branching: merge default into stable

This mark the start of the 5.9 freeze.

contrib/automation/linux-requirements-py3.5.txt

@@ -0,0 +1,194 b''
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --generate-hashes --output-file=contrib/automation/linux-requirements-py3.5.txt contrib/automation/linux-requirements.txt.in
+#
+astroid==2.4.2 \
+    --hash=sha256:2f4078c2a41bf377eea06d71c9d2ba4eb8f6b1af2135bec27bbbb7d8f12bb703 \
+    --hash=sha256:bc58d83eb610252fd8de6363e39d4f1d0619c894b0ed24603b881c02e64c7386
+    # via pylint
+docutils==0.17.1 \
+    --hash=sha256:686577d2e4c32380bb50cbb22f575ed742d58168cee37e99117a854bcd88f125 \
+    --hash=sha256:cf316c8370a737a022b72b56874f6602acf974a37a9fba42ec2876387549fc61
+    # via -r contrib/automation/linux-requirements.txt.in
+fuzzywuzzy==0.18.0 \
+    --hash=sha256:45016e92264780e58972dca1b3d939ac864b78437422beecebb3095f8efd00e8 \
+    --hash=sha256:928244b28db720d1e0ee7587acf660ea49d7e4c632569cad4f1cd7e68a5f0993
+    # via -r contrib/automation/linux-requirements.txt.in
+idna==3.1 \
+    --hash=sha256:5205d03e7bcbb919cc9c19885f9920d622ca52448306f2377daede5cf3faac16 \
+    --hash=sha256:c5b02147e01ea9920e6b0a3f1f7bb833612d507592c837a6c49552768f4054e1
+    # via yarl
+isort==4.3.21 \
+    --hash=sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1 \
+    --hash=sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd
+    # via
+    #   -r contrib/automation/linux-requirements.txt.in
+    #   pylint
+lazy-object-proxy==1.4.3 \
+    --hash=sha256:0c4b206227a8097f05c4dbdd323c50edf81f15db3b8dc064d08c62d37e1a504d \
+    --hash=sha256:194d092e6f246b906e8f70884e620e459fc54db3259e60cf69a4d66c3fda3449 \
+    --hash=sha256:1be7e4c9f96948003609aa6c974ae59830a6baecc5376c25c92d7d697e684c08 \
+    --hash=sha256:4677f594e474c91da97f489fea5b7daa17b5517190899cf213697e48d3902f5a \
+    --hash=sha256:48dab84ebd4831077b150572aec802f303117c8cc5c871e182447281ebf3ac50 \
+    --hash=sha256:5541cada25cd173702dbd99f8e22434105456314462326f06dba3e180f203dfd \
+    --hash=sha256:59f79fef100b09564bc2df42ea2d8d21a64fdcda64979c0fa3db7bdaabaf6239 \
+    --hash=sha256:8d859b89baf8ef7f8bc6b00aa20316483d67f0b1cbf422f5b4dc56701c8f2ffb \
+    --hash=sha256:9254f4358b9b541e3441b007a0ea0764b9d056afdeafc1a5569eee1cc6c1b9ea \
+    --hash=sha256:9651375199045a358eb6741df3e02a651e0330be090b3bc79f6d0de31a80ec3e \
+    --hash=sha256:97bb5884f6f1cdce0099f86b907aa41c970c3c672ac8b9c8352789e103cf3156 \
+    --hash=sha256:9b15f3f4c0f35727d3a0fba4b770b3c4ebbb1fa907dbcc046a1d2799f3edd142 \
+    --hash=sha256:a2238e9d1bb71a56cd710611a1614d1194dc10a175c1e08d75e1a7bcc250d442 \
+    --hash=sha256:a6ae12d08c0bf9909ce12385803a543bfe99b95fe01e752536a60af2b7797c62 \
+    --hash=sha256:ca0a928a3ddbc5725be2dd1cf895ec0a254798915fb3a36af0964a0a4149e3db \
+    --hash=sha256:cb2c7c57005a6804ab66f106ceb8482da55f5314b7fcb06551db1edae4ad1531 \
+    --hash=sha256:d74bb8693bf9cf75ac3b47a54d716bbb1a92648d5f781fc799347cfc95952383 \
+    --hash=sha256:d945239a5639b3ff35b70a88c5f2f491913eb94871780ebfabb2568bd58afc5a \
+    --hash=sha256:eba7011090323c1dadf18b3b689845fd96a61ba0a1dfbd7f24b921398affc357 \
+    --hash=sha256:efa1909120ce98bbb3777e8b6f92237f5d5c8ea6758efea36a473e1d38f7d3e4 \
+    --hash=sha256:f3900e8a5de27447acbf900b4750b0ddfd7ec1ea7fbaf11dfa911141bc522af0
+    # via astroid
+mccabe==0.6.1 \
+    --hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \
+    --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f
+    # via pylint
+multidict==5.0.2 \
+    --hash=sha256:060d68ae3e674c913ec41a464916f12c4d7ff17a3a9ebbf37ba7f2c681c2b33e \
+    --hash=sha256:06f39f0ddc308dab4e5fa282d145f90cd38d7ed75390fc83335636909a9ec191 \
+    --hash=sha256:17847fede1aafdb7e74e01bb34ab47a1a1ea726e8184c623c45d7e428d2d5d34 \
+    --hash=sha256:1cd102057b09223b919f9447c669cf2efabeefb42a42ae6233f25ffd7ee31a79 \
+    --hash=sha256:20cc9b2dd31761990abff7d0e63cd14dbfca4ebb52a77afc917b603473951a38 \
+    --hash=sha256:2576e30bbec004e863d87216bc34abe24962cc2e964613241a1c01c7681092ab \
+    --hash=sha256:2ab9cad4c5ef5c41e1123ed1f89f555aabefb9391d4e01fd6182de970b7267ed \
+    --hash=sha256:359ea00e1b53ceef282232308da9d9a3f60d645868a97f64df19485c7f9ef628 \
+    --hash=sha256:3e61cc244fd30bd9fdfae13bdd0c5ec65da51a86575ff1191255cae677045ffe \
+    --hash=sha256:43c7a87d8c31913311a1ab24b138254a0ee89142983b327a2c2eab7a7d10fea9 \
+    --hash=sha256:4a3f19da871befa53b48dd81ee48542f519beffa13090dc135fffc18d8fe36db \
+    --hash=sha256:4df708ef412fd9b59b7e6c77857e64c1f6b4c0116b751cb399384ec9a28baa66 \
+    --hash=sha256:59182e975b8c197d0146a003d0f0d5dc5487ce4899502061d8df585b0f51fba2 \
+    --hash=sha256:6128d2c0956fd60e39ec7d1c8f79426f0c915d36458df59ddd1f0cff0340305f \
+    --hash=sha256:6168839491a533fa75f3f5d48acbb829475e6c7d9fa5c6e245153b5f79b986a3 \
+    --hash=sha256:62abab8088704121297d39c8f47156cb8fab1da731f513e59ba73946b22cf3d0 \
+    --hash=sha256:653b2bbb0bbf282c37279dd04f429947ac92713049e1efc615f68d4e64b1dbc2 \
+    --hash=sha256:6566749cd78cb37cbf8e8171b5cd2cbfc03c99f0891de12255cf17a11c07b1a3 \
+    --hash=sha256:76cbdb22f48de64811f9ce1dd4dee09665f84f32d6a26de249a50c1e90e244e0 \
+    --hash=sha256:8efcf070d60fd497db771429b1c769a3783e3a0dd96c78c027e676990176adc5 \
+    --hash=sha256:8fa4549f341a057feec4c3139056ba73e17ed03a506469f447797a51f85081b5 \
+    --hash=sha256:9380b3f2b00b23a4106ba9dd022df3e6e2e84e1788acdbdd27603b621b3288df \
+    --hash=sha256:9ed9b280f7778ad6f71826b38a73c2fdca4077817c64bc1102fdada58e75c03c \
+    --hash=sha256:a7b8b5bd16376c8ac2977748bd978a200326af5145d8d0e7f799e2b355d425b6 \
+    --hash=sha256:af271c2540d1cd2a137bef8d95a8052230aa1cda26dd3b2c73d858d89993d518 \
+    --hash=sha256:b561e76c9e21402d9a446cdae13398f9942388b9bff529f32dfa46220af54d00 \
+    --hash=sha256:b82400ef848bbac6b9035a105ac6acaa1fb3eea0d164e35bbb21619b88e49fed \
+    --hash=sha256:b98af08d7bb37d3456a22f689819ea793e8d6961b9629322d7728c4039071641 \
+    --hash=sha256:c58e53e1c73109fdf4b759db9f2939325f510a8a5215135330fe6755921e4886 \
+    --hash=sha256:cbabfc12b401d074298bfda099c58dfa5348415ae2e4ec841290627cb7cb6b2e \
+    --hash=sha256:d4a6fb98e9e9be3f7d70fd3e852369c00a027bd5ed0f3e8ade3821bcad257408 \
+    --hash=sha256:d99da85d6890267292065e654a329e1d2f483a5d2485e347383800e616a8c0b1 \
+    --hash=sha256:e58db0e0d60029915f7fc95a8683fa815e204f2e1990f1fb46a7778d57ca8c35 \
+    --hash=sha256:e5bf89fe57f702a046c7ec718fe330ed50efd4bcf74722940db2eb0919cddb1c \
+    --hash=sha256:f612e8ef8408391a4a3366e3508bab8ef97b063b4918a317cb6e6de4415f01af \
+    --hash=sha256:f65a2442c113afde52fb09f9a6276bbc31da71add99dc76c3adf6083234e07c6 \
+    --hash=sha256:fa0503947a99a1be94f799fac89d67a5e20c333e78ddae16e8534b151cdc588a
+    # via yarl
+pyflakes==2.3.1 \
+    --hash=sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3 \
+    --hash=sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db
+    # via -r contrib/automation/linux-requirements.txt.in
+pygments==2.9.0 \
+    --hash=sha256:a18f47b506a429f6f4b9df81bb02beab9ca21d0a5fee38ed15aef65f0545519f \
+    --hash=sha256:d66e804411278594d764fc69ec36ec13d9ae9147193a1740cd34d272ca383b8e
+    # via -r contrib/automation/linux-requirements.txt.in
+pylint==2.6.2 \
+    --hash=sha256:718b74786ea7ed07aa0c58bf572154d4679f960d26e9641cc1de204a30b87fc9 \
+    --hash=sha256:e71c2e9614a4f06e36498f310027942b0f4f2fde20aebb01655b31edc63b9eaf
+    # via -r contrib/automation/linux-requirements.txt.in
+python-levenshtein==0.12.2 \
+    --hash=sha256:dc2395fbd148a1ab31090dd113c366695934b9e85fe5a4b2a032745efd0346f6
+    # via -r contrib/automation/linux-requirements.txt.in
+pyyaml==5.3.1 \
+    --hash=sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97 \
+    --hash=sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76 \
+    --hash=sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2 \
+    --hash=sha256:6034f55dab5fea9e53f436aa68fa3ace2634918e8b5994d82f3621c04ff5ed2e \
+    --hash=sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648 \
+    --hash=sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf \
+    --hash=sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f \
+    --hash=sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2 \
+    --hash=sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee \
+    --hash=sha256:ad9c67312c84def58f3c04504727ca879cb0013b2517c85a9a253f0cb6380c0a \
+    --hash=sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d \
+    --hash=sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c \
+    --hash=sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a
+    # via vcrpy
+six==1.16.0 \
+    --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
+    --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
+    # via
+    #   astroid
+    #   vcrpy
+toml==0.10.2 \
+    --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \
+    --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f
+    # via pylint
+typed-ast==1.4.3 ; python_version >= "3.0" and platform_python_implementation != "PyPy" \
+    --hash=sha256:01ae5f73431d21eead5015997ab41afa53aa1fbe252f9da060be5dad2c730ace \
+    --hash=sha256:067a74454df670dcaa4e59349a2e5c81e567d8d65458d480a5b3dfecec08c5ff \
+    --hash=sha256:0fb71b8c643187d7492c1f8352f2c15b4c4af3f6338f21681d3681b3dc31a266 \
+    --hash=sha256:1b3ead4a96c9101bef08f9f7d1217c096f31667617b58de957f690c92378b528 \
+    --hash=sha256:2068531575a125b87a41802130fa7e29f26c09a2833fea68d9a40cf33902eba6 \
+    --hash=sha256:209596a4ec71d990d71d5e0d312ac935d86930e6eecff6ccc7007fe54d703808 \
+    --hash=sha256:2c726c276d09fc5c414693a2de063f521052d9ea7c240ce553316f70656c84d4 \
+    --hash=sha256:398e44cd480f4d2b7ee8d98385ca104e35c81525dd98c519acff1b79bdaac363 \
+    --hash=sha256:52b1eb8c83f178ab787f3a4283f68258525f8d70f778a2f6dd54d3b5e5fb4341 \
+    --hash=sha256:5feca99c17af94057417d744607b82dd0a664fd5e4ca98061480fd8b14b18d04 \
+    --hash=sha256:7538e495704e2ccda9b234b82423a4038f324f3a10c43bc088a1636180f11a41 \
+    --hash=sha256:760ad187b1041a154f0e4d0f6aae3e40fdb51d6de16e5c99aedadd9246450e9e \
+    --hash=sha256:777a26c84bea6cd934422ac2e3b78863a37017618b6e5c08f92ef69853e765d3 \
+    --hash=sha256:95431a26309a21874005845c21118c83991c63ea800dd44843e42a916aec5899 \
+    --hash=sha256:9ad2c92ec681e02baf81fdfa056fe0d818645efa9af1f1cd5fd6f1bd2bdfd805 \
+    --hash=sha256:9c6d1a54552b5330bc657b7ef0eae25d00ba7ffe85d9ea8ae6540d2197a3788c \
+    --hash=sha256:aee0c1256be6c07bd3e1263ff920c325b59849dc95392a05f258bb9b259cf39c \
+    --hash=sha256:af3d4a73793725138d6b334d9d247ce7e5f084d96284ed23f22ee626a7b88e39 \
+    --hash=sha256:b36b4f3920103a25e1d5d024d155c504080959582b928e91cb608a65c3a49e1a \
+    --hash=sha256:b9574c6f03f685070d859e75c7f9eeca02d6933273b5e69572e5ff9d5e3931c3 \
+    --hash=sha256:bff6ad71c81b3bba8fa35f0f1921fb24ff4476235a6e94a26ada2e54370e6da7 \
+    --hash=sha256:c190f0899e9f9f8b6b7863debfb739abcb21a5c054f911ca3596d12b8a4c4c7f \
+    --hash=sha256:c907f561b1e83e93fad565bac5ba9c22d96a54e7ea0267c708bffe863cbe4075 \
+    --hash=sha256:cae53c389825d3b46fb37538441f75d6aecc4174f615d048321b716df2757fb0 \
+    --hash=sha256:dd4a21253f42b8d2b48410cb31fe501d32f8b9fbeb1f55063ad102fe9c425e40 \
+    --hash=sha256:dde816ca9dac1d9c01dd504ea5967821606f02e510438120091b84e852367428 \
+    --hash=sha256:f2362f3cb0f3172c42938946dbc5b7843c2a28aec307c49100c8b38764eb6927 \
+    --hash=sha256:f328adcfebed9f11301eaedfa48e15bdece9b519fb27e6a8c01aa52a17ec31b3 \
+    --hash=sha256:f8afcf15cc511ada719a88e013cec87c11aff7b91f019295eb4530f96fe5ef2f \
+    --hash=sha256:fb1bbeac803adea29cedd70781399c99138358c26d05fcbd23c13016b7f5ec65
+    # via
+    #   -r contrib/automation/linux-requirements.txt.in
+    #   astroid
+vcrpy==4.1.1 \
+    --hash=sha256:12c3fcdae7b88ecf11fc0d3e6d77586549d4575a2ceee18e82eee75c1f626162 \
+    --hash=sha256:57095bf22fc0a2d99ee9674cdafebed0f3ba763018582450706f7d3a74fff599
+    # via -r contrib/automation/linux-requirements.txt.in
+wrapt==1.12.1 \
+    --hash=sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7
+    # via
+    #   astroid
+    #   vcrpy
+yarl==1.3.0 \
+    --hash=sha256:024ecdc12bc02b321bc66b41327f930d1c2c543fa9a561b39861da9388ba7aa9 \
+    --hash=sha256:2f3010703295fbe1aec51023740871e64bb9664c789cba5a6bdf404e93f7568f \
+    --hash=sha256:3890ab952d508523ef4881457c4099056546593fa05e93da84c7250516e632eb \
+    --hash=sha256:3e2724eb9af5dc41648e5bb304fcf4891adc33258c6e14e2a7414ea32541e320 \
+    --hash=sha256:5badb97dd0abf26623a9982cd448ff12cb39b8e4c94032ccdedf22ce01a64842 \
+    --hash=sha256:73f447d11b530d860ca1e6b582f947688286ad16ca42256413083d13f260b7a0 \
+    --hash=sha256:7ab825726f2940c16d92aaec7d204cfc34ac26c0040da727cf8ba87255a33829 \
+    --hash=sha256:b25de84a8c20540531526dfbb0e2d2b648c13fd5dd126728c496d7c3fea33310 \
+    --hash=sha256:c6e341f5a6562af74ba55205dbd56d248daf1b5748ec48a0200ba227bb9e33f4 \
+    --hash=sha256:c9bb7c249c4432cd47e75af3864bc02d26c9594f49c82e2a28624417f0ae63b8 \
+    --hash=sha256:e060906c0c585565c718d1c3841747b61c5439af2211e185f6739a9412dfbde1
+    # via vcrpy
+
+# WARNING: The following packages were not pinned, but pip requires them to be
+# pinned when the requirements file includes hashes. Consider using the --allow-unsafe flag.
+# setuptools

contrib/automation/hgautomation/aws.py

@@ -1,1325 +1,1335 b''
 # aws.py - Automation code for Amazon Web Services
 #
 # Copyright 2019 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # no-check-code because Python 3 native.
 
 import contextlib
 import copy
 import hashlib
 import json
 import os
 import pathlib
 import subprocess
 import time
 
 import boto3
 import botocore.exceptions
 
 from .linux import BOOTSTRAP_DEBIAN
 from .ssh import (
     exec_command as ssh_exec_command,
     wait_for_ssh,
 )
 from .winrm import (
     run_powershell,
     wait_for_winrm,
 )
 
 
 SOURCE_ROOT = pathlib.Path(
     os.path.abspath(__file__)
 ).parent.parent.parent.parent
 
 INSTALL_WINDOWS_DEPENDENCIES = (
     SOURCE_ROOT / 'contrib' / 'install-windows-dependencies.ps1'
 )
 
 
 INSTANCE_TYPES_WITH_STORAGE = {
     'c5d',
     'd2',
     'h1',
     'i3',
     'm5ad',
     'm5d',
     'r5d',
     'r5ad',
     'x1',
     'z1d',
 }
 
 
 AMAZON_ACCOUNT_ID = '801119661308'
 DEBIAN_ACCOUNT_ID = '379101102735'
 DEBIAN_ACCOUNT_ID_2 = '136693071363'
 UBUNTU_ACCOUNT_ID = '099720109477'
 
 
 WINDOWS_BASE_IMAGE_NAME = 'Windows_Server-2019-English-Full-Base-*'
 
 
 KEY_PAIRS = {
     'automation',
 }
 
 
 SECURITY_GROUPS = {
     'linux-dev-1': {
         'description': 'Mercurial Linux instances that perform build/test automation',
         'ingress': [
             {
                 'FromPort': 22,
                 'ToPort': 22,
                 'IpProtocol': 'tcp',
                 'IpRanges': [
                     {
                         'CidrIp': '0.0.0.0/0',
                         'Description': 'SSH from entire Internet',
                     },
                 ],
             },
         ],
     },
     'windows-dev-1': {
         'description': 'Mercurial Windows instances that perform build automation',
         'ingress': [
             {
                 'FromPort': 22,
                 'ToPort': 22,
                 'IpProtocol': 'tcp',
                 'IpRanges': [
                     {
                         'CidrIp': '0.0.0.0/0',
                         'Description': 'SSH from entire Internet',
                     },
                 ],
             },
             {
                 'FromPort': 3389,
                 'ToPort': 3389,
                 'IpProtocol': 'tcp',
                 'IpRanges': [
                     {
                         'CidrIp': '0.0.0.0/0',
                         'Description': 'RDP from entire Internet',
                     },
                 ],
             },
             {
                 'FromPort': 5985,
                 'ToPort': 5986,
                 'IpProtocol': 'tcp',
                 'IpRanges': [
                     {
                         'CidrIp': '0.0.0.0/0',
                         'Description': 'PowerShell Remoting (Windows Remote Management)',
                     },
                 ],
             },
         ],
     },
 }
 
 
 IAM_ROLES = {
     'ephemeral-ec2-role-1': {
         'description': 'Mercurial temporary EC2 instances',
         'policy_arns': [
             'arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM',
         ],
     },
 }
 
 
 ASSUME_ROLE_POLICY_DOCUMENT = '''
 {
   "Version": "2012-10-17",
   "Statement": [
     {
       "Effect": "Allow",
       "Principal": {
         "Service": "ec2.amazonaws.com"
       },
       "Action": "sts:AssumeRole"
     }
   ]
 }
 '''.strip()
 
 
 IAM_INSTANCE_PROFILES = {
     'ephemeral-ec2-1': {
         'roles': [
             'ephemeral-ec2-role-1',
         ],
     }
 }
 
 
 # User Data for Windows EC2 instance. Mainly used to set the password
 # and configure WinRM.
 # Inspired by the User Data script used by Packer
 # (from https://www.packer.io/intro/getting-started/build-image.html).
 WINDOWS_USER_DATA = r'''
 <powershell>
 
 # TODO enable this once we figure out what is failing.
 #$ErrorActionPreference = "stop"
 
 # Set administrator password
 net user Administrator "%s"
 wmic useraccount where "name='Administrator'" set PasswordExpires=FALSE
 
 # First, make sure WinRM can't be connected to
 netsh advfirewall firewall set rule name="Windows Remote Management (HTTP-In)" new enable=yes action=block
 
 # Delete any existing WinRM listeners
 winrm delete winrm/config/listener?Address=*+Transport=HTTP  2>$Null
 winrm delete winrm/config/listener?Address=*+Transport=HTTPS 2>$Null
 
 # Create a new WinRM listener and configure
 winrm create winrm/config/listener?Address=*+Transport=HTTP
 winrm set winrm/config/winrs '@{MaxMemoryPerShellMB="0"}'
 winrm set winrm/config '@{MaxTimeoutms="7200000"}'
 winrm set winrm/config/service '@{AllowUnencrypted="true"}'
 winrm set winrm/config/service '@{MaxConcurrentOperationsPerUser="12000"}'
 winrm set winrm/config/service/auth '@{Basic="true"}'
 winrm set winrm/config/client/auth '@{Basic="true"}'
 
 # Configure UAC to allow privilege elevation in remote shells
 $Key = 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System'
 $Setting = 'LocalAccountTokenFilterPolicy'
 Set-ItemProperty -Path $Key -Name $Setting -Value 1 -Force
 
 # Avoid long usernames in the temp directory path because the '~' causes extra quoting in ssh output
 [System.Environment]::SetEnvironmentVariable('TMP', 'C:\Temp', [System.EnvironmentVariableTarget]::User)
 [System.Environment]::SetEnvironmentVariable('TEMP', 'C:\Temp', [System.EnvironmentVariableTarget]::User)
 
 # Configure and restart the WinRM Service; Enable the required firewall exception
 Stop-Service -Name WinRM
 Set-Service -Name WinRM -StartupType Automatic
 netsh advfirewall firewall set rule name="Windows Remote Management (HTTP-In)" new action=allow localip=any remoteip=any
 Start-Service -Name WinRM
 
 # Disable firewall on private network interfaces so prompts don't appear.
 Set-NetFirewallProfile -Name private -Enabled false
 </powershell>
 '''.lstrip()
 
 
 WINDOWS_BOOTSTRAP_POWERSHELL = '''
 Write-Output "installing PowerShell dependencies"
 Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force
 Set-PSRepository -Name PSGallery -InstallationPolicy Trusted
 Install-Module -Name OpenSSHUtils -RequiredVersion 0.0.2.0
 
 Write-Output "installing OpenSSL server"
 Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0
 # Various tools will attempt to use older versions of .NET. So we enable
 # the feature that provides them so it doesn't have to be auto-enabled
 # later.
 Write-Output "enabling .NET Framework feature"
 Install-WindowsFeature -Name Net-Framework-Core
 '''
 
 
 class AWSConnection:
     """Manages the state of a connection with AWS."""
 
     def __init__(self, automation, region: str, ensure_ec2_state: bool = True):
         self.automation = automation
         self.local_state_path = automation.state_path
 
         self.prefix = 'hg-'
 
         self.session = boto3.session.Session(region_name=region)
         self.ec2client = self.session.client('ec2')
         self.ec2resource = self.session.resource('ec2')
         self.iamclient = self.session.client('iam')
         self.iamresource = self.session.resource('iam')
         self.security_groups = {}
 
         if ensure_ec2_state:
             ensure_key_pairs(automation.state_path, self.ec2resource)
             self.security_groups = ensure_security_groups(self.ec2resource)
             ensure_iam_state(self.iamclient, self.iamresource)
 
     def key_pair_path_private(self, name):
         """Path to a key pair private key file."""
         return self.local_state_path / 'keys' / ('keypair-%s' % name)
 
     def key_pair_path_public(self, name):
         return self.local_state_path / 'keys' / ('keypair-%s.pub' % name)
 
 
 def rsa_key_fingerprint(p: pathlib.Path):
     """Compute the fingerprint of an RSA private key."""
 
     # TODO use rsa package.
     res = subprocess.run(
         [
             'openssl',
             'pkcs8',
             '-in',
             str(p),
             '-nocrypt',
             '-topk8',
             '-outform',
             'DER',
         ],
         capture_output=True,
         check=True,
     )
 
     sha1 = hashlib.sha1(res.stdout).hexdigest()
     return ':'.join(a + b for a, b in zip(sha1[::2], sha1[1::2]))
 
 
 def ensure_key_pairs(state_path: pathlib.Path, ec2resource, prefix='hg-'):
     remote_existing = {}
 
     for kpi in ec2resource.key_pairs.all():
         if kpi.name.startswith(prefix):
             remote_existing[kpi.name[len(prefix) :]] = kpi.key_fingerprint
 
     # Validate that we have these keys locally.
     key_path = state_path / 'keys'
     key_path.mkdir(exist_ok=True, mode=0o700)
 
     def remove_remote(name):
         print('deleting key pair %s' % name)
         key = ec2resource.KeyPair(name)
         key.delete()
 
     def remove_local(name):
         pub_full = key_path / ('keypair-%s.pub' % name)
         priv_full = key_path / ('keypair-%s' % name)
 
         print('removing %s' % pub_full)
         pub_full.unlink()
         print('removing %s' % priv_full)
         priv_full.unlink()
 
     local_existing = {}
 
     for f in sorted(os.listdir(key_path)):
         if not f.startswith('keypair-') or not f.endswith('.pub'):
             continue
 
         name = f[len('keypair-') : -len('.pub')]
 
         pub_full = key_path / f
         priv_full = key_path / ('keypair-%s' % name)
 
         with open(pub_full, 'r', encoding='ascii') as fh:
             data = fh.read()
 
         if not data.startswith('ssh-rsa '):
             print(
                 'unexpected format for key pair file: %s; removing' % pub_full
             )
             pub_full.unlink()
             priv_full.unlink()
             continue
 
         local_existing[name] = rsa_key_fingerprint(priv_full)
 
     for name in sorted(set(remote_existing) | set(local_existing)):
         if name not in local_existing:
             actual = '%s%s' % (prefix, name)
             print('remote key %s does not exist locally' % name)
             remove_remote(actual)
             del remote_existing[name]
 
         elif name not in remote_existing:
             print('local key %s does not exist remotely' % name)
             remove_local(name)
             del local_existing[name]
 
         elif remote_existing[name] != local_existing[name]:
             print(
                 'key fingerprint mismatch for %s; '
                 'removing from local and remote' % name
             )
             remove_local(name)
             remove_remote('%s%s' % (prefix, name))
             del local_existing[name]
             del remote_existing[name]
 
     missing = KEY_PAIRS - set(remote_existing)
 
     for name in sorted(missing):
         actual = '%s%s' % (prefix, name)
         print('creating key pair %s' % actual)
 
         priv_full = key_path / ('keypair-%s' % name)
         pub_full = key_path / ('keypair-%s.pub' % name)
 
         kp = ec2resource.create_key_pair(KeyName=actual)
 
         with priv_full.open('w', encoding='ascii') as fh:
             fh.write(kp.key_material)
             fh.write('\n')
 
         priv_full.chmod(0o0600)
 
         # SSH public key can be extracted via `ssh-keygen`.
         with pub_full.open('w', encoding='ascii') as fh:
             subprocess.run(
                 ['ssh-keygen', '-y', '-f', str(priv_full)],
                 stdout=fh,
                 check=True,
             )
 
         pub_full.chmod(0o0600)
 
 
 def delete_instance_profile(profile):
     for role in profile.roles:
         print(
             'removing role %s from instance profile %s'
             % (role.name, profile.name)
         )
         profile.remove_role(RoleName=role.name)
 
     print('deleting instance profile %s' % profile.name)
     profile.delete()
 
 
 def ensure_iam_state(iamclient, iamresource, prefix='hg-'):
     """Ensure IAM state is in sync with our canonical definition."""
 
     remote_profiles = {}
 
     for profile in iamresource.instance_profiles.all():
         if profile.name.startswith(prefix):
             remote_profiles[profile.name[len(prefix) :]] = profile
 
     for name in sorted(set(remote_profiles) - set(IAM_INSTANCE_PROFILES)):
         delete_instance_profile(remote_profiles[name])
         del remote_profiles[name]
 
     remote_roles = {}
 
     for role in iamresource.roles.all():
         if role.name.startswith(prefix):
             remote_roles[role.name[len(prefix) :]] = role
 
     for name in sorted(set(remote_roles) - set(IAM_ROLES)):
         role = remote_roles[name]
 
         print('removing role %s' % role.name)
         role.delete()
         del remote_roles[name]
 
     # We've purged remote state that doesn't belong. Create missing
     # instance profiles and roles.
     for name in sorted(set(IAM_INSTANCE_PROFILES) - set(remote_profiles)):
         actual = '%s%s' % (prefix, name)
         print('creating IAM instance profile %s' % actual)
 
         profile = iamresource.create_instance_profile(
             InstanceProfileName=actual
         )
         remote_profiles[name] = profile
 
         waiter = iamclient.get_waiter('instance_profile_exists')
         waiter.wait(InstanceProfileName=actual)
         print('IAM instance profile %s is available' % actual)
 
     for name in sorted(set(IAM_ROLES) - set(remote_roles)):
         entry = IAM_ROLES[name]
 
         actual = '%s%s' % (prefix, name)
         print('creating IAM role %s' % actual)
 
         role = iamresource.create_role(
             RoleName=actual,
             Description=entry['description'],
             AssumeRolePolicyDocument=ASSUME_ROLE_POLICY_DOCUMENT,
         )
 
         waiter = iamclient.get_waiter('role_exists')
         waiter.wait(RoleName=actual)
         print('IAM role %s is available' % actual)
 
         remote_roles[name] = role
 
         for arn in entry['policy_arns']:
             print('attaching policy %s to %s' % (arn, role.name))
             role.attach_policy(PolicyArn=arn)
 
     # Now reconcile state of profiles.
     for name, meta in sorted(IAM_INSTANCE_PROFILES.items()):
         profile = remote_profiles[name]
         wanted = {'%s%s' % (prefix, role) for role in meta['roles']}
         have = {role.name for role in profile.roles}
 
         for role in sorted(have - wanted):
             print('removing role %s from %s' % (role, profile.name))
             profile.remove_role(RoleName=role)
 
         for role in sorted(wanted - have):
             print('adding role %s to %s' % (role, profile.name))
             profile.add_role(RoleName=role)
 
 
 def find_image(ec2resource, owner_id, name, reverse_sort_field=None):
     """Find an AMI by its owner ID and name."""
 
     images = ec2resource.images.filter(
         Filters=[
             {
                 'Name': 'owner-id',
                 'Values': [owner_id],
             },
             {
                 'Name': 'state',
                 'Values': ['available'],
             },
             {
                 'Name': 'image-type',
                 'Values': ['machine'],
             },
             {
                 'Name': 'name',
                 'Values': [name],
             },
         ]
     )
 
     if reverse_sort_field:
         images = sorted(
             images,
             key=lambda image: getattr(image, reverse_sort_field),
             reverse=True,
         )
 
     for image in images:
         return image
 
     raise Exception('unable to find image for %s' % name)
 
 
 def ensure_security_groups(ec2resource, prefix='hg-'):
     """Ensure all necessary Mercurial security groups are present.
 
     All security groups are prefixed with ``hg-`` by default. Any security
     groups having this prefix but aren't in our list are deleted.
     """
     existing = {}
 
     for group in ec2resource.security_groups.all():
         if group.group_name.startswith(prefix):
             existing[group.group_name[len(prefix) :]] = group
 
     purge = set(existing) - set(SECURITY_GROUPS)
 
     for name in sorted(purge):
         group = existing[name]
         print('removing legacy security group: %s' % group.group_name)
         group.delete()
 
     security_groups = {}
 
     for name, group in sorted(SECURITY_GROUPS.items()):
         if name in existing:
             security_groups[name] = existing[name]
             continue
 
         actual = '%s%s' % (prefix, name)
         print('adding security group %s' % actual)
 
         group_res = ec2resource.create_security_group(
             Description=group['description'],
             GroupName=actual,
         )
 
         group_res.authorize_ingress(
             IpPermissions=group['ingress'],
         )
 
         security_groups[name] = group_res
 
     return security_groups
 
 
 def terminate_ec2_instances(ec2resource, prefix='hg-'):
     """Terminate all EC2 instances managed by us."""
     waiting = []
 
     for instance in ec2resource.instances.all():
         if instance.state['Name'] == 'terminated':
             continue
 
         for tag in instance.tags or []:
             if tag['Key'] == 'Name' and tag['Value'].startswith(prefix):
                 print('terminating %s' % instance.id)
                 instance.terminate()
                 waiting.append(instance)
 
     for instance in waiting:
         instance.wait_until_terminated()
 
 
 def remove_resources(c, prefix='hg-'):
     """Purge all of our resources in this EC2 region."""
     ec2resource = c.ec2resource
     iamresource = c.iamresource
 
     terminate_ec2_instances(ec2resource, prefix=prefix)
 
     for image in ec2resource.images.filter(Owners=['self']):
         if image.name.startswith(prefix):
             remove_ami(ec2resource, image)
 
     for group in ec2resource.security_groups.all():
         if group.group_name.startswith(prefix):
             print('removing security group %s' % group.group_name)
             group.delete()
 
     for profile in iamresource.instance_profiles.all():
         if profile.name.startswith(prefix):
             delete_instance_profile(profile)
 
     for role in iamresource.roles.all():
         if role.name.startswith(prefix):
             for p in role.attached_policies.all():
                 print('detaching policy %s from %s' % (p.arn, role.name))
                 role.detach_policy(PolicyArn=p.arn)
 
             print('removing role %s' % role.name)
             role.delete()
 
 
 def wait_for_ip_addresses(instances):
     """Wait for the public IP addresses of an iterable of instances."""
     for instance in instances:
         while True:
             if not instance.public_ip_address:
                 time.sleep(2)
                 instance.reload()
                 continue
 
             print(
                 'public IP address for %s: %s'
                 % (instance.id, instance.public_ip_address)
             )
             break
 
 
 def remove_ami(ec2resource, image):
     """Remove an AMI and its underlying snapshots."""
     snapshots = []
 
     for device in image.block_device_mappings:
         if 'Ebs' in device:
             snapshots.append(ec2resource.Snapshot(device['Ebs']['SnapshotId']))
 
     print('deregistering %s' % image.id)
     image.deregister()
 
     for snapshot in snapshots:
         print('deleting snapshot %s' % snapshot.id)
         snapshot.delete()
 
 
 def wait_for_ssm(ssmclient, instances):
     """Wait for SSM to come online for an iterable of instance IDs."""
     while True:
         res = ssmclient.describe_instance_information(
             Filters=[
                 {
                     'Key': 'InstanceIds',
                     'Values': [i.id for i in instances],
                 },
             ],
         )
 
         available = len(res['InstanceInformationList'])
         wanted = len(instances)
 
         print('%d/%d instances available in SSM' % (available, wanted))
 
         if available == wanted:
             return
 
         time.sleep(2)
 
 
 def run_ssm_command(ssmclient, instances, document_name, parameters):
     """Run a PowerShell script on an EC2 instance."""
 
     res = ssmclient.send_command(
         InstanceIds=[i.id for i in instances],
         DocumentName=document_name,
         Parameters=parameters,
         CloudWatchOutputConfig={
             'CloudWatchOutputEnabled': True,
         },
     )
 
     command_id = res['Command']['CommandId']
 
     for instance in instances:
         while True:
             try:
                 res = ssmclient.get_command_invocation(
                     CommandId=command_id,
                     InstanceId=instance.id,
                 )
             except botocore.exceptions.ClientError as e:
                 if e.response['Error']['Code'] == 'InvocationDoesNotExist':
                     print('could not find SSM command invocation; waiting')
                     time.sleep(1)
                     continue
                 else:
                     raise
 
             if res['Status'] == 'Success':
                 break
             elif res['Status'] in ('Pending', 'InProgress', 'Delayed'):
                 time.sleep(2)
             else:
                 raise Exception(
                     'command failed on %s: %s' % (instance.id, res['Status'])
                 )
 
 
 @contextlib.contextmanager
 def temporary_ec2_instances(ec2resource, config):
     """Create temporary EC2 instances.
 
     This is a proxy to ``ec2client.run_instances(**config)`` that takes care of
     managing the lifecycle of the instances.
 
     When the context manager exits, the instances are terminated.
 
     The context manager evaluates to the list of data structures
     describing each created instance. The instances may not be available
     for work immediately: it is up to the caller to wait for the instance
     to start responding.
     """
 
     ids = None
 
     try:
         res = ec2resource.create_instances(**config)
 
         ids = [i.id for i in res]
         print('started instances: %s' % ' '.join(ids))
 
         yield res
     finally:
         if ids:
             print('terminating instances: %s' % ' '.join(ids))
             for instance in res:
                 instance.terminate()
             print('terminated %d instances' % len(ids))
 
 
 @contextlib.contextmanager
 def create_temp_windows_ec2_instances(
     c: AWSConnection, config, bootstrap: bool = False
 ):
     """Create temporary Windows EC2 instances.
 
     This is a higher-level wrapper around ``create_temp_ec2_instances()`` that
     configures the Windows instance for Windows Remote Management. The emitted
     instances will have a ``winrm_client`` attribute containing a
     ``pypsrp.client.Client`` instance bound to the instance.
     """
     if 'IamInstanceProfile' in config:
         raise ValueError('IamInstanceProfile cannot be provided in config')
     if 'UserData' in config:
         raise ValueError('UserData cannot be provided in config')
 
     password = c.automation.default_password()
 
     config = copy.deepcopy(config)
     config['IamInstanceProfile'] = {
         'Name': 'hg-ephemeral-ec2-1',
     }
     config.setdefault('TagSpecifications', []).append(
         {
             'ResourceType': 'instance',
             'Tags': [{'Key': 'Name', 'Value': 'hg-temp-windows'}],
         }
     )
 
     if bootstrap:
         config['UserData'] = WINDOWS_USER_DATA % password
 
     with temporary_ec2_instances(c.ec2resource, config) as instances:
         wait_for_ip_addresses(instances)
 
         print('waiting for Windows Remote Management service...')
 
         for instance in instances:
             client = wait_for_winrm(
                 instance.public_ip_address, 'Administrator', password
             )
             print('established WinRM connection to %s' % instance.id)
             instance.winrm_client = client
 
         yield instances
 
 
 def resolve_fingerprint(fingerprint):
     fingerprint = json.dumps(fingerprint, sort_keys=True)
     return hashlib.sha256(fingerprint.encode('utf-8')).hexdigest()
 
 
 def find_and_reconcile_image(ec2resource, name, fingerprint):
     """Attempt to find an existing EC2 AMI with a name and fingerprint.
 
     If an image with the specified fingerprint is found, it is returned.
     Otherwise None is returned.
 
     Existing images for the specified name that don't have the specified
     fingerprint or are missing required metadata or deleted.
     """
     # Find existing AMIs with this name and delete the ones that are invalid.
     # Store a reference to a good image so it can be returned one the
     # image state is reconciled.
     images = ec2resource.images.filter(
         Filters=[{'Name': 'name', 'Values': [name]}]
     )
 
     existing_image = None
 
     for image in images:
         if image.tags is None:
             print(
                 'image %s for %s lacks required tags; removing'
                 % (image.id, image.name)
             )
             remove_ami(ec2resource, image)
         else:
             tags = {t['Key']: t['Value'] for t in image.tags}
 
             if tags.get('HGIMAGEFINGERPRINT') == fingerprint:
                 existing_image = image
             else:
                 print(
                     'image %s for %s has wrong fingerprint; removing'
                     % (image.id, image.name)
                 )
                 remove_ami(ec2resource, image)
 
     return existing_image
 
 
 def create_ami_from_instance(
     ec2client, instance, name, description, fingerprint
 ):
     """Create an AMI from a running instance.
 
     Returns the ``ec2resource.Image`` representing the created AMI.
     """
     instance.stop()
 
     ec2client.get_waiter('instance_stopped').wait(
         InstanceIds=[instance.id],
         WaiterConfig={
             'Delay': 5,
         },
     )
     print('%s is stopped' % instance.id)
 
     image = instance.create_image(
         Name=name,
         Description=description,
     )
 
     image.create_tags(
         Tags=[
             {
                 'Key': 'HGIMAGEFINGERPRINT',
                 'Value': fingerprint,
             },
         ]
     )
 
     print('waiting for image %s' % image.id)
 
     ec2client.get_waiter('image_available').wait(
         ImageIds=[image.id],
     )
 
     print('image %s available as %s' % (image.id, image.name))
 
     return image
 
 
 def ensure_linux_dev_ami(c: AWSConnection, distro='debian10', prefix='hg-'):
     """Ensures a Linux development AMI is available and up-to-date.
 
     Returns an ``ec2.Image`` of either an existing AMI or a newly-built one.
     """
     ec2client = c.ec2client
     ec2resource = c.ec2resource
 
     name = '%s%s-%s' % (prefix, 'linux-dev', distro)
 
     if distro == 'debian9':
         image = find_image(
             ec2resource,
             DEBIAN_ACCOUNT_ID,
             'debian-stretch-hvm-x86_64-gp2-2019-09-08-17994',
         )
         ssh_username = 'admin'
     elif distro == 'debian10':
         image = find_image(
             ec2resource,
             DEBIAN_ACCOUNT_ID_2,
             'debian-10-amd64-20190909-10',
         )
         ssh_username = 'admin'
     elif distro == 'ubuntu18.04':
         image = find_image(
             ec2resource,
             UBUNTU_ACCOUNT_ID,
             'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20190918',
         )
         ssh_username = 'ubuntu'
     elif distro == 'ubuntu19.04':
         image = find_image(
             ec2resource,
             UBUNTU_ACCOUNT_ID,
             'ubuntu/images/hvm-ssd/ubuntu-disco-19.04-amd64-server-20190918',
         )
         ssh_username = 'ubuntu'
     else:
         raise ValueError('unsupported Linux distro: %s' % distro)
 
     config = {
         'BlockDeviceMappings': [
             {
                 'DeviceName': image.block_device_mappings[0]['DeviceName'],
                 'Ebs': {
                     'DeleteOnTermination': True,
                     'VolumeSize': 10,
                     'VolumeType': 'gp2',
                 },
             },
         ],
         'EbsOptimized': True,
         'ImageId': image.id,
         'InstanceInitiatedShutdownBehavior': 'stop',
         # 8 VCPUs for compiling Python.
         'InstanceType': 't3.2xlarge',
         'KeyName': '%sautomation' % prefix,
         'MaxCount': 1,
         'MinCount': 1,
         'SecurityGroupIds': [c.security_groups['linux-dev-1'].id],
     }
 
     requirements2_path = (
         pathlib.Path(__file__).parent.parent / 'linux-requirements-py2.txt'
     )
     requirements3_path = (
         pathlib.Path(__file__).parent.parent / 'linux-requirements-py3.txt'
     )
+    requirements35_path = (
+        pathlib.Path(__file__).parent.parent / 'linux-requirements-py3.5.txt'
+    )
     with requirements2_path.open('r', encoding='utf-8') as fh:
         requirements2 = fh.read()
     with requirements3_path.open('r', encoding='utf-8') as fh:
         requirements3 = fh.read()
+    with requirements35_path.open('r', encoding='utf-8') as fh:
+        requirements35 = fh.read()
 
     # Compute a deterministic fingerprint to determine whether image needs to
     # be regenerated.
     fingerprint = resolve_fingerprint(
         {
             'instance_config': config,
             'bootstrap_script': BOOTSTRAP_DEBIAN,
             'requirements_py2': requirements2,
             'requirements_py3': requirements3,
+            'requirements_py35': requirements35,
         }
     )
 
     existing_image = find_and_reconcile_image(ec2resource, name, fingerprint)
 
     if existing_image:
         return existing_image
 
     print('no suitable %s image found; creating one...' % name)
 
     with temporary_ec2_instances(ec2resource, config) as instances:
         wait_for_ip_addresses(instances)
 
         instance = instances[0]
 
         client = wait_for_ssh(
             instance.public_ip_address,
             22,
             username=ssh_username,
             key_filename=str(c.key_pair_path_private('automation')),
         )
 
         home = '/home/%s' % ssh_username
 
         with client:
             print('connecting to SSH server')
             sftp = client.open_sftp()
 
             print('uploading bootstrap files')
             with sftp.open('%s/bootstrap' % home, 'wb') as fh:
                 fh.write(BOOTSTRAP_DEBIAN)
                 fh.chmod(0o0700)
 
             with sftp.open('%s/requirements-py2.txt' % home, 'wb') as fh:
                 fh.write(requirements2)
                 fh.chmod(0o0700)
 
             with sftp.open('%s/requirements-py3.txt' % home, 'wb') as fh:
                 fh.write(requirements3)
                 fh.chmod(0o0700)
 
+            with sftp.open('%s/requirements-py3.5.txt' % home, 'wb') as fh:
+                fh.write(requirements35)
+                fh.chmod(0o0700)
+
             print('executing bootstrap')
             chan, stdin, stdout = ssh_exec_command(
                 client, '%s/bootstrap' % home
             )
             stdin.close()
 
             for line in stdout:
                 print(line, end='')
 
             res = chan.recv_exit_status()
             if res:
                 raise Exception('non-0 exit from bootstrap: %d' % res)
 
             print(
                 'bootstrap completed; stopping %s to create %s'
                 % (instance.id, name)
             )
 
         return create_ami_from_instance(
             ec2client,
             instance,
             name,
             'Mercurial Linux development environment',
             fingerprint,
         )
 
 
 @contextlib.contextmanager
 def temporary_linux_dev_instances(
     c: AWSConnection,
     image,
     instance_type,
     prefix='hg-',
     ensure_extra_volume=False,
 ):
     """Create temporary Linux development EC2 instances.
 
     Context manager resolves to a list of ``ec2.Instance`` that were created
     and are running.
 
     ``ensure_extra_volume`` can be set to ``True`` to require that instances
     have a 2nd storage volume available other than the primary AMI volume.
     For instance types with instance storage, this does nothing special.
     But for instance types without instance storage, an additional EBS volume
     will be added to the instance.
 
     Instances have an ``ssh_client`` attribute containing a paramiko SSHClient
     instance bound to the instance.
 
     Instances have an ``ssh_private_key_path`` attributing containing the
     str path to the SSH private key to connect to the instance.
     """
 
     block_device_mappings = [
         {
             'DeviceName': image.block_device_mappings[0]['DeviceName'],
             'Ebs': {
                 'DeleteOnTermination': True,
                 'VolumeSize': 12,
                 'VolumeType': 'gp2',
             },
         }
     ]
 
     # This is not an exhaustive list of instance types having instance storage.
     # But
     if ensure_extra_volume and not instance_type.startswith(
         tuple(INSTANCE_TYPES_WITH_STORAGE)
     ):
         main_device = block_device_mappings[0]['DeviceName']
 
         if main_device == 'xvda':
             second_device = 'xvdb'
         elif main_device == '/dev/sda1':
             second_device = '/dev/sdb'
         else:
             raise ValueError(
                 'unhandled primary EBS device name: %s' % main_device
             )
 
         block_device_mappings.append(
             {
                 'DeviceName': second_device,
                 'Ebs': {
                     'DeleteOnTermination': True,
                     'VolumeSize': 8,
                     'VolumeType': 'gp2',
                 },
             }
         )
 
     config = {
         'BlockDeviceMappings': block_device_mappings,
         'EbsOptimized': True,
         'ImageId': image.id,
         'InstanceInitiatedShutdownBehavior': 'terminate',
         'InstanceType': instance_type,
         'KeyName': '%sautomation' % prefix,
         'MaxCount': 1,
         'MinCount': 1,
         'SecurityGroupIds': [c.security_groups['linux-dev-1'].id],
     }
 
     with temporary_ec2_instances(c.ec2resource, config) as instances:
         wait_for_ip_addresses(instances)
 
         ssh_private_key_path = str(c.key_pair_path_private('automation'))
 
         for instance in instances:
             client = wait_for_ssh(
                 instance.public_ip_address,
                 22,
                 username='hg',
                 key_filename=ssh_private_key_path,
             )
 
             instance.ssh_client = client
             instance.ssh_private_key_path = ssh_private_key_path
 
         try:
             yield instances
         finally:
             for instance in instances:
                 instance.ssh_client.close()
 
 
 def ensure_windows_dev_ami(
     c: AWSConnection,
     prefix='hg-',
     base_image_name=WINDOWS_BASE_IMAGE_NAME,
 ):
     """Ensure Windows Development AMI is available and up-to-date.
 
     If necessary, a modern AMI will be built by starting a temporary EC2
     instance and bootstrapping it.
 
     Obsolete AMIs will be deleted so there is only a single AMI having the
     desired name.
 
     Returns an ``ec2.Image`` of either an existing AMI or a newly-built
     one.
     """
     ec2client = c.ec2client
     ec2resource = c.ec2resource
     ssmclient = c.session.client('ssm')
 
     name = '%s%s' % (prefix, 'windows-dev')
 
     image = find_image(
         ec2resource,
         AMAZON_ACCOUNT_ID,
         base_image_name,
         reverse_sort_field="name",
     )
 
     config = {
         'BlockDeviceMappings': [
             {
                 'DeviceName': '/dev/sda1',
                 'Ebs': {
                     'DeleteOnTermination': True,
                     'VolumeSize': 32,
                     'VolumeType': 'gp2',
                 },
             }
         ],
         'ImageId': image.id,
         'InstanceInitiatedShutdownBehavior': 'stop',
         'InstanceType': 't3.medium',
         'KeyName': '%sautomation' % prefix,
         'MaxCount': 1,
         'MinCount': 1,
         'SecurityGroupIds': [c.security_groups['windows-dev-1'].id],
     }
 
     commands = [
         # Need to start the service so sshd_config is generated.
         'Start-Service sshd',
         'Write-Output "modifying sshd_config"',
         r'$content = Get-Content C:\ProgramData\ssh\sshd_config',
         '$content = $content -replace "Match Group administrators","" -replace "AuthorizedKeysFile __PROGRAMDATA__/ssh/administrators_authorized_keys",""',
         r'$content | Set-Content C:\ProgramData\ssh\sshd_config',
         'Import-Module OpenSSHUtils',
         r'Repair-SshdConfigPermission C:\ProgramData\ssh\sshd_config -Confirm:$false',
         'Restart-Service sshd',
         'Write-Output "installing OpenSSL client"',
         'Add-WindowsCapability -Online -Name OpenSSH.Client~~~~0.0.1.0',
         'Set-Service -Name sshd -StartupType "Automatic"',
         'Write-Output "OpenSSH server running"',
     ]
 
     with INSTALL_WINDOWS_DEPENDENCIES.open('r', encoding='utf-8') as fh:
         commands.extend(l.rstrip() for l in fh)
 
     # Schedule run of EC2Launch on next boot. This ensures that UserData
     # is executed.
     # We disable setComputerName because it forces a reboot.
     # We set an explicit admin password because this causes UserData to run
     # as Administrator instead of System.
     commands.extend(
         [
             r'''Set-Content -Path C:\ProgramData\Amazon\EC2-Windows\Launch\Config\LaunchConfig.json '''
             r'''-Value '{"setComputerName": false, "setWallpaper": true, "addDnsSuffixList": true, '''
             r'''"extendBootVolumeSize": true, "handleUserData": true, '''
             r'''"adminPasswordType": "Specify", "adminPassword": "%s"}' '''
             % c.automation.default_password(),
             r'C:\ProgramData\Amazon\EC2-Windows\Launch\Scripts\InitializeInstance.ps1 '
             r'–Schedule',
         ]
     )
 
     # Disable Windows Defender when bootstrapping because it just slows
     # things down.
     commands.insert(0, 'Set-MpPreference -DisableRealtimeMonitoring $true')
     commands.append('Set-MpPreference -DisableRealtimeMonitoring $false')
 
     # Compute a deterministic fingerprint to determine whether image needs
     # to be regenerated.
     fingerprint = resolve_fingerprint(
         {
             'instance_config': config,
             'user_data': WINDOWS_USER_DATA,
             'initial_bootstrap': WINDOWS_BOOTSTRAP_POWERSHELL,
             'bootstrap_commands': commands,
             'base_image_name': base_image_name,
         }
     )
 
     existing_image = find_and_reconcile_image(ec2resource, name, fingerprint)
 
     if existing_image:
         return existing_image
 
     print('no suitable Windows development image found; creating one...')
 
     with create_temp_windows_ec2_instances(
         c, config, bootstrap=True
     ) as instances:
         assert len(instances) == 1
         instance = instances[0]
 
         wait_for_ssm(ssmclient, [instance])
 
         # On first boot, install various Windows updates.
         # We would ideally use PowerShell Remoting for this. However, there are
         # trust issues that make it difficult to invoke Windows Update
         # remotely. So we use SSM, which has a mechanism for running Windows
         # Update.
         print('installing Windows features...')
         run_ssm_command(
             ssmclient,
             [instance],
             'AWS-RunPowerShellScript',
             {
                 'commands': WINDOWS_BOOTSTRAP_POWERSHELL.split('\n'),
             },
         )
 
         # Reboot so all updates are fully applied.
         #
         # We don't use instance.reboot() here because it is asynchronous and
         # we don't know when exactly the instance has rebooted. It could take
         # a while to stop and we may start trying to interact with the instance
         # before it has rebooted.
         print('rebooting instance %s' % instance.id)
         instance.stop()
         ec2client.get_waiter('instance_stopped').wait(
             InstanceIds=[instance.id],
             WaiterConfig={
                 'Delay': 5,
             },
         )
 
         instance.start()
         wait_for_ip_addresses([instance])
 
         # There is a race condition here between the User Data PS script running
         # and us connecting to WinRM. This can manifest as
         # "AuthorizationManager check failed" failures during run_powershell().
         # TODO figure out a workaround.
 
         print('waiting for Windows Remote Management to come back...')
         client = wait_for_winrm(
             instance.public_ip_address,
             'Administrator',
             c.automation.default_password(),
         )
         print('established WinRM connection to %s' % instance.id)
         instance.winrm_client = client
 
         print('bootstrapping instance...')
         run_powershell(instance.winrm_client, '\n'.join(commands))
 
         print('bootstrap completed; stopping %s to create image' % instance.id)
         return create_ami_from_instance(
             ec2client,
             instance,
             name,
             'Mercurial Windows development environment',
             fingerprint,
         )
 
 
 @contextlib.contextmanager
 def temporary_windows_dev_instances(
     c: AWSConnection,
     image,
     instance_type,
     prefix='hg-',
     disable_antivirus=False,
 ):
     """Create a temporary Windows development EC2 instance.
 
     Context manager resolves to the list of ``EC2.Instance`` that were created.
     """
     config = {
         'BlockDeviceMappings': [
             {
                 'DeviceName': '/dev/sda1',
                 'Ebs': {
                     'DeleteOnTermination': True,
                     'VolumeSize': 32,
                     'VolumeType': 'gp2',
                 },
             }
         ],
         'ImageId': image.id,
         'InstanceInitiatedShutdownBehavior': 'stop',
         'InstanceType': instance_type,
         'KeyName': '%sautomation' % prefix,
         'MaxCount': 1,
         'MinCount': 1,
         'SecurityGroupIds': [c.security_groups['windows-dev-1'].id],
     }
 
     with create_temp_windows_ec2_instances(c, config) as instances:
         if disable_antivirus:
             for instance in instances:
                 run_powershell(
                     instance.winrm_client,
                     'Set-MpPreference -DisableRealtimeMonitoring $true',
                 )
 
         yield instances

contrib/automation/hgautomation/linux.py

@@ -1,597 +1,621 b''
 # linux.py - Linux specific automation functionality
 #
 # Copyright 2019 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # no-check-code because Python 3 native.
 
 import os
 import pathlib
 import shlex
 import subprocess
 import tempfile
 
 from .ssh import exec_command
 
 
 # Linux distributions that are supported.
 DISTROS = {
     'debian9',
     'debian10',
     'ubuntu18.04',
     'ubuntu19.04',
 }
 
 INSTALL_PYTHONS = r'''
 PYENV2_VERSIONS="2.7.17 pypy2.7-7.2.0"
-PYENV3_VERSIONS="3.5.10 3.6.12 3.7.9 3.8.6 3.9.0 pypy3.5-7.0.0 pypy3.6-7.3.0"
+PYENV3_VERSIONS="3.5.10 3.6.13 3.7.10 3.8.10 3.9.5 pypy3.5-7.0.0 pypy3.6-7.3.3 pypy3.7-7.3.3"
 
 git clone https://github.com/pyenv/pyenv.git /hgdev/pyenv
 pushd /hgdev/pyenv
-git checkout 8ac91b4fd678a8c04356f5ec85cfcd565c265e9a
+git checkout 328fd42c3a2fbf14ae46dae2021a087fe27ba7e2
 popd
 
 export PYENV_ROOT="/hgdev/pyenv"
 export PATH="$PYENV_ROOT/bin:$PATH"
 
 # pip 19.2.3.
 PIP_SHA256=57e3643ff19f018f8a00dfaa6b7e4620e3c1a7a2171fd218425366ec006b3bfe
 wget -O get-pip.py --progress dot:mega https://github.com/pypa/get-pip/raw/309a56c5fd94bd1134053a541cb4657a4e47e09d/get-pip.py
 echo "${PIP_SHA256} get-pip.py" | sha256sum --check -
 
 VIRTUALENV_SHA256=f78d81b62d3147396ac33fc9d77579ddc42cc2a98dd9ea38886f616b33bc7fb2
 VIRTUALENV_TARBALL=virtualenv-16.7.5.tar.gz
 wget -O ${VIRTUALENV_TARBALL} --progress dot:mega https://files.pythonhosted.org/packages/66/f0/6867af06d2e2f511e4e1d7094ff663acdebc4f15d4a0cb0fed1007395124/${VIRTUALENV_TARBALL}
 echo "${VIRTUALENV_SHA256} ${VIRTUALENV_TARBALL}" | sha256sum --check -
 
 for v in ${PYENV2_VERSIONS}; do
     pyenv install -v ${v}
     ${PYENV_ROOT}/versions/${v}/bin/python get-pip.py
     ${PYENV_ROOT}/versions/${v}/bin/pip install ${VIRTUALENV_TARBALL}
     ${PYENV_ROOT}/versions/${v}/bin/pip install -r /hgdev/requirements-py2.txt
 done
 
 for v in ${PYENV3_VERSIONS}; do
     pyenv install -v ${v}
     ${PYENV_ROOT}/versions/${v}/bin/python get-pip.py
-    ${PYENV_ROOT}/versions/${v}/bin/pip install -r /hgdev/requirements-py3.txt
+
+    case ${v} in
+        3.5.*)
+            REQUIREMENTS=requirements-py3.5.txt
+            ;;
+        pypy3.5*)
+            REQUIREMENTS=requirements-py3.5.txt
+            ;;
+        *)
+            REQUIREMENTS=requirements-py3.txt
+            ;;
+    esac
+
+    ${PYENV_ROOT}/versions/${v}/bin/pip install -r /hgdev/${REQUIREMENTS}
 done
 
 pyenv global ${PYENV2_VERSIONS} ${PYENV3_VERSIONS} system
 '''.lstrip().replace(
     '\r\n', '\n'
 )
 
+INSTALL_PYOXIDIZER = r'''
+PYOXIDIZER_VERSION=0.16.0
+PYOXIDIZER_SHA256=8875471c270312fbb934007fd30f65f1904cc0f5da6188d61c90ed2129b9f9c1
+PYOXIDIZER_URL=https://github.com/indygreg/PyOxidizer/releases/download/pyoxidizer%2F${PYOXIDIZER_VERSION}/pyoxidizer-${PYOXIDIZER_VERSION}-linux_x86_64.zip
+
+wget -O pyoxidizer.zip --progress dot:mega ${PYOXIDIZER_URL}
+echo "${PYOXIDIZER_SHA256} pyoxidizer.zip" | sha256sum --check -
+
+unzip pyoxidizer.zip
+chmod +x pyoxidizer
+sudo mv pyoxidizer /usr/local/bin/pyoxidizer
+'''
 
 INSTALL_RUST = r'''
 RUSTUP_INIT_SHA256=a46fe67199b7bcbbde2dcbc23ae08db6f29883e260e23899a88b9073effc9076
 wget -O rustup-init --progress dot:mega https://static.rust-lang.org/rustup/archive/1.18.3/x86_64-unknown-linux-gnu/rustup-init
 echo "${RUSTUP_INIT_SHA256} rustup-init" | sha256sum --check -
 
 chmod +x rustup-init
 sudo -H -u hg -g hg ./rustup-init -y
-sudo -H -u hg -g hg /home/hg/.cargo/bin/rustup install 1.31.1 1.46.0
+sudo -H -u hg -g hg /home/hg/.cargo/bin/rustup install 1.41.1 1.52.0
 sudo -H -u hg -g hg /home/hg/.cargo/bin/rustup component add clippy
-
-sudo -H -u hg -g hg /home/hg/.cargo/bin/cargo install --version 0.10.3 pyoxidizer
 '''
 
 
 BOOTSTRAP_VIRTUALENV = r'''
 /usr/bin/virtualenv /hgdev/venv-bootstrap
 
 HG_SHA256=35fc8ba5e0379c1b3affa2757e83fb0509e8ac314cbd9f1fd133cf265d16e49f
 HG_TARBALL=mercurial-5.1.1.tar.gz
 
 wget -O ${HG_TARBALL} --progress dot:mega https://www.mercurial-scm.org/release/${HG_TARBALL}
 echo "${HG_SHA256} ${HG_TARBALL}" | sha256sum --check -
 
 /hgdev/venv-bootstrap/bin/pip install ${HG_TARBALL}
 '''.lstrip().replace(
     '\r\n', '\n'
 )
 
 
 BOOTSTRAP_DEBIAN = (
     r'''
 #!/bin/bash
 
 set -ex
 
 DISTRO=`grep DISTRIB_ID /etc/lsb-release  | awk -F= '{{print $2}}'`
 DEBIAN_VERSION=`cat /etc/debian_version`
 LSB_RELEASE=`lsb_release -cs`
 
 sudo /usr/sbin/groupadd hg
 sudo /usr/sbin/groupadd docker
 sudo /usr/sbin/useradd -g hg -G sudo,docker -d /home/hg -m -s /bin/bash hg
 sudo mkdir /home/hg/.ssh
 sudo cp ~/.ssh/authorized_keys /home/hg/.ssh/authorized_keys
 sudo chown -R hg:hg /home/hg/.ssh
 sudo chmod 700 /home/hg/.ssh
 sudo chmod 600 /home/hg/.ssh/authorized_keys
 
 cat << EOF | sudo tee /etc/sudoers.d/90-hg
 hg ALL=(ALL) NOPASSWD:ALL
 EOF
 
 sudo apt-get update
 sudo DEBIAN_FRONTEND=noninteractive apt-get -yq dist-upgrade
 
 # Install packages necessary to set up Docker Apt repo.
 sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install --no-install-recommends \
     apt-transport-https \
     gnupg
 
 cat > docker-apt-key << EOF
 -----BEGIN PGP PUBLIC KEY BLOCK-----
 
 mQINBFit2ioBEADhWpZ8/wvZ6hUTiXOwQHXMAlaFHcPH9hAtr4F1y2+OYdbtMuth
 lqqwp028AqyY+PRfVMtSYMbjuQuu5byyKR01BbqYhuS3jtqQmljZ/bJvXqnmiVXh
 38UuLa+z077PxyxQhu5BbqntTPQMfiyqEiU+BKbq2WmANUKQf+1AmZY/IruOXbnq
 L4C1+gJ8vfmXQt99npCaxEjaNRVYfOS8QcixNzHUYnb6emjlANyEVlZzeqo7XKl7
 UrwV5inawTSzWNvtjEjj4nJL8NsLwscpLPQUhTQ+7BbQXAwAmeHCUTQIvvWXqw0N
 cmhh4HgeQscQHYgOJjjDVfoY5MucvglbIgCqfzAHW9jxmRL4qbMZj+b1XoePEtht
 ku4bIQN1X5P07fNWzlgaRL5Z4POXDDZTlIQ/El58j9kp4bnWRCJW0lya+f8ocodo
 vZZ+Doi+fy4D5ZGrL4XEcIQP/Lv5uFyf+kQtl/94VFYVJOleAv8W92KdgDkhTcTD
 G7c0tIkVEKNUq48b3aQ64NOZQW7fVjfoKwEZdOqPE72Pa45jrZzvUFxSpdiNk2tZ
 XYukHjlxxEgBdC/J3cMMNRE1F4NCA3ApfV1Y7/hTeOnmDuDYwr9/obA8t016Yljj
 q5rdkywPf4JF8mXUW5eCN1vAFHxeg9ZWemhBtQmGxXnw9M+z6hWwc6ahmwARAQAB
 tCtEb2NrZXIgUmVsZWFzZSAoQ0UgZGViKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3
 BBMBCgAhBQJYrefAAhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEI2BgDwO
 v82IsskP/iQZo68flDQmNvn8X5XTd6RRaUH33kXYXquT6NkHJciS7E2gTJmqvMqd
 tI4mNYHCSEYxI5qrcYV5YqX9P6+Ko+vozo4nseUQLPH/ATQ4qL0Zok+1jkag3Lgk
 jonyUf9bwtWxFp05HC3GMHPhhcUSexCxQLQvnFWXD2sWLKivHp2fT8QbRGeZ+d3m
 6fqcd5Fu7pxsqm0EUDK5NL+nPIgYhN+auTrhgzhK1CShfGccM/wfRlei9Utz6p9P
 XRKIlWnXtT4qNGZNTN0tR+NLG/6Bqd8OYBaFAUcue/w1VW6JQ2VGYZHnZu9S8LMc
 FYBa5Ig9PxwGQOgq6RDKDbV+PqTQT5EFMeR1mrjckk4DQJjbxeMZbiNMG5kGECA8
 g383P3elhn03WGbEEa4MNc3Z4+7c236QI3xWJfNPdUbXRaAwhy/6rTSFbzwKB0Jm
 ebwzQfwjQY6f55MiI/RqDCyuPj3r3jyVRkK86pQKBAJwFHyqj9KaKXMZjfVnowLh
 9svIGfNbGHpucATqREvUHuQbNnqkCx8VVhtYkhDb9fEP2xBu5VvHbR+3nfVhMut5
 G34Ct5RS7Jt6LIfFdtcn8CaSas/l1HbiGeRgc70X/9aYx/V/CEJv0lIe8gP6uDoW
 FPIZ7d6vH+Vro6xuWEGiuMaiznap2KhZmpkgfupyFmplh0s6knymuQINBFit2ioB
 EADneL9S9m4vhU3blaRjVUUyJ7b/qTjcSylvCH5XUE6R2k+ckEZjfAMZPLpO+/tF
 M2JIJMD4SifKuS3xck9KtZGCufGmcwiLQRzeHF7vJUKrLD5RTkNi23ydvWZgPjtx
 Q+DTT1Zcn7BrQFY6FgnRoUVIxwtdw1bMY/89rsFgS5wwuMESd3Q2RYgb7EOFOpnu
 w6da7WakWf4IhnF5nsNYGDVaIHzpiqCl+uTbf1epCjrOlIzkZ3Z3Yk5CM/TiFzPk
 z2lLz89cpD8U+NtCsfagWWfjd2U3jDapgH+7nQnCEWpROtzaKHG6lA3pXdix5zG8
 eRc6/0IbUSWvfjKxLLPfNeCS2pCL3IeEI5nothEEYdQH6szpLog79xB9dVnJyKJb
 VfxXnseoYqVrRz2VVbUI5Blwm6B40E3eGVfUQWiux54DspyVMMk41Mx7QJ3iynIa
 1N4ZAqVMAEruyXTRTxc9XW0tYhDMA/1GYvz0EmFpm8LzTHA6sFVtPm/ZlNCX6P1X
 zJwrv7DSQKD6GGlBQUX+OeEJ8tTkkf8QTJSPUdh8P8YxDFS5EOGAvhhpMBYD42kQ
 pqXjEC+XcycTvGI7impgv9PDY1RCC1zkBjKPa120rNhv/hkVk/YhuGoajoHyy4h7
 ZQopdcMtpN2dgmhEegny9JCSwxfQmQ0zK0g7m6SHiKMwjwARAQABiQQ+BBgBCAAJ
 BQJYrdoqAhsCAikJEI2BgDwOv82IwV0gBBkBCAAGBQJYrdoqAAoJEH6gqcPyc/zY
 1WAP/2wJ+R0gE6qsce3rjaIz58PJmc8goKrir5hnElWhPgbq7cYIsW5qiFyLhkdp
 YcMmhD9mRiPpQn6Ya2w3e3B8zfIVKipbMBnke/ytZ9M7qHmDCcjoiSmwEXN3wKYI
 mD9VHONsl/CG1rU9Isw1jtB5g1YxuBA7M/m36XN6x2u+NtNMDB9P56yc4gfsZVES
 KA9v+yY2/l45L8d/WUkUi0YXomn6hyBGI7JrBLq0CX37GEYP6O9rrKipfz73XfO7
 JIGzOKZlljb/D9RX/g7nRbCn+3EtH7xnk+TK/50euEKw8SMUg147sJTcpQmv6UzZ
 cM4JgL0HbHVCojV4C/plELwMddALOFeYQzTif6sMRPf+3DSj8frbInjChC3yOLy0
 6br92KFom17EIj2CAcoeq7UPhi2oouYBwPxh5ytdehJkoo+sN7RIWua6P2WSmon5
 U888cSylXC0+ADFdgLX9K2zrDVYUG1vo8CX0vzxFBaHwN6Px26fhIT1/hYUHQR1z
 VfNDcyQmXqkOnZvvoMfz/Q0s9BhFJ/zU6AgQbIZE/hm1spsfgvtsD1frZfygXJ9f
 irP+MSAI80xHSf91qSRZOj4Pl3ZJNbq4yYxv0b1pkMqeGdjdCYhLU+LZ4wbQmpCk
 SVe2prlLureigXtmZfkqevRz7FrIZiu9ky8wnCAPwC7/zmS18rgP/17bOtL4/iIz
 QhxAAoAMWVrGyJivSkjhSGx1uCojsWfsTAm11P7jsruIL61ZzMUVE2aM3Pmj5G+W
 9AcZ58Em+1WsVnAXdUR//bMmhyr8wL/G1YO1V3JEJTRdxsSxdYa4deGBBY/Adpsw
 24jxhOJR+lsJpqIUeb999+R8euDhRHG9eFO7DRu6weatUJ6suupoDTRWtr/4yGqe
 dKxV3qQhNLSnaAzqW/1nA3iUB4k7kCaKZxhdhDbClf9P37qaRW467BLCVO/coL3y
 Vm50dwdrNtKpMBh3ZpbB1uJvgi9mXtyBOMJ3v8RZeDzFiG8HdCtg9RvIt/AIFoHR
 H3S+U79NT6i0KPzLImDfs8T7RlpyuMc4Ufs8ggyg9v3Ae6cN3eQyxcK3w0cbBwsh
 /nQNfsA6uu+9H7NhbehBMhYnpNZyrHzCmzyXkauwRAqoCbGCNykTRwsur9gS41TQ
 M8ssD1jFheOJf3hODnkKU+HKjvMROl1DK7zdmLdNzA1cvtZH/nCC9KPj1z8QC47S
 xx+dTZSx4ONAhwbS/LN3PoKtn8LPjY9NP9uDWI+TWYquS2U+KHDrBDlsgozDbs/O
 jCxcpDzNmXpWQHEtHU7649OXHP7UeNST1mCUCH5qdank0V1iejF6/CfTFU4MfcrG
 YT90qFF93M3v01BbxP+EIY2/9tiIPbrd
 =0YYh
 -----END PGP PUBLIC KEY BLOCK-----
 EOF
 
 sudo apt-key add docker-apt-key
 
 if [ "$LSB_RELEASE" = "stretch" ]; then
 cat << EOF | sudo tee -a /etc/apt/sources.list
 # Need backports for clang-format-6.0
 deb http://deb.debian.org/debian stretch-backports main
 EOF
 fi
 
 if [ "$LSB_RELEASE" = "stretch" -o "$LSB_RELEASE" = "buster" ]; then
 cat << EOF | sudo tee -a /etc/apt/sources.list
 # Sources are useful if we want to compile things locally.
 deb-src http://deb.debian.org/debian $LSB_RELEASE main
 deb-src http://security.debian.org/debian-security $LSB_RELEASE/updates main
 deb-src http://deb.debian.org/debian $LSB_RELEASE-updates main
 deb-src http://deb.debian.org/debian $LSB_RELEASE-backports main
 
 deb [arch=amd64] https://download.docker.com/linux/debian $LSB_RELEASE stable
 EOF
 
 elif [ "$DISTRO" = "Ubuntu" ]; then
 cat << EOF | sudo tee -a /etc/apt/sources.list
 deb [arch=amd64] https://download.docker.com/linux/ubuntu $LSB_RELEASE stable
 EOF
 
 fi
 
 sudo apt-get update
 
 PACKAGES="\
     awscli \
     btrfs-progs \
     build-essential \
     bzr \
     clang-format-6.0 \
     cvs \
     darcs \
     debhelper \
     devscripts \
     docker-ce \
     dpkg-dev \
     dstat \
     emacs \
     gettext \
     git \
     htop \
     iotop \
     jfsutils \
     libbz2-dev \
     libexpat1-dev \
     libffi-dev \
     libgdbm-dev \
     liblzma-dev \
     libncurses5-dev \
     libnss3-dev \
     libreadline-dev \
     libsqlite3-dev \
     libssl-dev \
     netbase \
     ntfs-3g \
     nvme-cli \
     pyflakes \
     pyflakes3 \
     pylint \
     pylint3 \
     python-all-dev \
     python-dev \
     python-docutils \
     python-fuzzywuzzy \
     python-pygments \
     python-subversion \
     python-vcr \
     python3-boto3 \
     python3-dev \
     python3-docutils \
     python3-fuzzywuzzy \
     python3-pygments \
     python3-vcr \
     python3-venv \
     rsync \
     sqlite3 \
     subversion \
     tcl-dev \
     tk-dev \
     tla \
     unzip \
     uuid-dev \
     vim \
     virtualenv \
     wget \
     xfsprogs \
     zip \
     zlib1g-dev"
 
 if [ "LSB_RELEASE" = "stretch" ]; then
     PACKAGES="$PACKAGES linux-perf"
 elif [ "$DISTRO" = "Ubuntu" ]; then
     PACKAGES="$PACKAGES linux-tools-common"
 fi
 
 # Monotone only available in older releases.
 if [ "$LSB_RELEASE" = "stretch" -o "$LSB_RELEASE" = "xenial" ]; then
     PACKAGES="$PACKAGES monotone"
 fi
 
 sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install --no-install-recommends $PACKAGES
 
 # Create clang-format symlink so test harness finds it.
 sudo update-alternatives --install /usr/bin/clang-format clang-format \
     /usr/bin/clang-format-6.0 1000
 
 sudo mkdir /hgdev
 # Will be normalized to hg:hg later.
 sudo chown `whoami` /hgdev
 
 {install_rust}
+{install_pyoxidizer}
 
-cp requirements-py2.txt /hgdev/requirements-py2.txt
-cp requirements-py3.txt /hgdev/requirements-py3.txt
+cp requirements-*.txt /hgdev/
 
 # Disable the pip version check because it uses the network and can
 # be annoying.
 cat << EOF | sudo tee -a /etc/pip.conf
 [global]
 disable-pip-version-check = True
 EOF
 
 {install_pythons}
 {bootstrap_virtualenv}
 
 /hgdev/venv-bootstrap/bin/hg clone https://www.mercurial-scm.org/repo/hg /hgdev/src
 
 # Mark the repo as non-publishing.
 cat >> /hgdev/src/.hg/hgrc << EOF
 [phases]
 publish = false
 EOF
 
 sudo chown -R hg:hg /hgdev
 '''.lstrip()
     .format(
         install_rust=INSTALL_RUST,
+        install_pyoxidizer=INSTALL_PYOXIDIZER,
         install_pythons=INSTALL_PYTHONS,
         bootstrap_virtualenv=BOOTSTRAP_VIRTUALENV,
     )
     .replace('\r\n', '\n')
 )
 
 
 # Prepares /hgdev for operations.
 PREPARE_HGDEV = '''
 #!/bin/bash
 
 set -e
 
 FS=$1
 
 ensure_device() {
     if [ -z "${DEVICE}" ]; then
         echo "could not find block device to format"
         exit 1
     fi
 }
 
 # Determine device to partition for extra filesystem.
 # If only 1 volume is present, it will be the root volume and
 # should be /dev/nvme0. If multiple volumes are present, the
 # root volume could be nvme0 or nvme1. Use whichever one doesn't have
 # a partition.
 if [ -e /dev/nvme1n1 ]; then
     if [ -e /dev/nvme0n1p1 ]; then
         DEVICE=/dev/nvme1n1
     else
         DEVICE=/dev/nvme0n1
     fi
 else
     DEVICE=
 fi
 
 sudo mkdir /hgwork
 
 if [ "${FS}" != "default" -a "${FS}" != "tmpfs" ]; then
     ensure_device
     echo "creating ${FS} filesystem on ${DEVICE}"
 fi
 
 if [ "${FS}" = "default" ]; then
     :
 
 elif [ "${FS}" = "btrfs" ]; then
     sudo mkfs.btrfs ${DEVICE}
     sudo mount ${DEVICE} /hgwork
 
 elif [ "${FS}" = "ext3" ]; then
     # lazy_journal_init speeds up filesystem creation at the expense of
     # integrity if things crash. We are an ephemeral instance, so we don't
     # care about integrity.
     sudo mkfs.ext3 -E lazy_journal_init=1 ${DEVICE}
     sudo mount ${DEVICE} /hgwork
 
 elif [ "${FS}" = "ext4" ]; then
     sudo mkfs.ext4 -E lazy_journal_init=1 ${DEVICE}
     sudo mount ${DEVICE} /hgwork
 
 elif [ "${FS}" = "jfs" ]; then
     sudo mkfs.jfs ${DEVICE}
     sudo mount ${DEVICE} /hgwork
 
 elif [ "${FS}" = "tmpfs" ]; then
     echo "creating tmpfs volume in /hgwork"
     sudo mount -t tmpfs -o size=1024M tmpfs /hgwork
 
 elif [ "${FS}" = "xfs" ]; then
     sudo mkfs.xfs ${DEVICE}
     sudo mount ${DEVICE} /hgwork
 
 else
     echo "unsupported filesystem: ${FS}"
     exit 1
 fi
 
 echo "/hgwork ready"
 
 sudo chown hg:hg /hgwork
 mkdir /hgwork/tmp
 chown hg:hg /hgwork/tmp
 
 rsync -a /hgdev/src /hgwork/
 '''.lstrip().replace(
     '\r\n', '\n'
 )
 
 
 HG_UPDATE_CLEAN = '''
 set -ex
 
 HG=/hgdev/venv-bootstrap/bin/hg
 
 cd /hgwork/src
 ${HG} --config extensions.purge= purge --all
 ${HG} update -C $1
 ${HG} log -r .
 '''.lstrip().replace(
     '\r\n', '\n'
 )
 
 
 def prepare_exec_environment(ssh_client, filesystem='default'):
     """Prepare an EC2 instance to execute things.
 
     The AMI has an ``/hgdev`` bootstrapped with various Python installs
     and a clone of the Mercurial repo.
 
     In EC2, EBS volumes launched from snapshots have wonky performance behavior.
     Notably, blocks have to be copied on first access, which makes volume
     I/O extremely slow on fresh volumes.
 
     Furthermore, we may want to run operations, tests, etc on alternative
     filesystems so we examine behavior on different filesystems.
 
     This function is used to facilitate executing operations on alternate
     volumes.
     """
     sftp = ssh_client.open_sftp()
 
     with sftp.open('/hgdev/prepare-hgdev', 'wb') as fh:
         fh.write(PREPARE_HGDEV)
         fh.chmod(0o0777)
 
     command = 'sudo /hgdev/prepare-hgdev %s' % filesystem
     chan, stdin, stdout = exec_command(ssh_client, command)
     stdin.close()
 
     for line in stdout:
         print(line, end='')
 
     res = chan.recv_exit_status()
 
     if res:
         raise Exception('non-0 exit code updating working directory; %d' % res)
 
 
 def synchronize_hg(
     source_path: pathlib.Path, ec2_instance, revision: str = None
 ):
     """Synchronize a local Mercurial source path to remote EC2 instance."""
 
     with tempfile.TemporaryDirectory() as temp_dir:
         temp_dir = pathlib.Path(temp_dir)
 
         ssh_dir = temp_dir / '.ssh'
         ssh_dir.mkdir()
         ssh_dir.chmod(0o0700)
 
         public_ip = ec2_instance.public_ip_address
 
         ssh_config = ssh_dir / 'config'
 
         with ssh_config.open('w', encoding='utf-8') as fh:
             fh.write('Host %s\n' % public_ip)
             fh.write('  User hg\n')
             fh.write('  StrictHostKeyChecking no\n')
             fh.write('  UserKnownHostsFile %s\n' % (ssh_dir / 'known_hosts'))
             fh.write('  IdentityFile %s\n' % ec2_instance.ssh_private_key_path)
 
         if not (source_path / '.hg').is_dir():
             raise Exception(
                 '%s is not a Mercurial repository; synchronization '
                 'not yet supported' % source_path
             )
 
         env = dict(os.environ)
         env['HGPLAIN'] = '1'
         env['HGENCODING'] = 'utf-8'
 
         hg_bin = source_path / 'hg'
 
         res = subprocess.run(
             ['python2.7', str(hg_bin), 'log', '-r', revision, '-T', '{node}'],
             cwd=str(source_path),
             env=env,
             check=True,
             capture_output=True,
         )
 
         full_revision = res.stdout.decode('ascii')
 
         args = [
             'python2.7',
             str(hg_bin),
             '--config',
             'ui.ssh=ssh -F %s' % ssh_config,
             '--config',
             'ui.remotecmd=/hgdev/venv-bootstrap/bin/hg',
             # Also ensure .hgtags changes are present so auto version
             # calculation works.
             'push',
             '-f',
             '-r',
             full_revision,
             '-r',
             'file(.hgtags)',
             'ssh://%s//hgwork/src' % public_ip,
         ]
 
         res = subprocess.run(args, cwd=str(source_path), env=env)
 
         # Allow 1 (no-op) to not trigger error.
         if res.returncode not in (0, 1):
             res.check_returncode()
 
         # TODO support synchronizing dirty working directory.
 
         sftp = ec2_instance.ssh_client.open_sftp()
 
         with sftp.open('/hgdev/hgup', 'wb') as fh:
             fh.write(HG_UPDATE_CLEAN)
             fh.chmod(0o0700)
 
         chan, stdin, stdout = exec_command(
             ec2_instance.ssh_client, '/hgdev/hgup %s' % full_revision
         )
         stdin.close()
 
         for line in stdout:
             print(line, end='')
 
         res = chan.recv_exit_status()
 
         if res:
             raise Exception(
                 'non-0 exit code updating working directory; %d' % res
             )
 
 
 def run_tests(ssh_client, python_version, test_flags=None):
     """Run tests on a remote Linux machine via an SSH client."""
     test_flags = test_flags or []
 
     print('running tests')
 
     if python_version == 'system2':
         python = '/usr/bin/python2'
     elif python_version == 'system3':
         python = '/usr/bin/python3'
     elif python_version.startswith('pypy'):
         python = '/hgdev/pyenv/shims/%s' % python_version
     else:
         python = '/hgdev/pyenv/shims/python%s' % python_version
 
     test_flags = ' '.join(shlex.quote(a) for a in test_flags)
 
     command = (
         '/bin/sh -c "export TMPDIR=/hgwork/tmp; '
         'cd /hgwork/src/tests && %s run-tests.py %s"' % (python, test_flags)
     )
 
     chan, stdin, stdout = exec_command(ssh_client, command)
 
     stdin.close()
 
     for line in stdout:
         print(line, end='')
 
     return chan.recv_exit_status()

contrib/automation/linux-requirements-py3.txt

@@ -1,215 +1,306 b''
 #
 # This file is autogenerated by pip-compile
 # To update, run:
 #
 #    pip-compile --generate-hashes --output-file=contrib/automation/linux-requirements-py3.txt contrib/automation/linux-requirements.txt.in
 #
 appdirs==1.4.4 \
     --hash=sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41 \
-    --hash=sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128 \
+    --hash=sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128
     # via black
-astroid==2.4.2 \
-    --hash=sha256:2f4078c2a41bf377eea06d71c9d2ba4eb8f6b1af2135bec27bbbb7d8f12bb703 \
-    --hash=sha256:bc58d83eb610252fd8de6363e39d4f1d0619c894b0ed24603b881c02e64c7386 \
+astroid==2.5.6 \
+    --hash=sha256:4db03ab5fc3340cf619dbc25e42c2cc3755154ce6009469766d7143d1fc2ee4e \
+    --hash=sha256:8a398dfce302c13f14bab13e2b14fe385d32b73f4e4853b9bdfb64598baa1975
     # via pylint
-attrs==20.2.0 \
-    --hash=sha256:26b54ddbbb9ee1d34d5d3668dd37d6cf74990ab23c828c2888dccdceee395594 \
-    --hash=sha256:fce7fc47dfc976152e82d53ff92fa0407700c21acd20886a13777a0d20e655dc \
+attrs==21.1.0 \
+    --hash=sha256:3901be1cb7c2a780f14668691474d9252c070a756be0a9ead98cfeabfa11aeb8 \
+    --hash=sha256:8ee1e5f5a1afc5b19bdfae4fdf0c35ed324074bdce3500c939842c8f818645d9
     # via black
 black==19.10b0 ; python_version >= "3.6" and platform_python_implementation != "PyPy" \
     --hash=sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b \
-    --hash=sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539 \
+    --hash=sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539
     # via -r contrib/automation/linux-requirements.txt.in
 click==7.1.2 \
     --hash=sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a \
-    --hash=sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc \
+    --hash=sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc
     # via black
-docutils==0.16 \
-    --hash=sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af \
-    --hash=sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc \
+docutils==0.17.1 \
+    --hash=sha256:686577d2e4c32380bb50cbb22f575ed742d58168cee37e99117a854bcd88f125 \
+    --hash=sha256:cf316c8370a737a022b72b56874f6602acf974a37a9fba42ec2876387549fc61
     # via -r contrib/automation/linux-requirements.txt.in
 fuzzywuzzy==0.18.0 \
     --hash=sha256:45016e92264780e58972dca1b3d939ac864b78437422beecebb3095f8efd00e8 \
-    --hash=sha256:928244b28db720d1e0ee7587acf660ea49d7e4c632569cad4f1cd7e68a5f0993 \
+    --hash=sha256:928244b28db720d1e0ee7587acf660ea49d7e4c632569cad4f1cd7e68a5f0993
     # via -r contrib/automation/linux-requirements.txt.in
-idna==2.10 \
-    --hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \
-    --hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0 \
+idna==3.1 \
+    --hash=sha256:5205d03e7bcbb919cc9c19885f9920d622ca52448306f2377daede5cf3faac16 \
+    --hash=sha256:c5b02147e01ea9920e6b0a3f1f7bb833612d507592c837a6c49552768f4054e1
     # via yarl
 isort==4.3.21 \
     --hash=sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1 \
-    --hash=sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd \
-    # via -r contrib/automation/linux-requirements.txt.in, pylint
-lazy-object-proxy==1.4.3 \
-    --hash=sha256:0c4b206227a8097f05c4dbdd323c50edf81f15db3b8dc064d08c62d37e1a504d \
-    --hash=sha256:194d092e6f246b906e8f70884e620e459fc54db3259e60cf69a4d66c3fda3449 \
-    --hash=sha256:1be7e4c9f96948003609aa6c974ae59830a6baecc5376c25c92d7d697e684c08 \
-    --hash=sha256:4677f594e474c91da97f489fea5b7daa17b5517190899cf213697e48d3902f5a \
-    --hash=sha256:48dab84ebd4831077b150572aec802f303117c8cc5c871e182447281ebf3ac50 \
-    --hash=sha256:5541cada25cd173702dbd99f8e22434105456314462326f06dba3e180f203dfd \
-    --hash=sha256:59f79fef100b09564bc2df42ea2d8d21a64fdcda64979c0fa3db7bdaabaf6239 \
-    --hash=sha256:8d859b89baf8ef7f8bc6b00aa20316483d67f0b1cbf422f5b4dc56701c8f2ffb \
-    --hash=sha256:9254f4358b9b541e3441b007a0ea0764b9d056afdeafc1a5569eee1cc6c1b9ea \
-    --hash=sha256:9651375199045a358eb6741df3e02a651e0330be090b3bc79f6d0de31a80ec3e \
-    --hash=sha256:97bb5884f6f1cdce0099f86b907aa41c970c3c672ac8b9c8352789e103cf3156 \
-    --hash=sha256:9b15f3f4c0f35727d3a0fba4b770b3c4ebbb1fa907dbcc046a1d2799f3edd142 \
-    --hash=sha256:a2238e9d1bb71a56cd710611a1614d1194dc10a175c1e08d75e1a7bcc250d442 \
-    --hash=sha256:a6ae12d08c0bf9909ce12385803a543bfe99b95fe01e752536a60af2b7797c62 \
-    --hash=sha256:ca0a928a3ddbc5725be2dd1cf895ec0a254798915fb3a36af0964a0a4149e3db \
-    --hash=sha256:cb2c7c57005a6804ab66f106ceb8482da55f5314b7fcb06551db1edae4ad1531 \
-    --hash=sha256:d74bb8693bf9cf75ac3b47a54d716bbb1a92648d5f781fc799347cfc95952383 \
-    --hash=sha256:d945239a5639b3ff35b70a88c5f2f491913eb94871780ebfabb2568bd58afc5a \
-    --hash=sha256:eba7011090323c1dadf18b3b689845fd96a61ba0a1dfbd7f24b921398affc357 \
-    --hash=sha256:efa1909120ce98bbb3777e8b6f92237f5d5c8ea6758efea36a473e1d38f7d3e4 \
-    --hash=sha256:f3900e8a5de27447acbf900b4750b0ddfd7ec1ea7fbaf11dfa911141bc522af0 \
+    --hash=sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd
+    # via
+    #   -r contrib/automation/linux-requirements.txt.in
+    #   pylint
+lazy-object-proxy==1.6.0 \
+    --hash=sha256:17e0967ba374fc24141738c69736da90e94419338fd4c7c7bef01ee26b339653 \
+    --hash=sha256:1fee665d2638491f4d6e55bd483e15ef21f6c8c2095f235fef72601021e64f61 \
+    --hash=sha256:22ddd618cefe54305df49e4c069fa65715be4ad0e78e8d252a33debf00f6ede2 \
+    --hash=sha256:24a5045889cc2729033b3e604d496c2b6f588c754f7a62027ad4437a7ecc4837 \
+    --hash=sha256:410283732af311b51b837894fa2f24f2c0039aa7f220135192b38fcc42bd43d3 \
+    --hash=sha256:4732c765372bd78a2d6b2150a6e99d00a78ec963375f236979c0626b97ed8e43 \
+    --hash=sha256:489000d368377571c6f982fba6497f2aa13c6d1facc40660963da62f5c379726 \
+    --hash=sha256:4f60460e9f1eb632584c9685bccea152f4ac2130e299784dbaf9fae9f49891b3 \
+    --hash=sha256:5743a5ab42ae40caa8421b320ebf3a998f89c85cdc8376d6b2e00bd12bd1b587 \
+    --hash=sha256:85fb7608121fd5621cc4377a8961d0b32ccf84a7285b4f1d21988b2eae2868e8 \
+    --hash=sha256:9698110e36e2df951c7c36b6729e96429c9c32b3331989ef19976592c5f3c77a \
+    --hash=sha256:9d397bf41caad3f489e10774667310d73cb9c4258e9aed94b9ec734b34b495fd \
+    --hash=sha256:b579f8acbf2bdd9ea200b1d5dea36abd93cabf56cf626ab9c744a432e15c815f \
+    --hash=sha256:b865b01a2e7f96db0c5d12cfea590f98d8c5ba64ad222300d93ce6ff9138bcad \
+    --hash=sha256:bf34e368e8dd976423396555078def5cfc3039ebc6fc06d1ae2c5a65eebbcde4 \
+    --hash=sha256:c6938967f8528b3668622a9ed3b31d145fab161a32f5891ea7b84f6b790be05b \
+    --hash=sha256:d1c2676e3d840852a2de7c7d5d76407c772927addff8d742b9808fe0afccebdf \
+    --hash=sha256:d7124f52f3bd259f510651450e18e0fd081ed82f3c08541dffc7b94b883aa981 \
+    --hash=sha256:d900d949b707778696fdf01036f58c9876a0d8bfe116e8d220cfd4b15f14e741 \
+    --hash=sha256:ebfd274dcd5133e0afae738e6d9da4323c3eb021b3e13052d8cbd0e457b1256e \
+    --hash=sha256:ed361bb83436f117f9917d282a456f9e5009ea12fd6de8742d1a4752c3017e93 \
+    --hash=sha256:f5144c75445ae3ca2057faac03fda5a902eff196702b0a24daf1d6ce0650514b
     # via astroid
 mccabe==0.6.1 \
     --hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \
-    --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f \
+    --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f
     # via pylint
-multidict==4.7.6 \
-    --hash=sha256:1ece5a3369835c20ed57adadc663400b5525904e53bae59ec854a5d36b39b21a \
-    --hash=sha256:275ca32383bc5d1894b6975bb4ca6a7ff16ab76fa622967625baeebcf8079000 \
-    --hash=sha256:3750f2205b800aac4bb03b5ae48025a64e474d2c6cc79547988ba1d4122a09e2 \
-    --hash=sha256:4538273208e7294b2659b1602490f4ed3ab1c8cf9dbdd817e0e9db8e64be2507 \
-    --hash=sha256:5141c13374e6b25fe6bf092052ab55c0c03d21bd66c94a0e3ae371d3e4d865a5 \
-    --hash=sha256:51a4d210404ac61d32dada00a50ea7ba412e6ea945bbe992e4d7a595276d2ec7 \
-    --hash=sha256:5cf311a0f5ef80fe73e4f4c0f0998ec08f954a6ec72b746f3c179e37de1d210d \
-    --hash=sha256:6513728873f4326999429a8b00fc7ceddb2509b01d5fd3f3be7881a257b8d463 \
-    --hash=sha256:7388d2ef3c55a8ba80da62ecfafa06a1c097c18032a501ffd4cabbc52d7f2b19 \
-    --hash=sha256:9456e90649005ad40558f4cf51dbb842e32807df75146c6d940b6f5abb4a78f3 \
-    --hash=sha256:c026fe9a05130e44157b98fea3ab12969e5b60691a276150db9eda71710cd10b \
-    --hash=sha256:d14842362ed4cf63751648e7672f7174c9818459d169231d03c56e84daf90b7c \
-    --hash=sha256:e0d072ae0f2a179c375f67e3da300b47e1a83293c554450b29c900e50afaae87 \
-    --hash=sha256:f07acae137b71af3bb548bd8da720956a3bc9f9a0b87733e0899226a2317aeb7 \
-    --hash=sha256:fbb77a75e529021e7c4a8d4e823d88ef4d23674a202be4f5addffc72cbb91430 \
-    --hash=sha256:fcfbb44c59af3f8ea984de67ec7c306f618a3ec771c2843804069917a8f2e255 \
-    --hash=sha256:feed85993dbdb1dbc29102f50bca65bdc68f2c0c8d352468c25b54874f23c39d \
+multidict==5.1.0 \
+    --hash=sha256:018132dbd8688c7a69ad89c4a3f39ea2f9f33302ebe567a879da8f4ca73f0d0a \
+    --hash=sha256:051012ccee979b2b06be928a6150d237aec75dd6bf2d1eeeb190baf2b05abc93 \
+    --hash=sha256:05c20b68e512166fddba59a918773ba002fdd77800cad9f55b59790030bab632 \
+    --hash=sha256:07b42215124aedecc6083f1ce6b7e5ec5b50047afa701f3442054373a6deb656 \
+    --hash=sha256:0e3c84e6c67eba89c2dbcee08504ba8644ab4284863452450520dad8f1e89b79 \
+    --hash=sha256:0e929169f9c090dae0646a011c8b058e5e5fb391466016b39d21745b48817fd7 \
+    --hash=sha256:1ab820665e67373de5802acae069a6a05567ae234ddb129f31d290fc3d1aa56d \
+    --hash=sha256:25b4e5f22d3a37ddf3effc0710ba692cfc792c2b9edfb9c05aefe823256e84d5 \
+    --hash=sha256:2e68965192c4ea61fff1b81c14ff712fc7dc15d2bd120602e4a3494ea6584224 \
+    --hash=sha256:2f1a132f1c88724674271d636e6b7351477c27722f2ed789f719f9e3545a3d26 \
+    --hash=sha256:37e5438e1c78931df5d3c0c78ae049092877e5e9c02dd1ff5abb9cf27a5914ea \
+    --hash=sha256:3a041b76d13706b7fff23b9fc83117c7b8fe8d5fe9e6be45eee72b9baa75f348 \
+    --hash=sha256:3a4f32116f8f72ecf2a29dabfb27b23ab7cdc0ba807e8459e59a93a9be9506f6 \
+    --hash=sha256:46c73e09ad374a6d876c599f2328161bcd95e280f84d2060cf57991dec5cfe76 \
+    --hash=sha256:46dd362c2f045095c920162e9307de5ffd0a1bfbba0a6e990b344366f55a30c1 \
+    --hash=sha256:4b186eb7d6ae7c06eb4392411189469e6a820da81447f46c0072a41c748ab73f \
+    --hash=sha256:54fd1e83a184e19c598d5e70ba508196fd0bbdd676ce159feb412a4a6664f952 \
+    --hash=sha256:585fd452dd7782130d112f7ddf3473ffdd521414674c33876187e101b588738a \
+    --hash=sha256:5cf3443199b83ed9e955f511b5b241fd3ae004e3cb81c58ec10f4fe47c7dce37 \
+    --hash=sha256:6a4d5ce640e37b0efcc8441caeea8f43a06addace2335bd11151bc02d2ee31f9 \
+    --hash=sha256:7df80d07818b385f3129180369079bd6934cf70469f99daaebfac89dca288359 \
+    --hash=sha256:806068d4f86cb06af37cd65821554f98240a19ce646d3cd24e1c33587f313eb8 \
+    --hash=sha256:830f57206cc96ed0ccf68304141fec9481a096c4d2e2831f311bde1c404401da \
+    --hash=sha256:929006d3c2d923788ba153ad0de8ed2e5ed39fdbe8e7be21e2f22ed06c6783d3 \
+    --hash=sha256:9436dc58c123f07b230383083855593550c4d301d2532045a17ccf6eca505f6d \
+    --hash=sha256:9dd6e9b1a913d096ac95d0399bd737e00f2af1e1594a787e00f7975778c8b2bf \
+    --hash=sha256:ace010325c787c378afd7f7c1ac66b26313b3344628652eacd149bdd23c68841 \
+    --hash=sha256:b47a43177a5e65b771b80db71e7be76c0ba23cc8aa73eeeb089ed5219cdbe27d \
+    --hash=sha256:b797515be8743b771aa868f83563f789bbd4b236659ba52243b735d80b29ed93 \
+    --hash=sha256:b7993704f1a4b204e71debe6095150d43b2ee6150fa4f44d6d966ec356a8d61f \
+    --hash=sha256:d5c65bdf4484872c4af3150aeebe101ba560dcfb34488d9a8ff8dbcd21079647 \
+    --hash=sha256:d81eddcb12d608cc08081fa88d046c78afb1bf8107e6feab5d43503fea74a635 \
+    --hash=sha256:dc862056f76443a0db4509116c5cd480fe1b6a2d45512a653f9a855cc0517456 \
+    --hash=sha256:ecc771ab628ea281517e24fd2c52e8f31c41e66652d07599ad8818abaad38cda \
+    --hash=sha256:f200755768dc19c6f4e2b672421e0ebb3dd54c38d5a4f262b872d8cfcc9e93b5 \
+    --hash=sha256:f21756997ad8ef815d8ef3d34edd98804ab5ea337feedcd62fb52d22bf531281 \
+    --hash=sha256:fc13a9524bc18b6fb6e0dbec3533ba0496bbed167c56d0aabefd965584557d80
     # via yarl
-pathspec==0.8.0 \
-    --hash=sha256:7d91249d21749788d07a2d0f94147accd8f845507400749ea19c1ec9054a12b0 \
-    --hash=sha256:da45173eb3a6f2a5a487efba21f050af2b41948be6ab52b6a1e3ff22bb8b7061 \
+pathspec==0.8.1 \
+    --hash=sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd \
+    --hash=sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d
     # via black
-pyflakes==2.2.0 \
-    --hash=sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92 \
-    --hash=sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8 \
+pyflakes==2.3.1 \
+    --hash=sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3 \
+    --hash=sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db
     # via -r contrib/automation/linux-requirements.txt.in
-pygments==2.7.1 \
-    --hash=sha256:307543fe65c0947b126e83dd5a61bd8acbd84abec11f43caebaf5534cbc17998 \
-    --hash=sha256:926c3f319eda178d1bd90851e4317e6d8cdb5e292a3386aac9bd75eca29cf9c7 \
+pygments==2.9.0 \
+    --hash=sha256:a18f47b506a429f6f4b9df81bb02beab9ca21d0a5fee38ed15aef65f0545519f \
+    --hash=sha256:d66e804411278594d764fc69ec36ec13d9ae9147193a1740cd34d272ca383b8e
     # via -r contrib/automation/linux-requirements.txt.in
-pylint==2.6.0 \
-    --hash=sha256:bb4a908c9dadbc3aac18860550e870f58e1a02c9f2c204fdf5693d73be061210 \
-    --hash=sha256:bfe68f020f8a0fece830a22dd4d5dddb4ecc6137db04face4c3420a46a52239f \
+pylint==2.8.2 \
+    --hash=sha256:586d8fa9b1891f4b725f587ef267abe2a1bad89d6b184520c7f07a253dd6e217 \
+    --hash=sha256:f7e2072654a6b6afdf5e2fb38147d3e2d2d43c89f648637baab63e026481279b
+    # via -r contrib/automation/linux-requirements.txt.in
+python-levenshtein==0.12.2 \
+    --hash=sha256:dc2395fbd148a1ab31090dd113c366695934b9e85fe5a4b2a032745efd0346f6
     # via -r contrib/automation/linux-requirements.txt.in
-python-levenshtein==0.12.0 \
-    --hash=sha256:033a11de5e3d19ea25c9302d11224e1a1898fe5abd23c61c7c360c25195e3eb1 \
-    # via -r contrib/automation/linux-requirements.txt.in
-pyyaml==5.3.1 \
-    --hash=sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97 \
-    --hash=sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76 \
-    --hash=sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2 \
-    --hash=sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648 \
-    --hash=sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf \
-    --hash=sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f \
-    --hash=sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2 \
-    --hash=sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee \
-    --hash=sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d \
-    --hash=sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c \
-    --hash=sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a \
+pyyaml==5.4.1 \
+    --hash=sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf \
+    --hash=sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696 \
+    --hash=sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393 \
+    --hash=sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77 \
+    --hash=sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922 \
+    --hash=sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5 \
+    --hash=sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8 \
+    --hash=sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10 \
+    --hash=sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc \
+    --hash=sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018 \
+    --hash=sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e \
+    --hash=sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253 \
+    --hash=sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347 \
+    --hash=sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183 \
+    --hash=sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541 \
+    --hash=sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb \
+    --hash=sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185 \
+    --hash=sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc \
+    --hash=sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db \
+    --hash=sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa \
+    --hash=sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46 \
+    --hash=sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122 \
+    --hash=sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b \
+    --hash=sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63 \
+    --hash=sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df \
+    --hash=sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc \
+    --hash=sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247 \
+    --hash=sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6 \
+    --hash=sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0
     # via vcrpy
-regex==2020.9.27 \
-    --hash=sha256:088afc8c63e7bd187a3c70a94b9e50ab3f17e1d3f52a32750b5b77dbe99ef5ef \
-    --hash=sha256:1fe0a41437bbd06063aa184c34804efa886bcc128222e9916310c92cd54c3b4c \
-    --hash=sha256:3d20024a70b97b4f9546696cbf2fd30bae5f42229fbddf8661261b1eaff0deb7 \
-    --hash=sha256:41bb65f54bba392643557e617316d0d899ed5b4946dccee1cb6696152b29844b \
-    --hash=sha256:4318d56bccfe7d43e5addb272406ade7a2274da4b70eb15922a071c58ab0108c \
-    --hash=sha256:4707f3695b34335afdfb09be3802c87fa0bc27030471dbc082f815f23688bc63 \
-    --hash=sha256:49f23ebd5ac073765ecbcf046edc10d63dcab2f4ae2bce160982cb30df0c0302 \
-    --hash=sha256:5533a959a1748a5c042a6da71fe9267a908e21eded7a4f373efd23a2cbdb0ecc \
-    --hash=sha256:5d892a4f1c999834eaa3c32bc9e8b976c5825116cde553928c4c8e7e48ebda67 \
-    --hash=sha256:5f18875ac23d9aa2f060838e8b79093e8bb2313dbaaa9f54c6d8e52a5df097be \
-    --hash=sha256:60b0e9e6dc45683e569ec37c55ac20c582973841927a85f2d8a7d20ee80216ab \
-    --hash=sha256:816064fc915796ea1f26966163f6845de5af78923dfcecf6551e095f00983650 \
-    --hash=sha256:84cada8effefe9a9f53f9b0d2ba9b7b6f5edf8d2155f9fdbe34616e06ececf81 \
-    --hash=sha256:84e9407db1b2eb368b7ecc283121b5e592c9aaedbe8c78b1a2f1102eb2e21d19 \
-    --hash=sha256:8d69cef61fa50c8133382e61fd97439de1ae623fe943578e477e76a9d9471637 \
-    --hash=sha256:9a02d0ae31d35e1ec12a4ea4d4cca990800f66a917d0fb997b20fbc13f5321fc \
-    --hash=sha256:9bc13e0d20b97ffb07821aa3e113f9998e84994fe4d159ffa3d3a9d1b805043b \
-    --hash=sha256:a6f32aea4260dfe0e55dc9733ea162ea38f0ea86aa7d0f77b15beac5bf7b369d \
-    --hash=sha256:ae91972f8ac958039920ef6e8769277c084971a142ce2b660691793ae44aae6b \
-    --hash=sha256:c570f6fa14b9c4c8a4924aaad354652366577b4f98213cf76305067144f7b100 \
-    --hash=sha256:c9443124c67b1515e4fe0bb0aa18df640965e1030f468a2a5dc2589b26d130ad \
-    --hash=sha256:d23a18037313714fb3bb5a94434d3151ee4300bae631894b1ac08111abeaa4a3 \
-    --hash=sha256:eaf548d117b6737df379fdd53bdde4f08870e66d7ea653e230477f071f861121 \
-    --hash=sha256:ebbe29186a3d9b0c591e71b7393f1ae08c83cb2d8e517d2a822b8f7ec99dfd8b \
-    --hash=sha256:eda4771e0ace7f67f58bc5b560e27fb20f32a148cbc993b0c3835970935c2707 \
-    --hash=sha256:f1b3afc574a3db3b25c89161059d857bd4909a1269b0b3cb3c904677c8c4a3f7 \
-    --hash=sha256:f2388013e68e750eaa16ccbea62d4130180c26abb1d8e5d584b9baf69672b30f \
+regex==2021.4.4 \
+    --hash=sha256:01afaf2ec48e196ba91b37451aa353cb7eda77efe518e481707e0515025f0cd5 \
+    --hash=sha256:11d773d75fa650cd36f68d7ca936e3c7afaae41b863b8c387a22aaa78d3c5c79 \
+    --hash=sha256:18c071c3eb09c30a264879f0d310d37fe5d3a3111662438889ae2eb6fc570c31 \
+    --hash=sha256:1e1c20e29358165242928c2de1482fb2cf4ea54a6a6dea2bd7a0e0d8ee321500 \
+    --hash=sha256:281d2fd05555079448537fe108d79eb031b403dac622621c78944c235f3fcf11 \
+    --hash=sha256:314d66636c494ed9c148a42731b3834496cc9a2c4251b1661e40936814542b14 \
+    --hash=sha256:32e65442138b7b76dd8173ffa2cf67356b7bc1768851dded39a7a13bf9223da3 \
+    --hash=sha256:339456e7d8c06dd36a22e451d58ef72cef293112b559010db3d054d5560ef439 \
+    --hash=sha256:3916d08be28a1149fb97f7728fca1f7c15d309a9f9682d89d79db75d5e52091c \
+    --hash=sha256:3a9cd17e6e5c7eb328517969e0cb0c3d31fd329298dd0c04af99ebf42e904f82 \
+    --hash=sha256:47bf5bf60cf04d72bf6055ae5927a0bd9016096bf3d742fa50d9bf9f45aa0711 \
+    --hash=sha256:4c46e22a0933dd783467cf32b3516299fb98cfebd895817d685130cc50cd1093 \
+    --hash=sha256:4c557a7b470908b1712fe27fb1ef20772b78079808c87d20a90d051660b1d69a \
+    --hash=sha256:52ba3d3f9b942c49d7e4bc105bb28551c44065f139a65062ab7912bef10c9afb \
+    --hash=sha256:563085e55b0d4fb8f746f6a335893bda5c2cef43b2f0258fe1020ab1dd874df8 \
+    --hash=sha256:598585c9f0af8374c28edd609eb291b5726d7cbce16be6a8b95aa074d252ee17 \
+    --hash=sha256:619d71c59a78b84d7f18891fe914446d07edd48dc8328c8e149cbe0929b4e000 \
+    --hash=sha256:67bdb9702427ceddc6ef3dc382455e90f785af4c13d495f9626861763ee13f9d \
+    --hash=sha256:6d1b01031dedf2503631d0903cb563743f397ccaf6607a5e3b19a3d76fc10480 \
+    --hash=sha256:741a9647fcf2e45f3a1cf0e24f5e17febf3efe8d4ba1281dcc3aa0459ef424dc \
+    --hash=sha256:7c2a1af393fcc09e898beba5dd59196edaa3116191cc7257f9224beaed3e1aa0 \
+    --hash=sha256:7d9884d86dd4dd489e981d94a65cd30d6f07203d90e98f6f657f05170f6324c9 \
+    --hash=sha256:90f11ff637fe8798933fb29f5ae1148c978cccb0452005bf4c69e13db951e765 \
+    --hash=sha256:919859aa909429fb5aa9cf8807f6045592c85ef56fdd30a9a3747e513db2536e \
+    --hash=sha256:96fcd1888ab4d03adfc9303a7b3c0bd78c5412b2bfbe76db5b56d9eae004907a \
+    --hash=sha256:97f29f57d5b84e73fbaf99ab3e26134e6687348e95ef6b48cfd2c06807005a07 \
+    --hash=sha256:980d7be47c84979d9136328d882f67ec5e50008681d94ecc8afa8a65ed1f4a6f \
+    --hash=sha256:a91aa8619b23b79bcbeb37abe286f2f408d2f2d6f29a17237afda55bb54e7aac \
+    --hash=sha256:ade17eb5d643b7fead300a1641e9f45401c98eee23763e9ed66a43f92f20b4a7 \
+    --hash=sha256:b9c3db21af35e3b3c05764461b262d6f05bbca08a71a7849fd79d47ba7bc33ed \
+    --hash=sha256:bd28bc2e3a772acbb07787c6308e00d9626ff89e3bfcdebe87fa5afbfdedf968 \
+    --hash=sha256:bf5824bfac591ddb2c1f0a5f4ab72da28994548c708d2191e3b87dd207eb3ad7 \
+    --hash=sha256:c0502c0fadef0d23b128605d69b58edb2c681c25d44574fc673b0e52dce71ee2 \
+    --hash=sha256:c38c71df845e2aabb7fb0b920d11a1b5ac8526005e533a8920aea97efb8ec6a4 \
+    --hash=sha256:ce15b6d103daff8e9fee13cf7f0add05245a05d866e73926c358e871221eae87 \
+    --hash=sha256:d3029c340cfbb3ac0a71798100ccc13b97dddf373a4ae56b6a72cf70dfd53bc8 \
+    --hash=sha256:e512d8ef5ad7b898cdb2d8ee1cb09a8339e4f8be706d27eaa180c2f177248a10 \
+    --hash=sha256:e8e5b509d5c2ff12f8418006d5a90e9436766133b564db0abaec92fd27fcee29 \
+    --hash=sha256:ee54ff27bf0afaf4c3b3a62bcd016c12c3fdb4ec4f413391a90bd38bc3624605 \
+    --hash=sha256:fa4537fb4a98fe8fde99626e4681cc644bdcf2a795038533f9f711513a862ae6 \
+    --hash=sha256:fd45ff9293d9274c5008a2054ecef86a9bfe819a67c7be1afb65e69b405b3042
     # via black
-six==1.15.0 \
-    --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
-    --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
-    # via astroid, vcrpy
-toml==0.10.1 \
-    --hash=sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f \
-    --hash=sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88 \
-    # via black, pylint
-typed-ast==1.4.1 ; python_version >= "3.0" and platform_python_implementation != "PyPy" \
-    --hash=sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355 \
-    --hash=sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919 \
-    --hash=sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa \
-    --hash=sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652 \
-    --hash=sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75 \
-    --hash=sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01 \
-    --hash=sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d \
-    --hash=sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1 \
-    --hash=sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907 \
-    --hash=sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c \
-    --hash=sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3 \
-    --hash=sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b \
-    --hash=sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614 \
-    --hash=sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb \
-    --hash=sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b \
-    --hash=sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41 \
-    --hash=sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6 \
-    --hash=sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34 \
-    --hash=sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe \
-    --hash=sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4 \
-    --hash=sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7 \
-    # via -r contrib/automation/linux-requirements.txt.in, astroid, black
-typing-extensions==3.7.4.3 \
-    --hash=sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918 \
-    --hash=sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c \
-    --hash=sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f \
+six==1.16.0 \
+    --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
+    --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
+    # via vcrpy
+toml==0.10.2 \
+    --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \
+    --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f
+    # via
+    #   black
+    #   pylint
+typed-ast==1.4.3 ; python_version >= "3.0" and platform_python_implementation != "PyPy" \
+    --hash=sha256:01ae5f73431d21eead5015997ab41afa53aa1fbe252f9da060be5dad2c730ace \
+    --hash=sha256:067a74454df670dcaa4e59349a2e5c81e567d8d65458d480a5b3dfecec08c5ff \
+    --hash=sha256:0fb71b8c643187d7492c1f8352f2c15b4c4af3f6338f21681d3681b3dc31a266 \
+    --hash=sha256:1b3ead4a96c9101bef08f9f7d1217c096f31667617b58de957f690c92378b528 \
+    --hash=sha256:2068531575a125b87a41802130fa7e29f26c09a2833fea68d9a40cf33902eba6 \
+    --hash=sha256:209596a4ec71d990d71d5e0d312ac935d86930e6eecff6ccc7007fe54d703808 \
+    --hash=sha256:2c726c276d09fc5c414693a2de063f521052d9ea7c240ce553316f70656c84d4 \
+    --hash=sha256:398e44cd480f4d2b7ee8d98385ca104e35c81525dd98c519acff1b79bdaac363 \
+    --hash=sha256:52b1eb8c83f178ab787f3a4283f68258525f8d70f778a2f6dd54d3b5e5fb4341 \
+    --hash=sha256:5feca99c17af94057417d744607b82dd0a664fd5e4ca98061480fd8b14b18d04 \
+    --hash=sha256:7538e495704e2ccda9b234b82423a4038f324f3a10c43bc088a1636180f11a41 \
+    --hash=sha256:760ad187b1041a154f0e4d0f6aae3e40fdb51d6de16e5c99aedadd9246450e9e \
+    --hash=sha256:777a26c84bea6cd934422ac2e3b78863a37017618b6e5c08f92ef69853e765d3 \
+    --hash=sha256:95431a26309a21874005845c21118c83991c63ea800dd44843e42a916aec5899 \
+    --hash=sha256:9ad2c92ec681e02baf81fdfa056fe0d818645efa9af1f1cd5fd6f1bd2bdfd805 \
+    --hash=sha256:9c6d1a54552b5330bc657b7ef0eae25d00ba7ffe85d9ea8ae6540d2197a3788c \
+    --hash=sha256:aee0c1256be6c07bd3e1263ff920c325b59849dc95392a05f258bb9b259cf39c \
+    --hash=sha256:af3d4a73793725138d6b334d9d247ce7e5f084d96284ed23f22ee626a7b88e39 \
+    --hash=sha256:b36b4f3920103a25e1d5d024d155c504080959582b928e91cb608a65c3a49e1a \
+    --hash=sha256:b9574c6f03f685070d859e75c7f9eeca02d6933273b5e69572e5ff9d5e3931c3 \
+    --hash=sha256:bff6ad71c81b3bba8fa35f0f1921fb24ff4476235a6e94a26ada2e54370e6da7 \
+    --hash=sha256:c190f0899e9f9f8b6b7863debfb739abcb21a5c054f911ca3596d12b8a4c4c7f \
+    --hash=sha256:c907f561b1e83e93fad565bac5ba9c22d96a54e7ea0267c708bffe863cbe4075 \
+    --hash=sha256:cae53c389825d3b46fb37538441f75d6aecc4174f615d048321b716df2757fb0 \
+    --hash=sha256:dd4a21253f42b8d2b48410cb31fe501d32f8b9fbeb1f55063ad102fe9c425e40 \
+    --hash=sha256:dde816ca9dac1d9c01dd504ea5967821606f02e510438120091b84e852367428 \
+    --hash=sha256:f2362f3cb0f3172c42938946dbc5b7843c2a28aec307c49100c8b38764eb6927 \
+    --hash=sha256:f328adcfebed9f11301eaedfa48e15bdece9b519fb27e6a8c01aa52a17ec31b3 \
+    --hash=sha256:f8afcf15cc511ada719a88e013cec87c11aff7b91f019295eb4530f96fe5ef2f \
+    --hash=sha256:fb1bbeac803adea29cedd70781399c99138358c26d05fcbd23c13016b7f5ec65
+    # via
+    #   -r contrib/automation/linux-requirements.txt.in
+    #   astroid
+    #   black
+typing-extensions==3.10.0.0 \
+    --hash=sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497 \
+    --hash=sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342 \
+    --hash=sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84
     # via yarl
-vcrpy==4.1.0 \
-    --hash=sha256:4138e79eb35981ad391406cbb7227bce7eba8bad788dcf1a89c2e4a8b740debe \
-    --hash=sha256:d833248442bbc560599add895c9ab0ef518676579e8dc72d8b0933bdb3880253 \
+vcrpy==4.1.1 \
+    --hash=sha256:12c3fcdae7b88ecf11fc0d3e6d77586549d4575a2ceee18e82eee75c1f626162 \
+    --hash=sha256:57095bf22fc0a2d99ee9674cdafebed0f3ba763018582450706f7d3a74fff599
     # via -r contrib/automation/linux-requirements.txt.in
 wrapt==1.12.1 \
-    --hash=sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7 \
-    # via astroid, vcrpy
-yarl==1.6.0 \
-    --hash=sha256:04a54f126a0732af75e5edc9addeaa2113e2ca7c6fce8974a63549a70a25e50e \
-    --hash=sha256:3cc860d72ed989f3b1f3abbd6ecf38e412de722fb38b8f1b1a086315cf0d69c5 \
-    --hash=sha256:5d84cc36981eb5a8533be79d6c43454c8e6a39ee3118ceaadbd3c029ab2ee580 \
-    --hash=sha256:5e447e7f3780f44f890360ea973418025e8c0cdcd7d6a1b221d952600fd945dc \
-    --hash=sha256:61d3ea3c175fe45f1498af868879c6ffeb989d4143ac542163c45538ba5ec21b \
-    --hash=sha256:67c5ea0970da882eaf9efcf65b66792557c526f8e55f752194eff8ec722c75c2 \
-    --hash=sha256:6f6898429ec3c4cfbef12907047136fd7b9e81a6ee9f105b45505e633427330a \
-    --hash=sha256:7ce35944e8e61927a8f4eb78f5bc5d1e6da6d40eadd77e3f79d4e9399e263921 \
-    --hash=sha256:b7c199d2cbaf892ba0f91ed36d12ff41ecd0dde46cbf64ff4bfe997a3ebc925e \
-    --hash=sha256:c15d71a640fb1f8e98a1423f9c64d7f1f6a3a168f803042eaf3a5b5022fde0c1 \
-    --hash=sha256:c22607421f49c0cb6ff3ed593a49b6a99c6ffdeaaa6c944cdda83c2393c8864d \
-    --hash=sha256:c604998ab8115db802cc55cb1b91619b2831a6128a62ca7eea577fc8ea4d3131 \
-    --hash=sha256:d088ea9319e49273f25b1c96a3763bf19a882cff774d1792ae6fba34bd40550a \
-    --hash=sha256:db9eb8307219d7e09b33bcb43287222ef35cbcf1586ba9472b0a4b833666ada1 \
-    --hash=sha256:e31fef4e7b68184545c3d68baec7074532e077bd1906b040ecfba659737df188 \
-    --hash=sha256:e32f0fb443afcfe7f01f95172b66f279938fbc6bdaebe294b0ff6747fb6db020 \
-    --hash=sha256:fcbe419805c9b20db9a51d33b942feddbf6e7fb468cb20686fd7089d4164c12a \
+    --hash=sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7
+    # via
+    #   astroid
+    #   vcrpy
+yarl==1.6.3 \
+    --hash=sha256:00d7ad91b6583602eb9c1d085a2cf281ada267e9a197e8b7cae487dadbfa293e \
+    --hash=sha256:0355a701b3998dcd832d0dc47cc5dedf3874f966ac7f870e0f3a6788d802d434 \
+    --hash=sha256:15263c3b0b47968c1d90daa89f21fcc889bb4b1aac5555580d74565de6836366 \
+    --hash=sha256:2ce4c621d21326a4a5500c25031e102af589edb50c09b321049e388b3934eec3 \
+    --hash=sha256:31ede6e8c4329fb81c86706ba8f6bf661a924b53ba191b27aa5fcee5714d18ec \
+    --hash=sha256:324ba3d3c6fee56e2e0b0d09bf5c73824b9f08234339d2b788af65e60040c959 \
+    --hash=sha256:329412812ecfc94a57cd37c9d547579510a9e83c516bc069470db5f75684629e \
+    --hash=sha256:4736eaee5626db8d9cda9eb5282028cc834e2aeb194e0d8b50217d707e98bb5c \
+    --hash=sha256:4953fb0b4fdb7e08b2f3b3be80a00d28c5c8a2056bb066169de00e6501b986b6 \
+    --hash=sha256:4c5bcfc3ed226bf6419f7a33982fb4b8ec2e45785a0561eb99274ebbf09fdd6a \
+    --hash=sha256:547f7665ad50fa8563150ed079f8e805e63dd85def6674c97efd78eed6c224a6 \
+    --hash=sha256:5b883e458058f8d6099e4420f0cc2567989032b5f34b271c0827de9f1079a424 \
+    --hash=sha256:63f90b20ca654b3ecc7a8d62c03ffa46999595f0167d6450fa8383bab252987e \
+    --hash=sha256:68dc568889b1c13f1e4745c96b931cc94fdd0defe92a72c2b8ce01091b22e35f \
+    --hash=sha256:69ee97c71fee1f63d04c945f56d5d726483c4762845400a6795a3b75d56b6c50 \
+    --hash=sha256:6d6283d8e0631b617edf0fd726353cb76630b83a089a40933043894e7f6721e2 \
+    --hash=sha256:72a660bdd24497e3e84f5519e57a9ee9220b6f3ac4d45056961bf22838ce20cc \
+    --hash=sha256:73494d5b71099ae8cb8754f1df131c11d433b387efab7b51849e7e1e851f07a4 \
+    --hash=sha256:7356644cbed76119d0b6bd32ffba704d30d747e0c217109d7979a7bc36c4d970 \
+    --hash=sha256:8a9066529240171b68893d60dca86a763eae2139dd42f42106b03cf4b426bf10 \
+    --hash=sha256:8aa3decd5e0e852dc68335abf5478a518b41bf2ab2f330fe44916399efedfae0 \
+    --hash=sha256:97b5bdc450d63c3ba30a127d018b866ea94e65655efaf889ebeabc20f7d12406 \
+    --hash=sha256:9ede61b0854e267fd565e7527e2f2eb3ef8858b301319be0604177690e1a3896 \
+    --hash=sha256:b2e9a456c121e26d13c29251f8267541bd75e6a1ccf9e859179701c36a078643 \
+    --hash=sha256:b5dfc9a40c198334f4f3f55880ecf910adebdcb2a0b9a9c23c9345faa9185721 \
+    --hash=sha256:bafb450deef6861815ed579c7a6113a879a6ef58aed4c3a4be54400ae8871478 \
+    --hash=sha256:c49ff66d479d38ab863c50f7bb27dee97c6627c5fe60697de15529da9c3de724 \
+    --hash=sha256:ce3beb46a72d9f2190f9e1027886bfc513702d748047b548b05dab7dfb584d2e \
+    --hash=sha256:d26608cf178efb8faa5ff0f2d2e77c208f471c5a3709e577a7b3fd0445703ac8 \
+    --hash=sha256:d597767fcd2c3dc49d6eea360c458b65643d1e4dbed91361cf5e36e53c1f8c96 \
+    --hash=sha256:d5c32c82990e4ac4d8150fd7652b972216b204de4e83a122546dce571c1bdf25 \
+    --hash=sha256:d8d07d102f17b68966e2de0e07bfd6e139c7c02ef06d3a0f8d2f0f055e13bb76 \
+    --hash=sha256:e46fba844f4895b36f4c398c5af062a9808d1f26b2999c58909517384d5deda2 \
+    --hash=sha256:e6b5460dc5ad42ad2b36cca524491dfcaffbfd9c8df50508bddc354e787b8dc2 \
+    --hash=sha256:f040bcc6725c821a4c0665f3aa96a4d0805a7aaf2caf266d256b8ed71b9f041c \
+    --hash=sha256:f0b059678fd549c66b89bed03efcabb009075bd131c248ecdf087bdb6faba24a \
+    --hash=sha256:fcbb48a93e8699eae920f8d92f7160c03567b421bc17362a9ffbbd706a816f71
     # via vcrpy
 
 # WARNING: The following packages were not pinned, but pip requires them to be
 # pinned when the requirements file includes hashes. Consider using the --allow-unsafe flag.
 # setuptools

contrib/check-code.py

@@ -1,1109 +1,1131 b''
 #!/usr/bin/env python3
 #
 # check-code - a style and portability checker for Mercurial
 #
 # Copyright 2010 Olivia Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """style and portability checker for Mercurial
 
 when a rule triggers wrong, do one of the following (prefer one from top):
  * do the work-around the rule suggests
  * doublecheck that it is a false match
  * improve the rule pattern
  * add an ignore pattern to the rule (3rd arg) which matches your good line
    (you can append a short comment and match this, like: #re-raises)
  * change the pattern to a warning and list the exception in test-check-code-hg
  * ONLY use no--check-code for skipping entire files from external sources
 """
 
 from __future__ import absolute_import, print_function
 import glob
 import keyword
 import optparse
 import os
 import re
 import sys
 
 if sys.version_info[0] < 3:
     opentext = open
 else:
 
     def opentext(f):
         return open(f, encoding='latin1')
 
 
 try:
     xrange
 except NameError:
     xrange = range
 try:
     import re2
 except ImportError:
     re2 = None
 
 import testparseutil
 
 
 def compilere(pat, multiline=False):
     if multiline:
         pat = '(?m)' + pat
     if re2:
         try:
             return re2.compile(pat)
         except re2.error:
             pass
     return re.compile(pat)
 
 
 # check "rules depending on implementation of repquote()" in each
 # patterns (especially pypats), before changing around repquote()
 _repquotefixedmap = {
     ' ': ' ',
     '\n': '\n',
     '.': 'p',
     ':': 'q',
     '%': '%',
     '\\': 'b',
     '*': 'A',
     '+': 'P',
     '-': 'M',
 }
 
 
 def _repquoteencodechr(i):
     if i > 255:
         return 'u'
     c = chr(i)
     if c in _repquotefixedmap:
         return _repquotefixedmap[c]
     if c.isalpha():
         return 'x'
     if c.isdigit():
         return 'n'
     return 'o'
 
 
 _repquotett = ''.join(_repquoteencodechr(i) for i in xrange(256))
 
 
 def repquote(m):
     t = m.group('text')
     t = t.translate(_repquotett)
     return m.group('quote') + t + m.group('quote')
 
 
 def reppython(m):
     comment = m.group('comment')
     if comment:
         l = len(comment.rstrip())
         return "#" * l + comment[l:]
     return repquote(m)
 
 
 def repcomment(m):
     return m.group(1) + "#" * len(m.group(2))
 
 
 def repccomment(m):
     t = re.sub(r"((?<=\n) )|\S", "x", m.group(2))
     return m.group(1) + t + "*/"
 
 
 def repcallspaces(m):
     t = re.sub(r"\n\s+", "\n", m.group(2))
     return m.group(1) + t
 
 
 def repinclude(m):
     return m.group(1) + "<foo>"
 
 
 def rephere(m):
     t = re.sub(r"\S", "x", m.group(2))
     return m.group(1) + t
 
 
 testpats = [
     [
         (r'\b(push|pop)d\b', "don't use 'pushd' or 'popd', use 'cd'"),
         (r'\W\$?\(\([^\)\n]*\)\)', "don't use (()) or $(()), use 'expr'"),
         (r'grep.*-q', "don't use 'grep -q', redirect to /dev/null"),
         (r'(?<!hg )grep.* -a', "don't use 'grep -a', use in-line python"),
         (r'sed.*-i', "don't use 'sed -i', use a temporary file"),
         (r'\becho\b.*\\n', "don't use 'echo \\n', use printf"),
         (r'echo -n', "don't use 'echo -n', use printf"),
         (r'(^|\|\s*)\bwc\b[^|]*$\n(?!.*\(re\))', "filter wc output"),
         (r'head -c', "don't use 'head -c', use 'dd'"),
         (r'tail -n', "don't use the '-n' option to tail, just use '-<num>'"),
         (r'sha1sum', "don't use sha1sum, use $TESTDIR/md5sum.py"),
         (r'\bls\b.*-\w*R', "don't use 'ls -R', use 'find'"),
         (r'printf.*[^\\]\\([1-9]|0\d)', r"don't use 'printf \NNN', use Python"),
         (r'printf.*[^\\]\\x', "don't use printf \\x, use Python"),
         (r'rm -rf \*', "don't use naked rm -rf, target a directory"),
         (
             r'\[[^\]]+==',
             '[ foo == bar ] is a bashism, use [ foo = bar ] instead',
         ),
         (
             r'(^|\|\s*)grep (-\w\s+)*[^|]*[(|]\w',
             "use egrep for extended grep syntax",
         ),
         (r'(^|\|\s*)e?grep .*\\S', "don't use \\S in regular expression"),
         (r'(?<!!)/bin/', "don't use explicit paths for tools"),
         (r'#!.*/bash', "don't use bash in shebang, use sh"),
         (r'[^\n]\Z', "no trailing newline"),
         (r'export .*=', "don't export and assign at once"),
         (r'^source\b', "don't use 'source', use '.'"),
         (r'touch -d', "don't use 'touch -d', use 'touch -t' instead"),
         (r'\bls +[^|\n-]+ +-', "options to 'ls' must come before filenames"),
         (r'[^>\n]>\s*\$HGRCPATH', "don't overwrite $HGRCPATH, append to it"),
         (r'^stop\(\)', "don't use 'stop' as a shell function name"),
         (r'(\[|\btest\b).*-e ', "don't use 'test -e', use 'test -f'"),
         (r'\[\[\s+[^\]]*\]\]', "don't use '[[ ]]', use '[ ]'"),
         (r'^alias\b.*=', "don't use alias, use a function"),
         (r'if\s*!', "don't use '!' to negate exit status"),
         (r'/dev/u?random', "don't use entropy, use /dev/zero"),
         (r'do\s*true;\s*done', "don't use true as loop body, use sleep 0"),
         (
             r'sed (-e )?\'(\d+|/[^/]*/)i(?!\\\n)',
             "put a backslash-escaped newline after sed 'i' command",
         ),
         (r'^diff *-\w*[uU].*$\n(^  \$ |^$)', "prefix diff -u/-U with cmp"),
         (r'^\s+(if)? diff *-\w*[uU]', "prefix diff -u/-U with cmp"),
         (r'[\s="`\']python\s(?!bindings)', "don't use 'python', use '$PYTHON'"),
         (r'seq ', "don't use 'seq', use $TESTDIR/seq.py"),
         (r'\butil\.Abort\b', "directly use error.Abort"),
         (r'\|&', "don't use |&, use 2>&1"),
         (r'\w =  +\w', "only one space after = allowed"),
         (
             r'\bsed\b.*[^\\]\\n',
             "don't use 'sed ... \\n', use a \\ and a newline",
         ),
         (r'env.*-u', "don't use 'env -u VAR', use 'unset VAR'"),
         (r'cp.* -r ', "don't use 'cp -r', use 'cp -R'"),
         (r'grep.* -[ABC]', "don't use grep's context flags"),
         (
             r'find.*-printf',
             "don't use 'find -printf', it doesn't exist on BSD find(1)",
         ),
         (r'\$RANDOM ', "don't use bash-only $RANDOM to generate random values"),
     ],
     # warnings
     [
         (r'^function', "don't use 'function', use old style"),
         (r'^diff.*-\w*N', "don't use 'diff -N'"),
         (r'\$PWD|\${PWD}', "don't use $PWD, use `pwd`"),
         (r'^([^"\'\n]|("[^"\n]*")|(\'[^\'\n]*\'))*\^', "^ must be quoted"),
         (r'kill (`|\$\()', "don't use kill, use killdaemons.py"),
     ],
 ]
 
 testfilters = [
     (r"( *)(#([^!][^\n]*\S)?)", repcomment),
     (r"<<(\S+)((.|\n)*?\n\1)", rephere),
 ]
 
 uprefix = r"^  \$ "
 utestpats = [
     [
         (r'^(\S.*||  [$>] \S.*)[ \t]\n', "trailing whitespace on non-output"),
         (
             uprefix + r'.*\|\s*sed[^|>\n]*\n',
             "use regex test output patterns instead of sed",
         ),
         (uprefix + r'(true|exit 0)', "explicit zero exit unnecessary"),
-        (uprefix + r'.*(?<!\[)\$\?', "explicit exit code checks unnecessary"),
         (
             uprefix + r'.*\|\| echo.*(fail|error)',
             "explicit exit code checks unnecessary",
         ),
         (uprefix + r'set -e', "don't use set -e"),
         (uprefix + r'(\s|fi\b|done\b)', "use > for continued lines"),
         (
             uprefix + r'.*:\.\S*/',
             "x:.y in a path does not work on msys, rewrite "
             "as x://.y, or see `hg log -k msys` for alternatives",
             r'-\S+:\.|' '# no-msys',  # -Rxxx
         ),  # in test-pull.t which is skipped on windows
         (
             r'^  [^$>].*27\.0\.0\.1',
             'use $LOCALIP not an explicit loopback address',
         ),
         (
             r'^  (?![>$] ).*\$LOCALIP.*[^)]$',
             'mark $LOCALIP output lines with (glob) to help tests in BSD jails',
         ),
         (
             r'^  (cat|find): .*: \$ENOENT\$',
             'use test -f to test for file existence',
         ),
         (
             r'^  diff -[^ -]*p',
             "don't use (external) diff with -p for portability",
         ),
         (r' readlink ', 'use readlink.py instead of readlink'),
         (
             r'^  [-+][-+][-+] .* [-+]0000 \(glob\)',
             "glob timezone field in diff output for portability",
         ),
         (
             r'^  @@ -[0-9]+ [+][0-9]+,[0-9]+ @@',
             "use '@@ -N* +N,n @@ (glob)' style chunk header for portability",
         ),
         (
             r'^  @@ -[0-9]+,[0-9]+ [+][0-9]+ @@',
             "use '@@ -N,n +N* @@ (glob)' style chunk header for portability",
         ),
         (
             r'^  @@ -[0-9]+ [+][0-9]+ @@',
             "use '@@ -N* +N* @@ (glob)' style chunk header for portability",
         ),
         (
             uprefix + r'hg( +-[^ ]+( +[^ ]+)?)* +extdiff'
             r'( +(-[^ po-]+|--(?!program|option)[^ ]+|[^-][^ ]*))*$',
             "use $RUNTESTDIR/pdiff via extdiff (or -o/-p for false-positives)",
         ),
     ],
     # warnings
     [
         (
             r'^  (?!.*\$LOCALIP)[^*?/\n]* \(glob\)$',
             "glob match with no glob string (?, *, /, and $LOCALIP)",
         ),
     ],
 ]
 
 # transform plain test rules to unified test's
 for i in [0, 1]:
     for tp in testpats[i]:
         p = tp[0]
         m = tp[1]
         if p.startswith('^'):
             p = "^  [$>] (%s)" % p[1:]
         else:
             p = "^  [$>] .*(%s)" % p
         utestpats[i].append((p, m) + tp[2:])
 
 # don't transform the following rules:
 # "  > \t" and "  \t" should be allowed in unified tests
 testpats[0].append((r'^( *)\t', "don't use tabs to indent"))
 utestpats[0].append((r'^( ?)\t', "don't use tabs to indent"))
 
 utestfilters = [
     (r"<<(\S+)((.|\n)*?\n  > \1)", rephere),
     (r"( +)(#([^!][^\n]*\S)?)", repcomment),
 ]
 
 # common patterns to check *.py
 commonpypats = [
     [
         (r'\\$', 'Use () to wrap long lines in Python, not \\'),
         (
             r'^\s*def\s*\w+\s*\(.*,\s*\(',
             "tuple parameter unpacking not available in Python 3+",
         ),
         (
             r'lambda\s*\(.*,.*\)',
             "tuple parameter unpacking not available in Python 3+",
         ),
         (r'(?<!def)\s+(cmp)\(', "cmp is not available in Python 3+"),
         (r'(?<!\.)\breduce\s*\(.*', "reduce is not available in Python 3+"),
         (
             r'\bdict\(.*=',
             'dict() is different in Py2 and 3 and is slower than {}',
             'dict-from-generator',
         ),
         (r'\.has_key\b', "dict.has_key is not available in Python 3+"),
         (r'\s<>\s', '<> operator is not available in Python 3+, use !='),
         (r'^\s*\t', "don't use tabs"),
         (r'\S;\s*\n', "semicolon"),
         (r'[^_]_\([ \t\n]*(?:"[^"]+"[ \t\n+]*)+%', "don't use % inside _()"),
         (r"[^_]_\([ \t\n]*(?:'[^']+'[ \t\n+]*)+%", "don't use % inside _()"),
         (r'(\w|\)),\w', "missing whitespace after ,"),
         (r'(\w|\))[+/*\-<>]\w', "missing whitespace in expression"),
         (r'\w\s=\s\s+\w', "gratuitous whitespace after ="),
         (
             (
                 # a line ending with a colon, potentially with trailing comments
                 r':([ \t]*#[^\n]*)?\n'
                 # one that is not a pass and not only a comment
                 r'(?P<indent>[ \t]+)[^#][^\n]+\n'
                 # more lines at the same indent level
                 r'((?P=indent)[^\n]+\n)*'
                 # a pass at the same indent level, which is bogus
                 r'(?P=indent)pass[ \t\n#]'
             ),
             'omit superfluous pass',
         ),
         (r'[^\n]\Z', "no trailing newline"),
         (r'(\S[ \t]+|^[ \t]+)\n', "trailing whitespace"),
         (
             r'^\s+(self\.)?[A-Za-z][a-z0-9]+[A-Z]\w* = ',
             "don't use camelcase in identifiers",
             r'#.*camelcase-required',
         ),
         (
             r'^\s*(if|while|def|class|except|try)\s[^[\n]*:\s*[^\\n]#\s]+',
             "linebreak after :",
         ),
         (
             r'class\s[^( \n]+:',
             "old-style class, use class foo(object)",
             r'#.*old-style',
         ),
         (
             r'class\s[^( \n]+\(\):',
             "class foo() creates old style object, use class foo(object)",
             r'#.*old-style',
         ),
         (
             r'\b(%s)\('
             % '|'.join(k for k in keyword.kwlist if k not in ('print', 'exec')),
             "Python keyword is not a function",
         ),
         #    (r'class\s[A-Z][^\(]*\((?!Exception)',
         #     "don't capitalize non-exception classes"),
         #    (r'in range\(', "use xrange"),
         #    (r'^\s*print\s+', "avoid using print in core and extensions"),
         (r'[\x80-\xff]', "non-ASCII character literal"),
         (r'("\')\.format\(', "str.format() has no bytes counterpart, use %"),
         (
             r'([\(\[][ \t]\S)|(\S[ \t][\)\]])',
             "gratuitous whitespace in () or []",
         ),
         #    (r'\s\s=', "gratuitous whitespace before ="),
         (
             r'[^>< ](\+=|-=|!=|<>|<=|>=|<<=|>>=|%=)\S',
             "missing whitespace around operator",
         ),
         (
             r'[^>< ](\+=|-=|!=|<>|<=|>=|<<=|>>=|%=)\s',
             "missing whitespace around operator",
         ),
         (
             r'\s(\+=|-=|!=|<>|<=|>=|<<=|>>=|%=)\S',
             "missing whitespace around operator",
         ),
         (r'[^^+=*/!<>&| %-](\s=|=\s)[^= ]', "wrong whitespace around ="),
         (
             r'\([^()]*( =[^=]|[^<>!=]= )',
             "no whitespace around = for named parameters",
         ),
         (
             r'raise [^,(]+, (\([^\)]+\)|[^,\(\)]+)$',
             "don't use old-style two-argument raise, use Exception(message)",
         ),
         (r' is\s+(not\s+)?["\'0-9-]', "object comparison with literal"),
         (
             r' [=!]=\s+(True|False|None)',
             "comparison with singleton, use 'is' or 'is not' instead",
         ),
         (
             r'^\s*(while|if) [01]:',
             "use True/False for constant Boolean expression",
         ),
         (r'^\s*if False(:| +and)', 'Remove code instead of using `if False`'),
         (
             r'(?:(?<!def)\s+|\()hasattr\(',
             'hasattr(foo, bar) is broken on py2, use util.safehasattr(foo, bar) '
             'instead',
             r'#.*hasattr-py3-only',
         ),
         (r'opener\([^)]*\).read\(', "use opener.read() instead"),
         (r'opener\([^)]*\).write\(', "use opener.write() instead"),
         (r'(?i)descend[e]nt', "the proper spelling is descendAnt"),
         (r'\.debug\(\_', "don't mark debug messages for translation"),
         (r'\.strip\(\)\.split\(\)', "no need to strip before splitting"),
         (r'^\s*except\s*:', "naked except clause", r'#.*re-raises'),
         (
             r'^\s*except\s([^\(,]+|\([^\)]+\))\s*,',
             'legacy exception syntax; use "as" instead of ","',
         ),
         (r'release\(.*wlock, .*lock\)', "wrong lock release order"),
         (r'\bdef\s+__bool__\b', "__bool__ should be __nonzero__ in Python 2"),
         (
             r'os\.path\.join\(.*, *(""|\'\')\)',
             "use pathutil.normasprefix(path) instead of os.path.join(path, '')",
         ),
         (r'\s0[0-7]+\b', 'legacy octal syntax; use "0o" prefix instead of "0"'),
         # XXX only catch mutable arguments on the first line of the definition
         (r'def.*[( ]\w+=\{\}', "don't use mutable default arguments"),
         (r'\butil\.Abort\b', "directly use error.Abort"),
         (
             r'^@(\w*\.)?cachefunc',
             "module-level @cachefunc is risky, please avoid",
         ),
         (
             r'^import Queue',
             "don't use Queue, use pycompat.queue.Queue + "
             "pycompat.queue.Empty",
         ),
         (
             r'^import cStringIO',
             "don't use cStringIO.StringIO, use util.stringio",
         ),
         (r'^import urllib', "don't use urllib, use util.urlreq/util.urlerr"),
         (
             r'^import SocketServer',
             "don't use SockerServer, use util.socketserver",
         ),
         (r'^import urlparse', "don't use urlparse, use util.urlreq"),
         (r'^import xmlrpclib', "don't use xmlrpclib, use util.xmlrpclib"),
         (r'^import cPickle', "don't use cPickle, use util.pickle"),
         (r'^import pickle', "don't use pickle, use util.pickle"),
         (r'^import httplib', "don't use httplib, use util.httplib"),
         (r'^import BaseHTTPServer', "use util.httpserver instead"),
         (
             r'^(from|import) mercurial\.(cext|pure|cffi)',
             "use mercurial.policy.importmod instead",
         ),
         (r'\.next\(\)', "don't use .next(), use next(...)"),
         (
             r'([a-z]*).revision\(\1\.node\(',
             "don't convert rev to node before passing to revision(nodeorrev)",
         ),
         (r'platform\.system\(\)', "don't use platform.system(), use pycompat"),
     ],
     # warnings
     [],
 ]
 
 # patterns to check normal *.py files
 pypats = [
     [
         # Ideally, these should be placed in "commonpypats" for
         # consistency of coding rules in Mercurial source tree.
         # But on the other hand, these are not so seriously required for
         # python code fragments embedded in test scripts. Fixing test
         # scripts for these patterns requires many changes, and has less
         # profit than effort.
         (r'raise Exception', "don't raise generic exceptions"),
         (r'[\s\(](open|file)\([^)]*\)\.read\(', "use util.readfile() instead"),
         (
             r'[\s\(](open|file)\([^)]*\)\.write\(',
             "use util.writefile() instead",
         ),
         (
             r'^[\s\(]*(open(er)?|file)\([^)]*\)(?!\.close\(\))',
             "always assign an opened file to a variable, and close it afterwards",
         ),
         (
             r'[\s\(](open|file)\([^)]*\)\.(?!close\(\))',
             "always assign an opened file to a variable, and close it afterwards",
         ),
         (r':\n(    )*( ){1,3}[^ ]', "must indent 4 spaces"),
         (r'^import atexit', "don't use atexit, use ui.atexit"),
         # rules depending on implementation of repquote()
         (
             r' x+[xpqo%APM][\'"]\n\s+[\'"]x',
             'string join across lines with no space',
         ),
         (
             r'''(?x)ui\.(status|progress|write|note|warn)\(
          [ \t\n#]*
          (?# any strings/comments might precede a string, which
            # contains translatable message)
          b?((['"]|\'\'\'|""")[ \npq%bAPMxno]*(['"]|\'\'\'|""")[ \t\n#]+)*
          (?# sequence consisting of below might precede translatable message
            # - formatting string: "% 10s", "%05d", "% -3.2f", "%*s", "%%" ...
            # - escaped character: "\\", "\n", "\0" ...
            # - character other than '%', 'b' as '\', and 'x' as alphabet)
          (['"]|\'\'\'|""")
          ((%([ n]?[PM]?([np]+|A))?x)|%%|b[bnx]|[ \nnpqAPMo])*x
          (?# this regexp can't use [^...] style,
            # because _preparepats forcibly adds "\n" into [^...],
            # even though this regexp wants match it against "\n")''',
             "missing _() in ui message (use () to hide false-positives)",
         ),
     ]
     + commonpypats[0],
     # warnings
     [
         # rules depending on implementation of repquote()
         (r'(^| )pp +xxxxqq[ \n][^\n]', "add two newlines after '.. note::'"),
     ]
     + commonpypats[1],
 ]
 
 # patterns to check *.py for embedded ones in test script
 embeddedpypats = [
     [] + commonpypats[0],
     # warnings
     [] + commonpypats[1],
 ]
 
 # common filters to convert *.py
 commonpyfilters = [
     (
         r"""(?msx)(?P<comment>\#.*?$)|
          ((?P<quote>('''|\"\"\"|(?<!')'(?!')|(?<!")"(?!")))
           (?P<text>(([^\\]|\\.)*?))
           (?P=quote))""",
         reppython,
     ),
 ]
 
+# pattern only for mercurial and extensions
+core_py_pats = [
+    [
+        # Windows tend to get confused about capitalization of the drive letter
+        #
+        # see mercurial.windows.abspath for details
+        (
+            r'os\.path\.abspath',
+            "use util.abspath instead (windows)",
+            r'#.*re-exports',
+        ),
+    ],
+    # warnings
+    [],
+]
+
 # filters to convert normal *.py files
 pyfilters = [] + commonpyfilters
 
 # non-filter patterns
 pynfpats = [
     [
         (r'pycompat\.osname\s*[=!]=\s*[\'"]nt[\'"]', "use pycompat.iswindows"),
         (r'pycompat\.osname\s*[=!]=\s*[\'"]posix[\'"]', "use pycompat.isposix"),
         (
             r'pycompat\.sysplatform\s*[!=]=\s*[\'"]darwin[\'"]',
             "use pycompat.isdarwin",
         ),
     ],
     # warnings
     [],
 ]
 
 # filters to convert *.py for embedded ones in test script
 embeddedpyfilters = [] + commonpyfilters
 
 # extension non-filter patterns
 pyextnfpats = [
     [(r'^"""\n?[A-Z]', "don't capitalize docstring title")],
     # warnings
     [],
 ]
 
 txtfilters = []
 
 txtpats = [
     [
         (r'\s$', 'trailing whitespace'),
         ('.. note::[ \n][^\n]', 'add two newlines after note::'),
     ],
     [],
 ]
 
 cpats = [
     [
         (r'//', "don't use //-style comments"),
         (r'\S\t', "don't use tabs except for indent"),
         (r'(\S[ \t]+|^[ \t]+)\n', "trailing whitespace"),
         (r'(while|if|do|for)\(', "use space after while/if/do/for"),
         (r'return\(', "return is not a function"),
         (r' ;', "no space before ;"),
         (r'[^;] \)', "no space before )"),
         (r'[)][{]', "space between ) and {"),
         (r'\w+\* \w+', "use int *foo, not int* foo"),
         (r'\W\([^\)]+\) \w+', "use (int)foo, not (int) foo"),
         (r'\w+ (\+\+|--)', "use foo++, not foo ++"),
         (r'\w,\w', "missing whitespace after ,"),
         (r'^[^#]\w[+/*]\w', "missing whitespace in expression"),
         (r'\w\s=\s\s+\w', "gratuitous whitespace after ="),
         (r'^#\s+\w', "use #foo, not # foo"),
         (r'[^\n]\Z', "no trailing newline"),
         (r'^\s*#import\b', "use only #include in standard C code"),
         (r'strcpy\(', "don't use strcpy, use strlcpy or memcpy"),
         (r'strcat\(', "don't use strcat"),
         # rules depending on implementation of repquote()
     ],
     # warnings
     [
         # rules depending on implementation of repquote()
     ],
 ]
 
 cfilters = [
     (r'(/\*)(((\*(?!/))|[^*])*)\*/', repccomment),
     (r'''(?P<quote>(?<!")")(?P<text>([^"]|\\")+)"(?!")''', repquote),
     (r'''(#\s*include\s+<)([^>]+)>''', repinclude),
     (r'(\()([^)]+\))', repcallspaces),
 ]
 
 inutilpats = [
     [
         (r'\bui\.', "don't use ui in util"),
     ],
     # warnings
     [],
 ]
 
 inrevlogpats = [
     [
         (r'\brepo\.', "don't use repo in revlog"),
     ],
     # warnings
     [],
 ]
 
 webtemplatefilters = []
 
 webtemplatepats = [
     [],
     [
         (
             r'{desc(\|(?!websub|firstline)[^\|]*)+}',
             'follow desc keyword with either firstline or websub',
         ),
     ],
 ]
 
 allfilesfilters = []
 
 allfilespats = [
     [
         (
             r'(http|https)://[a-zA-Z0-9./]*selenic.com/',
             'use mercurial-scm.org domain URL',
         ),
         (
             r'mercurial@selenic\.com',
             'use mercurial-scm.org domain for mercurial ML address',
         ),
         (
             r'mercurial-devel@selenic\.com',
             'use mercurial-scm.org domain for mercurial-devel ML address',
         ),
     ],
     # warnings
     [],
 ]
 
 py3pats = [
     [
         (
             r'os\.environ',
             "use encoding.environ instead (py3)",
             r'#.*re-exports',
         ),
         (r'os\.name', "use pycompat.osname instead (py3)"),
         (r'os\.getcwd', "use encoding.getcwd instead (py3)", r'#.*re-exports'),
         (r'os\.sep', "use pycompat.ossep instead (py3)"),
         (r'os\.pathsep', "use pycompat.ospathsep instead (py3)"),
         (r'os\.altsep', "use pycompat.osaltsep instead (py3)"),
         (r'sys\.platform', "use pycompat.sysplatform instead (py3)"),
         (r'getopt\.getopt', "use pycompat.getoptb instead (py3)"),
         (r'os\.getenv', "use encoding.environ.get instead"),
         (r'os\.setenv', "modifying the environ dict is not preferred"),
         (r'(?<!pycompat\.)xrange', "use pycompat.xrange instead (py3)"),
     ],
     # warnings
     [],
 ]
 
 checks = [
     ('python', r'.*\.(py|cgi)$', r'^#!.*python', pyfilters, pypats),
     ('python', r'.*\.(py|cgi)$', r'^#!.*python', [], pynfpats),
     ('python', r'.*hgext.*\.py$', '', [], pyextnfpats),
     (
         'python 3',
         r'.*(hgext|mercurial)/(?!demandimport|policy|pycompat).*\.py',
         '',
         pyfilters,
         py3pats,
     ),
+    (
+        'core files',
+        r'.*(hgext|mercurial)/(?!demandimport|policy|pycompat).*\.py',
+        '',
+        pyfilters,
+        core_py_pats,
+    ),
     ('test script', r'(.*/)?test-[^.~]*$', '', testfilters, testpats),
     ('c', r'.*\.[ch]$', '', cfilters, cpats),
     ('unified test', r'.*\.t$', '', utestfilters, utestpats),
     (
         'layering violation repo in revlog',
         r'mercurial/revlog\.py',
         '',
         pyfilters,
         inrevlogpats,
     ),
     (
         'layering violation ui in util',
         r'mercurial/util\.py',
         '',
         pyfilters,
         inutilpats,
     ),
     ('txt', r'.*\.txt$', '', txtfilters, txtpats),
     (
         'web template',
         r'mercurial/templates/.*\.tmpl',
         '',
         webtemplatefilters,
         webtemplatepats,
     ),
     ('all except for .po', r'.*(?<!\.po)$', '', allfilesfilters, allfilespats),
 ]
 
 # (desc,
 #  func to pick up embedded code fragments,
 #  list of patterns to convert target files
 #  list of patterns to detect errors/warnings)
 embeddedchecks = [
     (
         'embedded python',
         testparseutil.pyembedded,
         embeddedpyfilters,
         embeddedpypats,
     )
 ]
 
 
 def _preparepats():
     def preparefailandwarn(failandwarn):
         for pats in failandwarn:
             for i, pseq in enumerate(pats):
                 # fix-up regexes for multi-line searches
                 p = pseq[0]
                 # \s doesn't match \n (done in two steps)
                 # first, we replace \s that appears in a set already
                 p = re.sub(r'\[\\s', r'[ \\t', p)
                 # now we replace other \s instances.
                 p = re.sub(r'(?<!(\\|\[))\\s', r'[ \\t]', p)
                 # [^...] doesn't match newline
                 p = re.sub(r'(?<!\\)\[\^', r'[^\\n', p)
 
                 pats[i] = (re.compile(p, re.MULTILINE),) + pseq[1:]
 
     def preparefilters(filters):
         for i, flt in enumerate(filters):
             filters[i] = re.compile(flt[0]), flt[1]
 
     for cs in (checks, embeddedchecks):
         for c in cs:
             failandwarn = c[-1]
             preparefailandwarn(failandwarn)
 
             filters = c[-2]
             preparefilters(filters)
 
 
 class norepeatlogger(object):
     def __init__(self):
         self._lastseen = None
 
     def log(self, fname, lineno, line, msg, blame):
         """print error related a to given line of a given file.
 
         The faulty line will also be printed but only once in the case
         of multiple errors.
 
         :fname: filename
         :lineno: line number
         :line: actual content of the line
         :msg: error message
         """
         msgid = fname, lineno, line
         if msgid != self._lastseen:
             if blame:
                 print("%s:%d (%s):" % (fname, lineno, blame))
             else:
                 print("%s:%d:" % (fname, lineno))
             print(" > %s" % line)
             self._lastseen = msgid
         print(" " + msg)
 
 
 _defaultlogger = norepeatlogger()
 
 
 def getblame(f):
     lines = []
     for l in os.popen('hg annotate -un %s' % f):
         start, line = l.split(':', 1)
         user, rev = start.split()
         lines.append((line[1:-1], user, rev))
     return lines
 
 
 def checkfile(
     f,
     logfunc=_defaultlogger.log,
     maxerr=None,
     warnings=False,
     blame=False,
     debug=False,
     lineno=True,
 ):
     """checks style and portability of a given file
 
     :f: filepath
     :logfunc: function used to report error
               logfunc(filename, linenumber, linecontent, errormessage)
     :maxerr: number of error to display before aborting.
              Set to false (default) to report all errors
 
     return True if no error is found, False otherwise.
     """
     result = True
 
     try:
         with opentext(f) as fp:
             try:
                 pre = fp.read()
             except UnicodeDecodeError as e:
                 print("%s while reading %s" % (e, f))
                 return result
     except IOError as e:
         print("Skipping %s, %s" % (f, str(e).split(':', 1)[0]))
         return result
 
     # context information shared while single checkfile() invocation
     context = {'blamecache': None}
 
     for name, match, magic, filters, pats in checks:
         if debug:
             print(name, f)
         if not (re.match(match, f) or (magic and re.search(magic, pre))):
             if debug:
                 print(
                     "Skipping %s for %s it doesn't match %s" % (name, match, f)
                 )
             continue
         if "no-" "check-code" in pre:
             # If you're looking at this line, it's because a file has:
             # no- check- code
             # but the reason to output skipping is to make life for
             # tests easier. So, instead of writing it with a normal
             # spelling, we write it with the expected spelling from
             # tests/test-check-code.t
             print("Skipping %s it has no-che?k-code (glob)" % f)
             return "Skip"  # skip checking this file
 
         fc = _checkfiledata(
             name,
             f,
             pre,
             filters,
             pats,
             context,
             logfunc,
             maxerr,
             warnings,
             blame,
             debug,
             lineno,
         )
         if fc:
             result = False
 
     if f.endswith('.t') and "no-" "check-code" not in pre:
         if debug:
             print("Checking embedded code in %s" % f)
 
         prelines = pre.splitlines()
         embeddederros = []
         for name, embedded, filters, pats in embeddedchecks:
             # "reset curmax at each repetition" treats maxerr as "max
             # nubmer of errors in an actual file per entry of
             # (embedded)checks"
             curmaxerr = maxerr
 
             for found in embedded(f, prelines, embeddederros):
                 filename, starts, ends, code = found
                 fc = _checkfiledata(
                     name,
                     f,
                     code,
                     filters,
                     pats,
                     context,
                     logfunc,
                     curmaxerr,
                     warnings,
                     blame,
                     debug,
                     lineno,
                     offset=starts - 1,
                 )
                 if fc:
                     result = False
                     if curmaxerr:
                         if fc >= curmaxerr:
                             break
                         curmaxerr -= fc
 
     return result
 
 
 def _checkfiledata(
     name,
     f,
     filedata,
     filters,
     pats,
     context,
     logfunc,
     maxerr,
     warnings,
     blame,
     debug,
     lineno,
     offset=None,
 ):
     """Execute actual error check for file data
 
     :name: of the checking category
     :f: filepath
     :filedata: content of a file
     :filters: to be applied before checking
     :pats: to detect errors
     :context: a dict of information shared while single checkfile() invocation
               Valid keys: 'blamecache'.
     :logfunc: function used to report error
               logfunc(filename, linenumber, linecontent, errormessage)
     :maxerr: number of error to display before aborting, or False to
              report all errors
     :warnings: whether warning level checks should be applied
     :blame: whether blame information should be displayed at error reporting
     :debug: whether debug information should be displayed
     :lineno: whether lineno should be displayed at error reporting
     :offset: line number offset of 'filedata' in 'f' for checking
              an embedded code fragment, or None (offset=0 is different
              from offset=None)
 
     returns number of detected errors.
     """
     blamecache = context['blamecache']
     if offset is None:
         lineoffset = 0
     else:
         lineoffset = offset
 
     fc = 0
     pre = post = filedata
 
     if True:  # TODO: get rid of this redundant 'if' block
         for p, r in filters:
             post = re.sub(p, r, post)
         nerrs = len(pats[0])  # nerr elements are errors
         if warnings:
             pats = pats[0] + pats[1]
         else:
             pats = pats[0]
         # print post # uncomment to show filtered version
 
         if debug:
             print("Checking %s for %s" % (name, f))
 
         prelines = None
         errors = []
         for i, pat in enumerate(pats):
             if len(pat) == 3:
                 p, msg, ignore = pat
             else:
                 p, msg = pat
                 ignore = None
             if i >= nerrs:
                 msg = "warning: " + msg
 
             pos = 0
             n = 0
             for m in p.finditer(post):
                 if prelines is None:
                     prelines = pre.splitlines()
                     postlines = post.splitlines(True)
 
                 start = m.start()
                 while n < len(postlines):
                     step = len(postlines[n])
                     if pos + step > start:
                         break
                     pos += step
                     n += 1
                 l = prelines[n]
 
                 if ignore and re.search(ignore, l, re.MULTILINE):
                     if debug:
                         print(
                             "Skipping %s for %s:%s (ignore pattern)"
                             % (name, f, (n + lineoffset))
                         )
                     continue
                 bd = ""
                 if blame:
                     bd = 'working directory'
                     if blamecache is None:
                         blamecache = getblame(f)
                         context['blamecache'] = blamecache
                     if (n + lineoffset) < len(blamecache):
                         bl, bu, br = blamecache[(n + lineoffset)]
                         if offset is None and bl == l:
                             bd = '%s@%s' % (bu, br)
                         elif offset is not None and bl.endswith(l):
                             # "offset is not None" means "checking
                             # embedded code fragment". In this case,
                             # "l" does not have information about the
                             # beginning of an *original* line in the
                             # file (e.g. '  > ').
                             # Therefore, use "str.endswith()", and
                             # show "maybe" for a little loose
                             # examination.
                             bd = '%s@%s, maybe' % (bu, br)
 
                 errors.append((f, lineno and (n + lineoffset + 1), l, msg, bd))
 
         errors.sort()
         for e in errors:
             logfunc(*e)
             fc += 1
             if maxerr and fc >= maxerr:
                 print(" (too many errors, giving up)")
                 break
 
     return fc
 
 
 def main():
     parser = optparse.OptionParser("%prog [options] [files | -]")
     parser.add_option(
         "-w",
         "--warnings",
         action="store_true",
         help="include warning-level checks",
     )
     parser.add_option(
         "-p", "--per-file", type="int", help="max warnings per file"
     )
     parser.add_option(
         "-b",
         "--blame",
         action="store_true",
         help="use annotate to generate blame info",
     )
     parser.add_option(
         "", "--debug", action="store_true", help="show debug information"
     )
     parser.add_option(
         "",
         "--nolineno",
         action="store_false",
         dest='lineno',
         help="don't show line numbers",
     )
 
     parser.set_defaults(
         per_file=15, warnings=False, blame=False, debug=False, lineno=True
     )
     (options, args) = parser.parse_args()
 
     if len(args) == 0:
         check = glob.glob("*")
     elif args == ['-']:
         # read file list from stdin
         check = sys.stdin.read().splitlines()
     else:
         check = args
 
     _preparepats()
 
     ret = 0
     for f in check:
         if not checkfile(
             f,
             maxerr=options.per_file,
             warnings=options.warnings,
             blame=options.blame,
             debug=options.debug,
             lineno=options.lineno,
         ):
             ret = 1
     return ret
 
 
 if __name__ == "__main__":
     sys.exit(main())

contrib/chg/chg.c

@@ -1,551 +1,546 b''
 /*
  * A fast client for Mercurial command server
  *
  * Copyright (c) 2011 Yuya Nishihara <[e-mail]>
  *
  * This software may be used and distributed according to the terms of the
  * GNU General Public License version 2 or any later version.
  */
 
 #include <assert.h>
 #include <dirent.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <signal.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <sys/file.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <sys/un.h>
 #include <sys/wait.h>
 #include <time.h>
 #include <unistd.h>
 
 #include "hgclient.h"
 #include "procutil.h"
 #include "util.h"
 
 #ifndef PATH_MAX
 #define PATH_MAX 4096
 #endif
 
 struct cmdserveropts {
 	char sockname[PATH_MAX];
 	char initsockname[PATH_MAX];
 	char redirectsockname[PATH_MAX];
 	size_t argsize;
 	const char **args;
 };
 
 static void initcmdserveropts(struct cmdserveropts *opts)
 {
 	memset(opts, 0, sizeof(struct cmdserveropts));
 }
 
 static void freecmdserveropts(struct cmdserveropts *opts)
 {
 	free(opts->args);
 	opts->args = NULL;
 	opts->argsize = 0;
 }
 
 /*
  * Test if an argument is a sensitive flag that should be passed to the server.
  * Return 0 if not, otherwise the number of arguments starting from the current
  * one that should be passed to the server.
  */
 static size_t testsensitiveflag(const char *arg)
 {
 	static const struct {
 		const char *name;
 		size_t narg;
 	} flags[] = {
 	    {"--config", 1},     {"--cwd", 1},       {"--repo", 1},
 	    {"--repository", 1}, {"--traceback", 0}, {"-R", 1},
 	};
 	size_t i;
 	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); ++i) {
 		size_t len = strlen(flags[i].name);
 		size_t narg = flags[i].narg;
 		if (memcmp(arg, flags[i].name, len) == 0) {
 			if (arg[len] == '\0') {
 				/* --flag (value) */
 				return narg + 1;
 			} else if (arg[len] == '=' && narg > 0) {
 				/* --flag=value */
 				return 1;
 			} else if (flags[i].name[1] != '-') {
 				/* short flag */
 				return 1;
 			}
 		}
 	}
 	return 0;
 }
 
 /*
  * Parse argv[] and put sensitive flags to opts->args
  */
 static void setcmdserverargs(struct cmdserveropts *opts, int argc,
                              const char *argv[])
 {
 	size_t i, step;
 	opts->argsize = 0;
 	for (i = 0, step = 1; i < (size_t)argc; i += step, step = 1) {
 		if (!argv[i])
 			continue; /* pass clang-analyse */
 		if (strcmp(argv[i], "--") == 0)
 			break;
 		size_t n = testsensitiveflag(argv[i]);
 		if (n == 0 || i + n > (size_t)argc)
 			continue;
 		opts->args =
 		    reallocx(opts->args, (n + opts->argsize) * sizeof(char *));
 		memcpy(opts->args + opts->argsize, argv + i,
 		       sizeof(char *) * n);
 		opts->argsize += n;
 		step = n;
 	}
 }
 
 static void preparesockdir(const char *sockdir)
 {
 	int r;
 	r = mkdir(sockdir, 0700);
 	if (r < 0 && errno != EEXIST)
 		abortmsgerrno("cannot create sockdir %s", sockdir);
 
 	struct stat st;
 	r = lstat(sockdir, &st);
 	if (r < 0)
 		abortmsgerrno("cannot stat %s", sockdir);
 	if (!S_ISDIR(st.st_mode))
 		abortmsg("cannot create sockdir %s (file exists)", sockdir);
 	if (st.st_uid != geteuid() || st.st_mode & 0077)
 		abortmsg("insecure sockdir %s", sockdir);
 }
 
 /*
  * Check if a socket directory exists and is only owned by the current user.
  * Return 1 if so, 0 if not. This is used to check if XDG_RUNTIME_DIR can be
  * used or not. According to the specification [1], XDG_RUNTIME_DIR should be
  * ignored if the directory is not owned by the user with mode 0700.
  * [1]: https://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
  */
 static int checkruntimedir(const char *sockdir)
 {
 	struct stat st;
 	int r = lstat(sockdir, &st);
 	if (r < 0) /* ex. does not exist */
 		return 0;
 	if (!S_ISDIR(st.st_mode)) /* ex. is a file, not a directory */
 		return 0;
 	return st.st_uid == geteuid() && (st.st_mode & 0777) == 0700;
 }
 
 static void getdefaultsockdir(char sockdir[], size_t size)
 {
 	/* by default, put socket file in secure directory
 	 * (${XDG_RUNTIME_DIR}/chg, or /${TMPDIR:-tmp}/chg$UID)
 	 * (permission of socket file may be ignored on some Unices) */
 	const char *runtimedir = getenv("XDG_RUNTIME_DIR");
 	int r;
 	if (runtimedir && checkruntimedir(runtimedir)) {
 		r = snprintf(sockdir, size, "%s/chg", runtimedir);
 	} else {
 		const char *tmpdir = getenv("TMPDIR");
 		if (!tmpdir)
 			tmpdir = "/tmp";
 		r = snprintf(sockdir, size, "%s/chg%d", tmpdir, geteuid());
 	}
 	if (r < 0 || (size_t)r >= size)
 		abortmsg("too long TMPDIR (r = %d)", r);
 }
 
 static void setcmdserveropts(struct cmdserveropts *opts)
 {
 	int r;
 	char sockdir[PATH_MAX];
 	const char *envsockname = getenv("CHGSOCKNAME");
 	if (!envsockname) {
 		getdefaultsockdir(sockdir, sizeof(sockdir));
 		preparesockdir(sockdir);
 	}
 
 	const char *basename = (envsockname) ? envsockname : sockdir;
 	const char *sockfmt = (envsockname) ? "%s" : "%s/server";
 	r = snprintf(opts->sockname, sizeof(opts->sockname), sockfmt, basename);
 	if (r < 0 || (size_t)r >= sizeof(opts->sockname))
 		abortmsg("too long TMPDIR or CHGSOCKNAME (r = %d)", r);
 	r = snprintf(opts->initsockname, sizeof(opts->initsockname), "%s.%u",
 	             opts->sockname, (unsigned)getpid());
 	if (r < 0 || (size_t)r >= sizeof(opts->initsockname))
 		abortmsg("too long TMPDIR or CHGSOCKNAME (r = %d)", r);
 }
 
 /* If the current program is, say, /a/b/c/chg, returns /a/b/c/hg. */
 static char *getrelhgcmd(void)
 {
 	ssize_t n;
 	char *res, *slash;
 	int maxsize = 4096;
 	res = malloc(maxsize);
 	if (res == NULL)
 		goto cleanup;
 	n = readlink("/proc/self/exe", res, maxsize);
 	if (n < 0 || n >= maxsize)
 		goto cleanup;
 	res[n] = '\0';
 	slash = strrchr(res, '/');
 	if (slash == NULL)
 		goto cleanup;
 	/* 4 is strlen("/hg") + nul byte */
 	if (slash + 4 >= res + maxsize)
 		goto cleanup;
 	memcpy(slash, "/hg", 4);
 	return res;
 cleanup:
 	free(res);
 	return NULL;
 }
 
 static const char *gethgcmd(void)
 {
 	static const char *hgcmd = NULL;
 #ifdef HGPATHREL
 	int tryrelhgcmd = 1;
 #else
 	int tryrelhgcmd = 0;
 #endif
 	if (!hgcmd) {
 		hgcmd = getenv("CHGHG");
 		if (!hgcmd || hgcmd[0] == '\0')
 			hgcmd = getenv("HG");
 		if (tryrelhgcmd && (!hgcmd || hgcmd[0] == '\0'))
 			hgcmd = getrelhgcmd();
 		if (!hgcmd || hgcmd[0] == '\0')
 #ifdef HGPATH
 			hgcmd = (HGPATH);
 #else
 			hgcmd = "hg";
 #endif
 	}
 	return hgcmd;
 }
 
 static void execcmdserver(const struct cmdserveropts *opts)
 {
 	const char *hgcmd = gethgcmd();
 
 	const char *baseargv[] = {
-	    hgcmd,
-	    "serve",
-	    "--cmdserver",
-	    "chgunix",
-	    "--address",
-	    opts->initsockname,
-	    "--daemon-postexec",
+	    hgcmd,     "serve",     "--no-profile",     "--cmdserver",
+	    "chgunix", "--address", opts->initsockname, "--daemon-postexec",
 	    "chdir:/",
 	};
 	size_t baseargvsize = sizeof(baseargv) / sizeof(baseargv[0]);
 	size_t argsize = baseargvsize + opts->argsize + 1;
 
 	const char **argv = mallocx(sizeof(char *) * argsize);
 	memcpy(argv, baseargv, sizeof(baseargv));
 	if (opts->args) {
 		size_t size = sizeof(char *) * opts->argsize;
 		memcpy(argv + baseargvsize, opts->args, size);
 	}
 	argv[argsize - 1] = NULL;
 
 	const char *lc_ctype_env = getenv("LC_CTYPE");
 	if (lc_ctype_env == NULL) {
 		if (putenv("CHG_CLEAR_LC_CTYPE=") != 0)
 			abortmsgerrno("failed to putenv CHG_CLEAR_LC_CTYPE");
 	} else {
 		if (setenv("CHGORIG_LC_CTYPE", lc_ctype_env, 1) != 0) {
 			abortmsgerrno("failed to setenv CHGORIG_LC_CTYPE");
 		}
 	}
 
 	/* close any open files to avoid hanging locks */
 	DIR *dp = opendir("/proc/self/fd");
 	if (dp != NULL) {
 		debugmsg("closing files based on /proc contents");
 		struct dirent *de;
 		while ((de = readdir(dp))) {
 			errno = 0;
 			char *end;
 			long fd_value = strtol(de->d_name, &end, 10);
 			if (end == de->d_name) {
 				/* unable to convert to int (. or ..) */
 				continue;
 			}
 			if (errno == ERANGE) {
 				debugmsg("tried to parse %s, but range error "
 				         "occurred",
 				         de->d_name);
 				continue;
 			}
 			if (fd_value > STDERR_FILENO && fd_value != dirfd(dp)) {
 				debugmsg("closing fd %ld", fd_value);
 				int res = close(fd_value);
 				if (res) {
 					debugmsg("tried to close fd %ld: %d "
 					         "(errno: %d)",
 					         fd_value, res, errno);
 				}
 			}
 		}
 		closedir(dp);
 	}
 
 	if (putenv("CHGINTERNALMARK=") != 0)
 		abortmsgerrno("failed to putenv");
 	if (execvp(hgcmd, (char **)argv) < 0)
 		abortmsgerrno("failed to exec cmdserver");
 	free(argv);
 }
 
 /* Retry until we can connect to the server. Give up after some time. */
 static hgclient_t *retryconnectcmdserver(struct cmdserveropts *opts, pid_t pid)
 {
 	static const struct timespec sleepreq = {0, 10 * 1000000};
 	int pst = 0;
 
 	debugmsg("try connect to %s repeatedly", opts->initsockname);
 
 	unsigned int timeoutsec = 60; /* default: 60 seconds */
 	const char *timeoutenv = getenv("CHGTIMEOUT");
 	if (timeoutenv)
 		sscanf(timeoutenv, "%u", &timeoutsec);
 
 	for (unsigned int i = 0; !timeoutsec || i < timeoutsec * 100; i++) {
 		hgclient_t *hgc = hgc_open(opts->initsockname);
 		if (hgc) {
 			debugmsg("rename %s to %s", opts->initsockname,
 			         opts->sockname);
 			int r = rename(opts->initsockname, opts->sockname);
 			if (r != 0)
 				abortmsgerrno("cannot rename");
 			return hgc;
 		}
 
 		if (pid > 0) {
 			/* collect zombie if child process fails to start */
 			int r = waitpid(pid, &pst, WNOHANG);
 			if (r != 0)
 				goto cleanup;
 		}
 
 		nanosleep(&sleepreq, NULL);
 	}
 
 	abortmsg("timed out waiting for cmdserver %s", opts->initsockname);
 	return NULL;
 
 cleanup:
 	if (WIFEXITED(pst)) {
 		if (WEXITSTATUS(pst) == 0)
 			abortmsg("could not connect to cmdserver "
 			         "(exited with status 0)");
 		debugmsg("cmdserver exited with status %d", WEXITSTATUS(pst));
 		exit(WEXITSTATUS(pst));
 	} else if (WIFSIGNALED(pst)) {
 		abortmsg("cmdserver killed by signal %d", WTERMSIG(pst));
 	} else {
 		abortmsg("error while waiting for cmdserver");
 	}
 	return NULL;
 }
 
 /* Connect to a cmdserver. Will start a new server on demand. */
 static hgclient_t *connectcmdserver(struct cmdserveropts *opts)
 {
 	const char *sockname =
 	    opts->redirectsockname[0] ? opts->redirectsockname : opts->sockname;
 	debugmsg("try connect to %s", sockname);
 	hgclient_t *hgc = hgc_open(sockname);
 	if (hgc)
 		return hgc;
 
 	/* prevent us from being connected to an outdated server: we were
 	 * told by a server to redirect to opts->redirectsockname and that
 	 * address does not work. we do not want to connect to the server
 	 * again because it will probably tell us the same thing. */
 	if (sockname == opts->redirectsockname)
 		unlink(opts->sockname);
 
 	debugmsg("start cmdserver at %s", opts->initsockname);
 
 	pid_t pid = fork();
 	if (pid < 0)
 		abortmsg("failed to fork cmdserver process");
 	if (pid == 0) {
 		execcmdserver(opts);
 	} else {
 		hgc = retryconnectcmdserver(opts, pid);
 	}
 
 	return hgc;
 }
 
 static void killcmdserver(const struct cmdserveropts *opts)
 {
 	/* resolve config hash */
 	char *resolvedpath = realpath(opts->sockname, NULL);
 	if (resolvedpath) {
 		unlink(resolvedpath);
 		free(resolvedpath);
 	}
 }
 
 /* Run instructions sent from the server like unlink and set redirect path
  * Return 1 if reconnect is needed, otherwise 0 */
 static int runinstructions(struct cmdserveropts *opts, const char **insts)
 {
 	int needreconnect = 0;
 	if (!insts)
 		return needreconnect;
 
 	assert(insts);
 	opts->redirectsockname[0] = '\0';
 	const char **pinst;
 	for (pinst = insts; *pinst; pinst++) {
 		debugmsg("instruction: %s", *pinst);
 		if (strncmp(*pinst, "unlink ", 7) == 0) {
 			unlink(*pinst + 7);
 		} else if (strncmp(*pinst, "redirect ", 9) == 0) {
 			int r = snprintf(opts->redirectsockname,
 			                 sizeof(opts->redirectsockname), "%s",
 			                 *pinst + 9);
 			if (r < 0 || r >= (int)sizeof(opts->redirectsockname))
 				abortmsg("redirect path is too long (%d)", r);
 			needreconnect = 1;
 		} else if (strncmp(*pinst, "exit ", 5) == 0) {
 			int n = 0;
 			if (sscanf(*pinst + 5, "%d", &n) != 1)
 				abortmsg("cannot read the exit code");
 			exit(n);
 		} else if (strcmp(*pinst, "reconnect") == 0) {
 			needreconnect = 1;
 		} else {
 			abortmsg("unknown instruction: %s", *pinst);
 		}
 	}
 	return needreconnect;
 }
 
 /*
  * Test whether the command and the environment is unsupported or not.
  *
  * If any of the stdio file descriptors are not present (rare, but some tools
  * might spawn new processes without stdio instead of redirecting them to the
  * null device), then mark it as not supported because attachio won't work
  * correctly.
  *
  * The command list is not designed to cover all cases. But it's fast, and does
  * not depend on the server.
  */
 static int isunsupported(int argc, const char *argv[])
 {
 	enum {
 		SERVE = 1,
 		DAEMON = 2,
 		SERVEDAEMON = SERVE | DAEMON,
 	};
 	unsigned int state = 0;
 	int i;
 	/* use fcntl to test missing stdio fds */
 	if (fcntl(STDIN_FILENO, F_GETFD) == -1 ||
 	    fcntl(STDOUT_FILENO, F_GETFD) == -1 ||
 	    fcntl(STDERR_FILENO, F_GETFD) == -1) {
 		debugmsg("stdio fds are missing");
 		return 1;
 	}
 	for (i = 0; i < argc; ++i) {
 		if (strcmp(argv[i], "--") == 0)
 			break;
 		/*
 		 * there can be false positives but no false negative
 		 * we cannot assume `serve` will always be first argument
 		 * because global options can be passed before the command name
 		 */
 		if (strcmp("serve", argv[i]) == 0)
 			state |= SERVE;
 		else if (strcmp("-d", argv[i]) == 0 ||
 		         strcmp("--daemon", argv[i]) == 0)
 			state |= DAEMON;
 	}
 	return (state & SERVEDAEMON) == SERVEDAEMON;
 }
 
 static void execoriginalhg(const char *argv[])
 {
 	debugmsg("execute original hg");
 	if (execvp(gethgcmd(), (char **)argv) < 0)
 		abortmsgerrno("failed to exec original hg");
 }
 
 int main(int argc, const char *argv[], const char *envp[])
 {
 	if (getenv("CHGDEBUG"))
 		enabledebugmsg();
 
 	if (!getenv("HGPLAIN") && isatty(fileno(stderr)))
 		enablecolor();
 
 	if (getenv("CHGINTERNALMARK"))
 		abortmsg("chg started by chg detected.\n"
 		         "Please make sure ${HG:-hg} is not a symlink or "
 		         "wrapper to chg. Alternatively, set $CHGHG to the "
 		         "path of real hg.");
 
 	if (isunsupported(argc - 1, argv + 1))
 		execoriginalhg(argv);
 
 	struct cmdserveropts opts;
 	initcmdserveropts(&opts);
 	setcmdserveropts(&opts);
 	setcmdserverargs(&opts, argc, argv);
 
 	if (argc == 2) {
 		if (strcmp(argv[1], "--kill-chg-daemon") == 0) {
 			killcmdserver(&opts);
 			return 0;
 		}
 	}
 
 	hgclient_t *hgc;
 	size_t retry = 0;
 	while (1) {
 		hgc = connectcmdserver(&opts);
 		if (!hgc)
 			abortmsg("cannot open hg client");
 		hgc_setenv(hgc, envp);
 		const char **insts = hgc_validate(hgc, argv + 1, argc - 1);
 		int needreconnect = runinstructions(&opts, insts);
 		free(insts);
 		if (!needreconnect)
 			break;
 		hgc_close(hgc);
 		if (++retry > 10)
 			abortmsg("too many redirections.\n"
 			         "Please make sure %s is not a wrapper which "
 			         "changes sensitive environment variables "
 			         "before executing hg. If you have to use a "
 			         "wrapper, wrap chg instead of hg.",
 			         gethgcmd());
 	}
 
 	setupsignalhandler(hgc_peerpid(hgc), hgc_peerpgid(hgc));
 	atexit(waitpager);
 	int exitcode = hgc_runcommand(hgc, argv + 1, argc - 1);
 	restoresignalhandler();
 	hgc_close(hgc);
 	freecmdserveropts(&opts);
 
 	return exitcode;
 }

contrib/dirstatenonnormalcheck.py

@@ -1,65 +1,69 b''
 # dirstatenonnormalcheck.py - extension to check the consistency of the
 # dirstate's non-normal map
 #
 # For most operations on dirstate, this extensions checks that the nonnormalset
 # contains the right entries.
 # It compares the nonnormal file to a nonnormalset built from the map of all
 # the files in the dirstate to check that they contain the same files.
 
 from __future__ import absolute_import
 
 from mercurial import (
     dirstate,
     extensions,
+    pycompat,
 )
 
 
 def nonnormalentries(dmap):
     """Compute nonnormal entries from dirstate's dmap"""
     res = set()
     for f, e in dmap.iteritems():
-        if e[0] != b'n' or e[3] == -1:
+        if e.state != b'n' or e.mtime == -1:
             res.add(f)
     return res
 
 
 def checkconsistency(ui, orig, dmap, _nonnormalset, label):
     """Compute nonnormalset from dmap, check that it matches _nonnormalset"""
     nonnormalcomputedmap = nonnormalentries(dmap)
     if _nonnormalset != nonnormalcomputedmap:
-        ui.develwarn(b"%s call to %s\n" % (label, orig), config=b'dirstate')
+        b_orig = pycompat.sysbytes(repr(orig))
+        ui.develwarn(b"%s call to %s\n" % (label, b_orig), config=b'dirstate')
         ui.develwarn(b"inconsistency in nonnormalset\n", config=b'dirstate')
-        ui.develwarn(b"[nonnormalset] %s\n" % _nonnormalset, config=b'dirstate')
-        ui.develwarn(b"[map] %s\n" % nonnormalcomputedmap, config=b'dirstate')
+        b_nonnormal = pycompat.sysbytes(repr(_nonnormalset))
+        ui.develwarn(b"[nonnormalset] %s\n" % b_nonnormal, config=b'dirstate')
+        b_nonnormalcomputed = pycompat.sysbytes(repr(nonnormalcomputedmap))
+        ui.develwarn(b"[map] %s\n" % b_nonnormalcomputed, config=b'dirstate')
 
 
-def _checkdirstate(orig, self, arg):
+def _checkdirstate(orig, self, *args, **kwargs):
     """Check nonnormal set consistency before and after the call to orig"""
     checkconsistency(
         self._ui, orig, self._map, self._map.nonnormalset, b"before"
     )
-    r = orig(self, arg)
+    r = orig(self, *args, **kwargs)
     checkconsistency(
         self._ui, orig, self._map, self._map.nonnormalset, b"after"
     )
     return r
 
 
 def extsetup(ui):
     """Wrap functions modifying dirstate to check nonnormalset consistency"""
     dirstatecl = dirstate.dirstate
     devel = ui.configbool(b'devel', b'all-warnings')
     paranoid = ui.configbool(b'experimental', b'nonnormalparanoidcheck')
     if devel:
         extensions.wrapfunction(dirstatecl, '_writedirstate', _checkdirstate)
         if paranoid:
             # We don't do all these checks when paranoid is disable as it would
             # make the extension run very slowly on large repos
             extensions.wrapfunction(dirstatecl, 'normallookup', _checkdirstate)
             extensions.wrapfunction(dirstatecl, 'otherparent', _checkdirstate)
             extensions.wrapfunction(dirstatecl, 'normal', _checkdirstate)
             extensions.wrapfunction(dirstatecl, 'write', _checkdirstate)
             extensions.wrapfunction(dirstatecl, 'add', _checkdirstate)
             extensions.wrapfunction(dirstatecl, 'remove', _checkdirstate)
             extensions.wrapfunction(dirstatecl, 'merge', _checkdirstate)
             extensions.wrapfunction(dirstatecl, 'drop', _checkdirstate)

contrib/dumprevlog

@@ -1,47 +1,60 b''
 #!/usr/bin/env python3
 # Dump revlogs as raw data stream
 # $ find .hg/store/ -name "*.i" | xargs dumprevlog > repo.dump
 
 from __future__ import absolute_import, print_function
 
 import sys
 from mercurial.node import hex
 from mercurial import (
     encoding,
     pycompat,
     revlog,
 )
 from mercurial.utils import procutil
 
+from mercurial.revlogutils import (
+    constants as revlog_constants,
+)
+
 for fp in (sys.stdin, sys.stdout, sys.stderr):
     procutil.setbinary(fp)
 
 
 def binopen(path, mode=b'rb'):
     if b'b' not in mode:
         mode = mode + b'b'
     return open(path, pycompat.sysstr(mode))
 
 
 binopen.options = {}
 
 
 def printb(data, end=b'\n'):
     sys.stdout.flush()
     procutil.stdout.write(data + end)
 
 
 for f in sys.argv[1:]:
-    r = revlog.revlog(binopen, encoding.strtolocal(f))
+    localf = encoding.strtolocal(f)
+    if not localf.endswith(b'.i'):
+        print("file:", f, file=sys.stderr)
+        print("  invalid filename", file=sys.stderr)
+
+    r = revlog.revlog(
+        binopen,
+        target=(revlog_constants.KIND_OTHER, b'dump-revlog'),
+        radix=localf[:-2],
+    )
     print("file:", f)
     for i in r:
         n = r.node(i)
         p = r.parents(n)
         d = r.revision(n)
         printb(b"node: %s" % hex(n))
         printb(b"linkrev: %d" % r.linkrev(i))
         printb(b"parents: %s %s" % (hex(p[0]), hex(p[1])))
         printb(b"length: %d" % len(d))
         printb(b"-start-")
         printb(d)
         printb(b"-end-")

contrib/fuzz/mpatch_corpus.py

@@ -1,366 +1,371 b''
 from __future__ import absolute_import, print_function
 
 import argparse
+import os
 import struct
 import sys
 import zipfile
 
+# Add ../.. to sys.path as an absolute path so we can import hg modules
+hgloc = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
+sys.path[0:0] = [hgloc]
+
 from mercurial import (
     hg,
     ui as uimod,
 )
 
 ap = argparse.ArgumentParser()
 ap.add_argument("out", metavar="some.zip", type=str, nargs=1)
 args = ap.parse_args()
 
 
 if sys.version_info[0] < 3:
 
     class py2reprhack(object):
         def __repr__(self):
             """Py2 calls __repr__ for `bytes(foo)`, forward to __bytes__"""
             return self.__bytes__()
 
 
 else:
 
     class py2reprhack(object):
         """Not needed on py3."""
 
 
 class deltafrag(py2reprhack):
     def __init__(self, start, end, data):
         self.start = start
         self.end = end
         self.data = data
 
     def __bytes__(self):
         return (
             struct.pack(">lll", self.start, self.end, len(self.data))
             + self.data
         )
 
 
 class delta(py2reprhack):
     def __init__(self, frags):
         self.frags = frags
 
     def __bytes__(self):
         return b''.join(bytes(f) for f in self.frags)
 
 
 class corpus(py2reprhack):
     def __init__(self, base, deltas):
         self.base = base
         self.deltas = deltas
 
     def __bytes__(self):
         deltas = [bytes(d) for d in self.deltas]
         parts = (
             [
                 struct.pack(">B", len(deltas) + 1),
                 struct.pack(">H", len(self.base)),
             ]
             + [struct.pack(">H", len(d)) for d in deltas]
             + [self.base]
             + deltas
         )
         return b''.join(parts)
 
 
 with zipfile.ZipFile(args.out[0], "w", zipfile.ZIP_STORED) as zf:
     # Manually constructed entries
     zf.writestr(
         "one_delta_applies",
         bytes(corpus(b'a', [delta([deltafrag(0, 1, b'b')])])),
     )
     zf.writestr(
         "one_delta_starts_late",
         bytes(corpus(b'a', [delta([deltafrag(3, 1, b'b')])])),
     )
     zf.writestr(
         "one_delta_ends_late",
         bytes(corpus(b'a', [delta([deltafrag(0, 20, b'b')])])),
     )
 
     try:
         # Generated from repo data
         r = hg.repository(uimod.ui(), b'../..')
         fl = r.file(b'mercurial/manifest.py')
         rl = getattr(fl, '_revlog', fl)
         bins = rl._chunks(rl._deltachain(10)[0])
         zf.writestr('manifest_py_rev_10', bytes(corpus(bins[0], bins[1:])))
     except:  # skip this, so no re-raises
         print('skipping seed file from repo data')
     # Automatically discovered by running the fuzzer
     zf.writestr(
         "mpatch_decode_old_overread", b"\x02\x00\x00\x00\x02\x00\x00\x00"
     )
     # https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=8876
     zf.writestr(
         "mpatch_ossfuzz_getbe32_ubsan",
         b"\x02\x00\x00\x00\x0c    \xff\xff\xff\xff    ",
     )
     zf.writestr(
         "mpatch_apply_over_memcpy",
         b'\x13\x01\x00\x05\xd0\x00\x00\x00\x00\x00\x00\x00\x00\n \x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x8c\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00)\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00A\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x94\x18'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\xff\xfa\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x94\x18\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xfa\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x13\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00]\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00se\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00',
     )

contrib/heptapod-ci.yml

@@ -1,141 +1,174 b''
 stages:
   - tests
   - phabricator
 
 image: registry.heptapod.net/mercurial/ci-images/mercurial-core:$HG_CI_IMAGE_TAG
 
 variables:
     PYTHON: python
     TEST_HGMODULEPOLICY: "allow"
     HG_CI_IMAGE_TAG: "latest"
     TEST_HGTESTS_ALLOW_NETIO: "0"
 
 .runtests_template: &runtests
     stage: tests
     # The runner made a clone as root.
     # We make a new clone owned by user used to run the step.
     before_script:
       - hg clone . /tmp/mercurial-ci/ --noupdate --config phases.publish=no
       - hg -R /tmp/mercurial-ci/ update `hg log --rev '.' --template '{node}'`
       - cd /tmp/mercurial-ci/
       - ls -1 tests/test-check-*.* > /tmp/check-tests.txt
       - black --version
       - clang-format --version
     script:
         - echo "python used, $PYTHON"
         - echo "$RUNTEST_ARGS"
         - HGTESTS_ALLOW_NETIO="$TEST_HGTESTS_ALLOW_NETIO" HGMODULEPOLICY="$TEST_HGMODULEPOLICY" "$PYTHON" tests/run-tests.py --color=always $RUNTEST_ARGS
 
 checks-py2:
     <<: *runtests
     variables:
         RUNTEST_ARGS: "--time --test-list /tmp/check-tests.txt"
 
 checks-py3:
     <<: *runtests
     variables:
         RUNTEST_ARGS: "--time --test-list /tmp/check-tests.txt"
         PYTHON: python3
 
 rust-cargo-test-py2: &rust_cargo_test
     stage: tests
     script:
         - echo "python used, $PYTHON"
         - make rust-tests
 
 rust-cargo-test-py3:
     stage: tests
     <<: *rust_cargo_test
     variables:
         PYTHON: python3
 
 phabricator-refresh:
     stage: phabricator
     variables:
       DEFAULT_COMMENT: ":white_check_mark: refresh by Heptapod after a successful CI run (:octopus: :green_heart:)"
       STABLE_COMMENT: ":white_check_mark: refresh by Heptapod after a successful CI run (:octopus: :green_heart:)\n⚠ This patch is intended for stable ⚠\n{image https://media.giphy.com/media/nYI8SmmChYXK0/source.gif}"
     script:
       - |
         if [ `hg branch` == "stable" ]; then
             ./contrib/phab-refresh-stack.sh --comment "$STABLE_COMMENT";
         else
             ./contrib/phab-refresh-stack.sh --comment "$DEFAULT_COMMENT";
         fi
 
 test-py2:
     <<: *runtests
     variables:
         RUNTEST_ARGS: " --no-rust --blacklist /tmp/check-tests.txt"
         TEST_HGMODULEPOLICY: "c"
         TEST_HGTESTS_ALLOW_NETIO: "1"
 
 test-py3:
     <<: *runtests
     variables:
         RUNTEST_ARGS: " --no-rust --blacklist /tmp/check-tests.txt"
         PYTHON: python3
         TEST_HGMODULEPOLICY: "c"
         TEST_HGTESTS_ALLOW_NETIO: "1"
 
 test-py2-pure:
     <<: *runtests
     variables:
         RUNTEST_ARGS: "--pure --blacklist /tmp/check-tests.txt"
         TEST_HGMODULEPOLICY: "py"
 
 test-py3-pure:
     <<: *runtests
     variables:
         RUNTEST_ARGS: "--pure --blacklist /tmp/check-tests.txt"
         PYTHON: python3
         TEST_HGMODULEPOLICY: "py"
 
 test-py2-rust:
     <<: *runtests
     variables:
         HGWITHRUSTEXT: cpython
         RUNTEST_ARGS: "--rust --blacklist /tmp/check-tests.txt"
         TEST_HGMODULEPOLICY: "rust+c"
 
 test-py3-rust:
     <<: *runtests
     variables:
         HGWITHRUSTEXT: cpython
         RUNTEST_ARGS: "--rust --blacklist /tmp/check-tests.txt"
         PYTHON: python3
         TEST_HGMODULEPOLICY: "rust+c"
 
 test-py3-rhg:
     <<: *runtests
     variables:
         HGWITHRUSTEXT: cpython
         RUNTEST_ARGS: "--rust --rhg --blacklist /tmp/check-tests.txt"
         PYTHON: python3
         TEST_HGMODULEPOLICY: "rust+c"
 
 test-py2-chg:
     <<: *runtests
     variables:
         RUNTEST_ARGS: "--blacklist /tmp/check-tests.txt --chg"
         TEST_HGMODULEPOLICY: "c"
 
 test-py3-chg:
     <<: *runtests
     variables:
         PYTHON: python3
         RUNTEST_ARGS: "--blacklist /tmp/check-tests.txt --chg"
         TEST_HGMODULEPOLICY: "c"
 
 check-pytype-py3:
     extends: .runtests_template
     when: manual
     before_script:
       - hg clone . /tmp/mercurial-ci/ --noupdate --config phases.publish=no
       - hg -R /tmp/mercurial-ci/ update `hg log --rev '.' --template '{node}'`
       - cd /tmp/mercurial-ci/
       - make local PYTHON=$PYTHON
       - $PYTHON -m pip install --user -U pytype==2021.04.15
     variables:
         RUNTEST_ARGS: " --allow-slow-tests tests/test-check-pytype.t"
         PYTHON: python3
         TEST_HGMODULEPOLICY: "c"
+
+# `sh.exe --login` sets a couple of extra environment variables that are defined
+# in the MinGW shell, but switches CWD to /home/$username.  The previous value
+# is stored in OLDPWD.  Of the added variables, MSYSTEM is crucial to running
+# run-tests.py- it is needed to make run-tests.py generate a `python3` script
+# that satisfies the various shebang lines and delegates to `py -3`.
+.window_runtests_template: &windows_runtests
+    stage: tests
+    before_script:
+      # Temporary until this is adjusted in the environment
+      - $Env:TEMP="C:/Temp"
+      - $Env:TMP="C:/Temp"
+      # TODO: find/install cvs, bzr, perforce, gpg, sqlite3
+
+    script:
+        - echo "Entering script section"
+        - echo "python used, $Env:PYTHON"
+        - Invoke-Expression "$Env:PYTHON -V"
+        - Invoke-Expression "$Env:PYTHON -m black --version"
+        - echo "$Env:RUNTEST_ARGS"
+
+        - C:/MinGW/msys/1.0/bin/sh.exe --login -c 'cd "$OLDPWD" && HGTESTS_ALLOW_NETIO="$TEST_HGTESTS_ALLOW_NETIO" HGMODULEPOLICY="$TEST_HGMODULEPOLICY" $PYTHON tests/run-tests.py --color=always $RUNTEST_ARGS'
+
+windows-py3:
+    <<: *windows_runtests
+    when: manual
+    tags:
+      - windows
+    timeout: 2h
+    variables:
+        TEST_HGMODULEPOLICY: "c"
+        RUNTEST_ARGS: "--blacklist /tmp/check-tests.txt"
+        PYTHON: py -3

contrib/hg-ssh

@@ -1,111 +1,118 b''
 #!/usr/bin/env python3
 #
 # Copyright 2005-2007 by Intevation GmbH <[e-mail]>
 #
 # Author(s):
 # Thomas Arendsen Hein <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """
 hg-ssh - a wrapper for ssh access to a limited set of mercurial repos
 
 To be used in ~/.ssh/authorized_keys with the "command" option, see sshd(8):
 command="hg-ssh path/to/repo1 /path/to/repo2 ~/repo3 ~user/repo4" ssh-dss ...
 (probably together with these other useful options:
  no-port-forwarding,no-X11-forwarding,no-agent-forwarding)
 
 This allows pull/push over ssh from/to the repositories given as arguments.
 
 If all your repositories are subdirectories of a common directory, you can
 allow shorter paths with:
 command="cd path/to/my/repositories && hg-ssh repo1 subdir/repo2"
 
 You can use pattern matching of your normal shell, e.g.:
 command="cd repos && hg-ssh user/thomas/* projects/{mercurial,foo}"
 
 You can also add a --read-only flag to allow read-only access to a key, e.g.:
 command="hg-ssh --read-only repos/*"
 """
 from __future__ import absolute_import
 
 import os
+import re
 import shlex
 import sys
 
 # enable importing on demand to reduce startup time
 import hgdemandimport
 
 hgdemandimport.enable()
 
 from mercurial import (
     dispatch,
     pycompat,
     ui as uimod,
 )
 
 
 def main():
     # Prevent insertion/deletion of CRs
     dispatch.initstdio()
 
     cwd = os.getcwd()
+    if os.name == 'nt':
+        # os.getcwd() is inconsistent on the capitalization of the drive
+        # letter, so adjust it. see https://bugs.python.org/issue40368
+        if re.match('^[a-z]:', cwd):
+            cwd = cwd[0:1].upper() + cwd[1:]
+
     readonly = False
     args = sys.argv[1:]
     while len(args):
         if args[0] == '--read-only':
             readonly = True
             args.pop(0)
         else:
             break
     allowed_paths = [
         os.path.normpath(os.path.join(cwd, os.path.expanduser(path)))
         for path in args
     ]
     orig_cmd = os.getenv('SSH_ORIGINAL_COMMAND', '?')
     try:
         cmdargv = shlex.split(orig_cmd)
     except ValueError as e:
         sys.stderr.write('Illegal command "%s": %s\n' % (orig_cmd, e))
         sys.exit(255)
 
     if cmdargv[:2] == ['hg', '-R'] and cmdargv[3:] == ['serve', '--stdio']:
         path = cmdargv[2]
         repo = os.path.normpath(os.path.join(cwd, os.path.expanduser(path)))
         if repo in allowed_paths:
             cmd = [b'-R', pycompat.fsencode(repo), b'serve', b'--stdio']
             req = dispatch.request(cmd)
             if readonly:
                 if not req.ui:
                     req.ui = uimod.ui.load()
                 req.ui.setconfig(
                     b'hooks',
                     b'pretxnopen.hg-ssh',
                     b'python:__main__.rejectpush',
                     b'hg-ssh',
                 )
                 req.ui.setconfig(
                     b'hooks',
                     b'prepushkey.hg-ssh',
                     b'python:__main__.rejectpush',
                     b'hg-ssh',
                 )
             dispatch.dispatch(req)
         else:
             sys.stderr.write('Illegal repository "%s"\n' % repo)
             sys.exit(255)
     else:
         sys.stderr.write('Illegal command "%s"\n' % orig_cmd)
         sys.exit(255)
 
 
 def rejectpush(ui, **kwargs):
     ui.warn((b"Permission denied\n"))
     # mercurial hooks use unix process conventions for hook return values
     # so a truthy return means failure
     return True
 
 
 if __name__ == '__main__':
     main()

contrib/import-checker.py

@@ -1,821 +1,821 b''
 #!/usr/bin/env python3
 
 from __future__ import absolute_import, print_function
 
 import ast
 import collections
 import io
 import os
 import sys
 
 # Import a minimal set of stdlib modules needed for list_stdlib_modules()
 # to work when run from a virtualenv.  The modules were chosen empirically
 # so that the return value matches the return value without virtualenv.
 if True:  # disable lexical sorting checks
     try:
         import BaseHTTPServer as basehttpserver
     except ImportError:
         basehttpserver = None
     import zlib
 
 import testparseutil
 
 # Whitelist of modules that symbols can be directly imported from.
 allowsymbolimports = (
     '__future__',
-    'bzrlib',
+    'breezy',
     'hgclient',
     'mercurial',
     'mercurial.hgweb.common',
     'mercurial.hgweb.request',
     'mercurial.i18n',
     'mercurial.interfaces',
     'mercurial.node',
     'mercurial.pycompat',
     # for revlog to re-export constant to extensions
     'mercurial.revlogutils.constants',
     'mercurial.revlogutils.flagutil',
     # for cffi modules to re-export pure functions
     'mercurial.pure.base85',
     'mercurial.pure.bdiff',
     'mercurial.pure.mpatch',
     'mercurial.pure.osutil',
     'mercurial.pure.parsers',
     # third-party imports should be directly imported
     'mercurial.thirdparty',
     'mercurial.thirdparty.attr',
     'mercurial.thirdparty.zope',
     'mercurial.thirdparty.zope.interface',
 )
 
 # Whitelist of symbols that can be directly imported.
 directsymbols = ('demandimport',)
 
 # Modules that must be aliased because they are commonly confused with
 # common variables and can create aliasing and readability issues.
 requirealias = {
     'ui': 'uimod',
 }
 
 
 def usingabsolute(root):
     """Whether absolute imports are being used."""
     if sys.version_info[0] >= 3:
         return True
 
     for node in ast.walk(root):
         if isinstance(node, ast.ImportFrom):
             if node.module == '__future__':
                 for n in node.names:
                     if n.name == 'absolute_import':
                         return True
 
     return False
 
 
 def walklocal(root):
     """Recursively yield all descendant nodes but not in a different scope"""
     todo = collections.deque(ast.iter_child_nodes(root))
     yield root, False
     while todo:
         node = todo.popleft()
         newscope = isinstance(node, ast.FunctionDef)
         if not newscope:
             todo.extend(ast.iter_child_nodes(node))
         yield node, newscope
 
 
 def dotted_name_of_path(path):
     """Given a relative path to a source file, return its dotted module name.
 
     >>> dotted_name_of_path('mercurial/error.py')
     'mercurial.error'
     >>> dotted_name_of_path('zlibmodule.so')
     'zlib'
     """
     parts = path.replace(os.sep, '/').split('/')
     parts[-1] = parts[-1].split('.', 1)[0]  # remove .py and .so and .ARCH.so
     if parts[-1].endswith('module'):
         parts[-1] = parts[-1][:-6]
     return '.'.join(parts)
 
 
 def fromlocalfunc(modulename, localmods):
     """Get a function to examine which locally defined module the
     target source imports via a specified name.
 
     `modulename` is an `dotted_name_of_path()`-ed source file path,
     which may have `.__init__` at the end of it, of the target source.
 
     `localmods` is a set of absolute `dotted_name_of_path()`-ed source file
     paths of locally defined (= Mercurial specific) modules.
 
     This function assumes that module names not existing in
     `localmods` are from the Python standard library.
 
     This function returns the function, which takes `name` argument,
     and returns `(absname, dottedpath, hassubmod)` tuple if `name`
     matches against locally defined module. Otherwise, it returns
     False.
 
     It is assumed that `name` doesn't have `.__init__`.
 
     `absname` is an absolute module name of specified `name`
     (e.g. "hgext.convert"). This can be used to compose prefix for sub
     modules or so.
 
     `dottedpath` is a `dotted_name_of_path()`-ed source file path
     (e.g. "hgext.convert.__init__") of `name`. This is used to look
     module up in `localmods` again.
 
     `hassubmod` is whether it may have sub modules under it (for
     convenient, even though this is also equivalent to "absname !=
     dottednpath")
 
     >>> localmods = {'foo.__init__', 'foo.foo1',
     ...              'foo.bar.__init__', 'foo.bar.bar1',
     ...              'baz.__init__', 'baz.baz1'}
     >>> fromlocal = fromlocalfunc('foo.xxx', localmods)
     >>> # relative
     >>> fromlocal('foo1')
     ('foo.foo1', 'foo.foo1', False)
     >>> fromlocal('bar')
     ('foo.bar', 'foo.bar.__init__', True)
     >>> fromlocal('bar.bar1')
     ('foo.bar.bar1', 'foo.bar.bar1', False)
     >>> # absolute
     >>> fromlocal('baz')
     ('baz', 'baz.__init__', True)
     >>> fromlocal('baz.baz1')
     ('baz.baz1', 'baz.baz1', False)
     >>> # unknown = maybe standard library
     >>> fromlocal('os')
     False
     >>> fromlocal(None, 1)
     ('foo', 'foo.__init__', True)
     >>> fromlocal('foo1', 1)
     ('foo.foo1', 'foo.foo1', False)
     >>> fromlocal2 = fromlocalfunc('foo.xxx.yyy', localmods)
     >>> fromlocal2(None, 2)
     ('foo', 'foo.__init__', True)
     >>> fromlocal2('bar2', 1)
     False
     >>> fromlocal2('bar', 2)
     ('foo.bar', 'foo.bar.__init__', True)
     """
     if not isinstance(modulename, str):
         modulename = modulename.decode('ascii')
     prefix = '.'.join(modulename.split('.')[:-1])
     if prefix:
         prefix += '.'
 
     def fromlocal(name, level=0):
         # name is false value when relative imports are used.
         if not name:
             # If relative imports are used, level must not be absolute.
             assert level > 0
             candidates = ['.'.join(modulename.split('.')[:-level])]
         else:
             if not level:
                 # Check relative name first.
                 candidates = [prefix + name, name]
             else:
                 candidates = [
                     '.'.join(modulename.split('.')[:-level]) + '.' + name
                 ]
 
         for n in candidates:
             if n in localmods:
                 return (n, n, False)
             dottedpath = n + '.__init__'
             if dottedpath in localmods:
                 return (n, dottedpath, True)
         return False
 
     return fromlocal
 
 
 def populateextmods(localmods):
     """Populate C extension modules based on pure modules"""
     newlocalmods = set(localmods)
     for n in localmods:
         if n.startswith('mercurial.pure.'):
             m = n[len('mercurial.pure.') :]
             newlocalmods.add('mercurial.cext.' + m)
             newlocalmods.add('mercurial.cffi._' + m)
     return newlocalmods
 
 
 def list_stdlib_modules():
     """List the modules present in the stdlib.
 
     >>> py3 = sys.version_info[0] >= 3
     >>> mods = set(list_stdlib_modules())
     >>> 'BaseHTTPServer' in mods or py3
     True
 
     os.path isn't really a module, so it's missing:
 
     >>> 'os.path' in mods
     False
 
     sys requires special treatment, because it's baked into the
     interpreter, but it should still appear:
 
     >>> 'sys' in mods
     True
 
     >>> 'collections' in mods
     True
 
     >>> 'cStringIO' in mods or py3
     True
 
     >>> 'cffi' in mods
     True
     """
     for m in sys.builtin_module_names:
         yield m
     # These modules only exist on windows, but we should always
     # consider them stdlib.
     for m in ['msvcrt', '_winreg']:
         yield m
     yield '__builtin__'
     yield 'builtins'  # python3 only
     yield 'importlib.abc'  # python3 only
     yield 'importlib.machinery'  # python3 only
     yield 'importlib.util'  # python3 only
     for m in 'fcntl', 'grp', 'pwd', 'termios':  # Unix only
         yield m
     for m in 'cPickle', 'datetime':  # in Python (not C) on PyPy
         yield m
     for m in ['cffi']:
         yield m
     stdlib_prefixes = {sys.prefix, sys.exec_prefix}
     # We need to supplement the list of prefixes for the search to work
     # when run from within a virtualenv.
     for mod in (basehttpserver, zlib):
         if mod is None:
             continue
         try:
             # Not all module objects have a __file__ attribute.
             filename = mod.__file__
         except AttributeError:
             continue
         dirname = os.path.dirname(filename)
         for prefix in stdlib_prefixes:
             if dirname.startswith(prefix):
                 # Then this directory is redundant.
                 break
         else:
             stdlib_prefixes.add(dirname)
     sourceroot = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
     for libpath in sys.path:
         # We want to walk everything in sys.path that starts with something in
         # stdlib_prefixes, but not directories from the hg sources.
         if os.path.abspath(libpath).startswith(sourceroot) or not any(
             libpath.startswith(p) for p in stdlib_prefixes
         ):
             continue
         for top, dirs, files in os.walk(libpath):
             for i, d in reversed(list(enumerate(dirs))):
                 if (
                     not os.path.exists(os.path.join(top, d, '__init__.py'))
                     or top == libpath
                     and d in ('hgdemandimport', 'hgext', 'mercurial')
                 ):
                     del dirs[i]
             for name in files:
                 if not name.endswith(('.py', '.so', '.pyc', '.pyo', '.pyd')):
                     continue
                 if name.startswith('__init__.py'):
                     full_path = top
                 else:
                     full_path = os.path.join(top, name)
                 rel_path = full_path[len(libpath) + 1 :]
                 mod = dotted_name_of_path(rel_path)
                 yield mod
 
 
 stdlib_modules = set(list_stdlib_modules())
 
 
 def imported_modules(source, modulename, f, localmods, ignore_nested=False):
     """Given the source of a file as a string, yield the names
     imported by that file.
 
     Args:
       source: The python source to examine as a string.
       modulename: of specified python source (may have `__init__`)
       localmods: set of locally defined module names (may have `__init__`)
       ignore_nested: If true, import statements that do not start in
                      column zero will be ignored.
 
     Returns:
       A list of absolute module names imported by the given source.
 
     >>> f = 'foo/xxx.py'
     >>> modulename = 'foo.xxx'
     >>> localmods = {'foo.__init__': True,
     ...              'foo.foo1': True, 'foo.foo2': True,
     ...              'foo.bar.__init__': True, 'foo.bar.bar1': True,
     ...              'baz.__init__': True, 'baz.baz1': True }
     >>> # standard library (= not locally defined ones)
     >>> sorted(imported_modules(
     ...        'from stdlib1 import foo, bar; import stdlib2',
     ...        modulename, f, localmods))
     []
     >>> # relative importing
     >>> sorted(imported_modules(
     ...        'import foo1; from bar import bar1',
     ...        modulename, f, localmods))
     ['foo.bar.bar1', 'foo.foo1']
     >>> sorted(imported_modules(
     ...        'from bar.bar1 import name1, name2, name3',
     ...        modulename, f, localmods))
     ['foo.bar.bar1']
     >>> # absolute importing
     >>> sorted(imported_modules(
     ...        'from baz import baz1, name1',
     ...        modulename, f, localmods))
     ['baz.__init__', 'baz.baz1']
     >>> # mixed importing, even though it shouldn't be recommended
     >>> sorted(imported_modules(
     ...        'import stdlib, foo1, baz',
     ...        modulename, f, localmods))
     ['baz.__init__', 'foo.foo1']
     >>> # ignore_nested
     >>> sorted(imported_modules(
     ... '''import foo
     ... def wat():
     ...     import bar
     ... ''', modulename, f, localmods))
     ['foo.__init__', 'foo.bar.__init__']
     >>> sorted(imported_modules(
     ... '''import foo
     ... def wat():
     ...     import bar
     ... ''', modulename, f, localmods, ignore_nested=True))
     ['foo.__init__']
     """
     fromlocal = fromlocalfunc(modulename, localmods)
     for node in ast.walk(ast.parse(source, f)):
         if ignore_nested and getattr(node, 'col_offset', 0) > 0:
             continue
         if isinstance(node, ast.Import):
             for n in node.names:
                 found = fromlocal(n.name)
                 if not found:
                     # this should import standard library
                     continue
                 yield found[1]
         elif isinstance(node, ast.ImportFrom):
             found = fromlocal(node.module, node.level)
             if not found:
                 # this should import standard library
                 continue
 
             absname, dottedpath, hassubmod = found
             if not hassubmod:
                 # "dottedpath" is not a package; must be imported
                 yield dottedpath
                 # examination of "node.names" should be redundant
                 # e.g.: from mercurial.node import nullid, nullrev
                 continue
 
             modnotfound = False
             prefix = absname + '.'
             for n in node.names:
                 found = fromlocal(prefix + n.name)
                 if not found:
                     # this should be a function or a property of "node.module"
                     modnotfound = True
                     continue
                 yield found[1]
             if modnotfound and dottedpath != modulename:
                 # "dottedpath" is a package, but imported because of non-module
                 # lookup
                 # specifically allow "from . import foo" from __init__.py
                 yield dottedpath
 
 
 def verify_import_convention(module, source, localmods):
     """Verify imports match our established coding convention.
 
     We have 2 conventions: legacy and modern. The modern convention is in
     effect when using absolute imports.
 
     The legacy convention only looks for mixed imports. The modern convention
     is much more thorough.
     """
     root = ast.parse(source)
     absolute = usingabsolute(root)
 
     if absolute:
         return verify_modern_convention(module, root, localmods)
     else:
         return verify_stdlib_on_own_line(root)
 
 
 def verify_modern_convention(module, root, localmods, root_col_offset=0):
     """Verify a file conforms to the modern import convention rules.
 
     The rules of the modern convention are:
 
     * Ordering is stdlib followed by local imports. Each group is lexically
       sorted.
     * Importing multiple modules via "import X, Y" is not allowed: use
       separate import statements.
     * Importing multiple modules via "from X import ..." is allowed if using
       parenthesis and one entry per line.
     * Only 1 relative import statement per import level ("from .", "from ..")
       is allowed.
     * Relative imports from higher levels must occur before lower levels. e.g.
       "from .." must be before "from .".
     * Imports from peer packages should use relative import (e.g. do not
       "import mercurial.foo" from a "mercurial.*" module).
     * Symbols can only be imported from specific modules (see
       `allowsymbolimports`). For other modules, first import the module then
       assign the symbol to a module-level variable. In addition, these imports
       must be performed before other local imports. This rule only
       applies to import statements outside of any blocks.
     * Relative imports from the standard library are not allowed, unless that
       library is also a local module.
     * Certain modules must be aliased to alternate names to avoid aliasing
       and readability problems. See `requirealias`.
     """
     if not isinstance(module, str):
         module = module.decode('ascii')
     topmodule = module.split('.')[0]
     fromlocal = fromlocalfunc(module, localmods)
 
     # Whether a local/non-stdlib import has been performed.
     seenlocal = None
     # Whether a local/non-stdlib, non-symbol import has been seen.
     seennonsymbollocal = False
     # The last name to be imported (for sorting).
     lastname = None
     laststdlib = None
     # Relative import levels encountered so far.
     seenlevels = set()
 
     for node, newscope in walklocal(root):
 
         def msg(fmt, *args):
             return (fmt % args, node.lineno)
 
         if newscope:
             # Check for local imports in function
             for r in verify_modern_convention(
                 module, node, localmods, node.col_offset + 4
             ):
                 yield r
         elif isinstance(node, ast.Import):
             # Disallow "import foo, bar" and require separate imports
             # for each module.
             if len(node.names) > 1:
                 yield msg(
                     'multiple imported names: %s',
                     ', '.join(n.name for n in node.names),
                 )
 
             name = node.names[0].name
             asname = node.names[0].asname
 
             stdlib = name in stdlib_modules
 
             # Ignore sorting rules on imports inside blocks.
             if node.col_offset == root_col_offset:
                 if lastname and name < lastname and laststdlib == stdlib:
                     yield msg(
                         'imports not lexically sorted: %s < %s', name, lastname
                     )
 
             lastname = name
             laststdlib = stdlib
 
             # stdlib imports should be before local imports.
             if stdlib and seenlocal and node.col_offset == root_col_offset:
                 yield msg(
                     'stdlib import "%s" follows local import: %s',
                     name,
                     seenlocal,
                 )
 
             if not stdlib:
                 seenlocal = name
 
             # Import of sibling modules should use relative imports.
             topname = name.split('.')[0]
             if topname == topmodule:
                 yield msg('import should be relative: %s', name)
 
             if name in requirealias and asname != requirealias[name]:
                 yield msg(
                     '%s module must be "as" aliased to %s',
                     name,
                     requirealias[name],
                 )
 
         elif isinstance(node, ast.ImportFrom):
             # Resolve the full imported module name.
             if node.level > 0:
                 fullname = '.'.join(module.split('.')[: -node.level])
                 if node.module:
                     fullname += '.%s' % node.module
             else:
                 assert node.module
                 fullname = node.module
 
                 topname = fullname.split('.')[0]
                 if topname == topmodule:
                     yield msg('import should be relative: %s', fullname)
 
             # __future__ is special since it needs to come first and use
             # symbol import.
             if fullname != '__future__':
                 if not fullname or (
                     fullname in stdlib_modules
                     # allow standard 'from typing import ...' style
                     and fullname.startswith('.')
                     and fullname not in localmods
                     and fullname + '.__init__' not in localmods
                 ):
                     yield msg('relative import of stdlib module')
                 else:
                     seenlocal = fullname
 
             # Direct symbol import is only allowed from certain modules and
             # must occur before non-symbol imports.
             found = fromlocal(node.module, node.level)
             if found and found[2]:  # node.module is a package
                 prefix = found[0] + '.'
                 symbols = (
                     n.name for n in node.names if not fromlocal(prefix + n.name)
                 )
             else:
                 symbols = (n.name for n in node.names)
             symbols = [sym for sym in symbols if sym not in directsymbols]
             if node.module and node.col_offset == root_col_offset:
                 if symbols and fullname not in allowsymbolimports:
                     yield msg(
                         'direct symbol import %s from %s',
                         ', '.join(symbols),
                         fullname,
                     )
 
                 if symbols and seennonsymbollocal:
                     yield msg(
                         'symbol import follows non-symbol import: %s', fullname
                     )
             if not symbols and fullname not in stdlib_modules:
                 seennonsymbollocal = True
 
             if not node.module:
                 assert node.level
 
                 # Only allow 1 group per level.
                 if (
                     node.level in seenlevels
                     and node.col_offset == root_col_offset
                 ):
                     yield msg(
                         'multiple "from %s import" statements', '.' * node.level
                     )
 
                 # Higher-level groups come before lower-level groups.
                 if any(node.level > l for l in seenlevels):
                     yield msg(
                         'higher-level import should come first: %s', fullname
                     )
 
                 seenlevels.add(node.level)
 
             # Entries in "from .X import ( ... )" lists must be lexically
             # sorted.
             lastentryname = None
 
             for n in node.names:
                 if lastentryname and n.name < lastentryname:
                     yield msg(
                         'imports from %s not lexically sorted: %s < %s',
                         fullname,
                         n.name,
                         lastentryname,
                     )
 
                 lastentryname = n.name
 
                 if n.name in requirealias and n.asname != requirealias[n.name]:
                     yield msg(
                         '%s from %s must be "as" aliased to %s',
                         n.name,
                         fullname,
                         requirealias[n.name],
                     )
 
 
 def verify_stdlib_on_own_line(root):
     """Given some python source, verify that stdlib imports are done
     in separate statements from relative local module imports.
 
     >>> list(verify_stdlib_on_own_line(ast.parse('import sys, foo')))
     [('mixed imports\\n   stdlib:    sys\\n   relative:  foo', 1)]
     >>> list(verify_stdlib_on_own_line(ast.parse('import sys, os')))
     []
     >>> list(verify_stdlib_on_own_line(ast.parse('import foo, bar')))
     []
     """
     for node in ast.walk(root):
         if isinstance(node, ast.Import):
             from_stdlib = {False: [], True: []}
             for n in node.names:
                 from_stdlib[n.name in stdlib_modules].append(n.name)
             if from_stdlib[True] and from_stdlib[False]:
                 yield (
                     'mixed imports\n   stdlib:    %s\n   relative:  %s'
                     % (
                         ', '.join(sorted(from_stdlib[True])),
                         ', '.join(sorted(from_stdlib[False])),
                     ),
                     node.lineno,
                 )
 
 
 class CircularImport(Exception):
     pass
 
 
 def checkmod(mod, imports):
     shortest = {}
     visit = [[mod]]
     while visit:
         path = visit.pop(0)
         for i in sorted(imports.get(path[-1], [])):
             if len(path) < shortest.get(i, 1000):
                 shortest[i] = len(path)
                 if i in path:
                     if i == path[0]:
                         raise CircularImport(path)
                     continue
                 visit.append(path + [i])
 
 
 def rotatecycle(cycle):
     """arrange a cycle so that the lexicographically first module listed first
 
     >>> rotatecycle(['foo', 'bar'])
     ['bar', 'foo', 'bar']
     """
     lowest = min(cycle)
     idx = cycle.index(lowest)
     return cycle[idx:] + cycle[:idx] + [lowest]
 
 
 def find_cycles(imports):
     """Find cycles in an already-loaded import graph.
 
     All module names recorded in `imports` should be absolute one.
 
     >>> from __future__ import print_function
     >>> imports = {'top.foo': ['top.bar', 'os.path', 'top.qux'],
     ...            'top.bar': ['top.baz', 'sys'],
     ...            'top.baz': ['top.foo'],
     ...            'top.qux': ['top.foo']}
     >>> print('\\n'.join(sorted(find_cycles(imports))))
     top.bar -> top.baz -> top.foo -> top.bar
     top.foo -> top.qux -> top.foo
     """
     cycles = set()
     for mod in sorted(imports.keys()):
         try:
             checkmod(mod, imports)
         except CircularImport as e:
             cycle = e.args[0]
             cycles.add(" -> ".join(rotatecycle(cycle)))
     return cycles
 
 
 def _cycle_sortkey(c):
     return len(c), c
 
 
 def embedded(f, modname, src):
     """Extract embedded python code
 
     >>> def _forcestr(thing):
     ...     if not isinstance(thing, str):
     ...         return thing.decode('ascii')
     ...     return thing
     >>> def test(fn, lines):
     ...     for s, m, f, l in embedded(fn, b"example", lines):
     ...         print("%s %s %d" % (_forcestr(m), _forcestr(f), l))
     ...         print(repr(_forcestr(s)))
     >>> lines = [
     ...   'comment',
     ...   '  >>> from __future__ import print_function',
     ...   "  >>> ' multiline",
     ...   "  ... string'",
     ...   '  ',
     ...   'comment',
     ...   '  $ cat > foo.py <<EOF',
     ...   '  > from __future__ import print_function',
     ...   '  > EOF',
     ... ]
     >>> test(b"example.t", lines)
     example[2] doctest.py 1
     "from __future__ import print_function\\n' multiline\\nstring'\\n\\n"
     example[8] foo.py 7
     'from __future__ import print_function\\n'
     """
     errors = []
     for name, starts, ends, code in testparseutil.pyembedded(f, src, errors):
         if not name:
             # use 'doctest.py', in order to make already existing
             # doctest above pass instantly
             name = 'doctest.py'
         # "starts" is "line number" (1-origin), but embedded() is
         # expected to return "line offset" (0-origin). Therefore, this
         # yields "starts - 1".
         if not isinstance(modname, str):
             modname = modname.decode('utf8')
         yield code, "%s[%d]" % (modname, starts), name, starts - 1
 
 
 def sources(f, modname):
     """Yields possibly multiple sources from a filepath
 
     input: filepath, modulename
     yields:  script(string), modulename, filepath, linenumber
 
     For embedded scripts, the modulename and filepath will be different
     from the function arguments. linenumber is an offset relative to
     the input file.
     """
     py = False
     if not f.endswith('.t'):
         with open(f, 'rb') as src:
             yield src.read(), modname, f, 0
             py = True
     if py or f.endswith('.t'):
         # Strictly speaking we should sniff for the magic header that denotes
         # Python source file encoding. But in reality we don't use anything
         # other than ASCII (mainly) and UTF-8 (in a few exceptions), so
         # simplicity is fine.
         with io.open(f, 'r', encoding='utf-8') as src:
             for script, modname, t, line in embedded(f, modname, src):
                 yield script, modname.encode('utf8'), t, line
 
 
 def main(argv):
     if len(argv) < 2 or (argv[1] == '-' and len(argv) > 2):
         print('Usage: %s {-|file [file] [file] ...}')
         return 1
     if argv[1] == '-':
         argv = argv[:1]
         argv.extend(l.rstrip() for l in sys.stdin.readlines())
     localmodpaths = {}
     used_imports = {}
     any_errors = False
     for source_path in argv[1:]:
         modname = dotted_name_of_path(source_path)
         localmodpaths[modname] = source_path
     localmods = populateextmods(localmodpaths)
     for localmodname, source_path in sorted(localmodpaths.items()):
         if not isinstance(localmodname, bytes):
             # This is only safe because all hg's files are ascii
             localmodname = localmodname.encode('ascii')
         for src, modname, name, line in sources(source_path, localmodname):
             try:
                 used_imports[modname] = sorted(
                     imported_modules(
                         src, modname, name, localmods, ignore_nested=True
                     )
                 )
                 for error, lineno in verify_import_convention(
                     modname, src, localmods
                 ):
                     any_errors = True
                     print('%s:%d: %s' % (source_path, lineno + line, error))
             except SyntaxError as e:
                 print(
                     '%s:%d: SyntaxError: %s' % (source_path, e.lineno + line, e)
                 )
     cycles = find_cycles(used_imports)
     if cycles:
         firstmods = set()
         for c in sorted(cycles, key=_cycle_sortkey):
             first = c.split()[0]
             # As a rough cut, ignore any cycle that starts with the
             # same module as some other cycle. Otherwise we see lots
             # of cycles that are effectively duplicates.
             if first in firstmods:
                 continue
             print('Import cycle:', c)
             firstmods.add(first)
         any_errors = True
     return any_errors != 0
 
 
 if __name__ == '__main__':
     sys.exit(int(main(sys.argv)))

contrib/install-windows-dependencies.ps1

@@ -1,211 +1,215 b''
 # install-dependencies.ps1 - Install Windows dependencies for building Mercurial
 #
 # Copyright 2019 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # This script can be used to bootstrap a Mercurial build environment on
 # Windows.
 #
 # The script makes a lot of assumptions about how things should work.
 # For example, the install location of Python is hardcoded to c:\hgdev\*.
 #
 # The script should be executed from a PowerShell with elevated privileges
 # if you don't want to see a UAC prompt for various installers.
 #
 # The script is tested on Windows 10 and Windows Server 2019 (in EC2).
 
 $VS_BUILD_TOOLS_URL = "https://download.visualstudio.microsoft.com/download/pr/a1603c02-8a66-4b83-b821-811e3610a7c4/aa2db8bb39e0cbd23e9940d8951e0bc3/vs_buildtools.exe"
 $VS_BUILD_TOOLS_SHA256 = "911E292B8E6E5F46CBC17003BDCD2D27A70E616E8D5E6E69D5D489A605CAA139"
 
 $VC9_PYTHON_URL = "https://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi"
 $VC9_PYTHON_SHA256 = "070474db76a2e625513a5835df4595df9324d820f9cc97eab2a596dcbc2f5cbf"
 
 $PYTHON27_x64_URL = "https://www.python.org/ftp/python/2.7.18/python-2.7.18.amd64.msi"
 $PYTHON27_x64_SHA256 = "b74a3afa1e0bf2a6fc566a7b70d15c9bfabba3756fb077797d16fffa27800c05"
 $PYTHON27_X86_URL = "https://www.python.org/ftp/python/2.7.18/python-2.7.18.msi"
 $PYTHON27_X86_SHA256 = "d901802e90026e9bad76b8a81f8dd7e43c7d7e8269d9281c9e9df7a9c40480a9"
 
 $PYTHON37_x86_URL = "https://www.python.org/ftp/python/3.7.9/python-3.7.9.exe"
 $PYTHON37_x86_SHA256 = "769bb7c74ad1df6d7d74071cc16a984ff6182e4016e11b8949b93db487977220"
 $PYTHON37_X64_URL = "https://www.python.org/ftp/python/3.7.9/python-3.7.9-amd64.exe"
 $PYTHON37_x64_SHA256 = "e69ed52afb5a722e5c56f6c21d594e85c17cb29f12f18bb69751cf1714e0f987"
 
-$PYTHON38_x86_URL = "https://www.python.org/ftp/python/3.8.6/python-3.8.6.exe"
-$PYTHON38_x86_SHA256 = "287d5df01ff22ff09e6a487ae018603ee19eade71d462ec703850c96f1d5e8a0"
-$PYTHON38_x64_URL = "https://www.python.org/ftp/python/3.8.6/python-3.8.6-amd64.exe"
-$PYTHON38_x64_SHA256 = "328a257f189cb500606bb26ab0fbdd298ed0e05d8c36540a322a1744f489a0a0"
+$PYTHON38_x86_URL = "https://www.python.org/ftp/python/3.8.10/python-3.8.10.exe"
+$PYTHON38_x86_SHA256 = "ad07633a1f0cd795f3bf9da33729f662281df196b4567fa795829f3bb38a30ac"
+$PYTHON38_x64_URL = "https://www.python.org/ftp/python/3.8.10/python-3.8.10-amd64.exe"
+$PYTHON38_x64_SHA256 = "7628244cb53408b50639d2c1287c659f4e29d3dfdb9084b11aed5870c0c6a48a"
 
-$PYTHON39_x86_URL = "https://www.python.org/ftp/python/3.9.0/python-3.9.0.exe"
-$PYTHON39_x86_SHA256 = "a4c65917f4225d1543959342f0615c813a4e9e7ff1137c4394ff6a5290ac1913"
-$PYTHON39_x64_URL = "https://www.python.org/ftp/python/3.9.0/python-3.9.0-amd64.exe"
-$PYTHON39_x64_SHA256 = "fd2e2c6612d43bb6b213b72fc53f07d73d99059fa72c96e44bde12e7815073ae"
+$PYTHON39_x86_URL = "https://www.python.org/ftp/python/3.9.5/python-3.9.5.exe"
+$PYTHON39_x86_SHA256 = "505129081a839b699a6ab9064b441ad922ef03767b5dd4241fd0c2166baf64de"
+$PYTHON39_x64_URL = "https://www.python.org/ftp/python/3.9.5/python-3.9.5-amd64.exe"
+$PYTHON39_x64_SHA256 = "84d5243088ba00c11e51905c704dbe041040dfff044f4e1ce5476844ee2e6eac"
 
 # PIP 19.2.3.
 $PIP_URL = "https://github.com/pypa/get-pip/raw/309a56c5fd94bd1134053a541cb4657a4e47e09d/get-pip.py"
 $PIP_SHA256 = "57e3643ff19f018f8a00dfaa6b7e4620e3c1a7a2171fd218425366ec006b3bfe"
 
 $VIRTUALENV_URL = "https://files.pythonhosted.org/packages/66/f0/6867af06d2e2f511e4e1d7094ff663acdebc4f15d4a0cb0fed1007395124/virtualenv-16.7.5.tar.gz"
 $VIRTUALENV_SHA256 = "f78d81b62d3147396ac33fc9d77579ddc42cc2a98dd9ea38886f616b33bc7fb2"
 
 $INNO_SETUP_URL = "http://files.jrsoftware.org/is/5/innosetup-5.6.1-unicode.exe"
 $INNO_SETUP_SHA256 = "27D49E9BC769E9D1B214C153011978DB90DC01C2ACD1DDCD9ED7B3FE3B96B538"
 
 $MINGW_BIN_URL = "https://osdn.net/frs/redir.php?m=constant&f=mingw%2F68260%2Fmingw-get-0.6.3-mingw32-pre-20170905-1-bin.zip"
 $MINGW_BIN_SHA256 = "2AB8EFD7C7D1FC8EAF8B2FA4DA4EEF8F3E47768284C021599BC7435839A046DF"
 
 $MERCURIAL_WHEEL_FILENAME = "mercurial-5.1.2-cp27-cp27m-win_amd64.whl"
 $MERCURIAL_WHEEL_URL = "https://files.pythonhosted.org/packages/6d/47/e031e47f7fe9b16e4e3383da47e2b0a7eae6e603996bc67a03ec4fa1b3f4/$MERCURIAL_WHEEL_FILENAME"
 $MERCURIAL_WHEEL_SHA256 = "1d18c7f6ca1456f0f62ee65c9a50c14cbba48ce6e924930cdb10537f5c9eaf5f"
 
 $RUSTUP_INIT_URL = "https://static.rust-lang.org/rustup/archive/1.21.1/x86_64-pc-windows-gnu/rustup-init.exe"
 $RUSTUP_INIT_SHA256 = "d17df34ba974b9b19cf5c75883a95475aa22ddc364591d75d174090d55711c72"
 
+$PYOXIDIZER_URL = "https://github.com/indygreg/PyOxidizer/releases/download/pyoxidizer%2F0.16.0/PyOxidizer-0.16.0-x64.msi"
+$PYOXIDIZER_SHA256 = "2a9c58add9161c272c418d5e6dec13fbe648f624b5d26770190357e4d664f24e"
+
 # Writing progress slows down downloads substantially. So disable it.
 $progressPreference = 'silentlyContinue'
 
 function Secure-Download($url, $path, $sha256) {
     if (Test-Path -Path $path) {
         Get-FileHash -Path $path -Algorithm SHA256 -OutVariable hash
 
         if ($hash.Hash -eq $sha256) {
             Write-Output "SHA256 of $path verified as $sha256"
             return
         }
 
         Write-Output "hash mismatch on $path; downloading again"
     }
 
     Write-Output "downloading $url to $path"
     Invoke-WebRequest -Uri $url -OutFile $path
     Get-FileHash -Path $path -Algorithm SHA256 -OutVariable hash
 
     if ($hash.Hash -ne $sha256) {
         Remove-Item -Path $path
         throw "hash mismatch when downloading $url; got $($hash.Hash), expected $sha256"
     }
 }
 
 function Invoke-Process($path, $arguments) {
     $p = Start-Process -FilePath $path -ArgumentList $arguments -Wait -PassThru -WindowStyle Hidden
 
     if ($p.ExitCode -ne 0) {
         throw "process exited non-0: $($p.ExitCode)"
     }
 }
 
 function Install-Python3($name, $installer, $dest, $pip) {
     Write-Output "installing $name"
 
     # We hit this when running the script as part of Simple Systems Manager in
     # EC2. The Python 3 installer doesn't seem to like per-user installs
     # when running as the SYSTEM user. So enable global installs if executed in
     # this mode.
     if ($env:USERPROFILE -eq "C:\Windows\system32\config\systemprofile") {
         Write-Output "running with SYSTEM account; installing for all users"
         $allusers = "1"
     }
     else {
         $allusers = "0"
     }
 
     Invoke-Process $installer "/quiet TargetDir=${dest} InstallAllUsers=${allusers} AssociateFiles=0 CompileAll=0 PrependPath=0 Include_doc=0 Include_launcher=0 InstallLauncherAllUsers=0 Include_pip=0 Include_test=0"
     Invoke-Process ${dest}\python.exe $pip
 }
 
 function Install-Rust($prefix) {
     Write-Output "installing Rust"
     $Env:RUSTUP_HOME = "${prefix}\rustup"
     $Env:CARGO_HOME = "${prefix}\cargo"
 
     Invoke-Process "${prefix}\assets\rustup-init.exe" "-y --default-host x86_64-pc-windows-msvc"
     Invoke-Process "${prefix}\cargo\bin\rustup.exe" "target add i686-pc-windows-msvc"
-    Invoke-Process "${prefix}\cargo\bin\rustup.exe" "install 1.46.0"
+    Invoke-Process "${prefix}\cargo\bin\rustup.exe" "install 1.52.0"
     Invoke-Process "${prefix}\cargo\bin\rustup.exe" "component add clippy"
-
-    # Install PyOxidizer for packaging.
-    Invoke-Process "${prefix}\cargo\bin\cargo.exe" "install --version 0.10.3 pyoxidizer"
 }
 
 function Install-Dependencies($prefix) {
     if (!(Test-Path -Path $prefix\assets)) {
         New-Item -Path $prefix\assets -ItemType Directory
     }
 
     $pip = "${prefix}\assets\get-pip.py"
 
     Secure-Download $VC9_PYTHON_URL ${prefix}\assets\VCForPython27.msi $VC9_PYTHON_SHA256
     Secure-Download $PYTHON27_x86_URL ${prefix}\assets\python27-x86.msi $PYTHON27_x86_SHA256
     Secure-Download $PYTHON27_x64_URL ${prefix}\assets\python27-x64.msi $PYTHON27_x64_SHA256
     Secure-Download $PYTHON37_x86_URL ${prefix}\assets\python37-x86.exe $PYTHON37_x86_SHA256
     Secure-Download $PYTHON37_x64_URL ${prefix}\assets\python37-x64.exe $PYTHON37_x64_SHA256
     Secure-Download $PYTHON38_x86_URL ${prefix}\assets\python38-x86.exe $PYTHON38_x86_SHA256
     Secure-Download $PYTHON38_x64_URL ${prefix}\assets\python38-x64.exe $PYTHON38_x64_SHA256
     Secure-Download $PYTHON39_x86_URL ${prefix}\assets\python39-x86.exe $PYTHON39_x86_SHA256
     Secure-Download $PYTHON39_x64_URL ${prefix}\assets\python39-x64.exe $PYTHON39_x64_SHA256
     Secure-Download $PIP_URL ${pip} $PIP_SHA256
     Secure-Download $VIRTUALENV_URL ${prefix}\assets\virtualenv.tar.gz $VIRTUALENV_SHA256
     Secure-Download $VS_BUILD_TOOLS_URL ${prefix}\assets\vs_buildtools.exe $VS_BUILD_TOOLS_SHA256
     Secure-Download $INNO_SETUP_URL ${prefix}\assets\InnoSetup.exe $INNO_SETUP_SHA256
     Secure-Download $MINGW_BIN_URL ${prefix}\assets\mingw-get-bin.zip $MINGW_BIN_SHA256
     Secure-Download $MERCURIAL_WHEEL_URL ${prefix}\assets\${MERCURIAL_WHEEL_FILENAME} $MERCURIAL_WHEEL_SHA256
     Secure-Download $RUSTUP_INIT_URL ${prefix}\assets\rustup-init.exe $RUSTUP_INIT_SHA256
+    Secure-Download $PYOXIDIZER_URL ${prefix}\assets\PyOxidizer.msi $PYOXIDIZER_SHA256
 
     Write-Output "installing Python 2.7 32-bit"
     Invoke-Process msiexec.exe "/i ${prefix}\assets\python27-x86.msi /l* ${prefix}\assets\python27-x86.log /q TARGETDIR=${prefix}\python27-x86 ALLUSERS="
     Invoke-Process ${prefix}\python27-x86\python.exe ${prefix}\assets\get-pip.py
     Invoke-Process ${prefix}\python27-x86\Scripts\pip.exe "install ${prefix}\assets\virtualenv.tar.gz"
 
     Write-Output "installing Python 2.7 64-bit"
     Invoke-Process msiexec.exe "/i ${prefix}\assets\python27-x64.msi /l* ${prefix}\assets\python27-x64.log /q TARGETDIR=${prefix}\python27-x64 ALLUSERS="
     Invoke-Process ${prefix}\python27-x64\python.exe ${prefix}\assets\get-pip.py
     Invoke-Process ${prefix}\python27-x64\Scripts\pip.exe "install ${prefix}\assets\virtualenv.tar.gz"
 
     Install-Python3 "Python 3.7 32-bit" ${prefix}\assets\python37-x86.exe ${prefix}\python37-x86 ${pip}
     Install-Python3 "Python 3.7 64-bit" ${prefix}\assets\python37-x64.exe ${prefix}\python37-x64 ${pip}
     Install-Python3 "Python 3.8 32-bit" ${prefix}\assets\python38-x86.exe ${prefix}\python38-x86 ${pip}
     Install-Python3 "Python 3.8 64-bit" ${prefix}\assets\python38-x64.exe ${prefix}\python38-x64 ${pip}
     Install-Python3 "Python 3.9 32-bit" ${prefix}\assets\python39-x86.exe ${prefix}\python39-x86 ${pip}
     Install-Python3 "Python 3.9 64-bit" ${prefix}\assets\python39-x64.exe ${prefix}\python39-x64 ${pip}
 
     Write-Output "installing Visual Studio 2017 Build Tools and SDKs"
     Invoke-Process ${prefix}\assets\vs_buildtools.exe "--quiet --wait --norestart --nocache --channelUri https://aka.ms/vs/15/release/channel --add Microsoft.VisualStudio.Workload.MSBuildTools --add Microsoft.VisualStudio.Component.Windows10SDK.17763 --add Microsoft.VisualStudio.Workload.VCTools --add Microsoft.VisualStudio.Component.Windows10SDK --add Microsoft.VisualStudio.Component.VC.140"
 
+    Write-Output "installing PyOxidizer"
+    Invoke-Process msiexec.exe "/i ${prefix}\assets\PyOxidizer.msi /l* ${prefix}\assets\PyOxidizer.log /quiet"
+
     Install-Rust ${prefix}
 
     Write-Output "installing Visual C++ 9.0 for Python 2.7"
     Invoke-Process msiexec.exe "/i ${prefix}\assets\VCForPython27.msi /l* ${prefix}\assets\VCForPython27.log /q"
 
     Write-Output "installing Inno Setup"
     Invoke-Process ${prefix}\assets\InnoSetup.exe "/SP- /VERYSILENT /SUPPRESSMSGBOXES"
 
     Write-Output "extracting MinGW base archive"
     Expand-Archive -Path ${prefix}\assets\mingw-get-bin.zip -DestinationPath "${prefix}\MinGW" -Force
 
     Write-Output "updating MinGW package catalogs"
     Invoke-Process ${prefix}\MinGW\bin\mingw-get.exe "update"
 
     Write-Output "installing MinGW packages"
     Invoke-Process ${prefix}\MinGW\bin\mingw-get.exe "install msys-base msys-coreutils msys-diffutils msys-unzip"
 
     # Construct a virtualenv useful for bootstrapping. It conveniently contains a
     # Mercurial install.
     Write-Output "creating bootstrap virtualenv with Mercurial"
     Invoke-Process "$prefix\python27-x64\Scripts\virtualenv.exe" "${prefix}\venv-bootstrap"
     Invoke-Process "${prefix}\venv-bootstrap\Scripts\pip.exe" "install ${prefix}\assets\${MERCURIAL_WHEEL_FILENAME}"
 }
 
 function Clone-Mercurial-Repo($prefix, $repo_url, $dest) {
     Write-Output "cloning $repo_url to $dest"
     # TODO Figure out why CA verification isn't working in EC2 and remove
     # --insecure.
     Invoke-Process "${prefix}\venv-bootstrap\Scripts\hg.exe" "clone --insecure $repo_url $dest"
 
     # Mark repo as non-publishing by default for convenience.
     Add-Content -Path "$dest\.hg\hgrc" -Value "`n[phases]`npublish = false"
 }
 
 $prefix = "c:\hgdev"
 Install-Dependencies $prefix
 Clone-Mercurial-Repo $prefix "https://www.mercurial-scm.org/repo/hg" $prefix\src

contrib/packaging/hgpackaging/cli.py

@@ -1,187 +1,194 b''
 # cli.py - Command line interface for automation
 #
 # Copyright 2019 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # no-check-code because Python 3 native.
 
 import argparse
 import os
 import pathlib
 
 from . import (
     inno,
     wix,
 )
 
 HERE = pathlib.Path(os.path.abspath(os.path.dirname(__file__)))
 SOURCE_DIR = HERE.parent.parent.parent
 
 
 def build_inno(pyoxidizer_target=None, python=None, iscc=None, version=None):
     if not pyoxidizer_target and not python:
         raise Exception("--python required unless building with PyOxidizer")
 
     if python and not os.path.isabs(python):
         raise Exception("--python arg must be an absolute path")
 
     if iscc:
         iscc = pathlib.Path(iscc)
     else:
         iscc = (
             pathlib.Path(os.environ["ProgramFiles(x86)"])
             / "Inno Setup 5"
             / "ISCC.exe"
         )
 
     build_dir = SOURCE_DIR / "build"
 
     if pyoxidizer_target:
         inno.build_with_pyoxidizer(
             SOURCE_DIR, build_dir, pyoxidizer_target, iscc, version=version
         )
     else:
         inno.build_with_py2exe(
             SOURCE_DIR,
             build_dir,
             pathlib.Path(python),
             iscc,
             version=version,
         )
 
 
 def build_wix(
     name=None,
     pyoxidizer_target=None,
     python=None,
     version=None,
     sign_sn=None,
     sign_cert=None,
     sign_password=None,
     sign_timestamp_url=None,
     extra_packages_script=None,
     extra_wxs=None,
     extra_features=None,
+    extra_pyoxidizer_vars=None,
 ):
     if not pyoxidizer_target and not python:
         raise Exception("--python required unless building with PyOxidizer")
 
     if python and not os.path.isabs(python):
         raise Exception("--python arg must be an absolute path")
 
     kwargs = {
         "source_dir": SOURCE_DIR,
         "version": version,
     }
 
     if pyoxidizer_target:
         fn = wix.build_installer_pyoxidizer
         kwargs["target_triple"] = pyoxidizer_target
     else:
         fn = wix.build_installer_py2exe
         kwargs["python_exe"] = pathlib.Path(python)
 
     if extra_packages_script:
         if pyoxidizer_target:
             raise Exception(
                 "pyoxidizer does not support --extra-packages-script"
             )
         kwargs["extra_packages_script"] = extra_packages_script
     if extra_wxs:
         kwargs["extra_wxs"] = dict(
             thing.split("=") for thing in extra_wxs.split(",")
         )
     if extra_features:
         kwargs["extra_features"] = extra_features.split(",")
 
     if sign_sn or sign_cert:
         kwargs["signing_info"] = {
             "name": name,
             "subject_name": sign_sn,
             "cert_path": sign_cert,
             "cert_password": sign_password,
             "timestamp_url": sign_timestamp_url,
         }
 
-    fn(**kwargs)
+    fn(**kwargs, extra_pyoxidizer_vars=extra_pyoxidizer_vars)
 
 
 def get_parser():
     parser = argparse.ArgumentParser()
 
     subparsers = parser.add_subparsers()
 
     sp = subparsers.add_parser("inno", help="Build Inno Setup installer")
     sp.add_argument(
         "--pyoxidizer-target",
         choices={"i686-pc-windows-msvc", "x86_64-pc-windows-msvc"},
         help="Build with PyOxidizer targeting this host triple",
     )
     sp.add_argument("--python", help="path to python.exe to use")
     sp.add_argument("--iscc", help="path to iscc.exe to use")
     sp.add_argument(
         "--version",
         help="Mercurial version string to use "
         "(detected from __version__.py if not defined",
     )
     sp.set_defaults(func=build_inno)
 
     sp = subparsers.add_parser(
         "wix", help="Build Windows installer with WiX Toolset"
     )
     sp.add_argument("--name", help="Application name", default="Mercurial")
     sp.add_argument(
         "--pyoxidizer-target",
         choices={"i686-pc-windows-msvc", "x86_64-pc-windows-msvc"},
         help="Build with PyOxidizer targeting this host triple",
     )
     sp.add_argument("--python", help="Path to Python executable to use")
     sp.add_argument(
         "--sign-sn",
         help="Subject name (or fragment thereof) of certificate "
         "to use for signing",
     )
     sp.add_argument(
         "--sign-cert", help="Path to certificate to use for signing"
     )
     sp.add_argument("--sign-password", help="Password for signing certificate")
     sp.add_argument(
         "--sign-timestamp-url",
         help="URL of timestamp server to use for signing",
     )
     sp.add_argument("--version", help="Version string to use")
     sp.add_argument(
         "--extra-packages-script",
         help=(
             "Script to execute to include extra packages in " "py2exe binary."
         ),
     )
     sp.add_argument(
         "--extra-wxs", help="CSV of path_to_wxs_file=working_dir_for_wxs_file"
     )
     sp.add_argument(
         "--extra-features",
         help=(
             "CSV of extra feature names to include "
             "in the installer from the extra wxs files"
         ),
     )
+
+    sp.add_argument(
+        "--extra-pyoxidizer-vars",
+        help="json map of extra variables to pass to pyoxidizer",
+    )
+
     sp.set_defaults(func=build_wix)
 
     return parser
 
 
 def main():
     parser = get_parser()
     args = parser.parse_args()
 
     if not hasattr(args, "func"):
         parser.print_help()
         return
 
     kwargs = dict(vars(args))
     del kwargs["func"]
 
     args.func(**kwargs)

contrib/packaging/hgpackaging/inno.py

@@ -1,242 +1,244 b''
 # inno.py - Inno Setup functionality.
 #
 # Copyright 2019 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # no-check-code because Python 3 native.
 
 import os
 import pathlib
 import shutil
 import subprocess
 
 import jinja2
 
 from .py2exe import (
     build_py2exe,
     stage_install,
 )
-from .pyoxidizer import run_pyoxidizer
+from .pyoxidizer import create_pyoxidizer_install_layout
 from .util import (
     find_legacy_vc_runtime_files,
     normalize_windows_version,
     process_install_rules,
     read_version_py,
 )
 
 EXTRA_PACKAGES = {
     'dulwich',
     'keyring',
     'pygments',
     'win32ctypes',
 }
 
 EXTRA_INCLUDES = {
     '_curses',
     '_curses_panel',
 }
 
 EXTRA_INSTALL_RULES = [
     ('contrib/win32/mercurial.ini', 'defaultrc/mercurial.rc'),
 ]
 
 PACKAGE_FILES_METADATA = {
     'ReadMe.html': 'Flags: isreadme',
 }
 
 
 def build_with_py2exe(
     source_dir: pathlib.Path,
     build_dir: pathlib.Path,
     python_exe: pathlib.Path,
     iscc_exe: pathlib.Path,
     version=None,
 ):
     """Build the Inno installer using py2exe.
 
     Build files will be placed in ``build_dir``.
 
     py2exe's setup.py doesn't use setuptools. It doesn't have modern logic
     for finding the Python 2.7 toolchain. So, we require the environment
     to already be configured with an active toolchain.
     """
     if not iscc_exe.exists():
         raise Exception('%s does not exist' % iscc_exe)
 
     vc_x64 = r'\x64' in os.environ.get('LIB', '')
     arch = 'x64' if vc_x64 else 'x86'
     inno_build_dir = build_dir / ('inno-py2exe-%s' % arch)
     staging_dir = inno_build_dir / 'stage'
 
     requirements_txt = (
         source_dir / 'contrib' / 'packaging' / 'requirements-windows-py2.txt'
     )
 
     inno_build_dir.mkdir(parents=True, exist_ok=True)
 
     build_py2exe(
         source_dir,
         build_dir,
         python_exe,
         'inno',
         requirements_txt,
         extra_packages=EXTRA_PACKAGES,
         extra_includes=EXTRA_INCLUDES,
     )
 
     # Purge the staging directory for every build so packaging is
     # pristine.
     if staging_dir.exists():
         print('purging %s' % staging_dir)
         shutil.rmtree(staging_dir)
 
     # Now assemble all the packaged files into the staging directory.
     stage_install(source_dir, staging_dir)
 
     # We also install some extra files.
     process_install_rules(EXTRA_INSTALL_RULES, source_dir, staging_dir)
 
     # hg.exe depends on VC9 runtime DLLs. Copy those into place.
     for f in find_legacy_vc_runtime_files(vc_x64):
         if f.name.endswith('.manifest'):
             basename = 'Microsoft.VC90.CRT.manifest'
         else:
             basename = f.name
 
         dest_path = staging_dir / basename
 
         print('copying %s to %s' % (f, dest_path))
         shutil.copyfile(f, dest_path)
 
     build_installer(
         source_dir,
         inno_build_dir,
         staging_dir,
         iscc_exe,
         version,
         arch="x64" if vc_x64 else None,
         suffix="-python2",
     )
 
 
 def build_with_pyoxidizer(
     source_dir: pathlib.Path,
     build_dir: pathlib.Path,
     target_triple: str,
     iscc_exe: pathlib.Path,
     version=None,
 ):
     """Build the Inno installer using PyOxidizer."""
     if not iscc_exe.exists():
         raise Exception("%s does not exist" % iscc_exe)
 
     inno_build_dir = build_dir / ("inno-pyoxidizer-%s" % target_triple)
     staging_dir = inno_build_dir / "stage"
 
     inno_build_dir.mkdir(parents=True, exist_ok=True)
-    run_pyoxidizer(source_dir, inno_build_dir, staging_dir, target_triple)
+    create_pyoxidizer_install_layout(
+        source_dir, inno_build_dir, staging_dir, target_triple
+    )
 
     process_install_rules(EXTRA_INSTALL_RULES, source_dir, staging_dir)
 
     build_installer(
         source_dir,
         inno_build_dir,
         staging_dir,
         iscc_exe,
         version,
         arch="x64" if "x86_64" in target_triple else None,
     )
 
 
 def build_installer(
     source_dir: pathlib.Path,
     inno_build_dir: pathlib.Path,
     staging_dir: pathlib.Path,
     iscc_exe: pathlib.Path,
     version,
     arch=None,
     suffix="",
 ):
     """Build an Inno installer from staged Mercurial files.
 
     This function is agnostic about how to build Mercurial. It just
     cares that Mercurial files are in ``staging_dir``.
     """
     inno_source_dir = source_dir / "contrib" / "packaging" / "inno"
 
     # The final package layout is simply a mirror of the staging directory.
     package_files = []
     for root, dirs, files in os.walk(staging_dir):
         dirs.sort()
 
         root = pathlib.Path(root)
 
         for f in sorted(files):
             full = root / f
             rel = full.relative_to(staging_dir)
             if str(rel.parent) == '.':
                 dest_dir = '{app}'
             else:
                 dest_dir = '{app}\\%s' % rel.parent
 
             package_files.append(
                 {
                     'source': rel,
                     'dest_dir': dest_dir,
                     'metadata': PACKAGE_FILES_METADATA.get(str(rel), None),
                 }
             )
 
     print('creating installer')
 
     # Install Inno files by rendering a template.
     jinja_env = jinja2.Environment(
         loader=jinja2.FileSystemLoader(str(inno_source_dir)),
         # Need to change these to prevent conflict with Inno Setup.
         comment_start_string='{##',
         comment_end_string='##}',
     )
 
     try:
         template = jinja_env.get_template('mercurial.iss')
     except jinja2.TemplateSyntaxError as e:
         raise Exception(
             'template syntax error at %s:%d: %s'
             % (
                 e.name,
                 e.lineno,
                 e.message,
             )
         )
 
     content = template.render(package_files=package_files)
 
     with (inno_build_dir / 'mercurial.iss').open('w', encoding='utf-8') as fh:
         fh.write(content)
 
     # Copy additional files used by Inno.
     for p in ('mercurial.ico', 'postinstall.txt'):
         shutil.copyfile(
             source_dir / 'contrib' / 'win32' / p, inno_build_dir / p
         )
 
     args = [str(iscc_exe)]
 
     if arch:
         args.append('/dARCH=%s' % arch)
         args.append('/dSUFFIX=-%s%s' % (arch, suffix))
     else:
         args.append('/dSUFFIX=-x86%s' % suffix)
 
     if not version:
         version = read_version_py(source_dir)
 
     args.append('/dVERSION=%s' % version)
     args.append('/dQUAD_VERSION=%s' % normalize_windows_version(version))
 
     args.append('/Odist')
     args.append(str(inno_build_dir / 'mercurial.iss'))
 
     subprocess.run(args, cwd=str(source_dir), check=True)

contrib/packaging/hgpackaging/pyoxidizer.py

@@ -1,144 +1,180 b''
 # pyoxidizer.py - Packaging support for PyOxidizer
 #
 # Copyright 2020 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # no-check-code because Python 3 native.
 
 import os
 import pathlib
 import shutil
 import subprocess
 import sys
+import typing
 
 from .downloads import download_entry
 from .util import (
     extract_zip_to_directory,
     process_install_rules,
     find_vc_runtime_dll,
 )
 
 
 STAGING_RULES_WINDOWS = [
     ('contrib/bash_completion', 'contrib/'),
     ('contrib/hgk', 'contrib/hgk.tcl'),
     ('contrib/hgweb.fcgi', 'contrib/'),
     ('contrib/hgweb.wsgi', 'contrib/'),
     ('contrib/logo-droplets.svg', 'contrib/'),
     ('contrib/mercurial.el', 'contrib/'),
     ('contrib/mq.el', 'contrib/'),
     ('contrib/tcsh_completion', 'contrib/'),
     ('contrib/tcsh_completion_build.sh', 'contrib/'),
     ('contrib/vim/*', 'contrib/vim/'),
     ('contrib/win32/postinstall.txt', 'ReleaseNotes.txt'),
     ('contrib/win32/ReadMe.html', 'ReadMe.html'),
     ('contrib/xml.rnc', 'contrib/'),
     ('contrib/zsh_completion', 'contrib/'),
     ('doc/*.html', 'doc/'),
     ('doc/style.css', 'doc/'),
     ('COPYING', 'Copying.txt'),
 ]
 
 STAGING_RULES_APP = [
     ('lib/mercurial/helptext/**/*.txt', 'helptext/'),
     ('lib/mercurial/defaultrc/*.rc', 'defaultrc/'),
     ('lib/mercurial/locale/**/*', 'locale/'),
     ('lib/mercurial/templates/**/*', 'templates/'),
 ]
 
 STAGING_EXCLUDES_WINDOWS = [
     "doc/hg-ssh.8.html",
 ]
 
 
+def build_docs_html(source_dir: pathlib.Path):
+    """Ensures HTML documentation is built.
+
+    This will fail if docutils isn't available.
+
+    (The HTML docs aren't built as part of `pip install` so we need to build them
+    out of band.)
+    """
+    subprocess.run(
+        [sys.executable, str(source_dir / "setup.py"), "build_doc", "--html"],
+        cwd=str(source_dir),
+        check=True,
+    )
+
+
 def run_pyoxidizer(
     source_dir: pathlib.Path,
     build_dir: pathlib.Path,
-    out_dir: pathlib.Path,
     target_triple: str,
-):
-    """Build Mercurial with PyOxidizer and copy additional files into place.
+    build_vars: typing.Optional[typing.Dict[str, str]] = None,
+    target: typing.Optional[str] = None,
+) -> pathlib.Path:
+    """Run `pyoxidizer` in an environment with access to build dependencies.
 
-    After successful completion, ``out_dir`` contains files constituting a
-    Mercurial install.
+    Returns the output directory that pyoxidizer would have used for build
+    artifacts. Actual build artifacts are likely in a sub-directory with the
+    name of the pyoxidizer build target that was built.
     """
+    build_vars = build_vars or {}
+
     # We need to make gettext binaries available for compiling i18n files.
     gettext_pkg, gettext_entry = download_entry('gettext', build_dir)
     gettext_dep_pkg = download_entry('gettext-dep', build_dir)[0]
 
     gettext_root = build_dir / ('gettext-win-%s' % gettext_entry['version'])
 
     if not gettext_root.exists():
         extract_zip_to_directory(gettext_pkg, gettext_root)
         extract_zip_to_directory(gettext_dep_pkg, gettext_root)
 
     env = dict(os.environ)
     env["PATH"] = "%s%s%s" % (
         env["PATH"],
         os.pathsep,
         str(gettext_root / "bin"),
     )
 
     args = [
         "pyoxidizer",
         "build",
         "--path",
         str(source_dir / "rust" / "hgcli"),
         "--release",
         "--target-triple",
         target_triple,
     ]
 
+    for k, v in sorted(build_vars.items()):
+        args.extend(["--var", k, v])
+
+    if target:
+        args.append(target)
+
     subprocess.run(args, env=env, check=True)
 
+    return source_dir / "build" / "pyoxidizer" / target_triple / "release"
+
+
+def create_pyoxidizer_install_layout(
+    source_dir: pathlib.Path,
+    build_dir: pathlib.Path,
+    out_dir: pathlib.Path,
+    target_triple: str,
+):
+    """Build Mercurial with PyOxidizer and copy additional files into place.
+
+    After successful completion, ``out_dir`` contains files constituting a
+    Mercurial install.
+    """
+
+    run_pyoxidizer(source_dir, build_dir, target_triple)
+
     if "windows" in target_triple:
         target = "app_windows"
     else:
         target = "app_posix"
 
     build_dir = (
         source_dir / "build" / "pyoxidizer" / target_triple / "release" / target
     )
 
     if out_dir.exists():
         print("purging %s" % out_dir)
         shutil.rmtree(out_dir)
 
     # Now assemble all the files from PyOxidizer into the staging directory.
     shutil.copytree(build_dir, out_dir)
 
     # Move some of those files around. We can get rid of this once Mercurial
     # is taught to use the importlib APIs for reading resources.
     process_install_rules(STAGING_RULES_APP, build_dir, out_dir)
 
-    # We also need to run setup.py build_doc to produce html files,
-    # as they aren't built as part of ``pip install``.
-    # This will fail if docutils isn't installed.
-    subprocess.run(
-        [sys.executable, str(source_dir / "setup.py"), "build_doc", "--html"],
-        cwd=str(source_dir),
-        check=True,
-    )
+    build_docs_html(source_dir)
 
     if "windows" in target_triple:
         process_install_rules(STAGING_RULES_WINDOWS, source_dir, out_dir)
 
         # Write out a default editor.rc file to configure notepad as the
         # default editor.
         os.makedirs(out_dir / "defaultrc", exist_ok=True)
         with (out_dir / "defaultrc" / "editor.rc").open(
             "w", encoding="utf-8"
         ) as fh:
             fh.write("[ui]\neditor = notepad\n")
 
         for f in STAGING_EXCLUDES_WINDOWS:
             p = out_dir / f
             if p.exists():
                 print("removing %s" % p)
                 p.unlink()
 
         # Add vcruntimeXXX.dll next to executable.
         vc_runtime_dll = find_vc_runtime_dll(x64="x86_64" in target_triple)
         shutil.copy(vc_runtime_dll, out_dir / vc_runtime_dll.name)

contrib/packaging/hgpackaging/wix.py

@@ -1,547 +1,586 b''
 # wix.py - WiX installer functionality
 #
 # Copyright 2019 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # no-check-code because Python 3 native.
 
 import collections
+import json
 import os
 import pathlib
 import re
 import shutil
 import subprocess
 import typing
 import uuid
 import xml.dom.minidom
 
 from .downloads import download_entry
 from .py2exe import (
     build_py2exe,
     stage_install,
 )
-from .pyoxidizer import run_pyoxidizer
+from .pyoxidizer import (
+    build_docs_html,
+    create_pyoxidizer_install_layout,
+    run_pyoxidizer,
+)
 from .util import (
     extract_zip_to_directory,
     normalize_windows_version,
     process_install_rules,
     sign_with_signtool,
 )
 
 
 EXTRA_PACKAGES = {
     'dulwich',
     'distutils',
     'keyring',
     'pygments',
     'win32ctypes',
 }
 
 EXTRA_INCLUDES = {
     '_curses',
     '_curses_panel',
 }
 
 EXTRA_INSTALL_RULES = [
     ('contrib/packaging/wix/COPYING.rtf', 'COPYING.rtf'),
     ('contrib/win32/mercurial.ini', 'defaultrc/mercurial.rc'),
 ]
 
 STAGING_REMOVE_FILES = [
     # We use the RTF variant.
     'copying.txt',
 ]
 
 SHORTCUTS = {
     # hg.1.html'
     'hg.file.5d3e441c_28d9_5542_afd0_cdd4234f12d5': {
         'Name': 'Mercurial Command Reference',
     },
     # hgignore.5.html
     'hg.file.5757d8e0_f207_5e10_a2ec_3ba0a062f431': {
         'Name': 'Mercurial Ignore Files',
     },
     # hgrc.5.html
     'hg.file.92e605fd_1d1a_5dc6_9fc0_5d2998eb8f5e': {
         'Name': 'Mercurial Configuration Files',
     },
 }
 
 
 def find_version(source_dir: pathlib.Path):
     version_py = source_dir / 'mercurial' / '__version__.py'
 
     with version_py.open('r', encoding='utf-8') as fh:
         source = fh.read().strip()
 
     m = re.search('version = b"(.*)"', source)
     return m.group(1)
 
 
 def ensure_vc90_merge_modules(build_dir):
     x86 = (
         download_entry(
             'vc9-crt-x86-msm',
             build_dir,
             local_name='microsoft.vcxx.crt.x86_msm.msm',
         )[0],
         download_entry(
             'vc9-crt-x86-msm-policy',
             build_dir,
             local_name='policy.x.xx.microsoft.vcxx.crt.x86_msm.msm',
         )[0],
     )
 
     x64 = (
         download_entry(
             'vc9-crt-x64-msm',
             build_dir,
             local_name='microsoft.vcxx.crt.x64_msm.msm',
         )[0],
         download_entry(
             'vc9-crt-x64-msm-policy',
             build_dir,
             local_name='policy.x.xx.microsoft.vcxx.crt.x64_msm.msm',
         )[0],
     )
     return {
         'x86': x86,
         'x64': x64,
     }
 
 
 def run_candle(wix, cwd, wxs, source_dir, defines=None):
     args = [
         str(wix / 'candle.exe'),
         '-nologo',
         str(wxs),
         '-dSourceDir=%s' % source_dir,
     ]
 
     if defines:
         args.extend('-d%s=%s' % define for define in sorted(defines.items()))
 
     subprocess.run(args, cwd=str(cwd), check=True)
 
 
 def make_files_xml(staging_dir: pathlib.Path, is_x64) -> str:
     """Create XML string listing every file to be installed."""
 
     # We derive GUIDs from a deterministic file path identifier.
     # We shoehorn the name into something that looks like a URL because
     # the UUID namespaces are supposed to work that way (even though
     # the input data probably is never validated).
 
     doc = xml.dom.minidom.parseString(
         '<?xml version="1.0" encoding="utf-8"?>'
         '<Wix xmlns="http://schemas.microsoft.com/wix/2006/wi">'
         '</Wix>'
     )
 
     # Assemble the install layout by directory. This makes it easier to
     # emit XML, since each directory has separate entities.
     manifest = collections.defaultdict(dict)
 
     for root, dirs, files in os.walk(staging_dir):
         dirs.sort()
 
         root = pathlib.Path(root)
         rel_dir = root.relative_to(staging_dir)
 
         for i in range(len(rel_dir.parts)):
             parent = '/'.join(rel_dir.parts[0 : i + 1])
             manifest.setdefault(parent, {})
 
         for f in sorted(files):
             full = root / f
             manifest[str(rel_dir).replace('\\', '/')][full.name] = full
 
     component_groups = collections.defaultdict(list)
 
     # Now emit a <Fragment> for each directory.
     # Each directory is composed of a <DirectoryRef> pointing to its parent
     # and defines child <Directory>'s and a <Component> with all the files.
     for dir_name, entries in sorted(manifest.items()):
         # The directory id is derived from the path. But the root directory
         # is special.
         if dir_name == '.':
             parent_directory_id = 'INSTALLDIR'
         else:
             parent_directory_id = 'hg.dir.%s' % dir_name.replace(
                 '/', '.'
             ).replace('-', '_')
 
         fragment = doc.createElement('Fragment')
         directory_ref = doc.createElement('DirectoryRef')
         directory_ref.setAttribute('Id', parent_directory_id)
 
         # Add <Directory> entries for immediate children directories.
         for possible_child in sorted(manifest.keys()):
             if (
                 dir_name == '.'
                 and '/' not in possible_child
                 and possible_child != '.'
             ):
                 child_directory_id = ('hg.dir.%s' % possible_child).replace(
                     '-', '_'
                 )
                 name = possible_child
             else:
                 if not possible_child.startswith('%s/' % dir_name):
                     continue
                 name = possible_child[len(dir_name) + 1 :]
                 if '/' in name:
                     continue
 
                 child_directory_id = 'hg.dir.%s' % possible_child.replace(
                     '/', '.'
                 ).replace('-', '_')
 
             directory = doc.createElement('Directory')
             directory.setAttribute('Id', child_directory_id)
             directory.setAttribute('Name', name)
             directory_ref.appendChild(directory)
 
         # Add <Component>s for files in this directory.
         for rel, source_path in sorted(entries.items()):
             if dir_name == '.':
                 full_rel = rel
             else:
                 full_rel = '%s/%s' % (dir_name, rel)
 
             component_unique_id = (
                 'https://www.mercurial-scm.org/wix-installer/0/component/%s'
                 % full_rel
             )
             component_guid = uuid.uuid5(uuid.NAMESPACE_URL, component_unique_id)
             component_id = 'hg.component.%s' % str(component_guid).replace(
                 '-', '_'
             )
 
             component = doc.createElement('Component')
 
             component.setAttribute('Id', component_id)
             component.setAttribute('Guid', str(component_guid).upper())
             component.setAttribute('Win64', 'yes' if is_x64 else 'no')
 
             # Assign this component to a top-level group.
             if dir_name == '.':
                 component_groups['ROOT'].append(component_id)
             elif '/' in dir_name:
                 component_groups[dir_name[0 : dir_name.index('/')]].append(
                     component_id
                 )
             else:
                 component_groups[dir_name].append(component_id)
 
             unique_id = (
                 'https://www.mercurial-scm.org/wix-installer/0/%s' % full_rel
             )
             file_guid = uuid.uuid5(uuid.NAMESPACE_URL, unique_id)
 
             # IDs have length limits. So use GUID to derive them.
             file_guid_normalized = str(file_guid).replace('-', '_')
             file_id = 'hg.file.%s' % file_guid_normalized
 
             file_element = doc.createElement('File')
             file_element.setAttribute('Id', file_id)
             file_element.setAttribute('Source', str(source_path))
             file_element.setAttribute('KeyPath', 'yes')
             file_element.setAttribute('ReadOnly', 'yes')
 
             component.appendChild(file_element)
             directory_ref.appendChild(component)
 
         fragment.appendChild(directory_ref)
         doc.documentElement.appendChild(fragment)
 
     for group, component_ids in sorted(component_groups.items()):
         fragment = doc.createElement('Fragment')
         component_group = doc.createElement('ComponentGroup')
         component_group.setAttribute('Id', 'hg.group.%s' % group)
 
         for component_id in component_ids:
             component_ref = doc.createElement('ComponentRef')
             component_ref.setAttribute('Id', component_id)
             component_group.appendChild(component_ref)
 
         fragment.appendChild(component_group)
         doc.documentElement.appendChild(fragment)
 
     # Add <Shortcut> to files that have it defined.
     for file_id, metadata in sorted(SHORTCUTS.items()):
         els = doc.getElementsByTagName('File')
         els = [el for el in els if el.getAttribute('Id') == file_id]
 
         if not els:
             raise Exception('could not find File[Id=%s]' % file_id)
 
         for el in els:
             shortcut = doc.createElement('Shortcut')
             shortcut.setAttribute('Id', 'hg.shortcut.%s' % file_id)
             shortcut.setAttribute('Directory', 'ProgramMenuDir')
             shortcut.setAttribute('Icon', 'hgIcon.ico')
             shortcut.setAttribute('IconIndex', '0')
             shortcut.setAttribute('Advertise', 'yes')
             for k, v in sorted(metadata.items()):
                 shortcut.setAttribute(k, v)
 
             el.appendChild(shortcut)
 
     return doc.toprettyxml()
 
 
 def build_installer_py2exe(
     source_dir: pathlib.Path,
     python_exe: pathlib.Path,
     msi_name='mercurial',
     version=None,
     extra_packages_script=None,
     extra_wxs: typing.Optional[typing.Dict[str, str]] = None,
     extra_features: typing.Optional[typing.List[str]] = None,
     signing_info: typing.Optional[typing.Dict[str, str]] = None,
 ):
     """Build a WiX MSI installer using py2exe.
 
     ``source_dir`` is the path to the Mercurial source tree to use.
     ``arch`` is the target architecture. either ``x86`` or ``x64``.
     ``python_exe`` is the path to the Python executable to use/bundle.
     ``version`` is the Mercurial version string. If not defined,
     ``mercurial/__version__.py`` will be consulted.
     ``extra_packages_script`` is a command to be run to inject extra packages
     into the py2exe binary. It should stage packages into the virtualenv and
     print a null byte followed by a newline-separated list of packages that
     should be included in the exe.
     ``extra_wxs`` is a dict of {wxs_name: working_dir_for_wxs_build}.
     ``extra_features`` is a list of additional named Features to include in
     the build. These must match Feature names in one of the wxs scripts.
     """
     arch = 'x64' if r'\x64' in os.environ.get('LIB', '') else 'x86'
 
     hg_build_dir = source_dir / 'build'
 
     requirements_txt = (
         source_dir / 'contrib' / 'packaging' / 'requirements-windows-py2.txt'
     )
 
     build_py2exe(
         source_dir,
         hg_build_dir,
         python_exe,
         'wix',
         requirements_txt,
         extra_packages=EXTRA_PACKAGES,
         extra_packages_script=extra_packages_script,
         extra_includes=EXTRA_INCLUDES,
     )
 
     build_dir = hg_build_dir / ('wix-%s' % arch)
     staging_dir = build_dir / 'stage'
 
     build_dir.mkdir(exist_ok=True)
 
     # Purge the staging directory for every build so packaging is pristine.
     if staging_dir.exists():
         print('purging %s' % staging_dir)
         shutil.rmtree(staging_dir)
 
     stage_install(source_dir, staging_dir, lower_case=True)
 
     # We also install some extra files.
     process_install_rules(EXTRA_INSTALL_RULES, source_dir, staging_dir)
 
     # And remove some files we don't want.
     for f in STAGING_REMOVE_FILES:
         p = staging_dir / f
         if p.exists():
             print('removing %s' % p)
             p.unlink()
 
     return run_wix_packaging(
         source_dir,
         build_dir,
         staging_dir,
         arch,
         version=version,
         python2=True,
         msi_name=msi_name,
         suffix="-python2",
         extra_wxs=extra_wxs,
         extra_features=extra_features,
         signing_info=signing_info,
     )
 
 
 def build_installer_pyoxidizer(
     source_dir: pathlib.Path,
     target_triple: str,
     msi_name='mercurial',
     version=None,
     extra_wxs: typing.Optional[typing.Dict[str, str]] = None,
     extra_features: typing.Optional[typing.List[str]] = None,
     signing_info: typing.Optional[typing.Dict[str, str]] = None,
+    extra_pyoxidizer_vars=None,
 ):
     """Build a WiX MSI installer using PyOxidizer."""
     hg_build_dir = source_dir / "build"
     build_dir = hg_build_dir / ("wix-%s" % target_triple)
-    staging_dir = build_dir / "stage"
-
-    arch = "x64" if "x86_64" in target_triple else "x86"
 
     build_dir.mkdir(parents=True, exist_ok=True)
-    run_pyoxidizer(source_dir, build_dir, staging_dir, target_triple)
+
+    # Need to ensure docs HTML is built because this isn't done as part of
+    # `pip install Mercurial`.
+    build_docs_html(source_dir)
+
+    build_vars = {}
 
-    # We also install some extra files.
-    process_install_rules(EXTRA_INSTALL_RULES, source_dir, staging_dir)
+    if msi_name:
+        build_vars["MSI_NAME"] = msi_name
+
+    if version:
+        build_vars["VERSION"] = version
+
+    if extra_features:
+        build_vars["EXTRA_MSI_FEATURES"] = ";".join(extra_features)
 
-    # And remove some files we don't want.
-    for f in STAGING_REMOVE_FILES:
-        p = staging_dir / f
-        if p.exists():
-            print('removing %s' % p)
-            p.unlink()
+    if signing_info:
+        if signing_info["cert_path"]:
+            build_vars["SIGNING_PFX_PATH"] = signing_info["cert_path"]
+        if signing_info["cert_password"]:
+            build_vars["SIGNING_PFX_PASSWORD"] = signing_info["cert_password"]
+        if signing_info["subject_name"]:
+            build_vars["SIGNING_SUBJECT_NAME"] = signing_info["subject_name"]
+        if signing_info["timestamp_url"]:
+            build_vars["TIME_STAMP_SERVER_URL"] = signing_info["timestamp_url"]
 
-    return run_wix_packaging(
+    if extra_pyoxidizer_vars:
+        build_vars.update(json.loads(extra_pyoxidizer_vars))
+
+    if extra_wxs:
+        raise Exception(
+            "support for extra .wxs files has been temporarily dropped"
+        )
+
+    out_dir = run_pyoxidizer(
         source_dir,
         build_dir,
-        staging_dir,
-        arch,
-        version,
-        python2=False,
-        msi_name=msi_name,
-        extra_wxs=extra_wxs,
-        extra_features=extra_features,
-        signing_info=signing_info,
+        target_triple,
+        build_vars=build_vars,
+        target="msi",
     )
 
+    msi_dir = out_dir / "msi"
+    msi_files = [f for f in os.listdir(msi_dir) if f.endswith(".msi")]
+
+    if len(msi_files) != 1:
+        raise Exception("expected exactly 1 .msi file; got %d" % len(msi_files))
+
+    msi_filename = msi_files[0]
+
+    msi_path = msi_dir / msi_filename
+    dist_path = source_dir / "dist" / msi_filename
+
+    dist_path.parent.mkdir(parents=True, exist_ok=True)
+
+    shutil.copyfile(msi_path, dist_path)
+
+    return {
+        "msi_path": dist_path,
+    }
+
 
 def run_wix_packaging(
     source_dir: pathlib.Path,
     build_dir: pathlib.Path,
     staging_dir: pathlib.Path,
     arch: str,
     version: str,
     python2: bool,
     msi_name: typing.Optional[str] = "mercurial",
     suffix: str = "",
     extra_wxs: typing.Optional[typing.Dict[str, str]] = None,
     extra_features: typing.Optional[typing.List[str]] = None,
     signing_info: typing.Optional[typing.Dict[str, str]] = None,
 ):
     """Invokes WiX to package up a built Mercurial.
 
     ``signing_info`` is a dict defining properties to facilitate signing the
     installer. Recognized keys include ``name``, ``subject_name``,
     ``cert_path``, ``cert_password``, and ``timestamp_url``. If populated,
     we will sign both the hg.exe and the .msi using the signing credentials
     specified.
     """
 
     orig_version = version or find_version(source_dir)
     version = normalize_windows_version(orig_version)
     print('using version string: %s' % version)
     if version != orig_version:
         print('(normalized from: %s)' % orig_version)
 
     if signing_info:
         sign_with_signtool(
             staging_dir / "hg.exe",
             "%s %s" % (signing_info["name"], version),
             subject_name=signing_info["subject_name"],
             cert_path=signing_info["cert_path"],
             cert_password=signing_info["cert_password"],
             timestamp_url=signing_info["timestamp_url"],
         )
 
     wix_dir = source_dir / 'contrib' / 'packaging' / 'wix'
 
     wix_pkg, wix_entry = download_entry('wix', build_dir)
     wix_path = build_dir / ('wix-%s' % wix_entry['version'])
 
     if not wix_path.exists():
         extract_zip_to_directory(wix_pkg, wix_path)
 
     if python2:
         ensure_vc90_merge_modules(build_dir)
 
     source_build_rel = pathlib.Path(os.path.relpath(source_dir, build_dir))
 
     defines = {'Platform': arch}
 
     # Derive a .wxs file with the staged files.
     manifest_wxs = build_dir / 'stage.wxs'
     with manifest_wxs.open('w', encoding='utf-8') as fh:
         fh.write(make_files_xml(staging_dir, is_x64=arch == 'x64'))
 
     run_candle(wix_path, build_dir, manifest_wxs, staging_dir, defines=defines)
 
     for source, rel_path in sorted((extra_wxs or {}).items()):
         run_candle(wix_path, build_dir, source, rel_path, defines=defines)
 
     source = wix_dir / 'mercurial.wxs'
     defines['Version'] = version
     defines['Comments'] = 'Installs Mercurial version %s' % version
 
     if python2:
         defines["PythonVersion"] = "2"
         defines['VCRedistSrcDir'] = str(build_dir)
     else:
         defines["PythonVersion"] = "3"
 
     if (staging_dir / "lib").exists():
         defines["MercurialHasLib"] = "1"
 
     if extra_features:
         assert all(';' not in f for f in extra_features)
         defines['MercurialExtraFeatures'] = ';'.join(extra_features)
 
     run_candle(wix_path, build_dir, source, source_build_rel, defines=defines)
 
     msi_path = (
         source_dir
         / 'dist'
         / ('%s-%s-%s%s.msi' % (msi_name, orig_version, arch, suffix))
     )
 
     args = [
         str(wix_path / 'light.exe'),
         '-nologo',
         '-ext',
         'WixUIExtension',
         '-sw1076',
         '-spdb',
         '-o',
         str(msi_path),
     ]
 
     for source, rel_path in sorted((extra_wxs or {}).items()):
         assert source.endswith('.wxs')
         source = os.path.basename(source)
         args.append(str(build_dir / ('%s.wixobj' % source[:-4])))
 
     args.extend(
         [
             str(build_dir / 'stage.wixobj'),
             str(build_dir / 'mercurial.wixobj'),
         ]
     )
 
     subprocess.run(args, cwd=str(source_dir), check=True)
 
     print('%s created' % msi_path)
 
     if signing_info:
         sign_with_signtool(
             msi_path,
             "%s %s" % (signing_info["name"], version),
             subject_name=signing_info["subject_name"],
             cert_path=signing_info["cert_path"],
             cert_password=signing_info["cert_password"],
             timestamp_url=signing_info["timestamp_url"],
         )
 
     return {
         'msi_path': msi_path,
     }

contrib/packaging/wix/mercurial.wxs

@@ -1,153 +1,157 b''
 <?xml version='1.0' encoding='windows-1252'?>
 <Wix xmlns='http://schemas.microsoft.com/wix/2006/wi'>
 
   <!-- Copyright 2010 Steve Borho <[e-mail]>
 
   This software may be used and distributed according to the terms of the
   GNU General Public License version 2 or any later version. -->
 
   <?include guids.wxi ?>
   <?include defines.wxi ?>
 
   <?if $(var.Platform) = "x64" ?>
     <?define PFolder = ProgramFiles64Folder ?>
   <?else?>
     <?define PFolder = ProgramFilesFolder ?>
   <?endif?>
 
   <Product Id='*'
     Name='Mercurial $(var.Version) ($(var.Platform))'
     UpgradeCode='$(var.ProductUpgradeCode)'
     Language='1033' Codepage='1252' Version='$(var.Version)'
     Manufacturer='Olivia Mackall and others'>
 
     <Package Id='*'
       Keywords='Installer'
       Description="Mercurial distributed SCM (version $(var.Version))"
       Comments='$(var.Comments)'
       Platform='$(var.Platform)'
       Manufacturer='Olivia Mackall and others'
       InstallerVersion='300' Languages='1033' Compressed='yes' SummaryCodepage='1252' />
 
     <Media Id='1' Cabinet='mercurial.cab' EmbedCab='yes' DiskPrompt='CD-ROM #1'
            CompressionLevel='high' />
     <Property Id='DiskPrompt' Value="Mercurial $(var.Version) Installation [1]" />
 
     <Condition Message='Mercurial MSI installers require Windows XP or higher'>
         VersionNT >= 501
     </Condition>
 
     <Property Id="INSTALLDIR">
       <ComponentSearch Id='SearchForMainExecutableComponent'
                        Guid='$(var.ComponentMainExecutableGUID)'
                        Type='directory' />
     </Property>
 
     <!--Property Id='ARPCOMMENTS'>any comments</Property-->
     <Property Id='ARPCONTACT'>[e-mail]</Property>
     <Property Id='ARPHELPLINK'>https://mercurial-scm.org/wiki/</Property>
     <Property Id='ARPURLINFOABOUT'>https://mercurial-scm.org/about/</Property>
     <Property Id='ARPURLUPDATEINFO'>https://mercurial-scm.org/downloads/</Property>
     <Property Id='ARPHELPTELEPHONE'>https://mercurial-scm.org/wiki/Support</Property>
     <Property Id='ARPPRODUCTICON'>hgIcon.ico</Property>
 
     <Property Id='INSTALLEDMERCURIALPRODUCTS' Secure='yes'></Property>
     <Property Id='REINSTALLMODE'>amus</Property>
 
     <!--Auto-accept the license page-->
     <Property Id='LicenseAccepted'>1</Property>
 
     <Directory Id='TARGETDIR' Name='SourceDir'>
       <Directory Id='$(var.PFolder)' Name='PFiles'>
         <Directory Id='INSTALLDIR' Name='Mercurial'>
           <Component Id='MainExecutable' Guid='$(var.ComponentMainExecutableGUID)' Win64='$(var.IsX64)'>
             <CreateFolder />
             <Environment Id="Environment" Name="PATH" Part="last" System="yes"
                          Permanent="no" Value="[INSTALLDIR]" Action="set" />
           </Component>
         </Directory>
       </Directory>
 
       <Directory Id="ProgramMenuFolder" Name="Programs">
         <Directory Id="ProgramMenuDir" Name="Mercurial $(var.Version)">
           <Component Id="ProgramMenuDir" Guid="$(var.ProgramMenuDir.guid)" Win64='$(var.IsX64)'>
             <RemoveFolder Id='ProgramMenuDir' On='uninstall' />
             <RegistryValue Root='HKCU' Key='Software\Mercurial\InstallDir' Type='string'
                            Value='[INSTALLDIR]' KeyPath='yes' />
             <Shortcut Id='UrlShortcut' Directory='ProgramMenuDir' Name='Mercurial Web Site'
                       Target='[ARPHELPLINK]' Icon="hgIcon.ico" IconIndex='0' />
           </Component>
         </Directory>
       </Directory>
 
       <!-- Install VCRedist merge modules on Python 2. On Python 3,
            vcruntimeXXX.dll is part of the install layout and gets picked up
            as a regular file. -->
       <?if $(var.PythonVersion) = "2" ?>
         <?if $(var.Platform) = "x86" ?>
           <Merge Id='VCRuntime' DiskId='1' Language='1033'
                 SourceFile='$(var.VCRedistSrcDir)\microsoft.vcxx.crt.x86_msm.msm' />
           <Merge Id='VCRuntimePolicy' DiskId='1' Language='1033'
                 SourceFile='$(var.VCRedistSrcDir)\policy.x.xx.microsoft.vcxx.crt.x86_msm.msm' />
         <?else?>
           <Merge Id='VCRuntime' DiskId='1' Language='1033'
                 SourceFile='$(var.VCRedistSrcDir)\microsoft.vcxx.crt.x64_msm.msm' />
           <Merge Id='VCRuntimePolicy' DiskId='1' Language='1033'
                 SourceFile='$(var.VCRedistSrcDir)\policy.x.xx.microsoft.vcxx.crt.x64_msm.msm' />
         <?endif?>
       <?endif?>
     </Directory>
 
     <Feature Id='Complete' Title='Mercurial' Description='The complete package'
         Display='expand' Level='1' ConfigurableDirectory='INSTALLDIR' >
       <Feature Id='MainProgram' Title='Program' Description='Mercurial command line app'
              Level='1' Absent='disallow' >
         <ComponentRef Id='MainExecutable' />
         <ComponentRef Id='ProgramMenuDir' />
         <ComponentGroupRef Id="hg.group.ROOT" />
         <ComponentGroupRef Id="hg.group.defaultrc" />
         <ComponentGroupRef Id="hg.group.helptext" />
         <?ifdef MercurialHasLib?>
           <ComponentGroupRef Id="hg.group.lib" />
         <?endif?>
         <ComponentGroupRef Id="hg.group.templates" />
         <?if $(var.PythonVersion) = "2" ?>
           <MergeRef Id='VCRuntime' />
           <MergeRef Id='VCRuntimePolicy' />
         <?endif?>
       </Feature>
       <?ifdef MercurialExtraFeatures?>
         <?foreach EXTRAFEAT in $(var.MercurialExtraFeatures)?>
           <FeatureRef Id="$(var.EXTRAFEAT)" />
         <?endforeach?>
       <?endif?>
       <Feature Id='Locales' Title='Translations' Description='Translations' Level='1'>
         <ComponentGroupRef Id="hg.group.locale" />
       </Feature>
       <Feature Id='Documentation' Title='Documentation' Description='HTML man pages' Level='1'>
         <ComponentGroupRef Id="hg.group.doc" />
       </Feature>
       <Feature Id='Misc' Title='Miscellaneous' Description='Contributed scripts' Level='1'>
         <ComponentGroupRef Id="hg.group.contrib" />
       </Feature>
     </Feature>
 
     <UIRef Id="WixUI_FeatureTree" />
     <UIRef Id="WixUI_ErrorProgressText" />
 
+    <?ifdef PyOxidizer?>
+    <WixVariable Id="WixUILicenseRtf" Value="COPYING.rtf" />
+    <Icon Id="hgIcon.ico" SourceFile="mercurial.ico" />
+    <?else?>
     <WixVariable Id="WixUILicenseRtf" Value="contrib\packaging\wix\COPYING.rtf" />
-
     <Icon Id="hgIcon.ico" SourceFile="contrib/win32/mercurial.ico" />
+    <?endif?>
 
     <Upgrade Id='$(var.ProductUpgradeCode)'>
       <UpgradeVersion
         IncludeMinimum='yes' Minimum='0.0.0' IncludeMaximum='no' OnlyDetect='no'
         Property='INSTALLEDMERCURIALPRODUCTS' />
     </Upgrade>
 
     <InstallExecuteSequence>
       <RemoveExistingProducts After='InstallInitialize'/>
     </InstallExecuteSequence>
 
   </Product>
 </Wix>

contrib/perf.py

@@ -1,3928 +1,3980 b''
 # perf.py - performance test routines
 '''helper extension to measure performance
 
 Configurations
 ==============
 
 ``perf``
 --------
 
 ``all-timing``
     When set, additional statistics will be reported for each benchmark: best,
     worst, median average. If not set only the best timing is reported
     (default: off).
 
 ``presleep``
   number of second to wait before any group of runs (default: 1)
 
 ``pre-run``
   number of run to perform before starting measurement.
 
 ``profile-benchmark``
   Enable profiling for the benchmarked section.
   (The first iteration is benchmarked)
 
 ``run-limits``
   Control the number of runs each benchmark will perform. The option value
   should be a list of `<time>-<numberofrun>` pairs. After each run the
   conditions are considered in order with the following logic:
 
       If benchmark has been running for <time> seconds, and we have performed
       <numberofrun> iterations, stop the benchmark,
 
   The default value is: `3.0-100, 10.0-3`
 
 ``stub``
     When set, benchmarks will only be run once, useful for testing
     (default: off)
 '''
 
 # "historical portability" policy of perf.py:
 #
 # We have to do:
 # - make perf.py "loadable" with as wide Mercurial version as possible
 #   This doesn't mean that perf commands work correctly with that Mercurial.
 #   BTW, perf.py itself has been available since 1.1 (or eb240755386d).
 # - make historical perf command work correctly with as wide Mercurial
 #   version as possible
 #
 # We have to do, if possible with reasonable cost:
 # - make recent perf command for historical feature work correctly
 #   with early Mercurial
 #
 # We don't have to do:
 # - make perf command for recent feature work correctly with early
 #   Mercurial
 
 from __future__ import absolute_import
 import contextlib
 import functools
 import gc
 import os
 import random
 import shutil
 import struct
 import sys
 import tempfile
 import threading
 import time
+
+import mercurial.revlog
 from mercurial import (
     changegroup,
     cmdutil,
     commands,
     copies,
     error,
     extensions,
     hg,
     mdiff,
     merge,
-    revlog,
     util,
 )
 
 # for "historical portability":
 # try to import modules separately (in dict order), and ignore
 # failure, because these aren't available with early Mercurial
 try:
     from mercurial import branchmap  # since 2.5 (or bcee63733aad)
 except ImportError:
     pass
 try:
     from mercurial import obsolete  # since 2.3 (or ad0d6c2b3279)
 except ImportError:
     pass
 try:
     from mercurial import registrar  # since 3.7 (or 37d50250b696)
 
     dir(registrar)  # forcibly load it
 except ImportError:
     registrar = None
 try:
     from mercurial import repoview  # since 2.5 (or 3a6ddacb7198)
 except ImportError:
     pass
 try:
     from mercurial.utils import repoviewutil  # since 5.0
 except ImportError:
     repoviewutil = None
 try:
     from mercurial import scmutil  # since 1.9 (or 8b252e826c68)
 except ImportError:
     pass
 try:
     from mercurial import setdiscovery  # since 1.9 (or cb98fed52495)
 except ImportError:
     pass
 
 try:
     from mercurial import profiling
 except ImportError:
     profiling = None
 
+try:
+    from mercurial.revlogutils import constants as revlog_constants
+
+    perf_rl_kind = (revlog_constants.KIND_OTHER, b'created-by-perf')
+
+    def revlog(opener, *args, **kwargs):
+        return mercurial.revlog.revlog(opener, perf_rl_kind, *args, **kwargs)
+
+
+except (ImportError, AttributeError):
+    perf_rl_kind = None
+
+    def revlog(opener, *args, **kwargs):
+        return mercurial.revlog.revlog(opener, *args, **kwargs)
+
 
 def identity(a):
     return a
 
 
 try:
     from mercurial import pycompat
 
     getargspec = pycompat.getargspec  # added to module after 4.5
     _byteskwargs = pycompat.byteskwargs  # since 4.1 (or fbc3f73dc802)
     _sysstr = pycompat.sysstr  # since 4.0 (or 2219f4f82ede)
     _bytestr = pycompat.bytestr  # since 4.2 (or b70407bd84d5)
     _xrange = pycompat.xrange  # since 4.8 (or 7eba8f83129b)
     fsencode = pycompat.fsencode  # since 3.9 (or f4a5e0e86a7e)
     if pycompat.ispy3:
         _maxint = sys.maxsize  # per py3 docs for replacing maxint
     else:
         _maxint = sys.maxint
 except (NameError, ImportError, AttributeError):
     import inspect
 
     getargspec = inspect.getargspec
     _byteskwargs = identity
     _bytestr = str
     fsencode = identity  # no py3 support
     _maxint = sys.maxint  # no py3 support
     _sysstr = lambda x: x  # no py3 support
     _xrange = xrange
 
 try:
     # 4.7+
     queue = pycompat.queue.Queue
 except (NameError, AttributeError, ImportError):
     # <4.7.
     try:
         queue = pycompat.queue
     except (NameError, AttributeError, ImportError):
         import Queue as queue
 
 try:
     from mercurial import logcmdutil
 
     makelogtemplater = logcmdutil.maketemplater
 except (AttributeError, ImportError):
     try:
         makelogtemplater = cmdutil.makelogtemplater
     except (AttributeError, ImportError):
         makelogtemplater = None
 
 # for "historical portability":
 # define util.safehasattr forcibly, because util.safehasattr has been
 # available since 1.9.3 (or 94b200a11cf7)
 _undefined = object()
 
 
 def safehasattr(thing, attr):
     return getattr(thing, _sysstr(attr), _undefined) is not _undefined
 
 
 setattr(util, 'safehasattr', safehasattr)
 
 # for "historical portability":
 # define util.timer forcibly, because util.timer has been available
 # since ae5d60bb70c9
 if safehasattr(time, 'perf_counter'):
     util.timer = time.perf_counter
 elif os.name == b'nt':
     util.timer = time.clock
 else:
     util.timer = time.time
 
 # for "historical portability":
 # use locally defined empty option list, if formatteropts isn't
 # available, because commands.formatteropts has been available since
 # 3.2 (or 7a7eed5176a4), even though formatting itself has been
 # available since 2.2 (or ae5f92e154d3)
 formatteropts = getattr(
     cmdutil, "formatteropts", getattr(commands, "formatteropts", [])
 )
 
 # for "historical portability":
 # use locally defined option list, if debugrevlogopts isn't available,
 # because commands.debugrevlogopts has been available since 3.7 (or
 # 5606f7d0d063), even though cmdutil.openrevlog() has been available
 # since 1.9 (or a79fea6b3e77).
 revlogopts = getattr(
     cmdutil,
     "debugrevlogopts",
     getattr(
         commands,
         "debugrevlogopts",
         [
             (b'c', b'changelog', False, b'open changelog'),
             (b'm', b'manifest', False, b'open manifest'),
             (b'', b'dir', False, b'open directory manifest'),
         ],
     ),
 )
 
 cmdtable = {}
 
 # for "historical portability":
 # define parsealiases locally, because cmdutil.parsealiases has been
 # available since 1.5 (or 6252852b4332)
 def parsealiases(cmd):
     return cmd.split(b"|")
 
 
 if safehasattr(registrar, 'command'):
     command = registrar.command(cmdtable)
 elif safehasattr(cmdutil, 'command'):
     command = cmdutil.command(cmdtable)
     if 'norepo' not in getargspec(command).args:
         # for "historical portability":
         # wrap original cmdutil.command, because "norepo" option has
         # been available since 3.1 (or 75a96326cecb)
         _command = command
 
         def command(name, options=(), synopsis=None, norepo=False):
             if norepo:
                 commands.norepo += b' %s' % b' '.join(parsealiases(name))
             return _command(name, list(options), synopsis)
 
 
 else:
     # for "historical portability":
     # define "@command" annotation locally, because cmdutil.command
     # has been available since 1.9 (or 2daa5179e73f)
     def command(name, options=(), synopsis=None, norepo=False):
         def decorator(func):
             if synopsis:
                 cmdtable[name] = func, list(options), synopsis
             else:
                 cmdtable[name] = func, list(options)
             if norepo:
                 commands.norepo += b' %s' % b' '.join(parsealiases(name))
             return func
 
         return decorator
 
 
 try:
     import mercurial.registrar
     import mercurial.configitems
 
     configtable = {}
     configitem = mercurial.registrar.configitem(configtable)
     configitem(
         b'perf',
         b'presleep',
         default=mercurial.configitems.dynamicdefault,
         experimental=True,
     )
     configitem(
         b'perf',
         b'stub',
         default=mercurial.configitems.dynamicdefault,
         experimental=True,
     )
     configitem(
         b'perf',
         b'parentscount',
         default=mercurial.configitems.dynamicdefault,
         experimental=True,
     )
     configitem(
         b'perf',
         b'all-timing',
         default=mercurial.configitems.dynamicdefault,
         experimental=True,
     )
     configitem(
         b'perf',
         b'pre-run',
         default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf',
         b'profile-benchmark',
         default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf',
         b'run-limits',
         default=mercurial.configitems.dynamicdefault,
         experimental=True,
     )
 except (ImportError, AttributeError):
     pass
 except TypeError:
     # compatibility fix for a11fd395e83f
     # hg version: 5.2
     configitem(
         b'perf',
         b'presleep',
         default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf',
         b'stub',
         default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf',
         b'parentscount',
         default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf',
         b'all-timing',
         default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf',
         b'pre-run',
         default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf',
         b'profile-benchmark',
         default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf',
         b'run-limits',
         default=mercurial.configitems.dynamicdefault,
     )
 
 
 def getlen(ui):
     if ui.configbool(b"perf", b"stub", False):
         return lambda x: 1
     return len
 
 
 class noop(object):
     """dummy context manager"""
 
     def __enter__(self):
         pass
 
     def __exit__(self, *args):
         pass
 
 
 NOOPCTX = noop()
 
 
 def gettimer(ui, opts=None):
     """return a timer function and formatter: (timer, formatter)
 
     This function exists to gather the creation of formatter in a single
     place instead of duplicating it in all performance commands."""
 
     # enforce an idle period before execution to counteract power management
     # experimental config: perf.presleep
     time.sleep(getint(ui, b"perf", b"presleep", 1))
 
     if opts is None:
         opts = {}
     # redirect all to stderr unless buffer api is in use
     if not ui._buffers:
         ui = ui.copy()
         uifout = safeattrsetter(ui, b'fout', ignoremissing=True)
         if uifout:
             # for "historical portability":
             # ui.fout/ferr have been available since 1.9 (or 4e1ccd4c2b6d)
             uifout.set(ui.ferr)
 
     # get a formatter
     uiformatter = getattr(ui, 'formatter', None)
     if uiformatter:
         fm = uiformatter(b'perf', opts)
     else:
         # for "historical portability":
         # define formatter locally, because ui.formatter has been
         # available since 2.2 (or ae5f92e154d3)
         from mercurial import node
 
         class defaultformatter(object):
             """Minimized composition of baseformatter and plainformatter"""
 
             def __init__(self, ui, topic, opts):
                 self._ui = ui
                 if ui.debugflag:
                     self.hexfunc = node.hex
                 else:
                     self.hexfunc = node.short
 
             def __nonzero__(self):
                 return False
 
             __bool__ = __nonzero__
 
             def startitem(self):
                 pass
 
             def data(self, **data):
                 pass
 
             def write(self, fields, deftext, *fielddata, **opts):
                 self._ui.write(deftext % fielddata, **opts)
 
             def condwrite(self, cond, fields, deftext, *fielddata, **opts):
                 if cond:
                     self._ui.write(deftext % fielddata, **opts)
 
             def plain(self, text, **opts):
                 self._ui.write(text, **opts)
 
             def end(self):
                 pass
 
         fm = defaultformatter(ui, b'perf', opts)
 
     # stub function, runs code only once instead of in a loop
     # experimental config: perf.stub
     if ui.configbool(b"perf", b"stub", False):
         return functools.partial(stub_timer, fm), fm
 
     # experimental config: perf.all-timing
     displayall = ui.configbool(b"perf", b"all-timing", False)
 
     # experimental config: perf.run-limits
     limitspec = ui.configlist(b"perf", b"run-limits", [])
     limits = []
     for item in limitspec:
         parts = item.split(b'-', 1)
         if len(parts) < 2:
             ui.warn((b'malformatted run limit entry, missing "-": %s\n' % item))
             continue
         try:
             time_limit = float(_sysstr(parts[0]))
         except ValueError as e:
             ui.warn(
                 (
                     b'malformatted run limit entry, %s: %s\n'
                     % (_bytestr(e), item)
                 )
             )
             continue
         try:
             run_limit = int(_sysstr(parts[1]))
         except ValueError as e:
             ui.warn(
                 (
                     b'malformatted run limit entry, %s: %s\n'
                     % (_bytestr(e), item)
                 )
             )
             continue
         limits.append((time_limit, run_limit))
     if not limits:
         limits = DEFAULTLIMITS
 
     profiler = None
     if profiling is not None:
         if ui.configbool(b"perf", b"profile-benchmark", False):
             profiler = profiling.profile(ui)
 
     prerun = getint(ui, b"perf", b"pre-run", 0)
     t = functools.partial(
         _timer,
         fm,
         displayall=displayall,
         limits=limits,
         prerun=prerun,
         profiler=profiler,
     )
     return t, fm
 
 
 def stub_timer(fm, func, setup=None, title=None):
     if setup is not None:
         setup()
     func()
 
 
 @contextlib.contextmanager
 def timeone():
     r = []
     ostart = os.times()
     cstart = util.timer()
     yield r
     cstop = util.timer()
     ostop = os.times()
     a, b = ostart, ostop
     r.append((cstop - cstart, b[0] - a[0], b[1] - a[1]))
 
 
 # list of stop condition (elapsed time, minimal run count)
 DEFAULTLIMITS = (
     (3.0, 100),
     (10.0, 3),
 )
 
 
 def _timer(
     fm,
     func,
     setup=None,
     title=None,
     displayall=False,
     limits=DEFAULTLIMITS,
     prerun=0,
     profiler=None,
 ):
     gc.collect()
     results = []
     begin = util.timer()
     count = 0
     if profiler is None:
         profiler = NOOPCTX
     for i in range(prerun):
         if setup is not None:
             setup()
         func()
     keepgoing = True
     while keepgoing:
         if setup is not None:
             setup()
         with profiler:
             with timeone() as item:
                 r = func()
         profiler = NOOPCTX
         count += 1
         results.append(item[0])
         cstop = util.timer()
         # Look for a stop condition.
         elapsed = cstop - begin
         for t, mincount in limits:
             if elapsed >= t and count >= mincount:
                 keepgoing = False
                 break
 
     formatone(fm, results, title=title, result=r, displayall=displayall)
 
 
 def formatone(fm, timings, title=None, result=None, displayall=False):
 
     count = len(timings)
 
     fm.startitem()
 
     if title:
         fm.write(b'title', b'! %s\n', title)
     if result:
         fm.write(b'result', b'! result: %s\n', result)
 
     def display(role, entry):
         prefix = b''
         if role != b'best':
             prefix = b'%s.' % role
         fm.plain(b'!')
         fm.write(prefix + b'wall', b' wall %f', entry[0])
         fm.write(prefix + b'comb', b' comb %f', entry[1] + entry[2])
         fm.write(prefix + b'user', b' user %f', entry[1])
         fm.write(prefix + b'sys', b' sys %f', entry[2])
         fm.write(prefix + b'count', b' (%s of %%d)' % role, count)
         fm.plain(b'\n')
 
     timings.sort()
     min_val = timings[0]
     display(b'best', min_val)
     if displayall:
         max_val = timings[-1]
         display(b'max', max_val)
         avg = tuple([sum(x) / count for x in zip(*timings)])
         display(b'avg', avg)
         median = timings[len(timings) // 2]
         display(b'median', median)
 
 
 # utilities for historical portability
 
 
 def getint(ui, section, name, default):
     # for "historical portability":
     # ui.configint has been available since 1.9 (or fa2b596db182)
     v = ui.config(section, name, None)
     if v is None:
         return default
     try:
         return int(v)
     except ValueError:
         raise error.ConfigError(
             b"%s.%s is not an integer ('%s')" % (section, name, v)
         )
 
 
 def safeattrsetter(obj, name, ignoremissing=False):
     """Ensure that 'obj' has 'name' attribute before subsequent setattr
 
     This function is aborted, if 'obj' doesn't have 'name' attribute
     at runtime. This avoids overlooking removal of an attribute, which
     breaks assumption of performance measurement, in the future.
 
     This function returns the object to (1) assign a new value, and
     (2) restore an original value to the attribute.
 
     If 'ignoremissing' is true, missing 'name' attribute doesn't cause
     abortion, and this function returns None. This is useful to
     examine an attribute, which isn't ensured in all Mercurial
     versions.
     """
     if not util.safehasattr(obj, name):
         if ignoremissing:
             return None
         raise error.Abort(
             (
                 b"missing attribute %s of %s might break assumption"
                 b" of performance measurement"
             )
             % (name, obj)
         )
 
     origvalue = getattr(obj, _sysstr(name))
 
     class attrutil(object):
         def set(self, newvalue):
             setattr(obj, _sysstr(name), newvalue)
 
         def restore(self):
             setattr(obj, _sysstr(name), origvalue)
 
     return attrutil()
 
 
 # utilities to examine each internal API changes
 
 
 def getbranchmapsubsettable():
     # for "historical portability":
     # subsettable is defined in:
     # - branchmap since 2.9 (or 175c6fd8cacc)
     # - repoview since 2.5 (or 59a9f18d4587)
     # - repoviewutil since 5.0
     for mod in (branchmap, repoview, repoviewutil):
         subsettable = getattr(mod, 'subsettable', None)
         if subsettable:
             return subsettable
 
     # bisecting in bcee63733aad::59a9f18d4587 can reach here (both
     # branchmap and repoview modules exist, but subsettable attribute
     # doesn't)
     raise error.Abort(
         b"perfbranchmap not available with this Mercurial",
         hint=b"use 2.5 or later",
     )
 
 
 def getsvfs(repo):
     """Return appropriate object to access files under .hg/store"""
     # for "historical portability":
     # repo.svfs has been available since 2.3 (or 7034365089bf)
     svfs = getattr(repo, 'svfs', None)
     if svfs:
         return svfs
     else:
         return getattr(repo, 'sopener')
 
 
 def getvfs(repo):
     """Return appropriate object to access files under .hg"""
     # for "historical portability":
     # repo.vfs has been available since 2.3 (or 7034365089bf)
     vfs = getattr(repo, 'vfs', None)
     if vfs:
         return vfs
     else:
         return getattr(repo, 'opener')
 
 
 def repocleartagscachefunc(repo):
     """Return the function to clear tags cache according to repo internal API"""
     if util.safehasattr(repo, b'_tagscache'):  # since 2.0 (or 9dca7653b525)
         # in this case, setattr(repo, '_tagscache', None) or so isn't
         # correct way to clear tags cache, because existing code paths
         # expect _tagscache to be a structured object.
         def clearcache():
             # _tagscache has been filteredpropertycache since 2.5 (or
             # 98c867ac1330), and delattr() can't work in such case
             if '_tagscache' in vars(repo):
                 del repo.__dict__['_tagscache']
 
         return clearcache
 
     repotags = safeattrsetter(repo, b'_tags', ignoremissing=True)
     if repotags:  # since 1.4 (or 5614a628d173)
         return lambda: repotags.set(None)
 
     repotagscache = safeattrsetter(repo, b'tagscache', ignoremissing=True)
     if repotagscache:  # since 0.6 (or d7df759d0e97)
         return lambda: repotagscache.set(None)
 
     # Mercurial earlier than 0.6 (or d7df759d0e97) logically reaches
     # this point, but it isn't so problematic, because:
     # - repo.tags of such Mercurial isn't "callable", and repo.tags()
     #   in perftags() causes failure soon
     # - perf.py itself has been available since 1.1 (or eb240755386d)
     raise error.Abort(b"tags API of this hg command is unknown")
 
 
 # utilities to clear cache
 
 
 def clearfilecache(obj, attrname):
     unfiltered = getattr(obj, 'unfiltered', None)
     if unfiltered is not None:
         obj = obj.unfiltered()
     if attrname in vars(obj):
         delattr(obj, attrname)
     obj._filecache.pop(attrname, None)
 
 
 def clearchangelog(repo):
     if repo is not repo.unfiltered():
         object.__setattr__(repo, '_clcachekey', None)
         object.__setattr__(repo, '_clcache', None)
     clearfilecache(repo.unfiltered(), 'changelog')
 
 
 # perf commands
 
 
 @command(b'perf::walk|perfwalk', formatteropts)
 def perfwalk(ui, repo, *pats, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     m = scmutil.match(repo[None], pats, {})
     timer(
         lambda: len(
             list(
                 repo.dirstate.walk(m, subrepos=[], unknown=True, ignored=False)
             )
         )
     )
     fm.end()
 
 
 @command(b'perf::annotate|perfannotate', formatteropts)
 def perfannotate(ui, repo, f, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     fc = repo[b'.'][f]
     timer(lambda: len(fc.annotate(True)))
     fm.end()
 
 
 @command(
     b'perf::status|perfstatus',
     [
         (b'u', b'unknown', False, b'ask status to look for unknown files'),
         (b'', b'dirstate', False, b'benchmark the internal dirstate call'),
     ]
     + formatteropts,
 )
 def perfstatus(ui, repo, **opts):
     """benchmark the performance of a single status call
 
     The repository data are preserved between each call.
 
     By default, only the status of the tracked file are requested. If
     `--unknown` is passed, the "unknown" files are also tracked.
     """
     opts = _byteskwargs(opts)
     # m = match.always(repo.root, repo.getcwd())
     # timer(lambda: sum(map(len, repo.dirstate.status(m, [], False, False,
     #                                                False))))
     timer, fm = gettimer(ui, opts)
     if opts[b'dirstate']:
         dirstate = repo.dirstate
         m = scmutil.matchall(repo)
         unknown = opts[b'unknown']
 
         def status_dirstate():
             s = dirstate.status(
                 m, subrepos=[], ignored=False, clean=False, unknown=unknown
             )
             sum(map(bool, s))
 
         timer(status_dirstate)
     else:
         timer(lambda: sum(map(len, repo.status(unknown=opts[b'unknown']))))
     fm.end()
 
 
 @command(b'perf::addremove|perfaddremove', formatteropts)
 def perfaddremove(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     try:
         oldquiet = repo.ui.quiet
         repo.ui.quiet = True
         matcher = scmutil.match(repo[None])
         opts[b'dry_run'] = True
         if 'uipathfn' in getargspec(scmutil.addremove).args:
             uipathfn = scmutil.getuipathfn(repo)
             timer(lambda: scmutil.addremove(repo, matcher, b"", uipathfn, opts))
         else:
             timer(lambda: scmutil.addremove(repo, matcher, b"", opts))
     finally:
         repo.ui.quiet = oldquiet
         fm.end()
 
 
 def clearcaches(cl):
     # behave somewhat consistently across internal API changes
     if util.safehasattr(cl, b'clearcaches'):
         cl.clearcaches()
     elif util.safehasattr(cl, b'_nodecache'):
         # <= hg-5.2
         from mercurial.node import nullid, nullrev
 
         cl._nodecache = {nullid: nullrev}
         cl._nodepos = None
 
 
 @command(b'perf::heads|perfheads', formatteropts)
 def perfheads(ui, repo, **opts):
     """benchmark the computation of a changelog heads"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     cl = repo.changelog
 
     def s():
         clearcaches(cl)
 
     def d():
         len(cl.headrevs())
 
     timer(d, setup=s)
     fm.end()
 
 
 @command(
     b'perf::tags|perftags',
     formatteropts
     + [
         (b'', b'clear-revlogs', False, b'refresh changelog and manifest'),
     ],
 )
 def perftags(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     repocleartagscache = repocleartagscachefunc(repo)
     clearrevlogs = opts[b'clear_revlogs']
 
     def s():
         if clearrevlogs:
             clearchangelog(repo)
             clearfilecache(repo.unfiltered(), 'manifest')
         repocleartagscache()
 
     def t():
         return len(repo.tags())
 
     timer(t, setup=s)
     fm.end()
 
 
 @command(b'perf::ancestors|perfancestors', formatteropts)
 def perfancestors(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     heads = repo.changelog.headrevs()
 
     def d():
         for a in repo.changelog.ancestors(heads):
             pass
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::ancestorset|perfancestorset', formatteropts)
 def perfancestorset(ui, repo, revset, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     revs = repo.revs(revset)
     heads = repo.changelog.headrevs()
 
     def d():
         s = repo.changelog.ancestors(heads)
         for rev in revs:
             rev in s
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::discovery|perfdiscovery', formatteropts, b'PATH')
 def perfdiscovery(ui, repo, path, **opts):
     """benchmark discovery between local repo and the peer at given path"""
     repos = [repo, None]
     timer, fm = gettimer(ui, opts)
 
     try:
         from mercurial.utils.urlutil import get_unique_pull_path
 
         path = get_unique_pull_path(b'perfdiscovery', repo, ui, path)[0]
     except ImportError:
         path = ui.expandpath(path)
 
     def s():
         repos[1] = hg.peer(ui, opts, path)
 
     def d():
         setdiscovery.findcommonheads(ui, *repos)
 
     timer(d, setup=s)
     fm.end()
 
 
 @command(
     b'perf::bookmarks|perfbookmarks',
     formatteropts
     + [
         (b'', b'clear-revlogs', False, b'refresh changelog and manifest'),
     ],
 )
 def perfbookmarks(ui, repo, **opts):
     """benchmark parsing bookmarks from disk to memory"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
 
     clearrevlogs = opts[b'clear_revlogs']
 
     def s():
         if clearrevlogs:
             clearchangelog(repo)
         clearfilecache(repo, b'_bookmarks')
 
     def d():
         repo._bookmarks
 
     timer(d, setup=s)
     fm.end()
 
 
 @command(b'perf::bundleread|perfbundleread', formatteropts, b'BUNDLE')
 def perfbundleread(ui, repo, bundlepath, **opts):
     """Benchmark reading of bundle files.
 
     This command is meant to isolate the I/O part of bundle reading as
     much as possible.
     """
     from mercurial import (
         bundle2,
         exchange,
         streamclone,
     )
 
     opts = _byteskwargs(opts)
 
     def makebench(fn):
         def run():
             with open(bundlepath, b'rb') as fh:
                 bundle = exchange.readbundle(ui, fh, bundlepath)
                 fn(bundle)
 
         return run
 
     def makereadnbytes(size):
         def run():
             with open(bundlepath, b'rb') as fh:
                 bundle = exchange.readbundle(ui, fh, bundlepath)
                 while bundle.read(size):
                     pass
 
         return run
 
     def makestdioread(size):
         def run():
             with open(bundlepath, b'rb') as fh:
                 while fh.read(size):
                     pass
 
         return run
 
     # bundle1
 
     def deltaiter(bundle):
         for delta in bundle.deltaiter():
             pass
 
     def iterchunks(bundle):
         for chunk in bundle.getchunks():
             pass
 
     # bundle2
 
     def forwardchunks(bundle):
         for chunk in bundle._forwardchunks():
             pass
 
     def iterparts(bundle):
         for part in bundle.iterparts():
             pass
 
     def iterpartsseekable(bundle):
         for part in bundle.iterparts(seekable=True):
             pass
 
     def seek(bundle):
         for part in bundle.iterparts(seekable=True):
             part.seek(0, os.SEEK_END)
 
     def makepartreadnbytes(size):
         def run():
             with open(bundlepath, b'rb') as fh:
                 bundle = exchange.readbundle(ui, fh, bundlepath)
                 for part in bundle.iterparts():
                     while part.read(size):
                         pass
 
         return run
 
     benches = [
         (makestdioread(8192), b'read(8k)'),
         (makestdioread(16384), b'read(16k)'),
         (makestdioread(32768), b'read(32k)'),
         (makestdioread(131072), b'read(128k)'),
     ]
 
     with open(bundlepath, b'rb') as fh:
         bundle = exchange.readbundle(ui, fh, bundlepath)
 
         if isinstance(bundle, changegroup.cg1unpacker):
             benches.extend(
                 [
                     (makebench(deltaiter), b'cg1 deltaiter()'),
                     (makebench(iterchunks), b'cg1 getchunks()'),
                     (makereadnbytes(8192), b'cg1 read(8k)'),
                     (makereadnbytes(16384), b'cg1 read(16k)'),
                     (makereadnbytes(32768), b'cg1 read(32k)'),
                     (makereadnbytes(131072), b'cg1 read(128k)'),
                 ]
             )
         elif isinstance(bundle, bundle2.unbundle20):
             benches.extend(
                 [
                     (makebench(forwardchunks), b'bundle2 forwardchunks()'),
                     (makebench(iterparts), b'bundle2 iterparts()'),
                     (
                         makebench(iterpartsseekable),
                         b'bundle2 iterparts() seekable',
                     ),
                     (makebench(seek), b'bundle2 part seek()'),
                     (makepartreadnbytes(8192), b'bundle2 part read(8k)'),
                     (makepartreadnbytes(16384), b'bundle2 part read(16k)'),
                     (makepartreadnbytes(32768), b'bundle2 part read(32k)'),
                     (makepartreadnbytes(131072), b'bundle2 part read(128k)'),
                 ]
             )
         elif isinstance(bundle, streamclone.streamcloneapplier):
             raise error.Abort(b'stream clone bundles not supported')
         else:
             raise error.Abort(b'unhandled bundle type: %s' % type(bundle))
 
     for fn, title in benches:
         timer, fm = gettimer(ui, opts)
         timer(fn, title=title)
         fm.end()
 
 
 @command(
     b'perf::changegroupchangelog|perfchangegroupchangelog',
     formatteropts
     + [
         (b'', b'cgversion', b'02', b'changegroup version'),
         (b'r', b'rev', b'', b'revisions to add to changegroup'),
     ],
 )
 def perfchangegroupchangelog(ui, repo, cgversion=b'02', rev=None, **opts):
     """Benchmark producing a changelog group for a changegroup.
 
     This measures the time spent processing the changelog during a
     bundle operation. This occurs during `hg bundle` and on a server
     processing a `getbundle` wire protocol request (handles clones
     and pull requests).
 
     By default, all revisions are added to the changegroup.
     """
     opts = _byteskwargs(opts)
     cl = repo.changelog
     nodes = [cl.lookup(r) for r in repo.revs(rev or b'all()')]
     bundler = changegroup.getbundler(cgversion, repo)
 
     def d():
         state, chunks = bundler._generatechangelog(cl, nodes)
         for chunk in chunks:
             pass
 
     timer, fm = gettimer(ui, opts)
 
     # Terminal printing can interfere with timing. So disable it.
     with ui.configoverride({(b'progress', b'disable'): True}):
         timer(d)
 
     fm.end()
 
 
 @command(b'perf::dirs|perfdirs', formatteropts)
 def perfdirs(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     dirstate = repo.dirstate
     b'a' in dirstate
 
     def d():
         dirstate.hasdir(b'a')
-        del dirstate._map._dirs
+        try:
+            del dirstate._map._dirs
+        except AttributeError:
+            pass
 
     timer(d)
     fm.end()
 
 
 @command(
     b'perf::dirstate|perfdirstate',
     [
         (
             b'',
             b'iteration',
             None,
             b'benchmark a full iteration for the dirstate',
         ),
         (
             b'',
             b'contains',
             None,
             b'benchmark a large amount of `nf in dirstate` calls',
         ),
     ]
     + formatteropts,
 )
 def perfdirstate(ui, repo, **opts):
     """benchmap the time of various distate operations
 
     By default benchmark the time necessary to load a dirstate from scratch.
     The dirstate is loaded to the point were a "contains" request can be
     answered.
     """
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     b"a" in repo.dirstate
 
     if opts[b'iteration'] and opts[b'contains']:
         msg = b'only specify one of --iteration or --contains'
         raise error.Abort(msg)
 
     if opts[b'iteration']:
         setup = None
         dirstate = repo.dirstate
 
         def d():
             for f in dirstate:
                 pass
 
     elif opts[b'contains']:
         setup = None
         dirstate = repo.dirstate
         allfiles = list(dirstate)
         # also add file path that will be "missing" from the dirstate
         allfiles.extend([f[::-1] for f in allfiles])
 
         def d():
             for f in allfiles:
                 f in dirstate
 
     else:
 
         def setup():
             repo.dirstate.invalidate()
 
         def d():
             b"a" in repo.dirstate
 
     timer(d, setup=setup)
     fm.end()
 
 
 @command(b'perf::dirstatedirs|perfdirstatedirs', formatteropts)
 def perfdirstatedirs(ui, repo, **opts):
     """benchmap a 'dirstate.hasdir' call from an empty `dirs` cache"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     repo.dirstate.hasdir(b"a")
 
     def setup():
-        del repo.dirstate._map._dirs
+        try:
+            del repo.dirstate._map._dirs
+        except AttributeError:
+            pass
 
     def d():
         repo.dirstate.hasdir(b"a")
 
     timer(d, setup=setup)
     fm.end()
 
 
 @command(b'perf::dirstatefoldmap|perfdirstatefoldmap', formatteropts)
 def perfdirstatefoldmap(ui, repo, **opts):
     """benchmap a `dirstate._map.filefoldmap.get()` request
 
     The dirstate filefoldmap cache is dropped between every request.
     """
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     dirstate = repo.dirstate
     dirstate._map.filefoldmap.get(b'a')
 
     def setup():
         del dirstate._map.filefoldmap
 
     def d():
         dirstate._map.filefoldmap.get(b'a')
 
     timer(d, setup=setup)
     fm.end()
 
 
 @command(b'perf::dirfoldmap|perfdirfoldmap', formatteropts)
 def perfdirfoldmap(ui, repo, **opts):
     """benchmap a `dirstate._map.dirfoldmap.get()` request
 
     The dirstate dirfoldmap cache is dropped between every request.
     """
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     dirstate = repo.dirstate
     dirstate._map.dirfoldmap.get(b'a')
 
     def setup():
         del dirstate._map.dirfoldmap
-        del dirstate._map._dirs
+        try:
+            del dirstate._map._dirs
+        except AttributeError:
+            pass
 
     def d():
         dirstate._map.dirfoldmap.get(b'a')
 
     timer(d, setup=setup)
     fm.end()
 
 
 @command(b'perf::dirstatewrite|perfdirstatewrite', formatteropts)
 def perfdirstatewrite(ui, repo, **opts):
     """benchmap the time it take to write a dirstate on disk"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     ds = repo.dirstate
     b"a" in ds
 
     def setup():
         ds._dirty = True
 
     def d():
         ds.write(repo.currenttransaction())
 
     timer(d, setup=setup)
     fm.end()
 
 
 def _getmergerevs(repo, opts):
     """parse command argument to return rev involved in merge
 
     input: options dictionnary with `rev`, `from` and `bse`
     output: (localctx, otherctx, basectx)
     """
     if opts[b'from']:
         fromrev = scmutil.revsingle(repo, opts[b'from'])
         wctx = repo[fromrev]
     else:
         wctx = repo[None]
         # we don't want working dir files to be stat'd in the benchmark, so
         # prime that cache
         wctx.dirty()
     rctx = scmutil.revsingle(repo, opts[b'rev'], opts[b'rev'])
     if opts[b'base']:
         fromrev = scmutil.revsingle(repo, opts[b'base'])
         ancestor = repo[fromrev]
     else:
         ancestor = wctx.ancestor(rctx)
     return (wctx, rctx, ancestor)
 
 
 @command(
     b'perf::mergecalculate|perfmergecalculate',
     [
         (b'r', b'rev', b'.', b'rev to merge against'),
         (b'', b'from', b'', b'rev to merge from'),
         (b'', b'base', b'', b'the revision to use as base'),
     ]
     + formatteropts,
 )
 def perfmergecalculate(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
 
     wctx, rctx, ancestor = _getmergerevs(repo, opts)
 
     def d():
         # acceptremote is True because we don't want prompts in the middle of
         # our benchmark
         merge.calculateupdates(
             repo,
             wctx,
             rctx,
             [ancestor],
             branchmerge=False,
             force=False,
             acceptremote=True,
             followcopies=True,
         )
 
     timer(d)
     fm.end()
 
 
 @command(
     b'perf::mergecopies|perfmergecopies',
     [
         (b'r', b'rev', b'.', b'rev to merge against'),
         (b'', b'from', b'', b'rev to merge from'),
         (b'', b'base', b'', b'the revision to use as base'),
     ]
     + formatteropts,
 )
 def perfmergecopies(ui, repo, **opts):
     """measure runtime of `copies.mergecopies`"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     wctx, rctx, ancestor = _getmergerevs(repo, opts)
 
     def d():
         # acceptremote is True because we don't want prompts in the middle of
         # our benchmark
         copies.mergecopies(repo, wctx, rctx, ancestor)
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::pathcopies|perfpathcopies', [], b"REV REV")
 def perfpathcopies(ui, repo, rev1, rev2, **opts):
     """benchmark the copy tracing logic"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     ctx1 = scmutil.revsingle(repo, rev1, rev1)
     ctx2 = scmutil.revsingle(repo, rev2, rev2)
 
     def d():
         copies.pathcopies(ctx1, ctx2)
 
     timer(d)
     fm.end()
 
 
 @command(
     b'perf::phases|perfphases',
     [
         (b'', b'full', False, b'include file reading time too'),
     ],
     b"",
 )
 def perfphases(ui, repo, **opts):
     """benchmark phasesets computation"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     _phases = repo._phasecache
     full = opts.get(b'full')
 
     def d():
         phases = _phases
         if full:
             clearfilecache(repo, b'_phasecache')
             phases = repo._phasecache
         phases.invalidate()
         phases.loadphaserevs(repo)
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::phasesremote|perfphasesremote', [], b"[DEST]")
 def perfphasesremote(ui, repo, dest=None, **opts):
     """benchmark time needed to analyse phases of the remote server"""
     from mercurial.node import bin
     from mercurial import (
         exchange,
         hg,
         phases,
     )
 
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
 
     path = ui.getpath(dest, default=(b'default-push', b'default'))
     if not path:
         raise error.Abort(
             b'default repository not configured!',
             hint=b"see 'hg help config.paths'",
         )
     dest = path.pushloc or path.loc
     ui.statusnoi18n(b'analysing phase of %s\n' % util.hidepassword(dest))
     other = hg.peer(repo, opts, dest)
 
     # easier to perform discovery through the operation
     op = exchange.pushoperation(repo, other)
     exchange._pushdiscoverychangeset(op)
 
     remotesubset = op.fallbackheads
 
     with other.commandexecutor() as e:
         remotephases = e.callcommand(
             b'listkeys', {b'namespace': b'phases'}
         ).result()
     del other
     publishing = remotephases.get(b'publishing', False)
     if publishing:
         ui.statusnoi18n(b'publishing: yes\n')
     else:
         ui.statusnoi18n(b'publishing: no\n')
 
     has_node = getattr(repo.changelog.index, 'has_node', None)
     if has_node is None:
         has_node = repo.changelog.nodemap.__contains__
     nonpublishroots = 0
     for nhex, phase in remotephases.iteritems():
         if nhex == b'publishing':  # ignore data related to publish option
             continue
         node = bin(nhex)
         if has_node(node) and int(phase):
             nonpublishroots += 1
     ui.statusnoi18n(b'number of roots: %d\n' % len(remotephases))
     ui.statusnoi18n(b'number of known non public roots: %d\n' % nonpublishroots)
 
     def d():
         phases.remotephasessummary(repo, remotesubset, remotephases)
 
     timer(d)
     fm.end()
 
 
 @command(
     b'perf::manifest|perfmanifest',
     [
         (b'm', b'manifest-rev', False, b'Look up a manifest node revision'),
         (b'', b'clear-disk', False, b'clear on-disk caches too'),
     ]
     + formatteropts,
     b'REV|NODE',
 )
 def perfmanifest(ui, repo, rev, manifest_rev=False, clear_disk=False, **opts):
     """benchmark the time to read a manifest from disk and return a usable
     dict-like object
 
     Manifest caches are cleared before retrieval."""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     if not manifest_rev:
         ctx = scmutil.revsingle(repo, rev, rev)
         t = ctx.manifestnode()
     else:
         from mercurial.node import bin
 
         if len(rev) == 40:
             t = bin(rev)
         else:
             try:
                 rev = int(rev)
 
                 if util.safehasattr(repo.manifestlog, b'getstorage'):
                     t = repo.manifestlog.getstorage(b'').node(rev)
                 else:
                     t = repo.manifestlog._revlog.lookup(rev)
             except ValueError:
                 raise error.Abort(
                     b'manifest revision must be integer or full node'
                 )
 
     def d():
         repo.manifestlog.clearcaches(clear_persisted_data=clear_disk)
         repo.manifestlog[t].read()
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::changeset|perfchangeset', formatteropts)
 def perfchangeset(ui, repo, rev, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     n = scmutil.revsingle(repo, rev).node()
 
     def d():
         repo.changelog.read(n)
         # repo.changelog._cache = None
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::ignore|perfignore', formatteropts)
 def perfignore(ui, repo, **opts):
     """benchmark operation related to computing ignore"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     dirstate = repo.dirstate
 
     def setupone():
         dirstate.invalidate()
         clearfilecache(dirstate, b'_ignore')
 
     def runone():
         dirstate._ignore
 
     timer(runone, setup=setupone, title=b"load")
     fm.end()
 
 
 @command(
     b'perf::index|perfindex',
     [
         (b'', b'rev', [], b'revision to be looked up (default tip)'),
         (b'', b'no-lookup', None, b'do not revision lookup post creation'),
     ]
     + formatteropts,
 )
 def perfindex(ui, repo, **opts):
     """benchmark index creation time followed by a lookup
 
     The default is to look `tip` up. Depending on the index implementation,
     the revision looked up can matters. For example, an implementation
     scanning the index will have a faster lookup time for `--rev tip` than for
     `--rev 0`. The number of looked up revisions and their order can also
     matters.
 
     Example of useful set to test:
 
     * tip
     * 0
     * -10:
     * :10
     * -10: + :10
     * :10: + -10:
     * -10000:
     * -10000: + 0
 
     It is not currently possible to check for lookup of a missing node. For
     deeper lookup benchmarking, checkout the `perfnodemap` command."""
     import mercurial.revlog
 
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     mercurial.revlog._prereadsize = 2 ** 24  # disable lazy parser in old hg
     if opts[b'no_lookup']:
         if opts['rev']:
             raise error.Abort('--no-lookup and --rev are mutually exclusive')
         nodes = []
     elif not opts[b'rev']:
         nodes = [repo[b"tip"].node()]
     else:
         revs = scmutil.revrange(repo, opts[b'rev'])
         cl = repo.changelog
         nodes = [cl.node(r) for r in revs]
 
     unfi = repo.unfiltered()
     # find the filecache func directly
     # This avoid polluting the benchmark with the filecache logic
     makecl = unfi.__class__.changelog.func
 
     def setup():
         # probably not necessary, but for good measure
         clearchangelog(unfi)
 
     def d():
         cl = makecl(unfi)
         for n in nodes:
             cl.rev(n)
 
     timer(d, setup=setup)
     fm.end()
 
 
 @command(
     b'perf::nodemap|perfnodemap',
     [
         (b'', b'rev', [], b'revision to be looked up (default tip)'),
         (b'', b'clear-caches', True, b'clear revlog cache between calls'),
     ]
     + formatteropts,
 )
 def perfnodemap(ui, repo, **opts):
     """benchmark the time necessary to look up revision from a cold nodemap
 
     Depending on the implementation, the amount and order of revision we look
     up can varies. Example of useful set to test:
     * tip
     * 0
     * -10:
     * :10
     * -10: + :10
     * :10: + -10:
     * -10000:
     * -10000: + 0
 
     The command currently focus on valid binary lookup. Benchmarking for
     hexlookup, prefix lookup and missing lookup would also be valuable.
     """
     import mercurial.revlog
 
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     mercurial.revlog._prereadsize = 2 ** 24  # disable lazy parser in old hg
 
     unfi = repo.unfiltered()
     clearcaches = opts[b'clear_caches']
     # find the filecache func directly
     # This avoid polluting the benchmark with the filecache logic
     makecl = unfi.__class__.changelog.func
     if not opts[b'rev']:
         raise error.Abort(b'use --rev to specify revisions to look up')
     revs = scmutil.revrange(repo, opts[b'rev'])
     cl = repo.changelog
     nodes = [cl.node(r) for r in revs]
 
     # use a list to pass reference to a nodemap from one closure to the next
     nodeget = [None]
 
     def setnodeget():
         # probably not necessary, but for good measure
         clearchangelog(unfi)
         cl = makecl(unfi)
         if util.safehasattr(cl.index, 'get_rev'):
             nodeget[0] = cl.index.get_rev
         else:
             nodeget[0] = cl.nodemap.get
 
     def d():
         get = nodeget[0]
         for n in nodes:
             get(n)
 
     setup = None
     if clearcaches:
 
         def setup():
             setnodeget()
 
     else:
         setnodeget()
         d()  # prewarm the data structure
     timer(d, setup=setup)
     fm.end()
 
 
 @command(b'perf::startup|perfstartup', formatteropts)
 def perfstartup(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
 
     def d():
         if os.name != 'nt':
             os.system(
                 b"HGRCPATH= %s version -q > /dev/null" % fsencode(sys.argv[0])
             )
         else:
             os.environ['HGRCPATH'] = r' '
             os.system("%s version -q > NUL" % sys.argv[0])
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::parents|perfparents', formatteropts)
 def perfparents(ui, repo, **opts):
     """benchmark the time necessary to fetch one changeset's parents.
 
     The fetch is done using the `node identifier`, traversing all object layers
     from the repository object. The first N revisions will be used for this
     benchmark. N is controlled by the ``perf.parentscount`` config option
     (default: 1000).
     """
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     # control the number of commits perfparents iterates over
     # experimental config: perf.parentscount
     count = getint(ui, b"perf", b"parentscount", 1000)
     if len(repo.changelog) < count:
         raise error.Abort(b"repo needs %d commits for this test" % count)
     repo = repo.unfiltered()
     nl = [repo.changelog.node(i) for i in _xrange(count)]
 
     def d():
         for n in nl:
             repo.changelog.parents(n)
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::ctxfiles|perfctxfiles', formatteropts)
 def perfctxfiles(ui, repo, x, **opts):
     opts = _byteskwargs(opts)
     x = int(x)
     timer, fm = gettimer(ui, opts)
 
     def d():
         len(repo[x].files())
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::rawfiles|perfrawfiles', formatteropts)
 def perfrawfiles(ui, repo, x, **opts):
     opts = _byteskwargs(opts)
     x = int(x)
     timer, fm = gettimer(ui, opts)
     cl = repo.changelog
 
     def d():
         len(cl.read(x)[3])
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::lookup|perflookup', formatteropts)
 def perflookup(ui, repo, rev, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     timer(lambda: len(repo.lookup(rev)))
     fm.end()
 
 
 @command(
     b'perf::linelogedits|perflinelogedits',
     [
         (b'n', b'edits', 10000, b'number of edits'),
         (b'', b'max-hunk-lines', 10, b'max lines in a hunk'),
     ],
     norepo=True,
 )
 def perflinelogedits(ui, **opts):
     from mercurial import linelog
 
     opts = _byteskwargs(opts)
 
     edits = opts[b'edits']
     maxhunklines = opts[b'max_hunk_lines']
 
     maxb1 = 100000
     random.seed(0)
     randint = random.randint
     currentlines = 0
     arglist = []
     for rev in _xrange(edits):
         a1 = randint(0, currentlines)
         a2 = randint(a1, min(currentlines, a1 + maxhunklines))
         b1 = randint(0, maxb1)
         b2 = randint(b1, b1 + maxhunklines)
         currentlines += (b2 - b1) - (a2 - a1)
         arglist.append((rev, a1, a2, b1, b2))
 
     def d():
         ll = linelog.linelog()
         for args in arglist:
             ll.replacelines(*args)
 
     timer, fm = gettimer(ui, opts)
     timer(d)
     fm.end()
 
 
 @command(b'perf::revrange|perfrevrange', formatteropts)
 def perfrevrange(ui, repo, *specs, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     revrange = scmutil.revrange
     timer(lambda: len(revrange(repo, specs)))
     fm.end()
 
 
 @command(b'perf::nodelookup|perfnodelookup', formatteropts)
 def perfnodelookup(ui, repo, rev, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     import mercurial.revlog
 
     mercurial.revlog._prereadsize = 2 ** 24  # disable lazy parser in old hg
     n = scmutil.revsingle(repo, rev).node()
-    cl = mercurial.revlog.revlog(getsvfs(repo), b"00changelog.i")
+
+    try:
+        cl = revlog(getsvfs(repo), radix=b"00changelog")
+    except TypeError:
+        cl = revlog(getsvfs(repo), indexfile=b"00changelog.i")
 
     def d():
         cl.rev(n)
         clearcaches(cl)
 
     timer(d)
     fm.end()
 
 
 @command(
     b'perf::log|perflog',
     [(b'', b'rename', False, b'ask log to follow renames')] + formatteropts,
 )
 def perflog(ui, repo, rev=None, **opts):
     opts = _byteskwargs(opts)
     if rev is None:
         rev = []
     timer, fm = gettimer(ui, opts)
     ui.pushbuffer()
     timer(
         lambda: commands.log(
             ui, repo, rev=rev, date=b'', user=b'', copies=opts.get(b'rename')
         )
     )
     ui.popbuffer()
     fm.end()
 
 
 @command(b'perf::moonwalk|perfmoonwalk', formatteropts)
 def perfmoonwalk(ui, repo, **opts):
     """benchmark walking the changelog backwards
 
     This also loads the changelog data for each revision in the changelog.
     """
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
 
     def moonwalk():
         for i in repo.changelog.revs(start=(len(repo) - 1), stop=-1):
             ctx = repo[i]
             ctx.branch()  # read changelog data (in addition to the index)
 
     timer(moonwalk)
     fm.end()
 
 
 @command(
     b'perf::templating|perftemplating',
     [
         (b'r', b'rev', [], b'revisions to run the template on'),
     ]
     + formatteropts,
 )
 def perftemplating(ui, repo, testedtemplate=None, **opts):
     """test the rendering time of a given template"""
     if makelogtemplater is None:
         raise error.Abort(
             b"perftemplating not available with this Mercurial",
             hint=b"use 4.3 or later",
         )
 
     opts = _byteskwargs(opts)
 
     nullui = ui.copy()
     nullui.fout = open(os.devnull, 'wb')
     nullui.disablepager()
     revs = opts.get(b'rev')
     if not revs:
         revs = [b'all()']
     revs = list(scmutil.revrange(repo, revs))
 
     defaulttemplate = (
         b'{date|shortdate} [{rev}:{node|short}]'
         b' {author|person}: {desc|firstline}\n'
     )
     if testedtemplate is None:
         testedtemplate = defaulttemplate
     displayer = makelogtemplater(nullui, repo, testedtemplate)
 
     def format():
         for r in revs:
             ctx = repo[r]
             displayer.show(ctx)
             displayer.flush(ctx)
 
     timer, fm = gettimer(ui, opts)
     timer(format)
     fm.end()
 
 
 def _displaystats(ui, opts, entries, data):
     # use a second formatter because the data are quite different, not sure
     # how it flies with the templater.
     fm = ui.formatter(b'perf-stats', opts)
     for key, title in entries:
         values = data[key]
         nbvalues = len(data)
         values.sort()
         stats = {
             'key': key,
             'title': title,
             'nbitems': len(values),
             'min': values[0][0],
             '10%': values[(nbvalues * 10) // 100][0],
             '25%': values[(nbvalues * 25) // 100][0],
             '50%': values[(nbvalues * 50) // 100][0],
             '75%': values[(nbvalues * 75) // 100][0],
             '80%': values[(nbvalues * 80) // 100][0],
             '85%': values[(nbvalues * 85) // 100][0],
             '90%': values[(nbvalues * 90) // 100][0],
             '95%': values[(nbvalues * 95) // 100][0],
             '99%': values[(nbvalues * 99) // 100][0],
             'max': values[-1][0],
         }
         fm.startitem()
         fm.data(**stats)
         # make node pretty for the human output
         fm.plain('### %s (%d items)\n' % (title, len(values)))
         lines = [
             'min',
             '10%',
             '25%',
             '50%',
             '75%',
             '80%',
             '85%',
             '90%',
             '95%',
             '99%',
             'max',
         ]
         for l in lines:
             fm.plain('%s: %s\n' % (l, stats[l]))
     fm.end()
 
 
 @command(
     b'perf::helper-mergecopies|perfhelper-mergecopies',
     formatteropts
     + [
         (b'r', b'revs', [], b'restrict search to these revisions'),
         (b'', b'timing', False, b'provides extra data (costly)'),
         (b'', b'stats', False, b'provides statistic about the measured data'),
     ],
 )
 def perfhelpermergecopies(ui, repo, revs=[], **opts):
     """find statistics about potential parameters for `perfmergecopies`
 
     This command find (base, p1, p2) triplet relevant for copytracing
     benchmarking in the context of a merge.  It reports values for some of the
     parameters that impact merge copy tracing time during merge.
 
     If `--timing` is set, rename detection is run and the associated timing
     will be reported. The extra details come at the cost of slower command
     execution.
 
     Since rename detection is only run once, other factors might easily
     affect the precision of the timing. However it should give a good
     approximation of which revision triplets are very costly.
     """
     opts = _byteskwargs(opts)
     fm = ui.formatter(b'perf', opts)
     dotiming = opts[b'timing']
     dostats = opts[b'stats']
 
     output_template = [
         ("base", "%(base)12s"),
         ("p1", "%(p1.node)12s"),
         ("p2", "%(p2.node)12s"),
         ("p1.nb-revs", "%(p1.nbrevs)12d"),
         ("p1.nb-files", "%(p1.nbmissingfiles)12d"),
         ("p1.renames", "%(p1.renamedfiles)12d"),
         ("p1.time", "%(p1.time)12.3f"),
         ("p2.nb-revs", "%(p2.nbrevs)12d"),
         ("p2.nb-files", "%(p2.nbmissingfiles)12d"),
         ("p2.renames", "%(p2.renamedfiles)12d"),
         ("p2.time", "%(p2.time)12.3f"),
         ("renames", "%(nbrenamedfiles)12d"),
         ("total.time", "%(time)12.3f"),
     ]
     if not dotiming:
         output_template = [
             i
             for i in output_template
             if not ('time' in i[0] or 'renames' in i[0])
         ]
     header_names = [h for (h, v) in output_template]
     output = ' '.join([v for (h, v) in output_template]) + '\n'
     header = ' '.join(['%12s'] * len(header_names)) + '\n'
     fm.plain(header % tuple(header_names))
 
     if not revs:
         revs = ['all()']
     revs = scmutil.revrange(repo, revs)
 
     if dostats:
         alldata = {
             'nbrevs': [],
             'nbmissingfiles': [],
         }
         if dotiming:
             alldata['parentnbrenames'] = []
             alldata['totalnbrenames'] = []
             alldata['parenttime'] = []
             alldata['totaltime'] = []
 
     roi = repo.revs('merge() and %ld', revs)
     for r in roi:
         ctx = repo[r]
         p1 = ctx.p1()
         p2 = ctx.p2()
         bases = repo.changelog._commonancestorsheads(p1.rev(), p2.rev())
         for b in bases:
             b = repo[b]
             p1missing = copies._computeforwardmissing(b, p1)
             p2missing = copies._computeforwardmissing(b, p2)
             data = {
                 b'base': b.hex(),
                 b'p1.node': p1.hex(),
                 b'p1.nbrevs': len(repo.revs('only(%d, %d)', p1.rev(), b.rev())),
                 b'p1.nbmissingfiles': len(p1missing),
                 b'p2.node': p2.hex(),
                 b'p2.nbrevs': len(repo.revs('only(%d, %d)', p2.rev(), b.rev())),
                 b'p2.nbmissingfiles': len(p2missing),
             }
             if dostats:
                 if p1missing:
                     alldata['nbrevs'].append(
                         (data['p1.nbrevs'], b.hex(), p1.hex())
                     )
                     alldata['nbmissingfiles'].append(
                         (data['p1.nbmissingfiles'], b.hex(), p1.hex())
                     )
                 if p2missing:
                     alldata['nbrevs'].append(
                         (data['p2.nbrevs'], b.hex(), p2.hex())
                     )
                     alldata['nbmissingfiles'].append(
                         (data['p2.nbmissingfiles'], b.hex(), p2.hex())
                     )
             if dotiming:
                 begin = util.timer()
                 mergedata = copies.mergecopies(repo, p1, p2, b)
                 end = util.timer()
                 # not very stable timing since we did only one run
                 data['time'] = end - begin
                 # mergedata contains five dicts: "copy", "movewithdir",
                 # "diverge", "renamedelete" and "dirmove".
                 # The first 4 are about renamed file so lets count that.
                 renames = len(mergedata[0])
                 renames += len(mergedata[1])
                 renames += len(mergedata[2])
                 renames += len(mergedata[3])
                 data['nbrenamedfiles'] = renames
                 begin = util.timer()
                 p1renames = copies.pathcopies(b, p1)
                 end = util.timer()
                 data['p1.time'] = end - begin
                 begin = util.timer()
                 p2renames = copies.pathcopies(b, p2)
                 end = util.timer()
                 data['p2.time'] = end - begin
                 data['p1.renamedfiles'] = len(p1renames)
                 data['p2.renamedfiles'] = len(p2renames)
 
                 if dostats:
                     if p1missing:
                         alldata['parentnbrenames'].append(
                             (data['p1.renamedfiles'], b.hex(), p1.hex())
                         )
                         alldata['parenttime'].append(
                             (data['p1.time'], b.hex(), p1.hex())
                         )
                     if p2missing:
                         alldata['parentnbrenames'].append(
                             (data['p2.renamedfiles'], b.hex(), p2.hex())
                         )
                         alldata['parenttime'].append(
                             (data['p2.time'], b.hex(), p2.hex())
                         )
                     if p1missing or p2missing:
                         alldata['totalnbrenames'].append(
                             (
                                 data['nbrenamedfiles'],
                                 b.hex(),
                                 p1.hex(),
                                 p2.hex(),
                             )
                         )
                         alldata['totaltime'].append(
                             (data['time'], b.hex(), p1.hex(), p2.hex())
                         )
             fm.startitem()
             fm.data(**data)
             # make node pretty for the human output
             out = data.copy()
             out['base'] = fm.hexfunc(b.node())
             out['p1.node'] = fm.hexfunc(p1.node())
             out['p2.node'] = fm.hexfunc(p2.node())
             fm.plain(output % out)
 
     fm.end()
     if dostats:
         # use a second formatter because the data are quite different, not sure
         # how it flies with the templater.
         entries = [
             ('nbrevs', 'number of revision covered'),
             ('nbmissingfiles', 'number of missing files at head'),
         ]
         if dotiming:
             entries.append(
                 ('parentnbrenames', 'rename from one parent to base')
             )
             entries.append(('totalnbrenames', 'total number of renames'))
             entries.append(('parenttime', 'time for one parent'))
             entries.append(('totaltime', 'time for both parents'))
         _displaystats(ui, opts, entries, alldata)
 
 
 @command(
     b'perf::helper-pathcopies|perfhelper-pathcopies',
     formatteropts
     + [
         (b'r', b'revs', [], b'restrict search to these revisions'),
         (b'', b'timing', False, b'provides extra data (costly)'),
         (b'', b'stats', False, b'provides statistic about the measured data'),
     ],
 )
 def perfhelperpathcopies(ui, repo, revs=[], **opts):
     """find statistic about potential parameters for the `perftracecopies`
 
     This command find source-destination pair relevant for copytracing testing.
     It report value for some of the parameters that impact copy tracing time.
 
     If `--timing` is set, rename detection is run and the associated timing
     will be reported. The extra details comes at the cost of a slower command
     execution.
 
     Since the rename detection is only run once, other factors might easily
     affect the precision of the timing. However it should give a good
     approximation of which revision pairs are very costly.
     """
     opts = _byteskwargs(opts)
     fm = ui.formatter(b'perf', opts)
     dotiming = opts[b'timing']
     dostats = opts[b'stats']
 
     if dotiming:
         header = '%12s %12s %12s %12s %12s %12s\n'
         output = (
             "%(source)12s %(destination)12s "
             "%(nbrevs)12d %(nbmissingfiles)12d "
             "%(nbrenamedfiles)12d %(time)18.5f\n"
         )
         header_names = (
             "source",
             "destination",
             "nb-revs",
             "nb-files",
             "nb-renames",
             "time",
         )
         fm.plain(header % header_names)
     else:
         header = '%12s %12s %12s %12s\n'
         output = (
             "%(source)12s %(destination)12s "
             "%(nbrevs)12d %(nbmissingfiles)12d\n"
         )
         fm.plain(header % ("source", "destination", "nb-revs", "nb-files"))
 
     if not revs:
         revs = ['all()']
     revs = scmutil.revrange(repo, revs)
 
     if dostats:
         alldata = {
             'nbrevs': [],
             'nbmissingfiles': [],
         }
         if dotiming:
             alldata['nbrenames'] = []
             alldata['time'] = []
 
     roi = repo.revs('merge() and %ld', revs)
     for r in roi:
         ctx = repo[r]
         p1 = ctx.p1().rev()
         p2 = ctx.p2().rev()
         bases = repo.changelog._commonancestorsheads(p1, p2)
         for p in (p1, p2):
             for b in bases:
                 base = repo[b]
                 parent = repo[p]
                 missing = copies._computeforwardmissing(base, parent)
                 if not missing:
                     continue
                 data = {
                     b'source': base.hex(),
                     b'destination': parent.hex(),
                     b'nbrevs': len(repo.revs('only(%d, %d)', p, b)),
                     b'nbmissingfiles': len(missing),
                 }
                 if dostats:
                     alldata['nbrevs'].append(
                         (
                             data['nbrevs'],
                             base.hex(),
                             parent.hex(),
                         )
                     )
                     alldata['nbmissingfiles'].append(
                         (
                             data['nbmissingfiles'],
                             base.hex(),
                             parent.hex(),
                         )
                     )
                 if dotiming:
                     begin = util.timer()
                     renames = copies.pathcopies(base, parent)
                     end = util.timer()
                     # not very stable timing since we did only one run
                     data['time'] = end - begin
                     data['nbrenamedfiles'] = len(renames)
                     if dostats:
                         alldata['time'].append(
                             (
                                 data['time'],
                                 base.hex(),
                                 parent.hex(),
                             )
                         )
                         alldata['nbrenames'].append(
                             (
                                 data['nbrenamedfiles'],
                                 base.hex(),
                                 parent.hex(),
                             )
                         )
                 fm.startitem()
                 fm.data(**data)
                 out = data.copy()
                 out['source'] = fm.hexfunc(base.node())
                 out['destination'] = fm.hexfunc(parent.node())
                 fm.plain(output % out)
 
     fm.end()
     if dostats:
         entries = [
             ('nbrevs', 'number of revision covered'),
             ('nbmissingfiles', 'number of missing files at head'),
         ]
         if dotiming:
             entries.append(('nbrenames', 'renamed files'))
             entries.append(('time', 'time'))
         _displaystats(ui, opts, entries, alldata)
 
 
 @command(b'perf::cca|perfcca', formatteropts)
 def perfcca(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     timer(lambda: scmutil.casecollisionauditor(ui, False, repo.dirstate))
     fm.end()
 
 
 @command(b'perf::fncacheload|perffncacheload', formatteropts)
 def perffncacheload(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     s = repo.store
 
     def d():
         s.fncache._load()
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::fncachewrite|perffncachewrite', formatteropts)
 def perffncachewrite(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     s = repo.store
     lock = repo.lock()
     s.fncache._load()
     tr = repo.transaction(b'perffncachewrite')
     tr.addbackup(b'fncache')
 
     def d():
         s.fncache._dirty = True
         s.fncache.write(tr)
 
     timer(d)
     tr.close()
     lock.release()
     fm.end()
 
 
 @command(b'perf::fncacheencode|perffncacheencode', formatteropts)
 def perffncacheencode(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     s = repo.store
     s.fncache._load()
 
     def d():
         for p in s.fncache.entries:
             s.encode(p)
 
     timer(d)
     fm.end()
 
 
 def _bdiffworker(q, blocks, xdiff, ready, done):
     while not done.is_set():
         pair = q.get()
         while pair is not None:
             if xdiff:
                 mdiff.bdiff.xdiffblocks(*pair)
             elif blocks:
                 mdiff.bdiff.blocks(*pair)
             else:
                 mdiff.textdiff(*pair)
             q.task_done()
             pair = q.get()
         q.task_done()  # for the None one
         with ready:
             ready.wait()
 
 
 def _manifestrevision(repo, mnode):
     ml = repo.manifestlog
 
     if util.safehasattr(ml, b'getstorage'):
         store = ml.getstorage(b'')
     else:
         store = ml._revlog
 
     return store.revision(mnode)
 
 
 @command(
     b'perf::bdiff|perfbdiff',
     revlogopts
     + formatteropts
     + [
         (
             b'',
             b'count',
             1,
             b'number of revisions to test (when using --startrev)',
         ),
         (b'', b'alldata', False, b'test bdiffs for all associated revisions'),
         (b'', b'threads', 0, b'number of thread to use (disable with 0)'),
         (b'', b'blocks', False, b'test computing diffs into blocks'),
         (b'', b'xdiff', False, b'use xdiff algorithm'),
     ],
     b'-c|-m|FILE REV',
 )
 def perfbdiff(ui, repo, file_, rev=None, count=None, threads=0, **opts):
     """benchmark a bdiff between revisions
 
     By default, benchmark a bdiff between its delta parent and itself.
 
     With ``--count``, benchmark bdiffs between delta parents and self for N
     revisions starting at the specified revision.
 
     With ``--alldata``, assume the requested revision is a changeset and
     measure bdiffs for all changes related to that changeset (manifest
     and filelogs).
     """
     opts = _byteskwargs(opts)
 
     if opts[b'xdiff'] and not opts[b'blocks']:
         raise error.CommandError(b'perfbdiff', b'--xdiff requires --blocks')
 
     if opts[b'alldata']:
         opts[b'changelog'] = True
 
     if opts.get(b'changelog') or opts.get(b'manifest'):
         file_, rev = None, file_
     elif rev is None:
         raise error.CommandError(b'perfbdiff', b'invalid arguments')
 
     blocks = opts[b'blocks']
     xdiff = opts[b'xdiff']
     textpairs = []
 
     r = cmdutil.openrevlog(repo, b'perfbdiff', file_, opts)
 
     startrev = r.rev(r.lookup(rev))
     for rev in range(startrev, min(startrev + count, len(r) - 1)):
         if opts[b'alldata']:
             # Load revisions associated with changeset.
             ctx = repo[rev]
             mtext = _manifestrevision(repo, ctx.manifestnode())
             for pctx in ctx.parents():
                 pman = _manifestrevision(repo, pctx.manifestnode())
                 textpairs.append((pman, mtext))
 
             # Load filelog revisions by iterating manifest delta.
             man = ctx.manifest()
             pman = ctx.p1().manifest()
             for filename, change in pman.diff(man).items():
                 fctx = repo.file(filename)
                 f1 = fctx.revision(change[0][0] or -1)
                 f2 = fctx.revision(change[1][0] or -1)
                 textpairs.append((f1, f2))
         else:
             dp = r.deltaparent(rev)
             textpairs.append((r.revision(dp), r.revision(rev)))
 
     withthreads = threads > 0
     if not withthreads:
 
         def d():
             for pair in textpairs:
                 if xdiff:
                     mdiff.bdiff.xdiffblocks(*pair)
                 elif blocks:
                     mdiff.bdiff.blocks(*pair)
                 else:
                     mdiff.textdiff(*pair)
 
     else:
         q = queue()
         for i in _xrange(threads):
             q.put(None)
         ready = threading.Condition()
         done = threading.Event()
         for i in _xrange(threads):
             threading.Thread(
                 target=_bdiffworker, args=(q, blocks, xdiff, ready, done)
             ).start()
         q.join()
 
         def d():
             for pair in textpairs:
                 q.put(pair)
             for i in _xrange(threads):
                 q.put(None)
             with ready:
                 ready.notify_all()
             q.join()
 
     timer, fm = gettimer(ui, opts)
     timer(d)
     fm.end()
 
     if withthreads:
         done.set()
         for i in _xrange(threads):
             q.put(None)
         with ready:
             ready.notify_all()
 
 
 @command(
     b'perf::unidiff|perfunidiff',
     revlogopts
     + formatteropts
     + [
         (
             b'',
             b'count',
             1,
             b'number of revisions to test (when using --startrev)',
         ),
         (b'', b'alldata', False, b'test unidiffs for all associated revisions'),
     ],
     b'-c|-m|FILE REV',
 )
 def perfunidiff(ui, repo, file_, rev=None, count=None, **opts):
     """benchmark a unified diff between revisions
 
     This doesn't include any copy tracing - it's just a unified diff
     of the texts.
 
     By default, benchmark a diff between its delta parent and itself.
 
     With ``--count``, benchmark diffs between delta parents and self for N
     revisions starting at the specified revision.
 
     With ``--alldata``, assume the requested revision is a changeset and
     measure diffs for all changes related to that changeset (manifest
     and filelogs).
     """
     opts = _byteskwargs(opts)
     if opts[b'alldata']:
         opts[b'changelog'] = True
 
     if opts.get(b'changelog') or opts.get(b'manifest'):
         file_, rev = None, file_
     elif rev is None:
         raise error.CommandError(b'perfunidiff', b'invalid arguments')
 
     textpairs = []
 
     r = cmdutil.openrevlog(repo, b'perfunidiff', file_, opts)
 
     startrev = r.rev(r.lookup(rev))
     for rev in range(startrev, min(startrev + count, len(r) - 1)):
         if opts[b'alldata']:
             # Load revisions associated with changeset.
             ctx = repo[rev]
             mtext = _manifestrevision(repo, ctx.manifestnode())
             for pctx in ctx.parents():
                 pman = _manifestrevision(repo, pctx.manifestnode())
                 textpairs.append((pman, mtext))
 
             # Load filelog revisions by iterating manifest delta.
             man = ctx.manifest()
             pman = ctx.p1().manifest()
             for filename, change in pman.diff(man).items():
                 fctx = repo.file(filename)
                 f1 = fctx.revision(change[0][0] or -1)
                 f2 = fctx.revision(change[1][0] or -1)
                 textpairs.append((f1, f2))
         else:
             dp = r.deltaparent(rev)
             textpairs.append((r.revision(dp), r.revision(rev)))
 
     def d():
         for left, right in textpairs:
             # The date strings don't matter, so we pass empty strings.
             headerlines, hunks = mdiff.unidiff(
                 left, b'', right, b'', b'left', b'right', binary=False
             )
             # consume iterators in roughly the way patch.py does
             b'\n'.join(headerlines)
             b''.join(sum((list(hlines) for hrange, hlines in hunks), []))
 
     timer, fm = gettimer(ui, opts)
     timer(d)
     fm.end()
 
 
 @command(b'perf::diffwd|perfdiffwd', formatteropts)
 def perfdiffwd(ui, repo, **opts):
     """Profile diff of working directory changes"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     options = {
         'w': 'ignore_all_space',
         'b': 'ignore_space_change',
         'B': 'ignore_blank_lines',
     }
 
     for diffopt in ('', 'w', 'b', 'B', 'wB'):
         opts = {options[c]: b'1' for c in diffopt}
 
         def d():
             ui.pushbuffer()
             commands.diff(ui, repo, **opts)
             ui.popbuffer()
 
         diffopt = diffopt.encode('ascii')
         title = b'diffopts: %s' % (diffopt and (b'-' + diffopt) or b'none')
         timer(d, title=title)
     fm.end()
 
 
 @command(
     b'perf::revlogindex|perfrevlogindex',
     revlogopts + formatteropts,
     b'-c|-m|FILE',
 )
 def perfrevlogindex(ui, repo, file_=None, **opts):
     """Benchmark operations against a revlog index.
 
     This tests constructing a revlog instance, reading index data,
     parsing index data, and performing various operations related to
     index data.
     """
 
     opts = _byteskwargs(opts)
 
     rl = cmdutil.openrevlog(repo, b'perfrevlogindex', file_, opts)
 
     opener = getattr(rl, 'opener')  # trick linter
-    indexfile = rl.indexfile
+    # compat with hg <= 5.8
+    radix = getattr(rl, 'radix', None)
+    indexfile = getattr(rl, '_indexfile', None)
+    if indexfile is None:
+        # compatibility with <= hg-5.8
+        indexfile = getattr(rl, 'indexfile')
     data = opener.read(indexfile)
 
     header = struct.unpack(b'>I', data[0:4])[0]
     version = header & 0xFFFF
     if version == 1:
-        revlogio = revlog.revlogio()
         inline = header & (1 << 16)
     else:
         raise error.Abort(b'unsupported revlog version: %d' % version)
 
+    parse_index_v1 = getattr(mercurial.revlog, 'parse_index_v1', None)
+    if parse_index_v1 is None:
+        parse_index_v1 = mercurial.revlog.revlogio().parseindex
+
     rllen = len(rl)
 
     node0 = rl.node(0)
     node25 = rl.node(rllen // 4)
     node50 = rl.node(rllen // 2)
     node75 = rl.node(rllen // 4 * 3)
     node100 = rl.node(rllen - 1)
 
     allrevs = range(rllen)
     allrevsrev = list(reversed(allrevs))
     allnodes = [rl.node(rev) for rev in range(rllen)]
     allnodesrev = list(reversed(allnodes))
 
     def constructor():
-        revlog.revlog(opener, indexfile)
+        if radix is not None:
+            revlog(opener, radix=radix)
+        else:
+            # hg <= 5.8
+            revlog(opener, indexfile=indexfile)
 
     def read():
         with opener(indexfile) as fh:
             fh.read()
 
     def parseindex():
-        revlogio.parseindex(data, inline)
+        parse_index_v1(data, inline)
 
     def getentry(revornode):
-        index = revlogio.parseindex(data, inline)[0]
+        index = parse_index_v1(data, inline)[0]
         index[revornode]
 
     def getentries(revs, count=1):
-        index = revlogio.parseindex(data, inline)[0]
+        index = parse_index_v1(data, inline)[0]
 
         for i in range(count):
             for rev in revs:
                 index[rev]
 
     def resolvenode(node):
-        index = revlogio.parseindex(data, inline)[0]
+        index = parse_index_v1(data, inline)[0]
         rev = getattr(index, 'rev', None)
         if rev is None:
-            nodemap = getattr(
-                revlogio.parseindex(data, inline)[0], 'nodemap', None
-            )
+            nodemap = getattr(parse_index_v1(data, inline)[0], 'nodemap', None)
             # This only works for the C code.
             if nodemap is None:
                 return
             rev = nodemap.__getitem__
 
         try:
             rev(node)
         except error.RevlogError:
             pass
 
     def resolvenodes(nodes, count=1):
-        index = revlogio.parseindex(data, inline)[0]
+        index = parse_index_v1(data, inline)[0]
         rev = getattr(index, 'rev', None)
         if rev is None:
-            nodemap = getattr(
-                revlogio.parseindex(data, inline)[0], 'nodemap', None
-            )
+            nodemap = getattr(parse_index_v1(data, inline)[0], 'nodemap', None)
             # This only works for the C code.
             if nodemap is None:
                 return
             rev = nodemap.__getitem__
 
         for i in range(count):
             for node in nodes:
                 try:
                     rev(node)
                 except error.RevlogError:
                     pass
 
     benches = [
         (constructor, b'revlog constructor'),
         (read, b'read'),
         (parseindex, b'create index object'),
         (lambda: getentry(0), b'retrieve index entry for rev 0'),
         (lambda: resolvenode(b'a' * 20), b'look up missing node'),
         (lambda: resolvenode(node0), b'look up node at rev 0'),
         (lambda: resolvenode(node25), b'look up node at 1/4 len'),
         (lambda: resolvenode(node50), b'look up node at 1/2 len'),
         (lambda: resolvenode(node75), b'look up node at 3/4 len'),
         (lambda: resolvenode(node100), b'look up node at tip'),
         # 2x variation is to measure caching impact.
         (lambda: resolvenodes(allnodes), b'look up all nodes (forward)'),
         (lambda: resolvenodes(allnodes, 2), b'look up all nodes 2x (forward)'),
         (lambda: resolvenodes(allnodesrev), b'look up all nodes (reverse)'),
         (
             lambda: resolvenodes(allnodesrev, 2),
             b'look up all nodes 2x (reverse)',
         ),
         (lambda: getentries(allrevs), b'retrieve all index entries (forward)'),
         (
             lambda: getentries(allrevs, 2),
             b'retrieve all index entries 2x (forward)',
         ),
         (
             lambda: getentries(allrevsrev),
             b'retrieve all index entries (reverse)',
         ),
         (
             lambda: getentries(allrevsrev, 2),
             b'retrieve all index entries 2x (reverse)',
         ),
     ]
 
     for fn, title in benches:
         timer, fm = gettimer(ui, opts)
         timer(fn, title=title)
         fm.end()
 
 
 @command(
     b'perf::revlogrevisions|perfrevlogrevisions',
     revlogopts
     + formatteropts
     + [
         (b'd', b'dist', 100, b'distance between the revisions'),
         (b's', b'startrev', 0, b'revision to start reading at'),
         (b'', b'reverse', False, b'read in reverse'),
     ],
     b'-c|-m|FILE',
 )
 def perfrevlogrevisions(
     ui, repo, file_=None, startrev=0, reverse=False, **opts
 ):
     """Benchmark reading a series of revisions from a revlog.
 
     By default, we read every ``-d/--dist`` revision from 0 to tip of
     the specified revlog.
 
     The start revision can be defined via ``-s/--startrev``.
     """
     opts = _byteskwargs(opts)
 
     rl = cmdutil.openrevlog(repo, b'perfrevlogrevisions', file_, opts)
     rllen = getlen(ui)(rl)
 
     if startrev < 0:
         startrev = rllen + startrev
 
     def d():
         rl.clearcaches()
 
         beginrev = startrev
         endrev = rllen
         dist = opts[b'dist']
 
         if reverse:
             beginrev, endrev = endrev - 1, beginrev - 1
             dist = -1 * dist
 
         for x in _xrange(beginrev, endrev, dist):
             # Old revisions don't support passing int.
             n = rl.node(x)
             rl.revision(n)
 
     timer, fm = gettimer(ui, opts)
     timer(d)
     fm.end()
 
 
 @command(
     b'perf::revlogwrite|perfrevlogwrite',
     revlogopts
     + formatteropts
     + [
         (b's', b'startrev', 1000, b'revision to start writing at'),
         (b'', b'stoprev', -1, b'last revision to write'),
         (b'', b'count', 3, b'number of passes to perform'),
         (b'', b'details', False, b'print timing for every revisions tested'),
         (b'', b'source', b'full', b'the kind of data feed in the revlog'),
         (b'', b'lazydeltabase', True, b'try the provided delta first'),
         (b'', b'clear-caches', True, b'clear revlog cache between calls'),
     ],
     b'-c|-m|FILE',
 )
 def perfrevlogwrite(ui, repo, file_=None, startrev=1000, stoprev=-1, **opts):
     """Benchmark writing a series of revisions to a revlog.
 
     Possible source values are:
     * `full`: add from a full text (default).
     * `parent-1`: add from a delta to the first parent
     * `parent-2`: add from a delta to the second parent if it exists
                   (use a delta from the first parent otherwise)
     * `parent-smallest`: add from the smallest delta (either p1 or p2)
     * `storage`: add from the existing precomputed deltas
 
     Note: This performance command measures performance in a custom way. As a
     result some of the global configuration of the 'perf' command does not
     apply to it:
 
     * ``pre-run``: disabled
 
     * ``profile-benchmark``: disabled
 
     * ``run-limits``: disabled use --count instead
     """
     opts = _byteskwargs(opts)
 
     rl = cmdutil.openrevlog(repo, b'perfrevlogwrite', file_, opts)
     rllen = getlen(ui)(rl)
     if startrev < 0:
         startrev = rllen + startrev
     if stoprev < 0:
         stoprev = rllen + stoprev
 
     lazydeltabase = opts['lazydeltabase']
     source = opts['source']
     clearcaches = opts['clear_caches']
     validsource = (
         b'full',
         b'parent-1',
         b'parent-2',
         b'parent-smallest',
         b'storage',
     )
     if source not in validsource:
         raise error.Abort('invalid source type: %s' % source)
 
     ### actually gather results
     count = opts['count']
     if count <= 0:
         raise error.Abort('invalide run count: %d' % count)
     allresults = []
     for c in range(count):
         timing = _timeonewrite(
             ui,
             rl,
             source,
             startrev,
             stoprev,
             c + 1,
             lazydeltabase=lazydeltabase,
             clearcaches=clearcaches,
         )
         allresults.append(timing)
 
     ### consolidate the results in a single list
     results = []
     for idx, (rev, t) in enumerate(allresults[0]):
         ts = [t]
         for other in allresults[1:]:
             orev, ot = other[idx]
             assert orev == rev
             ts.append(ot)
         results.append((rev, ts))
     resultcount = len(results)
 
     ### Compute and display relevant statistics
 
     # get a formatter
     fm = ui.formatter(b'perf', opts)
     displayall = ui.configbool(b"perf", b"all-timing", False)
 
     # print individual details if requested
     if opts['details']:
         for idx, item in enumerate(results, 1):
             rev, data = item
             title = 'revisions #%d of %d, rev %d' % (idx, resultcount, rev)
             formatone(fm, data, title=title, displayall=displayall)
 
     # sorts results by median time
     results.sort(key=lambda x: sorted(x[1])[len(x[1]) // 2])
     # list of (name, index) to display)
     relevants = [
         ("min", 0),
         ("10%", resultcount * 10 // 100),
         ("25%", resultcount * 25 // 100),
         ("50%", resultcount * 70 // 100),
         ("75%", resultcount * 75 // 100),
         ("90%", resultcount * 90 // 100),
         ("95%", resultcount * 95 // 100),
         ("99%", resultcount * 99 // 100),
         ("99.9%", resultcount * 999 // 1000),
         ("99.99%", resultcount * 9999 // 10000),
         ("99.999%", resultcount * 99999 // 100000),
         ("max", -1),
     ]
     if not ui.quiet:
         for name, idx in relevants:
             data = results[idx]
             title = '%s of %d, rev %d' % (name, resultcount, data[0])
             formatone(fm, data[1], title=title, displayall=displayall)
 
     # XXX summing that many float will not be very precise, we ignore this fact
     # for now
     totaltime = []
     for item in allresults:
         totaltime.append(
             (
                 sum(x[1][0] for x in item),
                 sum(x[1][1] for x in item),
                 sum(x[1][2] for x in item),
             )
         )
     formatone(
         fm,
         totaltime,
         title="total time (%d revs)" % resultcount,
         displayall=displayall,
     )
     fm.end()
 
 
 class _faketr(object):
     def add(s, x, y, z=None):
         return None
 
 
 def _timeonewrite(
     ui,
     orig,
     source,
     startrev,
     stoprev,
     runidx=None,
     lazydeltabase=True,
     clearcaches=True,
 ):
     timings = []
     tr = _faketr()
     with _temprevlog(ui, orig, startrev) as dest:
         dest._lazydeltabase = lazydeltabase
         revs = list(orig.revs(startrev, stoprev))
         total = len(revs)
         topic = 'adding'
         if runidx is not None:
             topic += ' (run #%d)' % runidx
         # Support both old and new progress API
         if util.safehasattr(ui, 'makeprogress'):
             progress = ui.makeprogress(topic, unit='revs', total=total)
 
             def updateprogress(pos):
                 progress.update(pos)
 
             def completeprogress():
                 progress.complete()
 
         else:
 
             def updateprogress(pos):
                 ui.progress(topic, pos, unit='revs', total=total)
 
             def completeprogress():
                 ui.progress(topic, None, unit='revs', total=total)
 
         for idx, rev in enumerate(revs):
             updateprogress(idx)
             addargs, addkwargs = _getrevisionseed(orig, rev, tr, source)
             if clearcaches:
                 dest.index.clearcaches()
                 dest.clearcaches()
             with timeone() as r:
                 dest.addrawrevision(*addargs, **addkwargs)
             timings.append((rev, r[0]))
         updateprogress(total)
         completeprogress()
     return timings
 
 
 def _getrevisionseed(orig, rev, tr, source):
     from mercurial.node import nullid
 
     linkrev = orig.linkrev(rev)
     node = orig.node(rev)
     p1, p2 = orig.parents(node)
     flags = orig.flags(rev)
     cachedelta = None
     text = None
 
     if source == b'full':
         text = orig.revision(rev)
     elif source == b'parent-1':
         baserev = orig.rev(p1)
         cachedelta = (baserev, orig.revdiff(p1, rev))
     elif source == b'parent-2':
         parent = p2
         if p2 == nullid:
             parent = p1
         baserev = orig.rev(parent)
         cachedelta = (baserev, orig.revdiff(parent, rev))
     elif source == b'parent-smallest':
         p1diff = orig.revdiff(p1, rev)
         parent = p1
         diff = p1diff
         if p2 != nullid:
             p2diff = orig.revdiff(p2, rev)
             if len(p1diff) > len(p2diff):
                 parent = p2
                 diff = p2diff
         baserev = orig.rev(parent)
         cachedelta = (baserev, diff)
     elif source == b'storage':
         baserev = orig.deltaparent(rev)
         cachedelta = (baserev, orig.revdiff(orig.node(baserev), rev))
 
     return (
         (text, tr, linkrev, p1, p2),
         {'node': node, 'flags': flags, 'cachedelta': cachedelta},
     )
 
 
 @contextlib.contextmanager
 def _temprevlog(ui, orig, truncaterev):
     from mercurial import vfs as vfsmod
 
     if orig._inline:
         raise error.Abort('not supporting inline revlog (yet)')
     revlogkwargs = {}
     k = 'upperboundcomp'
     if util.safehasattr(orig, k):
         revlogkwargs[k] = getattr(orig, k)
 
-    origindexpath = orig.opener.join(orig.indexfile)
-    origdatapath = orig.opener.join(orig.datafile)
-    indexname = 'revlog.i'
-    dataname = 'revlog.d'
+    indexfile = getattr(orig, '_indexfile', None)
+    if indexfile is None:
+        # compatibility with <= hg-5.8
+        indexfile = getattr(orig, 'indexfile')
+    origindexpath = orig.opener.join(indexfile)
+
+    datafile = getattr(orig, '_datafile', getattr(orig, 'datafile'))
+    origdatapath = orig.opener.join(datafile)
+    radix = b'revlog'
+    indexname = b'revlog.i'
+    dataname = b'revlog.d'
 
     tmpdir = tempfile.mkdtemp(prefix='tmp-hgperf-')
     try:
         # copy the data file in a temporary directory
         ui.debug('copying data in %s\n' % tmpdir)
         destindexpath = os.path.join(tmpdir, 'revlog.i')
         destdatapath = os.path.join(tmpdir, 'revlog.d')
         shutil.copyfile(origindexpath, destindexpath)
         shutil.copyfile(origdatapath, destdatapath)
 
         # remove the data we want to add again
         ui.debug('truncating data to be rewritten\n')
         with open(destindexpath, 'ab') as index:
             index.seek(0)
             index.truncate(truncaterev * orig._io.size)
         with open(destdatapath, 'ab') as data:
             data.seek(0)
             data.truncate(orig.start(truncaterev))
 
         # instantiate a new revlog from the temporary copy
         ui.debug('truncating adding to be rewritten\n')
         vfs = vfsmod.vfs(tmpdir)
         vfs.options = getattr(orig.opener, 'options', None)
 
-        dest = revlog.revlog(
-            vfs, indexfile=indexname, datafile=dataname, **revlogkwargs
-        )
+        try:
+            dest = revlog(vfs, radix=radix, **revlogkwargs)
+        except TypeError:
+            dest = revlog(
+                vfs, indexfile=indexname, datafile=dataname, **revlogkwargs
+            )
         if dest._inline:
             raise error.Abort('not supporting inline revlog (yet)')
         # make sure internals are initialized
         dest.revision(len(dest) - 1)
         yield dest
         del dest, vfs
     finally:
         shutil.rmtree(tmpdir, True)
 
 
 @command(
     b'perf::revlogchunks|perfrevlogchunks',
     revlogopts
     + formatteropts
     + [
         (b'e', b'engines', b'', b'compression engines to use'),
         (b's', b'startrev', 0, b'revision to start at'),
     ],
     b'-c|-m|FILE',
 )
 def perfrevlogchunks(ui, repo, file_=None, engines=None, startrev=0, **opts):
     """Benchmark operations on revlog chunks.
 
     Logically, each revlog is a collection of fulltext revisions. However,
     stored within each revlog are "chunks" of possibly compressed data. This
     data needs to be read and decompressed or compressed and written.
 
     This command measures the time it takes to read+decompress and recompress
     chunks in a revlog. It effectively isolates I/O and compression performance.
     For measurements of higher-level operations like resolving revisions,
     see ``perfrevlogrevisions`` and ``perfrevlogrevision``.
     """
     opts = _byteskwargs(opts)
 
     rl = cmdutil.openrevlog(repo, b'perfrevlogchunks', file_, opts)
 
     # _chunkraw was renamed to _getsegmentforrevs.
     try:
         segmentforrevs = rl._getsegmentforrevs
     except AttributeError:
         segmentforrevs = rl._chunkraw
 
     # Verify engines argument.
     if engines:
         engines = {e.strip() for e in engines.split(b',')}
         for engine in engines:
             try:
                 util.compressionengines[engine]
             except KeyError:
                 raise error.Abort(b'unknown compression engine: %s' % engine)
     else:
         engines = []
         for e in util.compengines:
             engine = util.compengines[e]
             try:
                 if engine.available():
                     engine.revlogcompressor().compress(b'dummy')
                     engines.append(e)
             except NotImplementedError:
                 pass
 
     revs = list(rl.revs(startrev, len(rl) - 1))
 
     def rlfh(rl):
         if rl._inline:
-            return getsvfs(repo)(rl.indexfile)
+            indexfile = getattr(rl, '_indexfile', None)
+            if indexfile is None:
+                # compatibility with <= hg-5.8
+                indexfile = getattr(rl, 'indexfile')
+            return getsvfs(repo)(indexfile)
         else:
-            return getsvfs(repo)(rl.datafile)
+            datafile = getattr(rl, 'datafile', getattr(rl, 'datafile'))
+            return getsvfs(repo)(datafile)
 
     def doread():
         rl.clearcaches()
         for rev in revs:
             segmentforrevs(rev, rev)
 
     def doreadcachedfh():
         rl.clearcaches()
         fh = rlfh(rl)
         for rev in revs:
             segmentforrevs(rev, rev, df=fh)
 
     def doreadbatch():
         rl.clearcaches()
         segmentforrevs(revs[0], revs[-1])
 
     def doreadbatchcachedfh():
         rl.clearcaches()
         fh = rlfh(rl)
         segmentforrevs(revs[0], revs[-1], df=fh)
 
     def dochunk():
         rl.clearcaches()
         fh = rlfh(rl)
         for rev in revs:
             rl._chunk(rev, df=fh)
 
     chunks = [None]
 
     def dochunkbatch():
         rl.clearcaches()
         fh = rlfh(rl)
         # Save chunks as a side-effect.
         chunks[0] = rl._chunks(revs, df=fh)
 
     def docompress(compressor):
         rl.clearcaches()
 
         try:
             # Swap in the requested compression engine.
             oldcompressor = rl._compressor
             rl._compressor = compressor
             for chunk in chunks[0]:
                 rl.compress(chunk)
         finally:
             rl._compressor = oldcompressor
 
     benches = [
         (lambda: doread(), b'read'),
         (lambda: doreadcachedfh(), b'read w/ reused fd'),
         (lambda: doreadbatch(), b'read batch'),
         (lambda: doreadbatchcachedfh(), b'read batch w/ reused fd'),
         (lambda: dochunk(), b'chunk'),
         (lambda: dochunkbatch(), b'chunk batch'),
     ]
 
     for engine in sorted(engines):
         compressor = util.compengines[engine].revlogcompressor()
         benches.append(
             (
                 functools.partial(docompress, compressor),
                 b'compress w/ %s' % engine,
             )
         )
 
     for fn, title in benches:
         timer, fm = gettimer(ui, opts)
         timer(fn, title=title)
         fm.end()
 
 
 @command(
     b'perf::revlogrevision|perfrevlogrevision',
     revlogopts
     + formatteropts
     + [(b'', b'cache', False, b'use caches instead of clearing')],
     b'-c|-m|FILE REV',
 )
 def perfrevlogrevision(ui, repo, file_, rev=None, cache=None, **opts):
     """Benchmark obtaining a revlog revision.
 
     Obtaining a revlog revision consists of roughly the following steps:
 
     1. Compute the delta chain
     2. Slice the delta chain if applicable
     3. Obtain the raw chunks for that delta chain
     4. Decompress each raw chunk
     5. Apply binary patches to obtain fulltext
     6. Verify hash of fulltext
 
     This command measures the time spent in each of these phases.
     """
     opts = _byteskwargs(opts)
 
     if opts.get(b'changelog') or opts.get(b'manifest'):
         file_, rev = None, file_
     elif rev is None:
         raise error.CommandError(b'perfrevlogrevision', b'invalid arguments')
 
     r = cmdutil.openrevlog(repo, b'perfrevlogrevision', file_, opts)
 
     # _chunkraw was renamed to _getsegmentforrevs.
     try:
         segmentforrevs = r._getsegmentforrevs
     except AttributeError:
         segmentforrevs = r._chunkraw
 
     node = r.lookup(rev)
     rev = r.rev(node)
 
     def getrawchunks(data, chain):
         start = r.start
         length = r.length
         inline = r._inline
         try:
             iosize = r.index.entry_size
         except AttributeError:
             iosize = r._io.size
         buffer = util.buffer
 
         chunks = []
         ladd = chunks.append
         for idx, item in enumerate(chain):
             offset = start(item[0])
             bits = data[idx]
             for rev in item:
                 chunkstart = start(rev)
                 if inline:
                     chunkstart += (rev + 1) * iosize
                 chunklength = length(rev)
                 ladd(buffer(bits, chunkstart - offset, chunklength))
 
         return chunks
 
     def dodeltachain(rev):
         if not cache:
             r.clearcaches()
         r._deltachain(rev)
 
     def doread(chain):
         if not cache:
             r.clearcaches()
         for item in slicedchain:
             segmentforrevs(item[0], item[-1])
 
     def doslice(r, chain, size):
         for s in slicechunk(r, chain, targetsize=size):
             pass
 
     def dorawchunks(data, chain):
         if not cache:
             r.clearcaches()
         getrawchunks(data, chain)
 
     def dodecompress(chunks):
         decomp = r.decompress
         for chunk in chunks:
             decomp(chunk)
 
     def dopatch(text, bins):
         if not cache:
             r.clearcaches()
         mdiff.patches(text, bins)
 
     def dohash(text):
         if not cache:
             r.clearcaches()
         r.checkhash(text, node, rev=rev)
 
     def dorevision():
         if not cache:
             r.clearcaches()
         r.revision(node)
 
     try:
         from mercurial.revlogutils.deltas import slicechunk
     except ImportError:
         slicechunk = getattr(revlog, '_slicechunk', None)
 
     size = r.length(rev)
     chain = r._deltachain(rev)[0]
     if not getattr(r, '_withsparseread', False):
         slicedchain = (chain,)
     else:
         slicedchain = tuple(slicechunk(r, chain, targetsize=size))
     data = [segmentforrevs(seg[0], seg[-1])[1] for seg in slicedchain]
     rawchunks = getrawchunks(data, slicedchain)
     bins = r._chunks(chain)
     text = bytes(bins[0])
     bins = bins[1:]
     text = mdiff.patches(text, bins)
 
     benches = [
         (lambda: dorevision(), b'full'),
         (lambda: dodeltachain(rev), b'deltachain'),
         (lambda: doread(chain), b'read'),
     ]
 
     if getattr(r, '_withsparseread', False):
         slicing = (lambda: doslice(r, chain, size), b'slice-sparse-chain')
         benches.append(slicing)
 
     benches.extend(
         [
             (lambda: dorawchunks(data, slicedchain), b'rawchunks'),
             (lambda: dodecompress(rawchunks), b'decompress'),
             (lambda: dopatch(text, bins), b'patch'),
             (lambda: dohash(text), b'hash'),
         ]
     )
 
     timer, fm = gettimer(ui, opts)
     for fn, title in benches:
         timer(fn, title=title)
     fm.end()
 
 
 @command(
     b'perf::revset|perfrevset',
     [
         (b'C', b'clear', False, b'clear volatile cache between each call.'),
         (b'', b'contexts', False, b'obtain changectx for each revision'),
     ]
     + formatteropts,
     b"REVSET",
 )
 def perfrevset(ui, repo, expr, clear=False, contexts=False, **opts):
     """benchmark the execution time of a revset
 
     Use the --clean option if need to evaluate the impact of build volatile
     revisions set cache on the revset execution. Volatile cache hold filtered
     and obsolete related cache."""
     opts = _byteskwargs(opts)
 
     timer, fm = gettimer(ui, opts)
 
     def d():
         if clear:
             repo.invalidatevolatilesets()
         if contexts:
             for ctx in repo.set(expr):
                 pass
         else:
             for r in repo.revs(expr):
                 pass
 
     timer(d)
     fm.end()
 
 
 @command(
     b'perf::volatilesets|perfvolatilesets',
     [
         (b'', b'clear-obsstore', False, b'drop obsstore between each call.'),
     ]
     + formatteropts,
 )
 def perfvolatilesets(ui, repo, *names, **opts):
     """benchmark the computation of various volatile set
 
     Volatile set computes element related to filtering and obsolescence."""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     repo = repo.unfiltered()
 
     def getobs(name):
         def d():
             repo.invalidatevolatilesets()
             if opts[b'clear_obsstore']:
                 clearfilecache(repo, b'obsstore')
             obsolete.getrevs(repo, name)
 
         return d
 
     allobs = sorted(obsolete.cachefuncs)
     if names:
         allobs = [n for n in allobs if n in names]
 
     for name in allobs:
         timer(getobs(name), title=name)
 
     def getfiltered(name):
         def d():
             repo.invalidatevolatilesets()
             if opts[b'clear_obsstore']:
                 clearfilecache(repo, b'obsstore')
             repoview.filterrevs(repo, name)
 
         return d
 
     allfilter = sorted(repoview.filtertable)
     if names:
         allfilter = [n for n in allfilter if n in names]
 
     for name in allfilter:
         timer(getfiltered(name), title=name)
     fm.end()
 
 
 @command(
     b'perf::branchmap|perfbranchmap',
     [
         (b'f', b'full', False, b'Includes build time of subset'),
         (
             b'',
             b'clear-revbranch',
             False,
             b'purge the revbranch cache between computation',
         ),
     ]
     + formatteropts,
 )
 def perfbranchmap(ui, repo, *filternames, **opts):
     """benchmark the update of a branchmap
 
     This benchmarks the full repo.branchmap() call with read and write disabled
     """
     opts = _byteskwargs(opts)
     full = opts.get(b"full", False)
     clear_revbranch = opts.get(b"clear_revbranch", False)
     timer, fm = gettimer(ui, opts)
 
     def getbranchmap(filtername):
         """generate a benchmark function for the filtername"""
         if filtername is None:
             view = repo
         else:
             view = repo.filtered(filtername)
         if util.safehasattr(view._branchcaches, '_per_filter'):
             filtered = view._branchcaches._per_filter
         else:
             # older versions
             filtered = view._branchcaches
 
         def d():
             if clear_revbranch:
                 repo.revbranchcache()._clear()
             if full:
                 view._branchcaches.clear()
             else:
                 filtered.pop(filtername, None)
             view.branchmap()
 
         return d
 
     # add filter in smaller subset to bigger subset
     possiblefilters = set(repoview.filtertable)
     if filternames:
         possiblefilters &= set(filternames)
     subsettable = getbranchmapsubsettable()
     allfilters = []
     while possiblefilters:
         for name in possiblefilters:
             subset = subsettable.get(name)
             if subset not in possiblefilters:
                 break
         else:
             assert False, b'subset cycle %s!' % possiblefilters
         allfilters.append(name)
         possiblefilters.remove(name)
 
     # warm the cache
     if not full:
         for name in allfilters:
             repo.filtered(name).branchmap()
     if not filternames or b'unfiltered' in filternames:
         # add unfiltered
         allfilters.append(None)
 
     if util.safehasattr(branchmap.branchcache, 'fromfile'):
         branchcacheread = safeattrsetter(branchmap.branchcache, b'fromfile')
         branchcacheread.set(classmethod(lambda *args: None))
     else:
         # older versions
         branchcacheread = safeattrsetter(branchmap, b'read')
         branchcacheread.set(lambda *args: None)
     branchcachewrite = safeattrsetter(branchmap.branchcache, b'write')
     branchcachewrite.set(lambda *args: None)
     try:
         for name in allfilters:
             printname = name
             if name is None:
                 printname = b'unfiltered'
             timer(getbranchmap(name), title=printname)
     finally:
         branchcacheread.restore()
         branchcachewrite.restore()
     fm.end()
 
 
 @command(
     b'perf::branchmapupdate|perfbranchmapupdate',
     [
         (b'', b'base', [], b'subset of revision to start from'),
         (b'', b'target', [], b'subset of revision to end with'),
         (b'', b'clear-caches', False, b'clear cache between each runs'),
     ]
     + formatteropts,
 )
 def perfbranchmapupdate(ui, repo, base=(), target=(), **opts):
     """benchmark branchmap update from for <base> revs to <target> revs
 
     If `--clear-caches` is passed, the following items will be reset before
     each update:
         * the changelog instance and associated indexes
         * the rev-branch-cache instance
 
     Examples:
 
        # update for the one last revision
        $ hg perfbranchmapupdate --base 'not tip' --target 'tip'
 
        $ update for change coming with a new branch
        $ hg perfbranchmapupdate --base 'stable' --target 'default'
     """
     from mercurial import branchmap
     from mercurial import repoview
 
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     clearcaches = opts[b'clear_caches']
     unfi = repo.unfiltered()
     x = [None]  # used to pass data between closure
 
     # we use a `list` here to avoid possible side effect from smartset
     baserevs = list(scmutil.revrange(repo, base))
     targetrevs = list(scmutil.revrange(repo, target))
     if not baserevs:
         raise error.Abort(b'no revisions selected for --base')
     if not targetrevs:
         raise error.Abort(b'no revisions selected for --target')
 
     # make sure the target branchmap also contains the one in the base
     targetrevs = list(set(baserevs) | set(targetrevs))
     targetrevs.sort()
 
     cl = repo.changelog
     allbaserevs = list(cl.ancestors(baserevs, inclusive=True))
     allbaserevs.sort()
     alltargetrevs = frozenset(cl.ancestors(targetrevs, inclusive=True))
 
     newrevs = list(alltargetrevs.difference(allbaserevs))
     newrevs.sort()
 
     allrevs = frozenset(unfi.changelog.revs())
     basefilterrevs = frozenset(allrevs.difference(allbaserevs))
     targetfilterrevs = frozenset(allrevs.difference(alltargetrevs))
 
     def basefilter(repo, visibilityexceptions=None):
         return basefilterrevs
 
     def targetfilter(repo, visibilityexceptions=None):
         return targetfilterrevs
 
     msg = b'benchmark of branchmap with %d revisions with %d new ones\n'
     ui.status(msg % (len(allbaserevs), len(newrevs)))
     if targetfilterrevs:
         msg = b'(%d revisions still filtered)\n'
         ui.status(msg % len(targetfilterrevs))
 
     try:
         repoview.filtertable[b'__perf_branchmap_update_base'] = basefilter
         repoview.filtertable[b'__perf_branchmap_update_target'] = targetfilter
 
         baserepo = repo.filtered(b'__perf_branchmap_update_base')
         targetrepo = repo.filtered(b'__perf_branchmap_update_target')
 
         # try to find an existing branchmap to reuse
         subsettable = getbranchmapsubsettable()
         candidatefilter = subsettable.get(None)
         while candidatefilter is not None:
             candidatebm = repo.filtered(candidatefilter).branchmap()
             if candidatebm.validfor(baserepo):
                 filtered = repoview.filterrevs(repo, candidatefilter)
                 missing = [r for r in allbaserevs if r in filtered]
                 base = candidatebm.copy()
                 base.update(baserepo, missing)
                 break
             candidatefilter = subsettable.get(candidatefilter)
         else:
             # no suitable subset where found
             base = branchmap.branchcache()
             base.update(baserepo, allbaserevs)
 
         def setup():
             x[0] = base.copy()
             if clearcaches:
                 unfi._revbranchcache = None
                 clearchangelog(repo)
 
         def bench():
             x[0].update(targetrepo, newrevs)
 
         timer(bench, setup=setup)
         fm.end()
     finally:
         repoview.filtertable.pop(b'__perf_branchmap_update_base', None)
         repoview.filtertable.pop(b'__perf_branchmap_update_target', None)
 
 
 @command(
     b'perf::branchmapload|perfbranchmapload',
     [
         (b'f', b'filter', b'', b'Specify repoview filter'),
         (b'', b'list', False, b'List brachmap filter caches'),
         (b'', b'clear-revlogs', False, b'refresh changelog and manifest'),
     ]
     + formatteropts,
 )
 def perfbranchmapload(ui, repo, filter=b'', list=False, **opts):
     """benchmark reading the branchmap"""
     opts = _byteskwargs(opts)
     clearrevlogs = opts[b'clear_revlogs']
 
     if list:
         for name, kind, st in repo.cachevfs.readdir(stat=True):
             if name.startswith(b'branch2'):
                 filtername = name.partition(b'-')[2] or b'unfiltered'
                 ui.status(
                     b'%s - %s\n' % (filtername, util.bytecount(st.st_size))
                 )
         return
     if not filter:
         filter = None
     subsettable = getbranchmapsubsettable()
     if filter is None:
         repo = repo.unfiltered()
     else:
         repo = repoview.repoview(repo, filter)
 
     repo.branchmap()  # make sure we have a relevant, up to date branchmap
 
     try:
         fromfile = branchmap.branchcache.fromfile
     except AttributeError:
         # older versions
         fromfile = branchmap.read
 
     currentfilter = filter
     # try once without timer, the filter may not be cached
     while fromfile(repo) is None:
         currentfilter = subsettable.get(currentfilter)
         if currentfilter is None:
             raise error.Abort(
                 b'No branchmap cached for %s repo' % (filter or b'unfiltered')
             )
         repo = repo.filtered(currentfilter)
     timer, fm = gettimer(ui, opts)
 
     def setup():
         if clearrevlogs:
             clearchangelog(repo)
 
     def bench():
         fromfile(repo)
 
     timer(bench, setup=setup)
     fm.end()
 
 
 @command(b'perf::loadmarkers|perfloadmarkers')
 def perfloadmarkers(ui, repo):
     """benchmark the time to parse the on-disk markers for a repo
 
     Result is the number of markers in the repo."""
     timer, fm = gettimer(ui)
     svfs = getsvfs(repo)
     timer(lambda: len(obsolete.obsstore(repo, svfs)))
     fm.end()
 
 
 @command(
     b'perf::lrucachedict|perflrucachedict',
     formatteropts
     + [
         (b'', b'costlimit', 0, b'maximum total cost of items in cache'),
         (b'', b'mincost', 0, b'smallest cost of items in cache'),
         (b'', b'maxcost', 100, b'maximum cost of items in cache'),
         (b'', b'size', 4, b'size of cache'),
         (b'', b'gets', 10000, b'number of key lookups'),
         (b'', b'sets', 10000, b'number of key sets'),
         (b'', b'mixed', 10000, b'number of mixed mode operations'),
         (
             b'',
             b'mixedgetfreq',
             50,
             b'frequency of get vs set ops in mixed mode',
         ),
     ],
     norepo=True,
 )
 def perflrucache(
     ui,
     mincost=0,
     maxcost=100,
     costlimit=0,
     size=4,
     gets=10000,
     sets=10000,
     mixed=10000,
     mixedgetfreq=50,
     **opts
 ):
     opts = _byteskwargs(opts)
 
     def doinit():
         for i in _xrange(10000):
             util.lrucachedict(size)
 
     costrange = list(range(mincost, maxcost + 1))
 
     values = []
     for i in _xrange(size):
         values.append(random.randint(0, _maxint))
 
     # Get mode fills the cache and tests raw lookup performance with no
     # eviction.
     getseq = []
     for i in _xrange(gets):
         getseq.append(random.choice(values))
 
     def dogets():
         d = util.lrucachedict(size)
         for v in values:
             d[v] = v
         for key in getseq:
             value = d[key]
             value  # silence pyflakes warning
 
     def dogetscost():
         d = util.lrucachedict(size, maxcost=costlimit)
         for i, v in enumerate(values):
             d.insert(v, v, cost=costs[i])
         for key in getseq:
             try:
                 value = d[key]
                 value  # silence pyflakes warning
             except KeyError:
                 pass
 
     # Set mode tests insertion speed with cache eviction.
     setseq = []
     costs = []
     for i in _xrange(sets):
         setseq.append(random.randint(0, _maxint))
         costs.append(random.choice(costrange))
 
     def doinserts():
         d = util.lrucachedict(size)
         for v in setseq:
             d.insert(v, v)
 
     def doinsertscost():
         d = util.lrucachedict(size, maxcost=costlimit)
         for i, v in enumerate(setseq):
             d.insert(v, v, cost=costs[i])
 
     def dosets():
         d = util.lrucachedict(size)
         for v in setseq:
             d[v] = v
 
     # Mixed mode randomly performs gets and sets with eviction.
     mixedops = []
     for i in _xrange(mixed):
         r = random.randint(0, 100)
         if r < mixedgetfreq:
             op = 0
         else:
             op = 1
 
         mixedops.append(
             (op, random.randint(0, size * 2), random.choice(costrange))
         )
 
     def domixed():
         d = util.lrucachedict(size)
 
         for op, v, cost in mixedops:
             if op == 0:
                 try:
                     d[v]
                 except KeyError:
                     pass
             else:
                 d[v] = v
 
     def domixedcost():
         d = util.lrucachedict(size, maxcost=costlimit)
 
         for op, v, cost in mixedops:
             if op == 0:
                 try:
                     d[v]
                 except KeyError:
                     pass
             else:
                 d.insert(v, v, cost=cost)
 
     benches = [
         (doinit, b'init'),
     ]
 
     if costlimit:
         benches.extend(
             [
                 (dogetscost, b'gets w/ cost limit'),
                 (doinsertscost, b'inserts w/ cost limit'),
                 (domixedcost, b'mixed w/ cost limit'),
             ]
         )
     else:
         benches.extend(
             [
                 (dogets, b'gets'),
                 (doinserts, b'inserts'),
                 (dosets, b'sets'),
                 (domixed, b'mixed'),
             ]
         )
 
     for fn, title in benches:
         timer, fm = gettimer(ui, opts)
         timer(fn, title=title)
         fm.end()
 
 
 @command(
     b'perf::write|perfwrite',
     formatteropts
     + [
         (b'', b'write-method', b'write', b'ui write method'),
         (b'', b'nlines', 100, b'number of lines'),
         (b'', b'nitems', 100, b'number of items (per line)'),
         (b'', b'item', b'x', b'item that is written'),
         (b'', b'batch-line', None, b'pass whole line to write method at once'),
         (b'', b'flush-line', None, b'flush after each line'),
     ],
 )
 def perfwrite(ui, repo, **opts):
     """microbenchmark ui.write (and others)"""
     opts = _byteskwargs(opts)
 
     write = getattr(ui, _sysstr(opts[b'write_method']))
     nlines = int(opts[b'nlines'])
     nitems = int(opts[b'nitems'])
     item = opts[b'item']
     batch_line = opts.get(b'batch_line')
     flush_line = opts.get(b'flush_line')
 
     if batch_line:
         line = item * nitems + b'\n'
 
     def benchmark():
         for i in pycompat.xrange(nlines):
             if batch_line:
                 write(line)
             else:
                 for i in pycompat.xrange(nitems):
                     write(item)
                 write(b'\n')
             if flush_line:
                 ui.flush()
         ui.flush()
 
     timer, fm = gettimer(ui, opts)
     timer(benchmark)
     fm.end()
 
 
 def uisetup(ui):
     if util.safehasattr(cmdutil, b'openrevlog') and not util.safehasattr(
         commands, b'debugrevlogopts'
     ):
         # for "historical portability":
         # In this case, Mercurial should be 1.9 (or a79fea6b3e77) -
         # 3.7 (or 5606f7d0d063). Therefore, '--dir' option for
         # openrevlog() should cause failure, because it has been
         # available since 3.5 (or 49c583ca48c4).
         def openrevlog(orig, repo, cmd, file_, opts):
             if opts.get(b'dir') and not util.safehasattr(repo, b'dirlog'):
                 raise error.Abort(
                     b"This version doesn't support --dir option",
                     hint=b"use 3.5 or later",
                 )
             return orig(repo, cmd, file_, opts)
 
         extensions.wrapfunction(cmdutil, b'openrevlog', openrevlog)
 
 
 @command(
     b'perf::progress|perfprogress',
     formatteropts
     + [
         (b'', b'topic', b'topic', b'topic for progress messages'),
         (b'c', b'total', 1000000, b'total value we are progressing to'),
     ],
     norepo=True,
 )
 def perfprogress(ui, topic=None, total=None, **opts):
     """printing of progress bars"""
     opts = _byteskwargs(opts)
 
     timer, fm = gettimer(ui, opts)
 
     def doprogress():
         with ui.makeprogress(topic, total=total) as progress:
             for i in _xrange(total):
                 progress.increment()
 
     timer(doprogress)
     fm.end()

contrib/undumprevlog

@@ -1,48 +1,57 b''
 #!/usr/bin/env python3
 # Undump a dump from dumprevlog
 # $ hg init
 # $ undumprevlog < repo.dump
 
 from __future__ import absolute_import, print_function
 
 import sys
 from mercurial.node import bin
 from mercurial import (
     encoding,
     revlog,
     transaction,
     vfs as vfsmod,
 )
 from mercurial.utils import procutil
 
+from mercurial.revlogutils import (
+    constants as revlog_constants,
+)
+
 for fp in (sys.stdin, sys.stdout, sys.stderr):
     procutil.setbinary(fp)
 
 opener = vfsmod.vfs(b'.', False)
 tr = transaction.transaction(
     sys.stderr.write, opener, {b'store': opener}, b"undump.journal"
 )
 while True:
     l = sys.stdin.readline()
     if not l:
         break
     if l.startswith("file:"):
         f = encoding.strtolocal(l[6:-1])
-        r = revlog.revlog(opener, f)
+        assert f.endswith(b'.i')
+        r = revlog.revlog(
+            opener,
+            target=(revlog_constants.KIND_OTHER, b'undump-revlog'),
+            radix=f[:-2],
+        )
         procutil.stdout.write(b'%s\n' % f)
     elif l.startswith("node:"):
         n = bin(l[6:-1])
     elif l.startswith("linkrev:"):
         lr = int(l[9:-1])
     elif l.startswith("parents:"):
         p = l[9:-1].split()
         p1 = bin(p[0])
         p2 = bin(p[1])
     elif l.startswith("length:"):
         length = int(l[8:-1])
         sys.stdin.readline()  # start marker
         d = encoding.strtolocal(sys.stdin.read(length))
         sys.stdin.readline()  # end marker
         r.addrevision(d, tr, lr, p1, p2)
 
 tr.close()

hgext/absorb.py

@@ -1,1166 +1,1165 b''
 # absorb.py
 #
 # Copyright 2016 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """apply working directory changes to changesets (EXPERIMENTAL)
 
 The absorb extension provides a command to use annotate information to
 amend modified chunks into the corresponding non-public changesets.
 
 ::
 
     [absorb]
     # only check 50 recent non-public changesets at most
     max-stack-size = 50
     # whether to add noise to new commits to avoid obsolescence cycle
     add-noise = 1
     # make `amend --correlated` a shortcut to the main command
     amend-flag = correlated
 
     [color]
     absorb.description = yellow
     absorb.node = blue bold
     absorb.path = bold
 """
 
 # TODO:
 #  * Rename config items to [commands] namespace
 #  * Converge getdraftstack() with other code in core
 #  * move many attributes on fixupstate to be private
 
 from __future__ import absolute_import
 
 import collections
 
 from mercurial.i18n import _
 from mercurial.node import (
     hex,
-    nullid,
     short,
 )
 from mercurial import (
     cmdutil,
     commands,
     context,
     crecord,
     error,
     linelog,
     mdiff,
     obsolete,
     patch,
     phases,
     pycompat,
     registrar,
     rewriteutil,
     scmutil,
     util,
 )
 from mercurial.utils import stringutil
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 configitem(b'absorb', b'add-noise', default=True)
 configitem(b'absorb', b'amend-flag', default=None)
 configitem(b'absorb', b'max-stack-size', default=50)
 
 colortable = {
     b'absorb.description': b'yellow',
     b'absorb.node': b'blue bold',
     b'absorb.path': b'bold',
 }
 
 defaultdict = collections.defaultdict
 
 
 class nullui(object):
     """blank ui object doing nothing"""
 
     debugflag = False
     verbose = False
     quiet = True
 
     def __getitem__(name):
         def nullfunc(*args, **kwds):
             return
 
         return nullfunc
 
 
 class emptyfilecontext(object):
     """minimal filecontext representing an empty file"""
 
     def __init__(self, repo):
         self._repo = repo
 
     def data(self):
         return b''
 
     def node(self):
-        return nullid
+        return self._repo.nullid
 
 
 def uniq(lst):
     """list -> list. remove duplicated items without changing the order"""
     seen = set()
     result = []
     for x in lst:
         if x not in seen:
             seen.add(x)
             result.append(x)
     return result
 
 
 def getdraftstack(headctx, limit=None):
     """(ctx, int?) -> [ctx]. get a linear stack of non-public changesets.
 
     changesets are sorted in topo order, oldest first.
     return at most limit items, if limit is a positive number.
 
     merges are considered as non-draft as well. i.e. every commit
     returned has and only has 1 parent.
     """
     ctx = headctx
     result = []
     while ctx.phase() != phases.public:
         if limit and len(result) >= limit:
             break
         parents = ctx.parents()
         if len(parents) != 1:
             break
         result.append(ctx)
         ctx = parents[0]
     result.reverse()
     return result
 
 
 def getfilestack(stack, path, seenfctxs=None):
     """([ctx], str, set) -> [fctx], {ctx: fctx}
 
     stack is a list of contexts, from old to new. usually they are what
     "getdraftstack" returns.
 
     follows renames, but not copies.
 
     seenfctxs is a set of filecontexts that will be considered "immutable".
     they are usually what this function returned in earlier calls, useful
     to avoid issues that a file was "moved" to multiple places and was then
     modified differently, like: "a" was copied to "b", "a" was also copied to
     "c" and then "a" was deleted, then both "b" and "c" were "moved" from "a"
     and we enforce only one of them to be able to affect "a"'s content.
 
     return an empty list and an empty dict, if the specified path does not
     exist in stack[-1] (the top of the stack).
 
     otherwise, return a list of de-duplicated filecontexts, and the map to
     convert ctx in the stack to fctx, for possible mutable fctxs. the first item
     of the list would be outside the stack and should be considered immutable.
     the remaining items are within the stack.
 
     for example, given the following changelog and corresponding filelog
     revisions:
 
       changelog: 3----4----5----6----7
       filelog:   x    0----1----1----2 (x: no such file yet)
 
     - if stack = [5, 6, 7], returns ([0, 1, 2], {5: 1, 6: 1, 7: 2})
     - if stack = [3, 4, 5], returns ([e, 0, 1], {4: 0, 5: 1}), where "e" is a
       dummy empty filecontext.
     - if stack = [2], returns ([], {})
     - if stack = [7], returns ([1, 2], {7: 2})
     - if stack = [6, 7], returns ([1, 2], {6: 1, 7: 2}), although {6: 1} can be
       removed, since 1 is immutable.
     """
     if seenfctxs is None:
         seenfctxs = set()
     assert stack
 
     if path not in stack[-1]:
         return [], {}
 
     fctxs = []
     fctxmap = {}
 
     pctx = stack[0].p1()  # the public (immutable) ctx we stop at
     for ctx in reversed(stack):
         if path not in ctx:  # the file is added in the next commit
             pctx = ctx
             break
         fctx = ctx[path]
         fctxs.append(fctx)
         if fctx in seenfctxs:  # treat fctx as the immutable one
             pctx = None  # do not add another immutable fctx
             break
         fctxmap[ctx] = fctx  # only for mutable fctxs
         copy = fctx.copysource()
         if copy:
             path = copy  # follow rename
             if path in ctx:  # but do not follow copy
                 pctx = ctx.p1()
                 break
 
     if pctx is not None:  # need an extra immutable fctx
         if path in pctx:
             fctxs.append(pctx[path])
         else:
             fctxs.append(emptyfilecontext(pctx.repo()))
 
     fctxs.reverse()
     # note: we rely on a property of hg: filerev is not reused for linear
     # history. i.e. it's impossible to have:
     #   changelog:  4----5----6 (linear, no merges)
     #   filelog:    1----2----1
     #                         ^ reuse filerev (impossible)
     # because parents are part of the hash. if that's not true, we need to
     # remove uniq and find a different way to identify fctxs.
     return uniq(fctxs), fctxmap
 
 
 class overlaystore(patch.filestore):
     """read-only, hybrid store based on a dict and ctx.
     memworkingcopy: {path: content}, overrides file contents.
     """
 
     def __init__(self, basectx, memworkingcopy):
         self.basectx = basectx
         self.memworkingcopy = memworkingcopy
 
     def getfile(self, path):
         """comply with mercurial.patch.filestore.getfile"""
         if path not in self.basectx:
             return None, None, None
         fctx = self.basectx[path]
         if path in self.memworkingcopy:
             content = self.memworkingcopy[path]
         else:
             content = fctx.data()
         mode = (fctx.islink(), fctx.isexec())
         copy = fctx.copysource()
         return content, mode, copy
 
 
 def overlaycontext(memworkingcopy, ctx, parents=None, extra=None, desc=None):
     """({path: content}, ctx, (p1node, p2node)?, {}?) -> memctx
     memworkingcopy overrides file contents.
     """
     # parents must contain 2 items: (node1, node2)
     if parents is None:
         parents = ctx.repo().changelog.parents(ctx.node())
     if extra is None:
         extra = ctx.extra()
     if desc is None:
         desc = ctx.description()
     date = ctx.date()
     user = ctx.user()
     files = set(ctx.files()).union(memworkingcopy)
     store = overlaystore(ctx, memworkingcopy)
     return context.memctx(
         repo=ctx.repo(),
         parents=parents,
         text=desc,
         files=files,
         filectxfn=store,
         user=user,
         date=date,
         branch=None,
         extra=extra,
     )
 
 
 class filefixupstate(object):
     """state needed to apply fixups to a single file
 
     internally, it keeps file contents of several revisions and a linelog.
 
     the linelog uses odd revision numbers for original contents (fctxs passed
     to __init__), and even revision numbers for fixups, like:
 
         linelog rev 1: self.fctxs[0] (from an immutable "public" changeset)
         linelog rev 2: fixups made to self.fctxs[0]
         linelog rev 3: self.fctxs[1] (a child of fctxs[0])
         linelog rev 4: fixups made to self.fctxs[1]
         ...
 
     a typical use is like:
 
         1. call diffwith, to calculate self.fixups
         2. (optionally), present self.fixups to the user, or change it
         3. call apply, to apply changes
         4. read results from "finalcontents", or call getfinalcontent
     """
 
     def __init__(self, fctxs, path, ui=None, opts=None):
         """([fctx], ui or None) -> None
 
         fctxs should be linear, and sorted by topo order - oldest first.
         fctxs[0] will be considered as "immutable" and will not be changed.
         """
         self.fctxs = fctxs
         self.path = path
         self.ui = ui or nullui()
         self.opts = opts or {}
 
         # following fields are built from fctxs. they exist for perf reason
         self.contents = [f.data() for f in fctxs]
         self.contentlines = pycompat.maplist(mdiff.splitnewlines, self.contents)
         self.linelog = self._buildlinelog()
         if self.ui.debugflag:
             assert self._checkoutlinelog() == self.contents
 
         # following fields will be filled later
         self.chunkstats = [0, 0]  # [adopted, total : int]
         self.targetlines = []  # [str]
         self.fixups = []  # [(linelog rev, a1, a2, b1, b2)]
         self.finalcontents = []  # [str]
         self.ctxaffected = set()
 
     def diffwith(self, targetfctx, fm=None):
         """calculate fixups needed by examining the differences between
         self.fctxs[-1] and targetfctx, chunk by chunk.
 
         targetfctx is the target state we move towards. we may or may not be
         able to get there because not all modified chunks can be amended into
         a non-public fctx unambiguously.
 
         call this only once, before apply().
 
         update self.fixups, self.chunkstats, and self.targetlines.
         """
         a = self.contents[-1]
         alines = self.contentlines[-1]
         b = targetfctx.data()
         blines = mdiff.splitnewlines(b)
         self.targetlines = blines
 
         self.linelog.annotate(self.linelog.maxrev)
         annotated = self.linelog.annotateresult  # [(linelog rev, linenum)]
         assert len(annotated) == len(alines)
         # add a dummy end line to make insertion at the end easier
         if annotated:
             dummyendline = (annotated[-1][0], annotated[-1][1] + 1)
             annotated.append(dummyendline)
 
         # analyse diff blocks
         for chunk in self._alldiffchunks(a, b, alines, blines):
             newfixups = self._analysediffchunk(chunk, annotated)
             self.chunkstats[0] += bool(newfixups)  # 1 or 0
             self.chunkstats[1] += 1
             self.fixups += newfixups
             if fm is not None:
                 self._showchanges(fm, alines, blines, chunk, newfixups)
 
     def apply(self):
         """apply self.fixups. update self.linelog, self.finalcontents.
 
         call this only once, before getfinalcontent(), after diffwith().
         """
         # the following is unnecessary, as it's done by "diffwith":
         #   self.linelog.annotate(self.linelog.maxrev)
         for rev, a1, a2, b1, b2 in reversed(self.fixups):
             blines = self.targetlines[b1:b2]
             if self.ui.debugflag:
                 idx = (max(rev - 1, 0)) // 2
                 self.ui.write(
                     _(b'%s: chunk %d:%d -> %d lines\n')
                     % (short(self.fctxs[idx].node()), a1, a2, len(blines))
                 )
             self.linelog.replacelines(rev, a1, a2, b1, b2)
         if self.opts.get(b'edit_lines', False):
             self.finalcontents = self._checkoutlinelogwithedits()
         else:
             self.finalcontents = self._checkoutlinelog()
 
     def getfinalcontent(self, fctx):
         """(fctx) -> str. get modified file content for a given filecontext"""
         idx = self.fctxs.index(fctx)
         return self.finalcontents[idx]
 
     def _analysediffchunk(self, chunk, annotated):
         """analyse a different chunk and return new fixups found
 
         return [] if no lines from the chunk can be safely applied.
 
         the chunk (or lines) cannot be safely applied, if, for example:
           - the modified (deleted) lines belong to a public changeset
             (self.fctxs[0])
           - the chunk is a pure insertion and the adjacent lines (at most 2
             lines) belong to different non-public changesets, or do not belong
             to any non-public changesets.
           - the chunk is modifying lines from different changesets.
             in this case, if the number of lines deleted equals to the number
             of lines added, assume it's a simple 1:1 map (could be wrong).
             otherwise, give up.
           - the chunk is modifying lines from a single non-public changeset,
             but other revisions touch the area as well. i.e. the lines are
             not continuous as seen from the linelog.
         """
         a1, a2, b1, b2 = chunk
         # find involved indexes from annotate result
         involved = annotated[a1:a2]
         if not involved and annotated:  # a1 == a2 and a is not empty
             # pure insertion, check nearby lines. ignore lines belong
             # to the public (first) changeset (i.e. annotated[i][0] == 1)
             nearbylinenums = {a2, max(0, a1 - 1)}
             involved = [
                 annotated[i] for i in nearbylinenums if annotated[i][0] != 1
             ]
         involvedrevs = list({r for r, l in involved})
         newfixups = []
         if len(involvedrevs) == 1 and self._iscontinuous(a1, a2 - 1, True):
             # chunk belongs to a single revision
             rev = involvedrevs[0]
             if rev > 1:
                 fixuprev = rev + 1
                 newfixups.append((fixuprev, a1, a2, b1, b2))
         elif a2 - a1 == b2 - b1 or b1 == b2:
             # 1:1 line mapping, or chunk was deleted
             for i in pycompat.xrange(a1, a2):
                 rev, linenum = annotated[i]
                 if rev > 1:
                     if b1 == b2:  # deletion, simply remove that single line
                         nb1 = nb2 = 0
                     else:  # 1:1 line mapping, change the corresponding rev
                         nb1 = b1 + i - a1
                         nb2 = nb1 + 1
                     fixuprev = rev + 1
                     newfixups.append((fixuprev, i, i + 1, nb1, nb2))
         return self._optimizefixups(newfixups)
 
     @staticmethod
     def _alldiffchunks(a, b, alines, blines):
         """like mdiff.allblocks, but only care about differences"""
         blocks = mdiff.allblocks(a, b, lines1=alines, lines2=blines)
         for chunk, btype in blocks:
             if btype != b'!':
                 continue
             yield chunk
 
     def _buildlinelog(self):
         """calculate the initial linelog based on self.content{,line}s.
         this is similar to running a partial "annotate".
         """
         llog = linelog.linelog()
         a, alines = b'', []
         for i in pycompat.xrange(len(self.contents)):
             b, blines = self.contents[i], self.contentlines[i]
             llrev = i * 2 + 1
             chunks = self._alldiffchunks(a, b, alines, blines)
             for a1, a2, b1, b2 in reversed(list(chunks)):
                 llog.replacelines(llrev, a1, a2, b1, b2)
             a, alines = b, blines
         return llog
 
     def _checkoutlinelog(self):
         """() -> [str]. check out file contents from linelog"""
         contents = []
         for i in pycompat.xrange(len(self.contents)):
             rev = (i + 1) * 2
             self.linelog.annotate(rev)
             content = b''.join(map(self._getline, self.linelog.annotateresult))
             contents.append(content)
         return contents
 
     def _checkoutlinelogwithedits(self):
         """() -> [str]. prompt all lines for edit"""
         alllines = self.linelog.getalllines()
         # header
         editortext = (
             _(
                 b'HG: editing %s\nHG: "y" means the line to the right '
                 b'exists in the changeset to the top\nHG:\n'
             )
             % self.fctxs[-1].path()
         )
         # [(idx, fctx)]. hide the dummy emptyfilecontext
         visiblefctxs = [
             (i, f)
             for i, f in enumerate(self.fctxs)
             if not isinstance(f, emptyfilecontext)
         ]
         for i, (j, f) in enumerate(visiblefctxs):
             editortext += _(b'HG: %s/%s %s %s\n') % (
                 b'|' * i,
                 b'-' * (len(visiblefctxs) - i + 1),
                 short(f.node()),
                 f.description().split(b'\n', 1)[0],
             )
         editortext += _(b'HG: %s\n') % (b'|' * len(visiblefctxs))
         # figure out the lifetime of a line, this is relatively inefficient,
         # but probably fine
         lineset = defaultdict(lambda: set())  # {(llrev, linenum): {llrev}}
         for i, f in visiblefctxs:
             self.linelog.annotate((i + 1) * 2)
             for l in self.linelog.annotateresult:
                 lineset[l].add(i)
         # append lines
         for l in alllines:
             editortext += b'    %s : %s' % (
                 b''.join(
                     [
                         (b'y' if i in lineset[l] else b' ')
                         for i, _f in visiblefctxs
                     ]
                 ),
                 self._getline(l),
             )
         # run editor
         editedtext = self.ui.edit(editortext, b'', action=b'absorb')
         if not editedtext:
             raise error.InputError(_(b'empty editor text'))
         # parse edited result
         contents = [b''] * len(self.fctxs)
         leftpadpos = 4
         colonpos = leftpadpos + len(visiblefctxs) + 1
         for l in mdiff.splitnewlines(editedtext):
             if l.startswith(b'HG:'):
                 continue
             if l[colonpos - 1 : colonpos + 2] != b' : ':
                 raise error.InputError(_(b'malformed line: %s') % l)
             linecontent = l[colonpos + 2 :]
             for i, ch in enumerate(
                 pycompat.bytestr(l[leftpadpos : colonpos - 1])
             ):
                 if ch == b'y':
                     contents[visiblefctxs[i][0]] += linecontent
         # chunkstats is hard to calculate if anything changes, therefore
         # set them to just a simple value (1, 1).
         if editedtext != editortext:
             self.chunkstats = [1, 1]
         return contents
 
     def _getline(self, lineinfo):
         """((rev, linenum)) -> str. convert rev+line number to line content"""
         rev, linenum = lineinfo
         if rev & 1:  # odd: original line taken from fctxs
             return self.contentlines[rev // 2][linenum]
         else:  # even: fixup line from targetfctx
             return self.targetlines[linenum]
 
     def _iscontinuous(self, a1, a2, closedinterval=False):
         """(a1, a2 : int) -> bool
 
         check if these lines are continuous. i.e. no other insertions or
         deletions (from other revisions) among these lines.
 
         closedinterval decides whether a2 should be included or not. i.e. is
         it [a1, a2), or [a1, a2] ?
         """
         if a1 >= a2:
             return True
         llog = self.linelog
         offset1 = llog.getoffset(a1)
         offset2 = llog.getoffset(a2) + int(closedinterval)
         linesinbetween = llog.getalllines(offset1, offset2)
         return len(linesinbetween) == a2 - a1 + int(closedinterval)
 
     def _optimizefixups(self, fixups):
         """[(rev, a1, a2, b1, b2)] -> [(rev, a1, a2, b1, b2)].
         merge adjacent fixups to make them less fragmented.
         """
         result = []
         pcurrentchunk = [[-1, -1, -1, -1, -1]]
 
         def pushchunk():
             if pcurrentchunk[0][0] != -1:
                 result.append(tuple(pcurrentchunk[0]))
 
         for i, chunk in enumerate(fixups):
             rev, a1, a2, b1, b2 = chunk
             lastrev = pcurrentchunk[0][0]
             lasta2 = pcurrentchunk[0][2]
             lastb2 = pcurrentchunk[0][4]
             if (
                 a1 == lasta2
                 and b1 == lastb2
                 and rev == lastrev
                 and self._iscontinuous(max(a1 - 1, 0), a1)
             ):
                 # merge into currentchunk
                 pcurrentchunk[0][2] = a2
                 pcurrentchunk[0][4] = b2
             else:
                 pushchunk()
                 pcurrentchunk[0] = list(chunk)
         pushchunk()
         return result
 
     def _showchanges(self, fm, alines, blines, chunk, fixups):
         def trim(line):
             if line.endswith(b'\n'):
                 line = line[:-1]
             return line
 
         # this is not optimized for perf but _showchanges only gets executed
         # with an extra command-line flag.
         a1, a2, b1, b2 = chunk
         aidxs, bidxs = [0] * (a2 - a1), [0] * (b2 - b1)
         for idx, fa1, fa2, fb1, fb2 in fixups:
             for i in pycompat.xrange(fa1, fa2):
                 aidxs[i - a1] = (max(idx, 1) - 1) // 2
             for i in pycompat.xrange(fb1, fb2):
                 bidxs[i - b1] = (max(idx, 1) - 1) // 2
 
         fm.startitem()
         fm.write(
             b'hunk',
             b'        %s\n',
             b'@@ -%d,%d +%d,%d @@' % (a1, a2 - a1, b1, b2 - b1),
             label=b'diff.hunk',
         )
         fm.data(path=self.path, linetype=b'hunk')
 
         def writeline(idx, diffchar, line, linetype, linelabel):
             fm.startitem()
             node = b''
             if idx:
                 ctx = self.fctxs[idx]
                 fm.context(fctx=ctx)
                 node = ctx.hex()
                 self.ctxaffected.add(ctx.changectx())
             fm.write(b'node', b'%-7.7s ', node, label=b'absorb.node')
             fm.write(
                 b'diffchar ' + linetype,
                 b'%s%s\n',
                 diffchar,
                 line,
                 label=linelabel,
             )
             fm.data(path=self.path, linetype=linetype)
 
         for i in pycompat.xrange(a1, a2):
             writeline(
                 aidxs[i - a1],
                 b'-',
                 trim(alines[i]),
                 b'deleted',
                 b'diff.deleted',
             )
         for i in pycompat.xrange(b1, b2):
             writeline(
                 bidxs[i - b1],
                 b'+',
                 trim(blines[i]),
                 b'inserted',
                 b'diff.inserted',
             )
 
 
 class fixupstate(object):
     """state needed to run absorb
 
     internally, it keeps paths and filefixupstates.
 
     a typical use is like filefixupstates:
 
         1. call diffwith, to calculate fixups
         2. (optionally), present fixups to the user, or edit fixups
         3. call apply, to apply changes to memory
         4. call commit, to commit changes to hg database
     """
 
     def __init__(self, stack, ui=None, opts=None):
         """([ctx], ui or None) -> None
 
         stack: should be linear, and sorted by topo order - oldest first.
         all commits in stack are considered mutable.
         """
         assert stack
         self.ui = ui or nullui()
         self.opts = opts or {}
         self.stack = stack
         self.repo = stack[-1].repo().unfiltered()
 
         # following fields will be filled later
         self.paths = []  # [str]
         self.status = None  # ctx.status output
         self.fctxmap = {}  # {path: {ctx: fctx}}
         self.fixupmap = {}  # {path: filefixupstate}
         self.replacemap = {}  # {oldnode: newnode or None}
         self.finalnode = None  # head after all fixups
         self.ctxaffected = set()  # ctx that will be absorbed into
 
     def diffwith(self, targetctx, match=None, fm=None):
         """diff and prepare fixups. update self.fixupmap, self.paths"""
         # only care about modified files
         self.status = self.stack[-1].status(targetctx, match)
         self.paths = []
         # but if --edit-lines is used, the user may want to edit files
         # even if they are not modified
         editopt = self.opts.get(b'edit_lines')
         if not self.status.modified and editopt and match:
             interestingpaths = match.files()
         else:
             interestingpaths = self.status.modified
         # prepare the filefixupstate
         seenfctxs = set()
         # sorting is necessary to eliminate ambiguity for the "double move"
         # case: "hg cp A B; hg cp A C; hg rm A", then only "B" can affect "A".
         for path in sorted(interestingpaths):
             self.ui.debug(b'calculating fixups for %s\n' % path)
             targetfctx = targetctx[path]
             fctxs, ctx2fctx = getfilestack(self.stack, path, seenfctxs)
             # ignore symbolic links or binary, or unchanged files
             if any(
                 f.islink() or stringutil.binary(f.data())
                 for f in [targetfctx] + fctxs
                 if not isinstance(f, emptyfilecontext)
             ):
                 continue
             if targetfctx.data() == fctxs[-1].data() and not editopt:
                 continue
             seenfctxs.update(fctxs[1:])
             self.fctxmap[path] = ctx2fctx
             fstate = filefixupstate(fctxs, path, ui=self.ui, opts=self.opts)
             if fm is not None:
                 fm.startitem()
                 fm.plain(b'showing changes for ')
                 fm.write(b'path', b'%s\n', path, label=b'absorb.path')
                 fm.data(linetype=b'path')
             fstate.diffwith(targetfctx, fm)
             self.fixupmap[path] = fstate
             self.paths.append(path)
             self.ctxaffected.update(fstate.ctxaffected)
 
     def apply(self):
         """apply fixups to individual filefixupstates"""
         for path, state in pycompat.iteritems(self.fixupmap):
             if self.ui.debugflag:
                 self.ui.write(_(b'applying fixups to %s\n') % path)
             state.apply()
 
     @property
     def chunkstats(self):
         """-> {path: chunkstats}. collect chunkstats from filefixupstates"""
         return {
             path: state.chunkstats
             for path, state in pycompat.iteritems(self.fixupmap)
         }
 
     def commit(self):
         """commit changes. update self.finalnode, self.replacemap"""
         with self.repo.transaction(b'absorb') as tr:
             self._commitstack()
             self._movebookmarks(tr)
             if self.repo[b'.'].node() in self.replacemap:
                 self._moveworkingdirectoryparent()
             self._cleanupoldcommits()
         return self.finalnode
 
     def printchunkstats(self):
         """print things like '1 of 2 chunk(s) applied'"""
         ui = self.ui
         chunkstats = self.chunkstats
         if ui.verbose:
             # chunkstats for each file
             for path, stat in pycompat.iteritems(chunkstats):
                 if stat[0]:
                     ui.write(
                         _(b'%s: %d of %d chunk(s) applied\n')
                         % (path, stat[0], stat[1])
                     )
         elif not ui.quiet:
             # a summary for all files
             stats = chunkstats.values()
             applied, total = (sum(s[i] for s in stats) for i in (0, 1))
             ui.write(_(b'%d of %d chunk(s) applied\n') % (applied, total))
 
     def _commitstack(self):
         """make new commits. update self.finalnode, self.replacemap.
         it is splitted from "commit" to avoid too much indentation.
         """
         # last node (20-char) committed by us
         lastcommitted = None
         # p1 which overrides the parent of the next commit, "None" means use
         # the original parent unchanged
         nextp1 = None
         for ctx in self.stack:
             memworkingcopy = self._getnewfilecontents(ctx)
             if not memworkingcopy and not lastcommitted:
                 # nothing changed, nothing commited
                 nextp1 = ctx
                 continue
             willbecomenoop = ctx.files() and self._willbecomenoop(
                 memworkingcopy, ctx, nextp1
             )
             if self.skip_empty_successor and willbecomenoop:
                 # changeset is no longer necessary
                 self.replacemap[ctx.node()] = None
                 msg = _(b'became empty and was dropped')
             else:
                 # changeset needs re-commit
                 nodestr = self._commitsingle(memworkingcopy, ctx, p1=nextp1)
                 lastcommitted = self.repo[nodestr]
                 nextp1 = lastcommitted
                 self.replacemap[ctx.node()] = lastcommitted.node()
                 if memworkingcopy:
                     if willbecomenoop:
                         msg = _(b'%d file(s) changed, became empty as %s')
                     else:
                         msg = _(b'%d file(s) changed, became %s')
                     msg = msg % (
                         len(memworkingcopy),
                         self._ctx2str(lastcommitted),
                     )
                 else:
                     msg = _(b'became %s') % self._ctx2str(lastcommitted)
             if self.ui.verbose and msg:
                 self.ui.write(_(b'%s: %s\n') % (self._ctx2str(ctx), msg))
         self.finalnode = lastcommitted and lastcommitted.node()
 
     def _ctx2str(self, ctx):
         if self.ui.debugflag:
             return b'%d:%s' % (ctx.rev(), ctx.hex())
         else:
             return b'%d:%s' % (ctx.rev(), short(ctx.node()))
 
     def _getnewfilecontents(self, ctx):
         """(ctx) -> {path: str}
 
         fetch file contents from filefixupstates.
         return the working copy overrides - files different from ctx.
         """
         result = {}
         for path in self.paths:
             ctx2fctx = self.fctxmap[path]  # {ctx: fctx}
             if ctx not in ctx2fctx:
                 continue
             fctx = ctx2fctx[ctx]
             content = fctx.data()
             newcontent = self.fixupmap[path].getfinalcontent(fctx)
             if content != newcontent:
                 result[fctx.path()] = newcontent
         return result
 
     def _movebookmarks(self, tr):
         repo = self.repo
         needupdate = [
             (name, self.replacemap[hsh])
             for name, hsh in pycompat.iteritems(repo._bookmarks)
             if hsh in self.replacemap
         ]
         changes = []
         for name, hsh in needupdate:
             if hsh:
                 changes.append((name, hsh))
                 if self.ui.verbose:
                     self.ui.write(
                         _(b'moving bookmark %s to %s\n') % (name, hex(hsh))
                     )
             else:
                 changes.append((name, None))
                 if self.ui.verbose:
                     self.ui.write(_(b'deleting bookmark %s\n') % name)
         repo._bookmarks.applychanges(repo, tr, changes)
 
     def _moveworkingdirectoryparent(self):
         if not self.finalnode:
             # Find the latest not-{obsoleted,stripped} parent.
             revs = self.repo.revs(b'max(::. - %ln)', self.replacemap.keys())
             ctx = self.repo[revs.first()]
             self.finalnode = ctx.node()
         else:
             ctx = self.repo[self.finalnode]
 
         dirstate = self.repo.dirstate
         # dirstate.rebuild invalidates fsmonitorstate, causing "hg status" to
         # be slow. in absorb's case, no need to invalidate fsmonitorstate.
         noop = lambda: 0
         restore = noop
         if util.safehasattr(dirstate, '_fsmonitorstate'):
             bak = dirstate._fsmonitorstate.invalidate
 
             def restore():
                 dirstate._fsmonitorstate.invalidate = bak
 
             dirstate._fsmonitorstate.invalidate = noop
         try:
             with dirstate.parentchange():
                 dirstate.rebuild(ctx.node(), ctx.manifest(), self.paths)
         finally:
             restore()
 
     @staticmethod
     def _willbecomenoop(memworkingcopy, ctx, pctx=None):
         """({path: content}, ctx, ctx) -> bool. test if a commit will be noop
 
         if it will become an empty commit (does not change anything, after the
         memworkingcopy overrides), return True. otherwise return False.
         """
         if not pctx:
             parents = ctx.parents()
             if len(parents) != 1:
                 return False
             pctx = parents[0]
         if ctx.branch() != pctx.branch():
             return False
         if ctx.extra().get(b'close'):
             return False
         # ctx changes more files (not a subset of memworkingcopy)
         if not set(ctx.files()).issubset(set(memworkingcopy)):
             return False
         for path, content in pycompat.iteritems(memworkingcopy):
             if path not in pctx or path not in ctx:
                 return False
             fctx = ctx[path]
             pfctx = pctx[path]
             if pfctx.flags() != fctx.flags():
                 return False
             if pfctx.data() != content:
                 return False
         return True
 
     def _commitsingle(self, memworkingcopy, ctx, p1=None):
         """(ctx, {path: content}, node) -> node. make a single commit
 
         the commit is a clone from ctx, with a (optionally) different p1, and
         different file contents replaced by memworkingcopy.
         """
-        parents = p1 and (p1, nullid)
+        parents = p1 and (p1, self.repo.nullid)
         extra = ctx.extra()
         if self._useobsolete and self.ui.configbool(b'absorb', b'add-noise'):
             extra[b'absorb_source'] = ctx.hex()
 
         desc = rewriteutil.update_hash_refs(
             ctx.repo(),
             ctx.description(),
             {
                 oldnode: [newnode]
                 for oldnode, newnode in self.replacemap.items()
             },
         )
         mctx = overlaycontext(
             memworkingcopy, ctx, parents, extra=extra, desc=desc
         )
         return mctx.commit()
 
     @util.propertycache
     def _useobsolete(self):
         """() -> bool"""
         return obsolete.isenabled(self.repo, obsolete.createmarkersopt)
 
     def _cleanupoldcommits(self):
         replacements = {
             k: ([v] if v is not None else [])
             for k, v in pycompat.iteritems(self.replacemap)
         }
         if replacements:
             scmutil.cleanupnodes(
                 self.repo, replacements, operation=b'absorb', fixphase=True
             )
 
     @util.propertycache
     def skip_empty_successor(self):
         return rewriteutil.skip_empty_successor(self.ui, b'absorb')
 
 
 def _parsechunk(hunk):
     """(crecord.uihunk or patch.recordhunk) -> (path, (a1, a2, [bline]))"""
     if type(hunk) not in (crecord.uihunk, patch.recordhunk):
         return None, None
     path = hunk.header.filename()
     a1 = hunk.fromline + len(hunk.before) - 1
     # remove before and after context
     hunk.before = hunk.after = []
     buf = util.stringio()
     hunk.write(buf)
     patchlines = mdiff.splitnewlines(buf.getvalue())
     # hunk.prettystr() will update hunk.removed
     a2 = a1 + hunk.removed
     blines = [l[1:] for l in patchlines[1:] if not l.startswith(b'-')]
     return path, (a1, a2, blines)
 
 
 def overlaydiffcontext(ctx, chunks):
     """(ctx, [crecord.uihunk]) -> memctx
 
     return a memctx with some [1] patches (chunks) applied to ctx.
     [1]: modifications are handled. renames, mode changes, etc. are ignored.
     """
     # sadly the applying-patch logic is hardly reusable, and messy:
     # 1. the core logic "_applydiff" is too heavy - it writes .rej files, it
     #    needs a file stream of a patch and will re-parse it, while we have
     #    structured hunk objects at hand.
     # 2. a lot of different implementations about "chunk" (patch.hunk,
     #    patch.recordhunk, crecord.uihunk)
     # as we only care about applying changes to modified files, no mode
     # change, no binary diff, and no renames, it's probably okay to
     # re-invent the logic using much simpler code here.
     memworkingcopy = {}  # {path: content}
     patchmap = defaultdict(lambda: [])  # {path: [(a1, a2, [bline])]}
     for path, info in map(_parsechunk, chunks):
         if not path or not info:
             continue
         patchmap[path].append(info)
     for path, patches in pycompat.iteritems(patchmap):
         if path not in ctx or not patches:
             continue
         patches.sort(reverse=True)
         lines = mdiff.splitnewlines(ctx[path].data())
         for a1, a2, blines in patches:
             lines[a1:a2] = blines
         memworkingcopy[path] = b''.join(lines)
     return overlaycontext(memworkingcopy, ctx)
 
 
 def absorb(ui, repo, stack=None, targetctx=None, pats=None, opts=None):
     """pick fixup chunks from targetctx, apply them to stack.
 
     if targetctx is None, the working copy context will be used.
     if stack is None, the current draft stack will be used.
     return fixupstate.
     """
     if stack is None:
         limit = ui.configint(b'absorb', b'max-stack-size')
         headctx = repo[b'.']
         if len(headctx.parents()) > 1:
             raise error.InputError(_(b'cannot absorb into a merge'))
         stack = getdraftstack(headctx, limit)
         if limit and len(stack) >= limit:
             ui.warn(
                 _(
                     b'absorb: only the recent %d changesets will '
                     b'be analysed\n'
                 )
                 % limit
             )
     if not stack:
         raise error.InputError(_(b'no mutable changeset to change'))
     if targetctx is None:  # default to working copy
         targetctx = repo[None]
     if pats is None:
         pats = ()
     if opts is None:
         opts = {}
     state = fixupstate(stack, ui=ui, opts=opts)
     matcher = scmutil.match(targetctx, pats, opts)
     if opts.get(b'interactive'):
         diff = patch.diff(repo, stack[-1].node(), targetctx.node(), matcher)
         origchunks = patch.parsepatch(diff)
         chunks = cmdutil.recordfilter(ui, origchunks, matcher)[0]
         targetctx = overlaydiffcontext(stack[-1], chunks)
     fm = None
     if opts.get(b'print_changes') or not opts.get(b'apply_changes'):
         fm = ui.formatter(b'absorb', opts)
     state.diffwith(targetctx, matcher, fm)
     if fm is not None:
         fm.startitem()
         fm.write(
             b"count", b"\n%d changesets affected\n", len(state.ctxaffected)
         )
         fm.data(linetype=b'summary')
         for ctx in reversed(stack):
             if ctx not in state.ctxaffected:
                 continue
             fm.startitem()
             fm.context(ctx=ctx)
             fm.data(linetype=b'changeset')
             fm.write(b'node', b'%-7.7s ', ctx.hex(), label=b'absorb.node')
             descfirstline = ctx.description().splitlines()[0]
             fm.write(
                 b'descfirstline',
                 b'%s\n',
                 descfirstline,
                 label=b'absorb.description',
             )
         fm.end()
     if not opts.get(b'dry_run'):
         if (
             not opts.get(b'apply_changes')
             and state.ctxaffected
             and ui.promptchoice(
                 b"apply changes (y/N)? $$ &Yes $$ &No", default=1
             )
         ):
             raise error.CanceledError(_(b'absorb cancelled\n'))
 
         state.apply()
         if state.commit():
             state.printchunkstats()
         elif not ui.quiet:
             ui.write(_(b'nothing applied\n'))
     return state
 
 
 @command(
     b'absorb',
     [
         (
             b'a',
             b'apply-changes',
             None,
             _(b'apply changes without prompting for confirmation'),
         ),
         (
             b'p',
             b'print-changes',
             None,
             _(b'always print which changesets are modified by which changes'),
         ),
         (
             b'i',
             b'interactive',
             None,
             _(b'interactively select which chunks to apply'),
         ),
         (
             b'e',
             b'edit-lines',
             None,
             _(
                 b'edit what lines belong to which changesets before commit '
                 b'(EXPERIMENTAL)'
             ),
         ),
     ]
     + commands.dryrunopts
     + commands.templateopts
     + commands.walkopts,
     _(b'hg absorb [OPTION] [FILE]...'),
     helpcategory=command.CATEGORY_COMMITTING,
     helpbasic=True,
 )
 def absorbcmd(ui, repo, *pats, **opts):
     """incorporate corrections into the stack of draft changesets
 
     absorb analyzes each change in your working directory and attempts to
     amend the changed lines into the changesets in your stack that first
     introduced those lines.
 
     If absorb cannot find an unambiguous changeset to amend for a change,
     that change will be left in the working directory, untouched. They can be
     observed by :hg:`status` or :hg:`diff` afterwards. In other words,
     absorb does not write to the working directory.
 
     Changesets outside the revset `::. and not public() and not merge()` will
     not be changed.
 
     Changesets that become empty after applying the changes will be deleted.
 
     By default, absorb will show what it plans to do and prompt for
     confirmation.  If you are confident that the changes will be absorbed
     to the correct place, run :hg:`absorb -a` to apply the changes
     immediately.
 
     Returns 0 on success, 1 if all chunks were ignored and nothing amended.
     """
     opts = pycompat.byteskwargs(opts)
 
     with repo.wlock(), repo.lock():
         if not opts[b'dry_run']:
             cmdutil.checkunfinished(repo)
 
         state = absorb(ui, repo, pats=pats, opts=opts)
         if sum(s[0] for s in state.chunkstats.values()) == 0:
             return 1

hgext/amend.py

@@ -1,76 +1,74 b''
 # amend.py - provide the amend command
 #
 # Copyright 2017 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 """provide the amend command (EXPERIMENTAL)
 
 This extension provides an ``amend`` command that is similar to
 ``commit --amend`` but does not prompt an editor.
 """
 
 from __future__ import absolute_import
 
 from mercurial.i18n import _
 from mercurial import (
     cmdutil,
     commands,
-    pycompat,
     registrar,
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 
 @command(
     b'amend',
     [
         (
             b'A',
             b'addremove',
             None,
             _(b'mark new/missing files as added/removed before committing'),
         ),
         (b'e', b'edit', None, _(b'invoke editor on commit messages')),
         (b'i', b'interactive', None, _(b'use interactive mode')),
         (
             b'',
             b'close-branch',
             None,
             _(b'mark a branch as closed, hiding it from the branch list'),
         ),
         (b's', b'secret', None, _(b'use the secret phase for committing')),
         (b'n', b'note', b'', _(b'store a note on the amend')),
     ]
     + cmdutil.walkopts
     + cmdutil.commitopts
     + cmdutil.commitopts2
     + cmdutil.commitopts3,
     _(b'[OPTION]... [FILE]...'),
     helpcategory=command.CATEGORY_COMMITTING,
     inferrepo=True,
 )
 def amend(ui, repo, *pats, **opts):
     """amend the working copy parent with all or specified outstanding changes
 
     Similar to :hg:`commit --amend`, but reuse the commit message without
     invoking editor, unless ``--edit`` was set.
 
     See :hg:`help commit` for more details.
     """
-    opts = pycompat.byteskwargs(opts)
-    cmdutil.checknotesize(ui, opts)
+    cmdutil.check_note_size(opts)
 
     with repo.wlock(), repo.lock():
-        if not opts.get(b'logfile'):
-            opts[b'message'] = opts.get(b'message') or repo[b'.'].description()
-        opts[b'amend'] = True
-        return commands._docommit(ui, repo, *pats, **pycompat.strkwargs(opts))
+        if not opts.get('logfile'):
+            opts['message'] = opts.get('message') or repo[b'.'].description()
+        opts['amend'] = True
+        return commands._docommit(ui, repo, *pats, **opts)

hgext/convert/bzr.py

@@ -1,335 +1,339 b''
 # bzr.py - bzr support for the convert extension
 #
 #  Copyright 2008, 2009 Marek Kubica <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
-# This module is for handling 'bzr', that was formerly known as Bazaar-NG;
-# it cannot access 'bar' repositories, but they were never used very much
+# This module is for handling Breezy imports or `brz`, but it's also compatible
+# with Bazaar or `bzr`, that was formerly known as Bazaar-NG;
+# it cannot access `bar` repositories, but they were never used very much.
 from __future__ import absolute_import
 
 import os
 
 from mercurial.i18n import _
 from mercurial import (
     demandimport,
     error,
     pycompat,
+    util,
 )
 from . import common
 
+
 # these do not work with demandimport, blacklist
 demandimport.IGNORES.update(
     [
-        b'bzrlib.transactions',
-        b'bzrlib.urlutils',
+        b'breezy.transactions',
+        b'breezy.urlutils',
         b'ElementPath',
     ]
 )
 
 try:
     # bazaar imports
-    import bzrlib.bzrdir
-    import bzrlib.errors
-    import bzrlib.revision
-    import bzrlib.revisionspec
+    import breezy.bzr.bzrdir
+    import breezy.errors
+    import breezy.revision
+    import breezy.revisionspec
 
-    bzrdir = bzrlib.bzrdir
-    errors = bzrlib.errors
-    revision = bzrlib.revision
-    revisionspec = bzrlib.revisionspec
+    bzrdir = breezy.bzr.bzrdir
+    errors = breezy.errors
+    revision = breezy.revision
+    revisionspec = breezy.revisionspec
     revisionspec.RevisionSpec
 except ImportError:
     pass
 
-supportedkinds = (b'file', b'symlink')
+supportedkinds = ('file', 'symlink')
 
 
 class bzr_source(common.converter_source):
     """Reads Bazaar repositories by using the Bazaar Python libraries"""
 
     def __init__(self, ui, repotype, path, revs=None):
         super(bzr_source, self).__init__(ui, repotype, path, revs=revs)
 
         if not os.path.exists(os.path.join(path, b'.bzr')):
             raise common.NoRepo(
                 _(b'%s does not look like a Bazaar repository') % path
             )
 
         try:
-            # access bzrlib stuff
+            # access breezy stuff
             bzrdir
         except NameError:
             raise common.NoRepo(_(b'Bazaar modules could not be loaded'))
 
-        path = os.path.abspath(path)
+        path = util.abspath(path)
         self._checkrepotype(path)
         try:
-            self.sourcerepo = bzrdir.BzrDir.open(path).open_repository()
+            bzr_dir = bzrdir.BzrDir.open(path.decode())
+            self.sourcerepo = bzr_dir.open_repository()
         except errors.NoRepositoryPresent:
             raise common.NoRepo(
                 _(b'%s does not look like a Bazaar repository') % path
             )
         self._parentids = {}
         self._saverev = ui.configbool(b'convert', b'bzr.saverev')
 
     def _checkrepotype(self, path):
         # Lightweight checkouts detection is informational but probably
         # fragile at API level. It should not terminate the conversion.
         try:
-            dir = bzrdir.BzrDir.open_containing(path)[0]
+            dir = bzrdir.BzrDir.open_containing(path.decode())[0]
             try:
                 tree = dir.open_workingtree(recommend_upgrade=False)
                 branch = tree.branch
             except (errors.NoWorkingTree, errors.NotLocalUrl):
                 tree = None
                 branch = dir.open_branch()
             if (
                 tree is not None
-                and tree.bzrdir.root_transport.base
-                != branch.bzrdir.root_transport.base
+                and tree.controldir.root_transport.base
+                != branch.controldir.root_transport.base
             ):
                 self.ui.warn(
                     _(
                         b'warning: lightweight checkouts may cause '
                         b'conversion failures, try with a regular '
                         b'branch instead.\n'
                     )
                 )
         except Exception:
             self.ui.note(_(b'bzr source type could not be determined\n'))
 
     def before(self):
         """Before the conversion begins, acquire a read lock
         for all the operations that might need it. Fortunately
         read locks don't block other reads or writes to the
         repository, so this shouldn't have any impact on the usage of
         the source repository.
 
         The alternative would be locking on every operation that
         needs locks (there are currently two: getting the file and
         getting the parent map) and releasing immediately after,
         but this approach can take even 40% longer."""
         self.sourcerepo.lock_read()
 
     def after(self):
         self.sourcerepo.unlock()
 
     def _bzrbranches(self):
         return self.sourcerepo.find_branches(using=True)
 
     def getheads(self):
         if not self.revs:
             # Set using=True to avoid nested repositories (see issue3254)
             heads = sorted([b.last_revision() for b in self._bzrbranches()])
         else:
             revid = None
             for branch in self._bzrbranches():
                 try:
-                    r = revisionspec.RevisionSpec.from_string(self.revs[0])
+                    revspec = self.revs[0].decode()
+                    r = revisionspec.RevisionSpec.from_string(revspec)
                     info = r.in_history(branch)
                 except errors.BzrError:
                     pass
                 revid = info.rev_id
             if revid is None:
                 raise error.Abort(
                     _(b'%s is not a valid revision') % self.revs[0]
                 )
             heads = [revid]
         # Empty repositories return 'null:', which cannot be retrieved
         heads = [h for h in heads if h != b'null:']
         return heads
 
     def getfile(self, name, rev):
+        name = name.decode()
         revtree = self.sourcerepo.revision_tree(rev)
-        fileid = revtree.path2id(name.decode(self.encoding or b'utf-8'))
-        kind = None
-        if fileid is not None:
-            kind = revtree.kind(fileid)
+
+        try:
+            kind = revtree.kind(name)
+        except breezy.errors.NoSuchFile:
+            return None, None
         if kind not in supportedkinds:
             # the file is not available anymore - was deleted
             return None, None
-        mode = self._modecache[(name, rev)]
-        if kind == b'symlink':
-            target = revtree.get_symlink_target(fileid)
+        mode = self._modecache[(name.encode(), rev)]
+        if kind == 'symlink':
+            target = revtree.get_symlink_target(name)
             if target is None:
                 raise error.Abort(
                     _(b'%s.%s symlink has no target') % (name, rev)
                 )
-            return target, mode
+            return target.encode(), mode
         else:
-            sio = revtree.get_file(fileid)
+            sio = revtree.get_file(name)
             return sio.read(), mode
 
     def getchanges(self, version, full):
         if full:
             raise error.Abort(_(b"convert from cvs does not support --full"))
         self._modecache = {}
         self._revtree = self.sourcerepo.revision_tree(version)
         # get the parentids from the cache
         parentids = self._parentids.pop(version)
         # only diff against first parent id
         prevtree = self.sourcerepo.revision_tree(parentids[0])
         files, changes = self._gettreechanges(self._revtree, prevtree)
         return files, changes, set()
 
     def getcommit(self, version):
         rev = self.sourcerepo.get_revision(version)
         # populate parent id cache
         if not rev.parent_ids:
             parents = []
             self._parentids[version] = (revision.NULL_REVISION,)
         else:
             parents = self._filterghosts(rev.parent_ids)
             self._parentids[version] = parents
 
-        branch = self.recode(rev.properties.get(b'branch-nick', u'default'))
-        if branch == b'trunk':
-            branch = b'default'
+        branch = rev.properties.get('branch-nick', 'default')
+        if branch == 'trunk':
+            branch = 'default'
         return common.commit(
             parents=parents,
             date=b'%d %d' % (rev.timestamp, -rev.timezone),
             author=self.recode(rev.committer),
             desc=self.recode(rev.message),
-            branch=branch,
+            branch=branch.encode('utf8'),
             rev=version,
             saverev=self._saverev,
         )
 
     def gettags(self):
         bytetags = {}
         for branch in self._bzrbranches():
             if not branch.supports_tags():
                 return {}
             tagdict = branch.tags.get_tag_dict()
             for name, rev in pycompat.iteritems(tagdict):
                 bytetags[self.recode(name)] = rev
         return bytetags
 
     def getchangedfiles(self, rev, i):
         self._modecache = {}
         curtree = self.sourcerepo.revision_tree(rev)
         if i is not None:
             parentid = self._parentids[rev][i]
         else:
             # no parent id, get the empty revision
             parentid = revision.NULL_REVISION
 
         prevtree = self.sourcerepo.revision_tree(parentid)
         changes = [e[0] for e in self._gettreechanges(curtree, prevtree)[0]]
         return changes
 
     def _gettreechanges(self, current, origin):
         revid = current._revision_id
         changes = []
         renames = {}
         seen = set()
 
         # Fall back to the deprecated attribute for legacy installations.
         try:
             inventory = origin.root_inventory
         except AttributeError:
             inventory = origin.inventory
 
         # Process the entries by reverse lexicographic name order to
         # handle nested renames correctly, most specific first.
+
+        def key(c):
+            return c.path[0] or c.path[1] or ""
+
         curchanges = sorted(
             current.iter_changes(origin),
-            key=lambda c: c[1][0] or c[1][1],
+            key=key,
             reverse=True,
         )
-        for (
-            fileid,
-            paths,
-            changed_content,
-            versioned,
-            parent,
-            name,
-            kind,
-            executable,
-        ) in curchanges:
-
+        for change in curchanges:
+            paths = change.path
+            kind = change.kind
+            executable = change.executable
             if paths[0] == u'' or paths[1] == u'':
                 # ignore changes to tree root
                 continue
 
             # bazaar tracks directories, mercurial does not, so
             # we have to rename the directory contents
-            if kind[1] == b'directory':
-                if kind[0] not in (None, b'directory'):
+            if kind[1] == 'directory':
+                if kind[0] not in (None, 'directory'):
                     # Replacing 'something' with a directory, record it
                     # so it can be removed.
                     changes.append((self.recode(paths[0]), revid))
 
-                if kind[0] == b'directory' and None not in paths:
+                if kind[0] == 'directory' and None not in paths:
                     renaming = paths[0] != paths[1]
                     # neither an add nor an delete - a move
                     # rename all directory contents manually
                     subdir = inventory.path2id(paths[0])
                     # get all child-entries of the directory
                     for name, entry in inventory.iter_entries(subdir):
                         # hg does not track directory renames
-                        if entry.kind == b'directory':
+                        if entry.kind == 'directory':
                             continue
-                        frompath = self.recode(paths[0] + b'/' + name)
+                        frompath = self.recode(paths[0] + '/' + name)
                         if frompath in seen:
                             # Already handled by a more specific change entry
                             # This is important when you have:
                             # a => b
                             # a/c => a/c
                             # Here a/c must not be renamed into b/c
                             continue
                         seen.add(frompath)
                         if not renaming:
                             continue
-                        topath = self.recode(paths[1] + b'/' + name)
+                        topath = self.recode(paths[1] + '/' + name)
                         # register the files as changed
                         changes.append((frompath, revid))
                         changes.append((topath, revid))
                         # add to mode cache
                         mode = (
                             (entry.executable and b'x')
-                            or (entry.kind == b'symlink' and b's')
+                            or (entry.kind == 'symlink' and b's')
                             or b''
                         )
                         self._modecache[(topath, revid)] = mode
                         # register the change as move
                         renames[topath] = frompath
 
                 # no further changes, go to the next change
                 continue
 
             # we got unicode paths, need to convert them
             path, topath = paths
             if path is not None:
                 path = self.recode(path)
             if topath is not None:
                 topath = self.recode(topath)
             seen.add(path or topath)
 
             if topath is None:
                 # file deleted
                 changes.append((path, revid))
                 continue
 
             # renamed
             if path and path != topath:
                 renames[topath] = path
                 changes.append((path, revid))
 
             # populate the mode cache
             kind, executable = [e[1] for e in (kind, executable)]
-            mode = (executable and b'x') or (kind == b'symlink' and b'l') or b''
+            mode = (executable and b'x') or (kind == 'symlink' and b'l') or b''
             self._modecache[(topath, revid)] = mode
             changes.append((topath, revid))
 
         return changes, renames
 
     def _filterghosts(self, ids):
         """Filters out ghost revisions which hg does not support, see
         <http://bazaar-vcs.org/GhostRevision>
         """
         parentmap = self.sourcerepo.get_parent_map(ids)
         parents = tuple([parent for parent in ids if parent in parentmap])
         return parents

hgext/convert/git.py

@@ -1,531 +1,532 b''
 # git.py - git support for the convert extension
 #
 #  Copyright 2005-2009 Olivia Mackall <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
 
 import os
 
 from mercurial.i18n import _
-from mercurial.node import nullhex
+from mercurial.node import sha1nodeconstants
 from mercurial import (
     config,
     error,
     pycompat,
+    util,
 )
 
 from . import common
 
 
 class submodule(object):
     def __init__(self, path, node, url):
         self.path = path
         self.node = node
         self.url = url
 
     def hgsub(self):
         return b"%s = [git]%s" % (self.path, self.url)
 
     def hgsubstate(self):
         return b"%s %s" % (self.node, self.path)
 
 
 # Keys in extra fields that should not be copied if the user requests.
 bannedextrakeys = {
     # Git commit object built-ins.
     b'tree',
     b'parent',
     b'author',
     b'committer',
     # Mercurial built-ins.
     b'branch',
     b'close',
 }
 
 
 class convert_git(common.converter_source, common.commandline):
     # Windows does not support GIT_DIR= construct while other systems
     # cannot remove environment variable. Just assume none have
     # both issues.
 
     def _gitcmd(self, cmd, *args, **kwargs):
         return cmd(b'--git-dir=%s' % self.path, *args, **kwargs)
 
     def gitrun0(self, *args, **kwargs):
         return self._gitcmd(self.run0, *args, **kwargs)
 
     def gitrun(self, *args, **kwargs):
         return self._gitcmd(self.run, *args, **kwargs)
 
     def gitrunlines0(self, *args, **kwargs):
         return self._gitcmd(self.runlines0, *args, **kwargs)
 
     def gitrunlines(self, *args, **kwargs):
         return self._gitcmd(self.runlines, *args, **kwargs)
 
     def gitpipe(self, *args, **kwargs):
         return self._gitcmd(self._run3, *args, **kwargs)
 
     def __init__(self, ui, repotype, path, revs=None):
         super(convert_git, self).__init__(ui, repotype, path, revs=revs)
         common.commandline.__init__(self, ui, b'git')
 
         # Pass an absolute path to git to prevent from ever being interpreted
         # as a URL
-        path = os.path.abspath(path)
+        path = util.abspath(path)
 
         if os.path.isdir(path + b"/.git"):
             path += b"/.git"
         if not os.path.exists(path + b"/objects"):
             raise common.NoRepo(
                 _(b"%s does not look like a Git repository") % path
             )
 
         # The default value (50) is based on the default for 'git diff'.
         similarity = ui.configint(b'convert', b'git.similarity')
         if similarity < 0 or similarity > 100:
             raise error.Abort(_(b'similarity must be between 0 and 100'))
         if similarity > 0:
             self.simopt = [b'-C%d%%' % similarity]
             findcopiesharder = ui.configbool(
                 b'convert', b'git.findcopiesharder'
             )
             if findcopiesharder:
                 self.simopt.append(b'--find-copies-harder')
 
             renamelimit = ui.configint(b'convert', b'git.renamelimit')
             self.simopt.append(b'-l%d' % renamelimit)
         else:
             self.simopt = []
 
         common.checktool(b'git', b'git')
 
         self.path = path
         self.submodules = []
 
         self.catfilepipe = self.gitpipe(b'cat-file', b'--batch')
 
         self.copyextrakeys = self.ui.configlist(b'convert', b'git.extrakeys')
         banned = set(self.copyextrakeys) & bannedextrakeys
         if banned:
             raise error.Abort(
                 _(b'copying of extra key is forbidden: %s')
                 % _(b', ').join(sorted(banned))
             )
 
         committeractions = self.ui.configlist(
             b'convert', b'git.committeractions'
         )
 
         messagedifferent = None
         messagealways = None
         for a in committeractions:
             if a.startswith((b'messagedifferent', b'messagealways')):
                 k = a
                 v = None
                 if b'=' in a:
                     k, v = a.split(b'=', 1)
 
                 if k == b'messagedifferent':
                     messagedifferent = v or b'committer:'
                 elif k == b'messagealways':
                     messagealways = v or b'committer:'
 
         if messagedifferent and messagealways:
             raise error.Abort(
                 _(
                     b'committeractions cannot define both '
                     b'messagedifferent and messagealways'
                 )
             )
 
         dropcommitter = b'dropcommitter' in committeractions
         replaceauthor = b'replaceauthor' in committeractions
 
         if dropcommitter and replaceauthor:
             raise error.Abort(
                 _(
                     b'committeractions cannot define both '
                     b'dropcommitter and replaceauthor'
                 )
             )
 
         if dropcommitter and messagealways:
             raise error.Abort(
                 _(
                     b'committeractions cannot define both '
                     b'dropcommitter and messagealways'
                 )
             )
 
         if not messagedifferent and not messagealways:
             messagedifferent = b'committer:'
 
         self.committeractions = {
             b'dropcommitter': dropcommitter,
             b'replaceauthor': replaceauthor,
             b'messagedifferent': messagedifferent,
             b'messagealways': messagealways,
         }
 
     def after(self):
         for f in self.catfilepipe:
             f.close()
 
     def getheads(self):
         if not self.revs:
             output, status = self.gitrun(
                 b'rev-parse', b'--branches', b'--remotes'
             )
             heads = output.splitlines()
             if status:
                 raise error.Abort(_(b'cannot retrieve git heads'))
         else:
             heads = []
             for rev in self.revs:
                 rawhead, ret = self.gitrun(b'rev-parse', b'--verify', rev)
                 heads.append(rawhead[:-1])
                 if ret:
                     raise error.Abort(_(b'cannot retrieve git head "%s"') % rev)
         return heads
 
     def catfile(self, rev, ftype):
-        if rev == nullhex:
+        if rev == sha1nodeconstants.nullhex:
             raise IOError
         self.catfilepipe[0].write(rev + b'\n')
         self.catfilepipe[0].flush()
         info = self.catfilepipe[1].readline().split()
         if info[1] != ftype:
             raise error.Abort(
                 _(b'cannot read %r object at %s')
                 % (pycompat.bytestr(ftype), rev)
             )
         size = int(info[2])
         data = self.catfilepipe[1].read(size)
         if len(data) < size:
             raise error.Abort(
                 _(b'cannot read %r object at %s: unexpected size')
                 % (ftype, rev)
             )
         # read the trailing newline
         self.catfilepipe[1].read(1)
         return data
 
     def getfile(self, name, rev):
-        if rev == nullhex:
+        if rev == sha1nodeconstants.nullhex:
             return None, None
         if name == b'.hgsub':
             data = b'\n'.join([m.hgsub() for m in self.submoditer()])
             mode = b''
         elif name == b'.hgsubstate':
             data = b'\n'.join([m.hgsubstate() for m in self.submoditer()])
             mode = b''
         else:
             data = self.catfile(rev, b"blob")
             mode = self.modecache[(name, rev)]
         return data, mode
 
     def submoditer(self):
-        null = nullhex
+        null = sha1nodeconstants.nullhex
         for m in sorted(self.submodules, key=lambda p: p.path):
             if m.node != null:
                 yield m
 
     def parsegitmodules(self, content):
         """Parse the formatted .gitmodules file, example file format:
         [submodule "sub"]\n
         \tpath = sub\n
         \turl = git://giturl\n
         """
         self.submodules = []
         c = config.config()
         # Each item in .gitmodules starts with whitespace that cant be parsed
         c.parse(
             b'.gitmodules',
             b'\n'.join(line.strip() for line in content.split(b'\n')),
         )
         for sec in c.sections():
             # turn the config object into a real dict
             s = dict(c.items(sec))
             if b'url' in s and b'path' in s:
                 self.submodules.append(submodule(s[b'path'], b'', s[b'url']))
 
     def retrievegitmodules(self, version):
         modules, ret = self.gitrun(
             b'show', b'%s:%s' % (version, b'.gitmodules')
         )
         if ret:
             # This can happen if a file is in the repo that has permissions
             # 160000, but there is no .gitmodules file.
             self.ui.warn(
                 _(b"warning: cannot read submodules config file in %s\n")
                 % version
             )
             return
 
         try:
             self.parsegitmodules(modules)
         except error.ParseError:
             self.ui.warn(
                 _(b"warning: unable to parse .gitmodules in %s\n") % version
             )
             return
 
         for m in self.submodules:
             node, ret = self.gitrun(b'rev-parse', b'%s:%s' % (version, m.path))
             if ret:
                 continue
             m.node = node.strip()
 
     def getchanges(self, version, full):
         if full:
             raise error.Abort(_(b"convert from git does not support --full"))
         self.modecache = {}
         cmd = (
             [b'diff-tree', b'-z', b'--root', b'-m', b'-r']
             + self.simopt
             + [version]
         )
         output, status = self.gitrun(*cmd)
         if status:
             raise error.Abort(_(b'cannot read changes in %s') % version)
         changes = []
         copies = {}
         seen = set()
         entry = None
         subexists = [False]
         subdeleted = [False]
         difftree = output.split(b'\x00')
         lcount = len(difftree)
         i = 0
 
         skipsubmodules = self.ui.configbool(b'convert', b'git.skipsubmodules')
 
         def add(entry, f, isdest):
             seen.add(f)
             h = entry[3]
             p = entry[1] == b"100755"
             s = entry[1] == b"120000"
             renamesource = not isdest and entry[4][0] == b'R'
 
             if f == b'.gitmodules':
                 if skipsubmodules:
                     return
 
                 subexists[0] = True
                 if entry[4] == b'D' or renamesource:
                     subdeleted[0] = True
-                    changes.append((b'.hgsub', nullhex))
+                    changes.append((b'.hgsub', sha1nodeconstants.nullhex))
                 else:
                     changes.append((b'.hgsub', b''))
             elif entry[1] == b'160000' or entry[0] == b':160000':
                 if not skipsubmodules:
                     subexists[0] = True
             else:
                 if renamesource:
-                    h = nullhex
+                    h = sha1nodeconstants.nullhex
                 self.modecache[(f, h)] = (p and b"x") or (s and b"l") or b""
                 changes.append((f, h))
 
         while i < lcount:
             l = difftree[i]
             i += 1
             if not entry:
                 if not l.startswith(b':'):
                     continue
                 entry = tuple(pycompat.bytestr(p) for p in l.split())
                 continue
             f = l
             if entry[4][0] == b'C':
                 copysrc = f
                 copydest = difftree[i]
                 i += 1
                 f = copydest
                 copies[copydest] = copysrc
             if f not in seen:
                 add(entry, f, False)
             # A file can be copied multiple times, or modified and copied
             # simultaneously. So f can be repeated even if fdest isn't.
             if entry[4][0] == b'R':
                 # rename: next line is the destination
                 fdest = difftree[i]
                 i += 1
                 if fdest not in seen:
                     add(entry, fdest, True)
                     # .gitmodules isn't imported at all, so it being copied to
                     # and fro doesn't really make sense
                     if f != b'.gitmodules' and fdest != b'.gitmodules':
                         copies[fdest] = f
             entry = None
 
         if subexists[0]:
             if subdeleted[0]:
-                changes.append((b'.hgsubstate', nullhex))
+                changes.append((b'.hgsubstate', sha1nodeconstants.nullhex))
             else:
                 self.retrievegitmodules(version)
                 changes.append((b'.hgsubstate', b''))
         return (changes, copies, set())
 
     def getcommit(self, version):
         c = self.catfile(version, b"commit")  # read the commit hash
         end = c.find(b"\n\n")
         message = c[end + 2 :]
         message = self.recode(message)
         l = c[:end].splitlines()
         parents = []
         author = committer = None
         extra = {}
         for e in l[1:]:
             n, v = e.split(b" ", 1)
             if n == b"author":
                 p = v.split()
                 tm, tz = p[-2:]
                 author = b" ".join(p[:-2])
                 if author[0] == b"<":
                     author = author[1:-1]
                 author = self.recode(author)
             if n == b"committer":
                 p = v.split()
                 tm, tz = p[-2:]
                 committer = b" ".join(p[:-2])
                 if committer[0] == b"<":
                     committer = committer[1:-1]
                 committer = self.recode(committer)
             if n == b"parent":
                 parents.append(v)
             if n in self.copyextrakeys:
                 extra[n] = v
 
         if self.committeractions[b'dropcommitter']:
             committer = None
         elif self.committeractions[b'replaceauthor']:
             author = committer
 
         if committer:
             messagealways = self.committeractions[b'messagealways']
             messagedifferent = self.committeractions[b'messagedifferent']
             if messagealways:
                 message += b'\n%s %s\n' % (messagealways, committer)
             elif messagedifferent and author != committer:
                 message += b'\n%s %s\n' % (messagedifferent, committer)
 
         tzs, tzh, tzm = tz[-5:-4] + b"1", tz[-4:-2], tz[-2:]
         tz = -int(tzs) * (int(tzh) * 3600 + int(tzm))
         date = tm + b" " + (b"%d" % tz)
         saverev = self.ui.configbool(b'convert', b'git.saverev')
 
         c = common.commit(
             parents=parents,
             date=date,
             author=author,
             desc=message,
             rev=version,
             extra=extra,
             saverev=saverev,
         )
         return c
 
     def numcommits(self):
         output, ret = self.gitrunlines(b'rev-list', b'--all')
         if ret:
             raise error.Abort(
                 _(b'cannot retrieve number of commits in %s') % self.path
             )
         return len(output)
 
     def gettags(self):
         tags = {}
         alltags = {}
         output, status = self.gitrunlines(b'ls-remote', b'--tags', self.path)
 
         if status:
             raise error.Abort(_(b'cannot read tags from %s') % self.path)
         prefix = b'refs/tags/'
 
         # Build complete list of tags, both annotated and bare ones
         for line in output:
             line = line.strip()
             if line.startswith(b"error:") or line.startswith(b"fatal:"):
                 raise error.Abort(_(b'cannot read tags from %s') % self.path)
             node, tag = line.split(None, 1)
             if not tag.startswith(prefix):
                 continue
             alltags[tag[len(prefix) :]] = node
 
         # Filter out tag objects for annotated tag refs
         for tag in alltags:
             if tag.endswith(b'^{}'):
                 tags[tag[:-3]] = alltags[tag]
             else:
                 if tag + b'^{}' in alltags:
                     continue
                 else:
                     tags[tag] = alltags[tag]
 
         return tags
 
     def getchangedfiles(self, version, i):
         changes = []
         if i is None:
             output, status = self.gitrunlines(
                 b'diff-tree', b'--root', b'-m', b'-r', version
             )
             if status:
                 raise error.Abort(_(b'cannot read changes in %s') % version)
             for l in output:
                 if b"\t" not in l:
                     continue
                 m, f = l[:-1].split(b"\t")
                 changes.append(f)
         else:
             output, status = self.gitrunlines(
                 b'diff-tree',
                 b'--name-only',
                 b'--root',
                 b'-r',
                 version,
                 b'%s^%d' % (version, i + 1),
                 b'--',
             )
             if status:
                 raise error.Abort(_(b'cannot read changes in %s') % version)
             changes = [f.rstrip(b'\n') for f in output]
 
         return changes
 
     def getbookmarks(self):
         bookmarks = {}
 
         # Handle local and remote branches
         remoteprefix = self.ui.config(b'convert', b'git.remoteprefix')
         reftypes = [
             # (git prefix, hg prefix)
             (b'refs/remotes/origin/', remoteprefix + b'/'),
             (b'refs/heads/', b''),
         ]
 
         exclude = {
             b'refs/remotes/origin/HEAD',
         }
 
         try:
             output, status = self.gitrunlines(b'show-ref')
             for line in output:
                 line = line.strip()
                 rev, name = line.split(None, 1)
                 # Process each type of branch
                 for gitprefix, hgprefix in reftypes:
                     if not name.startswith(gitprefix) or name in exclude:
                         continue
                     name = b'%s%s' % (hgprefix, name[len(gitprefix) :])
                     bookmarks[name] = rev
         except Exception:
             pass
 
         return bookmarks
 
     def checkrevformat(self, revstr, mapname=b'splicemap'):
         """git revision string is a 40 byte hex"""
         self.checkhexformat(revstr, mapname)