tests: port test-pathencode.py to Python 3

# skip-blame uninteresting changes, mainly b prefixes

Differential Revision: https://phab.mercurial-scm.org/D3475

contrib/python3-whitelist

@@ -1,463 +1,464
 test-abort-checkin.t
 test-add.t
 test-addremove-similar.t
 test-addremove.t
 test-amend-subrepo.t
 test-amend.t
 test-ancestor.py
 test-annotate.py
 test-annotate.t
 test-archive-symlinks.t
 test-atomictempfile.py
 test-audit-path.t
 test-audit-subrepo.t
 test-automv.t
 test-backout.t
 test-backwards-remove.t
 test-basic.t
 test-bheads.t
 test-bisect.t
 test-bisect2.t
 test-bisect3.t
 test-blackbox.t
 test-bookmarks-current.t
 test-bookmarks-merge.t
 test-bookmarks-rebase.t
 test-bookmarks-strip.t
 test-bookmarks.t
 test-branch-change.t
 test-branch-option.t
 test-branch-tag-confict.t
 test-branches.t
 test-bundle-phases.t
 test-bundle-type.t
 test-bundle-vs-outgoing.t
 test-bundle2-multiple-changegroups.t
 test-cappedreader.py
 test-casecollision.t
 test-cat.t
 test-censor.t
 test-changelog-exec.t
 test-check-commit.t
 test-check-execute.t
 test-check-interfaces.py
 test-check-module-imports.t
 test-check-pyflakes.t
 test-check-pylint.t
 test-check-shbang.t
 test-children.t
 test-clone-cgi.t
 test-clone-pull-corruption.t
 test-clone-r.t
 test-clone-update-order.t
 test-command-template.t
 test-commit-amend.t
 test-commit-interactive.t
 test-commit-multiple.t
 test-commit-unresolved.t
 test-commit.t
 test-committer.t
 test-completion.t
 test-config-env.py
 test-config.t
 test-conflict.t
 test-confused-revert.t
 test-contrib-check-code.t
 test-contrib-check-commit.t
 test-convert-authormap.t
 test-convert-clonebranches.t
 test-convert-datesort.t
 test-convert-filemap.t
 test-convert-hg-sink.t
 test-convert-hg-source.t
 test-convert-hg-startrev.t
 test-convert-splicemap.t
 test-convert-tagsbranch-topology.t
 test-copy-move-merge.t
 test-copy.t
 test-copytrace-heuristics.t
 test-debugbuilddag.t
 test-debugbundle.t
 test-debugextensions.t
 test-debugindexdot.t
 test-debugrename.t
 test-default-push.t
 test-diff-binary-file.t
 test-diff-change.t
 test-diff-copy-depth.t
 test-diff-hashes.t
 test-diff-ignore-whitespace.t
 test-diff-indent-heuristic.t
 test-diff-issue2761.t
 test-diff-newlines.t
 test-diff-reverse.t
 test-diff-subdir.t
 test-diff-unified.t
 test-diff-upgrade.t
 test-diffdir.t
 test-diffstat.t
 test-directaccess.t
 test-dirstate-backup.t
 test-dirstate-nonnormalset.t
 test-dirstate.t
 test-doctest.py
 test-double-merge.t
 test-drawdag.t
 test-duplicateoptions.py
 test-editor-filename.t
 test-empty-dir.t
 test-empty-file.t
 test-empty-group.t
 test-empty.t
 test-encode.t
 test-encoding-func.py
 test-encoding.t
 test-eol-add.t
 test-eol-clone.t
 test-eol-hook.t
 test-eol-tag.t
 test-eol-update.t
 test-excessive-merge.t
 test-exchange-obsmarkers-case-A1.t
 test-exchange-obsmarkers-case-A2.t
 test-exchange-obsmarkers-case-A3.t
 test-exchange-obsmarkers-case-A4.t
 test-exchange-obsmarkers-case-A5.t
 test-exchange-obsmarkers-case-A6.t
 test-exchange-obsmarkers-case-A7.t
 test-exchange-obsmarkers-case-B1.t
 test-exchange-obsmarkers-case-B2.t
 test-exchange-obsmarkers-case-B3.t
 test-exchange-obsmarkers-case-B4.t
 test-exchange-obsmarkers-case-B5.t
 test-exchange-obsmarkers-case-B6.t
 test-exchange-obsmarkers-case-B7.t
 test-exchange-obsmarkers-case-C1.t
 test-exchange-obsmarkers-case-C2.t
 test-exchange-obsmarkers-case-C3.t
 test-exchange-obsmarkers-case-C4.t
 test-exchange-obsmarkers-case-D1.t
 test-exchange-obsmarkers-case-D2.t
 test-exchange-obsmarkers-case-D3.t
 test-exchange-obsmarkers-case-D4.t
 test-execute-bit.t
 test-export.t
 test-extdata.t
 test-extdiff.t
 test-extra-filelog-entry.t
 test-filebranch.t
 test-fileset-generated.t
 test-fix-topology.t
 test-flags.t
 test-generaldelta.t
 test-getbundle.t
 test-git-export.t
 test-glog-topological.t
 test-gpg.t
 test-graft.t
 test-hg-parseurl.py
 test-hghave.t
 test-hgignore.t
 test-hgk.t
 test-hgweb-bundle.t
 test-hgweb-descend-empties.t
 test-hgweb-empty.t
 test-hgweb-removed.t
 test-hgwebdir-paths.py
 test-hgwebdirsym.t
 test-histedit-arguments.t
 test-histedit-base.t
 test-histedit-bookmark-motion.t
 test-histedit-commute.t
 test-histedit-drop.t
 test-histedit-edit.t
 test-histedit-fold-non-commute.t
 test-histedit-fold.t
 test-histedit-no-change.t
 test-histedit-non-commute-abort.t
 test-histedit-non-commute.t
 test-histedit-obsolete.t
 test-histedit-outgoing.t
 test-histedit-templates.t
 test-http-branchmap.t
 test-http-bundle1.t
 test-http-clone-r.t
 test-http.t
 test-hybridencode.py
 test-identify.t
 test-import-unknown.t
 test-import.t
 test-imports-checker.t
 test-incoming-outgoing.t
 test-inherit-mode.t
 test-issue1089.t
 test-issue1102.t
 test-issue1175.t
 test-issue1306.t
 test-issue1438.t
 test-issue1502.t
 test-issue1802.t
 test-issue1877.t
 test-issue1993.t
 test-issue2137.t
 test-issue3084.t
 test-issue4074.t
 test-issue522.t
 test-issue586.t
 test-issue612.t
 test-issue619.t
 test-issue660.t
 test-issue672.t
 test-issue842.t
 test-journal-exists.t
 test-journal-share.t
 test-journal.t
 test-largefiles-cache.t
 test-largefiles-misc.t
 test-largefiles-small-disk.t
 test-largefiles-update.t
 test-largefiles.t
 test-lfs-largefiles.t
 test-linerange.py
 test-locate.t
 test-lock-badness.t
 test-log-linerange.t
 test-log.t
 test-logexchange.t
 test-lrucachedict.py
 test-mactext.t
 test-mailmap.t
 test-manifest-merging.t
 test-manifest.py
 test-manifest.t
 test-match.py
 test-mdiff.py
 test-merge-changedelete.t
 test-merge-closedheads.t
 test-merge-commit.t
 test-merge-criss-cross.t
 test-merge-default.t
 test-merge-force.t
 test-merge-halt.t
 test-merge-internal-tools-pattern.t
 test-merge-local.t
 test-merge-remove.t
 test-merge-revert.t
 test-merge-revert2.t
 test-merge-subrepos.t
 test-merge-symlinks.t
 test-merge-tools.t
 test-merge-types.t
 test-merge1.t
 test-merge10.t
 test-merge2.t
 test-merge4.t
 test-merge5.t
 test-merge6.t
 test-merge7.t
 test-merge8.t
 test-merge9.t
 test-minifileset.py
 test-mq-git.t
 test-mq-header-date.t
 test-mq-header-from.t
 test-mq-merge.t
 test-mq-pull-from-bundle.t
 test-mq-qclone-http.t
 test-mq-qdelete.t
 test-mq-qdiff.t
 test-mq-qfold.t
 test-mq-qgoto.t
 test-mq-qimport-fail-cleanup.t
 test-mq-qnew.t
 test-mq-qpush-exact.t
 test-mq-qqueue.t
 test-mq-qrefresh-interactive.t
 test-mq-qrefresh-replace-log-message.t
 test-mq-qrefresh.t
 test-mq-qrename.t
 test-mq-qsave.t
 test-mq-safety.t
 test-mq-subrepo.t
 test-mq-symlinks.t
 test-mv-cp-st-diff.t
 test-narrow-archive.t
 test-narrow-clone-no-ellipsis.t
 test-narrow-clone-non-narrow-server.t
 test-narrow-clone-nonlinear.t
 test-narrow-clone.t
 test-narrow-commit.t
 test-narrow-copies.t
 test-narrow-debugcommands.t
 test-narrow-debugrebuilddirstate.t
 test-narrow-exchange-merges.t
 test-narrow-exchange.t
 test-narrow-expanddirstate.t
 test-narrow-merge.t
 test-narrow-patch.t
 test-narrow-patterns.t
 test-narrow-pull.t
 test-narrow-rebase.t
 test-narrow-shallow-merges.t
 test-narrow-shallow.t
 test-narrow-strip.t
 test-narrow-update.t
 test-nested-repo.t
 test-newbranch.t
 test-obshistory.t
 test-obsmarker-template.t
 test-obsmarkers-effectflag.t
 test-obsolete-bundle-strip.t
 test-obsolete-changeset-exchange.t
 test-obsolete-checkheads.t
 test-obsolete-distributed.t
 test-obsolete-tag-cache.t
 test-parents.t
 test-pathconflicts-merge.t
 test-pathconflicts-update.t
+test-pathencode.py
 test-pending.t
 test-permissions.t
 test-phases.t
 test-pull-branch.t
 test-pull-http.t
 test-pull-permission.t
 test-pull-pull-corruption.t
 test-pull-r.t
 test-pull-update.t
 test-purge.t
 test-push-checkheads-partial-C1.t
 test-push-checkheads-partial-C2.t
 test-push-checkheads-partial-C3.t
 test-push-checkheads-partial-C4.t
 test-push-checkheads-pruned-B1.t
 test-push-checkheads-pruned-B2.t
 test-push-checkheads-pruned-B3.t
 test-push-checkheads-pruned-B4.t
 test-push-checkheads-pruned-B5.t
 test-push-checkheads-pruned-B6.t
 test-push-checkheads-pruned-B7.t
 test-push-checkheads-pruned-B8.t
 test-push-checkheads-superceed-A1.t
 test-push-checkheads-superceed-A2.t
 test-push-checkheads-superceed-A3.t
 test-push-checkheads-superceed-A4.t
 test-push-checkheads-superceed-A5.t
 test-push-checkheads-superceed-A6.t
 test-push-checkheads-superceed-A7.t
 test-push-checkheads-superceed-A8.t
 test-push-checkheads-unpushed-D1.t
 test-push-checkheads-unpushed-D2.t
 test-push-checkheads-unpushed-D3.t
 test-push-checkheads-unpushed-D4.t
 test-push-checkheads-unpushed-D5.t
 test-push-checkheads-unpushed-D6.t
 test-push-checkheads-unpushed-D7.t
 test-push-http.t
 test-push-warn.t
 test-pushvars.t
 test-rebase-abort.t
 test-rebase-base-flag.t
 test-rebase-bookmarks.t
 test-rebase-brute-force.t
 test-rebase-cache.t
 test-rebase-check-restore.t
 test-rebase-collapse.t
 test-rebase-conflicts.t
 test-rebase-dest.t
 test-rebase-detach.t
 test-rebase-emptycommit.t
 test-rebase-inmemory.t
 test-rebase-interruptions.t
 test-rebase-issue-noparam-single-rev.t
 test-rebase-legacy.t
 test-rebase-mq-skip.t
 test-rebase-mq.t
 test-rebase-named-branches.t
 test-rebase-newancestor.t
 test-rebase-obsolete.t
 test-rebase-parameters.t
 test-rebase-partial.t
 test-rebase-pull.t
 test-rebase-rename.t
 test-rebase-scenario-global.t
 test-rebase-templates.t
 test-rebase-transaction.t
 test-record.t
 test-relink.t
 test-remove.t
 test-rename-after-merge.t
 test-rename-dir-merge.t
 test-rename-merge1.t
 test-rename.t
 test-repair-strip.t
 test-repo-compengines.t
 test-resolve.t
 test-revert-flags.t
 test-revert-unknown.t
 test-revlog-ancestry.py
 test-revlog-group-emptyiter.t
 test-revlog-mmapindex.t
 test-revlog-packentry.t
 test-revset-dirstate-parents.t
 test-revset-legacy-lookup.t
 test-revset-outgoing.t
 test-rollback.t
 test-run-tests.py
 test-run-tests.t
 test-schemes.t
 test-serve.t
 test-setdiscovery.t
 test-share.t
 test-shelve.t
 test-show-stack.t
 test-show-work.t
 test-show.t
 test-simple-update.t
 test-single-head.t
 test-sparse-clear.t
 test-sparse-import.t
 test-sparse-merges.t
 test-sparse-profiles.t
 test-sparse-requirement.t
 test-sparse-verbose-json.t
 test-ssh-clone-r.t
 test-ssh-proto.t
 test-sshserver.py
 test-stack.t
 test-status-rev.t
 test-status-terse.t
 test-strip-cross.t
 test-strip.t
 test-subrepo-deep-nested-change.t
 test-subrepo-missing.t
 test-subrepo-recursion.t
 test-subrepo-relative-path.t
 test-subrepo.t
 test-symlinks.t
 test-tag.t
 test-tags.t
 test-template-engine.t
 test-treemanifest.t
 test-unamend.t
 test-uncommit.t
 test-unified-test.t
 test-unrelated-pull.t
 test-up-local-change.t
 test-update-branches.t
 test-update-dest.t
 test-update-issue1456.t
 test-update-names.t
 test-update-reverse.t
 test-upgrade-repo.t
 test-url-download.t
 test-url-rev.t
 test-url.py
 test-username-newline.t
 test-verify.t
 test-walkrepo.py
 test-websub.t
 test-win32text.t
 test-wireproto-clientreactor.py
 test-wireproto-framing.py
 test-wireproto-serverreactor.py
 test-xdg.t

tests/test-pathencode.py

@@ -1,209 +1,210
 # This is a randomized test that generates different pathnames every
 # time it is invoked, and tests the encoding of those pathnames.
 #
 # It uses a simple probabilistic model to generate valid pathnames
 # that have proven likely to expose bugs and divergent behavior in
 # different encoding implementations.
 
 from __future__ import absolute_import, print_function
 
 import binascii
 import collections
 import itertools
 import math
 import os
 import random
 import sys
 import time
 from mercurial import (
+    pycompat,
     store,
 )
 
 try:
     xrange
 except NameError:
     xrange = range
 
-validchars = set(map(chr, range(0, 256)))
+validchars = set(map(pycompat.bytechr, range(0, 256)))
 alphanum = range(ord('A'), ord('Z'))
 
-for c in '\0/':
+for c in (b'\0', b'/'):
     validchars.remove(c)
 
-winreserved = ('aux con prn nul'.split() +
-               ['com%d' % i for i in xrange(1, 10)] +
-               ['lpt%d' % i for i in xrange(1, 10)])
+winreserved = (b'aux con prn nul'.split() +
+               [b'com%d' % i for i in xrange(1, 10)] +
+               [b'lpt%d' % i for i in xrange(1, 10)])
 
 def casecombinations(names):
     '''Build all case-diddled combinations of names.'''
 
     combos = set()
 
     for r in names:
         for i in xrange(len(r) + 1):
             for c in itertools.combinations(xrange(len(r)), i):
                 d = r
                 for j in c:
-                    d = ''.join((d[:j], d[j].upper(), d[j + 1:]))
+                    d = b''.join((d[:j], d[j:j + 1].upper(), d[j + 1:]))
                 combos.add(d)
     return sorted(combos)
 
 def buildprobtable(fp, cmd='hg manifest tip'):
     '''Construct and print a table of probabilities for path name
     components.  The numbers are percentages.'''
 
     counts = collections.defaultdict(lambda: 0)
     for line in os.popen(cmd).read().splitlines():
         if line[-2:] in ('.i', '.d'):
             line = line[:-2]
         if line.startswith('data/'):
             line = line[5:]
         for c in line:
             counts[c] += 1
     for c in '\r/\n':
         counts.pop(c, None)
     t = sum(counts.itervalues()) / 100.0
     fp.write('probtable = (')
     for i, (k, v) in enumerate(sorted(counts.items(), key=lambda x: x[1],
                                       reverse=True)):
         if (i % 5) == 0:
             fp.write('\n    ')
         vt = v / t
         if vt < 0.0005:
             break
         fp.write('(%r, %.03f), ' % (k, vt))
     fp.write('\n    )\n')
 
 # A table of character frequencies (as percentages), gleaned by
 # looking at filelog names from a real-world, very large repo.
 
 probtable = (
-    ('t', 9.828), ('e', 9.042), ('s', 8.011), ('a', 6.801), ('i', 6.618),
-    ('g', 5.053), ('r', 5.030), ('o', 4.887), ('p', 4.363), ('n', 4.258),
-    ('l', 3.830), ('h', 3.693), ('_', 3.659), ('.', 3.377), ('m', 3.194),
-    ('u', 2.364), ('d', 2.296), ('c', 2.163), ('b', 1.739), ('f', 1.625),
-    ('6', 0.666), ('j', 0.610), ('y', 0.554), ('x', 0.487), ('w', 0.477),
-    ('k', 0.476), ('v', 0.473), ('3', 0.336), ('1', 0.335), ('2', 0.326),
-    ('4', 0.310), ('5', 0.305), ('9', 0.302), ('8', 0.300), ('7', 0.299),
-    ('q', 0.298), ('0', 0.250), ('z', 0.223), ('-', 0.118), ('C', 0.095),
-    ('T', 0.087), ('F', 0.085), ('B', 0.077), ('S', 0.076), ('P', 0.076),
-    ('L', 0.059), ('A', 0.058), ('N', 0.051), ('D', 0.049), ('M', 0.046),
-    ('E', 0.039), ('I', 0.035), ('R', 0.035), ('G', 0.028), ('U', 0.026),
-    ('W', 0.025), ('O', 0.017), ('V', 0.015), ('H', 0.013), ('Q', 0.011),
-    ('J', 0.007), ('K', 0.005), ('+', 0.004), ('X', 0.003), ('Y', 0.001),
+    (b't', 9.828), (b'e', 9.042), (b's', 8.011), (b'a', 6.801), (b'i', 6.618),
+    (b'g', 5.053), (b'r', 5.030), (b'o', 4.887), (b'p', 4.363), (b'n', 4.258),
+    (b'l', 3.830), (b'h', 3.693), (b'_', 3.659), (b'.', 3.377), (b'm', 3.194),
+    (b'u', 2.364), (b'd', 2.296), (b'c', 2.163), (b'b', 1.739), (b'f', 1.625),
+    (b'6', 0.666), (b'j', 0.610), (b'y', 0.554), (b'x', 0.487), (b'w', 0.477),
+    (b'k', 0.476), (b'v', 0.473), (b'3', 0.336), (b'1', 0.335), (b'2', 0.326),
+    (b'4', 0.310), (b'5', 0.305), (b'9', 0.302), (b'8', 0.300), (b'7', 0.299),
+    (b'q', 0.298), (b'0', 0.250), (b'z', 0.223), (b'-', 0.118), (b'C', 0.095),
+    (b'T', 0.087), (b'F', 0.085), (b'B', 0.077), (b'S', 0.076), (b'P', 0.076),
+    (b'L', 0.059), (b'A', 0.058), (b'N', 0.051), (b'D', 0.049), (b'M', 0.046),
+    (b'E', 0.039), (b'I', 0.035), (b'R', 0.035), (b'G', 0.028), (b'U', 0.026),
+    (b'W', 0.025), (b'O', 0.017), (b'V', 0.015), (b'H', 0.013), (b'Q', 0.011),
+    (b'J', 0.007), (b'K', 0.005), (b'+', 0.004), (b'X', 0.003), (b'Y', 0.001),
     )
 
 for c, _ in probtable:
     validchars.remove(c)
 validchars = list(validchars)
 
 def pickfrom(rng, table):
     c = 0
     r = rng.random() * sum(i[1] for i in table)
     for i, p in table:
         c += p
         if c >= r:
             return i
 
 reservedcombos = casecombinations(winreserved)
 
 # The first component of a name following a slash.
 
 firsttable = (
     (lambda rng: pickfrom(rng, probtable), 90),
     (lambda rng: rng.choice(validchars), 5),
     (lambda rng: rng.choice(reservedcombos), 5),
     )
 
 # Components of a name following the first.
 
 resttable = firsttable[:-1]
 
 # Special suffixes.
 
-internalsuffixcombos = casecombinations('.hg .i .d'.split())
+internalsuffixcombos = casecombinations(b'.hg .i .d'.split())
 
 # The last component of a path, before a slash or at the end of a name.
 
 lasttable = resttable + (
-    (lambda rng: '', 95),
+    (lambda rng: b'', 95),
     (lambda rng: rng.choice(internalsuffixcombos), 5),
     )
 
 def makepart(rng, k):
     '''Construct a part of a pathname, without slashes.'''
 
     p = pickfrom(rng, firsttable)(rng)
     l = len(p)
     ps = [p]
     maxl = rng.randint(1, k)
     while l < maxl:
         p = pickfrom(rng, resttable)(rng)
         l += len(p)
         ps.append(p)
     ps.append(pickfrom(rng, lasttable)(rng))
-    return ''.join(ps)
+    return b''.join(ps)
 
 def makepath(rng, j, k):
     '''Construct a complete pathname.'''
 
-    return ('data/' + '/'.join(makepart(rng, k) for _ in xrange(j)) +
-            rng.choice(['.d', '.i']))
+    return (b'data/' + b'/'.join(makepart(rng, k) for _ in xrange(j)) +
+            rng.choice([b'.d', b'.i']))
 
 def genpath(rng, count):
     '''Generate random pathnames with gradually increasing lengths.'''
 
     mink, maxk = 1, 4096
     def steps():
         for i in xrange(count):
             yield mink + int(round(math.sqrt((maxk - mink) * float(i) / count)))
     for k in steps():
         x = rng.randint(1, k)
         y = rng.randint(1, k)
         yield makepath(rng, x, y)
 
 def runtests(rng, seed, count):
     nerrs = 0
     for p in genpath(rng, count):
         h = store._pathencode(p)    # uses C implementation, if available
         r = store._hybridencode(p, True) # reference implementation in Python
         if h != r:
             if nerrs == 0:
                 print('seed:', hex(seed)[:-1], file=sys.stderr)
             print("\np: '%s'" % p.encode("string_escape"), file=sys.stderr)
             print("h: '%s'" % h.encode("string_escape"), file=sys.stderr)
             print("r: '%s'" % r.encode("string_escape"), file=sys.stderr)
             nerrs += 1
     return nerrs
 
 def main():
     import getopt
 
     # Empirically observed to take about a second to run
     count = 100
     seed = None
     opts, args = getopt.getopt(sys.argv[1:], 'c:s:',
                                ['build', 'count=', 'seed='])
     for o, a in opts:
         if o in ('-c', '--count'):
             count = int(a)
         elif o in ('-s', '--seed'):
             seed = int(a, base=0) # accepts base 10 or 16 strings
         elif o == '--build':
             buildprobtable(sys.stdout,
                            'find .hg/store/data -type f && '
                            'cat .hg/store/fncache 2>/dev/null')
             sys.exit(0)
 
     if seed is None:
         try:
             seed = int(binascii.hexlify(os.urandom(16)), 16)
         except AttributeError:
             seed = int(time.time() * 1000)
 
     rng = random.Random(seed)
     if runtests(rng, seed, count):
         sys.exit(1)
 
 if __name__ == '__main__':
     main()