global: bulk replace simple pycompat.iteritems(x) with x.items()

pycompat.iteritems() just calls .items().

This commit applies a regular expression search and replace to convert
simple instances of pycompat.iteritems() with .items(). There are still
a handful of calls to pycompat.iteritems() remaining. But these all have
more complicated expressions that I wasn't comfortable performing an
automated replace on. In addition, some simple replacements were withheld
because they broke pytype. These will be handled by their own changesets.

Differential Revision: https://phab.mercurial-scm.org/D12318

hgext/absorb.py

@@ -1,1164 +1,1161 b''
 # absorb.py
 #
 # Copyright 2016 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """apply working directory changes to changesets (EXPERIMENTAL)
 
 The absorb extension provides a command to use annotate information to
 amend modified chunks into the corresponding non-public changesets.
 
 ::
 
     [absorb]
     # only check 50 recent non-public changesets at most
     max-stack-size = 50
     # whether to add noise to new commits to avoid obsolescence cycle
     add-noise = 1
     # make `amend --correlated` a shortcut to the main command
     amend-flag = correlated
 
     [color]
     absorb.description = yellow
     absorb.node = blue bold
     absorb.path = bold
 """
 
 # TODO:
 #  * Rename config items to [commands] namespace
 #  * Converge getdraftstack() with other code in core
 #  * move many attributes on fixupstate to be private
 
 
 import collections
 
 from mercurial.i18n import _
 from mercurial.node import (
     hex,
     short,
 )
 from mercurial import (
     cmdutil,
     commands,
     context,
     crecord,
     error,
     linelog,
     mdiff,
     obsolete,
     patch,
     phases,
     pycompat,
     registrar,
     rewriteutil,
     scmutil,
     util,
 )
 from mercurial.utils import stringutil
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 configitem(b'absorb', b'add-noise', default=True)
 configitem(b'absorb', b'amend-flag', default=None)
 configitem(b'absorb', b'max-stack-size', default=50)
 
 colortable = {
     b'absorb.description': b'yellow',
     b'absorb.node': b'blue bold',
     b'absorb.path': b'bold',
 }
 
 defaultdict = collections.defaultdict
 
 
 class nullui(object):
     """blank ui object doing nothing"""
 
     debugflag = False
     verbose = False
     quiet = True
 
     def __getitem__(name):
         def nullfunc(*args, **kwds):
             return
 
         return nullfunc
 
 
 class emptyfilecontext(object):
     """minimal filecontext representing an empty file"""
 
     def __init__(self, repo):
         self._repo = repo
 
     def data(self):
         return b''
 
     def node(self):
         return self._repo.nullid
 
 
 def uniq(lst):
     """list -> list. remove duplicated items without changing the order"""
     seen = set()
     result = []
     for x in lst:
         if x not in seen:
             seen.add(x)
             result.append(x)
     return result
 
 
 def getdraftstack(headctx, limit=None):
     """(ctx, int?) -> [ctx]. get a linear stack of non-public changesets.
 
     changesets are sorted in topo order, oldest first.
     return at most limit items, if limit is a positive number.
 
     merges are considered as non-draft as well. i.e. every commit
     returned has and only has 1 parent.
     """
     ctx = headctx
     result = []
     while ctx.phase() != phases.public:
         if limit and len(result) >= limit:
             break
         parents = ctx.parents()
         if len(parents) != 1:
             break
         result.append(ctx)
         ctx = parents[0]
     result.reverse()
     return result
 
 
 def getfilestack(stack, path, seenfctxs=None):
     """([ctx], str, set) -> [fctx], {ctx: fctx}
 
     stack is a list of contexts, from old to new. usually they are what
     "getdraftstack" returns.
 
     follows renames, but not copies.
 
     seenfctxs is a set of filecontexts that will be considered "immutable".
     they are usually what this function returned in earlier calls, useful
     to avoid issues that a file was "moved" to multiple places and was then
     modified differently, like: "a" was copied to "b", "a" was also copied to
     "c" and then "a" was deleted, then both "b" and "c" were "moved" from "a"
     and we enforce only one of them to be able to affect "a"'s content.
 
     return an empty list and an empty dict, if the specified path does not
     exist in stack[-1] (the top of the stack).
 
     otherwise, return a list of de-duplicated filecontexts, and the map to
     convert ctx in the stack to fctx, for possible mutable fctxs. the first item
     of the list would be outside the stack and should be considered immutable.
     the remaining items are within the stack.
 
     for example, given the following changelog and corresponding filelog
     revisions:
 
       changelog: 3----4----5----6----7
       filelog:   x    0----1----1----2 (x: no such file yet)
 
     - if stack = [5, 6, 7], returns ([0, 1, 2], {5: 1, 6: 1, 7: 2})
     - if stack = [3, 4, 5], returns ([e, 0, 1], {4: 0, 5: 1}), where "e" is a
       dummy empty filecontext.
     - if stack = [2], returns ([], {})
     - if stack = [7], returns ([1, 2], {7: 2})
     - if stack = [6, 7], returns ([1, 2], {6: 1, 7: 2}), although {6: 1} can be
       removed, since 1 is immutable.
     """
     if seenfctxs is None:
         seenfctxs = set()
     assert stack
 
     if path not in stack[-1]:
         return [], {}
 
     fctxs = []
     fctxmap = {}
 
     pctx = stack[0].p1()  # the public (immutable) ctx we stop at
     for ctx in reversed(stack):
         if path not in ctx:  # the file is added in the next commit
             pctx = ctx
             break
         fctx = ctx[path]
         fctxs.append(fctx)
         if fctx in seenfctxs:  # treat fctx as the immutable one
             pctx = None  # do not add another immutable fctx
             break
         fctxmap[ctx] = fctx  # only for mutable fctxs
         copy = fctx.copysource()
         if copy:
             path = copy  # follow rename
             if path in ctx:  # but do not follow copy
                 pctx = ctx.p1()
                 break
 
     if pctx is not None:  # need an extra immutable fctx
         if path in pctx:
             fctxs.append(pctx[path])
         else:
             fctxs.append(emptyfilecontext(pctx.repo()))
 
     fctxs.reverse()
     # note: we rely on a property of hg: filerev is not reused for linear
     # history. i.e. it's impossible to have:
     #   changelog:  4----5----6 (linear, no merges)
     #   filelog:    1----2----1
     #                         ^ reuse filerev (impossible)
     # because parents are part of the hash. if that's not true, we need to
     # remove uniq and find a different way to identify fctxs.
     return uniq(fctxs), fctxmap
 
 
 class overlaystore(patch.filestore):
     """read-only, hybrid store based on a dict and ctx.
     memworkingcopy: {path: content}, overrides file contents.
     """
 
     def __init__(self, basectx, memworkingcopy):
         self.basectx = basectx
         self.memworkingcopy = memworkingcopy
 
     def getfile(self, path):
         """comply with mercurial.patch.filestore.getfile"""
         if path not in self.basectx:
             return None, None, None
         fctx = self.basectx[path]
         if path in self.memworkingcopy:
             content = self.memworkingcopy[path]
         else:
             content = fctx.data()
         mode = (fctx.islink(), fctx.isexec())
         copy = fctx.copysource()
         return content, mode, copy
 
 
 def overlaycontext(memworkingcopy, ctx, parents=None, extra=None, desc=None):
     """({path: content}, ctx, (p1node, p2node)?, {}?) -> memctx
     memworkingcopy overrides file contents.
     """
     # parents must contain 2 items: (node1, node2)
     if parents is None:
         parents = ctx.repo().changelog.parents(ctx.node())
     if extra is None:
         extra = ctx.extra()
     if desc is None:
         desc = ctx.description()
     date = ctx.date()
     user = ctx.user()
     files = set(ctx.files()).union(memworkingcopy)
     store = overlaystore(ctx, memworkingcopy)
     return context.memctx(
         repo=ctx.repo(),
         parents=parents,
         text=desc,
         files=files,
         filectxfn=store,
         user=user,
         date=date,
         branch=None,
         extra=extra,
     )
 
 
 class filefixupstate(object):
     """state needed to apply fixups to a single file
 
     internally, it keeps file contents of several revisions and a linelog.
 
     the linelog uses odd revision numbers for original contents (fctxs passed
     to __init__), and even revision numbers for fixups, like:
 
         linelog rev 1: self.fctxs[0] (from an immutable "public" changeset)
         linelog rev 2: fixups made to self.fctxs[0]
         linelog rev 3: self.fctxs[1] (a child of fctxs[0])
         linelog rev 4: fixups made to self.fctxs[1]
         ...
 
     a typical use is like:
 
         1. call diffwith, to calculate self.fixups
         2. (optionally), present self.fixups to the user, or change it
         3. call apply, to apply changes
         4. read results from "finalcontents", or call getfinalcontent
     """
 
     def __init__(self, fctxs, path, ui=None, opts=None):
         """([fctx], ui or None) -> None
 
         fctxs should be linear, and sorted by topo order - oldest first.
         fctxs[0] will be considered as "immutable" and will not be changed.
         """
         self.fctxs = fctxs
         self.path = path
         self.ui = ui or nullui()
         self.opts = opts or {}
 
         # following fields are built from fctxs. they exist for perf reason
         self.contents = [f.data() for f in fctxs]
         self.contentlines = pycompat.maplist(mdiff.splitnewlines, self.contents)
         self.linelog = self._buildlinelog()
         if self.ui.debugflag:
             assert self._checkoutlinelog() == self.contents
 
         # following fields will be filled later
         self.chunkstats = [0, 0]  # [adopted, total : int]
         self.targetlines = []  # [str]
         self.fixups = []  # [(linelog rev, a1, a2, b1, b2)]
         self.finalcontents = []  # [str]
         self.ctxaffected = set()
 
     def diffwith(self, targetfctx, fm=None):
         """calculate fixups needed by examining the differences between
         self.fctxs[-1] and targetfctx, chunk by chunk.
 
         targetfctx is the target state we move towards. we may or may not be
         able to get there because not all modified chunks can be amended into
         a non-public fctx unambiguously.
 
         call this only once, before apply().
 
         update self.fixups, self.chunkstats, and self.targetlines.
         """
         a = self.contents[-1]
         alines = self.contentlines[-1]
         b = targetfctx.data()
         blines = mdiff.splitnewlines(b)
         self.targetlines = blines
 
         self.linelog.annotate(self.linelog.maxrev)
         annotated = self.linelog.annotateresult  # [(linelog rev, linenum)]
         assert len(annotated) == len(alines)
         # add a dummy end line to make insertion at the end easier
         if annotated:
             dummyendline = (annotated[-1][0], annotated[-1][1] + 1)
             annotated.append(dummyendline)
 
         # analyse diff blocks
         for chunk in self._alldiffchunks(a, b, alines, blines):
             newfixups = self._analysediffchunk(chunk, annotated)
             self.chunkstats[0] += bool(newfixups)  # 1 or 0
             self.chunkstats[1] += 1
             self.fixups += newfixups
             if fm is not None:
                 self._showchanges(fm, alines, blines, chunk, newfixups)
 
     def apply(self):
         """apply self.fixups. update self.linelog, self.finalcontents.
 
         call this only once, before getfinalcontent(), after diffwith().
         """
         # the following is unnecessary, as it's done by "diffwith":
         #   self.linelog.annotate(self.linelog.maxrev)
         for rev, a1, a2, b1, b2 in reversed(self.fixups):
             blines = self.targetlines[b1:b2]
             if self.ui.debugflag:
                 idx = (max(rev - 1, 0)) // 2
                 self.ui.write(
                     _(b'%s: chunk %d:%d -> %d lines\n')
                     % (short(self.fctxs[idx].node()), a1, a2, len(blines))
                 )
             self.linelog.replacelines(rev, a1, a2, b1, b2)
         if self.opts.get(b'edit_lines', False):
             self.finalcontents = self._checkoutlinelogwithedits()
         else:
             self.finalcontents = self._checkoutlinelog()
 
     def getfinalcontent(self, fctx):
         """(fctx) -> str. get modified file content for a given filecontext"""
         idx = self.fctxs.index(fctx)
         return self.finalcontents[idx]
 
     def _analysediffchunk(self, chunk, annotated):
         """analyse a different chunk and return new fixups found
 
         return [] if no lines from the chunk can be safely applied.
 
         the chunk (or lines) cannot be safely applied, if, for example:
           - the modified (deleted) lines belong to a public changeset
             (self.fctxs[0])
           - the chunk is a pure insertion and the adjacent lines (at most 2
             lines) belong to different non-public changesets, or do not belong
             to any non-public changesets.
           - the chunk is modifying lines from different changesets.
             in this case, if the number of lines deleted equals to the number
             of lines added, assume it's a simple 1:1 map (could be wrong).
             otherwise, give up.
           - the chunk is modifying lines from a single non-public changeset,
             but other revisions touch the area as well. i.e. the lines are
             not continuous as seen from the linelog.
         """
         a1, a2, b1, b2 = chunk
         # find involved indexes from annotate result
         involved = annotated[a1:a2]
         if not involved and annotated:  # a1 == a2 and a is not empty
             # pure insertion, check nearby lines. ignore lines belong
             # to the public (first) changeset (i.e. annotated[i][0] == 1)
             nearbylinenums = {a2, max(0, a1 - 1)}
             involved = [
                 annotated[i] for i in nearbylinenums if annotated[i][0] != 1
             ]
         involvedrevs = list({r for r, l in involved})
         newfixups = []
         if len(involvedrevs) == 1 and self._iscontinuous(a1, a2 - 1, True):
             # chunk belongs to a single revision
             rev = involvedrevs[0]
             if rev > 1:
                 fixuprev = rev + 1
                 newfixups.append((fixuprev, a1, a2, b1, b2))
         elif a2 - a1 == b2 - b1 or b1 == b2:
             # 1:1 line mapping, or chunk was deleted
             for i in pycompat.xrange(a1, a2):
                 rev, linenum = annotated[i]
                 if rev > 1:
                     if b1 == b2:  # deletion, simply remove that single line
                         nb1 = nb2 = 0
                     else:  # 1:1 line mapping, change the corresponding rev
                         nb1 = b1 + i - a1
                         nb2 = nb1 + 1
                     fixuprev = rev + 1
                     newfixups.append((fixuprev, i, i + 1, nb1, nb2))
         return self._optimizefixups(newfixups)
 
     @staticmethod
     def _alldiffchunks(a, b, alines, blines):
         """like mdiff.allblocks, but only care about differences"""
         blocks = mdiff.allblocks(a, b, lines1=alines, lines2=blines)
         for chunk, btype in blocks:
             if btype != b'!':
                 continue
             yield chunk
 
     def _buildlinelog(self):
         """calculate the initial linelog based on self.content{,line}s.
         this is similar to running a partial "annotate".
         """
         llog = linelog.linelog()
         a, alines = b'', []
         for i in pycompat.xrange(len(self.contents)):
             b, blines = self.contents[i], self.contentlines[i]
             llrev = i * 2 + 1
             chunks = self._alldiffchunks(a, b, alines, blines)
             for a1, a2, b1, b2 in reversed(list(chunks)):
                 llog.replacelines(llrev, a1, a2, b1, b2)
             a, alines = b, blines
         return llog
 
     def _checkoutlinelog(self):
         """() -> [str]. check out file contents from linelog"""
         contents = []
         for i in pycompat.xrange(len(self.contents)):
             rev = (i + 1) * 2
             self.linelog.annotate(rev)
             content = b''.join(map(self._getline, self.linelog.annotateresult))
             contents.append(content)
         return contents
 
     def _checkoutlinelogwithedits(self):
         """() -> [str]. prompt all lines for edit"""
         alllines = self.linelog.getalllines()
         # header
         editortext = (
             _(
                 b'HG: editing %s\nHG: "y" means the line to the right '
                 b'exists in the changeset to the top\nHG:\n'
             )
             % self.fctxs[-1].path()
         )
         # [(idx, fctx)]. hide the dummy emptyfilecontext
         visiblefctxs = [
             (i, f)
             for i, f in enumerate(self.fctxs)
             if not isinstance(f, emptyfilecontext)
         ]
         for i, (j, f) in enumerate(visiblefctxs):
             editortext += _(b'HG: %s/%s %s %s\n') % (
                 b'|' * i,
                 b'-' * (len(visiblefctxs) - i + 1),
                 short(f.node()),
                 f.description().split(b'\n', 1)[0],
             )
         editortext += _(b'HG: %s\n') % (b'|' * len(visiblefctxs))
         # figure out the lifetime of a line, this is relatively inefficient,
         # but probably fine
         lineset = defaultdict(lambda: set())  # {(llrev, linenum): {llrev}}
         for i, f in visiblefctxs:
             self.linelog.annotate((i + 1) * 2)
             for l in self.linelog.annotateresult:
                 lineset[l].add(i)
         # append lines
         for l in alllines:
             editortext += b'    %s : %s' % (
                 b''.join(
                     [
                         (b'y' if i in lineset[l] else b' ')
                         for i, _f in visiblefctxs
                     ]
                 ),
                 self._getline(l),
             )
         # run editor
         editedtext = self.ui.edit(editortext, b'', action=b'absorb')
         if not editedtext:
             raise error.InputError(_(b'empty editor text'))
         # parse edited result
         contents = [b''] * len(self.fctxs)
         leftpadpos = 4
         colonpos = leftpadpos + len(visiblefctxs) + 1
         for l in mdiff.splitnewlines(editedtext):
             if l.startswith(b'HG:'):
                 continue
             if l[colonpos - 1 : colonpos + 2] != b' : ':
                 raise error.InputError(_(b'malformed line: %s') % l)
             linecontent = l[colonpos + 2 :]
             for i, ch in enumerate(
                 pycompat.bytestr(l[leftpadpos : colonpos - 1])
             ):
                 if ch == b'y':
                     contents[visiblefctxs[i][0]] += linecontent
         # chunkstats is hard to calculate if anything changes, therefore
         # set them to just a simple value (1, 1).
         if editedtext != editortext:
             self.chunkstats = [1, 1]
         return contents
 
     def _getline(self, lineinfo):
         """((rev, linenum)) -> str. convert rev+line number to line content"""
         rev, linenum = lineinfo
         if rev & 1:  # odd: original line taken from fctxs
             return self.contentlines[rev // 2][linenum]
         else:  # even: fixup line from targetfctx
             return self.targetlines[linenum]
 
     def _iscontinuous(self, a1, a2, closedinterval=False):
         """(a1, a2 : int) -> bool
 
         check if these lines are continuous. i.e. no other insertions or
         deletions (from other revisions) among these lines.
 
         closedinterval decides whether a2 should be included or not. i.e. is
         it [a1, a2), or [a1, a2] ?
         """
         if a1 >= a2:
             return True
         llog = self.linelog
         offset1 = llog.getoffset(a1)
         offset2 = llog.getoffset(a2) + int(closedinterval)
         linesinbetween = llog.getalllines(offset1, offset2)
         return len(linesinbetween) == a2 - a1 + int(closedinterval)
 
     def _optimizefixups(self, fixups):
         """[(rev, a1, a2, b1, b2)] -> [(rev, a1, a2, b1, b2)].
         merge adjacent fixups to make them less fragmented.
         """
         result = []
         pcurrentchunk = [[-1, -1, -1, -1, -1]]
 
         def pushchunk():
             if pcurrentchunk[0][0] != -1:
                 result.append(tuple(pcurrentchunk[0]))
 
         for i, chunk in enumerate(fixups):
             rev, a1, a2, b1, b2 = chunk
             lastrev = pcurrentchunk[0][0]
             lasta2 = pcurrentchunk[0][2]
             lastb2 = pcurrentchunk[0][4]
             if (
                 a1 == lasta2
                 and b1 == lastb2
                 and rev == lastrev
                 and self._iscontinuous(max(a1 - 1, 0), a1)
             ):
                 # merge into currentchunk
                 pcurrentchunk[0][2] = a2
                 pcurrentchunk[0][4] = b2
             else:
                 pushchunk()
                 pcurrentchunk[0] = list(chunk)
         pushchunk()
         return result
 
     def _showchanges(self, fm, alines, blines, chunk, fixups):
         def trim(line):
             if line.endswith(b'\n'):
                 line = line[:-1]
             return line
 
         # this is not optimized for perf but _showchanges only gets executed
         # with an extra command-line flag.
         a1, a2, b1, b2 = chunk
         aidxs, bidxs = [0] * (a2 - a1), [0] * (b2 - b1)
         for idx, fa1, fa2, fb1, fb2 in fixups:
             for i in pycompat.xrange(fa1, fa2):
                 aidxs[i - a1] = (max(idx, 1) - 1) // 2
             for i in pycompat.xrange(fb1, fb2):
                 bidxs[i - b1] = (max(idx, 1) - 1) // 2
 
         fm.startitem()
         fm.write(
             b'hunk',
             b'        %s\n',
             b'@@ -%d,%d +%d,%d @@' % (a1, a2 - a1, b1, b2 - b1),
             label=b'diff.hunk',
         )
         fm.data(path=self.path, linetype=b'hunk')
 
         def writeline(idx, diffchar, line, linetype, linelabel):
             fm.startitem()
             node = b''
             if idx:
                 ctx = self.fctxs[idx]
                 fm.context(fctx=ctx)
                 node = ctx.hex()
                 self.ctxaffected.add(ctx.changectx())
             fm.write(b'node', b'%-7.7s ', node, label=b'absorb.node')
             fm.write(
                 b'diffchar ' + linetype,
                 b'%s%s\n',
                 diffchar,
                 line,
                 label=linelabel,
             )
             fm.data(path=self.path, linetype=linetype)
 
         for i in pycompat.xrange(a1, a2):
             writeline(
                 aidxs[i - a1],
                 b'-',
                 trim(alines[i]),
                 b'deleted',
                 b'diff.deleted',
             )
         for i in pycompat.xrange(b1, b2):
             writeline(
                 bidxs[i - b1],
                 b'+',
                 trim(blines[i]),
                 b'inserted',
                 b'diff.inserted',
             )
 
 
 class fixupstate(object):
     """state needed to run absorb
 
     internally, it keeps paths and filefixupstates.
 
     a typical use is like filefixupstates:
 
         1. call diffwith, to calculate fixups
         2. (optionally), present fixups to the user, or edit fixups
         3. call apply, to apply changes to memory
         4. call commit, to commit changes to hg database
     """
 
     def __init__(self, stack, ui=None, opts=None):
         """([ctx], ui or None) -> None
 
         stack: should be linear, and sorted by topo order - oldest first.
         all commits in stack are considered mutable.
         """
         assert stack
         self.ui = ui or nullui()
         self.opts = opts or {}
         self.stack = stack
         self.repo = stack[-1].repo().unfiltered()
 
         # following fields will be filled later
         self.paths = []  # [str]
         self.status = None  # ctx.status output
         self.fctxmap = {}  # {path: {ctx: fctx}}
         self.fixupmap = {}  # {path: filefixupstate}
         self.replacemap = {}  # {oldnode: newnode or None}
         self.finalnode = None  # head after all fixups
         self.ctxaffected = set()  # ctx that will be absorbed into
 
     def diffwith(self, targetctx, match=None, fm=None):
         """diff and prepare fixups. update self.fixupmap, self.paths"""
         # only care about modified files
         self.status = self.stack[-1].status(targetctx, match)
         self.paths = []
         # but if --edit-lines is used, the user may want to edit files
         # even if they are not modified
         editopt = self.opts.get(b'edit_lines')
         if not self.status.modified and editopt and match:
             interestingpaths = match.files()
         else:
             interestingpaths = self.status.modified
         # prepare the filefixupstate
         seenfctxs = set()
         # sorting is necessary to eliminate ambiguity for the "double move"
         # case: "hg cp A B; hg cp A C; hg rm A", then only "B" can affect "A".
         for path in sorted(interestingpaths):
             self.ui.debug(b'calculating fixups for %s\n' % path)
             targetfctx = targetctx[path]
             fctxs, ctx2fctx = getfilestack(self.stack, path, seenfctxs)
             # ignore symbolic links or binary, or unchanged files
             if any(
                 f.islink() or stringutil.binary(f.data())
                 for f in [targetfctx] + fctxs
                 if not isinstance(f, emptyfilecontext)
             ):
                 continue
             if targetfctx.data() == fctxs[-1].data() and not editopt:
                 continue
             seenfctxs.update(fctxs[1:])
             self.fctxmap[path] = ctx2fctx
             fstate = filefixupstate(fctxs, path, ui=self.ui, opts=self.opts)
             if fm is not None:
                 fm.startitem()
                 fm.plain(b'showing changes for ')
                 fm.write(b'path', b'%s\n', path, label=b'absorb.path')
                 fm.data(linetype=b'path')
             fstate.diffwith(targetfctx, fm)
             self.fixupmap[path] = fstate
             self.paths.append(path)
             self.ctxaffected.update(fstate.ctxaffected)
 
     def apply(self):
         """apply fixups to individual filefixupstates"""
-        for path, state in pycompat.iteritems(self.fixupmap):
+        for path, state in self.fixupmap.items():
             if self.ui.debugflag:
                 self.ui.write(_(b'applying fixups to %s\n') % path)
             state.apply()
 
     @property
     def chunkstats(self):
         """-> {path: chunkstats}. collect chunkstats from filefixupstates"""
-        return {
-            path: state.chunkstats
-            for path, state in pycompat.iteritems(self.fixupmap)
-        }
+        return {path: state.chunkstats for path, state in self.fixupmap.items()}
 
     def commit(self):
         """commit changes. update self.finalnode, self.replacemap"""
         with self.repo.transaction(b'absorb') as tr:
             self._commitstack()
             self._movebookmarks(tr)
             if self.repo[b'.'].node() in self.replacemap:
                 self._moveworkingdirectoryparent()
             self._cleanupoldcommits()
         return self.finalnode
 
     def printchunkstats(self):
         """print things like '1 of 2 chunk(s) applied'"""
         ui = self.ui
         chunkstats = self.chunkstats
         if ui.verbose:
             # chunkstats for each file
-            for path, stat in pycompat.iteritems(chunkstats):
+            for path, stat in chunkstats.items():
                 if stat[0]:
                     ui.write(
                         _(b'%s: %d of %d chunk(s) applied\n')
                         % (path, stat[0], stat[1])
                     )
         elif not ui.quiet:
             # a summary for all files
             stats = chunkstats.values()
             applied, total = (sum(s[i] for s in stats) for i in (0, 1))
             ui.write(_(b'%d of %d chunk(s) applied\n') % (applied, total))
 
     def _commitstack(self):
         """make new commits. update self.finalnode, self.replacemap.
         it is splitted from "commit" to avoid too much indentation.
         """
         # last node (20-char) committed by us
         lastcommitted = None
         # p1 which overrides the parent of the next commit, "None" means use
         # the original parent unchanged
         nextp1 = None
         for ctx in self.stack:
             memworkingcopy = self._getnewfilecontents(ctx)
             if not memworkingcopy and not lastcommitted:
                 # nothing changed, nothing commited
                 nextp1 = ctx
                 continue
             willbecomenoop = ctx.files() and self._willbecomenoop(
                 memworkingcopy, ctx, nextp1
             )
             if self.skip_empty_successor and willbecomenoop:
                 # changeset is no longer necessary
                 self.replacemap[ctx.node()] = None
                 msg = _(b'became empty and was dropped')
             else:
                 # changeset needs re-commit
                 nodestr = self._commitsingle(memworkingcopy, ctx, p1=nextp1)
                 lastcommitted = self.repo[nodestr]
                 nextp1 = lastcommitted
                 self.replacemap[ctx.node()] = lastcommitted.node()
                 if memworkingcopy:
                     if willbecomenoop:
                         msg = _(b'%d file(s) changed, became empty as %s')
                     else:
                         msg = _(b'%d file(s) changed, became %s')
                     msg = msg % (
                         len(memworkingcopy),
                         self._ctx2str(lastcommitted),
                     )
                 else:
                     msg = _(b'became %s') % self._ctx2str(lastcommitted)
             if self.ui.verbose and msg:
                 self.ui.write(_(b'%s: %s\n') % (self._ctx2str(ctx), msg))
         self.finalnode = lastcommitted and lastcommitted.node()
 
     def _ctx2str(self, ctx):
         if self.ui.debugflag:
             return b'%d:%s' % (ctx.rev(), ctx.hex())
         else:
             return b'%d:%s' % (ctx.rev(), short(ctx.node()))
 
     def _getnewfilecontents(self, ctx):
         """(ctx) -> {path: str}
 
         fetch file contents from filefixupstates.
         return the working copy overrides - files different from ctx.
         """
         result = {}
         for path in self.paths:
             ctx2fctx = self.fctxmap[path]  # {ctx: fctx}
             if ctx not in ctx2fctx:
                 continue
             fctx = ctx2fctx[ctx]
             content = fctx.data()
             newcontent = self.fixupmap[path].getfinalcontent(fctx)
             if content != newcontent:
                 result[fctx.path()] = newcontent
         return result
 
     def _movebookmarks(self, tr):
         repo = self.repo
         needupdate = [
             (name, self.replacemap[hsh])
-            for name, hsh in pycompat.iteritems(repo._bookmarks)
+            for name, hsh in repo._bookmarks.items()
             if hsh in self.replacemap
         ]
         changes = []
         for name, hsh in needupdate:
             if hsh:
                 changes.append((name, hsh))
                 if self.ui.verbose:
                     self.ui.write(
                         _(b'moving bookmark %s to %s\n') % (name, hex(hsh))
                     )
             else:
                 changes.append((name, None))
                 if self.ui.verbose:
                     self.ui.write(_(b'deleting bookmark %s\n') % name)
         repo._bookmarks.applychanges(repo, tr, changes)
 
     def _moveworkingdirectoryparent(self):
         if not self.finalnode:
             # Find the latest not-{obsoleted,stripped} parent.
             revs = self.repo.revs(b'max(::. - %ln)', self.replacemap.keys())
             ctx = self.repo[revs.first()]
             self.finalnode = ctx.node()
         else:
             ctx = self.repo[self.finalnode]
 
         dirstate = self.repo.dirstate
         # dirstate.rebuild invalidates fsmonitorstate, causing "hg status" to
         # be slow. in absorb's case, no need to invalidate fsmonitorstate.
         noop = lambda: 0
         restore = noop
         if util.safehasattr(dirstate, '_fsmonitorstate'):
             bak = dirstate._fsmonitorstate.invalidate
 
             def restore():
                 dirstate._fsmonitorstate.invalidate = bak
 
             dirstate._fsmonitorstate.invalidate = noop
         try:
             with dirstate.parentchange():
                 dirstate.rebuild(ctx.node(), ctx.manifest(), self.paths)
         finally:
             restore()
 
     @staticmethod
     def _willbecomenoop(memworkingcopy, ctx, pctx=None):
         """({path: content}, ctx, ctx) -> bool. test if a commit will be noop
 
         if it will become an empty commit (does not change anything, after the
         memworkingcopy overrides), return True. otherwise return False.
         """
         if not pctx:
             parents = ctx.parents()
             if len(parents) != 1:
                 return False
             pctx = parents[0]
         if ctx.branch() != pctx.branch():
             return False
         if ctx.extra().get(b'close'):
             return False
         # ctx changes more files (not a subset of memworkingcopy)
         if not set(ctx.files()).issubset(set(memworkingcopy)):
             return False
-        for path, content in pycompat.iteritems(memworkingcopy):
+        for path, content in memworkingcopy.items():
             if path not in pctx or path not in ctx:
                 return False
             fctx = ctx[path]
             pfctx = pctx[path]
             if pfctx.flags() != fctx.flags():
                 return False
             if pfctx.data() != content:
                 return False
         return True
 
     def _commitsingle(self, memworkingcopy, ctx, p1=None):
         """(ctx, {path: content}, node) -> node. make a single commit
 
         the commit is a clone from ctx, with a (optionally) different p1, and
         different file contents replaced by memworkingcopy.
         """
         parents = p1 and (p1, self.repo.nullid)
         extra = ctx.extra()
         if self._useobsolete and self.ui.configbool(b'absorb', b'add-noise'):
             extra[b'absorb_source'] = ctx.hex()
 
         desc = rewriteutil.update_hash_refs(
             ctx.repo(),
             ctx.description(),
             {
                 oldnode: [newnode]
                 for oldnode, newnode in self.replacemap.items()
             },
         )
         mctx = overlaycontext(
             memworkingcopy, ctx, parents, extra=extra, desc=desc
         )
         return mctx.commit()
 
     @util.propertycache
     def _useobsolete(self):
         """() -> bool"""
         return obsolete.isenabled(self.repo, obsolete.createmarkersopt)
 
     def _cleanupoldcommits(self):
         replacements = {
             k: ([v] if v is not None else [])
-            for k, v in pycompat.iteritems(self.replacemap)
+            for k, v in self.replacemap.items()
         }
         if replacements:
             scmutil.cleanupnodes(
                 self.repo, replacements, operation=b'absorb', fixphase=True
             )
 
     @util.propertycache
     def skip_empty_successor(self):
         return rewriteutil.skip_empty_successor(self.ui, b'absorb')
 
 
 def _parsechunk(hunk):
     """(crecord.uihunk or patch.recordhunk) -> (path, (a1, a2, [bline]))"""
     if type(hunk) not in (crecord.uihunk, patch.recordhunk):
         return None, None
     path = hunk.header.filename()
     a1 = hunk.fromline + len(hunk.before) - 1
     # remove before and after context
     hunk.before = hunk.after = []
     buf = util.stringio()
     hunk.write(buf)
     patchlines = mdiff.splitnewlines(buf.getvalue())
     # hunk.prettystr() will update hunk.removed
     a2 = a1 + hunk.removed
     blines = [l[1:] for l in patchlines[1:] if not l.startswith(b'-')]
     return path, (a1, a2, blines)
 
 
 def overlaydiffcontext(ctx, chunks):
     """(ctx, [crecord.uihunk]) -> memctx
 
     return a memctx with some [1] patches (chunks) applied to ctx.
     [1]: modifications are handled. renames, mode changes, etc. are ignored.
     """
     # sadly the applying-patch logic is hardly reusable, and messy:
     # 1. the core logic "_applydiff" is too heavy - it writes .rej files, it
     #    needs a file stream of a patch and will re-parse it, while we have
     #    structured hunk objects at hand.
     # 2. a lot of different implementations about "chunk" (patch.hunk,
     #    patch.recordhunk, crecord.uihunk)
     # as we only care about applying changes to modified files, no mode
     # change, no binary diff, and no renames, it's probably okay to
     # re-invent the logic using much simpler code here.
     memworkingcopy = {}  # {path: content}
     patchmap = defaultdict(lambda: [])  # {path: [(a1, a2, [bline])]}
     for path, info in map(_parsechunk, chunks):
         if not path or not info:
             continue
         patchmap[path].append(info)
-    for path, patches in pycompat.iteritems(patchmap):
+    for path, patches in patchmap.items():
         if path not in ctx or not patches:
             continue
         patches.sort(reverse=True)
         lines = mdiff.splitnewlines(ctx[path].data())
         for a1, a2, blines in patches:
             lines[a1:a2] = blines
         memworkingcopy[path] = b''.join(lines)
     return overlaycontext(memworkingcopy, ctx)
 
 
 def absorb(ui, repo, stack=None, targetctx=None, pats=None, opts=None):
     """pick fixup chunks from targetctx, apply them to stack.
 
     if targetctx is None, the working copy context will be used.
     if stack is None, the current draft stack will be used.
     return fixupstate.
     """
     if stack is None:
         limit = ui.configint(b'absorb', b'max-stack-size')
         headctx = repo[b'.']
         if len(headctx.parents()) > 1:
             raise error.InputError(_(b'cannot absorb into a merge'))
         stack = getdraftstack(headctx, limit)
         if limit and len(stack) >= limit:
             ui.warn(
                 _(
                     b'absorb: only the recent %d changesets will '
                     b'be analysed\n'
                 )
                 % limit
             )
     if not stack:
         raise error.InputError(_(b'no mutable changeset to change'))
     if targetctx is None:  # default to working copy
         targetctx = repo[None]
     if pats is None:
         pats = ()
     if opts is None:
         opts = {}
     state = fixupstate(stack, ui=ui, opts=opts)
     matcher = scmutil.match(targetctx, pats, opts)
     if opts.get(b'interactive'):
         diff = patch.diff(repo, stack[-1].node(), targetctx.node(), matcher)
         origchunks = patch.parsepatch(diff)
         chunks = cmdutil.recordfilter(ui, origchunks, matcher)[0]
         targetctx = overlaydiffcontext(stack[-1], chunks)
     fm = None
     if opts.get(b'print_changes') or not opts.get(b'apply_changes'):
         fm = ui.formatter(b'absorb', opts)
     state.diffwith(targetctx, matcher, fm)
     if fm is not None:
         fm.startitem()
         fm.write(
             b"count", b"\n%d changesets affected\n", len(state.ctxaffected)
         )
         fm.data(linetype=b'summary')
         for ctx in reversed(stack):
             if ctx not in state.ctxaffected:
                 continue
             fm.startitem()
             fm.context(ctx=ctx)
             fm.data(linetype=b'changeset')
             fm.write(b'node', b'%-7.7s ', ctx.hex(), label=b'absorb.node')
             descfirstline = ctx.description().splitlines()[0]
             fm.write(
                 b'descfirstline',
                 b'%s\n',
                 descfirstline,
                 label=b'absorb.description',
             )
         fm.end()
     if not opts.get(b'dry_run'):
         if (
             not opts.get(b'apply_changes')
             and state.ctxaffected
             and ui.promptchoice(
                 b"apply changes (y/N)? $$ &Yes $$ &No", default=1
             )
         ):
             raise error.CanceledError(_(b'absorb cancelled\n'))
 
         state.apply()
         if state.commit():
             state.printchunkstats()
         elif not ui.quiet:
             ui.write(_(b'nothing applied\n'))
     return state
 
 
 @command(
     b'absorb',
     [
         (
             b'a',
             b'apply-changes',
             None,
             _(b'apply changes without prompting for confirmation'),
         ),
         (
             b'p',
             b'print-changes',
             None,
             _(b'always print which changesets are modified by which changes'),
         ),
         (
             b'i',
             b'interactive',
             None,
             _(b'interactively select which chunks to apply'),
         ),
         (
             b'e',
             b'edit-lines',
             None,
             _(
                 b'edit what lines belong to which changesets before commit '
                 b'(EXPERIMENTAL)'
             ),
         ),
     ]
     + commands.dryrunopts
     + commands.templateopts
     + commands.walkopts,
     _(b'hg absorb [OPTION] [FILE]...'),
     helpcategory=command.CATEGORY_COMMITTING,
     helpbasic=True,
 )
 def absorbcmd(ui, repo, *pats, **opts):
     """incorporate corrections into the stack of draft changesets
 
     absorb analyzes each change in your working directory and attempts to
     amend the changed lines into the changesets in your stack that first
     introduced those lines.
 
     If absorb cannot find an unambiguous changeset to amend for a change,
     that change will be left in the working directory, untouched. They can be
     observed by :hg:`status` or :hg:`diff` afterwards. In other words,
     absorb does not write to the working directory.
 
     Changesets outside the revset `::. and not public() and not merge()` will
     not be changed.
 
     Changesets that become empty after applying the changes will be deleted.
 
     By default, absorb will show what it plans to do and prompt for
     confirmation.  If you are confident that the changes will be absorbed
     to the correct place, run :hg:`absorb -a` to apply the changes
     immediately.
 
     Returns 0 on success, 1 if all chunks were ignored and nothing amended.
     """
     opts = pycompat.byteskwargs(opts)
 
     with repo.wlock(), repo.lock():
         if not opts[b'dry_run']:
             cmdutil.checkunfinished(repo)
 
         state = absorb(ui, repo, pats=pats, opts=opts)
         if sum(s[0] for s in state.chunkstats.values()) == 0:
             return 1

hgext/convert/bzr.py

@@ -1,338 +1,337 b''
 # bzr.py - bzr support for the convert extension
 #
 #  Copyright 2008, 2009 Marek Kubica <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # This module is for handling Breezy imports or `brz`, but it's also compatible
 # with Bazaar or `bzr`, that was formerly known as Bazaar-NG;
 # it cannot access `bar` repositories, but they were never used very much.
 
 import os
 
 from mercurial.i18n import _
 from mercurial import (
     demandimport,
     error,
-    pycompat,
     util,
 )
 from . import common
 
 
 # these do not work with demandimport, blacklist
 demandimport.IGNORES.update(
     [
         b'breezy.transactions',
         b'breezy.urlutils',
         b'ElementPath',
     ]
 )
 
 try:
     # bazaar imports
     import breezy.bzr.bzrdir
     import breezy.errors
     import breezy.revision
     import breezy.revisionspec
 
     bzrdir = breezy.bzr.bzrdir
     errors = breezy.errors
     revision = breezy.revision
     revisionspec = breezy.revisionspec
     revisionspec.RevisionSpec
 except ImportError:
     pass
 
 supportedkinds = ('file', 'symlink')
 
 
 class bzr_source(common.converter_source):
     """Reads Bazaar repositories by using the Bazaar Python libraries"""
 
     def __init__(self, ui, repotype, path, revs=None):
         super(bzr_source, self).__init__(ui, repotype, path, revs=revs)
 
         if not os.path.exists(os.path.join(path, b'.bzr')):
             raise common.NoRepo(
                 _(b'%s does not look like a Bazaar repository') % path
             )
 
         try:
             # access breezy stuff
             bzrdir
         except NameError:
             raise common.NoRepo(_(b'Bazaar modules could not be loaded'))
 
         path = util.abspath(path)
         self._checkrepotype(path)
         try:
             bzr_dir = bzrdir.BzrDir.open(path.decode())
             self.sourcerepo = bzr_dir.open_repository()
         except errors.NoRepositoryPresent:
             raise common.NoRepo(
                 _(b'%s does not look like a Bazaar repository') % path
             )
         self._parentids = {}
         self._saverev = ui.configbool(b'convert', b'bzr.saverev')
 
     def _checkrepotype(self, path):
         # Lightweight checkouts detection is informational but probably
         # fragile at API level. It should not terminate the conversion.
         try:
             dir = bzrdir.BzrDir.open_containing(path.decode())[0]
             try:
                 tree = dir.open_workingtree(recommend_upgrade=False)
                 branch = tree.branch
             except (errors.NoWorkingTree, errors.NotLocalUrl):
                 tree = None
                 branch = dir.open_branch()
             if (
                 tree is not None
                 and tree.controldir.root_transport.base
                 != branch.controldir.root_transport.base
             ):
                 self.ui.warn(
                     _(
                         b'warning: lightweight checkouts may cause '
                         b'conversion failures, try with a regular '
                         b'branch instead.\n'
                     )
                 )
         except Exception:
             self.ui.note(_(b'bzr source type could not be determined\n'))
 
     def before(self):
         """Before the conversion begins, acquire a read lock
         for all the operations that might need it. Fortunately
         read locks don't block other reads or writes to the
         repository, so this shouldn't have any impact on the usage of
         the source repository.
 
         The alternative would be locking on every operation that
         needs locks (there are currently two: getting the file and
         getting the parent map) and releasing immediately after,
         but this approach can take even 40% longer."""
         self.sourcerepo.lock_read()
 
     def after(self):
         self.sourcerepo.unlock()
 
     def _bzrbranches(self):
         return self.sourcerepo.find_branches(using=True)
 
     def getheads(self):
         if not self.revs:
             # Set using=True to avoid nested repositories (see issue3254)
             heads = sorted([b.last_revision() for b in self._bzrbranches()])
         else:
             revid = None
             for branch in self._bzrbranches():
                 try:
                     revspec = self.revs[0].decode()
                     r = revisionspec.RevisionSpec.from_string(revspec)
                     info = r.in_history(branch)
                 except errors.BzrError:
                     pass
                 revid = info.rev_id
             if revid is None:
                 raise error.Abort(
                     _(b'%s is not a valid revision') % self.revs[0]
                 )
             heads = [revid]
         # Empty repositories return 'null:', which cannot be retrieved
         heads = [h for h in heads if h != b'null:']
         return heads
 
     def getfile(self, name, rev):
         name = name.decode()
         revtree = self.sourcerepo.revision_tree(rev)
 
         try:
             kind = revtree.kind(name)
         except breezy.errors.NoSuchFile:
             return None, None
         if kind not in supportedkinds:
             # the file is not available anymore - was deleted
             return None, None
         mode = self._modecache[(name.encode(), rev)]
         if kind == 'symlink':
             target = revtree.get_symlink_target(name)
             if target is None:
                 raise error.Abort(
                     _(b'%s.%s symlink has no target') % (name, rev)
                 )
             return target.encode(), mode
         else:
             sio = revtree.get_file(name)
             return sio.read(), mode
 
     def getchanges(self, version, full):
         if full:
             raise error.Abort(_(b"convert from cvs does not support --full"))
         self._modecache = {}
         self._revtree = self.sourcerepo.revision_tree(version)
         # get the parentids from the cache
         parentids = self._parentids.pop(version)
         # only diff against first parent id
         prevtree = self.sourcerepo.revision_tree(parentids[0])
         files, changes = self._gettreechanges(self._revtree, prevtree)
         return files, changes, set()
 
     def getcommit(self, version):
         rev = self.sourcerepo.get_revision(version)
         # populate parent id cache
         if not rev.parent_ids:
             parents = []
             self._parentids[version] = (revision.NULL_REVISION,)
         else:
             parents = self._filterghosts(rev.parent_ids)
             self._parentids[version] = parents
 
         branch = rev.properties.get('branch-nick', 'default')
         if branch == 'trunk':
             branch = 'default'
         return common.commit(
             parents=parents,
             date=b'%d %d' % (rev.timestamp, -rev.timezone),
             author=self.recode(rev.committer),
             desc=self.recode(rev.message),
             branch=branch.encode('utf8'),
             rev=version,
             saverev=self._saverev,
         )
 
     def gettags(self):
         bytetags = {}
         for branch in self._bzrbranches():
             if not branch.supports_tags():
                 return {}
             tagdict = branch.tags.get_tag_dict()
-            for name, rev in pycompat.iteritems(tagdict):
+            for name, rev in tagdict.items():
                 bytetags[self.recode(name)] = rev
         return bytetags
 
     def getchangedfiles(self, rev, i):
         self._modecache = {}
         curtree = self.sourcerepo.revision_tree(rev)
         if i is not None:
             parentid = self._parentids[rev][i]
         else:
             # no parent id, get the empty revision
             parentid = revision.NULL_REVISION
 
         prevtree = self.sourcerepo.revision_tree(parentid)
         changes = [e[0] for e in self._gettreechanges(curtree, prevtree)[0]]
         return changes
 
     def _gettreechanges(self, current, origin):
         revid = current._revision_id
         changes = []
         renames = {}
         seen = set()
 
         # Fall back to the deprecated attribute for legacy installations.
         try:
             inventory = origin.root_inventory
         except AttributeError:
             inventory = origin.inventory
 
         # Process the entries by reverse lexicographic name order to
         # handle nested renames correctly, most specific first.
 
         def key(c):
             return c.path[0] or c.path[1] or ""
 
         curchanges = sorted(
             current.iter_changes(origin),
             key=key,
             reverse=True,
         )
         for change in curchanges:
             paths = change.path
             kind = change.kind
             executable = change.executable
             if paths[0] == u'' or paths[1] == u'':
                 # ignore changes to tree root
                 continue
 
             # bazaar tracks directories, mercurial does not, so
             # we have to rename the directory contents
             if kind[1] == 'directory':
                 if kind[0] not in (None, 'directory'):
                     # Replacing 'something' with a directory, record it
                     # so it can be removed.
                     changes.append((self.recode(paths[0]), revid))
 
                 if kind[0] == 'directory' and None not in paths:
                     renaming = paths[0] != paths[1]
                     # neither an add nor an delete - a move
                     # rename all directory contents manually
                     subdir = inventory.path2id(paths[0])
                     # get all child-entries of the directory
                     for name, entry in inventory.iter_entries(subdir):
                         # hg does not track directory renames
                         if entry.kind == 'directory':
                             continue
                         frompath = self.recode(paths[0] + '/' + name)
                         if frompath in seen:
                             # Already handled by a more specific change entry
                             # This is important when you have:
                             # a => b
                             # a/c => a/c
                             # Here a/c must not be renamed into b/c
                             continue
                         seen.add(frompath)
                         if not renaming:
                             continue
                         topath = self.recode(paths[1] + '/' + name)
                         # register the files as changed
                         changes.append((frompath, revid))
                         changes.append((topath, revid))
                         # add to mode cache
                         mode = (
                             (entry.executable and b'x')
                             or (entry.kind == 'symlink' and b's')
                             or b''
                         )
                         self._modecache[(topath, revid)] = mode
                         # register the change as move
                         renames[topath] = frompath
 
                 # no further changes, go to the next change
                 continue
 
             # we got unicode paths, need to convert them
             path, topath = paths
             if path is not None:
                 path = self.recode(path)
             if topath is not None:
                 topath = self.recode(topath)
             seen.add(path or topath)
 
             if topath is None:
                 # file deleted
                 changes.append((path, revid))
                 continue
 
             # renamed
             if path and path != topath:
                 renames[topath] = path
                 changes.append((path, revid))
 
             # populate the mode cache
             kind, executable = [e[1] for e in (kind, executable)]
             mode = (executable and b'x') or (kind == 'symlink' and b'l') or b''
             self._modecache[(topath, revid)] = mode
             changes.append((topath, revid))
 
         return changes, renames
 
     def _filterghosts(self, ids):
         """Filters out ghost revisions which hg does not support, see
         <http://bazaar-vcs.org/GhostRevision>
         """
         parentmap = self.sourcerepo.get_parent_map(ids)
         parents = tuple([parent for parent in ids if parent in parentmap])
         return parents

hgext/convert/common.py

@@ -1,597 +1,597 b''
 # common.py - common code for the convert extension
 #
 #  Copyright 2005-2009 Olivia Mackall <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 import base64
 import datetime
 import errno
 import os
 import pickle
 import re
 import shlex
 import subprocess
 
 from mercurial.i18n import _
 from mercurial.pycompat import open
 from mercurial import (
     encoding,
     error,
     phases,
     pycompat,
     util,
 )
 from mercurial.utils import procutil
 
 propertycache = util.propertycache
 
 
 def _encodeornone(d):
     if d is None:
         return
     return d.encode('latin1')
 
 
 class _shlexpy3proxy(object):
     def __init__(self, l):
         self._l = l
 
     def __iter__(self):
         return (_encodeornone(v) for v in self._l)
 
     def get_token(self):
         return _encodeornone(self._l.get_token())
 
     @property
     def infile(self):
         return self._l.infile or b'<unknown>'
 
     @property
     def lineno(self):
         return self._l.lineno
 
 
 def shlexer(data=None, filepath=None, wordchars=None, whitespace=None):
     if data is None:
         if pycompat.ispy3:
             data = open(filepath, b'r', encoding='latin1')
         else:
             data = open(filepath, b'r')
     else:
         if filepath is not None:
             raise error.ProgrammingError(
                 b'shlexer only accepts data or filepath, not both'
             )
         if pycompat.ispy3:
             data = data.decode('latin1')
     l = shlex.shlex(data, infile=filepath, posix=True)
     if whitespace is not None:
         l.whitespace_split = True
         if pycompat.ispy3:
             l.whitespace += whitespace.decode('latin1')
         else:
             l.whitespace += whitespace
     if wordchars is not None:
         if pycompat.ispy3:
             l.wordchars += wordchars.decode('latin1')
         else:
             l.wordchars += wordchars
     if pycompat.ispy3:
         return _shlexpy3proxy(l)
     return l
 
 
 if pycompat.ispy3:
     base64_encodebytes = base64.encodebytes
     base64_decodebytes = base64.decodebytes
 else:
     base64_encodebytes = base64.encodestring
     base64_decodebytes = base64.decodestring
 
 
 def encodeargs(args):
     def encodearg(s):
         lines = base64_encodebytes(s)
         lines = [l.splitlines()[0] for l in pycompat.iterbytestr(lines)]
         return b''.join(lines)
 
     s = pickle.dumps(args)
     return encodearg(s)
 
 
 def decodeargs(s):
     s = base64_decodebytes(s)
     return pickle.loads(s)
 
 
 class MissingTool(Exception):
     pass
 
 
 def checktool(exe, name=None, abort=True):
     name = name or exe
     if not procutil.findexe(exe):
         if abort:
             exc = error.Abort
         else:
             exc = MissingTool
         raise exc(_(b'cannot find required "%s" tool') % name)
 
 
 class NoRepo(Exception):
     pass
 
 
 SKIPREV = b'SKIP'
 
 
 class commit(object):
     def __init__(
         self,
         author,
         date,
         desc,
         parents,
         branch=None,
         rev=None,
         extra=None,
         sortkey=None,
         saverev=True,
         phase=phases.draft,
         optparents=None,
         ctx=None,
     ):
         self.author = author or b'unknown'
         self.date = date or b'0 0'
         self.desc = desc
         self.parents = parents  # will be converted and used as parents
         self.optparents = optparents or []  # will be used if already converted
         self.branch = branch
         self.rev = rev
         self.extra = extra or {}
         self.sortkey = sortkey
         self.saverev = saverev
         self.phase = phase
         self.ctx = ctx  # for hg to hg conversions
 
 
 class converter_source(object):
     """Conversion source interface"""
 
     def __init__(self, ui, repotype, path=None, revs=None):
         """Initialize conversion source (or raise NoRepo("message")
         exception if path is not a valid repository)"""
         self.ui = ui
         self.path = path
         self.revs = revs
         self.repotype = repotype
 
         self.encoding = b'utf-8'
 
     def checkhexformat(self, revstr, mapname=b'splicemap'):
         """fails if revstr is not a 40 byte hex. mercurial and git both uses
         such format for their revision numbering
         """
         if not re.match(br'[0-9a-fA-F]{40,40}$', revstr):
             raise error.Abort(
                 _(b'%s entry %s is not a valid revision identifier')
                 % (mapname, revstr)
             )
 
     def before(self):
         pass
 
     def after(self):
         pass
 
     def targetfilebelongstosource(self, targetfilename):
         """Returns true if the given targetfile belongs to the source repo. This
         is useful when only a subdirectory of the target belongs to the source
         repo."""
         # For normal full repo converts, this is always True.
         return True
 
     def setrevmap(self, revmap):
         """set the map of already-converted revisions"""
 
     def getheads(self):
         """Return a list of this repository's heads"""
         raise NotImplementedError
 
     def getfile(self, name, rev):
         """Return a pair (data, mode) where data is the file content
         as a string and mode one of '', 'x' or 'l'. rev is the
         identifier returned by a previous call to getchanges().
         Data is None if file is missing/deleted in rev.
         """
         raise NotImplementedError
 
     def getchanges(self, version, full):
         """Returns a tuple of (files, copies, cleanp2).
 
         files is a sorted list of (filename, id) tuples for all files
         changed between version and its first parent returned by
         getcommit(). If full, all files in that revision is returned.
         id is the source revision id of the file.
 
         copies is a dictionary of dest: source
 
         cleanp2 is the set of files filenames that are clean against p2.
         (Files that are clean against p1 are already not in files (unless
         full). This makes it possible to handle p2 clean files similarly.)
         """
         raise NotImplementedError
 
     def getcommit(self, version):
         """Return the commit object for version"""
         raise NotImplementedError
 
     def numcommits(self):
         """Return the number of commits in this source.
 
         If unknown, return None.
         """
         return None
 
     def gettags(self):
         """Return the tags as a dictionary of name: revision
 
         Tag names must be UTF-8 strings.
         """
         raise NotImplementedError
 
     def recode(self, s, encoding=None):
         if not encoding:
             encoding = self.encoding or b'utf-8'
 
         if isinstance(s, pycompat.unicode):
             return s.encode("utf-8")
         try:
             return s.decode(pycompat.sysstr(encoding)).encode("utf-8")
         except UnicodeError:
             try:
                 return s.decode("latin-1").encode("utf-8")
             except UnicodeError:
                 return s.decode(pycompat.sysstr(encoding), "replace").encode(
                     "utf-8"
                 )
 
     def getchangedfiles(self, rev, i):
         """Return the files changed by rev compared to parent[i].
 
         i is an index selecting one of the parents of rev.  The return
         value should be the list of files that are different in rev and
         this parent.
 
         If rev has no parents, i is None.
 
         This function is only needed to support --filemap
         """
         raise NotImplementedError
 
     def converted(self, rev, sinkrev):
         '''Notify the source that a revision has been converted.'''
 
     def hasnativeorder(self):
         """Return true if this source has a meaningful, native revision
         order. For instance, Mercurial revisions are store sequentially
         while there is no such global ordering with Darcs.
         """
         return False
 
     def hasnativeclose(self):
         """Return true if this source has ability to close branch."""
         return False
 
     def lookuprev(self, rev):
         """If rev is a meaningful revision reference in source, return
         the referenced identifier in the same format used by getcommit().
         return None otherwise.
         """
         return None
 
     def getbookmarks(self):
         """Return the bookmarks as a dictionary of name: revision
 
         Bookmark names are to be UTF-8 strings.
         """
         return {}
 
     def checkrevformat(self, revstr, mapname=b'splicemap'):
         """revstr is a string that describes a revision in the given
         source control system.  Return true if revstr has correct
         format.
         """
         return True
 
 
 class converter_sink(object):
     """Conversion sink (target) interface"""
 
     def __init__(self, ui, repotype, path):
         """Initialize conversion sink (or raise NoRepo("message")
         exception if path is not a valid repository)
 
         created is a list of paths to remove if a fatal error occurs
         later"""
         self.ui = ui
         self.path = path
         self.created = []
         self.repotype = repotype
 
     def revmapfile(self):
         """Path to a file that will contain lines
         source_rev_id sink_rev_id
         mapping equivalent revision identifiers for each system."""
         raise NotImplementedError
 
     def authorfile(self):
         """Path to a file that will contain lines
         srcauthor=dstauthor
         mapping equivalent authors identifiers for each system."""
         return None
 
     def putcommit(
         self, files, copies, parents, commit, source, revmap, full, cleanp2
     ):
         """Create a revision with all changed files listed in 'files'
         and having listed parents. 'commit' is a commit object
         containing at a minimum the author, date, and message for this
         changeset.  'files' is a list of (path, version) tuples,
         'copies' is a dictionary mapping destinations to sources,
         'source' is the source repository, and 'revmap' is a mapfile
         of source revisions to converted revisions. Only getfile() and
         lookuprev() should be called on 'source'. 'full' means that 'files'
         is complete and all other files should be removed.
         'cleanp2' is a set of the filenames that are unchanged from p2
         (only in the common merge case where there two parents).
 
         Note that the sink repository is not told to update itself to
         a particular revision (or even what that revision would be)
         before it receives the file data.
         """
         raise NotImplementedError
 
     def puttags(self, tags):
         """Put tags into sink.
 
         tags: {tagname: sink_rev_id, ...} where tagname is an UTF-8 string.
         Return a pair (tag_revision, tag_parent_revision), or (None, None)
         if nothing was changed.
         """
         raise NotImplementedError
 
     def setbranch(self, branch, pbranches):
         """Set the current branch name. Called before the first putcommit
         on the branch.
         branch: branch name for subsequent commits
         pbranches: (converted parent revision, parent branch) tuples"""
 
     def setfilemapmode(self, active):
         """Tell the destination that we're using a filemap
 
         Some converter_sources (svn in particular) can claim that a file
         was changed in a revision, even if there was no change.  This method
         tells the destination that we're using a filemap and that it should
         filter empty revisions.
         """
 
     def before(self):
         pass
 
     def after(self):
         pass
 
     def putbookmarks(self, bookmarks):
         """Put bookmarks into sink.
 
         bookmarks: {bookmarkname: sink_rev_id, ...}
         where bookmarkname is an UTF-8 string.
         """
 
     def hascommitfrommap(self, rev):
         """Return False if a rev mentioned in a filemap is known to not be
         present."""
         raise NotImplementedError
 
     def hascommitforsplicemap(self, rev):
         """This method is for the special needs for splicemap handling and not
         for general use. Returns True if the sink contains rev, aborts on some
         special cases."""
         raise NotImplementedError
 
 
 class commandline(object):
     def __init__(self, ui, command):
         self.ui = ui
         self.command = command
 
     def prerun(self):
         pass
 
     def postrun(self):
         pass
 
     def _cmdline(self, cmd, *args, **kwargs):
         kwargs = pycompat.byteskwargs(kwargs)
         cmdline = [self.command, cmd] + list(args)
-        for k, v in pycompat.iteritems(kwargs):
+        for k, v in kwargs.items():
             if len(k) == 1:
                 cmdline.append(b'-' + k)
             else:
                 cmdline.append(b'--' + k.replace(b'_', b'-'))
             try:
                 if len(k) == 1:
                     cmdline.append(b'' + v)
                 else:
                     cmdline[-1] += b'=' + v
             except TypeError:
                 pass
         cmdline = [procutil.shellquote(arg) for arg in cmdline]
         if not self.ui.debugflag:
             cmdline += [b'2>', pycompat.bytestr(os.devnull)]
         cmdline = b' '.join(cmdline)
         return cmdline
 
     def _run(self, cmd, *args, **kwargs):
         def popen(cmdline):
             p = subprocess.Popen(
                 procutil.tonativestr(cmdline),
                 shell=True,
                 bufsize=-1,
                 close_fds=procutil.closefds,
                 stdout=subprocess.PIPE,
             )
             return p
 
         return self._dorun(popen, cmd, *args, **kwargs)
 
     def _run2(self, cmd, *args, **kwargs):
         return self._dorun(procutil.popen2, cmd, *args, **kwargs)
 
     def _run3(self, cmd, *args, **kwargs):
         return self._dorun(procutil.popen3, cmd, *args, **kwargs)
 
     def _dorun(self, openfunc, cmd, *args, **kwargs):
         cmdline = self._cmdline(cmd, *args, **kwargs)
         self.ui.debug(b'running: %s\n' % (cmdline,))
         self.prerun()
         try:
             return openfunc(cmdline)
         finally:
             self.postrun()
 
     def run(self, cmd, *args, **kwargs):
         p = self._run(cmd, *args, **kwargs)
         output = p.communicate()[0]
         self.ui.debug(output)
         return output, p.returncode
 
     def runlines(self, cmd, *args, **kwargs):
         p = self._run(cmd, *args, **kwargs)
         output = p.stdout.readlines()
         p.wait()
         self.ui.debug(b''.join(output))
         return output, p.returncode
 
     def checkexit(self, status, output=b''):
         if status:
             if output:
                 self.ui.warn(_(b'%s error:\n') % self.command)
                 self.ui.warn(output)
             msg = procutil.explainexit(status)
             raise error.Abort(b'%s %s' % (self.command, msg))
 
     def run0(self, cmd, *args, **kwargs):
         output, status = self.run(cmd, *args, **kwargs)
         self.checkexit(status, output)
         return output
 
     def runlines0(self, cmd, *args, **kwargs):
         output, status = self.runlines(cmd, *args, **kwargs)
         self.checkexit(status, b''.join(output))
         return output
 
     @propertycache
     def argmax(self):
         # POSIX requires at least 4096 bytes for ARG_MAX
         argmax = 4096
         try:
             argmax = os.sysconf("SC_ARG_MAX")
         except (AttributeError, ValueError):
             pass
 
         # Windows shells impose their own limits on command line length,
         # down to 2047 bytes for cmd.exe under Windows NT/2k and 2500 bytes
         # for older 4nt.exe. See http://support.microsoft.com/kb/830473 for
         # details about cmd.exe limitations.
 
         # Since ARG_MAX is for command line _and_ environment, lower our limit
         # (and make happy Windows shells while doing this).
         return argmax // 2 - 1
 
     def _limit_arglist(self, arglist, cmd, *args, **kwargs):
         cmdlen = len(self._cmdline(cmd, *args, **kwargs))
         limit = self.argmax - cmdlen
         numbytes = 0
         fl = []
         for fn in arglist:
             b = len(fn) + 3
             if numbytes + b < limit or len(fl) == 0:
                 fl.append(fn)
                 numbytes += b
             else:
                 yield fl
                 fl = [fn]
                 numbytes = b
         if fl:
             yield fl
 
     def xargs(self, arglist, cmd, *args, **kwargs):
         for l in self._limit_arglist(arglist, cmd, *args, **kwargs):
             self.run0(cmd, *(list(args) + l), **kwargs)
 
 
 class mapfile(dict):
     def __init__(self, ui, path):
         super(mapfile, self).__init__()
         self.ui = ui
         self.path = path
         self.fp = None
         self.order = []
         self._read()
 
     def _read(self):
         if not self.path:
             return
         try:
             fp = open(self.path, b'rb')
         except IOError as err:
             if err.errno != errno.ENOENT:
                 raise
             return
         for i, line in enumerate(util.iterfile(fp)):
             line = line.splitlines()[0].rstrip()
             if not line:
                 # Ignore blank lines
                 continue
             try:
                 key, value = line.rsplit(b' ', 1)
             except ValueError:
                 raise error.Abort(
                     _(b'syntax error in %s(%d): key/value pair expected')
                     % (self.path, i + 1)
                 )
             if key not in self:
                 self.order.append(key)
             super(mapfile, self).__setitem__(key, value)
         fp.close()
 
     def __setitem__(self, key, value):
         if self.fp is None:
             try:
                 self.fp = open(self.path, b'ab')
             except IOError as err:
                 raise error.Abort(
                     _(b'could not open map file %r: %s')
                     % (self.path, encoding.strtolocal(err.strerror))
                 )
         self.fp.write(util.tonativeeol(b'%s %s\n' % (key, value)))
         self.fp.flush()
         super(mapfile, self).__setitem__(key, value)
 
     def close(self):
         if self.fp:
             self.fp.close()
             self.fp = None
 
 
 def makedatetimestamp(t):
     """Like dateutil.makedate() but for time t instead of current time"""
     delta = datetime.datetime.utcfromtimestamp(
         t
     ) - datetime.datetime.fromtimestamp(t)
     tz = delta.days * 86400 + delta.seconds
     return t, tz

hgext/convert/convcmd.py

@@ -1,669 +1,667 b''
 # convcmd - convert extension commands definition
 #
 # Copyright 2005-2007 Olivia Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 import collections
 import os
 import shutil
 
 from mercurial.i18n import _
 from mercurial.pycompat import open
 from mercurial import (
     encoding,
     error,
     hg,
     pycompat,
     scmutil,
     util,
 )
 from mercurial.utils import dateutil
 
 from . import (
     bzr,
     common,
     cvs,
     darcs,
     filemap,
     git,
     gnuarch,
     hg as hgconvert,
     monotone,
     p4,
     subversion,
 )
 
 mapfile = common.mapfile
 MissingTool = common.MissingTool
 NoRepo = common.NoRepo
 SKIPREV = common.SKIPREV
 
 bzr_source = bzr.bzr_source
 convert_cvs = cvs.convert_cvs
 convert_git = git.convert_git
 darcs_source = darcs.darcs_source
 gnuarch_source = gnuarch.gnuarch_source
 mercurial_sink = hgconvert.mercurial_sink
 mercurial_source = hgconvert.mercurial_source
 monotone_source = monotone.monotone_source
 p4_source = p4.p4_source
 svn_sink = subversion.svn_sink
 svn_source = subversion.svn_source
 
 orig_encoding = b'ascii'
 
 
 def readauthormap(ui, authorfile, authors=None):
     if authors is None:
         authors = {}
     with open(authorfile, b'rb') as afile:
         for line in afile:
 
             line = line.strip()
             if not line or line.startswith(b'#'):
                 continue
 
             try:
                 srcauthor, dstauthor = line.split(b'=', 1)
             except ValueError:
                 msg = _(b'ignoring bad line in author map file %s: %s\n')
                 ui.warn(msg % (authorfile, line.rstrip()))
                 continue
 
             srcauthor = srcauthor.strip()
             dstauthor = dstauthor.strip()
             if authors.get(srcauthor) in (None, dstauthor):
                 msg = _(b'mapping author %s to %s\n')
                 ui.debug(msg % (srcauthor, dstauthor))
                 authors[srcauthor] = dstauthor
                 continue
 
             m = _(b'overriding mapping for author %s, was %s, will be %s\n')
             ui.status(m % (srcauthor, authors[srcauthor], dstauthor))
     return authors
 
 
 def recode(s):
     if isinstance(s, pycompat.unicode):
         return s.encode(pycompat.sysstr(orig_encoding), 'replace')
     else:
         return s.decode('utf-8').encode(
             pycompat.sysstr(orig_encoding), 'replace'
         )
 
 
 def mapbranch(branch, branchmap):
     """
     >>> bmap = {b'default': b'branch1'}
     >>> for i in [b'', None]:
     ...     mapbranch(i, bmap)
     'branch1'
     'branch1'
     >>> bmap = {b'None': b'branch2'}
     >>> for i in [b'', None]:
     ...     mapbranch(i, bmap)
     'branch2'
     'branch2'
     >>> bmap = {b'None': b'branch3', b'default': b'branch4'}
     >>> for i in [b'None', b'', None, b'default', b'branch5']:
     ...     mapbranch(i, bmap)
     'branch3'
     'branch4'
     'branch4'
     'branch4'
     'branch5'
     """
     # If branch is None or empty, this commit is coming from the source
     # repository's default branch and destined for the default branch in the
     # destination repository. For such commits, using a literal "default"
     # in branchmap below allows the user to map "default" to an alternate
     # default branch in the destination repository.
     branch = branchmap.get(branch or b'default', branch)
     # At some point we used "None" literal to denote the default branch,
     # attempt to use that for backward compatibility.
     if not branch:
         branch = branchmap.get(b'None', branch)
     return branch
 
 
 source_converters = [
     (b'cvs', convert_cvs, b'branchsort'),
     (b'git', convert_git, b'branchsort'),
     (b'svn', svn_source, b'branchsort'),
     (b'hg', mercurial_source, b'sourcesort'),
     (b'darcs', darcs_source, b'branchsort'),
     (b'mtn', monotone_source, b'branchsort'),
     (b'gnuarch', gnuarch_source, b'branchsort'),
     (b'bzr', bzr_source, b'branchsort'),
     (b'p4', p4_source, b'branchsort'),
 ]
 
 sink_converters = [
     (b'hg', mercurial_sink),
     (b'svn', svn_sink),
 ]
 
 
 def convertsource(ui, path, type, revs):
     exceptions = []
     if type and type not in [s[0] for s in source_converters]:
         raise error.Abort(_(b'%s: invalid source repository type') % type)
     for name, source, sortmode in source_converters:
         try:
             if not type or name == type:
                 return source(ui, name, path, revs), sortmode
         except (NoRepo, MissingTool) as inst:
             exceptions.append(inst)
     if not ui.quiet:
         for inst in exceptions:
             ui.write(b"%s\n" % pycompat.bytestr(inst.args[0]))
     raise error.Abort(_(b'%s: missing or unsupported repository') % path)
 
 
 def convertsink(ui, path, type):
     if type and type not in [s[0] for s in sink_converters]:
         raise error.Abort(_(b'%s: invalid destination repository type') % type)
     for name, sink in sink_converters:
         try:
             if not type or name == type:
                 return sink(ui, name, path)
         except NoRepo as inst:
             ui.note(_(b"convert: %s\n") % inst)
         except MissingTool as inst:
             raise error.Abort(b'%s\n' % inst)
     raise error.Abort(_(b'%s: unknown repository type') % path)
 
 
 class progresssource(object):
     def __init__(self, ui, source, filecount):
         self.ui = ui
         self.source = source
         self.progress = ui.makeprogress(
             _(b'getting files'), unit=_(b'files'), total=filecount
         )
 
     def getfile(self, file, rev):
         self.progress.increment(item=file)
         return self.source.getfile(file, rev)
 
     def targetfilebelongstosource(self, targetfilename):
         return self.source.targetfilebelongstosource(targetfilename)
 
     def lookuprev(self, rev):
         return self.source.lookuprev(rev)
 
     def close(self):
         self.progress.complete()
 
 
 class converter(object):
     def __init__(self, ui, source, dest, revmapfile, opts):
 
         self.source = source
         self.dest = dest
         self.ui = ui
         self.opts = opts
         self.commitcache = {}
         self.authors = {}
         self.authorfile = None
 
         # Record converted revisions persistently: maps source revision
         # ID to target revision ID (both strings).  (This is how
         # incremental conversions work.)
         self.map = mapfile(ui, revmapfile)
 
         # Read first the dst author map if any
         authorfile = self.dest.authorfile()
         if authorfile and os.path.exists(authorfile):
             self.readauthormap(authorfile)
         # Extend/Override with new author map if necessary
         if opts.get(b'authormap'):
             self.readauthormap(opts.get(b'authormap'))
             self.authorfile = self.dest.authorfile()
 
         self.splicemap = self.parsesplicemap(opts.get(b'splicemap'))
         self.branchmap = mapfile(ui, opts.get(b'branchmap'))
 
     def parsesplicemap(self, path):
         """check and validate the splicemap format and
         return a child/parents dictionary.
         Format checking has two parts.
         1. generic format which is same across all source types
         2. specific format checking which may be different for
            different source type.  This logic is implemented in
            checkrevformat function in source files like
            hg.py, subversion.py etc.
         """
 
         if not path:
             return {}
         m = {}
         try:
             fp = open(path, b'rb')
             for i, line in enumerate(util.iterfile(fp)):
                 line = line.splitlines()[0].rstrip()
                 if not line:
                     # Ignore blank lines
                     continue
                 # split line
                 lex = common.shlexer(data=line, whitespace=b',')
                 line = list(lex)
                 # check number of parents
                 if not (2 <= len(line) <= 3):
                     raise error.Abort(
                         _(
                             b'syntax error in %s(%d): child parent1'
                             b'[,parent2] expected'
                         )
                         % (path, i + 1)
                     )
                 for part in line:
                     self.source.checkrevformat(part)
                 child, p1, p2 = line[0], line[1:2], line[2:]
                 if p1 == p2:
                     m[child] = p1
                 else:
                     m[child] = p1 + p2
         # if file does not exist or error reading, exit
         except IOError:
             raise error.Abort(
                 _(b'splicemap file not found or error reading %s:') % path
             )
         return m
 
     def walktree(self, heads):
         """Return a mapping that identifies the uncommitted parents of every
         uncommitted changeset."""
         visit = list(heads)
         known = set()
         parents = {}
         numcommits = self.source.numcommits()
         progress = self.ui.makeprogress(
             _(b'scanning'), unit=_(b'revisions'), total=numcommits
         )
         while visit:
             n = visit.pop(0)
             if n in known:
                 continue
             if n in self.map:
                 m = self.map[n]
                 if m == SKIPREV or self.dest.hascommitfrommap(m):
                     continue
             known.add(n)
             progress.update(len(known))
             commit = self.cachecommit(n)
             parents[n] = []
             for p in commit.parents:
                 parents[n].append(p)
                 visit.append(p)
         progress.complete()
 
         return parents
 
     def mergesplicemap(self, parents, splicemap):
         """A splicemap redefines child/parent relationships. Check the
         map contains valid revision identifiers and merge the new
         links in the source graph.
         """
         for c in sorted(splicemap):
             if c not in parents:
                 if not self.dest.hascommitforsplicemap(self.map.get(c, c)):
                     # Could be in source but not converted during this run
                     self.ui.warn(
                         _(
                             b'splice map revision %s is not being '
                             b'converted, ignoring\n'
                         )
                         % c
                     )
                 continue
             pc = []
             for p in splicemap[c]:
                 # We do not have to wait for nodes already in dest.
                 if self.dest.hascommitforsplicemap(self.map.get(p, p)):
                     continue
                 # Parent is not in dest and not being converted, not good
                 if p not in parents:
                     raise error.Abort(_(b'unknown splice map parent: %s') % p)
                 pc.append(p)
             parents[c] = pc
 
     def toposort(self, parents, sortmode):
         """Return an ordering such that every uncommitted changeset is
         preceded by all its uncommitted ancestors."""
 
         def mapchildren(parents):
             """Return a (children, roots) tuple where 'children' maps parent
             revision identifiers to children ones, and 'roots' is the list of
             revisions without parents. 'parents' must be a mapping of revision
             identifier to its parents ones.
             """
             visit = collections.deque(sorted(parents))
             seen = set()
             children = {}
             roots = []
 
             while visit:
                 n = visit.popleft()
                 if n in seen:
                     continue
                 seen.add(n)
                 # Ensure that nodes without parents are present in the
                 # 'children' mapping.
                 children.setdefault(n, [])
                 hasparent = False
                 for p in parents[n]:
                     if p not in self.map:
                         visit.append(p)
                         hasparent = True
                     children.setdefault(p, []).append(n)
                 if not hasparent:
                     roots.append(n)
 
             return children, roots
 
         # Sort functions are supposed to take a list of revisions which
         # can be converted immediately and pick one
 
         def makebranchsorter():
             """If the previously converted revision has a child in the
             eligible revisions list, pick it. Return the list head
             otherwise. Branch sort attempts to minimize branch
             switching, which is harmful for Mercurial backend
             compression.
             """
             prev = [None]
 
             def picknext(nodes):
                 next = nodes[0]
                 for n in nodes:
                     if prev[0] in parents[n]:
                         next = n
                         break
                 prev[0] = next
                 return next
 
             return picknext
 
         def makesourcesorter():
             """Source specific sort."""
             keyfn = lambda n: self.commitcache[n].sortkey
 
             def picknext(nodes):
                 return sorted(nodes, key=keyfn)[0]
 
             return picknext
 
         def makeclosesorter():
             """Close order sort."""
             keyfn = lambda n: (
                 b'close' not in self.commitcache[n].extra,
                 self.commitcache[n].sortkey,
             )
 
             def picknext(nodes):
                 return sorted(nodes, key=keyfn)[0]
 
             return picknext
 
         def makedatesorter():
             """Sort revisions by date."""
             dates = {}
 
             def getdate(n):
                 if n not in dates:
                     dates[n] = dateutil.parsedate(self.commitcache[n].date)
                 return dates[n]
 
             def picknext(nodes):
                 return min([(getdate(n), n) for n in nodes])[1]
 
             return picknext
 
         if sortmode == b'branchsort':
             picknext = makebranchsorter()
         elif sortmode == b'datesort':
             picknext = makedatesorter()
         elif sortmode == b'sourcesort':
             picknext = makesourcesorter()
         elif sortmode == b'closesort':
             picknext = makeclosesorter()
         else:
             raise error.Abort(_(b'unknown sort mode: %s') % sortmode)
 
         children, actives = mapchildren(parents)
 
         s = []
         pendings = {}
         while actives:
             n = picknext(actives)
             actives.remove(n)
             s.append(n)
 
             # Update dependents list
             for c in children.get(n, []):
                 if c not in pendings:
                     pendings[c] = [p for p in parents[c] if p not in self.map]
                 try:
                     pendings[c].remove(n)
                 except ValueError:
                     raise error.Abort(
                         _(b'cycle detected between %s and %s')
                         % (recode(c), recode(n))
                     )
                 if not pendings[c]:
                     # Parents are converted, node is eligible
                     actives.insert(0, c)
                     pendings[c] = None
 
         if len(s) != len(parents):
             raise error.Abort(_(b"not all revisions were sorted"))
 
         return s
 
     def writeauthormap(self):
         authorfile = self.authorfile
         if authorfile:
             self.ui.status(_(b'writing author map file %s\n') % authorfile)
             ofile = open(authorfile, b'wb+')
             for author in self.authors:
                 ofile.write(
                     util.tonativeeol(
                         b"%s=%s\n" % (author, self.authors[author])
                     )
                 )
             ofile.close()
 
     def readauthormap(self, authorfile):
         self.authors = readauthormap(self.ui, authorfile, self.authors)
 
     def cachecommit(self, rev):
         commit = self.source.getcommit(rev)
         commit.author = self.authors.get(commit.author, commit.author)
         commit.branch = mapbranch(commit.branch, self.branchmap)
         self.commitcache[rev] = commit
         return commit
 
     def copy(self, rev):
         commit = self.commitcache[rev]
         full = self.opts.get(b'full')
         changes = self.source.getchanges(rev, full)
         if isinstance(changes, bytes):
             if changes == SKIPREV:
                 dest = SKIPREV
             else:
                 dest = self.map[changes]
             self.map[rev] = dest
             return
         files, copies, cleanp2 = changes
         pbranches = []
         if commit.parents:
             for prev in commit.parents:
                 if prev not in self.commitcache:
                     self.cachecommit(prev)
                 pbranches.append(
                     (self.map[prev], self.commitcache[prev].branch)
                 )
         self.dest.setbranch(commit.branch, pbranches)
         try:
             parents = self.splicemap[rev]
             self.ui.status(
                 _(b'spliced in %s as parents of %s\n')
                 % (_(b' and ').join(parents), rev)
             )
             parents = [self.map.get(p, p) for p in parents]
         except KeyError:
             parents = [b[0] for b in pbranches]
             parents.extend(
                 self.map[x] for x in commit.optparents if x in self.map
             )
         if len(pbranches) != 2:
             cleanp2 = set()
         if len(parents) < 3:
             source = progresssource(self.ui, self.source, len(files))
         else:
             # For an octopus merge, we end up traversing the list of
             # changed files N-1 times. This tweak to the number of
             # files makes it so the progress bar doesn't overflow
             # itself.
             source = progresssource(
                 self.ui, self.source, len(files) * (len(parents) - 1)
             )
         newnode = self.dest.putcommit(
             files, copies, parents, commit, source, self.map, full, cleanp2
         )
         source.close()
         self.source.converted(rev, newnode)
         self.map[rev] = newnode
 
     def convert(self, sortmode):
         try:
             self.source.before()
             self.dest.before()
             self.source.setrevmap(self.map)
             self.ui.status(_(b"scanning source...\n"))
             heads = self.source.getheads()
             parents = self.walktree(heads)
             self.mergesplicemap(parents, self.splicemap)
             self.ui.status(_(b"sorting...\n"))
             t = self.toposort(parents, sortmode)
             num = len(t)
             c = None
 
             self.ui.status(_(b"converting...\n"))
             progress = self.ui.makeprogress(
                 _(b'converting'), unit=_(b'revisions'), total=len(t)
             )
             for i, c in enumerate(t):
                 num -= 1
                 desc = self.commitcache[c].desc
                 if b"\n" in desc:
                     desc = desc.splitlines()[0]
                 # convert log message to local encoding without using
                 # tolocal() because the encoding.encoding convert()
                 # uses is 'utf-8'
                 self.ui.status(b"%d %s\n" % (num, recode(desc)))
                 self.ui.note(_(b"source: %s\n") % recode(c))
                 progress.update(i)
                 self.copy(c)
             progress.complete()
 
             if not self.ui.configbool(b'convert', b'skiptags'):
                 tags = self.source.gettags()
                 ctags = {}
                 for k in tags:
                     v = tags[k]
                     if self.map.get(v, SKIPREV) != SKIPREV:
                         ctags[k] = self.map[v]
 
                 if c and ctags:
                     nrev, tagsparent = self.dest.puttags(ctags)
                     if nrev and tagsparent:
                         # write another hash correspondence to override the
                         # previous one so we don't end up with extra tag heads
                         tagsparents = [
-                            e
-                            for e in pycompat.iteritems(self.map)
-                            if e[1] == tagsparent
+                            e for e in self.map.items() if e[1] == tagsparent
                         ]
                         if tagsparents:
                             self.map[tagsparents[0][0]] = nrev
 
             bookmarks = self.source.getbookmarks()
             cbookmarks = {}
             for k in bookmarks:
                 v = bookmarks[k]
                 if self.map.get(v, SKIPREV) != SKIPREV:
                     cbookmarks[k] = self.map[v]
 
             if c and cbookmarks:
                 self.dest.putbookmarks(cbookmarks)
 
             self.writeauthormap()
         finally:
             self.cleanup()
 
     def cleanup(self):
         try:
             self.dest.after()
         finally:
             self.source.after()
         self.map.close()
 
 
 def convert(ui, src, dest=None, revmapfile=None, **opts):
     opts = pycompat.byteskwargs(opts)
     global orig_encoding
     orig_encoding = encoding.encoding
     encoding.encoding = b'UTF-8'
 
     # support --authors as an alias for --authormap
     if not opts.get(b'authormap'):
         opts[b'authormap'] = opts.get(b'authors')
 
     if not dest:
         dest = hg.defaultdest(src) + b"-hg"
         ui.status(_(b"assuming destination %s\n") % dest)
 
     destc = convertsink(ui, dest, opts.get(b'dest_type'))
     destc = scmutil.wrapconvertsink(destc)
 
     try:
         srcc, defaultsort = convertsource(
             ui, src, opts.get(b'source_type'), opts.get(b'rev')
         )
     except Exception:
         for path in destc.created:
             shutil.rmtree(path, True)
         raise
 
     sortmodes = (b'branchsort', b'datesort', b'sourcesort', b'closesort')
     sortmode = [m for m in sortmodes if opts.get(m)]
     if len(sortmode) > 1:
         raise error.Abort(_(b'more than one sort mode specified'))
     if sortmode:
         sortmode = sortmode[0]
     else:
         sortmode = defaultsort
 
     if sortmode == b'sourcesort' and not srcc.hasnativeorder():
         raise error.Abort(
             _(b'--sourcesort is not supported by this data source')
         )
     if sortmode == b'closesort' and not srcc.hasnativeclose():
         raise error.Abort(
             _(b'--closesort is not supported by this data source')
         )
 
     fmap = opts.get(b'filemap')
     if fmap:
         srcc = filemap.filemap_source(ui, srcc, fmap)
         destc.setfilemapmode(True)
 
     if not revmapfile:
         revmapfile = destc.revmapfile()
 
     c = converter(ui, srcc, destc, revmapfile, opts)
     c.convert(sortmode)

hgext/convert/cvsps.py

@@ -1,1068 +1,1068 b''
 # Mercurial built-in replacement for cvsps.
 #
 # Copyright 2008, Frank Kingswood <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 import functools
 import os
 import pickle
 import re
 
 from mercurial.i18n import _
 from mercurial.pycompat import open
 from mercurial import (
     encoding,
     error,
     hook,
     pycompat,
     util,
 )
 from mercurial.utils import (
     dateutil,
     procutil,
     stringutil,
 )
 
 
 class logentry(object):
     """Class logentry has the following attributes:
     .author    - author name as CVS knows it
     .branch    - name of branch this revision is on
     .branches  - revision tuple of branches starting at this revision
     .comment   - commit message
     .commitid  - CVS commitid or None
     .date      - the commit date as a (time, tz) tuple
     .dead      - true if file revision is dead
     .file      - Name of file
     .lines     - a tuple (+lines, -lines) or None
     .parent    - Previous revision of this entry
     .rcs       - name of file as returned from CVS
     .revision  - revision number as tuple
     .tags      - list of tags on the file
     .synthetic - is this a synthetic "file ... added on ..." revision?
     .mergepoint - the branch that has been merged from (if present in
                   rlog output) or None
     .branchpoints - the branches that start at the current entry or empty
     """
 
     def __init__(self, **entries):
         self.synthetic = False
         self.__dict__.update(entries)
 
     def __repr__(self):
         items = ("%s=%r" % (k, self.__dict__[k]) for k in sorted(self.__dict__))
         return "%s(%s)" % (type(self).__name__, ", ".join(items))
 
 
 class logerror(Exception):
     pass
 
 
 def getrepopath(cvspath):
     """Return the repository path from a CVS path.
 
     >>> getrepopath(b'/foo/bar')
     '/foo/bar'
     >>> getrepopath(b'c:/foo/bar')
     '/foo/bar'
     >>> getrepopath(b':pserver:10/foo/bar')
     '/foo/bar'
     >>> getrepopath(b':pserver:10c:/foo/bar')
     '/foo/bar'
     >>> getrepopath(b':pserver:/foo/bar')
     '/foo/bar'
     >>> getrepopath(b':pserver:c:/foo/bar')
     '/foo/bar'
     >>> getrepopath(b':pserver:[e-mail]:/foo/bar')
     '/foo/bar'
     >>> getrepopath(b':pserver:[e-mail]:c:/foo/bar')
     '/foo/bar'
     >>> getrepopath(b'user@server/path/to/repository')
     '/path/to/repository'
     """
     # According to CVS manual, CVS paths are expressed like:
     # [:method:][[user][:password]@]hostname[:[port]]/path/to/repository
     #
     # CVSpath is splitted into parts and then position of the first occurrence
     # of the '/' char after the '@' is located. The solution is the rest of the
     # string after that '/' sign including it
 
     parts = cvspath.split(b':')
     atposition = parts[-1].find(b'@')
     start = 0
 
     if atposition != -1:
         start = atposition
 
     repopath = parts[-1][parts[-1].find(b'/', start) :]
     return repopath
 
 
 def createlog(ui, directory=None, root=b"", rlog=True, cache=None):
     '''Collect the CVS rlog'''
 
     # Because we store many duplicate commit log messages, reusing strings
     # saves a lot of memory and pickle storage space.
     _scache = {}
 
     def scache(s):
         """return a shared version of a string"""
         return _scache.setdefault(s, s)
 
     ui.status(_(b'collecting CVS rlog\n'))
 
     log = []  # list of logentry objects containing the CVS state
 
     # patterns to match in CVS (r)log output, by state of use
     re_00 = re.compile(b'RCS file: (.+)$')
     re_01 = re.compile(b'cvs \\[r?log aborted\\]: (.+)$')
     re_02 = re.compile(b'cvs (r?log|server): (.+)\n$')
     re_03 = re.compile(
         b"(Cannot access.+CVSROOT)|(can't create temporary directory.+)$"
     )
     re_10 = re.compile(b'Working file: (.+)$')
     re_20 = re.compile(b'symbolic names:')
     re_30 = re.compile(b'\t(.+): ([\\d.]+)$')
     re_31 = re.compile(b'----------------------------$')
     re_32 = re.compile(
         b'======================================='
         b'======================================$'
     )
     re_50 = re.compile(br'revision ([\d.]+)(\s+locked by:\s+.+;)?$')
     re_60 = re.compile(
         br'date:\s+(.+);\s+author:\s+(.+);\s+state:\s+(.+?);'
         br'(\s+lines:\s+(\+\d+)?\s+(-\d+)?;)?'
         br'(\s+commitid:\s+([^;]+);)?'
         br'(.*mergepoint:\s+([^;]+);)?'
     )
     re_70 = re.compile(b'branches: (.+);$')
 
     file_added_re = re.compile(br'file [^/]+ was (initially )?added on branch')
 
     prefix = b''  # leading path to strip of what we get from CVS
 
     if directory is None:
         # Current working directory
 
         # Get the real directory in the repository
         try:
             with open(os.path.join(b'CVS', b'Repository'), b'rb') as f:
                 prefix = f.read().strip()
             directory = prefix
             if prefix == b".":
                 prefix = b""
         except IOError:
             raise logerror(_(b'not a CVS sandbox'))
 
         if prefix and not prefix.endswith(pycompat.ossep):
             prefix += pycompat.ossep
 
         # Use the Root file in the sandbox, if it exists
         try:
             root = open(os.path.join(b'CVS', b'Root'), b'rb').read().strip()
         except IOError:
             pass
 
     if not root:
         root = encoding.environ.get(b'CVSROOT', b'')
 
     # read log cache if one exists
     oldlog = []
     date = None
 
     if cache:
         cachedir = os.path.expanduser(b'~/.hg.cvsps')
         if not os.path.exists(cachedir):
             os.mkdir(cachedir)
 
         # The cvsps cache pickle needs a uniquified name, based on the
         # repository location. The address may have all sort of nasties
         # in it, slashes, colons and such. So here we take just the
         # alphanumeric characters, concatenated in a way that does not
         # mix up the various components, so that
         #    :pserver:user@server:/path
         # and
         #    /pserver/user/server/path
         # are mapped to different cache file names.
         cachefile = root.split(b":") + [directory, b"cache"]
         cachefile = [b'-'.join(re.findall(br'\w+', s)) for s in cachefile if s]
         cachefile = os.path.join(
             cachedir, b'.'.join([s for s in cachefile if s])
         )
 
     if cache == b'update':
         try:
             ui.note(_(b'reading cvs log cache %s\n') % cachefile)
             oldlog = pickle.load(open(cachefile, b'rb'))
             for e in oldlog:
                 if not (
                     util.safehasattr(e, b'branchpoints')
                     and util.safehasattr(e, b'commitid')
                     and util.safehasattr(e, b'mergepoint')
                 ):
                     ui.status(_(b'ignoring old cache\n'))
                     oldlog = []
                     break
 
             ui.note(_(b'cache has %d log entries\n') % len(oldlog))
         except Exception as e:
             ui.note(_(b'error reading cache: %r\n') % e)
 
         if oldlog:
             date = oldlog[-1].date  # last commit date as a (time,tz) tuple
             date = dateutil.datestr(date, b'%Y/%m/%d %H:%M:%S %1%2')
 
     # build the CVS commandline
     cmd = [b'cvs', b'-q']
     if root:
         cmd.append(b'-d%s' % root)
         p = util.normpath(getrepopath(root))
         if not p.endswith(b'/'):
             p += b'/'
         if prefix:
             # looks like normpath replaces "" by "."
             prefix = p + util.normpath(prefix)
         else:
             prefix = p
     cmd.append([b'log', b'rlog'][rlog])
     if date:
         # no space between option and date string
         cmd.append(b'-d>%s' % date)
     cmd.append(directory)
 
     # state machine begins here
     tags = {}  # dictionary of revisions on current file with their tags
     branchmap = {}  # mapping between branch names and revision numbers
     rcsmap = {}
     state = 0
     store = False  # set when a new record can be appended
 
     cmd = [procutil.shellquote(arg) for arg in cmd]
     ui.note(_(b"running %s\n") % (b' '.join(cmd)))
     ui.debug(b"prefix=%r directory=%r root=%r\n" % (prefix, directory, root))
 
     pfp = procutil.popen(b' '.join(cmd), b'rb')
     peek = util.fromnativeeol(pfp.readline())
     while True:
         line = peek
         if line == b'':
             break
         peek = util.fromnativeeol(pfp.readline())
         if line.endswith(b'\n'):
             line = line[:-1]
         # ui.debug('state=%d line=%r\n' % (state, line))
 
         if state == 0:
             # initial state, consume input until we see 'RCS file'
             match = re_00.match(line)
             if match:
                 rcs = match.group(1)
                 tags = {}
                 if rlog:
                     filename = util.normpath(rcs[:-2])
                     if filename.startswith(prefix):
                         filename = filename[len(prefix) :]
                     if filename.startswith(b'/'):
                         filename = filename[1:]
                     if filename.startswith(b'Attic/'):
                         filename = filename[6:]
                     else:
                         filename = filename.replace(b'/Attic/', b'/')
                     state = 2
                     continue
                 state = 1
                 continue
             match = re_01.match(line)
             if match:
                 raise logerror(match.group(1))
             match = re_02.match(line)
             if match:
                 raise logerror(match.group(2))
             if re_03.match(line):
                 raise logerror(line)
 
         elif state == 1:
             # expect 'Working file' (only when using log instead of rlog)
             match = re_10.match(line)
             assert match, _(b'RCS file must be followed by working file')
             filename = util.normpath(match.group(1))
             state = 2
 
         elif state == 2:
             # expect 'symbolic names'
             if re_20.match(line):
                 branchmap = {}
                 state = 3
 
         elif state == 3:
             # read the symbolic names and store as tags
             match = re_30.match(line)
             if match:
                 rev = [int(x) for x in match.group(2).split(b'.')]
 
                 # Convert magic branch number to an odd-numbered one
                 revn = len(rev)
                 if revn > 3 and (revn % 2) == 0 and rev[-2] == 0:
                     rev = rev[:-2] + rev[-1:]
                 rev = tuple(rev)
 
                 if rev not in tags:
                     tags[rev] = []
                 tags[rev].append(match.group(1))
                 branchmap[match.group(1)] = match.group(2)
 
             elif re_31.match(line):
                 state = 5
             elif re_32.match(line):
                 state = 0
 
         elif state == 4:
             # expecting '------' separator before first revision
             if re_31.match(line):
                 state = 5
             else:
                 assert not re_32.match(line), _(
                     b'must have at least some revisions'
                 )
 
         elif state == 5:
             # expecting revision number and possibly (ignored) lock indication
             # we create the logentry here from values stored in states 0 to 4,
             # as this state is re-entered for subsequent revisions of a file.
             match = re_50.match(line)
             assert match, _(b'expected revision number')
             e = logentry(
                 rcs=scache(rcs),
                 file=scache(filename),
                 revision=tuple([int(x) for x in match.group(1).split(b'.')]),
                 branches=[],
                 parent=None,
                 commitid=None,
                 mergepoint=None,
                 branchpoints=set(),
             )
 
             state = 6
 
         elif state == 6:
             # expecting date, author, state, lines changed
             match = re_60.match(line)
             assert match, _(b'revision must be followed by date line')
             d = match.group(1)
             if d[2] == b'/':
                 # Y2K
                 d = b'19' + d
 
             if len(d.split()) != 3:
                 # cvs log dates always in GMT
                 d = d + b' UTC'
             e.date = dateutil.parsedate(
                 d,
                 [
                     b'%y/%m/%d %H:%M:%S',
                     b'%Y/%m/%d %H:%M:%S',
                     b'%Y-%m-%d %H:%M:%S',
                 ],
             )
             e.author = scache(match.group(2))
             e.dead = match.group(3).lower() == b'dead'
 
             if match.group(5):
                 if match.group(6):
                     e.lines = (int(match.group(5)), int(match.group(6)))
                 else:
                     e.lines = (int(match.group(5)), 0)
             elif match.group(6):
                 e.lines = (0, int(match.group(6)))
             else:
                 e.lines = None
 
             if match.group(7):  # cvs 1.12 commitid
                 e.commitid = match.group(8)
 
             if match.group(9):  # cvsnt mergepoint
                 myrev = match.group(10).split(b'.')
                 if len(myrev) == 2:  # head
                     e.mergepoint = b'HEAD'
                 else:
                     myrev = b'.'.join(myrev[:-2] + [b'0', myrev[-2]])
                     branches = [b for b in branchmap if branchmap[b] == myrev]
                     assert len(branches) == 1, (
                         b'unknown branch: %s' % e.mergepoint
                     )
                     e.mergepoint = branches[0]
 
             e.comment = []
             state = 7
 
         elif state == 7:
             # read the revision numbers of branches that start at this revision
             # or store the commit log message otherwise
             m = re_70.match(line)
             if m:
                 e.branches = [
                     tuple([int(y) for y in x.strip().split(b'.')])
                     for x in m.group(1).split(b';')
                 ]
                 state = 8
             elif re_31.match(line) and re_50.match(peek):
                 state = 5
                 store = True
             elif re_32.match(line):
                 state = 0
                 store = True
             else:
                 e.comment.append(line)
 
         elif state == 8:
             # store commit log message
             if re_31.match(line):
                 cpeek = peek
                 if cpeek.endswith(b'\n'):
                     cpeek = cpeek[:-1]
                 if re_50.match(cpeek):
                     state = 5
                     store = True
                 else:
                     e.comment.append(line)
             elif re_32.match(line):
                 state = 0
                 store = True
             else:
                 e.comment.append(line)
 
         # When a file is added on a branch B1, CVS creates a synthetic
         # dead trunk revision 1.1 so that the branch has a root.
         # Likewise, if you merge such a file to a later branch B2 (one
         # that already existed when the file was added on B1), CVS
         # creates a synthetic dead revision 1.1.x.1 on B2.  Don't drop
         # these revisions now, but mark them synthetic so
         # createchangeset() can take care of them.
         if (
             store
             and e.dead
             and e.revision[-1] == 1
             and len(e.comment) == 1  # 1.1 or 1.1.x.1
             and file_added_re.match(e.comment[0])
         ):
             ui.debug(
                 b'found synthetic revision in %s: %r\n' % (e.rcs, e.comment[0])
             )
             e.synthetic = True
 
         if store:
             # clean up the results and save in the log.
             store = False
             e.tags = sorted([scache(x) for x in tags.get(e.revision, [])])
             e.comment = scache(b'\n'.join(e.comment))
 
             revn = len(e.revision)
             if revn > 3 and (revn % 2) == 0:
                 e.branch = tags.get(e.revision[:-1], [None])[0]
             else:
                 e.branch = None
 
             # find the branches starting from this revision
             branchpoints = set()
-            for branch, revision in pycompat.iteritems(branchmap):
+            for branch, revision in branchmap.items():
                 revparts = tuple([int(i) for i in revision.split(b'.')])
                 if len(revparts) < 2:  # bad tags
                     continue
                 if revparts[-2] == 0 and revparts[-1] % 2 == 0:
                     # normal branch
                     if revparts[:-2] == e.revision:
                         branchpoints.add(branch)
                 elif revparts == (1, 1, 1):  # vendor branch
                     if revparts in e.branches:
                         branchpoints.add(branch)
             e.branchpoints = branchpoints
 
             log.append(e)
 
             rcsmap[e.rcs.replace(b'/Attic/', b'/')] = e.rcs
 
             if len(log) % 100 == 0:
                 ui.status(
                     stringutil.ellipsis(b'%d %s' % (len(log), e.file), 80)
                     + b'\n'
                 )
 
     log.sort(key=lambda x: (x.rcs, x.revision))
 
     # find parent revisions of individual files
     versions = {}
     for e in sorted(oldlog, key=lambda x: (x.rcs, x.revision)):
         rcs = e.rcs.replace(b'/Attic/', b'/')
         if rcs in rcsmap:
             e.rcs = rcsmap[rcs]
         branch = e.revision[:-1]
         versions[(e.rcs, branch)] = e.revision
 
     for e in log:
         branch = e.revision[:-1]
         p = versions.get((e.rcs, branch), None)
         if p is None:
             p = e.revision[:-2]
         e.parent = p
         versions[(e.rcs, branch)] = e.revision
 
     # update the log cache
     if cache:
         if log:
             # join up the old and new logs
             log.sort(key=lambda x: x.date)
 
             if oldlog and oldlog[-1].date >= log[0].date:
                 raise logerror(
                     _(
                         b'log cache overlaps with new log entries,'
                         b' re-run without cache.'
                     )
                 )
 
             log = oldlog + log
 
             # write the new cachefile
             ui.note(_(b'writing cvs log cache %s\n') % cachefile)
             pickle.dump(log, open(cachefile, b'wb'))
         else:
             log = oldlog
 
     ui.status(_(b'%d log entries\n') % len(log))
 
     encodings = ui.configlist(b'convert', b'cvsps.logencoding')
     if encodings:
 
         def revstr(r):
             # this is needed, because logentry.revision is a tuple of "int"
             # (e.g. (1, 2) for "1.2")
             return b'.'.join(pycompat.maplist(pycompat.bytestr, r))
 
         for entry in log:
             comment = entry.comment
             for e in encodings:
                 try:
                     entry.comment = comment.decode(pycompat.sysstr(e)).encode(
                         'utf-8'
                     )
                     if ui.debugflag:
                         ui.debug(
                             b"transcoding by %s: %s of %s\n"
                             % (e, revstr(entry.revision), entry.file)
                         )
                     break
                 except UnicodeDecodeError:
                     pass  # try next encoding
                 except LookupError as inst:  # unknown encoding, maybe
                     raise error.Abort(
                         pycompat.bytestr(inst),
                         hint=_(
                             b'check convert.cvsps.logencoding configuration'
                         ),
                     )
             else:
                 raise error.Abort(
                     _(
                         b"no encoding can transcode"
                         b" CVS log message for %s of %s"
                     )
                     % (revstr(entry.revision), entry.file),
                     hint=_(b'check convert.cvsps.logencoding configuration'),
                 )
 
     hook.hook(ui, None, b"cvslog", True, log=log)
 
     return log
 
 
 class changeset(object):
     """Class changeset has the following attributes:
     .id        - integer identifying this changeset (list index)
     .author    - author name as CVS knows it
     .branch    - name of branch this changeset is on, or None
     .comment   - commit message
     .commitid  - CVS commitid or None
     .date      - the commit date as a (time,tz) tuple
     .entries   - list of logentry objects in this changeset
     .parents   - list of one or two parent changesets
     .tags      - list of tags on this changeset
     .synthetic - from synthetic revision "file ... added on branch ..."
     .mergepoint- the branch that has been merged from or None
     .branchpoints- the branches that start at the current entry or empty
     """
 
     def __init__(self, **entries):
         self.id = None
         self.synthetic = False
         self.__dict__.update(entries)
 
     def __repr__(self):
         items = (
             b"%s=%r" % (k, self.__dict__[k]) for k in sorted(self.__dict__)
         )
         return b"%s(%s)" % (type(self).__name__, b", ".join(items))
 
 
 def createchangeset(ui, log, fuzz=60, mergefrom=None, mergeto=None):
     '''Convert log into changesets.'''
 
     ui.status(_(b'creating changesets\n'))
 
     # try to order commitids by date
     mindate = {}
     for e in log:
         if e.commitid:
             if e.commitid not in mindate:
                 mindate[e.commitid] = e.date
             else:
                 mindate[e.commitid] = min(e.date, mindate[e.commitid])
 
     # Merge changesets
     log.sort(
         key=lambda x: (
             mindate.get(x.commitid, (-1, 0)),
             x.commitid or b'',
             x.comment,
             x.author,
             x.branch or b'',
             x.date,
             x.branchpoints,
         )
     )
 
     changesets = []
     files = set()
     c = None
     for i, e in enumerate(log):
 
         # Check if log entry belongs to the current changeset or not.
 
         # Since CVS is file-centric, two different file revisions with
         # different branchpoints should be treated as belonging to two
         # different changesets (and the ordering is important and not
         # honoured by cvsps at this point).
         #
         # Consider the following case:
         # foo 1.1 branchpoints: [MYBRANCH]
         # bar 1.1 branchpoints: [MYBRANCH, MYBRANCH2]
         #
         # Here foo is part only of MYBRANCH, but not MYBRANCH2, e.g. a
         # later version of foo may be in MYBRANCH2, so foo should be the
         # first changeset and bar the next and MYBRANCH and MYBRANCH2
         # should both start off of the bar changeset. No provisions are
         # made to ensure that this is, in fact, what happens.
         if not (
             c
             and e.branchpoints == c.branchpoints
             and (  # cvs commitids
                 (e.commitid is not None and e.commitid == c.commitid)
                 or (  # no commitids, use fuzzy commit detection
                     (e.commitid is None or c.commitid is None)
                     and e.comment == c.comment
                     and e.author == c.author
                     and e.branch == c.branch
                     and (
                         (c.date[0] + c.date[1])
                         <= (e.date[0] + e.date[1])
                         <= (c.date[0] + c.date[1]) + fuzz
                     )
                     and e.file not in files
                 )
             )
         ):
             c = changeset(
                 comment=e.comment,
                 author=e.author,
                 branch=e.branch,
                 date=e.date,
                 entries=[],
                 mergepoint=e.mergepoint,
                 branchpoints=e.branchpoints,
                 commitid=e.commitid,
             )
             changesets.append(c)
 
             files = set()
             if len(changesets) % 100 == 0:
                 t = b'%d %s' % (len(changesets), repr(e.comment)[1:-1])
                 ui.status(stringutil.ellipsis(t, 80) + b'\n')
 
         c.entries.append(e)
         files.add(e.file)
         c.date = e.date  # changeset date is date of latest commit in it
 
     # Mark synthetic changesets
 
     for c in changesets:
         # Synthetic revisions always get their own changeset, because
         # the log message includes the filename.  E.g. if you add file3
         # and file4 on a branch, you get four log entries and three
         # changesets:
         #   "File file3 was added on branch ..." (synthetic, 1 entry)
         #   "File file4 was added on branch ..." (synthetic, 1 entry)
         #   "Add file3 and file4 to fix ..."     (real, 2 entries)
         # Hence the check for 1 entry here.
         c.synthetic = len(c.entries) == 1 and c.entries[0].synthetic
 
     # Sort files in each changeset
 
     def entitycompare(l, r):
         """Mimic cvsps sorting order"""
         l = l.file.split(b'/')
         r = r.file.split(b'/')
         nl = len(l)
         nr = len(r)
         n = min(nl, nr)
         for i in range(n):
             if i + 1 == nl and nl < nr:
                 return -1
             elif i + 1 == nr and nl > nr:
                 return +1
             elif l[i] < r[i]:
                 return -1
             elif l[i] > r[i]:
                 return +1
         return 0
 
     for c in changesets:
         c.entries.sort(key=functools.cmp_to_key(entitycompare))
 
     # Sort changesets by date
 
     odd = set()
 
     def cscmp(l, r):
         d = sum(l.date) - sum(r.date)
         if d:
             return d
 
         # detect vendor branches and initial commits on a branch
         le = {}
         for e in l.entries:
             le[e.rcs] = e.revision
         re = {}
         for e in r.entries:
             re[e.rcs] = e.revision
 
         d = 0
         for e in l.entries:
             if re.get(e.rcs, None) == e.parent:
                 assert not d
                 d = 1
                 break
 
         for e in r.entries:
             if le.get(e.rcs, None) == e.parent:
                 if d:
                     odd.add((l, r))
                 d = -1
                 break
         # By this point, the changesets are sufficiently compared that
         # we don't really care about ordering. However, this leaves
         # some race conditions in the tests, so we compare on the
         # number of files modified, the files contained in each
         # changeset, and the branchpoints in the change to ensure test
         # output remains stable.
 
         # recommended replacement for cmp from
         # https://docs.python.org/3.0/whatsnew/3.0.html
         c = lambda x, y: (x > y) - (x < y)
         # Sort bigger changes first.
         if not d:
             d = c(len(l.entries), len(r.entries))
         # Try sorting by filename in the change.
         if not d:
             d = c([e.file for e in l.entries], [e.file for e in r.entries])
         # Try and put changes without a branch point before ones with
         # a branch point.
         if not d:
             d = c(len(l.branchpoints), len(r.branchpoints))
         return d
 
     changesets.sort(key=functools.cmp_to_key(cscmp))
 
     # Collect tags
 
     globaltags = {}
     for c in changesets:
         for e in c.entries:
             for tag in e.tags:
                 # remember which is the latest changeset to have this tag
                 globaltags[tag] = c
 
     for c in changesets:
         tags = set()
         for e in c.entries:
             tags.update(e.tags)
         # remember tags only if this is the latest changeset to have it
         c.tags = sorted(tag for tag in tags if globaltags[tag] is c)
 
     # Find parent changesets, handle {{mergetobranch BRANCHNAME}}
     # by inserting dummy changesets with two parents, and handle
     # {{mergefrombranch BRANCHNAME}} by setting two parents.
 
     if mergeto is None:
         mergeto = br'{{mergetobranch ([-\w]+)}}'
     if mergeto:
         mergeto = re.compile(mergeto)
 
     if mergefrom is None:
         mergefrom = br'{{mergefrombranch ([-\w]+)}}'
     if mergefrom:
         mergefrom = re.compile(mergefrom)
 
     versions = {}  # changeset index where we saw any particular file version
     branches = {}  # changeset index where we saw a branch
     n = len(changesets)
     i = 0
     while i < n:
         c = changesets[i]
 
         for f in c.entries:
             versions[(f.rcs, f.revision)] = i
 
         p = None
         if c.branch in branches:
             p = branches[c.branch]
         else:
             # first changeset on a new branch
             # the parent is a changeset with the branch in its
             # branchpoints such that it is the latest possible
             # commit without any intervening, unrelated commits.
 
             for candidate in pycompat.xrange(i):
                 if c.branch not in changesets[candidate].branchpoints:
                     if p is not None:
                         break
                     continue
                 p = candidate
 
         c.parents = []
         if p is not None:
             p = changesets[p]
 
             # Ensure no changeset has a synthetic changeset as a parent.
             while p.synthetic:
                 assert len(p.parents) <= 1, _(
                     b'synthetic changeset cannot have multiple parents'
                 )
                 if p.parents:
                     p = p.parents[0]
                 else:
                     p = None
                     break
 
             if p is not None:
                 c.parents.append(p)
 
         if c.mergepoint:
             if c.mergepoint == b'HEAD':
                 c.mergepoint = None
             c.parents.append(changesets[branches[c.mergepoint]])
 
         if mergefrom:
             m = mergefrom.search(c.comment)
             if m:
                 m = m.group(1)
                 if m == b'HEAD':
                     m = None
                 try:
                     candidate = changesets[branches[m]]
                 except KeyError:
                     ui.warn(
                         _(
                             b"warning: CVS commit message references "
                             b"non-existent branch %r:\n%s\n"
                         )
                         % (pycompat.bytestr(m), c.comment)
                     )
                 if m in branches and c.branch != m and not candidate.synthetic:
                     c.parents.append(candidate)
 
         if mergeto:
             m = mergeto.search(c.comment)
             if m:
                 if m.groups():
                     m = m.group(1)
                     if m == b'HEAD':
                         m = None
                 else:
                     m = None  # if no group found then merge to HEAD
                 if m in branches and c.branch != m:
                     # insert empty changeset for merge
                     cc = changeset(
                         author=c.author,
                         branch=m,
                         date=c.date,
                         comment=b'convert-repo: CVS merge from branch %s'
                         % c.branch,
                         entries=[],
                         tags=[],
                         parents=[changesets[branches[m]], c],
                     )
                     changesets.insert(i + 1, cc)
                     branches[m] = i + 1
 
                     # adjust our loop counters now we have inserted a new entry
                     n += 1
                     i += 2
                     continue
 
         branches[c.branch] = i
         i += 1
 
     # Drop synthetic changesets (safe now that we have ensured no other
     # changesets can have them as parents).
     i = 0
     while i < len(changesets):
         if changesets[i].synthetic:
             del changesets[i]
         else:
             i += 1
 
     # Number changesets
 
     for i, c in enumerate(changesets):
         c.id = i + 1
 
     if odd:
         for l, r in odd:
             if l.id is not None and r.id is not None:
                 ui.warn(
                     _(b'changeset %d is both before and after %d\n')
                     % (l.id, r.id)
                 )
 
     ui.status(_(b'%d changeset entries\n') % len(changesets))
 
     hook.hook(ui, None, b"cvschangesets", True, changesets=changesets)
 
     return changesets
 
 
 def debugcvsps(ui, *args, **opts):
     """Read CVS rlog for current directory or named path in
     repository, and convert the log to changesets based on matching
     commit log entries and dates.
     """
     opts = pycompat.byteskwargs(opts)
     if opts[b"new_cache"]:
         cache = b"write"
     elif opts[b"update_cache"]:
         cache = b"update"
     else:
         cache = None
 
     revisions = opts[b"revisions"]
 
     try:
         if args:
             log = []
             for d in args:
                 log += createlog(ui, d, root=opts[b"root"], cache=cache)
         else:
             log = createlog(ui, root=opts[b"root"], cache=cache)
     except logerror as e:
         ui.write(b"%r\n" % e)
         return
 
     changesets = createchangeset(ui, log, opts[b"fuzz"])
     del log
 
     # Print changesets (optionally filtered)
 
     off = len(revisions)
     branches = {}  # latest version number in each branch
     ancestors = {}  # parent branch
     for cs in changesets:
 
         if opts[b"ancestors"]:
             if cs.branch not in branches and cs.parents and cs.parents[0].id:
                 ancestors[cs.branch] = (
                     changesets[cs.parents[0].id - 1].branch,
                     cs.parents[0].id,
                 )
             branches[cs.branch] = cs.id
 
         # limit by branches
         if (
             opts[b"branches"]
             and (cs.branch or b'HEAD') not in opts[b"branches"]
         ):
             continue
 
         if not off:
             # Note: trailing spaces on several lines here are needed to have
             #       bug-for-bug compatibility with cvsps.
             ui.write(b'---------------------\n')
             ui.write((b'PatchSet %d \n' % cs.id))
             ui.write(
                 (
                     b'Date: %s\n'
                     % dateutil.datestr(cs.date, b'%Y/%m/%d %H:%M:%S %1%2')
                 )
             )
             ui.write((b'Author: %s\n' % cs.author))
             ui.write((b'Branch: %s\n' % (cs.branch or b'HEAD')))
             ui.write(
                 (
                     b'Tag%s: %s \n'
                     % (
                         [b'', b's'][len(cs.tags) > 1],
                         b','.join(cs.tags) or b'(none)',
                     )
                 )
             )
             if cs.branchpoints:
                 ui.writenoi18n(
                     b'Branchpoints: %s \n' % b', '.join(sorted(cs.branchpoints))
                 )
             if opts[b"parents"] and cs.parents:
                 if len(cs.parents) > 1:
                     ui.write(
                         (
                             b'Parents: %s\n'
                             % (b','.join([(b"%d" % p.id) for p in cs.parents]))
                         )
                     )
                 else:
                     ui.write((b'Parent: %d\n' % cs.parents[0].id))
 
             if opts[b"ancestors"]:
                 b = cs.branch
                 r = []
                 while b:
                     b, c = ancestors[b]
                     r.append(b'%s:%d:%d' % (b or b"HEAD", c, branches[b]))
                 if r:
                     ui.write((b'Ancestors: %s\n' % (b','.join(r))))
 
             ui.writenoi18n(b'Log:\n')
             ui.write(b'%s\n\n' % cs.comment)
             ui.writenoi18n(b'Members: \n')
             for f in cs.entries:
                 fn = f.file
                 if fn.startswith(opts[b"prefix"]):
                     fn = fn[len(opts[b"prefix"]) :]
                 ui.write(
                     b'\t%s:%s->%s%s \n'
                     % (
                         fn,
                         b'.'.join([b"%d" % x for x in f.parent]) or b'INITIAL',
                         b'.'.join([(b"%d" % x) for x in f.revision]),
                         [b'', b'(DEAD)'][f.dead],
                     )
                 )
             ui.write(b'\n')
 
         # have we seen the start tag?
         if revisions and off:
             if revisions[0] == (b"%d" % cs.id) or revisions[0] in cs.tags:
                 off = False
 
         # see if we reached the end tag
         if len(revisions) > 1 and not off:
             if revisions[1] == (b"%d" % cs.id) or revisions[1] in cs.tags:
                 break

hgext/convert/filemap.py

@@ -1,497 +1,497 b''
 # Copyright 2007 Bryan O'Sullivan <[e-mail]>
 # Copyright 2007 Alexis S. L. Carvalho <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 
 import posixpath
 
 from mercurial.i18n import _
 from mercurial import (
     error,
     pycompat,
 )
 from . import common
 
 SKIPREV = common.SKIPREV
 
 
 def rpairs(path):
     """Yield tuples with path split at '/', starting with the full path.
     No leading, trailing or double '/', please.
     >>> for x in rpairs(b'foo/bar/baz'): print(x)
     ('foo/bar/baz', '')
     ('foo/bar', 'baz')
     ('foo', 'bar/baz')
     ('.', 'foo/bar/baz')
     """
     i = len(path)
     while i != -1:
         yield path[:i], path[i + 1 :]
         i = path.rfind(b'/', 0, i)
     yield b'.', path
 
 
 def normalize(path):
     """We use posixpath.normpath to support cross-platform path format.
     However, it doesn't handle None input. So we wrap it up."""
     if path is None:
         return None
     return posixpath.normpath(path)
 
 
 class filemapper(object):
     """Map and filter filenames when importing.
     A name can be mapped to itself, a new name, or None (omit from new
     repository)."""
 
     def __init__(self, ui, path=None):
         self.ui = ui
         self.include = {}
         self.exclude = {}
         self.rename = {}
         self.targetprefixes = None
         if path:
             if self.parse(path):
                 raise error.Abort(_(b'errors in filemap'))
 
     def parse(self, path):
         errs = 0
 
         def check(name, mapping, listname):
             if not name:
                 self.ui.warn(
                     _(b'%s:%d: path to %s is missing\n')
                     % (lex.infile, lex.lineno, listname)
                 )
                 return 1
             if name in mapping:
                 self.ui.warn(
                     _(b'%s:%d: %r already in %s list\n')
                     % (lex.infile, lex.lineno, name, listname)
                 )
                 return 1
             if name.startswith(b'/') or name.endswith(b'/') or b'//' in name:
                 self.ui.warn(
                     _(b'%s:%d: superfluous / in %s %r\n')
                     % (lex.infile, lex.lineno, listname, pycompat.bytestr(name))
                 )
                 return 1
             return 0
 
         lex = common.shlexer(
             filepath=path, wordchars=b'!@#$%^&*()-=+[]{}|;:,./<>?'
         )
         cmd = lex.get_token()
         while cmd:
             if cmd == b'include':
                 name = normalize(lex.get_token())
                 errs += check(name, self.exclude, b'exclude')
                 self.include[name] = name
             elif cmd == b'exclude':
                 name = normalize(lex.get_token())
                 errs += check(name, self.include, b'include')
                 errs += check(name, self.rename, b'rename')
                 self.exclude[name] = name
             elif cmd == b'rename':
                 src = normalize(lex.get_token())
                 dest = normalize(lex.get_token())
                 errs += check(src, self.exclude, b'exclude')
                 self.rename[src] = dest
             elif cmd == b'source':
                 errs += self.parse(normalize(lex.get_token()))
             else:
                 self.ui.warn(
                     _(b'%s:%d: unknown directive %r\n')
                     % (lex.infile, lex.lineno, pycompat.bytestr(cmd))
                 )
                 errs += 1
             cmd = lex.get_token()
         return errs
 
     def lookup(self, name, mapping):
         name = normalize(name)
         for pre, suf in rpairs(name):
             try:
                 return mapping[pre], pre, suf
             except KeyError:
                 pass
         return b'', name, b''
 
     def istargetfile(self, filename):
         """Return true if the given target filename is covered as a destination
         of the filemap. This is useful for identifying what parts of the target
         repo belong to the source repo and what parts don't."""
         if self.targetprefixes is None:
             self.targetprefixes = set()
-            for before, after in pycompat.iteritems(self.rename):
+            for before, after in self.rename.items():
                 self.targetprefixes.add(after)
 
         # If "." is a target, then all target files are considered from the
         # source.
         if not self.targetprefixes or b'.' in self.targetprefixes:
             return True
 
         filename = normalize(filename)
         for pre, suf in rpairs(filename):
             # This check is imperfect since it doesn't account for the
             # include/exclude list, but it should work in filemaps that don't
             # apply include/exclude to the same source directories they are
             # renaming.
             if pre in self.targetprefixes:
                 return True
         return False
 
     def __call__(self, name):
         if self.include:
             inc = self.lookup(name, self.include)[0]
         else:
             inc = name
         if self.exclude:
             exc = self.lookup(name, self.exclude)[0]
         else:
             exc = b''
         if (not self.include and exc) or (len(inc) <= len(exc)):
             return None
         newpre, pre, suf = self.lookup(name, self.rename)
         if newpre:
             if newpre == b'.':
                 return suf
             if suf:
                 if newpre.endswith(b'/'):
                     return newpre + suf
                 return newpre + b'/' + suf
             return newpre
         return name
 
     def active(self):
         return bool(self.include or self.exclude or self.rename)
 
 
 # This class does two additional things compared to a regular source:
 #
 # - Filter and rename files.  This is mostly wrapped by the filemapper
 #   class above. We hide the original filename in the revision that is
 #   returned by getchanges to be able to find things later in getfile.
 #
 # - Return only revisions that matter for the files we're interested in.
 #   This involves rewriting the parents of the original revision to
 #   create a graph that is restricted to those revisions.
 #
 #   This set of revisions includes not only revisions that directly
 #   touch files we're interested in, but also merges that merge two
 #   or more interesting revisions.
 
 
 class filemap_source(common.converter_source):
     def __init__(self, ui, baseconverter, filemap):
         super(filemap_source, self).__init__(ui, baseconverter.repotype)
         self.base = baseconverter
         self.filemapper = filemapper(ui, filemap)
         self.commits = {}
         # if a revision rev has parent p in the original revision graph, then
         # rev will have parent self.parentmap[p] in the restricted graph.
         self.parentmap = {}
         # self.wantedancestors[rev] is the set of all ancestors of rev that
         # are in the restricted graph.
         self.wantedancestors = {}
         self.convertedorder = None
         self._rebuilt = False
         self.origparents = {}
         self.children = {}
         self.seenchildren = {}
         # experimental config: convert.ignoreancestorcheck
         self.ignoreancestorcheck = self.ui.configbool(
             b'convert', b'ignoreancestorcheck'
         )
 
     def before(self):
         self.base.before()
 
     def after(self):
         self.base.after()
 
     def setrevmap(self, revmap):
         # rebuild our state to make things restartable
         #
         # To avoid calling getcommit for every revision that has already
         # been converted, we rebuild only the parentmap, delaying the
         # rebuild of wantedancestors until we need it (i.e. until a
         # merge).
         #
         # We assume the order argument lists the revisions in
         # topological order, so that we can infer which revisions were
         # wanted by previous runs.
         self._rebuilt = not revmap
         seen = {SKIPREV: SKIPREV}
         dummyset = set()
         converted = []
         for rev in revmap.order:
             mapped = revmap[rev]
             wanted = mapped not in seen
             if wanted:
                 seen[mapped] = rev
                 self.parentmap[rev] = rev
             else:
                 self.parentmap[rev] = seen[mapped]
             self.wantedancestors[rev] = dummyset
             arg = seen[mapped]
             if arg == SKIPREV:
                 arg = None
             converted.append((rev, wanted, arg))
         self.convertedorder = converted
         return self.base.setrevmap(revmap)
 
     def rebuild(self):
         if self._rebuilt:
             return True
         self._rebuilt = True
         self.parentmap.clear()
         self.wantedancestors.clear()
         self.seenchildren.clear()
         for rev, wanted, arg in self.convertedorder:
             if rev not in self.origparents:
                 try:
                     self.origparents[rev] = self.getcommit(rev).parents
                 except error.RepoLookupError:
                     self.ui.debug(b"unknown revmap source: %s\n" % rev)
                     continue
             if arg is not None:
                 self.children[arg] = self.children.get(arg, 0) + 1
 
         for rev, wanted, arg in self.convertedorder:
             try:
                 parents = self.origparents[rev]
             except KeyError:
                 continue  # unknown revmap source
             if wanted:
                 self.mark_wanted(rev, parents)
             else:
                 self.mark_not_wanted(rev, arg)
             self._discard(arg, *parents)
 
         return True
 
     def getheads(self):
         return self.base.getheads()
 
     def getcommit(self, rev):
         # We want to save a reference to the commit objects to be able
         # to rewrite their parents later on.
         c = self.commits[rev] = self.base.getcommit(rev)
         for p in c.parents:
             self.children[p] = self.children.get(p, 0) + 1
         return c
 
     def numcommits(self):
         return self.base.numcommits()
 
     def _cachedcommit(self, rev):
         if rev in self.commits:
             return self.commits[rev]
         return self.base.getcommit(rev)
 
     def _discard(self, *revs):
         for r in revs:
             if r is None:
                 continue
             self.seenchildren[r] = self.seenchildren.get(r, 0) + 1
             if self.seenchildren[r] == self.children[r]:
                 self.wantedancestors.pop(r, None)
                 self.parentmap.pop(r, None)
                 del self.seenchildren[r]
                 if self._rebuilt:
                     del self.children[r]
 
     def wanted(self, rev, i):
         # Return True if we're directly interested in rev.
         #
         # i is an index selecting one of the parents of rev (if rev
         # has no parents, i is None).  getchangedfiles will give us
         # the list of files that are different in rev and in the parent
         # indicated by i.  If we're interested in any of these files,
         # we're interested in rev.
         try:
             files = self.base.getchangedfiles(rev, i)
         except NotImplementedError:
             raise error.Abort(_(b"source repository doesn't support --filemap"))
         for f in files:
             if self.filemapper(f):
                 return True
 
         # The include directive is documented to include nothing else (though
         # valid branch closes are included).
         if self.filemapper.include:
             return False
 
         # Allow empty commits in the source revision through.  The getchanges()
         # method doesn't even bother calling this if it determines that the
         # close marker is significant (i.e. all of the branch ancestors weren't
         # eliminated).  Therefore if there *is* a close marker, getchanges()
         # doesn't consider it significant, and this revision should be dropped.
         return not files and b'close' not in self.commits[rev].extra
 
     def mark_not_wanted(self, rev, p):
         # Mark rev as not interesting and update data structures.
 
         if p is None:
             # A root revision. Use SKIPREV to indicate that it doesn't
             # map to any revision in the restricted graph.  Put SKIPREV
             # in the set of wanted ancestors to simplify code elsewhere
             self.parentmap[rev] = SKIPREV
             self.wantedancestors[rev] = {SKIPREV}
             return
 
         # Reuse the data from our parent.
         self.parentmap[rev] = self.parentmap[p]
         self.wantedancestors[rev] = self.wantedancestors[p]
 
     def mark_wanted(self, rev, parents):
         # Mark rev ss wanted and update data structures.
 
         # rev will be in the restricted graph, so children of rev in
         # the original graph should still have rev as a parent in the
         # restricted graph.
         self.parentmap[rev] = rev
 
         # The set of wanted ancestors of rev is the union of the sets
         # of wanted ancestors of its parents. Plus rev itself.
         wrev = set()
         for p in parents:
             if p in self.wantedancestors:
                 wrev.update(self.wantedancestors[p])
             else:
                 self.ui.warn(
                     _(b'warning: %s parent %s is missing\n') % (rev, p)
                 )
         wrev.add(rev)
         self.wantedancestors[rev] = wrev
 
     def getchanges(self, rev, full):
         parents = self.commits[rev].parents
         if len(parents) > 1 and not self.ignoreancestorcheck:
             self.rebuild()
 
         # To decide whether we're interested in rev we:
         #
         # - calculate what parents rev will have if it turns out we're
         #   interested in it.  If it's going to have more than 1 parent,
         #   we're interested in it.
         #
         # - otherwise, we'll compare it with the single parent we found.
         #   If any of the files we're interested in is different in the
         #   the two revisions, we're interested in rev.
 
         # A parent p is interesting if its mapped version (self.parentmap[p]):
         # - is not SKIPREV
         # - is still not in the list of parents (we don't want duplicates)
         # - is not an ancestor of the mapped versions of the other parents or
         #   there is no parent in the same branch than the current revision.
         mparents = []
         knownparents = set()
         branch = self.commits[rev].branch
         hasbranchparent = False
         for i, p1 in enumerate(parents):
             mp1 = self.parentmap[p1]
             if mp1 == SKIPREV or mp1 in knownparents:
                 continue
 
             isancestor = not self.ignoreancestorcheck and any(
                 p2
                 for p2 in parents
                 if p1 != p2
                 and mp1 != self.parentmap[p2]
                 and mp1 in self.wantedancestors[p2]
             )
             if not isancestor and not hasbranchparent and len(parents) > 1:
                 # This could be expensive, avoid unnecessary calls.
                 if self._cachedcommit(p1).branch == branch:
                     hasbranchparent = True
             mparents.append((p1, mp1, i, isancestor))
             knownparents.add(mp1)
         # Discard parents ancestors of other parents if there is a
         # non-ancestor one on the same branch than current revision.
         if hasbranchparent:
             mparents = [p for p in mparents if not p[3]]
         wp = None
         if mparents:
             wp = max(p[2] for p in mparents)
             mparents = [p[1] for p in mparents]
         elif parents:
             wp = 0
 
         self.origparents[rev] = parents
 
         closed = False
         if b'close' in self.commits[rev].extra:
             # A branch closing revision is only useful if one of its
             # parents belong to the branch being closed
             pbranches = [self._cachedcommit(p).branch for p in mparents]
             if branch in pbranches:
                 closed = True
 
         if len(mparents) < 2 and not closed and not self.wanted(rev, wp):
             # We don't want this revision.
             # Update our state and tell the convert process to map this
             # revision to the same revision its parent as mapped to.
             p = None
             if parents:
                 p = parents[wp]
             self.mark_not_wanted(rev, p)
             self.convertedorder.append((rev, False, p))
             self._discard(*parents)
             return self.parentmap[rev]
 
         # We want this revision.
         # Rewrite the parents of the commit object
         self.commits[rev].parents = mparents
         self.mark_wanted(rev, parents)
         self.convertedorder.append((rev, True, None))
         self._discard(*parents)
 
         # Get the real changes and do the filtering/mapping. To be
         # able to get the files later on in getfile, we hide the
         # original filename in the rev part of the return value.
         changes, copies, cleanp2 = self.base.getchanges(rev, full)
         files = {}
         ncleanp2 = set(cleanp2)
         for f, r in changes:
             newf = self.filemapper(f)
             if newf and (newf != f or newf not in files):
                 files[newf] = (f, r)
                 if newf != f:
                     ncleanp2.discard(f)
         files = sorted(files.items())
 
         ncopies = {}
         for c in copies:
             newc = self.filemapper(c)
             if newc:
                 newsource = self.filemapper(copies[c])
                 if newsource:
                     ncopies[newc] = newsource
 
         return files, ncopies, ncleanp2
 
     def targetfilebelongstosource(self, targetfilename):
         return self.filemapper.istargetfile(targetfilename)
 
     def getfile(self, name, rev):
         realname, realrev = rev
         return self.base.getfile(realname, realrev)
 
     def gettags(self):
         return self.base.gettags()
 
     def hasnativeorder(self):
         return self.base.hasnativeorder()
 
     def lookuprev(self, rev):
         return self.base.lookuprev(rev)
 
     def getbookmarks(self):
         return self.base.getbookmarks()
 
     def converted(self, rev, sinkrev):
         self.base.converted(rev, sinkrev)

hgext/convert/hg.py

@@ -1,732 +1,732 b''
 # hg.py - hg backend for convert extension
 #
 #  Copyright 2005-2009 Olivia Mackall <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # Notes for hg->hg conversion:
 #
 # * Old versions of Mercurial didn't trim the whitespace from the ends
 #   of commit messages, but new versions do.  Changesets created by
 #   those older versions, then converted, may thus have different
 #   hashes for changesets that are otherwise identical.
 #
 # * Using "--config convert.hg.saverev=true" will make the source
 #   identifier to be stored in the converted revision. This will cause
 #   the converted revision to have a different identity than the
 #   source.
 
 import os
 import re
 import time
 
 from mercurial.i18n import _
 from mercurial.pycompat import open
 from mercurial.node import (
     bin,
     hex,
     sha1nodeconstants,
 )
 from mercurial import (
     bookmarks,
     context,
     error,
     exchange,
     hg,
     lock as lockmod,
     logcmdutil,
     merge as mergemod,
     mergestate,
     phases,
     pycompat,
     util,
 )
 from mercurial.utils import dateutil
 
 stringio = util.stringio
 
 from . import common
 
 mapfile = common.mapfile
 NoRepo = common.NoRepo
 
 sha1re = re.compile(br'\b[0-9a-f]{12,40}\b')
 
 
 class mercurial_sink(common.converter_sink):
     def __init__(self, ui, repotype, path):
         common.converter_sink.__init__(self, ui, repotype, path)
         self.branchnames = ui.configbool(b'convert', b'hg.usebranchnames')
         self.clonebranches = ui.configbool(b'convert', b'hg.clonebranches')
         self.tagsbranch = ui.config(b'convert', b'hg.tagsbranch')
         self.lastbranch = None
         if os.path.isdir(path) and len(os.listdir(path)) > 0:
             try:
                 self.repo = hg.repository(self.ui, path)
                 if not self.repo.local():
                     raise NoRepo(
                         _(b'%s is not a local Mercurial repository') % path
                     )
             except error.RepoError as err:
                 ui.traceback()
                 raise NoRepo(err.args[0])
         else:
             try:
                 ui.status(_(b'initializing destination %s repository\n') % path)
                 self.repo = hg.repository(self.ui, path, create=True)
                 if not self.repo.local():
                     raise NoRepo(
                         _(b'%s is not a local Mercurial repository') % path
                     )
                 self.created.append(path)
             except error.RepoError:
                 ui.traceback()
                 raise NoRepo(
                     _(b"could not create hg repository %s as sink") % path
                 )
         self.lock = None
         self.wlock = None
         self.filemapmode = False
         self.subrevmaps = {}
 
     def before(self):
         self.ui.debug(b'run hg sink pre-conversion action\n')
         self.wlock = self.repo.wlock()
         self.lock = self.repo.lock()
 
     def after(self):
         self.ui.debug(b'run hg sink post-conversion action\n')
         if self.lock:
             self.lock.release()
         if self.wlock:
             self.wlock.release()
 
     def revmapfile(self):
         return self.repo.vfs.join(b"shamap")
 
     def authorfile(self):
         return self.repo.vfs.join(b"authormap")
 
     def setbranch(self, branch, pbranches):
         if not self.clonebranches:
             return
 
         setbranch = branch != self.lastbranch
         self.lastbranch = branch
         if not branch:
             branch = b'default'
         pbranches = [(b[0], b[1] and b[1] or b'default') for b in pbranches]
 
         branchpath = os.path.join(self.path, branch)
         if setbranch:
             self.after()
             try:
                 self.repo = hg.repository(self.ui, branchpath)
             except Exception:
                 self.repo = hg.repository(self.ui, branchpath, create=True)
             self.before()
 
         # pbranches may bring revisions from other branches (merge parents)
         # Make sure we have them, or pull them.
         missings = {}
         for b in pbranches:
             try:
                 self.repo.lookup(b[0])
             except Exception:
                 missings.setdefault(b[1], []).append(b[0])
 
         if missings:
             self.after()
-            for pbranch, heads in sorted(pycompat.iteritems(missings)):
+            for pbranch, heads in sorted(missings.items()):
                 pbranchpath = os.path.join(self.path, pbranch)
                 prepo = hg.peer(self.ui, {}, pbranchpath)
                 self.ui.note(
                     _(b'pulling from %s into %s\n') % (pbranch, branch)
                 )
                 exchange.pull(
                     self.repo, prepo, heads=[prepo.lookup(h) for h in heads]
                 )
             self.before()
 
     def _rewritetags(self, source, revmap, data):
         fp = stringio()
         for line in data.splitlines():
             s = line.split(b' ', 1)
             if len(s) != 2:
                 self.ui.warn(_(b'invalid tag entry: "%s"\n') % line)
                 fp.write(b'%s\n' % line)  # Bogus, but keep for hash stability
                 continue
             revid = revmap.get(source.lookuprev(s[0]))
             if not revid:
                 if s[0] == sha1nodeconstants.nullhex:
                     revid = s[0]
                 else:
                     # missing, but keep for hash stability
                     self.ui.warn(_(b'missing tag entry: "%s"\n') % line)
                     fp.write(b'%s\n' % line)
                     continue
             fp.write(b'%s %s\n' % (revid, s[1]))
         return fp.getvalue()
 
     def _rewritesubstate(self, source, data):
         fp = stringio()
         for line in data.splitlines():
             s = line.split(b' ', 1)
             if len(s) != 2:
                 continue
 
             revid = s[0]
             subpath = s[1]
             if revid != sha1nodeconstants.nullhex:
                 revmap = self.subrevmaps.get(subpath)
                 if revmap is None:
                     revmap = mapfile(
                         self.ui, self.repo.wjoin(subpath, b'.hg/shamap')
                     )
                     self.subrevmaps[subpath] = revmap
 
                     # It is reasonable that one or more of the subrepos don't
                     # need to be converted, in which case they can be cloned
                     # into place instead of converted.  Therefore, only warn
                     # once.
                     msg = _(b'no ".hgsubstate" updates will be made for "%s"\n')
                     if len(revmap) == 0:
                         sub = self.repo.wvfs.reljoin(subpath, b'.hg')
 
                         if self.repo.wvfs.exists(sub):
                             self.ui.warn(msg % subpath)
 
                 newid = revmap.get(revid)
                 if not newid:
                     if len(revmap) > 0:
                         self.ui.warn(
                             _(b"%s is missing from %s/.hg/shamap\n")
                             % (revid, subpath)
                         )
                 else:
                     revid = newid
 
             fp.write(b'%s %s\n' % (revid, subpath))
 
         return fp.getvalue()
 
     def _calculatemergedfiles(self, source, p1ctx, p2ctx):
         """Calculates the files from p2 that we need to pull in when merging p1
         and p2, given that the merge is coming from the given source.
 
         This prevents us from losing files that only exist in the target p2 and
         that don't come from the source repo (like if you're merging multiple
         repositories together).
         """
         anc = [p1ctx.ancestor(p2ctx)]
         # Calculate what files are coming from p2
         # TODO: mresult.commitinfo might be able to get that info
         mresult = mergemod.calculateupdates(
             self.repo,
             p1ctx,
             p2ctx,
             anc,
             branchmerge=True,
             force=True,
             acceptremote=False,
             followcopies=False,
         )
 
         for file, (action, info, msg) in mresult.filemap():
             if source.targetfilebelongstosource(file):
                 # If the file belongs to the source repo, ignore the p2
                 # since it will be covered by the existing fileset.
                 continue
 
             # If the file requires actual merging, abort. We don't have enough
             # context to resolve merges correctly.
             if action in mergestate.CONVERT_MERGE_ACTIONS:
                 raise error.Abort(
                     _(
                         b"unable to convert merge commit "
                         b"since target parents do not merge cleanly (file "
                         b"%s, parents %s and %s)"
                     )
                     % (file, p1ctx, p2ctx)
                 )
             elif action == mergestate.ACTION_KEEP:
                 # 'keep' means nothing changed from p1
                 continue
             else:
                 # Any other change means we want to take the p2 version
                 yield file
 
     def putcommit(
         self, files, copies, parents, commit, source, revmap, full, cleanp2
     ):
         files = dict(files)
 
         def getfilectx(repo, memctx, f):
             if p2ctx and f in p2files and f not in copies:
                 self.ui.debug(b'reusing %s from p2\n' % f)
                 try:
                     return p2ctx[f]
                 except error.ManifestLookupError:
                     # If the file doesn't exist in p2, then we're syncing a
                     # delete, so just return None.
                     return None
             try:
                 v = files[f]
             except KeyError:
                 return None
             data, mode = source.getfile(f, v)
             if data is None:
                 return None
             if f == b'.hgtags':
                 data = self._rewritetags(source, revmap, data)
             if f == b'.hgsubstate':
                 data = self._rewritesubstate(source, data)
             return context.memfilectx(
                 self.repo,
                 memctx,
                 f,
                 data,
                 b'l' in mode,
                 b'x' in mode,
                 copies.get(f),
             )
 
         pl = []
         for p in parents:
             if p not in pl:
                 pl.append(p)
         parents = pl
         nparents = len(parents)
         if self.filemapmode and nparents == 1:
             m1node = self.repo.changelog.read(bin(parents[0]))[0]
             parent = parents[0]
 
         if len(parents) < 2:
             parents.append(self.repo.nullid)
         if len(parents) < 2:
             parents.append(self.repo.nullid)
         p2 = parents.pop(0)
 
         text = commit.desc
 
         sha1s = re.findall(sha1re, text)
         for sha1 in sha1s:
             oldrev = source.lookuprev(sha1)
             newrev = revmap.get(oldrev)
             if newrev is not None:
                 text = text.replace(sha1, newrev[: len(sha1)])
 
         extra = commit.extra.copy()
 
         sourcename = self.repo.ui.config(b'convert', b'hg.sourcename')
         if sourcename:
             extra[b'convert_source'] = sourcename
 
         for label in (
             b'source',
             b'transplant_source',
             b'rebase_source',
             b'intermediate-source',
         ):
             node = extra.get(label)
 
             if node is None:
                 continue
 
             # Only transplant stores its reference in binary
             if label == b'transplant_source':
                 node = hex(node)
 
             newrev = revmap.get(node)
             if newrev is not None:
                 if label == b'transplant_source':
                     newrev = bin(newrev)
 
                 extra[label] = newrev
 
         if self.branchnames and commit.branch:
             extra[b'branch'] = commit.branch
         if commit.rev and commit.saverev:
             extra[b'convert_revision'] = commit.rev
 
         while parents:
             p1 = p2
             p2 = parents.pop(0)
             p1ctx = self.repo[p1]
             p2ctx = None
             if p2 != self.repo.nullid:
                 p2ctx = self.repo[p2]
             fileset = set(files)
             if full:
                 fileset.update(self.repo[p1])
                 fileset.update(self.repo[p2])
 
             if p2ctx:
                 p2files = set(cleanp2)
                 for file in self._calculatemergedfiles(source, p1ctx, p2ctx):
                     p2files.add(file)
                     fileset.add(file)
 
             ctx = context.memctx(
                 self.repo,
                 (p1, p2),
                 text,
                 fileset,
                 getfilectx,
                 commit.author,
                 commit.date,
                 extra,
             )
 
             # We won't know if the conversion changes the node until after the
             # commit, so copy the source's phase for now.
             self.repo.ui.setconfig(
                 b'phases',
                 b'new-commit',
                 phases.phasenames[commit.phase],
                 b'convert',
             )
 
             with self.repo.transaction(b"convert") as tr:
                 if self.repo.ui.config(b'convert', b'hg.preserve-hash'):
                     origctx = commit.ctx
                 else:
                     origctx = None
                 node = hex(self.repo.commitctx(ctx, origctx=origctx))
 
                 # If the node value has changed, but the phase is lower than
                 # draft, set it back to draft since it hasn't been exposed
                 # anywhere.
                 if commit.rev != node:
                     ctx = self.repo[node]
                     if ctx.phase() < phases.draft:
                         phases.registernew(
                             self.repo, tr, phases.draft, [ctx.rev()]
                         )
 
             text = b"(octopus merge fixup)\n"
             p2 = node
 
         if self.filemapmode and nparents == 1:
             man = self.repo.manifestlog.getstorage(b'')
             mnode = self.repo.changelog.read(bin(p2))[0]
             closed = b'close' in commit.extra
             if not closed and not man.cmp(m1node, man.revision(mnode)):
                 self.ui.status(_(b"filtering out empty revision\n"))
                 self.repo.rollback(force=True)
                 return parent
         return p2
 
     def puttags(self, tags):
         tagparent = self.repo.branchtip(self.tagsbranch, ignoremissing=True)
         tagparent = tagparent or self.repo.nullid
 
         oldlines = set()
         for branch, heads in pycompat.iteritems(self.repo.branchmap()):
             for h in heads:
                 if b'.hgtags' in self.repo[h]:
                     oldlines.update(
                         set(self.repo[h][b'.hgtags'].data().splitlines(True))
                     )
         oldlines = sorted(list(oldlines))
 
         newlines = sorted([(b"%s %s\n" % (tags[tag], tag)) for tag in tags])
         if newlines == oldlines:
             return None, None
 
         # if the old and new tags match, then there is nothing to update
         oldtags = set()
         newtags = set()
         for line in oldlines:
             s = line.strip().split(b' ', 1)
             if len(s) != 2:
                 continue
             oldtags.add(s[1])
         for line in newlines:
             s = line.strip().split(b' ', 1)
             if len(s) != 2:
                 continue
             if s[1] not in oldtags:
                 newtags.add(s[1].strip())
 
         if not newtags:
             return None, None
 
         data = b"".join(newlines)
 
         def getfilectx(repo, memctx, f):
             return context.memfilectx(repo, memctx, f, data, False, False, None)
 
         self.ui.status(_(b"updating tags\n"))
         date = b"%d 0" % int(time.mktime(time.gmtime()))
         extra = {b'branch': self.tagsbranch}
         ctx = context.memctx(
             self.repo,
             (tagparent, None),
             b"update tags",
             [b".hgtags"],
             getfilectx,
             b"convert-repo",
             date,
             extra,
         )
         node = self.repo.commitctx(ctx)
         return hex(node), hex(tagparent)
 
     def setfilemapmode(self, active):
         self.filemapmode = active
 
     def putbookmarks(self, updatedbookmark):
         if not len(updatedbookmark):
             return
         wlock = lock = tr = None
         try:
             wlock = self.repo.wlock()
             lock = self.repo.lock()
             tr = self.repo.transaction(b'bookmark')
             self.ui.status(_(b"updating bookmarks\n"))
             destmarks = self.repo._bookmarks
             changes = [
                 (bookmark, bin(updatedbookmark[bookmark]))
                 for bookmark in updatedbookmark
             ]
             destmarks.applychanges(self.repo, tr, changes)
             tr.close()
         finally:
             lockmod.release(lock, wlock, tr)
 
     def hascommitfrommap(self, rev):
         # the exact semantics of clonebranches is unclear so we can't say no
         return rev in self.repo or self.clonebranches
 
     def hascommitforsplicemap(self, rev):
         if rev not in self.repo and self.clonebranches:
             raise error.Abort(
                 _(
                     b'revision %s not found in destination '
                     b'repository (lookups with clonebranches=true '
                     b'are not implemented)'
                 )
                 % rev
             )
         return rev in self.repo
 
 
 class mercurial_source(common.converter_source):
     def __init__(self, ui, repotype, path, revs=None):
         common.converter_source.__init__(self, ui, repotype, path, revs)
         self.ignoreerrors = ui.configbool(b'convert', b'hg.ignoreerrors')
         self.ignored = set()
         self.saverev = ui.configbool(b'convert', b'hg.saverev')
         try:
             self.repo = hg.repository(self.ui, path)
             # try to provoke an exception if this isn't really a hg
             # repo, but some other bogus compatible-looking url
             if not self.repo.local():
                 raise error.RepoError
         except error.RepoError:
             ui.traceback()
             raise NoRepo(_(b"%s is not a local Mercurial repository") % path)
         self.lastrev = None
         self.lastctx = None
         self._changescache = None, None
         self.convertfp = None
         # Restrict converted revisions to startrev descendants
         startnode = ui.config(b'convert', b'hg.startrev')
         hgrevs = ui.config(b'convert', b'hg.revs')
         if hgrevs is None:
             if startnode is not None:
                 try:
                     startnode = self.repo.lookup(startnode)
                 except error.RepoError:
                     raise error.Abort(
                         _(b'%s is not a valid start revision') % startnode
                     )
                 startrev = self.repo.changelog.rev(startnode)
                 children = {startnode: 1}
                 for r in self.repo.changelog.descendants([startrev]):
                     children[self.repo.changelog.node(r)] = 1
                 self.keep = children.__contains__
             else:
                 self.keep = util.always
             if revs:
                 self._heads = [self.repo.lookup(r) for r in revs]
             else:
                 self._heads = self.repo.heads()
         else:
             if revs or startnode is not None:
                 raise error.Abort(
                     _(
                         b'hg.revs cannot be combined with '
                         b'hg.startrev or --rev'
                     )
                 )
             nodes = set()
             parents = set()
             for r in logcmdutil.revrange(self.repo, [hgrevs]):
                 ctx = self.repo[r]
                 nodes.add(ctx.node())
                 parents.update(p.node() for p in ctx.parents())
             self.keep = nodes.__contains__
             self._heads = nodes - parents
 
     def _changectx(self, rev):
         if self.lastrev != rev:
             self.lastctx = self.repo[rev]
             self.lastrev = rev
         return self.lastctx
 
     def _parents(self, ctx):
         return [p for p in ctx.parents() if p and self.keep(p.node())]
 
     def getheads(self):
         return [hex(h) for h in self._heads if self.keep(h)]
 
     def getfile(self, name, rev):
         try:
             fctx = self._changectx(rev)[name]
             return fctx.data(), fctx.flags()
         except error.LookupError:
             return None, None
 
     def _changedfiles(self, ctx1, ctx2):
         ma, r = [], []
         maappend = ma.append
         rappend = r.append
         d = ctx1.manifest().diff(ctx2.manifest())
-        for f, ((node1, flag1), (node2, flag2)) in pycompat.iteritems(d):
+        for f, ((node1, flag1), (node2, flag2)) in d.items():
             if node2 is None:
                 rappend(f)
             else:
                 maappend(f)
         return ma, r
 
     def getchanges(self, rev, full):
         ctx = self._changectx(rev)
         parents = self._parents(ctx)
         if full or not parents:
             files = copyfiles = ctx.manifest()
         if parents:
             if self._changescache[0] == rev:
                 ma, r = self._changescache[1]
             else:
                 ma, r = self._changedfiles(parents[0], ctx)
             if not full:
                 files = ma + r
             copyfiles = ma
         # _getcopies() is also run for roots and before filtering so missing
         # revlogs are detected early
         copies = self._getcopies(ctx, parents, copyfiles)
         cleanp2 = set()
         if len(parents) == 2:
             d = parents[1].manifest().diff(ctx.manifest(), clean=True)
-            for f, value in pycompat.iteritems(d):
+            for f, value in d.items():
                 if value is None:
                     cleanp2.add(f)
         changes = [(f, rev) for f in files if f not in self.ignored]
         changes.sort()
         return changes, copies, cleanp2
 
     def _getcopies(self, ctx, parents, files):
         copies = {}
         for name in files:
             if name in self.ignored:
                 continue
             try:
                 copysource = ctx.filectx(name).copysource()
                 if copysource in self.ignored:
                     continue
                 # Ignore copy sources not in parent revisions
                 if not any(copysource in p for p in parents):
                     continue
                 copies[name] = copysource
             except TypeError:
                 pass
             except error.LookupError as e:
                 if not self.ignoreerrors:
                     raise
                 self.ignored.add(name)
                 self.ui.warn(_(b'ignoring: %s\n') % e)
         return copies
 
     def getcommit(self, rev):
         ctx = self._changectx(rev)
         _parents = self._parents(ctx)
         parents = [p.hex() for p in _parents]
         optparents = [p.hex() for p in ctx.parents() if p and p not in _parents]
         crev = rev
 
         return common.commit(
             author=ctx.user(),
             date=dateutil.datestr(ctx.date(), b'%Y-%m-%d %H:%M:%S %1%2'),
             desc=ctx.description(),
             rev=crev,
             parents=parents,
             optparents=optparents,
             branch=ctx.branch(),
             extra=ctx.extra(),
             sortkey=ctx.rev(),
             saverev=self.saverev,
             phase=ctx.phase(),
             ctx=ctx,
         )
 
     def numcommits(self):
         return len(self.repo)
 
     def gettags(self):
         # This will get written to .hgtags, filter non global tags out.
         tags = [
             t
             for t in self.repo.tagslist()
             if self.repo.tagtype(t[0]) == b'global'
         ]
         return {name: hex(node) for name, node in tags if self.keep(node)}
 
     def getchangedfiles(self, rev, i):
         ctx = self._changectx(rev)
         parents = self._parents(ctx)
         if not parents and i is None:
             i = 0
             ma, r = ctx.manifest().keys(), []
         else:
             i = i or 0
             ma, r = self._changedfiles(parents[i], ctx)
         ma, r = [[f for f in l if f not in self.ignored] for l in (ma, r)]
 
         if i == 0:
             self._changescache = (rev, (ma, r))
 
         return ma + r
 
     def converted(self, rev, destrev):
         if self.convertfp is None:
             self.convertfp = open(self.repo.vfs.join(b'shamap'), b'ab')
         self.convertfp.write(util.tonativeeol(b'%s %s\n' % (destrev, rev)))
         self.convertfp.flush()
 
     def before(self):
         self.ui.debug(b'run hg source pre-conversion action\n')
 
     def after(self):
         self.ui.debug(b'run hg source post-conversion action\n')
 
     def hasnativeorder(self):
         return True
 
     def hasnativeclose(self):
         return True
 
     def lookuprev(self, rev):
         try:
             return hex(self.repo.lookup(rev))
         except (error.RepoError, error.LookupError):
             return None
 
     def getbookmarks(self):
         return bookmarks.listbookmarks(self.repo)
 
     def checkrevformat(self, revstr, mapname=b'splicemap'):
         """Mercurial, revision string is a 40 byte hex"""
         self.checkhexformat(revstr, mapname)

hgext/convert/monotone.py

@@ -1,410 +1,410 b''
 # monotone.py - monotone support for the convert extension
 #
 #  Copyright 2008, 2009 Mikkel Fahnoe Jorgensen <[e-mail]> and
 #  others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 import os
 import re
 
 from mercurial.i18n import _
 from mercurial.pycompat import open
 from mercurial import (
     error,
     pycompat,
 )
 from mercurial.utils import dateutil
 
 from . import common
 
 
 class monotone_source(common.converter_source, common.commandline):
     def __init__(self, ui, repotype, path=None, revs=None):
         common.converter_source.__init__(self, ui, repotype, path, revs)
         if revs and len(revs) > 1:
             raise error.Abort(
                 _(
                     b'monotone source does not support specifying '
                     b'multiple revs'
                 )
             )
         common.commandline.__init__(self, ui, b'mtn')
 
         self.ui = ui
         self.path = path
         self.automatestdio = False
         self.revs = revs
 
         norepo = common.NoRepo(
             _(b"%s does not look like a monotone repository") % path
         )
         if not os.path.exists(os.path.join(path, b'_MTN')):
             # Could be a monotone repository (SQLite db file)
             try:
                 f = open(path, b'rb')
                 header = f.read(16)
                 f.close()
             except IOError:
                 header = b''
             if header != b'SQLite format 3\x00':
                 raise norepo
 
         # regular expressions for parsing monotone output
         space = br'\s*'
         name = br'\s+"((?:\\"|[^"])*)"\s*'
         value = name
         revision = br'\s+\[(\w+)\]\s*'
         lines = br'(?:.|\n)+'
 
         self.dir_re = re.compile(space + b"dir" + name)
         self.file_re = re.compile(
             space + b"file" + name + b"content" + revision
         )
         self.add_file_re = re.compile(
             space + b"add_file" + name + b"content" + revision
         )
         self.patch_re = re.compile(
             space + b"patch" + name + b"from" + revision + b"to" + revision
         )
         self.rename_re = re.compile(space + b"rename" + name + b"to" + name)
         self.delete_re = re.compile(space + b"delete" + name)
         self.tag_re = re.compile(space + b"tag" + name + b"revision" + revision)
         self.cert_re = re.compile(
             lines + space + b"name" + name + b"value" + value
         )
 
         attr = space + b"file" + lines + space + b"attr" + space
         self.attr_execute_re = re.compile(
             attr + b'"mtn:execute"' + space + b'"true"'
         )
 
         # cached data
         self.manifest_rev = None
         self.manifest = None
         self.files = None
         self.dirs = None
 
         common.checktool(b'mtn', abort=False)
 
     def mtnrun(self, *args, **kwargs):
         if self.automatestdio:
             return self.mtnrunstdio(*args, **kwargs)
         else:
             return self.mtnrunsingle(*args, **kwargs)
 
     def mtnrunsingle(self, *args, **kwargs):
         kwargs['d'] = self.path
         return self.run0(b'automate', *args, **kwargs)
 
     def mtnrunstdio(self, *args, **kwargs):
         # Prepare the command in automate stdio format
         kwargs = pycompat.byteskwargs(kwargs)
         command = []
-        for k, v in pycompat.iteritems(kwargs):
+        for k, v in kwargs.items():
             command.append(b"%d:%s" % (len(k), k))
             if v:
                 command.append(b"%d:%s" % (len(v), v))
         if command:
             command.insert(0, b'o')
             command.append(b'e')
 
         command.append(b'l')
         for arg in args:
             command.append(b"%d:%s" % (len(arg), arg))
         command.append(b'e')
         command = b''.join(command)
 
         self.ui.debug(b"mtn: sending '%s'\n" % command)
         self.mtnwritefp.write(command)
         self.mtnwritefp.flush()
 
         return self.mtnstdioreadcommandoutput(command)
 
     def mtnstdioreadpacket(self):
         read = None
         commandnbr = b''
         while read != b':':
             read = self.mtnreadfp.read(1)
             if not read:
                 raise error.Abort(_(b'bad mtn packet - no end of commandnbr'))
             commandnbr += read
         commandnbr = commandnbr[:-1]
 
         stream = self.mtnreadfp.read(1)
         if stream not in b'mewptl':
             raise error.Abort(
                 _(b'bad mtn packet - bad stream type %s') % stream
             )
 
         read = self.mtnreadfp.read(1)
         if read != b':':
             raise error.Abort(_(b'bad mtn packet - no divider before size'))
 
         read = None
         lengthstr = b''
         while read != b':':
             read = self.mtnreadfp.read(1)
             if not read:
                 raise error.Abort(_(b'bad mtn packet - no end of packet size'))
             lengthstr += read
         try:
             length = pycompat.long(lengthstr[:-1])
         except TypeError:
             raise error.Abort(
                 _(b'bad mtn packet - bad packet size %s') % lengthstr
             )
 
         read = self.mtnreadfp.read(length)
         if len(read) != length:
             raise error.Abort(
                 _(
                     b"bad mtn packet - unable to read full packet "
                     b"read %s of %s"
                 )
                 % (len(read), length)
             )
 
         return (commandnbr, stream, length, read)
 
     def mtnstdioreadcommandoutput(self, command):
         retval = []
         while True:
             commandnbr, stream, length, output = self.mtnstdioreadpacket()
             self.ui.debug(
                 b'mtn: read packet %s:%s:%d\n' % (commandnbr, stream, length)
             )
 
             if stream == b'l':
                 # End of command
                 if output != b'0':
                     raise error.Abort(
                         _(b"mtn command '%s' returned %s") % (command, output)
                     )
                 break
             elif stream in b'ew':
                 # Error, warning output
                 self.ui.warn(_(b'%s error:\n') % self.command)
                 self.ui.warn(output)
             elif stream == b'p':
                 # Progress messages
                 self.ui.debug(b'mtn: ' + output)
             elif stream == b'm':
                 # Main stream - command output
                 retval.append(output)
 
         return b''.join(retval)
 
     def mtnloadmanifest(self, rev):
         if self.manifest_rev == rev:
             return
         self.manifest = self.mtnrun(b"get_manifest_of", rev).split(b"\n\n")
         self.manifest_rev = rev
         self.files = {}
         self.dirs = {}
 
         for e in self.manifest:
             m = self.file_re.match(e)
             if m:
                 attr = b""
                 name = m.group(1)
                 node = m.group(2)
                 if self.attr_execute_re.match(e):
                     attr += b"x"
                 self.files[name] = (node, attr)
             m = self.dir_re.match(e)
             if m:
                 self.dirs[m.group(1)] = True
 
     def mtnisfile(self, name, rev):
         # a non-file could be a directory or a deleted or renamed file
         self.mtnloadmanifest(rev)
         return name in self.files
 
     def mtnisdir(self, name, rev):
         self.mtnloadmanifest(rev)
         return name in self.dirs
 
     def mtngetcerts(self, rev):
         certs = {
             b"author": b"<missing>",
             b"date": b"<missing>",
             b"changelog": b"<missing>",
             b"branch": b"<missing>",
         }
         certlist = self.mtnrun(b"certs", rev)
         # mtn < 0.45:
         #   key "[e-mail]"
         # mtn >= 0.45:
         #   key [ff58a7ffb771907c4ff68995eada1c4da068d328]
         certlist = re.split(br'\n\n {6}key ["\[]', certlist)
         for e in certlist:
             m = self.cert_re.match(e)
             if m:
                 name, value = m.groups()
                 value = value.replace(br'\"', b'"')
                 value = value.replace(br'\\', b'\\')
                 certs[name] = value
         # Monotone may have subsecond dates: 2005-02-05T09:39:12.364306
         # and all times are stored in UTC
         certs[b"date"] = certs[b"date"].split(b'.')[0] + b" UTC"
         return certs
 
     # implement the converter_source interface:
 
     def getheads(self):
         if not self.revs:
             return self.mtnrun(b"leaves").splitlines()
         else:
             return self.revs
 
     def getchanges(self, rev, full):
         if full:
             raise error.Abort(
                 _(b"convert from monotone does not support --full")
             )
         revision = self.mtnrun(b"get_revision", rev).split(b"\n\n")
         files = {}
         ignoremove = {}
         renameddirs = []
         copies = {}
         for e in revision:
             m = self.add_file_re.match(e)
             if m:
                 files[m.group(1)] = rev
                 ignoremove[m.group(1)] = rev
             m = self.patch_re.match(e)
             if m:
                 files[m.group(1)] = rev
             # Delete/rename is handled later when the convert engine
             # discovers an IOError exception from getfile,
             # but only if we add the "from" file to the list of changes.
             m = self.delete_re.match(e)
             if m:
                 files[m.group(1)] = rev
             m = self.rename_re.match(e)
             if m:
                 toname = m.group(2)
                 fromname = m.group(1)
                 if self.mtnisfile(toname, rev):
                     ignoremove[toname] = 1
                     copies[toname] = fromname
                     files[toname] = rev
                     files[fromname] = rev
                 elif self.mtnisdir(toname, rev):
                     renameddirs.append((fromname, toname))
 
         # Directory renames can be handled only once we have recorded
         # all new files
         for fromdir, todir in renameddirs:
             renamed = {}
             for tofile in self.files:
                 if tofile in ignoremove:
                     continue
                 if tofile.startswith(todir + b'/'):
                     renamed[tofile] = fromdir + tofile[len(todir) :]
                     # Avoid chained moves like:
                     # d1(/a) => d3/d1(/a)
                     # d2 => d3
                     ignoremove[tofile] = 1
             for tofile, fromfile in renamed.items():
                 self.ui.debug(
                     b"copying file in renamed directory from '%s' to '%s'"
                     % (fromfile, tofile),
                     b'\n',
                 )
                 files[tofile] = rev
                 copies[tofile] = fromfile
             for fromfile in renamed.values():
                 files[fromfile] = rev
 
         return (files.items(), copies, set())
 
     def getfile(self, name, rev):
         if not self.mtnisfile(name, rev):
             return None, None
         try:
             data = self.mtnrun(b"get_file_of", name, r=rev)
         except Exception:
             return None, None
         self.mtnloadmanifest(rev)
         node, attr = self.files.get(name, (None, b""))
         return data, attr
 
     def getcommit(self, rev):
         extra = {}
         certs = self.mtngetcerts(rev)
         if certs.get(b'suspend') == certs[b"branch"]:
             extra[b'close'] = b'1'
         dateformat = b"%Y-%m-%dT%H:%M:%S"
         return common.commit(
             author=certs[b"author"],
             date=dateutil.datestr(dateutil.strdate(certs[b"date"], dateformat)),
             desc=certs[b"changelog"],
             rev=rev,
             parents=self.mtnrun(b"parents", rev).splitlines(),
             branch=certs[b"branch"],
             extra=extra,
         )
 
     def gettags(self):
         tags = {}
         for e in self.mtnrun(b"tags").split(b"\n\n"):
             m = self.tag_re.match(e)
             if m:
                 tags[m.group(1)] = m.group(2)
         return tags
 
     def getchangedfiles(self, rev, i):
         # This function is only needed to support --filemap
         # ... and we don't support that
         raise NotImplementedError
 
     def before(self):
         # Check if we have a new enough version to use automate stdio
         try:
             versionstr = self.mtnrunsingle(b"interface_version")
             version = float(versionstr)
         except Exception:
             raise error.Abort(
                 _(b"unable to determine mtn automate interface version")
             )
 
         if version >= 12.0:
             self.automatestdio = True
             self.ui.debug(
                 b"mtn automate version %f - using automate stdio\n" % version
             )
 
             # launch the long-running automate stdio process
             self.mtnwritefp, self.mtnreadfp = self._run2(
                 b'automate', b'stdio', b'-d', self.path
             )
             # read the headers
             read = self.mtnreadfp.readline()
             if read != b'format-version: 2\n':
                 raise error.Abort(
                     _(b'mtn automate stdio header unexpected: %s') % read
                 )
             while read != b'\n':
                 read = self.mtnreadfp.readline()
                 if not read:
                     raise error.Abort(
                         _(
                             b"failed to reach end of mtn automate "
                             b"stdio headers"
                         )
                     )
         else:
             self.ui.debug(
                 b"mtn automate version %s - not using automate stdio "
                 b"(automate >= 12.0 - mtn >= 0.46 is needed)\n" % version
             )
 
     def after(self):
         if self.automatestdio:
             self.mtnwritefp.close()
             self.mtnwritefp = None
             self.mtnreadfp.close()
             self.mtnreadfp = None

hgext/convert/subversion.py

@@ -1,1740 +1,1740 b''
 # Subversion 1.4/1.5 Python API backend
 #
 # Copyright(C) 2007 Daniel Holth et al
 
 import codecs
 import locale
 import os
 import pickle
 import re
 import xml.dom.minidom
 
 from mercurial.i18n import _
 from mercurial.pycompat import open
 from mercurial import (
     encoding,
     error,
     pycompat,
     util,
     vfs as vfsmod,
 )
 from mercurial.utils import (
     dateutil,
     procutil,
     stringutil,
 )
 
 from . import common
 
 stringio = util.stringio
 propertycache = util.propertycache
 urlerr = util.urlerr
 urlreq = util.urlreq
 
 commandline = common.commandline
 commit = common.commit
 converter_sink = common.converter_sink
 converter_source = common.converter_source
 decodeargs = common.decodeargs
 encodeargs = common.encodeargs
 makedatetimestamp = common.makedatetimestamp
 mapfile = common.mapfile
 MissingTool = common.MissingTool
 NoRepo = common.NoRepo
 
 # Subversion stuff. Works best with very recent Python SVN bindings
 # e.g. SVN 1.5 or backports. Thanks to the bzr folks for enhancing
 # these bindings.
 
 try:
     import svn
     import svn.client
     import svn.core
     import svn.ra
     import svn.delta
     from . import transport
     import warnings
 
     warnings.filterwarnings(
         'ignore', module='svn.core', category=DeprecationWarning
     )
     svn.core.SubversionException  # trigger import to catch error
 
 except ImportError:
     svn = None
 
 
 # In Subversion, paths and URLs are Unicode (encoded as UTF-8), which
 # Subversion converts from / to native strings when interfacing with the OS.
 # When passing paths and URLs to Subversion, we have to recode them such that
 # it roundstrips with what Subversion is doing.
 
 fsencoding = None
 
 
 def init_fsencoding():
     global fsencoding, fsencoding_is_utf8
     if fsencoding is not None:
         return
     if pycompat.iswindows:
         # On Windows, filenames are Unicode, but we store them using the MBCS
         # encoding.
         fsencoding = 'mbcs'
     else:
         # This is the encoding used to convert UTF-8 back to natively-encoded
         # strings in Subversion 1.14.0 or earlier with APR 1.7.0 or earlier.
         with util.with_lc_ctype():
             fsencoding = locale.nl_langinfo(locale.CODESET) or 'ISO-8859-1'
     fsencoding = codecs.lookup(fsencoding).name
     fsencoding_is_utf8 = fsencoding == codecs.lookup('utf-8').name
 
 
 def fs2svn(s):
     if fsencoding_is_utf8:
         return s
     else:
         return s.decode(fsencoding).encode('utf-8')
 
 
 def formatsvndate(date):
     return dateutil.datestr(date, b'%Y-%m-%dT%H:%M:%S.000000Z')
 
 
 def parsesvndate(s):
     # Example SVN datetime. Includes microseconds.
     # ISO-8601 conformant
     # '2007-01-04T17:35:00.902377Z'
     return dateutil.parsedate(s[:19] + b' UTC', [b'%Y-%m-%dT%H:%M:%S'])
 
 
 class SvnPathNotFound(Exception):
     pass
 
 
 def revsplit(rev):
     """Parse a revision string and return (uuid, path, revnum).
     >>> revsplit(b'svn:a2147622-4a9f-4db4-a8d3-13562ff547b2'
     ...          b'/proj%20B/mytrunk/mytrunk@1')
     ('a2147622-4a9f-4db4-a8d3-13562ff547b2', '/proj%20B/mytrunk/mytrunk', 1)
     >>> revsplit(b'svn:8af66a51-67f5-4354-b62c-98d67cc7be1d@1')
     ('', '', 1)
     >>> revsplit(b'@7')
     ('', '', 7)
     >>> revsplit(b'7')
     ('', '', 0)
     >>> revsplit(b'bad')
     ('', '', 0)
     """
     parts = rev.rsplit(b'@', 1)
     revnum = 0
     if len(parts) > 1:
         revnum = int(parts[1])
     parts = parts[0].split(b'/', 1)
     uuid = b''
     mod = b''
     if len(parts) > 1 and parts[0].startswith(b'svn:'):
         uuid = parts[0][4:]
         mod = b'/' + parts[1]
     return uuid, mod, revnum
 
 
 def quote(s):
     # As of svn 1.7, many svn calls expect "canonical" paths. In
     # theory, we should call svn.core.*canonicalize() on all paths
     # before passing them to the API.  Instead, we assume the base url
     # is canonical and copy the behaviour of svn URL encoding function
     # so we can extend it safely with new components. The "safe"
     # characters were taken from the "svn_uri__char_validity" table in
     # libsvn_subr/path.c.
     return urlreq.quote(s, b"!$&'()*+,-./:=@_~")
 
 
 def geturl(path):
     """Convert path or URL to a SVN URL, encoded in UTF-8.
 
     This can raise UnicodeDecodeError if the path or URL can't be converted to
     unicode using `fsencoding`.
     """
     try:
         return svn.client.url_from_path(
             svn.core.svn_path_canonicalize(fs2svn(path))
         )
     except svn.core.SubversionException:
         # svn.client.url_from_path() fails with local repositories
         pass
     if os.path.isdir(path):
         path = os.path.normpath(util.abspath(path))
         if pycompat.iswindows:
             path = b'/' + util.normpath(path)
         # Module URL is later compared with the repository URL returned
         # by svn API, which is UTF-8.
         path = fs2svn(path)
         path = b'file://%s' % quote(path)
     return svn.core.svn_path_canonicalize(path)
 
 
 def optrev(number):
     optrev = svn.core.svn_opt_revision_t()
     optrev.kind = svn.core.svn_opt_revision_number
     optrev.value.number = number
     return optrev
 
 
 class changedpath(object):
     def __init__(self, p):
         self.copyfrom_path = p.copyfrom_path
         self.copyfrom_rev = p.copyfrom_rev
         self.action = p.action
 
 
 def get_log_child(
     fp,
     url,
     paths,
     start,
     end,
     limit=0,
     discover_changed_paths=True,
     strict_node_history=False,
 ):
     protocol = -1
 
     def receiver(orig_paths, revnum, author, date, message, pool):
         paths = {}
         if orig_paths is not None:
-            for k, v in pycompat.iteritems(orig_paths):
+            for k, v in orig_paths.items():
                 paths[k] = changedpath(v)
         pickle.dump((paths, revnum, author, date, message), fp, protocol)
 
     try:
         # Use an ra of our own so that our parent can consume
         # our results without confusing the server.
         t = transport.SvnRaTransport(url=url)
         svn.ra.get_log(
             t.ra,
             paths,
             start,
             end,
             limit,
             discover_changed_paths,
             strict_node_history,
             receiver,
         )
     except IOError:
         # Caller may interrupt the iteration
         pickle.dump(None, fp, protocol)
     except Exception as inst:
         pickle.dump(stringutil.forcebytestr(inst), fp, protocol)
     else:
         pickle.dump(None, fp, protocol)
     fp.flush()
     # With large history, cleanup process goes crazy and suddenly
     # consumes *huge* amount of memory. The output file being closed,
     # there is no need for clean termination.
     os._exit(0)
 
 
 def debugsvnlog(ui, **opts):
     """Fetch SVN log in a subprocess and channel them back to parent to
     avoid memory collection issues.
     """
     with util.with_lc_ctype():
         if svn is None:
             raise error.Abort(
                 _(b'debugsvnlog could not load Subversion python bindings')
             )
 
         args = decodeargs(ui.fin.read())
         get_log_child(ui.fout, *args)
 
 
 class logstream(object):
     """Interruptible revision log iterator."""
 
     def __init__(self, stdout):
         self._stdout = stdout
 
     def __iter__(self):
         while True:
             try:
                 entry = pickle.load(self._stdout)
             except EOFError:
                 raise error.Abort(
                     _(
                         b'Mercurial failed to run itself, check'
                         b' hg executable is in PATH'
                     )
                 )
             try:
                 orig_paths, revnum, author, date, message = entry
             except (TypeError, ValueError):
                 if entry is None:
                     break
                 raise error.Abort(_(b"log stream exception '%s'") % entry)
             yield entry
 
     def close(self):
         if self._stdout:
             self._stdout.close()
             self._stdout = None
 
 
 class directlogstream(list):
     """Direct revision log iterator.
     This can be used for debugging and development but it will probably leak
     memory and is not suitable for real conversions."""
 
     def __init__(
         self,
         url,
         paths,
         start,
         end,
         limit=0,
         discover_changed_paths=True,
         strict_node_history=False,
     ):
         def receiver(orig_paths, revnum, author, date, message, pool):
             paths = {}
             if orig_paths is not None:
-                for k, v in pycompat.iteritems(orig_paths):
+                for k, v in orig_paths.items():
                     paths[k] = changedpath(v)
             self.append((paths, revnum, author, date, message))
 
         # Use an ra of our own so that our parent can consume
         # our results without confusing the server.
         t = transport.SvnRaTransport(url=url)
         svn.ra.get_log(
             t.ra,
             paths,
             start,
             end,
             limit,
             discover_changed_paths,
             strict_node_history,
             receiver,
         )
 
     def close(self):
         pass
 
 
 # Check to see if the given path is a local Subversion repo. Verify this by
 # looking for several svn-specific files and directories in the given
 # directory.
 def filecheck(ui, path, proto):
     for x in (b'locks', b'hooks', b'format', b'db'):
         if not os.path.exists(os.path.join(path, x)):
             return False
     return True
 
 
 # Check to see if a given path is the root of an svn repo over http. We verify
 # this by requesting a version-controlled URL we know can't exist and looking
 # for the svn-specific "not found" XML.
 def httpcheck(ui, path, proto):
     try:
         opener = urlreq.buildopener()
         rsp = opener.open(
             pycompat.strurl(b'%s://%s/!svn/ver/0/.svn' % (proto, path)), b'rb'
         )
         data = rsp.read()
     except urlerr.httperror as inst:
         if inst.code != 404:
             # Except for 404 we cannot know for sure this is not an svn repo
             ui.warn(
                 _(
                     b'svn: cannot probe remote repository, assume it could '
                     b'be a subversion repository. Use --source-type if you '
                     b'know better.\n'
                 )
             )
             return True
         data = inst.fp.read()
     except Exception:
         # Could be urlerr.urlerror if the URL is invalid or anything else.
         return False
     return b'<m:human-readable errcode="160013">' in data
 
 
 protomap = {
     b'http': httpcheck,
     b'https': httpcheck,
     b'file': filecheck,
 }
 
 
 class NonUtf8PercentEncodedBytes(Exception):
     pass
 
 
 # Subversion paths are Unicode. Since the percent-decoding is done on
 # UTF-8-encoded strings, percent-encoded bytes are interpreted as UTF-8.
 def url2pathname_like_subversion(unicodepath):
     if pycompat.ispy3:
         # On Python 3, we have to pass unicode to urlreq.url2pathname().
         # Percent-decoded bytes get decoded using UTF-8 and the 'replace' error
         # handler.
         unicodepath = urlreq.url2pathname(unicodepath)
         if u'\N{REPLACEMENT CHARACTER}' in unicodepath:
             raise NonUtf8PercentEncodedBytes
         else:
             return unicodepath
     else:
         # If we passed unicode on Python 2, it would be converted using the
         # latin-1 encoding. Therefore, we pass UTF-8-encoded bytes.
         unicodepath = urlreq.url2pathname(unicodepath.encode('utf-8'))
         try:
             return unicodepath.decode('utf-8')
         except UnicodeDecodeError:
             raise NonUtf8PercentEncodedBytes
 
 
 def issvnurl(ui, url):
     try:
         proto, path = url.split(b'://', 1)
         if proto == b'file':
             if (
                 pycompat.iswindows
                 and path[:1] == b'/'
                 and path[1:2].isalpha()
                 and path[2:6].lower() == b'%3a/'
             ):
                 path = path[:2] + b':/' + path[6:]
             try:
                 unicodepath = path.decode(fsencoding)
             except UnicodeDecodeError:
                 ui.warn(
                     _(
                         b'Subversion requires that file URLs can be converted '
                         b'to Unicode using the current locale encoding (%s)\n'
                     )
                     % pycompat.sysbytes(fsencoding)
                 )
                 return False
             try:
                 unicodepath = url2pathname_like_subversion(unicodepath)
             except NonUtf8PercentEncodedBytes:
                 ui.warn(
                     _(
                         b'Subversion does not support non-UTF-8 '
                         b'percent-encoded bytes in file URLs\n'
                     )
                 )
                 return False
             # Below, we approximate how Subversion checks the path. On Unix, we
             # should therefore convert the path to bytes using `fsencoding`
             # (like Subversion does). On Windows, the right thing would
             # actually be to leave the path as unicode. For now, we restrict
             # the path to MBCS.
             path = unicodepath.encode(fsencoding)
     except ValueError:
         proto = b'file'
         path = util.abspath(url)
         try:
             path.decode(fsencoding)
         except UnicodeDecodeError:
             ui.warn(
                 _(
                     b'Subversion requires that paths can be converted to '
                     b'Unicode using the current locale encoding (%s)\n'
                 )
                 % pycompat.sysbytes(fsencoding)
             )
             return False
     if proto == b'file':
         path = util.pconvert(path)
     elif proto in (b'http', 'https'):
         if not encoding.isasciistr(path):
             ui.warn(
                 _(
                     b"Subversion sources don't support non-ASCII characters in "
                     b"HTTP(S) URLs. Please percent-encode them.\n"
                 )
             )
             return False
     check = protomap.get(proto, lambda *args: False)
     while b'/' in path:
         if check(ui, path, proto):
             return True
         path = path.rsplit(b'/', 1)[0]
     return False
 
 
 # SVN conversion code stolen from bzr-svn and tailor
 #
 # Subversion looks like a versioned filesystem, branches structures
 # are defined by conventions and not enforced by the tool. First,
 # we define the potential branches (modules) as "trunk" and "branches"
 # children directories. Revisions are then identified by their
 # module and revision number (and a repository identifier).
 #
 # The revision graph is really a tree (or a forest). By default, a
 # revision parent is the previous revision in the same module. If the
 # module directory is copied/moved from another module then the
 # revision is the module root and its parent the source revision in
 # the parent module. A revision has at most one parent.
 #
 class svn_source(converter_source):
     def __init__(self, ui, repotype, url, revs=None):
         super(svn_source, self).__init__(ui, repotype, url, revs=revs)
 
         init_fsencoding()
         if not (
             url.startswith(b'svn://')
             or url.startswith(b'svn+ssh://')
             or (
                 os.path.exists(url)
                 and os.path.exists(os.path.join(url, b'.svn'))
             )
             or issvnurl(ui, url)
         ):
             raise NoRepo(
                 _(b"%s does not look like a Subversion repository") % url
             )
         if svn is None:
             raise MissingTool(_(b'could not load Subversion python bindings'))
 
         try:
             version = svn.core.SVN_VER_MAJOR, svn.core.SVN_VER_MINOR
             if version < (1, 4):
                 raise MissingTool(
                     _(
                         b'Subversion python bindings %d.%d found, '
                         b'1.4 or later required'
                     )
                     % version
                 )
         except AttributeError:
             raise MissingTool(
                 _(
                     b'Subversion python bindings are too old, 1.4 '
                     b'or later required'
                 )
             )
 
         self.lastrevs = {}
 
         latest = None
         try:
             # Support file://path@rev syntax. Useful e.g. to convert
             # deleted branches.
             at = url.rfind(b'@')
             if at >= 0:
                 latest = int(url[at + 1 :])
                 url = url[:at]
         except ValueError:
             pass
         self.url = geturl(url)
         self.encoding = b'UTF-8'  # Subversion is always nominal UTF-8
         try:
             with util.with_lc_ctype():
                 self.transport = transport.SvnRaTransport(url=self.url)
                 self.ra = self.transport.ra
                 self.ctx = self.transport.client
                 self.baseurl = svn.ra.get_repos_root(self.ra)
                 # Module is either empty or a repository path starting with
                 # a slash and not ending with a slash.
                 self.module = urlreq.unquote(self.url[len(self.baseurl) :])
                 self.prevmodule = None
                 self.rootmodule = self.module
                 self.commits = {}
                 self.paths = {}
                 self.uuid = svn.ra.get_uuid(self.ra)
         except svn.core.SubversionException:
             ui.traceback()
             svnversion = b'%d.%d.%d' % (
                 svn.core.SVN_VER_MAJOR,
                 svn.core.SVN_VER_MINOR,
                 svn.core.SVN_VER_MICRO,
             )
             raise NoRepo(
                 _(
                     b"%s does not look like a Subversion repository "
                     b"to libsvn version %s"
                 )
                 % (self.url, svnversion)
             )
 
         if revs:
             if len(revs) > 1:
                 raise error.Abort(
                     _(
                         b'subversion source does not support '
                         b'specifying multiple revisions'
                     )
                 )
             try:
                 latest = int(revs[0])
             except ValueError:
                 raise error.Abort(
                     _(b'svn: revision %s is not an integer') % revs[0]
                 )
 
         trunkcfg = self.ui.config(b'convert', b'svn.trunk')
         if trunkcfg is None:
             trunkcfg = b'trunk'
         self.trunkname = trunkcfg.strip(b'/')
         self.startrev = self.ui.config(b'convert', b'svn.startrev')
         try:
             self.startrev = int(self.startrev)
             if self.startrev < 0:
                 self.startrev = 0
         except ValueError:
             raise error.Abort(
                 _(b'svn: start revision %s is not an integer') % self.startrev
             )
 
         try:
             with util.with_lc_ctype():
                 self.head = self.latest(self.module, latest)
         except SvnPathNotFound:
             self.head = None
         if not self.head:
             raise error.Abort(
                 _(b'no revision found in module %s') % self.module
             )
         self.last_changed = self.revnum(self.head)
 
         self._changescache = (None, None)
 
         if os.path.exists(os.path.join(url, b'.svn/entries')):
             self.wc = url
         else:
             self.wc = None
         self.convertfp = None
 
     def before(self):
         self.with_lc_ctype = util.with_lc_ctype()
         self.with_lc_ctype.__enter__()
 
     def after(self):
         self.with_lc_ctype.__exit__(None, None, None)
 
     def setrevmap(self, revmap):
         lastrevs = {}
         for revid in revmap:
             uuid, module, revnum = revsplit(revid)
             lastrevnum = lastrevs.setdefault(module, revnum)
             if revnum > lastrevnum:
                 lastrevs[module] = revnum
         self.lastrevs = lastrevs
 
     def exists(self, path, optrev):
         try:
             svn.client.ls(
                 self.url.rstrip(b'/') + b'/' + quote(path),
                 optrev,
                 False,
                 self.ctx,
             )
             return True
         except svn.core.SubversionException:
             return False
 
     def getheads(self):
         def isdir(path, revnum):
             kind = self._checkpath(path, revnum)
             return kind == svn.core.svn_node_dir
 
         def getcfgpath(name, rev):
             cfgpath = self.ui.config(b'convert', b'svn.' + name)
             if cfgpath is not None and cfgpath.strip() == b'':
                 return None
             path = (cfgpath or name).strip(b'/')
             if not self.exists(path, rev):
                 if self.module.endswith(path) and name == b'trunk':
                     # we are converting from inside this directory
                     return None
                 if cfgpath:
                     raise error.Abort(
                         _(b'expected %s to be at %r, but not found')
                         % (name, path)
                     )
                 return None
             self.ui.note(
                 _(b'found %s at %r\n') % (name, pycompat.bytestr(path))
             )
             return path
 
         rev = optrev(self.last_changed)
         oldmodule = b''
         trunk = getcfgpath(b'trunk', rev)
         self.tags = getcfgpath(b'tags', rev)
         branches = getcfgpath(b'branches', rev)
 
         # If the project has a trunk or branches, we will extract heads
         # from them. We keep the project root otherwise.
         if trunk:
             oldmodule = self.module or b''
             self.module += b'/' + trunk
             self.head = self.latest(self.module, self.last_changed)
             if not self.head:
                 raise error.Abort(
                     _(b'no revision found in module %s') % self.module
                 )
 
         # First head in the list is the module's head
         self.heads = [self.head]
         if self.tags is not None:
             self.tags = b'%s/%s' % (oldmodule, (self.tags or b'tags'))
 
         # Check if branches bring a few more heads to the list
         if branches:
             rpath = self.url.strip(b'/')
             branchnames = svn.client.ls(
                 rpath + b'/' + quote(branches), rev, False, self.ctx
             )
             for branch in sorted(branchnames):
                 module = b'%s/%s/%s' % (oldmodule, branches, branch)
                 if not isdir(module, self.last_changed):
                     continue
                 brevid = self.latest(module, self.last_changed)
                 if not brevid:
                     self.ui.note(_(b'ignoring empty branch %s\n') % branch)
                     continue
                 self.ui.note(
                     _(b'found branch %s at %d\n')
                     % (branch, self.revnum(brevid))
                 )
                 self.heads.append(brevid)
 
         if self.startrev and self.heads:
             if len(self.heads) > 1:
                 raise error.Abort(
                     _(
                         b'svn: start revision is not supported '
                         b'with more than one branch'
                     )
                 )
             revnum = self.revnum(self.heads[0])
             if revnum < self.startrev:
                 raise error.Abort(
                     _(b'svn: no revision found after start revision %d')
                     % self.startrev
                 )
 
         return self.heads
 
     def _getchanges(self, rev, full):
         (paths, parents) = self.paths[rev]
         copies = {}
         if parents:
             files, self.removed, copies = self.expandpaths(rev, paths, parents)
         if full or not parents:
             # Perform a full checkout on roots
             uuid, module, revnum = revsplit(rev)
             entries = svn.client.ls(
                 self.baseurl + quote(module), optrev(revnum), True, self.ctx
             )
             files = [
                 n
-                for n, e in pycompat.iteritems(entries)
+                for n, e in entries.items()
                 if e.kind == svn.core.svn_node_file
             ]
             self.removed = set()
 
         files.sort()
         files = pycompat.ziplist(files, [rev] * len(files))
         return (files, copies)
 
     def getchanges(self, rev, full):
         # reuse cache from getchangedfiles
         if self._changescache[0] == rev and not full:
             (files, copies) = self._changescache[1]
         else:
             (files, copies) = self._getchanges(rev, full)
             # caller caches the result, so free it here to release memory
             del self.paths[rev]
         return (files, copies, set())
 
     def getchangedfiles(self, rev, i):
         # called from filemap - cache computed values for reuse in getchanges
         (files, copies) = self._getchanges(rev, False)
         self._changescache = (rev, (files, copies))
         return [f[0] for f in files]
 
     def getcommit(self, rev):
         if rev not in self.commits:
             uuid, module, revnum = revsplit(rev)
             self.module = module
             self.reparent(module)
             # We assume that:
             # - requests for revisions after "stop" come from the
             # revision graph backward traversal. Cache all of them
             # down to stop, they will be used eventually.
             # - requests for revisions before "stop" come to get
             # isolated branches parents. Just fetch what is needed.
             stop = self.lastrevs.get(module, 0)
             if revnum < stop:
                 stop = revnum + 1
             self._fetch_revisions(revnum, stop)
             if rev not in self.commits:
                 raise error.Abort(_(b'svn: revision %s not found') % revnum)
         revcommit = self.commits[rev]
         # caller caches the result, so free it here to release memory
         del self.commits[rev]
         return revcommit
 
     def checkrevformat(self, revstr, mapname=b'splicemap'):
         """fails if revision format does not match the correct format"""
         if not re.match(
             br'svn:[0-9a-f]{8,8}-[0-9a-f]{4,4}-'
             br'[0-9a-f]{4,4}-[0-9a-f]{4,4}-[0-9a-f]'
             br'{12,12}(.*)@[0-9]+$',
             revstr,
         ):
             raise error.Abort(
                 _(b'%s entry %s is not a valid revision identifier')
                 % (mapname, revstr)
             )
 
     def numcommits(self):
         return int(self.head.rsplit(b'@', 1)[1]) - self.startrev
 
     def gettags(self):
         tags = {}
         if self.tags is None:
             return tags
 
         # svn tags are just a convention, project branches left in a
         # 'tags' directory. There is no other relationship than
         # ancestry, which is expensive to discover and makes them hard
         # to update incrementally.  Worse, past revisions may be
         # referenced by tags far away in the future, requiring a deep
         # history traversal on every calculation.  Current code
         # performs a single backward traversal, tracking moves within
         # the tags directory (tag renaming) and recording a new tag
         # everytime a project is copied from outside the tags
         # directory. It also lists deleted tags, this behaviour may
         # change in the future.
         pendings = []
         tagspath = self.tags
         start = svn.ra.get_latest_revnum(self.ra)
         stream = self._getlog([self.tags], start, self.startrev)
         try:
             for entry in stream:
                 origpaths, revnum, author, date, message = entry
                 if not origpaths:
                     origpaths = []
                 copies = [
                     (e.copyfrom_path, e.copyfrom_rev, p)
-                    for p, e in pycompat.iteritems(origpaths)
+                    for p, e in origpaths.items()
                     if e.copyfrom_path
                 ]
                 # Apply moves/copies from more specific to general
                 copies.sort(reverse=True)
 
                 srctagspath = tagspath
                 if copies and copies[-1][2] == tagspath:
                     # Track tags directory moves
                     srctagspath = copies.pop()[0]
 
                 for source, sourcerev, dest in copies:
                     if not dest.startswith(tagspath + b'/'):
                         continue
                     for tag in pendings:
                         if tag[0].startswith(dest):
                             tagpath = source + tag[0][len(dest) :]
                             tag[:2] = [tagpath, sourcerev]
                             break
                     else:
                         pendings.append([source, sourcerev, dest])
 
                 # Filter out tags with children coming from different
                 # parts of the repository like:
                 # /tags/tag.1 (from /trunk:10)
                 # /tags/tag.1/foo (from /branches/foo:12)
                 # Here/tags/tag.1 discarded as well as its children.
                 # It happens with tools like cvs2svn. Such tags cannot
                 # be represented in mercurial.
                 addeds = {
                     p: e.copyfrom_path
-                    for p, e in pycompat.iteritems(origpaths)
+                    for p, e in origpaths.items()
                     if e.action == b'A' and e.copyfrom_path
                 }
                 badroots = set()
                 for destroot in addeds:
                     for source, sourcerev, dest in pendings:
                         if not dest.startswith(
                             destroot + b'/'
                         ) or source.startswith(addeds[destroot] + b'/'):
                             continue
                         badroots.add(destroot)
                         break
 
                 for badroot in badroots:
                     pendings = [
                         p
                         for p in pendings
                         if p[2] != badroot
                         and not p[2].startswith(badroot + b'/')
                     ]
 
                 # Tell tag renamings from tag creations
                 renamings = []
                 for source, sourcerev, dest in pendings:
                     tagname = dest.split(b'/')[-1]
                     if source.startswith(srctagspath):
                         renamings.append([source, sourcerev, tagname])
                         continue
                     if tagname in tags:
                         # Keep the latest tag value
                         continue
                     # From revision may be fake, get one with changes
                     try:
                         tagid = self.latest(source, sourcerev)
                         if tagid and tagname not in tags:
                             tags[tagname] = tagid
                     except SvnPathNotFound:
                         # It happens when we are following directories
                         # we assumed were copied with their parents
                         # but were really created in the tag
                         # directory.
                         pass
                 pendings = renamings
                 tagspath = srctagspath
         finally:
             stream.close()
         return tags
 
     def converted(self, rev, destrev):
         if not self.wc:
             return
         if self.convertfp is None:
             self.convertfp = open(
                 os.path.join(self.wc, b'.svn', b'hg-shamap'), b'ab'
             )
         self.convertfp.write(
             util.tonativeeol(b'%s %d\n' % (destrev, self.revnum(rev)))
         )
         self.convertfp.flush()
 
     def revid(self, revnum, module=None):
         return b'svn:%s%s@%d' % (self.uuid, module or self.module, revnum)
 
     def revnum(self, rev):
         return int(rev.split(b'@')[-1])
 
     def latest(self, path, stop=None):
         """Find the latest revid affecting path, up to stop revision
         number. If stop is None, default to repository latest
         revision. It may return a revision in a different module,
         since a branch may be moved without a change being
         reported. Return None if computed module does not belong to
         rootmodule subtree.
         """
 
         def findchanges(path, start, stop=None):
             stream = self._getlog([path], start, stop or 1)
             try:
                 for entry in stream:
                     paths, revnum, author, date, message = entry
                     if stop is None and paths:
                         # We do not know the latest changed revision,
                         # keep the first one with changed paths.
                         break
                     if stop is not None and revnum <= stop:
                         break
 
                     for p in paths:
                         if not path.startswith(p) or not paths[p].copyfrom_path:
                             continue
                         newpath = paths[p].copyfrom_path + path[len(p) :]
                         self.ui.debug(
                             b"branch renamed from %s to %s at %d\n"
                             % (path, newpath, revnum)
                         )
                         path = newpath
                         break
                 if not paths:
                     revnum = None
                 return revnum, path
             finally:
                 stream.close()
 
         if not path.startswith(self.rootmodule):
             # Requests on foreign branches may be forbidden at server level
             self.ui.debug(b'ignoring foreign branch %r\n' % path)
             return None
 
         if stop is None:
             stop = svn.ra.get_latest_revnum(self.ra)
         try:
             prevmodule = self.reparent(b'')
             dirent = svn.ra.stat(self.ra, path.strip(b'/'), stop)
             self.reparent(prevmodule)
         except svn.core.SubversionException:
             dirent = None
         if not dirent:
             raise SvnPathNotFound(
                 _(b'%s not found up to revision %d') % (path, stop)
             )
 
         # stat() gives us the previous revision on this line of
         # development, but it might be in *another module*. Fetch the
         # log and detect renames down to the latest revision.
         revnum, realpath = findchanges(path, stop, dirent.created_rev)
         if revnum is None:
             # Tools like svnsync can create empty revision, when
             # synchronizing only a subtree for instance. These empty
             # revisions created_rev still have their original values
             # despite all changes having disappeared and can be
             # returned by ra.stat(), at least when stating the root
             # module. In that case, do not trust created_rev and scan
             # the whole history.
             revnum, realpath = findchanges(path, stop)
             if revnum is None:
                 self.ui.debug(b'ignoring empty branch %r\n' % realpath)
                 return None
 
         if not realpath.startswith(self.rootmodule):
             self.ui.debug(b'ignoring foreign branch %r\n' % realpath)
             return None
         return self.revid(revnum, realpath)
 
     def reparent(self, module):
         """Reparent the svn transport and return the previous parent."""
         if self.prevmodule == module:
             return module
         svnurl = self.baseurl + quote(module)
         prevmodule = self.prevmodule
         if prevmodule is None:
             prevmodule = b''
         self.ui.debug(b"reparent to %s\n" % svnurl)
         svn.ra.reparent(self.ra, svnurl)
         self.prevmodule = module
         return prevmodule
 
     def expandpaths(self, rev, paths, parents):
         changed, removed = set(), set()
         copies = {}
 
         new_module, revnum = revsplit(rev)[1:]
         if new_module != self.module:
             self.module = new_module
             self.reparent(self.module)
 
         progress = self.ui.makeprogress(
             _(b'scanning paths'), unit=_(b'paths'), total=len(paths)
         )
         for i, (path, ent) in enumerate(paths):
             progress.update(i, item=path)
             entrypath = self.getrelpath(path)
 
             kind = self._checkpath(entrypath, revnum)
             if kind == svn.core.svn_node_file:
                 changed.add(self.recode(entrypath))
                 if not ent.copyfrom_path or not parents:
                     continue
                 # Copy sources not in parent revisions cannot be
                 # represented, ignore their origin for now
                 pmodule, prevnum = revsplit(parents[0])[1:]
                 if ent.copyfrom_rev < prevnum:
                     continue
                 copyfrom_path = self.getrelpath(ent.copyfrom_path, pmodule)
                 if not copyfrom_path:
                     continue
                 self.ui.debug(
                     b"copied to %s from %s@%d\n"
                     % (entrypath, copyfrom_path, ent.copyfrom_rev)
                 )
                 copies[self.recode(entrypath)] = self.recode(copyfrom_path)
             elif kind == 0:  # gone, but had better be a deleted *file*
                 self.ui.debug(b"gone from %d\n" % ent.copyfrom_rev)
                 pmodule, prevnum = revsplit(parents[0])[1:]
                 parentpath = pmodule + b"/" + entrypath
                 fromkind = self._checkpath(entrypath, prevnum, pmodule)
 
                 if fromkind == svn.core.svn_node_file:
                     removed.add(self.recode(entrypath))
                 elif fromkind == svn.core.svn_node_dir:
                     oroot = parentpath.strip(b'/')
                     nroot = path.strip(b'/')
                     children = self._iterfiles(oroot, prevnum)
                     for childpath in children:
                         childpath = childpath.replace(oroot, nroot)
                         childpath = self.getrelpath(b"/" + childpath, pmodule)
                         if childpath:
                             removed.add(self.recode(childpath))
                 else:
                     self.ui.debug(
                         b'unknown path in revision %d: %s\n' % (revnum, path)
                     )
             elif kind == svn.core.svn_node_dir:
                 if ent.action == b'M':
                     # If the directory just had a prop change,
                     # then we shouldn't need to look for its children.
                     continue
                 if ent.action == b'R' and parents:
                     # If a directory is replacing a file, mark the previous
                     # file as deleted
                     pmodule, prevnum = revsplit(parents[0])[1:]
                     pkind = self._checkpath(entrypath, prevnum, pmodule)
                     if pkind == svn.core.svn_node_file:
                         removed.add(self.recode(entrypath))
                     elif pkind == svn.core.svn_node_dir:
                         # We do not know what files were kept or removed,
                         # mark them all as changed.
                         for childpath in self._iterfiles(pmodule, prevnum):
                             childpath = self.getrelpath(b"/" + childpath)
                             if childpath:
                                 changed.add(self.recode(childpath))
 
                 for childpath in self._iterfiles(path, revnum):
                     childpath = self.getrelpath(b"/" + childpath)
                     if childpath:
                         changed.add(self.recode(childpath))
 
                 # Handle directory copies
                 if not ent.copyfrom_path or not parents:
                     continue
                 # Copy sources not in parent revisions cannot be
                 # represented, ignore their origin for now
                 pmodule, prevnum = revsplit(parents[0])[1:]
                 if ent.copyfrom_rev < prevnum:
                     continue
                 copyfrompath = self.getrelpath(ent.copyfrom_path, pmodule)
                 if not copyfrompath:
                     continue
                 self.ui.debug(
                     b"mark %s came from %s:%d\n"
                     % (path, copyfrompath, ent.copyfrom_rev)
                 )
                 children = self._iterfiles(ent.copyfrom_path, ent.copyfrom_rev)
                 for childpath in children:
                     childpath = self.getrelpath(b"/" + childpath, pmodule)
                     if not childpath:
                         continue
                     copytopath = path + childpath[len(copyfrompath) :]
                     copytopath = self.getrelpath(copytopath)
                     copies[self.recode(copytopath)] = self.recode(childpath)
 
         progress.complete()
         changed.update(removed)
         return (list(changed), removed, copies)
 
     def _fetch_revisions(self, from_revnum, to_revnum):
         if from_revnum < to_revnum:
             from_revnum, to_revnum = to_revnum, from_revnum
 
         self.child_cset = None
 
         def parselogentry(orig_paths, revnum, author, date, message):
             """Return the parsed commit object or None, and True if
             the revision is a branch root.
             """
             self.ui.debug(
                 b"parsing revision %d (%d changes)\n"
                 % (revnum, len(orig_paths))
             )
 
             branched = False
             rev = self.revid(revnum)
             # branch log might return entries for a parent we already have
 
             if rev in self.commits or revnum < to_revnum:
                 return None, branched
 
             parents = []
             # check whether this revision is the start of a branch or part
             # of a branch renaming
-            orig_paths = sorted(pycompat.iteritems(orig_paths))
+            orig_paths = sorted(orig_paths.items())
             root_paths = [
                 (p, e) for p, e in orig_paths if self.module.startswith(p)
             ]
             if root_paths:
                 path, ent = root_paths[-1]
                 if ent.copyfrom_path:
                     branched = True
                     newpath = ent.copyfrom_path + self.module[len(path) :]
                     # ent.copyfrom_rev may not be the actual last revision
                     previd = self.latest(newpath, ent.copyfrom_rev)
                     if previd is not None:
                         prevmodule, prevnum = revsplit(previd)[1:]
                         if prevnum >= self.startrev:
                             parents = [previd]
                             self.ui.note(
                                 _(b'found parent of branch %s at %d: %s\n')
                                 % (self.module, prevnum, prevmodule)
                             )
                 else:
                     self.ui.debug(b"no copyfrom path, don't know what to do.\n")
 
             paths = []
             # filter out unrelated paths
             for path, ent in orig_paths:
                 if self.getrelpath(path) is None:
                     continue
                 paths.append((path, ent))
 
             date = parsesvndate(date)
             if self.ui.configbool(b'convert', b'localtimezone'):
                 date = makedatetimestamp(date[0])
 
             if message:
                 log = self.recode(message)
             else:
                 log = b''
 
             if author:
                 author = self.recode(author)
             else:
                 author = b''
 
             try:
                 branch = self.module.split(b"/")[-1]
                 if branch == self.trunkname:
                     branch = None
             except IndexError:
                 branch = None
 
             cset = commit(
                 author=author,
                 date=dateutil.datestr(date, b'%Y-%m-%d %H:%M:%S %1%2'),
                 desc=log,
                 parents=parents,
                 branch=branch,
                 rev=rev,
             )
 
             self.commits[rev] = cset
             # The parents list is *shared* among self.paths and the
             # commit object. Both will be updated below.
             self.paths[rev] = (paths, cset.parents)
             if self.child_cset and not self.child_cset.parents:
                 self.child_cset.parents[:] = [rev]
             self.child_cset = cset
             return cset, branched
 
         self.ui.note(
             _(b'fetching revision log for "%s" from %d to %d\n')
             % (self.module, from_revnum, to_revnum)
         )
 
         try:
             firstcset = None
             lastonbranch = False
             stream = self._getlog([self.module], from_revnum, to_revnum)
             try:
                 for entry in stream:
                     paths, revnum, author, date, message = entry
                     if revnum < self.startrev:
                         lastonbranch = True
                         break
                     if not paths:
                         self.ui.debug(b'revision %d has no entries\n' % revnum)
                         # If we ever leave the loop on an empty
                         # revision, do not try to get a parent branch
                         lastonbranch = lastonbranch or revnum == 0
                         continue
                     cset, lastonbranch = parselogentry(
                         paths, revnum, author, date, message
                     )
                     if cset:
                         firstcset = cset
                     if lastonbranch:
                         break
             finally:
                 stream.close()
 
             if not lastonbranch and firstcset and not firstcset.parents:
                 # The first revision of the sequence (the last fetched one)
                 # has invalid parents if not a branch root. Find the parent
                 # revision now, if any.
                 try:
                     firstrevnum = self.revnum(firstcset.rev)
                     if firstrevnum > 1:
                         latest = self.latest(self.module, firstrevnum - 1)
                         if latest:
                             firstcset.parents.append(latest)
                 except SvnPathNotFound:
                     pass
         except svn.core.SubversionException as xxx_todo_changeme:
             (inst, num) = xxx_todo_changeme.args
             if num == svn.core.SVN_ERR_FS_NO_SUCH_REVISION:
                 raise error.Abort(
                     _(b'svn: branch has no revision %s') % to_revnum
                 )
             raise
 
     def getfile(self, file, rev):
         # TODO: ra.get_file transmits the whole file instead of diffs.
         if file in self.removed:
             return None, None
         try:
             new_module, revnum = revsplit(rev)[1:]
             if self.module != new_module:
                 self.module = new_module
                 self.reparent(self.module)
             io = stringio()
             info = svn.ra.get_file(self.ra, file, revnum, io)
             data = io.getvalue()
             # ra.get_file() seems to keep a reference on the input buffer
             # preventing collection. Release it explicitly.
             io.close()
             if isinstance(info, list):
                 info = info[-1]
             mode = (b"svn:executable" in info) and b'x' or b''
             mode = (b"svn:special" in info) and b'l' or mode
         except svn.core.SubversionException as e:
             notfound = (
                 svn.core.SVN_ERR_FS_NOT_FOUND,
                 svn.core.SVN_ERR_RA_DAV_PATH_NOT_FOUND,
             )
             if e.apr_err in notfound:  # File not found
                 return None, None
             raise
         if mode == b'l':
             link_prefix = b"link "
             if data.startswith(link_prefix):
                 data = data[len(link_prefix) :]
         return data, mode
 
     def _iterfiles(self, path, revnum):
         """Enumerate all files in path at revnum, recursively."""
         path = path.strip(b'/')
         pool = svn.core.Pool()
         rpath = b'/'.join([self.baseurl, quote(path)]).strip(b'/')
         entries = svn.client.ls(rpath, optrev(revnum), True, self.ctx, pool)
         if path:
             path += b'/'
         return (
             (path + p)
-            for p, e in pycompat.iteritems(entries)
+            for p, e in entries.items()
             if e.kind == svn.core.svn_node_file
         )
 
     def getrelpath(self, path, module=None):
         if module is None:
             module = self.module
         # Given the repository url of this wc, say
         #   "http://server/plone/CMFPlone/branches/Plone-2_0-branch"
         # extract the "entry" portion (a relative path) from what
         # svn log --xml says, i.e.
         #   "/CMFPlone/branches/Plone-2_0-branch/tests/PloneTestCase.py"
         # that is to say "tests/PloneTestCase.py"
         if path.startswith(module):
             relative = path.rstrip(b'/')[len(module) :]
             if relative.startswith(b'/'):
                 return relative[1:]
             elif relative == b'':
                 return relative
 
         # The path is outside our tracked tree...
         self.ui.debug(
             b'%r is not under %r, ignoring\n'
             % (pycompat.bytestr(path), pycompat.bytestr(module))
         )
         return None
 
     def _checkpath(self, path, revnum, module=None):
         if module is not None:
             prevmodule = self.reparent(b'')
             path = module + b'/' + path
         try:
             # ra.check_path does not like leading slashes very much, it leads
             # to PROPFIND subversion errors
             return svn.ra.check_path(self.ra, path.strip(b'/'), revnum)
         finally:
             if module is not None:
                 self.reparent(prevmodule)
 
     def _getlog(
         self,
         paths,
         start,
         end,
         limit=0,
         discover_changed_paths=True,
         strict_node_history=False,
     ):
         # Normalize path names, svn >= 1.5 only wants paths relative to
         # supplied URL
         relpaths = []
         for p in paths:
             if not p.startswith(b'/'):
                 p = self.module + b'/' + p
             relpaths.append(p.strip(b'/'))
         args = [
             self.baseurl,
             relpaths,
             start,
             end,
             limit,
             discover_changed_paths,
             strict_node_history,
         ]
         # developer config: convert.svn.debugsvnlog
         if not self.ui.configbool(b'convert', b'svn.debugsvnlog'):
             return directlogstream(*args)
         arg = encodeargs(args)
         hgexe = procutil.hgexecutable()
         cmd = b'%s debugsvnlog' % procutil.shellquote(hgexe)
         stdin, stdout = procutil.popen2(cmd)
         stdin.write(arg)
         try:
             stdin.close()
         except IOError:
             raise error.Abort(
                 _(
                     b'Mercurial failed to run itself, check'
                     b' hg executable is in PATH'
                 )
             )
         return logstream(stdout)
 
 
 pre_revprop_change_template = b'''#!/bin/sh
 
 REPOS="$1"
 REV="$2"
 USER="$3"
 PROPNAME="$4"
 ACTION="$5"
 
 %(rules)s
 
 echo "Changing prohibited revision property" >&2
 exit 1
 '''
 
 
 def gen_pre_revprop_change_hook(prop_actions_allowed):
     rules = []
     for action, propname in prop_actions_allowed:
         rules.append(
             (
                 b'if [ "$ACTION" = "%s" -a "$PROPNAME" = "%s" ]; '
                 b'then exit 0; fi'
             )
             % (action, propname)
         )
     return pre_revprop_change_template % {b'rules': b'\n'.join(rules)}
 
 
 class svn_sink(converter_sink, commandline):
     commit_re = re.compile(br'Committed revision (\d+).', re.M)
     uuid_re = re.compile(br'Repository UUID:\s*(\S+)', re.M)
 
     def prerun(self):
         if self.wc:
             os.chdir(self.wc)
 
     def postrun(self):
         if self.wc:
             os.chdir(self.cwd)
 
     def join(self, name):
         return os.path.join(self.wc, b'.svn', name)
 
     def revmapfile(self):
         return self.join(b'hg-shamap')
 
     def authorfile(self):
         return self.join(b'hg-authormap')
 
     def __init__(self, ui, repotype, path):
 
         converter_sink.__init__(self, ui, repotype, path)
         commandline.__init__(self, ui, b'svn')
         self.delete = []
         self.setexec = []
         self.delexec = []
         self.copies = []
         self.wc = None
         self.cwd = encoding.getcwd()
 
         created = False
         if os.path.isfile(os.path.join(path, b'.svn', b'entries')):
             self.wc = os.path.realpath(path)
             self.run0(b'update')
         else:
             if not re.search(br'^(file|http|https|svn|svn\+ssh)://', path):
                 path = os.path.realpath(path)
                 if os.path.isdir(os.path.dirname(path)):
                     if not os.path.exists(
                         os.path.join(path, b'db', b'fs-type')
                     ):
                         ui.status(
                             _(b"initializing svn repository '%s'\n")
                             % os.path.basename(path)
                         )
                         commandline(ui, b'svnadmin').run0(b'create', path)
                         created = path
                     path = util.normpath(path)
                     if not path.startswith(b'/'):
                         path = b'/' + path
                     path = b'file://' + path
 
             wcpath = os.path.join(
                 encoding.getcwd(), os.path.basename(path) + b'-wc'
             )
             ui.status(
                 _(b"initializing svn working copy '%s'\n")
                 % os.path.basename(wcpath)
             )
             self.run0(b'checkout', path, wcpath)
 
             self.wc = wcpath
         self.opener = vfsmod.vfs(self.wc)
         self.wopener = vfsmod.vfs(self.wc)
         self.childmap = mapfile(ui, self.join(b'hg-childmap'))
         if util.checkexec(self.wc):
             self.is_exec = util.isexec
         else:
             self.is_exec = None
 
         if created:
             prop_actions_allowed = [
                 (b'M', b'svn:log'),
                 (b'A', b'hg:convert-branch'),
                 (b'A', b'hg:convert-rev'),
             ]
 
             if self.ui.configbool(
                 b'convert', b'svn.dangerous-set-commit-dates'
             ):
                 prop_actions_allowed.append((b'M', b'svn:date'))
 
             hook = os.path.join(created, b'hooks', b'pre-revprop-change')
             fp = open(hook, b'wb')
             fp.write(gen_pre_revprop_change_hook(prop_actions_allowed))
             fp.close()
             util.setflags(hook, False, True)
 
         output = self.run0(b'info')
         self.uuid = self.uuid_re.search(output).group(1).strip()
 
     def wjoin(self, *names):
         return os.path.join(self.wc, *names)
 
     @propertycache
     def manifest(self):
         # As of svn 1.7, the "add" command fails when receiving
         # already tracked entries, so we have to track and filter them
         # ourselves.
         m = set()
         output = self.run0(b'ls', recursive=True, xml=True)
         doc = xml.dom.minidom.parseString(output)
         for e in doc.getElementsByTagName('entry'):
             for n in e.childNodes:
                 if n.nodeType != n.ELEMENT_NODE or n.tagName != 'name':
                     continue
                 name = ''.join(
                     c.data for c in n.childNodes if c.nodeType == c.TEXT_NODE
                 )
                 # Entries are compared with names coming from
                 # mercurial, so bytes with undefined encoding. Our
                 # best bet is to assume they are in local
                 # encoding. They will be passed to command line calls
                 # later anyway, so they better be.
                 m.add(encoding.unitolocal(name))
                 break
         return m
 
     def putfile(self, filename, flags, data):
         if b'l' in flags:
             self.wopener.symlink(data, filename)
         else:
             try:
                 if os.path.islink(self.wjoin(filename)):
                     os.unlink(filename)
             except OSError:
                 pass
 
             if self.is_exec:
                 # We need to check executability of the file before the change,
                 # because `vfs.write` is able to reset exec bit.
                 wasexec = False
                 if os.path.exists(self.wjoin(filename)):
                     wasexec = self.is_exec(self.wjoin(filename))
 
             self.wopener.write(filename, data)
 
             if self.is_exec:
                 if wasexec:
                     if b'x' not in flags:
                         self.delexec.append(filename)
                 else:
                     if b'x' in flags:
                         self.setexec.append(filename)
                 util.setflags(self.wjoin(filename), False, b'x' in flags)
 
     def _copyfile(self, source, dest):
         # SVN's copy command pukes if the destination file exists, but
         # our copyfile method expects to record a copy that has
         # already occurred.  Cross the semantic gap.
         wdest = self.wjoin(dest)
         exists = os.path.lexists(wdest)
         if exists:
             fd, tempname = pycompat.mkstemp(
                 prefix=b'hg-copy-', dir=os.path.dirname(wdest)
             )
             os.close(fd)
             os.unlink(tempname)
             os.rename(wdest, tempname)
         try:
             self.run0(b'copy', source, dest)
         finally:
             self.manifest.add(dest)
             if exists:
                 try:
                     os.unlink(wdest)
                 except OSError:
                     pass
                 os.rename(tempname, wdest)
 
     def dirs_of(self, files):
         dirs = set()
         for f in files:
             if os.path.isdir(self.wjoin(f)):
                 dirs.add(f)
             i = len(f)
             for i in iter(lambda: f.rfind(b'/', 0, i), -1):
                 dirs.add(f[:i])
         return dirs
 
     def add_dirs(self, files):
         add_dirs = [
             d for d in sorted(self.dirs_of(files)) if d not in self.manifest
         ]
         if add_dirs:
             self.manifest.update(add_dirs)
             self.xargs(add_dirs, b'add', non_recursive=True, quiet=True)
         return add_dirs
 
     def add_files(self, files):
         files = [f for f in files if f not in self.manifest]
         if files:
             self.manifest.update(files)
             self.xargs(files, b'add', quiet=True)
         return files
 
     def addchild(self, parent, child):
         self.childmap[parent] = child
 
     def revid(self, rev):
         return b"svn:%s@%s" % (self.uuid, rev)
 
     def putcommit(
         self, files, copies, parents, commit, source, revmap, full, cleanp2
     ):
         for parent in parents:
             try:
                 return self.revid(self.childmap[parent])
             except KeyError:
                 pass
 
         # Apply changes to working copy
         for f, v in files:
             data, mode = source.getfile(f, v)
             if data is None:
                 self.delete.append(f)
             else:
                 self.putfile(f, mode, data)
                 if f in copies:
                     self.copies.append([copies[f], f])
         if full:
             self.delete.extend(sorted(self.manifest.difference(files)))
         files = [f[0] for f in files]
 
         entries = set(self.delete)
         files = frozenset(files)
         entries.update(self.add_dirs(files.difference(entries)))
         if self.copies:
             for s, d in self.copies:
                 self._copyfile(s, d)
             self.copies = []
         if self.delete:
             self.xargs(self.delete, b'delete')
             for f in self.delete:
                 self.manifest.remove(f)
             self.delete = []
         entries.update(self.add_files(files.difference(entries)))
         if self.delexec:
             self.xargs(self.delexec, b'propdel', b'svn:executable')
             self.delexec = []
         if self.setexec:
             self.xargs(self.setexec, b'propset', b'svn:executable', b'*')
             self.setexec = []
 
         fd, messagefile = pycompat.mkstemp(prefix=b'hg-convert-')
         fp = os.fdopen(fd, 'wb')
         fp.write(util.tonativeeol(commit.desc))
         fp.close()
         try:
             output = self.run0(
                 b'commit',
                 username=stringutil.shortuser(commit.author),
                 file=messagefile,
                 encoding=b'utf-8',
             )
             try:
                 rev = self.commit_re.search(output).group(1)
             except AttributeError:
                 if not files:
                     return parents[0] if parents else b'None'
                 self.ui.warn(_(b'unexpected svn output:\n'))
                 self.ui.warn(output)
                 raise error.Abort(_(b'unable to cope with svn output'))
             if commit.rev:
                 self.run(
                     b'propset',
                     b'hg:convert-rev',
                     commit.rev,
                     revprop=True,
                     revision=rev,
                 )
             if commit.branch and commit.branch != b'default':
                 self.run(
                     b'propset',
                     b'hg:convert-branch',
                     commit.branch,
                     revprop=True,
                     revision=rev,
                 )
 
             if self.ui.configbool(
                 b'convert', b'svn.dangerous-set-commit-dates'
             ):
                 # Subverson always uses UTC to represent date and time
                 date = dateutil.parsedate(commit.date)
                 date = (date[0], 0)
 
                 # The only way to set date and time for svn commit is to use propset after commit is done
                 self.run(
                     b'propset',
                     b'svn:date',
                     formatsvndate(date),
                     revprop=True,
                     revision=rev,
                 )
 
             for parent in parents:
                 self.addchild(parent, rev)
             return self.revid(rev)
         finally:
             os.unlink(messagefile)
 
     def puttags(self, tags):
         self.ui.warn(_(b'writing Subversion tags is not yet implemented\n'))
         return None, None
 
     def hascommitfrommap(self, rev):
         # We trust that revisions referenced in a map still is present
         # TODO: implement something better if necessary and feasible
         return True
 
     def hascommitforsplicemap(self, rev):
         # This is not correct as one can convert to an existing subversion
         # repository and childmap would not list all revisions. Too bad.
         if rev in self.childmap:
             return True
         raise error.Abort(
             _(
                 b'splice map revision %s not found in subversion '
                 b'child map (revision lookups are not implemented)'
             )
             % rev
         )

hgext/eol.py

@@ -1,479 +1,479 b''
 """automatically manage newlines in repository files
 
 This extension allows you to manage the type of line endings (CRLF or
 LF) that are used in the repository and in the local working
 directory. That way you can get CRLF line endings on Windows and LF on
 Unix/Mac, thereby letting everybody use their OS native line endings.
 
 The extension reads its configuration from a versioned ``.hgeol``
 configuration file found in the root of the working directory. The
 ``.hgeol`` file use the same syntax as all other Mercurial
 configuration files. It uses two sections, ``[patterns]`` and
 ``[repository]``.
 
 The ``[patterns]`` section specifies how line endings should be
 converted between the working directory and the repository. The format is
 specified by a file pattern. The first match is used, so put more
 specific patterns first. The available line endings are ``LF``,
 ``CRLF``, and ``BIN``.
 
 Files with the declared format of ``CRLF`` or ``LF`` are always
 checked out and stored in the repository in that format and files
 declared to be binary (``BIN``) are left unchanged. Additionally,
 ``native`` is an alias for checking out in the platform's default line
 ending: ``LF`` on Unix (including Mac OS X) and ``CRLF`` on
 Windows. Note that ``BIN`` (do nothing to line endings) is Mercurial's
 default behavior; it is only needed if you need to override a later,
 more general pattern.
 
 The optional ``[repository]`` section specifies the line endings to
 use for files stored in the repository. It has a single setting,
 ``native``, which determines the storage line endings for files
 declared as ``native`` in the ``[patterns]`` section. It can be set to
 ``LF`` or ``CRLF``. The default is ``LF``. For example, this means
 that on Windows, files configured as ``native`` (``CRLF`` by default)
 will be converted to ``LF`` when stored in the repository. Files
 declared as ``LF``, ``CRLF``, or ``BIN`` in the ``[patterns]`` section
 are always stored as-is in the repository.
 
 Example versioned ``.hgeol`` file::
 
   [patterns]
   **.py = native
   **.vcproj = CRLF
   **.txt = native
   Makefile = LF
   **.jpg = BIN
 
   [repository]
   native = LF
 
 .. note::
 
    The rules will first apply when files are touched in the working
    directory, e.g. by updating to null and back to tip to touch all files.
 
 The extension uses an optional ``[eol]`` section read from both the
 normal Mercurial configuration files and the ``.hgeol`` file, with the
 latter overriding the former. You can use that section to control the
 overall behavior. There are three settings:
 
 - ``eol.native`` (default ``os.linesep``) can be set to ``LF`` or
   ``CRLF`` to override the default interpretation of ``native`` for
   checkout. This can be used with :hg:`archive` on Unix, say, to
   generate an archive where files have line endings for Windows.
 
 - ``eol.only-consistent`` (default True) can be set to False to make
   the extension convert files with inconsistent EOLs. Inconsistent
   means that there is both ``CRLF`` and ``LF`` present in the file.
   Such files are normally not touched under the assumption that they
   have mixed EOLs on purpose.
 
 - ``eol.fix-trailing-newline`` (default False) can be set to True to
   ensure that converted files end with a EOL character (either ``\\n``
   or ``\\r\\n`` as per the configured patterns).
 
 The extension provides ``cleverencode:`` and ``cleverdecode:`` filters
 like the deprecated win32text extension does. This means that you can
 disable win32text and enable eol and your filters will still work. You
 only need to these filters until you have prepared a ``.hgeol`` file.
 
 The ``win32text.forbid*`` hooks provided by the win32text extension
 have been unified into a single hook named ``eol.checkheadshook``. The
 hook will lookup the expected line endings from the ``.hgeol`` file,
 which means you must migrate to a ``.hgeol`` file first before using
 the hook. ``eol.checkheadshook`` only checks heads, intermediate
 invalid revisions will be pushed. To forbid them completely, use the
 ``eol.checkallhook`` hook. These hooks are best used as
 ``pretxnchangegroup`` hooks.
 
 See :hg:`help patterns` for more information about the glob patterns
 used.
 """
 
 
 import os
 import re
 from mercurial.i18n import _
 from mercurial import (
     config,
     error as errormod,
     extensions,
     match,
     pycompat,
     registrar,
     scmutil,
     util,
 )
 from mercurial.utils import stringutil
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 configitem(
     b'eol',
     b'fix-trailing-newline',
     default=False,
 )
 configitem(
     b'eol',
     b'native',
     default=pycompat.oslinesep,
 )
 configitem(
     b'eol',
     b'only-consistent',
     default=True,
 )
 
 # Matches a lone LF, i.e., one that is not part of CRLF.
 singlelf = re.compile(b'(^|[^\r])\n')
 
 
 def inconsistenteol(data):
     return b'\r\n' in data and singlelf.search(data)
 
 
 def tolf(s, params, ui, **kwargs):
     """Filter to convert to LF EOLs."""
     if stringutil.binary(s):
         return s
     if ui.configbool(b'eol', b'only-consistent') and inconsistenteol(s):
         return s
     if (
         ui.configbool(b'eol', b'fix-trailing-newline')
         and s
         and not s.endswith(b'\n')
     ):
         s = s + b'\n'
     return util.tolf(s)
 
 
 def tocrlf(s, params, ui, **kwargs):
     """Filter to convert to CRLF EOLs."""
     if stringutil.binary(s):
         return s
     if ui.configbool(b'eol', b'only-consistent') and inconsistenteol(s):
         return s
     if (
         ui.configbool(b'eol', b'fix-trailing-newline')
         and s
         and not s.endswith(b'\n')
     ):
         s = s + b'\n'
     return util.tocrlf(s)
 
 
 def isbinary(s, params, ui, **kwargs):
     """Filter to do nothing with the file."""
     return s
 
 
 filters = {
     b'to-lf': tolf,
     b'to-crlf': tocrlf,
     b'is-binary': isbinary,
     # The following provide backwards compatibility with win32text
     b'cleverencode:': tolf,
     b'cleverdecode:': tocrlf,
 }
 
 
 class eolfile(object):
     def __init__(self, ui, root, data):
         self._decode = {
             b'LF': b'to-lf',
             b'CRLF': b'to-crlf',
             b'BIN': b'is-binary',
         }
         self._encode = {
             b'LF': b'to-lf',
             b'CRLF': b'to-crlf',
             b'BIN': b'is-binary',
         }
 
         self.cfg = config.config()
         # Our files should not be touched. The pattern must be
         # inserted first override a '** = native' pattern.
         self.cfg.set(b'patterns', b'.hg*', b'BIN', b'eol')
         # We can then parse the user's patterns.
         self.cfg.parse(b'.hgeol', data)
 
         isrepolf = self.cfg.get(b'repository', b'native') != b'CRLF'
         self._encode[b'NATIVE'] = isrepolf and b'to-lf' or b'to-crlf'
         iswdlf = ui.config(b'eol', b'native') in (b'LF', b'\n')
         self._decode[b'NATIVE'] = iswdlf and b'to-lf' or b'to-crlf'
 
         include = []
         exclude = []
         self.patterns = []
         for pattern, style in self.cfg.items(b'patterns'):
             key = style.upper()
             if key == b'BIN':
                 exclude.append(pattern)
             else:
                 include.append(pattern)
             m = match.match(root, b'', [pattern])
             self.patterns.append((pattern, key, m))
         # This will match the files for which we need to care
         # about inconsistent newlines.
         self.match = match.match(root, b'', [], include, exclude)
 
     def copytoui(self, ui):
         newpatterns = {pattern for pattern, key, m in self.patterns}
         for section in (b'decode', b'encode'):
             for oldpattern, _filter in ui.configitems(section):
                 if oldpattern not in newpatterns:
                     if ui.configsource(section, oldpattern) == b'eol':
                         ui.setconfig(section, oldpattern, b'!', b'eol')
         for pattern, key, m in self.patterns:
             try:
                 ui.setconfig(b'decode', pattern, self._decode[key], b'eol')
                 ui.setconfig(b'encode', pattern, self._encode[key], b'eol')
             except KeyError:
                 ui.warn(
                     _(b"ignoring unknown EOL style '%s' from %s\n")
                     % (key, self.cfg.source(b'patterns', pattern))
                 )
         # eol.only-consistent can be specified in ~/.hgrc or .hgeol
         for k, v in self.cfg.items(b'eol'):
             ui.setconfig(b'eol', k, v, b'eol')
 
     def checkrev(self, repo, ctx, files):
         failed = []
         for f in files or ctx.files():
             if f not in ctx:
                 continue
             for pattern, key, m in self.patterns:
                 if not m(f):
                     continue
                 target = self._encode[key]
                 data = ctx[f].data()
                 if (
                     target == b"to-lf"
                     and b"\r\n" in data
                     or target == b"to-crlf"
                     and singlelf.search(data)
                 ):
                     failed.append((f, target, bytes(ctx)))
                 break
         return failed
 
 
 def parseeol(ui, repo, nodes):
     try:
         for node in nodes:
             try:
                 if node is None:
                     # Cannot use workingctx.data() since it would load
                     # and cache the filters before we configure them.
                     data = repo.wvfs(b'.hgeol').read()
                 else:
                     data = repo[node][b'.hgeol'].data()
                 return eolfile(ui, repo.root, data)
             except (IOError, LookupError):
                 pass
     except errormod.ConfigError as inst:
         ui.warn(
             _(
                 b"warning: ignoring .hgeol file due to parse error "
                 b"at %s: %s\n"
             )
             % (inst.location, inst.message)
         )
     return None
 
 
 def ensureenabled(ui):
     """make sure the extension is enabled when used as hook
 
     When eol is used through hooks, the extension is never formally loaded and
     enabled. This has some side effect, for example the config declaration is
     never loaded. This function ensure the extension is enabled when running
     hooks.
     """
     if b'eol' in ui._knownconfig:
         return
     ui.setconfig(b'extensions', b'eol', b'', source=b'internal')
     extensions.loadall(ui, [b'eol'])
 
 
 def _checkhook(ui, repo, node, headsonly):
     # Get revisions to check and touched files at the same time
     ensureenabled(ui)
     files = set()
     revs = set()
     for rev in pycompat.xrange(repo[node].rev(), len(repo)):
         revs.add(rev)
         if headsonly:
             ctx = repo[rev]
             files.update(ctx.files())
             for pctx in ctx.parents():
                 revs.discard(pctx.rev())
     failed = []
     for rev in revs:
         ctx = repo[rev]
         eol = parseeol(ui, repo, [ctx.node()])
         if eol:
             failed.extend(eol.checkrev(repo, ctx, files))
 
     if failed:
         eols = {b'to-lf': b'CRLF', b'to-crlf': b'LF'}
         msgs = []
         for f, target, node in sorted(failed):
             msgs.append(
                 _(b"  %s in %s should not have %s line endings")
                 % (f, node, eols[target])
             )
         raise errormod.Abort(
             _(b"end-of-line check failed:\n") + b"\n".join(msgs)
         )
 
 
 def checkallhook(ui, repo, node, hooktype, **kwargs):
     """verify that files have expected EOLs"""
     _checkhook(ui, repo, node, False)
 
 
 def checkheadshook(ui, repo, node, hooktype, **kwargs):
     """verify that files have expected EOLs"""
     _checkhook(ui, repo, node, True)
 
 
 # "checkheadshook" used to be called "hook"
 hook = checkheadshook
 
 
 def preupdate(ui, repo, hooktype, parent1, parent2):
     p1node = scmutil.resolvehexnodeidprefix(repo, parent1)
     repo.loadeol([p1node])
     return False
 
 
 def uisetup(ui):
     ui.setconfig(b'hooks', b'preupdate.eol', preupdate, b'eol')
 
 
 def extsetup(ui):
     try:
         extensions.find(b'win32text')
         ui.warn(
             _(
                 b"the eol extension is incompatible with the "
                 b"win32text extension\n"
             )
         )
     except KeyError:
         pass
 
 
 def reposetup(ui, repo):
     uisetup(repo.ui)
 
     if not repo.local():
         return
-    for name, fn in pycompat.iteritems(filters):
+    for name, fn in filters.items():
         repo.adddatafilter(name, fn)
 
     ui.setconfig(b'patch', b'eol', b'auto', b'eol')
 
     class eolrepo(repo.__class__):
         def loadeol(self, nodes):
             eol = parseeol(self.ui, self, nodes)
             if eol is None:
                 return None
             eol.copytoui(self.ui)
             return eol.match
 
         def _hgcleardirstate(self):
             self._eolmatch = self.loadeol([None])
             if not self._eolmatch:
                 self._eolmatch = util.never
                 return
 
             oldeol = None
             try:
                 cachemtime = os.path.getmtime(self.vfs.join(b"eol.cache"))
             except OSError:
                 cachemtime = 0
             else:
                 olddata = self.vfs.read(b"eol.cache")
                 if olddata:
                     oldeol = eolfile(self.ui, self.root, olddata)
 
             try:
                 eolmtime = os.path.getmtime(self.wjoin(b".hgeol"))
             except OSError:
                 eolmtime = 0
 
             if eolmtime >= cachemtime and eolmtime > 0:
                 self.ui.debug(b"eol: detected change in .hgeol\n")
 
                 hgeoldata = self.wvfs.read(b'.hgeol')
                 neweol = eolfile(self.ui, self.root, hgeoldata)
 
                 wlock = None
                 try:
                     wlock = self.wlock()
                     for f in self.dirstate:
                         if not self.dirstate.get_entry(f).maybe_clean:
                             continue
                         if oldeol is not None:
                             if not oldeol.match(f) and not neweol.match(f):
                                 continue
                             oldkey = None
                             for pattern, key, m in oldeol.patterns:
                                 if m(f):
                                     oldkey = key
                                     break
                             newkey = None
                             for pattern, key, m in neweol.patterns:
                                 if m(f):
                                     newkey = key
                                     break
                             if oldkey == newkey:
                                 continue
                         # all normal files need to be looked at again since
                         # the new .hgeol file specify a different filter
                         self.dirstate.set_possibly_dirty(f)
                     # Write the cache to update mtime and cache .hgeol
                     with self.vfs(b"eol.cache", b"w") as f:
                         f.write(hgeoldata)
                 except errormod.LockUnavailable:
                     # If we cannot lock the repository and clear the
                     # dirstate, then a commit might not see all files
                     # as modified. But if we cannot lock the
                     # repository, then we can also not make a commit,
                     # so ignore the error.
                     pass
                 finally:
                     if wlock is not None:
                         wlock.release()
 
         def commitctx(self, ctx, error=False, origctx=None):
             for f in sorted(ctx.added() + ctx.modified()):
                 if not self._eolmatch(f):
                     continue
                 fctx = ctx[f]
                 if fctx is None:
                     continue
                 data = fctx.data()
                 if stringutil.binary(data):
                     # We should not abort here, since the user should
                     # be able to say "** = native" to automatically
                     # have all non-binary files taken care of.
                     continue
                 if inconsistenteol(data):
                     raise errormod.Abort(
                         _(b"inconsistent newline style in %s\n") % f
                     )
             return super(eolrepo, self).commitctx(ctx, error, origctx)
 
     repo.__class__ = eolrepo
     repo._hgcleardirstate()

hgext/fastannotate/context.py

@@ -1,858 +1,858 b''
 # Copyright 2016-present Facebook. All Rights Reserved.
 #
 # context: context needed to annotate a file
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 
 import collections
 import contextlib
 import os
 
 from mercurial.i18n import _
 from mercurial.pycompat import (
     getattr,
     open,
     setattr,
 )
 from mercurial.node import (
     bin,
     hex,
     short,
 )
 from mercurial import (
     error,
     linelog as linelogmod,
     lock as lockmod,
     mdiff,
     pycompat,
     scmutil,
     util,
 )
 from mercurial.utils import (
     hashutil,
     stringutil,
 )
 
 from . import (
     error as faerror,
     revmap as revmapmod,
 )
 
 # given path, get filelog, cached
 @util.lrucachefunc
 def _getflog(repo, path):
     return repo.file(path)
 
 
 # extracted from mercurial.context.basefilectx.annotate
 def _parents(f, follow=True):
     # Cut _descendantrev here to mitigate the penalty of lazy linkrev
     # adjustment. Otherwise, p._adjustlinkrev() would walk changelog
     # from the topmost introrev (= srcrev) down to p.linkrev() if it
     # isn't an ancestor of the srcrev.
     f._changeid
     pl = f.parents()
 
     # Don't return renamed parents if we aren't following.
     if not follow:
         pl = [p for p in pl if p.path() == f.path()]
 
     # renamed filectx won't have a filelog yet, so set it
     # from the cache to save time
     for p in pl:
         if not '_filelog' in p.__dict__:
             p._filelog = _getflog(f._repo, p.path())
 
     return pl
 
 
 # extracted from mercurial.context.basefilectx.annotate. slightly modified
 # so it takes a fctx instead of a pair of text and fctx.
 def _decorate(fctx):
     text = fctx.data()
     linecount = text.count(b'\n')
     if text and not text.endswith(b'\n'):
         linecount += 1
     return ([(fctx, i) for i in pycompat.xrange(linecount)], text)
 
 
 # extracted from mercurial.context.basefilectx.annotate. slightly modified
 # so it takes an extra "blocks" parameter calculated elsewhere, instead of
 # calculating diff here.
 def _pair(parent, child, blocks):
     for (a1, a2, b1, b2), t in blocks:
         # Changed blocks ('!') or blocks made only of blank lines ('~')
         # belong to the child.
         if t == b'=':
             child[0][b1:b2] = parent[0][a1:a2]
     return child
 
 
 # like scmutil.revsingle, but with lru cache, so their states (like manifests)
 # could be reused
 _revsingle = util.lrucachefunc(scmutil.revsingle)
 
 
 def resolvefctx(repo, rev, path, resolverev=False, adjustctx=None):
     """(repo, str, str) -> fctx
 
     get the filectx object from repo, rev, path, in an efficient way.
 
     if resolverev is True, "rev" is a revision specified by the revset
     language, otherwise "rev" is a nodeid, or a revision number that can
     be consumed by repo.__getitem__.
 
     if adjustctx is not None, the returned fctx will point to a changeset
     that introduces the change (last modified the file). if adjustctx
     is 'linkrev', trust the linkrev and do not adjust it. this is noticeably
     faster for big repos but is incorrect for some cases.
     """
     if resolverev and not isinstance(rev, int) and rev is not None:
         ctx = _revsingle(repo, rev)
     else:
         ctx = repo[rev]
 
     # If we don't need to adjust the linkrev, create the filectx using the
     # changectx instead of using ctx[path]. This means it already has the
     # changectx information, so blame -u will be able to look directly at the
     # commitctx object instead of having to resolve it by going through the
     # manifest. In a lazy-manifest world this can prevent us from downloading a
     # lot of data.
     if adjustctx is None:
         # ctx.rev() is None means it's the working copy, which is a special
         # case.
         if ctx.rev() is None:
             fctx = ctx[path]
         else:
             fctx = repo.filectx(path, changeid=ctx.rev())
     else:
         fctx = ctx[path]
         if adjustctx == b'linkrev':
             introrev = fctx.linkrev()
         else:
             introrev = fctx.introrev()
         if introrev != ctx.rev():
             fctx._changeid = introrev
             fctx._changectx = repo[introrev]
     return fctx
 
 
 # like mercurial.store.encodedir, but use linelog suffixes: .m, .l, .lock
 def encodedir(path):
     return (
         path.replace(b'.hg/', b'.hg.hg/')
         .replace(b'.l/', b'.l.hg/')
         .replace(b'.m/', b'.m.hg/')
         .replace(b'.lock/', b'.lock.hg/')
     )
 
 
 def hashdiffopts(diffopts):
     diffoptstr = stringutil.pprint(
         sorted((k, getattr(diffopts, k)) for k in mdiff.diffopts.defaults)
     )
     return hex(hashutil.sha1(diffoptstr).digest())[:6]
 
 
 _defaultdiffopthash = hashdiffopts(mdiff.defaultopts)
 
 
 class annotateopts(object):
     """like mercurial.mdiff.diffopts, but is for annotate
 
     followrename: follow renames, like "hg annotate -f"
     followmerge: follow p2 of a merge changeset, otherwise p2 is ignored
     """
 
     defaults = {
         b'diffopts': None,
         b'followrename': True,
         b'followmerge': True,
     }
 
     def __init__(self, **opts):
         opts = pycompat.byteskwargs(opts)
-        for k, v in pycompat.iteritems(self.defaults):
+        for k, v in self.defaults.items():
             setattr(self, k, opts.get(k, v))
 
     @util.propertycache
     def shortstr(self):
         """represent opts in a short string, suitable for a directory name"""
         result = b''
         if not self.followrename:
             result += b'r0'
         if not self.followmerge:
             result += b'm0'
         if self.diffopts is not None:
             assert isinstance(self.diffopts, mdiff.diffopts)
             diffopthash = hashdiffopts(self.diffopts)
             if diffopthash != _defaultdiffopthash:
                 result += b'i' + diffopthash
         return result or b'default'
 
 
 defaultopts = annotateopts()
 
 
 class _annotatecontext(object):
     """do not use this class directly as it does not use lock to protect
     writes. use "with annotatecontext(...)" instead.
     """
 
     def __init__(self, repo, path, linelogpath, revmappath, opts):
         self.repo = repo
         self.ui = repo.ui
         self.path = path
         self.opts = opts
         self.linelogpath = linelogpath
         self.revmappath = revmappath
         self._linelog = None
         self._revmap = None
         self._node2path = {}  # {str: str}
 
     @property
     def linelog(self):
         if self._linelog is None:
             if os.path.exists(self.linelogpath):
                 with open(self.linelogpath, b'rb') as f:
                     try:
                         self._linelog = linelogmod.linelog.fromdata(f.read())
                     except linelogmod.LineLogError:
                         self._linelog = linelogmod.linelog()
             else:
                 self._linelog = linelogmod.linelog()
         return self._linelog
 
     @property
     def revmap(self):
         if self._revmap is None:
             self._revmap = revmapmod.revmap(self.revmappath)
         return self._revmap
 
     def close(self):
         if self._revmap is not None:
             self._revmap.flush()
             self._revmap = None
         if self._linelog is not None:
             with open(self.linelogpath, b'wb') as f:
                 f.write(self._linelog.encode())
             self._linelog = None
 
     __del__ = close
 
     def rebuild(self):
         """delete linelog and revmap, useful for rebuilding"""
         self.close()
         self._node2path.clear()
         _unlinkpaths([self.revmappath, self.linelogpath])
 
     @property
     def lastnode(self):
         """return last node in revmap, or None if revmap is empty"""
         if self._revmap is None:
             # fast path, read revmap without loading its full content
             return revmapmod.getlastnode(self.revmappath)
         else:
             return self._revmap.rev2hsh(self._revmap.maxrev)
 
     def isuptodate(self, master, strict=True):
         """return True if the revmap / linelog is up-to-date, or the file
         does not exist in the master revision. False otherwise.
 
         it tries to be fast and could return false negatives, because of the
         use of linkrev instead of introrev.
 
         useful for both server and client to decide whether to update
         fastannotate cache or not.
 
         if strict is True, even if fctx exists in the revmap, but is not the
         last node, isuptodate will return False. it's good for performance - no
         expensive check was done.
 
         if strict is False, if fctx exists in the revmap, this function may
         return True. this is useful for the client to skip downloading the
         cache if the client's master is behind the server's.
         """
         lastnode = self.lastnode
         try:
             f = self._resolvefctx(master, resolverev=True)
             # choose linkrev instead of introrev as the check is meant to be
             # *fast*.
             linknode = self.repo.changelog.node(f.linkrev())
             if not strict and lastnode and linknode != lastnode:
                 # check if f.node() is in the revmap. note: this loads the
                 # revmap and can be slow.
                 return self.revmap.hsh2rev(linknode) is not None
             # avoid resolving old manifest, or slow adjustlinkrev to be fast,
             # false negatives are acceptable in this case.
             return linknode == lastnode
         except LookupError:
             # master does not have the file, or the revmap is ahead
             return True
 
     def annotate(self, rev, master=None, showpath=False, showlines=False):
         """incrementally update the cache so it includes revisions in the main
         branch till 'master'. and run annotate on 'rev', which may or may not be
         included in the main branch.
 
         if master is None, do not update linelog.
 
         the first value returned is the annotate result, it is [(node, linenum)]
         by default. [(node, linenum, path)] if showpath is True.
 
         if showlines is True, a second value will be returned, it is a list of
         corresponding line contents.
         """
 
         # the fast path test requires commit hash, convert rev number to hash,
         # so it may hit the fast path. note: in the "fctx" mode, the "annotate"
         # command could give us a revision number even if the user passes a
         # commit hash.
         if isinstance(rev, int):
             rev = hex(self.repo.changelog.node(rev))
 
         # fast path: if rev is in the main branch already
         directly, revfctx = self.canannotatedirectly(rev)
         if directly:
             if self.ui.debugflag:
                 self.ui.debug(
                     b'fastannotate: %s: using fast path '
                     b'(resolved fctx: %s)\n'
                     % (
                         self.path,
                         stringutil.pprint(util.safehasattr(revfctx, b'node')),
                     )
                 )
             return self.annotatedirectly(revfctx, showpath, showlines)
 
         # resolve master
         masterfctx = None
         if master:
             try:
                 masterfctx = self._resolvefctx(
                     master, resolverev=True, adjustctx=True
                 )
             except LookupError:  # master does not have the file
                 pass
             else:
                 if masterfctx in self.revmap:  # no need to update linelog
                     masterfctx = None
 
         #                  ... - @ <- rev (can be an arbitrary changeset,
         #                 /                not necessarily a descendant
         #      master -> o                 of master)
         #                |
         #     a merge -> o         'o': new changesets in the main branch
         #                |\        '#': revisions in the main branch that
         #                o *            exist in linelog / revmap
         #                | .       '*': changesets in side branches, or
         # last master -> # .            descendants of master
         #                | .
         #                # *       joint: '#', and is a parent of a '*'
         #                |/
         #     a joint -> # ^^^^ --- side branches
         #                |
         #                ^ --- main branch (in linelog)
 
         # these DFSes are similar to the traditional annotate algorithm.
         # we cannot really reuse the code for perf reason.
 
         # 1st DFS calculates merges, joint points, and needed.
         # "needed" is a simple reference counting dict to free items in
         # "hist", reducing its memory usage otherwise could be huge.
         initvisit = [revfctx]
         if masterfctx:
             if masterfctx.rev() is None:
                 raise error.Abort(
                     _(b'cannot update linelog to wdir()'),
                     hint=_(b'set fastannotate.mainbranch'),
                 )
             initvisit.append(masterfctx)
         visit = initvisit[:]
         pcache = {}
         needed = {revfctx: 1}
         hist = {}  # {fctx: ([(llrev or fctx, linenum)], text)}
         while visit:
             f = visit.pop()
             if f in pcache or f in hist:
                 continue
             if f in self.revmap:  # in the old main branch, it's a joint
                 llrev = self.revmap.hsh2rev(f.node())
                 self.linelog.annotate(llrev)
                 result = self.linelog.annotateresult
                 hist[f] = (result, f.data())
                 continue
             pl = self._parentfunc(f)
             pcache[f] = pl
             for p in pl:
                 needed[p] = needed.get(p, 0) + 1
                 if p not in pcache:
                     visit.append(p)
 
         # 2nd (simple) DFS calculates new changesets in the main branch
         # ('o' nodes in # the above graph), so we know when to update linelog.
         newmainbranch = set()
         f = masterfctx
         while f and f not in self.revmap:
             newmainbranch.add(f)
             pl = pcache[f]
             if pl:
                 f = pl[0]
             else:
                 f = None
                 break
 
         # f, if present, is the position where the last build stopped at, and
         # should be the "master" last time. check to see if we can continue
         # building the linelog incrementally. (we cannot if diverged)
         if masterfctx is not None:
             self._checklastmasterhead(f)
 
         if self.ui.debugflag:
             if newmainbranch:
                 self.ui.debug(
                     b'fastannotate: %s: %d new changesets in the main'
                     b' branch\n' % (self.path, len(newmainbranch))
                 )
             elif not hist:  # no joints, no updates
                 self.ui.debug(
                     b'fastannotate: %s: linelog cannot help in '
                     b'annotating this revision\n' % self.path
                 )
 
         # prepare annotateresult so we can update linelog incrementally
         self.linelog.annotate(self.linelog.maxrev)
 
         # 3rd DFS does the actual annotate
         visit = initvisit[:]
         progress = self.ui.makeprogress(
             b'building cache', total=len(newmainbranch)
         )
         while visit:
             f = visit[-1]
             if f in hist:
                 visit.pop()
                 continue
 
             ready = True
             pl = pcache[f]
             for p in pl:
                 if p not in hist:
                     ready = False
                     visit.append(p)
             if not ready:
                 continue
 
             visit.pop()
             blocks = None  # mdiff blocks, used for appending linelog
             ismainbranch = f in newmainbranch
             # curr is the same as the traditional annotate algorithm,
             # if we only care about linear history (do not follow merge),
             # then curr is not actually used.
             assert f not in hist
             curr = _decorate(f)
             for i, p in enumerate(pl):
                 bs = list(self._diffblocks(hist[p][1], curr[1]))
                 if i == 0 and ismainbranch:
                     blocks = bs
                 curr = _pair(hist[p], curr, bs)
                 if needed[p] == 1:
                     del hist[p]
                     del needed[p]
                 else:
                     needed[p] -= 1
 
             hist[f] = curr
             del pcache[f]
 
             if ismainbranch:  # need to write to linelog
                 progress.increment()
                 bannotated = None
                 if len(pl) == 2 and self.opts.followmerge:  # merge
                     bannotated = curr[0]
                 if blocks is None:  # no parents, add an empty one
                     blocks = list(self._diffblocks(b'', curr[1]))
                 self._appendrev(f, blocks, bannotated)
             elif showpath:  # not append linelog, but we need to record path
                 self._node2path[f.node()] = f.path()
 
         progress.complete()
 
         result = [
             ((self.revmap.rev2hsh(fr) if isinstance(fr, int) else fr.node()), l)
             for fr, l in hist[revfctx][0]
         ]  # [(node, linenumber)]
         return self._refineannotateresult(result, revfctx, showpath, showlines)
 
     def canannotatedirectly(self, rev):
         """(str) -> bool, fctx or node.
         return (True, f) if we can annotate without updating the linelog, pass
         f to annotatedirectly.
         return (False, f) if we need extra calculation. f is the fctx resolved
         from rev.
         """
         result = True
         f = None
         if not isinstance(rev, int) and rev is not None:
             hsh = {20: bytes, 40: bin}.get(len(rev), lambda x: None)(rev)
             if hsh is not None and (hsh, self.path) in self.revmap:
                 f = hsh
         if f is None:
             adjustctx = b'linkrev' if self._perfhack else True
             f = self._resolvefctx(rev, adjustctx=adjustctx, resolverev=True)
             result = f in self.revmap
             if not result and self._perfhack:
                 # redo the resolution without perfhack - as we are going to
                 # do write operations, we need a correct fctx.
                 f = self._resolvefctx(rev, adjustctx=True, resolverev=True)
         return result, f
 
     def annotatealllines(self, rev, showpath=False, showlines=False):
         """(rev : str) -> [(node : str, linenum : int, path : str)]
 
         the result has the same format with annotate, but include all (including
         deleted) lines up to rev. call this after calling annotate(rev, ...) for
         better performance and accuracy.
         """
         revfctx = self._resolvefctx(rev, resolverev=True, adjustctx=True)
 
         # find a chain from rev to anything in the mainbranch
         if revfctx not in self.revmap:
             chain = [revfctx]
             a = b''
             while True:
                 f = chain[-1]
                 pl = self._parentfunc(f)
                 if not pl:
                     break
                 if pl[0] in self.revmap:
                     a = pl[0].data()
                     break
                 chain.append(pl[0])
 
             # both self.linelog and self.revmap is backed by filesystem. now
             # we want to modify them but do not want to write changes back to
             # files. so we create in-memory objects and copy them. it's like
             # a "fork".
             linelog = linelogmod.linelog()
             linelog.copyfrom(self.linelog)
             linelog.annotate(linelog.maxrev)
             revmap = revmapmod.revmap()
             revmap.copyfrom(self.revmap)
 
             for f in reversed(chain):
                 b = f.data()
                 blocks = list(self._diffblocks(a, b))
                 self._doappendrev(linelog, revmap, f, blocks)
                 a = b
         else:
             # fastpath: use existing linelog, revmap as we don't write to them
             linelog = self.linelog
             revmap = self.revmap
 
         lines = linelog.getalllines()
         hsh = revfctx.node()
         llrev = revmap.hsh2rev(hsh)
         result = [(revmap.rev2hsh(r), l) for r, l in lines if r <= llrev]
         # cannot use _refineannotateresult since we need custom logic for
         # resolving line contents
         if showpath:
             result = self._addpathtoresult(result, revmap)
         if showlines:
             linecontents = self._resolvelines(result, revmap, linelog)
             result = (result, linecontents)
         return result
 
     def _resolvelines(self, annotateresult, revmap, linelog):
         """(annotateresult) -> [line]. designed for annotatealllines.
         this is probably the most inefficient code in the whole fastannotate
         directory. but we have made a decision that the linelog does not
         store line contents. so getting them requires random accesses to
         the revlog data, since they can be many, it can be very slow.
         """
         # [llrev]
         revs = [revmap.hsh2rev(l[0]) for l in annotateresult]
         result = [None] * len(annotateresult)
         # {(rev, linenum): [lineindex]}
         key2idxs = collections.defaultdict(list)
         for i in pycompat.xrange(len(result)):
             key2idxs[(revs[i], annotateresult[i][1])].append(i)
         while key2idxs:
             # find an unresolved line and its linelog rev to annotate
             hsh = None
             try:
-                for (rev, _linenum), idxs in pycompat.iteritems(key2idxs):
+                for (rev, _linenum), idxs in key2idxs.items():
                     if revmap.rev2flag(rev) & revmapmod.sidebranchflag:
                         continue
                     hsh = annotateresult[idxs[0]][0]
                     break
             except StopIteration:  # no more unresolved lines
                 return result
             if hsh is None:
                 # the remaining key2idxs are not in main branch, resolving them
                 # using the hard way...
                 revlines = {}
-                for (rev, linenum), idxs in pycompat.iteritems(key2idxs):
+                for (rev, linenum), idxs in key2idxs.items():
                     if rev not in revlines:
                         hsh = annotateresult[idxs[0]][0]
                         if self.ui.debugflag:
                             self.ui.debug(
                                 b'fastannotate: reading %s line #%d '
                                 b'to resolve lines %r\n'
                                 % (short(hsh), linenum, idxs)
                             )
                         fctx = self._resolvefctx(hsh, revmap.rev2path(rev))
                         lines = mdiff.splitnewlines(fctx.data())
                         revlines[rev] = lines
                     for idx in idxs:
                         result[idx] = revlines[rev][linenum]
                 assert all(x is not None for x in result)
                 return result
 
             # run the annotate and the lines should match to the file content
             self.ui.debug(
                 b'fastannotate: annotate %s to resolve lines\n' % short(hsh)
             )
             linelog.annotate(rev)
             fctx = self._resolvefctx(hsh, revmap.rev2path(rev))
             annotated = linelog.annotateresult
             lines = mdiff.splitnewlines(fctx.data())
             if len(lines) != len(annotated):
                 raise faerror.CorruptedFileError(b'unexpected annotated lines')
             # resolve lines from the annotate result
             for i, line in enumerate(lines):
                 k = annotated[i]
                 if k in key2idxs:
                     for idx in key2idxs[k]:
                         result[idx] = line
                     del key2idxs[k]
         return result
 
     def annotatedirectly(self, f, showpath, showlines):
         """like annotate, but when we know that f is in linelog.
         f can be either a 20-char str (node) or a fctx. this is for perf - in
         the best case, the user provides a node and we don't need to read the
         filelog or construct any filecontext.
         """
         if isinstance(f, bytes):
             hsh = f
         else:
             hsh = f.node()
         llrev = self.revmap.hsh2rev(hsh)
         if not llrev:
             raise faerror.CorruptedFileError(b'%s is not in revmap' % hex(hsh))
         if (self.revmap.rev2flag(llrev) & revmapmod.sidebranchflag) != 0:
             raise faerror.CorruptedFileError(
                 b'%s is not in revmap mainbranch' % hex(hsh)
             )
         self.linelog.annotate(llrev)
         result = [
             (self.revmap.rev2hsh(r), l) for r, l in self.linelog.annotateresult
         ]
         return self._refineannotateresult(result, f, showpath, showlines)
 
     def _refineannotateresult(self, result, f, showpath, showlines):
         """add the missing path or line contents, they can be expensive.
         f could be either node or fctx.
         """
         if showpath:
             result = self._addpathtoresult(result)
         if showlines:
             if isinstance(f, bytes):  # f: node or fctx
                 llrev = self.revmap.hsh2rev(f)
                 fctx = self._resolvefctx(f, self.revmap.rev2path(llrev))
             else:
                 fctx = f
             lines = mdiff.splitnewlines(fctx.data())
             if len(lines) != len(result):  # linelog is probably corrupted
                 raise faerror.CorruptedFileError()
             result = (result, lines)
         return result
 
     def _appendrev(self, fctx, blocks, bannotated=None):
         self._doappendrev(self.linelog, self.revmap, fctx, blocks, bannotated)
 
     def _diffblocks(self, a, b):
         return mdiff.allblocks(a, b, self.opts.diffopts)
 
     @staticmethod
     def _doappendrev(linelog, revmap, fctx, blocks, bannotated=None):
         """append a revision to linelog and revmap"""
 
         def getllrev(f):
             """(fctx) -> int"""
             # f should not be a linelog revision
             if isinstance(f, int):
                 raise error.ProgrammingError(b'f should not be an int')
             # f is a fctx, allocate linelog rev on demand
             hsh = f.node()
             rev = revmap.hsh2rev(hsh)
             if rev is None:
                 rev = revmap.append(hsh, sidebranch=True, path=f.path())
             return rev
 
         # append sidebranch revisions to revmap
         siderevs = []
         siderevmap = {}  # node: int
         if bannotated is not None:
             for (a1, a2, b1, b2), op in blocks:
                 if op != b'=':
                     # f could be either linelong rev, or fctx.
                     siderevs += [
                         f
                         for f, l in bannotated[b1:b2]
                         if not isinstance(f, int)
                     ]
         siderevs = set(siderevs)
         if fctx in siderevs:  # mainnode must be appended seperately
             siderevs.remove(fctx)
         for f in siderevs:
             siderevmap[f] = getllrev(f)
 
         # the changeset in the main branch, could be a merge
         llrev = revmap.append(fctx.node(), path=fctx.path())
         siderevmap[fctx] = llrev
 
         for (a1, a2, b1, b2), op in reversed(blocks):
             if op == b'=':
                 continue
             if bannotated is None:
                 linelog.replacelines(llrev, a1, a2, b1, b2)
             else:
                 blines = [
                     ((r if isinstance(r, int) else siderevmap[r]), l)
                     for r, l in bannotated[b1:b2]
                 ]
                 linelog.replacelines_vec(llrev, a1, a2, blines)
 
     def _addpathtoresult(self, annotateresult, revmap=None):
         """(revmap, [(node, linenum)]) -> [(node, linenum, path)]"""
         if revmap is None:
             revmap = self.revmap
 
         def _getpath(nodeid):
             path = self._node2path.get(nodeid)
             if path is None:
                 path = revmap.rev2path(revmap.hsh2rev(nodeid))
                 self._node2path[nodeid] = path
             return path
 
         return [(n, l, _getpath(n)) for n, l in annotateresult]
 
     def _checklastmasterhead(self, fctx):
         """check if fctx is the master's head last time, raise if not"""
         if fctx is None:
             llrev = 0
         else:
             llrev = self.revmap.hsh2rev(fctx.node())
             if not llrev:
                 raise faerror.CannotReuseError()
         if self.linelog.maxrev != llrev:
             raise faerror.CannotReuseError()
 
     @util.propertycache
     def _parentfunc(self):
         """-> (fctx) -> [fctx]"""
         followrename = self.opts.followrename
         followmerge = self.opts.followmerge
 
         def parents(f):
             pl = _parents(f, follow=followrename)
             if not followmerge:
                 pl = pl[:1]
             return pl
 
         return parents
 
     @util.propertycache
     def _perfhack(self):
         return self.ui.configbool(b'fastannotate', b'perfhack')
 
     def _resolvefctx(self, rev, path=None, **kwds):
         return resolvefctx(self.repo, rev, (path or self.path), **kwds)
 
 
 def _unlinkpaths(paths):
     """silent, best-effort unlink"""
     for path in paths:
         try:
             util.unlink(path)
         except OSError:
             pass
 
 
 class pathhelper(object):
     """helper for getting paths for lockfile, linelog and revmap"""
 
     def __init__(self, repo, path, opts=defaultopts):
         # different options use different directories
         self._vfspath = os.path.join(
             b'fastannotate', opts.shortstr, encodedir(path)
         )
         self._repo = repo
 
     @property
     def dirname(self):
         return os.path.dirname(self._repo.vfs.join(self._vfspath))
 
     @property
     def linelogpath(self):
         return self._repo.vfs.join(self._vfspath + b'.l')
 
     def lock(self):
         return lockmod.lock(self._repo.vfs, self._vfspath + b'.lock')
 
     @property
     def revmappath(self):
         return self._repo.vfs.join(self._vfspath + b'.m')
 
 
 @contextlib.contextmanager
 def annotatecontext(repo, path, opts=defaultopts, rebuild=False):
     """context needed to perform (fast) annotate on a file
 
     an annotatecontext of a single file consists of two structures: the
     linelog and the revmap. this function takes care of locking. only 1
     process is allowed to write that file's linelog and revmap at a time.
 
     when something goes wrong, this function will assume the linelog and the
     revmap are in a bad state, and remove them from disk.
 
     use this function in the following way:
 
         with annotatecontext(...) as actx:
             actx. ....
     """
     helper = pathhelper(repo, path, opts)
     util.makedirs(helper.dirname)
     revmappath = helper.revmappath
     linelogpath = helper.linelogpath
     actx = None
     try:
         with helper.lock():
             actx = _annotatecontext(repo, path, linelogpath, revmappath, opts)
             if rebuild:
                 actx.rebuild()
             yield actx
     except Exception:
         if actx is not None:
             actx.rebuild()
         repo.ui.debug(b'fastannotate: %s: cache broken and deleted\n' % path)
         raise
     finally:
         if actx is not None:
             actx.close()
 
 
 def fctxannotatecontext(fctx, follow=True, diffopts=None, rebuild=False):
     """like annotatecontext but get the context from a fctx. convenient when
     used in fctx.annotate
     """
     repo = fctx._repo
     path = fctx._path
     if repo.ui.configbool(b'fastannotate', b'forcefollow', True):
         follow = True
     aopts = annotateopts(diffopts=diffopts, followrename=follow)
     return annotatecontext(repo, path, aopts, rebuild)

hgext/fastannotate/protocol.py

@@ -1,260 +1,259 b''
 # Copyright 2016-present Facebook. All Rights Reserved.
 #
 # protocol: logic for a server providing fastannotate support
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 import contextlib
 import os
 
 from mercurial.i18n import _
 from mercurial.pycompat import open
 from mercurial import (
     error,
     extensions,
     hg,
-    pycompat,
     util,
     wireprotov1peer,
     wireprotov1server,
 )
 from mercurial.utils import (
     urlutil,
 )
 from . import context
 
 # common
 
 
 def _getmaster(ui):
     """get the mainbranch, and enforce it is set"""
     master = ui.config(b'fastannotate', b'mainbranch')
     if not master:
         raise error.Abort(
             _(
                 b'fastannotate.mainbranch is required '
                 b'for both the client and the server'
             )
         )
     return master
 
 
 # server-side
 
 
 def _capabilities(orig, repo, proto):
     result = orig(repo, proto)
     result.append(b'getannotate')
     return result
 
 
 def _getannotate(repo, proto, path, lastnode):
     # output:
     #   FILE := vfspath + '\0' + str(size) + '\0' + content
     #   OUTPUT := '' | FILE + OUTPUT
     result = b''
     buildondemand = repo.ui.configbool(
         b'fastannotate', b'serverbuildondemand', True
     )
     with context.annotatecontext(repo, path) as actx:
         if buildondemand:
             # update before responding to the client
             master = _getmaster(repo.ui)
             try:
                 if not actx.isuptodate(master):
                     actx.annotate(master, master)
             except Exception:
                 # non-fast-forward move or corrupted. rebuild automically.
                 actx.rebuild()
                 try:
                     actx.annotate(master, master)
                 except Exception:
                     actx.rebuild()  # delete files
             finally:
                 # although the "with" context will also do a close/flush, we
                 # need to do it early so we can send the correct respond to
                 # client.
                 actx.close()
         # send back the full content of revmap and linelog, in the future we
         # may want to do some rsync-like fancy updating.
         # the lastnode check is not necessary if the client and the server
         # agree where the main branch is.
         if actx.lastnode != lastnode:
             for p in [actx.revmappath, actx.linelogpath]:
                 if not os.path.exists(p):
                     continue
                 with open(p, b'rb') as f:
                     content = f.read()
                 vfsbaselen = len(repo.vfs.base + b'/')
                 relpath = p[vfsbaselen:]
                 result += b'%s\0%d\0%s' % (relpath, len(content), content)
     return result
 
 
 def _registerwireprotocommand():
     if b'getannotate' in wireprotov1server.commands:
         return
     wireprotov1server.wireprotocommand(b'getannotate', b'path lastnode')(
         _getannotate
     )
 
 
 def serveruisetup(ui):
     _registerwireprotocommand()
     extensions.wrapfunction(wireprotov1server, b'_capabilities', _capabilities)
 
 
 # client-side
 
 
 def _parseresponse(payload):
     result = {}
     i = 0
     l = len(payload) - 1
     state = 0  # 0: vfspath, 1: size
     vfspath = size = b''
     while i < l:
         ch = payload[i : i + 1]
         if ch == b'\0':
             if state == 1:
                 result[vfspath] = payload[i + 1 : i + 1 + int(size)]
                 i += int(size)
                 state = 0
                 vfspath = size = b''
             elif state == 0:
                 state = 1
         else:
             if state == 1:
                 size += ch
             elif state == 0:
                 vfspath += ch
         i += 1
     return result
 
 
 def peersetup(ui, peer):
     class fastannotatepeer(peer.__class__):
         @wireprotov1peer.batchable
         def getannotate(self, path, lastnode=None):
             if not self.capable(b'getannotate'):
                 ui.warn(_(b'remote peer cannot provide annotate cache\n'))
                 return None, None
             else:
                 args = {b'path': path, b'lastnode': lastnode or b''}
                 return args, _parseresponse
 
     peer.__class__ = fastannotatepeer
 
 
 @contextlib.contextmanager
 def annotatepeer(repo):
     ui = repo.ui
 
     remotedest = ui.config(b'fastannotate', b'remotepath', b'default')
     r = urlutil.get_unique_pull_path(b'fastannotate', repo, ui, remotedest)
     remotepath = r[0]
     peer = hg.peer(ui, {}, remotepath)
 
     try:
         yield peer
     finally:
         peer.close()
 
 
 def clientfetch(repo, paths, lastnodemap=None, peer=None):
     """download annotate cache from the server for paths"""
     if not paths:
         return
 
     if peer is None:
         with annotatepeer(repo) as peer:
             return clientfetch(repo, paths, lastnodemap, peer)
 
     if lastnodemap is None:
         lastnodemap = {}
 
     ui = repo.ui
     results = []
     with peer.commandexecutor() as batcher:
         ui.debug(b'fastannotate: requesting %d files\n' % len(paths))
         for p in paths:
             results.append(
                 batcher.callcommand(
                     b'getannotate',
                     {b'path': p, b'lastnode': lastnodemap.get(p)},
                 )
             )
 
         for result in results:
             r = result.result()
             # TODO: pconvert these paths on the server?
-            r = {util.pconvert(p): v for p, v in pycompat.iteritems(r)}
+            r = {util.pconvert(p): v for p, v in r.items()}
             for path in sorted(r):
                 # ignore malicious paths
                 if not path.startswith(b'fastannotate/') or b'/../' in (
                     path + b'/'
                 ):
                     ui.debug(
                         b'fastannotate: ignored malicious path %s\n' % path
                     )
                     continue
                 content = r[path]
                 if ui.debugflag:
                     ui.debug(
                         b'fastannotate: writing %d bytes to %s\n'
                         % (len(content), path)
                     )
                 repo.vfs.makedirs(os.path.dirname(path))
                 with repo.vfs(path, b'wb') as f:
                     f.write(content)
 
 
 def _filterfetchpaths(repo, paths):
     """return a subset of paths whose history is long and need to fetch linelog
     from the server. works with remotefilelog and non-remotefilelog repos.
     """
     threshold = repo.ui.configint(b'fastannotate', b'clientfetchthreshold', 10)
     if threshold <= 0:
         return paths
 
     result = []
     for path in paths:
         try:
             if len(repo.file(path)) >= threshold:
                 result.append(path)
         except Exception:  # file not found etc.
             result.append(path)
 
     return result
 
 
 def localreposetup(ui, repo):
     class fastannotaterepo(repo.__class__):
         def prefetchfastannotate(self, paths, peer=None):
             master = _getmaster(self.ui)
             needupdatepaths = []
             lastnodemap = {}
             try:
                 for path in _filterfetchpaths(self, paths):
                     with context.annotatecontext(self, path) as actx:
                         if not actx.isuptodate(master, strict=False):
                             needupdatepaths.append(path)
                             lastnodemap[path] = actx.lastnode
                 if needupdatepaths:
                     clientfetch(self, needupdatepaths, lastnodemap, peer)
             except Exception as ex:
                 # could be directory not writable or so, not fatal
                 self.ui.debug(b'fastannotate: prefetch failed: %r\n' % ex)
 
     repo.__class__ = fastannotaterepo
 
 
 def clientreposetup(ui, repo):
     _registerwireprotocommand()
     if repo.local():
         localreposetup(ui, repo)
     # TODO: this mutates global state, but only if at least one repo
     # has the extension enabled. This is probably bad for hgweb.
     if peersetup not in hg.wirepeersetupfuncs:
         hg.wirepeersetupfuncs.append(peersetup)

hgext/fix.py

@@ -1,957 +1,955 b''
 # fix - rewrite file content in changesets and working copy
 #
 # Copyright 2018 Google LLC.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 """rewrite file content in changesets or working copy (EXPERIMENTAL)
 
 Provides a command that runs configured tools on the contents of modified files,
 writing back any fixes to the working copy or replacing changesets.
 
 Here is an example configuration that causes :hg:`fix` to apply automatic
 formatting fixes to modified lines in C++ code::
 
   [fix]
   clang-format:command=clang-format --assume-filename={rootpath}
   clang-format:linerange=--lines={first}:{last}
   clang-format:pattern=set:**.cpp or **.hpp
 
 The :command suboption forms the first part of the shell command that will be
 used to fix a file. The content of the file is passed on standard input, and the
 fixed file content is expected on standard output. Any output on standard error
 will be displayed as a warning. If the exit status is not zero, the file will
 not be affected. A placeholder warning is displayed if there is a non-zero exit
 status but no standard error output. Some values may be substituted into the
 command::
 
   {rootpath}  The path of the file being fixed, relative to the repo root
   {basename}  The name of the file being fixed, without the directory path
 
 If the :linerange suboption is set, the tool will only be run if there are
 changed lines in a file. The value of this suboption is appended to the shell
 command once for every range of changed lines in the file. Some values may be
 substituted into the command::
 
   {first}   The 1-based line number of the first line in the modified range
   {last}    The 1-based line number of the last line in the modified range
 
 Deleted sections of a file will be ignored by :linerange, because there is no
 corresponding line range in the version being fixed.
 
 By default, tools that set :linerange will only be executed if there is at least
 one changed line range. This is meant to prevent accidents like running a code
 formatter in such a way that it unexpectedly reformats the whole file. If such a
 tool needs to operate on unchanged files, it should set the :skipclean suboption
 to false.
 
 The :pattern suboption determines which files will be passed through each
 configured tool. See :hg:`help patterns` for possible values. However, all
 patterns are relative to the repo root, even if that text says they are relative
 to the current working directory. If there are file arguments to :hg:`fix`, the
 intersection of these patterns is used.
 
 There is also a configurable limit for the maximum size of file that will be
 processed by :hg:`fix`::
 
   [fix]
   maxfilesize = 2MB
 
 Normally, execution of configured tools will continue after a failure (indicated
 by a non-zero exit status). It can also be configured to abort after the first
 such failure, so that no files will be affected if any tool fails. This abort
 will also cause :hg:`fix` to exit with a non-zero status::
 
   [fix]
   failure = abort
 
 When multiple tools are configured to affect a file, they execute in an order
 defined by the :priority suboption. The priority suboption has a default value
 of zero for each tool. Tools are executed in order of descending priority. The
 execution order of tools with equal priority is unspecified. For example, you
 could use the 'sort' and 'head' utilities to keep only the 10 smallest numbers
 in a text file by ensuring that 'sort' runs before 'head'::
 
   [fix]
   sort:command = sort -n
   head:command = head -n 10
   sort:pattern = numbers.txt
   head:pattern = numbers.txt
   sort:priority = 2
   head:priority = 1
 
 To account for changes made by each tool, the line numbers used for incremental
 formatting are recomputed before executing the next tool. So, each tool may see
 different values for the arguments added by the :linerange suboption.
 
 Each fixer tool is allowed to return some metadata in addition to the fixed file
 content. The metadata must be placed before the file content on stdout,
 separated from the file content by a zero byte. The metadata is parsed as a JSON
 value (so, it should be UTF-8 encoded and contain no zero bytes). A fixer tool
 is expected to produce this metadata encoding if and only if the :metadata
 suboption is true::
 
   [fix]
   tool:command = tool --prepend-json-metadata
   tool:metadata = true
 
 The metadata values are passed to hooks, which can be used to print summaries or
 perform other post-fixing work. The supported hooks are::
 
   "postfixfile"
     Run once for each file in each revision where any fixer tools made changes
     to the file content. Provides "$HG_REV" and "$HG_PATH" to identify the file,
     and "$HG_METADATA" with a map of fixer names to metadata values from fixer
     tools that affected the file. Fixer tools that didn't affect the file have a
     value of None. Only fixer tools that executed are present in the metadata.
 
   "postfix"
     Run once after all files and revisions have been handled. Provides
     "$HG_REPLACEMENTS" with information about what revisions were created and
     made obsolete. Provides a boolean "$HG_WDIRWRITTEN" to indicate whether any
     files in the working copy were updated. Provides a list "$HG_METADATA"
     mapping fixer tool names to lists of metadata values returned from
     executions that modified a file. This aggregates the same metadata
     previously passed to the "postfixfile" hook.
 
 Fixer tools are run in the repository's root directory. This allows them to read
 configuration files from the working copy, or even write to the working copy.
 The working copy is not updated to match the revision being fixed. In fact,
 several revisions may be fixed in parallel. Writes to the working copy are not
 amended into the revision being fixed; fixer tools should always write fixed
 file content back to stdout as documented above.
 """
 
 
 import collections
 import itertools
 import os
 import re
 import subprocess
 
 from mercurial.i18n import _
 from mercurial.node import (
     nullid,
     nullrev,
     wdirrev,
 )
 
 from mercurial.utils import procutil
 
 from mercurial import (
     cmdutil,
     context,
     copies,
     error,
     logcmdutil,
     match as matchmod,
     mdiff,
     merge,
     mergestate as mergestatemod,
     pycompat,
     registrar,
     rewriteutil,
     scmutil,
     util,
     worker,
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 # Register the suboptions allowed for each configured fixer, and default values.
 FIXER_ATTRS = {
     b'command': None,
     b'linerange': None,
     b'pattern': None,
     b'priority': 0,
     b'metadata': False,
     b'skipclean': True,
     b'enabled': True,
 }
 
 for key, default in FIXER_ATTRS.items():
     configitem(b'fix', b'.*:%s$' % key, default=default, generic=True)
 
 # A good default size allows most source code files to be fixed, but avoids
 # letting fixer tools choke on huge inputs, which could be surprising to the
 # user.
 configitem(b'fix', b'maxfilesize', default=b'2MB')
 
 # Allow fix commands to exit non-zero if an executed fixer tool exits non-zero.
 # This helps users do shell scripts that stop when a fixer tool signals a
 # problem.
 configitem(b'fix', b'failure', default=b'continue')
 
 
 def checktoolfailureaction(ui, message, hint=None):
     """Abort with 'message' if fix.failure=abort"""
     action = ui.config(b'fix', b'failure')
     if action not in (b'continue', b'abort'):
         raise error.Abort(
             _(b'unknown fix.failure action: %s') % (action,),
             hint=_(b'use "continue" or "abort"'),
         )
     if action == b'abort':
         raise error.Abort(message, hint=hint)
 
 
 allopt = (b'', b'all', False, _(b'fix all non-public non-obsolete revisions'))
 baseopt = (
     b'',
     b'base',
     [],
     _(
         b'revisions to diff against (overrides automatic '
         b'selection, and applies to every revision being '
         b'fixed)'
     ),
     _(b'REV'),
 )
 revopt = (b'r', b'rev', [], _(b'revisions to fix (ADVANCED)'), _(b'REV'))
 sourceopt = (
     b's',
     b'source',
     [],
     _(b'fix the specified revisions and their descendants'),
     _(b'REV'),
 )
 wdiropt = (b'w', b'working-dir', False, _(b'fix the working directory'))
 wholeopt = (b'', b'whole', False, _(b'always fix every line of a file'))
 usage = _(b'[OPTION]... [FILE]...')
 
 
 @command(
     b'fix',
     [allopt, baseopt, revopt, sourceopt, wdiropt, wholeopt],
     usage,
     helpcategory=command.CATEGORY_FILE_CONTENTS,
 )
 def fix(ui, repo, *pats, **opts):
     """rewrite file content in changesets or working directory
 
     Runs any configured tools to fix the content of files. Only affects files
     with changes, unless file arguments are provided. Only affects changed lines
     of files, unless the --whole flag is used. Some tools may always affect the
     whole file regardless of --whole.
 
     If --working-dir is used, files with uncommitted changes in the working copy
     will be fixed. Note that no backup are made.
 
     If revisions are specified with --source, those revisions and their
     descendants will be checked, and they may be replaced with new revisions
     that have fixed file content. By automatically including the descendants,
     no merging, rebasing, or evolution will be required. If an ancestor of the
     working copy is included, then the working copy itself will also be fixed,
     and the working copy will be updated to the fixed parent.
 
     When determining what lines of each file to fix at each revision, the whole
     set of revisions being fixed is considered, so that fixes to earlier
     revisions are not forgotten in later ones. The --base flag can be used to
     override this default behavior, though it is not usually desirable to do so.
     """
     opts = pycompat.byteskwargs(opts)
     cmdutil.check_at_most_one_arg(opts, b'all', b'source', b'rev')
     cmdutil.check_incompatible_arguments(
         opts, b'working_dir', [b'all', b'source']
     )
 
     with repo.wlock(), repo.lock(), repo.transaction(b'fix'):
         revstofix = getrevstofix(ui, repo, opts)
         basectxs = getbasectxs(repo, opts, revstofix)
         workqueue, numitems = getworkqueue(
             ui, repo, pats, opts, revstofix, basectxs
         )
         basepaths = getbasepaths(repo, opts, workqueue, basectxs)
         fixers = getfixers(ui)
 
         # Rather than letting each worker independently fetch the files
         # (which also would add complications for shared/keepalive
         # connections), prefetch them all first.
         _prefetchfiles(repo, workqueue, basepaths)
 
         # There are no data dependencies between the workers fixing each file
         # revision, so we can use all available parallelism.
         def getfixes(items):
             for srcrev, path, dstrevs in items:
                 ctx = repo[srcrev]
                 olddata = ctx[path].data()
                 metadata, newdata = fixfile(
                     ui,
                     repo,
                     opts,
                     fixers,
                     ctx,
                     path,
                     basepaths,
                     basectxs[srcrev],
                 )
                 # We ungroup the work items now, because the code that consumes
                 # these results has to handle each dstrev separately, and in
                 # topological order. Because these are handled in topological
                 # order, it's important that we pass around references to
                 # "newdata" instead of copying it. Otherwise, we would be
                 # keeping more copies of file content in memory at a time than
                 # if we hadn't bothered to group/deduplicate the work items.
                 data = newdata if newdata != olddata else None
                 for dstrev in dstrevs:
                     yield (dstrev, path, metadata, data)
 
         results = worker.worker(
             ui, 1.0, getfixes, tuple(), workqueue, threadsafe=False
         )
 
         # We have to hold on to the data for each successor revision in memory
         # until all its parents are committed. We ensure this by committing and
         # freeing memory for the revisions in some topological order. This
         # leaves a little bit of memory efficiency on the table, but also makes
         # the tests deterministic. It might also be considered a feature since
         # it makes the results more easily reproducible.
         filedata = collections.defaultdict(dict)
         aggregatemetadata = collections.defaultdict(list)
         replacements = {}
         wdirwritten = False
         commitorder = sorted(revstofix, reverse=True)
         with ui.makeprogress(
             topic=_(b'fixing'), unit=_(b'files'), total=sum(numitems.values())
         ) as progress:
             for rev, path, filerevmetadata, newdata in results:
                 progress.increment(item=path)
                 for fixername, fixermetadata in filerevmetadata.items():
                     aggregatemetadata[fixername].append(fixermetadata)
                 if newdata is not None:
                     filedata[rev][path] = newdata
                     hookargs = {
                         b'rev': rev,
                         b'path': path,
                         b'metadata': filerevmetadata,
                     }
                     repo.hook(
                         b'postfixfile',
                         throw=False,
                         **pycompat.strkwargs(hookargs)
                     )
                 numitems[rev] -= 1
                 # Apply the fixes for this and any other revisions that are
                 # ready and sitting at the front of the queue. Using a loop here
                 # prevents the queue from being blocked by the first revision to
                 # be ready out of order.
                 while commitorder and not numitems[commitorder[-1]]:
                     rev = commitorder.pop()
                     ctx = repo[rev]
                     if rev == wdirrev:
                         writeworkingdir(repo, ctx, filedata[rev], replacements)
                         wdirwritten = bool(filedata[rev])
                     else:
                         replacerev(ui, repo, ctx, filedata[rev], replacements)
                     del filedata[rev]
 
         cleanup(repo, replacements, wdirwritten)
         hookargs = {
             b'replacements': replacements,
             b'wdirwritten': wdirwritten,
             b'metadata': aggregatemetadata,
         }
         repo.hook(b'postfix', throw=True, **pycompat.strkwargs(hookargs))
 
 
 def cleanup(repo, replacements, wdirwritten):
     """Calls scmutil.cleanupnodes() with the given replacements.
 
     "replacements" is a dict from nodeid to nodeid, with one key and one value
     for every revision that was affected by fixing. This is slightly different
     from cleanupnodes().
 
     "wdirwritten" is a bool which tells whether the working copy was affected by
     fixing, since it has no entry in "replacements".
 
     Useful as a hook point for extending "hg fix" with output summarizing the
     effects of the command, though we choose not to output anything here.
     """
-    replacements = {
-        prec: [succ] for prec, succ in pycompat.iteritems(replacements)
-    }
+    replacements = {prec: [succ] for prec, succ in replacements.items()}
     scmutil.cleanupnodes(repo, replacements, b'fix', fixphase=True)
 
 
 def getworkqueue(ui, repo, pats, opts, revstofix, basectxs):
     """Constructs a list of files to fix and which revisions each fix applies to
 
     To avoid duplicating work, there is usually only one work item for each file
     revision that might need to be fixed. There can be multiple work items per
     file revision if the same file needs to be fixed in multiple changesets with
     different baserevs. Each work item also contains a list of changesets where
     the file's data should be replaced with the fixed data. The work items for
     earlier changesets come earlier in the work queue, to improve pipelining by
     allowing the first changeset to be replaced while fixes are still being
     computed for later changesets.
 
     Also returned is a map from changesets to the count of work items that might
     affect each changeset. This is used later to count when all of a changeset's
     work items have been finished, without having to inspect the remaining work
     queue in each worker subprocess.
 
     The example work item (1, "foo/bar.txt", (1, 2, 3)) means that the data of
     bar.txt should be read from revision 1, then fixed, and written back to
     revisions 1, 2 and 3. Revision 1 is called the "srcrev" and the list of
     revisions is called the "dstrevs". In practice the srcrev is always one of
     the dstrevs, and we make that choice when constructing the work item so that
     the choice can't be made inconsistently later on. The dstrevs should all
     have the same file revision for the given path, so the choice of srcrev is
     arbitrary. The wdirrev can be a dstrev and a srcrev.
     """
     dstrevmap = collections.defaultdict(list)
     numitems = collections.defaultdict(int)
     maxfilesize = ui.configbytes(b'fix', b'maxfilesize')
     for rev in sorted(revstofix):
         fixctx = repo[rev]
         match = scmutil.match(fixctx, pats, opts)
         for path in sorted(
             pathstofix(ui, repo, pats, opts, match, basectxs[rev], fixctx)
         ):
             fctx = fixctx[path]
             if fctx.islink():
                 continue
             if fctx.size() > maxfilesize:
                 ui.warn(
                     _(b'ignoring file larger than %s: %s\n')
                     % (util.bytecount(maxfilesize), path)
                 )
                 continue
             baserevs = tuple(ctx.rev() for ctx in basectxs[rev])
             dstrevmap[(fctx.filerev(), baserevs, path)].append(rev)
             numitems[rev] += 1
     workqueue = [
         (min(dstrevs), path, dstrevs)
         for (_filerev, _baserevs, path), dstrevs in dstrevmap.items()
     ]
     # Move work items for earlier changesets to the front of the queue, so we
     # might be able to replace those changesets (in topological order) while
     # we're still processing later work items. Note the min() in the previous
     # expression, which means we don't need a custom comparator here. The path
     # is also important in the sort order to make the output order stable. There
     # are some situations where this doesn't help much, but some situations
     # where it lets us buffer O(1) files instead of O(n) files.
     workqueue.sort()
     return workqueue, numitems
 
 
 def getrevstofix(ui, repo, opts):
     """Returns the set of revision numbers that should be fixed"""
     if opts[b'all']:
         revs = repo.revs(b'(not public() and not obsolete()) or wdir()')
     elif opts[b'source']:
         source_revs = logcmdutil.revrange(repo, opts[b'source'])
         revs = set(repo.revs(b'(%ld::) - obsolete()', source_revs))
         if wdirrev in source_revs:
             # `wdir()::` is currently empty, so manually add wdir
             revs.add(wdirrev)
         if repo[b'.'].rev() in revs:
             revs.add(wdirrev)
     else:
         revs = set(logcmdutil.revrange(repo, opts[b'rev']))
         if opts.get(b'working_dir'):
             revs.add(wdirrev)
     # Allow fixing only wdir() even if there's an unfinished operation
     if not (len(revs) == 1 and wdirrev in revs):
         cmdutil.checkunfinished(repo)
         rewriteutil.precheck(repo, revs, b'fix')
     if (
         wdirrev in revs
         and mergestatemod.mergestate.read(repo).unresolvedcount()
     ):
         raise error.Abort(b'unresolved conflicts', hint=b"use 'hg resolve'")
     if not revs:
         raise error.Abort(
             b'no changesets specified', hint=b'use --source or --working-dir'
         )
     return revs
 
 
 def pathstofix(ui, repo, pats, opts, match, basectxs, fixctx):
     """Returns the set of files that should be fixed in a context
 
     The result depends on the base contexts; we include any file that has
     changed relative to any of the base contexts. Base contexts should be
     ancestors of the context being fixed.
     """
     files = set()
     for basectx in basectxs:
         stat = basectx.status(
             fixctx, match=match, listclean=bool(pats), listunknown=bool(pats)
         )
         files.update(
             set(
                 itertools.chain(
                     stat.added, stat.modified, stat.clean, stat.unknown
                 )
             )
         )
     return files
 
 
 def lineranges(opts, path, basepaths, basectxs, fixctx, content2):
     """Returns the set of line ranges that should be fixed in a file
 
     Of the form [(10, 20), (30, 40)].
 
     This depends on the given base contexts; we must consider lines that have
     changed versus any of the base contexts, and whether the file has been
     renamed versus any of them.
 
     Another way to understand this is that we exclude line ranges that are
     common to the file in all base contexts.
     """
     if opts.get(b'whole'):
         # Return a range containing all lines. Rely on the diff implementation's
         # idea of how many lines are in the file, instead of reimplementing it.
         return difflineranges(b'', content2)
 
     rangeslist = []
     for basectx in basectxs:
         basepath = basepaths.get((basectx.rev(), fixctx.rev(), path), path)
 
         if basepath in basectx:
             content1 = basectx[basepath].data()
         else:
             content1 = b''
         rangeslist.extend(difflineranges(content1, content2))
     return unionranges(rangeslist)
 
 
 def getbasepaths(repo, opts, workqueue, basectxs):
     if opts.get(b'whole'):
         # Base paths will never be fetched for line range determination.
         return {}
 
     basepaths = {}
     for srcrev, path, _dstrevs in workqueue:
         fixctx = repo[srcrev]
         for basectx in basectxs[srcrev]:
             basepath = copies.pathcopies(basectx, fixctx).get(path, path)
             if basepath in basectx:
                 basepaths[(basectx.rev(), fixctx.rev(), path)] = basepath
     return basepaths
 
 
 def unionranges(rangeslist):
     """Return the union of some closed intervals
 
     >>> unionranges([])
     []
     >>> unionranges([(1, 100)])
     [(1, 100)]
     >>> unionranges([(1, 100), (1, 100)])
     [(1, 100)]
     >>> unionranges([(1, 100), (2, 100)])
     [(1, 100)]
     >>> unionranges([(1, 99), (1, 100)])
     [(1, 100)]
     >>> unionranges([(1, 100), (40, 60)])
     [(1, 100)]
     >>> unionranges([(1, 49), (50, 100)])
     [(1, 100)]
     >>> unionranges([(1, 48), (50, 100)])
     [(1, 48), (50, 100)]
     >>> unionranges([(1, 2), (3, 4), (5, 6)])
     [(1, 6)]
     """
     rangeslist = sorted(set(rangeslist))
     unioned = []
     if rangeslist:
         unioned, rangeslist = [rangeslist[0]], rangeslist[1:]
     for a, b in rangeslist:
         c, d = unioned[-1]
         if a > d + 1:
             unioned.append((a, b))
         else:
             unioned[-1] = (c, max(b, d))
     return unioned
 
 
 def difflineranges(content1, content2):
     """Return list of line number ranges in content2 that differ from content1.
 
     Line numbers are 1-based. The numbers are the first and last line contained
     in the range. Single-line ranges have the same line number for the first and
     last line. Excludes any empty ranges that result from lines that are only
     present in content1. Relies on mdiff's idea of where the line endings are in
     the string.
 
     >>> from mercurial import pycompat
     >>> lines = lambda s: b'\\n'.join([c for c in pycompat.iterbytestr(s)])
     >>> difflineranges2 = lambda a, b: difflineranges(lines(a), lines(b))
     >>> difflineranges2(b'', b'')
     []
     >>> difflineranges2(b'a', b'')
     []
     >>> difflineranges2(b'', b'A')
     [(1, 1)]
     >>> difflineranges2(b'a', b'a')
     []
     >>> difflineranges2(b'a', b'A')
     [(1, 1)]
     >>> difflineranges2(b'ab', b'')
     []
     >>> difflineranges2(b'', b'AB')
     [(1, 2)]
     >>> difflineranges2(b'abc', b'ac')
     []
     >>> difflineranges2(b'ab', b'aCb')
     [(2, 2)]
     >>> difflineranges2(b'abc', b'aBc')
     [(2, 2)]
     >>> difflineranges2(b'ab', b'AB')
     [(1, 2)]
     >>> difflineranges2(b'abcde', b'aBcDe')
     [(2, 2), (4, 4)]
     >>> difflineranges2(b'abcde', b'aBCDe')
     [(2, 4)]
     """
     ranges = []
     for lines, kind in mdiff.allblocks(content1, content2):
         firstline, lastline = lines[2:4]
         if kind == b'!' and firstline != lastline:
             ranges.append((firstline + 1, lastline))
     return ranges
 
 
 def getbasectxs(repo, opts, revstofix):
     """Returns a map of the base contexts for each revision
 
     The base contexts determine which lines are considered modified when we
     attempt to fix just the modified lines in a file. It also determines which
     files we attempt to fix, so it is important to compute this even when
     --whole is used.
     """
     # The --base flag overrides the usual logic, and we give every revision
     # exactly the set of baserevs that the user specified.
     if opts.get(b'base'):
         baserevs = set(logcmdutil.revrange(repo, opts.get(b'base')))
         if not baserevs:
             baserevs = {nullrev}
         basectxs = {repo[rev] for rev in baserevs}
         return {rev: basectxs for rev in revstofix}
 
     # Proceed in topological order so that we can easily determine each
     # revision's baserevs by looking at its parents and their baserevs.
     basectxs = collections.defaultdict(set)
     for rev in sorted(revstofix):
         ctx = repo[rev]
         for pctx in ctx.parents():
             if pctx.rev() in basectxs:
                 basectxs[rev].update(basectxs[pctx.rev()])
             else:
                 basectxs[rev].add(pctx)
     return basectxs
 
 
 def _prefetchfiles(repo, workqueue, basepaths):
     toprefetch = set()
 
     # Prefetch the files that will be fixed.
     for srcrev, path, _dstrevs in workqueue:
         if srcrev == wdirrev:
             continue
         toprefetch.add((srcrev, path))
 
     # Prefetch the base contents for lineranges().
     for (baserev, fixrev, path), basepath in basepaths.items():
         toprefetch.add((baserev, basepath))
 
     if toprefetch:
         scmutil.prefetchfiles(
             repo,
             [
                 (rev, scmutil.matchfiles(repo, [path]))
                 for rev, path in toprefetch
             ],
         )
 
 
 def fixfile(ui, repo, opts, fixers, fixctx, path, basepaths, basectxs):
     """Run any configured fixers that should affect the file in this context
 
     Returns the file content that results from applying the fixers in some order
     starting with the file's content in the fixctx. Fixers that support line
     ranges will affect lines that have changed relative to any of the basectxs
     (i.e. they will only avoid lines that are common to all basectxs).
 
     A fixer tool's stdout will become the file's new content if and only if it
     exits with code zero. The fixer tool's working directory is the repository's
     root.
     """
     metadata = {}
     newdata = fixctx[path].data()
-    for fixername, fixer in pycompat.iteritems(fixers):
+    for fixername, fixer in fixers.items():
         if fixer.affects(opts, fixctx, path):
             ranges = lineranges(
                 opts, path, basepaths, basectxs, fixctx, newdata
             )
             command = fixer.command(ui, path, ranges)
             if command is None:
                 continue
             ui.debug(b'subprocess: %s\n' % (command,))
             proc = subprocess.Popen(
                 procutil.tonativestr(command),
                 shell=True,
                 cwd=procutil.tonativestr(repo.root),
                 stdin=subprocess.PIPE,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE,
             )
             stdout, stderr = proc.communicate(newdata)
             if stderr:
                 showstderr(ui, fixctx.rev(), fixername, stderr)
             newerdata = stdout
             if fixer.shouldoutputmetadata():
                 try:
                     metadatajson, newerdata = stdout.split(b'\0', 1)
                     metadata[fixername] = pycompat.json_loads(metadatajson)
                 except ValueError:
                     ui.warn(
                         _(b'ignored invalid output from fixer tool: %s\n')
                         % (fixername,)
                     )
                     continue
             else:
                 metadata[fixername] = None
             if proc.returncode == 0:
                 newdata = newerdata
             else:
                 if not stderr:
                     message = _(b'exited with status %d\n') % (proc.returncode,)
                     showstderr(ui, fixctx.rev(), fixername, message)
                 checktoolfailureaction(
                     ui,
                     _(b'no fixes will be applied'),
                     hint=_(
                         b'use --config fix.failure=continue to apply any '
                         b'successful fixes anyway'
                     ),
                 )
     return metadata, newdata
 
 
 def showstderr(ui, rev, fixername, stderr):
     """Writes the lines of the stderr string as warnings on the ui
 
     Uses the revision number and fixername to give more context to each line of
     the error message. Doesn't include file names, since those take up a lot of
     space and would tend to be included in the error message if they were
     relevant.
     """
     for line in re.split(b'[\r\n]+', stderr):
         if line:
             ui.warn(b'[')
             if rev is None:
                 ui.warn(_(b'wdir'), label=b'evolve.rev')
             else:
                 ui.warn(b'%d' % rev, label=b'evolve.rev')
             ui.warn(b'] %s: %s\n' % (fixername, line))
 
 
 def writeworkingdir(repo, ctx, filedata, replacements):
     """Write new content to the working copy and check out the new p1 if any
 
     We check out a new revision if and only if we fixed something in both the
     working directory and its parent revision. This avoids the need for a full
     update/merge, and means that the working directory simply isn't affected
     unless the --working-dir flag is given.
 
     Directly updates the dirstate for the affected files.
     """
-    for path, data in pycompat.iteritems(filedata):
+    for path, data in filedata.items():
         fctx = ctx[path]
         fctx.write(data, fctx.flags())
 
     oldp1 = repo.dirstate.p1()
     newp1 = replacements.get(oldp1, oldp1)
     if newp1 != oldp1:
         assert repo.dirstate.p2() == nullid
         with repo.dirstate.parentchange():
             scmutil.movedirstate(repo, repo[newp1])
 
 
 def replacerev(ui, repo, ctx, filedata, replacements):
     """Commit a new revision like the given one, but with file content changes
 
     "ctx" is the original revision to be replaced by a modified one.
 
     "filedata" is a dict that maps paths to their new file content. All other
     paths will be recreated from the original revision without changes.
     "filedata" may contain paths that didn't exist in the original revision;
     they will be added.
 
     "replacements" is a dict that maps a single node to a single node, and it is
     updated to indicate the original revision is replaced by the newly created
     one. No entry is added if the replacement's node already exists.
 
     The new revision has the same parents as the old one, unless those parents
     have already been replaced, in which case those replacements are the parents
     of this new revision. Thus, if revisions are replaced in topological order,
     there is no need to rebase them into the original topology later.
     """
 
     p1rev, p2rev = repo.changelog.parentrevs(ctx.rev())
     p1ctx, p2ctx = repo[p1rev], repo[p2rev]
     newp1node = replacements.get(p1ctx.node(), p1ctx.node())
     newp2node = replacements.get(p2ctx.node(), p2ctx.node())
 
     # We don't want to create a revision that has no changes from the original,
     # but we should if the original revision's parent has been replaced.
     # Otherwise, we would produce an orphan that needs no actual human
     # intervention to evolve. We can't rely on commit() to avoid creating the
     # un-needed revision because the extra field added below produces a new hash
     # regardless of file content changes.
     if (
         not filedata
         and p1ctx.node() not in replacements
         and p2ctx.node() not in replacements
     ):
         return
 
     extra = ctx.extra().copy()
     extra[b'fix_source'] = ctx.hex()
 
     wctx = context.overlayworkingctx(repo)
     wctx.setbase(repo[newp1node])
     merge.revert_to(ctx, wc=wctx)
     copies.graftcopies(wctx, ctx, ctx.p1())
 
     for path in filedata.keys():
         fctx = ctx[path]
         copysource = fctx.copysource()
         wctx.write(path, filedata[path], flags=fctx.flags())
         if copysource:
             wctx.markcopied(path, copysource)
 
     desc = rewriteutil.update_hash_refs(
         repo,
         ctx.description(),
         {oldnode: [newnode] for oldnode, newnode in replacements.items()},
     )
 
     memctx = wctx.tomemctx(
         text=desc,
         branch=ctx.branch(),
         extra=extra,
         date=ctx.date(),
         parents=(newp1node, newp2node),
         user=ctx.user(),
     )
 
     sucnode = memctx.commit()
     prenode = ctx.node()
     if prenode == sucnode:
         ui.debug(b'node %s already existed\n' % (ctx.hex()))
     else:
         replacements[ctx.node()] = sucnode
 
 
 def getfixers(ui):
     """Returns a map of configured fixer tools indexed by their names
 
     Each value is a Fixer object with methods that implement the behavior of the
     fixer's config suboptions. Does not validate the config values.
     """
     fixers = {}
     for name in fixernames(ui):
         enabled = ui.configbool(b'fix', name + b':enabled')
         command = ui.config(b'fix', name + b':command')
         pattern = ui.config(b'fix', name + b':pattern')
         linerange = ui.config(b'fix', name + b':linerange')
         priority = ui.configint(b'fix', name + b':priority')
         metadata = ui.configbool(b'fix', name + b':metadata')
         skipclean = ui.configbool(b'fix', name + b':skipclean')
         # Don't use a fixer if it has no pattern configured. It would be
         # dangerous to let it affect all files. It would be pointless to let it
         # affect no files. There is no reasonable subset of files to use as the
         # default.
         if command is None:
             ui.warn(
                 _(b'fixer tool has no command configuration: %s\n') % (name,)
             )
         elif pattern is None:
             ui.warn(
                 _(b'fixer tool has no pattern configuration: %s\n') % (name,)
             )
         elif not enabled:
             ui.debug(b'ignoring disabled fixer tool: %s\n' % (name,))
         else:
             fixers[name] = Fixer(
                 command, pattern, linerange, priority, metadata, skipclean
             )
     return collections.OrderedDict(
         sorted(fixers.items(), key=lambda item: item[1]._priority, reverse=True)
     )
 
 
 def fixernames(ui):
     """Returns the names of [fix] config options that have suboptions"""
     names = set()
     for k, v in ui.configitems(b'fix'):
         if b':' in k:
             names.add(k.split(b':', 1)[0])
     return names
 
 
 class Fixer(object):
     """Wraps the raw config values for a fixer with methods"""
 
     def __init__(
         self, command, pattern, linerange, priority, metadata, skipclean
     ):
         self._command = command
         self._pattern = pattern
         self._linerange = linerange
         self._priority = priority
         self._metadata = metadata
         self._skipclean = skipclean
 
     def affects(self, opts, fixctx, path):
         """Should this fixer run on the file at the given path and context?"""
         repo = fixctx.repo()
         matcher = matchmod.match(
             repo.root, repo.root, [self._pattern], ctx=fixctx
         )
         return matcher(path)
 
     def shouldoutputmetadata(self):
         """Should the stdout of this fixer start with JSON and a null byte?"""
         return self._metadata
 
     def command(self, ui, path, ranges):
         """A shell command to use to invoke this fixer on the given file/lines
 
         May return None if there is no appropriate command to run for the given
         parameters.
         """
         expand = cmdutil.rendercommandtemplate
         parts = [
             expand(
                 ui,
                 self._command,
                 {b'rootpath': path, b'basename': os.path.basename(path)},
             )
         ]
         if self._linerange:
             if self._skipclean and not ranges:
                 # No line ranges to fix, so don't run the fixer.
                 return None
             for first, last in ranges:
                 parts.append(
                     expand(
                         ui, self._linerange, {b'first': first, b'last': last}
                     )
                 )
         return b' '.join(parts)

hgext/fsmonitor/__init__.py

@@ -1,1004 +1,1000 b''
 # __init__.py - fsmonitor initialization and overrides
 #
 # Copyright 2013-2016 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''Faster status operations with the Watchman file monitor (EXPERIMENTAL)
 
 Integrates the file-watching program Watchman with Mercurial to produce faster
 status results.
 
 On a particular Linux system, for a real-world repository with over 400,000
 files hosted on ext4, vanilla `hg status` takes 1.3 seconds. On the same
 system, with fsmonitor it takes about 0.3 seconds.
 
 fsmonitor requires no configuration -- it will tell Watchman about your
 repository as necessary. You'll need to install Watchman from
 https://facebook.github.io/watchman/ and make sure it is in your PATH.
 
 fsmonitor is incompatible with the largefiles and eol extensions, and
 will disable itself if any of those are active.
 
 The following configuration options exist:
 
 ::
 
     [fsmonitor]
     mode = {off, on, paranoid}
 
 When `mode = off`, fsmonitor will disable itself (similar to not loading the
 extension at all). When `mode = on`, fsmonitor will be enabled (the default).
 When `mode = paranoid`, fsmonitor will query both Watchman and the filesystem,
 and ensure that the results are consistent.
 
 ::
 
     [fsmonitor]
     timeout = (float)
 
 A value, in seconds, that determines how long fsmonitor will wait for Watchman
 to return results. Defaults to `2.0`.
 
 ::
 
     [fsmonitor]
     blacklistusers = (list of userids)
 
 A list of usernames for which fsmonitor will disable itself altogether.
 
 ::
 
     [fsmonitor]
     walk_on_invalidate = (boolean)
 
 Whether or not to walk the whole repo ourselves when our cached state has been
 invalidated, for example when Watchman has been restarted or .hgignore rules
 have been changed. Walking the repo in that case can result in competing for
 I/O with Watchman. For large repos it is recommended to set this value to
 false. You may wish to set this to true if you have a very fast filesystem
 that can outpace the IPC overhead of getting the result data for the full repo
 from Watchman. Defaults to false.
 
 ::
 
     [fsmonitor]
     warn_when_unused = (boolean)
 
 Whether to print a warning during certain operations when fsmonitor would be
 beneficial to performance but isn't enabled.
 
 ::
 
     [fsmonitor]
     warn_update_file_count = (integer)
     # or when mercurial is built with rust support
     warn_update_file_count_rust = (integer)
 
 If ``warn_when_unused`` is set and fsmonitor isn't enabled, a warning will
 be printed during working directory updates if this many files will be
 created.
 '''
 
 # Platforms Supported
 # ===================
 #
 # **Linux:** *Stable*. Watchman and fsmonitor are both known to work reliably,
 #   even under severe loads.
 #
 # **Mac OS X:** *Stable*. The Mercurial test suite passes with fsmonitor
 #   turned on, on case-insensitive HFS+. There has been a reasonable amount of
 #   user testing under normal loads.
 #
 # **Solaris, BSD:** *Alpha*. watchman and fsmonitor are believed to work, but
 #   very little testing has been done.
 #
 # **Windows:** *Alpha*. Not in a release version of watchman or fsmonitor yet.
 #
 # Known Issues
 # ============
 #
 # * fsmonitor will disable itself if any of the following extensions are
 #   enabled: largefiles, inotify, eol; or if the repository has subrepos.
 # * fsmonitor will produce incorrect results if nested repos that are not
 #   subrepos exist. *Workaround*: add nested repo paths to your `.hgignore`.
 #
 # The issues related to nested repos and subrepos are probably not fundamental
 # ones. Patches to fix them are welcome.
 
 
 import codecs
 import os
 import stat
 import sys
 import tempfile
 import weakref
 
 from mercurial.i18n import _
 from mercurial.node import hex
 from mercurial.pycompat import open
 from mercurial import (
     context,
     encoding,
     error,
     extensions,
     localrepo,
     merge,
     pathutil,
     pycompat,
     registrar,
     scmutil,
     util,
 )
 from mercurial import match as matchmod
 from mercurial.utils import (
     hashutil,
     stringutil,
 )
 
 from . import (
     pywatchman,
     state,
     watchmanclient,
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 configitem(
     b'fsmonitor',
     b'mode',
     default=b'on',
 )
 configitem(
     b'fsmonitor',
     b'walk_on_invalidate',
     default=False,
 )
 configitem(
     b'fsmonitor',
     b'timeout',
     default=b'2',
 )
 configitem(
     b'fsmonitor',
     b'blacklistusers',
     default=list,
 )
 configitem(
     b'fsmonitor',
     b'watchman_exe',
     default=b'watchman',
 )
 configitem(
     b'fsmonitor',
     b'verbose',
     default=True,
     experimental=True,
 )
 configitem(
     b'experimental',
     b'fsmonitor.transaction_notify',
     default=False,
 )
 
 # This extension is incompatible with the following blacklisted extensions
 # and will disable itself when encountering one of these:
 _blacklist = [b'largefiles', b'eol']
 
 
 def debuginstall(ui, fm):
     fm.write(
         b"fsmonitor-watchman",
         _(b"fsmonitor checking for watchman binary... (%s)\n"),
         ui.configpath(b"fsmonitor", b"watchman_exe"),
     )
     root = tempfile.mkdtemp()
     c = watchmanclient.client(ui, root)
     err = None
     try:
         v = c.command(b"version")
         fm.write(
             b"fsmonitor-watchman-version",
             _(b" watchman binary version %s\n"),
             pycompat.bytestr(v["version"]),
         )
     except watchmanclient.Unavailable as e:
         err = stringutil.forcebytestr(e)
     fm.condwrite(
         err,
         b"fsmonitor-watchman-error",
         _(b" watchman binary missing or broken: %s\n"),
         err,
     )
     return 1 if err else 0
 
 
 def _handleunavailable(ui, state, ex):
     """Exception handler for Watchman interaction exceptions"""
     if isinstance(ex, watchmanclient.Unavailable):
         # experimental config: fsmonitor.verbose
         if ex.warn and ui.configbool(b'fsmonitor', b'verbose'):
             if b'illegal_fstypes' not in stringutil.forcebytestr(ex):
                 ui.warn(stringutil.forcebytestr(ex) + b'\n')
         if ex.invalidate:
             state.invalidate()
         # experimental config: fsmonitor.verbose
         if ui.configbool(b'fsmonitor', b'verbose'):
             ui.log(
                 b'fsmonitor',
                 b'Watchman unavailable: %s\n',
                 stringutil.forcebytestr(ex.msg),
             )
     else:
         ui.log(
             b'fsmonitor',
             b'Watchman exception: %s\n',
             stringutil.forcebytestr(ex),
         )
 
 
 def _hashignore(ignore):
     """Calculate hash for ignore patterns and filenames
 
     If this information changes between Mercurial invocations, we can't
     rely on Watchman information anymore and have to re-scan the working
     copy.
 
     """
     sha1 = hashutil.sha1()
     sha1.update(pycompat.byterepr(ignore))
     return pycompat.sysbytes(sha1.hexdigest())
 
 
 _watchmanencoding = pywatchman.encoding.get_local_encoding()
 _fsencoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
 _fixencoding = codecs.lookup(_watchmanencoding) != codecs.lookup(_fsencoding)
 
 
 def _watchmantofsencoding(path):
     """Fix path to match watchman and local filesystem encoding
 
     watchman's paths encoding can differ from filesystem encoding. For example,
     on Windows, it's always utf-8.
     """
     try:
         decoded = path.decode(_watchmanencoding)
     except UnicodeDecodeError as e:
         raise error.Abort(
             stringutil.forcebytestr(e), hint=b'watchman encoding error'
         )
 
     try:
         encoded = decoded.encode(_fsencoding, 'strict')
     except UnicodeEncodeError as e:
         raise error.Abort(stringutil.forcebytestr(e))
 
     return encoded
 
 
 def overridewalk(orig, self, match, subrepos, unknown, ignored, full=True):
     """Replacement for dirstate.walk, hooking into Watchman.
 
     Whenever full is False, ignored is False, and the Watchman client is
     available, use Watchman combined with saved state to possibly return only a
     subset of files."""
 
     def bail(reason):
         self._ui.debug(b'fsmonitor: fallback to core status, %s\n' % reason)
         return orig(match, subrepos, unknown, ignored, full=True)
 
     if full:
         return bail(b'full rewalk requested')
     if ignored:
         return bail(b'listing ignored files')
     if not self._watchmanclient.available():
         return bail(b'client unavailable')
     state = self._fsmonitorstate
     clock, ignorehash, notefiles = state.get()
     if not clock:
         if state.walk_on_invalidate:
             return bail(b'no clock')
         # Initial NULL clock value, see
         # https://facebook.github.io/watchman/docs/clockspec.html
         clock = b'c:0:0'
         notefiles = []
 
     ignore = self._ignore
     dirignore = self._dirignore
     if unknown:
         if _hashignore(ignore) != ignorehash and clock != b'c:0:0':
             # ignore list changed -- can't rely on Watchman state any more
             if state.walk_on_invalidate:
                 return bail(b'ignore rules changed')
             notefiles = []
             clock = b'c:0:0'
     else:
         # always ignore
         ignore = util.always
         dirignore = util.always
 
     matchfn = match.matchfn
     matchalways = match.always()
     dmap = self._map
     if util.safehasattr(dmap, b'_map'):
         # for better performance, directly access the inner dirstate map if the
         # standard dirstate implementation is in use.
         dmap = dmap._map
     nonnormalset = {
         f
         for f, e in self._map.items()
         if e.v1_state() != b"n" or e.v1_mtime() == -1
     }
 
     copymap = self._map.copymap
     getkind = stat.S_IFMT
     dirkind = stat.S_IFDIR
     regkind = stat.S_IFREG
     lnkkind = stat.S_IFLNK
     join = self._join
     normcase = util.normcase
     fresh_instance = False
 
     exact = skipstep3 = False
     if match.isexact():  # match.exact
         exact = True
         dirignore = util.always  # skip step 2
     elif match.prefix():  # match.match, no patterns
         skipstep3 = True
 
     if not exact and self._checkcase:
         # note that even though we could receive directory entries, we're only
         # interested in checking if a file with the same name exists. So only
         # normalize files if possible.
         normalize = self._normalizefile
         skipstep3 = False
     else:
         normalize = None
 
     # step 1: find all explicit files
     results, work, dirsnotfound = self._walkexplicit(match, subrepos)
 
     skipstep3 = skipstep3 and not (work or dirsnotfound)
     work = [d for d in work if not dirignore(d[0])]
 
     if not work and (exact or skipstep3):
         for s in subrepos:
             del results[s]
         del results[b'.hg']
         return results
 
     # step 2: query Watchman
     try:
         # Use the user-configured timeout for the query.
         # Add a little slack over the top of the user query to allow for
         # overheads while transferring the data
         self._watchmanclient.settimeout(state.timeout + 0.1)
         result = self._watchmanclient.command(
             b'query',
             {
                 b'fields': [b'mode', b'mtime', b'size', b'exists', b'name'],
                 b'since': clock,
                 b'expression': [
                     b'not',
                     [
                         b'anyof',
                         [b'dirname', b'.hg'],
                         [b'name', b'.hg', b'wholename'],
                     ],
                 ],
                 b'sync_timeout': int(state.timeout * 1000),
                 b'empty_on_fresh_instance': state.walk_on_invalidate,
             },
         )
     except Exception as ex:
         _handleunavailable(self._ui, state, ex)
         self._watchmanclient.clearconnection()
         return bail(b'exception during run')
     else:
         # We need to propagate the last observed clock up so that we
         # can use it for our next query
         state.setlastclock(pycompat.sysbytes(result[b'clock']))
         if result[b'is_fresh_instance']:
             if state.walk_on_invalidate:
                 state.invalidate()
                 return bail(b'fresh instance')
             fresh_instance = True
             # Ignore any prior noteable files from the state info
             notefiles = []
 
     # for file paths which require normalization and we encounter a case
     # collision, we store our own foldmap
     if normalize:
         foldmap = {normcase(k): k for k in results}
 
     switch_slashes = pycompat.ossep == b'\\'
     # The order of the results is, strictly speaking, undefined.
     # For case changes on a case insensitive filesystem we may receive
     # two entries, one with exists=True and another with exists=False.
     # The exists=True entries in the same response should be interpreted
     # as being happens-after the exists=False entries due to the way that
     # Watchman tracks files.  We use this property to reconcile deletes
     # for name case changes.
     for entry in result[b'files']:
         fname = entry[b'name']
 
         # Watchman always give us a str. Normalize to bytes on Python 3
         # using Watchman's encoding, if needed.
         if not isinstance(fname, bytes):
             fname = fname.encode(_watchmanencoding)
 
         if _fixencoding:
             fname = _watchmantofsencoding(fname)
 
         if switch_slashes:
             fname = fname.replace(b'\\', b'/')
         if normalize:
             normed = normcase(fname)
             fname = normalize(fname, True, True)
             foldmap[normed] = fname
         fmode = entry[b'mode']
         fexists = entry[b'exists']
         kind = getkind(fmode)
 
         if b'/.hg/' in fname or fname.endswith(b'/.hg'):
             return bail(b'nested-repo-detected')
 
         if not fexists:
             # if marked as deleted and we don't already have a change
             # record, mark it as deleted.  If we already have an entry
             # for fname then it was either part of walkexplicit or was
             # an earlier result that was a case change
             if (
                 fname not in results
                 and fname in dmap
                 and (matchalways or matchfn(fname))
             ):
                 results[fname] = None
         elif kind == dirkind:
             if fname in dmap and (matchalways or matchfn(fname)):
                 results[fname] = None
         elif kind == regkind or kind == lnkkind:
             if fname in dmap:
                 if matchalways or matchfn(fname):
                     results[fname] = entry
             elif (matchalways or matchfn(fname)) and not ignore(fname):
                 results[fname] = entry
         elif fname in dmap and (matchalways or matchfn(fname)):
             results[fname] = None
 
     # step 3: query notable files we don't already know about
     # XXX try not to iterate over the entire dmap
     if normalize:
         # any notable files that have changed case will already be handled
         # above, so just check membership in the foldmap
         notefiles = {
             normalize(f, True, True)
             for f in notefiles
             if normcase(f) not in foldmap
         }
     visit = {
         f
         for f in notefiles
         if (f not in results and matchfn(f) and (f in dmap or not ignore(f)))
     }
 
     if not fresh_instance:
         if matchalways:
             visit.update(f for f in nonnormalset if f not in results)
             visit.update(f for f in copymap if f not in results)
         else:
             visit.update(
                 f for f in nonnormalset if f not in results and matchfn(f)
             )
             visit.update(f for f in copymap if f not in results and matchfn(f))
     else:
         if matchalways:
-            visit.update(
-                f for f, st in pycompat.iteritems(dmap) if f not in results
-            )
+            visit.update(f for f, st in dmap.items() if f not in results)
             visit.update(f for f in copymap if f not in results)
         else:
             visit.update(
-                f
-                for f, st in pycompat.iteritems(dmap)
-                if f not in results and matchfn(f)
+                f for f, st in dmap.items() if f not in results and matchfn(f)
             )
             visit.update(f for f in copymap if f not in results and matchfn(f))
 
     audit = pathutil.pathauditor(self._root, cached=True).check
     auditpass = [f for f in visit if audit(f)]
     auditpass.sort()
     auditfail = visit.difference(auditpass)
     for f in auditfail:
         results[f] = None
 
     nf = iter(auditpass)
     for st in util.statfiles([join(f) for f in auditpass]):
         f = next(nf)
         if st or f in dmap:
             results[f] = st
 
     for s in subrepos:
         del results[s]
     del results[b'.hg']
     return results
 
 
 def overridestatus(
     orig,
     self,
     node1=b'.',
     node2=None,
     match=None,
     ignored=False,
     clean=False,
     unknown=False,
     listsubrepos=False,
 ):
     listignored = ignored
     listclean = clean
     listunknown = unknown
 
     def _cmpsets(l1, l2):
         try:
             if b'FSMONITOR_LOG_FILE' in encoding.environ:
                 fn = encoding.environ[b'FSMONITOR_LOG_FILE']
                 f = open(fn, b'wb')
             else:
                 fn = b'fsmonitorfail.log'
                 f = self.vfs.open(fn, b'wb')
         except (IOError, OSError):
             self.ui.warn(_(b'warning: unable to write to %s\n') % fn)
             return
 
         try:
             for i, (s1, s2) in enumerate(zip(l1, l2)):
                 if set(s1) != set(s2):
                     f.write(b'sets at position %d are unequal\n' % i)
                     f.write(b'watchman returned: %r\n' % s1)
                     f.write(b'stat returned: %r\n' % s2)
         finally:
             f.close()
 
     if isinstance(node1, context.changectx):
         ctx1 = node1
     else:
         ctx1 = self[node1]
     if isinstance(node2, context.changectx):
         ctx2 = node2
     else:
         ctx2 = self[node2]
 
     working = ctx2.rev() is None
     parentworking = working and ctx1 == self[b'.']
     match = match or matchmod.always()
 
     # Maybe we can use this opportunity to update Watchman's state.
     # Mercurial uses workingcommitctx and/or memctx to represent the part of
     # the workingctx that is to be committed. So don't update the state in
     # that case.
     # HG_PENDING is set in the environment when the dirstate is being updated
     # in the middle of a transaction; we must not update our state in that
     # case, or we risk forgetting about changes in the working copy.
     updatestate = (
         parentworking
         and match.always()
         and not isinstance(ctx2, (context.workingcommitctx, context.memctx))
         and b'HG_PENDING' not in encoding.environ
     )
 
     try:
         if self._fsmonitorstate.walk_on_invalidate:
             # Use a short timeout to query the current clock.  If that
             # takes too long then we assume that the service will be slow
             # to answer our query.
             # walk_on_invalidate indicates that we prefer to walk the
             # tree ourselves because we can ignore portions that Watchman
             # cannot and we tend to be faster in the warmer buffer cache
             # cases.
             self._watchmanclient.settimeout(0.1)
         else:
             # Give Watchman more time to potentially complete its walk
             # and return the initial clock.  In this mode we assume that
             # the filesystem will be slower than parsing a potentially
             # very large Watchman result set.
             self._watchmanclient.settimeout(self._fsmonitorstate.timeout + 0.1)
         startclock = self._watchmanclient.getcurrentclock()
     except Exception as ex:
         self._watchmanclient.clearconnection()
         _handleunavailable(self.ui, self._fsmonitorstate, ex)
         # boo, Watchman failed. bail
         return orig(
             node1,
             node2,
             match,
             listignored,
             listclean,
             listunknown,
             listsubrepos,
         )
 
     if updatestate:
         # We need info about unknown files. This may make things slower the
         # first time, but whatever.
         stateunknown = True
     else:
         stateunknown = listunknown
 
     if updatestate:
         ps = poststatus(startclock)
         self.addpostdsstatus(ps)
 
     r = orig(
         node1, node2, match, listignored, listclean, stateunknown, listsubrepos
     )
     modified, added, removed, deleted, unknown, ignored, clean = r
 
     if not listunknown:
         unknown = []
 
     # don't do paranoid checks if we're not going to query Watchman anyway
     full = listclean or match.traversedir is not None
     if self._fsmonitorstate.mode == b'paranoid' and not full:
         # run status again and fall back to the old walk this time
         self.dirstate._fsmonitordisable = True
 
         # shut the UI up
         quiet = self.ui.quiet
         self.ui.quiet = True
         fout, ferr = self.ui.fout, self.ui.ferr
         self.ui.fout = self.ui.ferr = open(os.devnull, b'wb')
 
         try:
             rv2 = orig(
                 node1,
                 node2,
                 match,
                 listignored,
                 listclean,
                 listunknown,
                 listsubrepos,
             )
         finally:
             self.dirstate._fsmonitordisable = False
             self.ui.quiet = quiet
             self.ui.fout, self.ui.ferr = fout, ferr
 
         # clean isn't tested since it's set to True above
         with self.wlock():
             _cmpsets(
                 [modified, added, removed, deleted, unknown, ignored, clean],
                 rv2,
             )
         modified, added, removed, deleted, unknown, ignored, clean = rv2
 
     return scmutil.status(
         modified, added, removed, deleted, unknown, ignored, clean
     )
 
 
 class poststatus(object):
     def __init__(self, startclock):
         self._startclock = pycompat.sysbytes(startclock)
 
     def __call__(self, wctx, status):
         clock = wctx.repo()._fsmonitorstate.getlastclock() or self._startclock
         hashignore = _hashignore(wctx.repo().dirstate._ignore)
         notefiles = (
             status.modified
             + status.added
             + status.removed
             + status.deleted
             + status.unknown
         )
         wctx.repo()._fsmonitorstate.set(clock, hashignore, notefiles)
 
 
 def makedirstate(repo, dirstate):
     class fsmonitordirstate(dirstate.__class__):
         def _fsmonitorinit(self, repo):
             # _fsmonitordisable is used in paranoid mode
             self._fsmonitordisable = False
             self._fsmonitorstate = repo._fsmonitorstate
             self._watchmanclient = repo._watchmanclient
             self._repo = weakref.proxy(repo)
 
         def walk(self, *args, **kwargs):
             orig = super(fsmonitordirstate, self).walk
             if self._fsmonitordisable:
                 return orig(*args, **kwargs)
             return overridewalk(orig, self, *args, **kwargs)
 
         def rebuild(self, *args, **kwargs):
             self._fsmonitorstate.invalidate()
             return super(fsmonitordirstate, self).rebuild(*args, **kwargs)
 
         def invalidate(self, *args, **kwargs):
             self._fsmonitorstate.invalidate()
             return super(fsmonitordirstate, self).invalidate(*args, **kwargs)
 
     dirstate.__class__ = fsmonitordirstate
     dirstate._fsmonitorinit(repo)
 
 
 def wrapdirstate(orig, self):
     ds = orig(self)
     # only override the dirstate when Watchman is available for the repo
     if util.safehasattr(self, b'_fsmonitorstate'):
         makedirstate(self, ds)
     return ds
 
 
 def extsetup(ui):
     extensions.wrapfilecache(
         localrepo.localrepository, b'dirstate', wrapdirstate
     )
     if pycompat.isdarwin:
         # An assist for avoiding the dangling-symlink fsevents bug
         extensions.wrapfunction(os, b'symlink', wrapsymlink)
 
     extensions.wrapfunction(merge, b'_update', wrapupdate)
 
 
 def wrapsymlink(orig, source, link_name):
     """if we create a dangling symlink, also touch the parent dir
     to encourage fsevents notifications to work more correctly"""
     try:
         return orig(source, link_name)
     finally:
         try:
             os.utime(os.path.dirname(link_name), None)
         except OSError:
             pass
 
 
 class state_update(object):
     """This context manager is responsible for dispatching the state-enter
     and state-leave signals to the watchman service. The enter and leave
     methods can be invoked manually (for scenarios where context manager
     semantics are not possible). If parameters oldnode and newnode are None,
     they will be populated based on current working copy in enter and
     leave, respectively. Similarly, if the distance is none, it will be
     calculated based on the oldnode and newnode in the leave method."""
 
     def __init__(
         self,
         repo,
         name,
         oldnode=None,
         newnode=None,
         distance=None,
         partial=False,
     ):
         self.repo = repo.unfiltered()
         self.name = name
         self.oldnode = oldnode
         self.newnode = newnode
         self.distance = distance
         self.partial = partial
         self._lock = None
         self.need_leave = False
 
     def __enter__(self):
         self.enter()
 
     def enter(self):
         # Make sure we have a wlock prior to sending notifications to watchman.
         # We don't want to race with other actors. In the update case,
         # merge.update is going to take the wlock almost immediately. We are
         # effectively extending the lock around several short sanity checks.
         if self.oldnode is None:
             self.oldnode = self.repo[b'.'].node()
 
         if self.repo.currentwlock() is None:
             if util.safehasattr(self.repo, b'wlocknostateupdate'):
                 self._lock = self.repo.wlocknostateupdate()
             else:
                 self._lock = self.repo.wlock()
         self.need_leave = self._state(b'state-enter', hex(self.oldnode))
         return self
 
     def __exit__(self, type_, value, tb):
         abort = True if type_ else False
         self.exit(abort=abort)
 
     def exit(self, abort=False):
         try:
             if self.need_leave:
                 status = b'failed' if abort else b'ok'
                 if self.newnode is None:
                     self.newnode = self.repo[b'.'].node()
                 if self.distance is None:
                     self.distance = calcdistance(
                         self.repo, self.oldnode, self.newnode
                     )
                 self._state(b'state-leave', hex(self.newnode), status=status)
         finally:
             self.need_leave = False
             if self._lock:
                 self._lock.release()
 
     def _state(self, cmd, commithash, status=b'ok'):
         if not util.safehasattr(self.repo, b'_watchmanclient'):
             return False
         try:
             self.repo._watchmanclient.command(
                 cmd,
                 {
                     b'name': self.name,
                     b'metadata': {
                         # the target revision
                         b'rev': commithash,
                         # approximate number of commits between current and target
                         b'distance': self.distance if self.distance else 0,
                         # success/failure (only really meaningful for state-leave)
                         b'status': status,
                         # whether the working copy parent is changing
                         b'partial': self.partial,
                     },
                 },
             )
             return True
         except Exception as e:
             # Swallow any errors; fire and forget
             self.repo.ui.log(
                 b'watchman', b'Exception %s while running %s\n', e, cmd
             )
             return False
 
 
 # Estimate the distance between two nodes
 def calcdistance(repo, oldnode, newnode):
     anc = repo.changelog.ancestor(oldnode, newnode)
     ancrev = repo[anc].rev()
     distance = abs(repo[oldnode].rev() - ancrev) + abs(
         repo[newnode].rev() - ancrev
     )
     return distance
 
 
 # Bracket working copy updates with calls to the watchman state-enter
 # and state-leave commands.  This allows clients to perform more intelligent
 # settling during bulk file change scenarios
 # https://facebook.github.io/watchman/docs/cmd/subscribe.html#advanced-settling
 def wrapupdate(
     orig,
     repo,
     node,
     branchmerge,
     force,
     ancestor=None,
     mergeancestor=False,
     labels=None,
     matcher=None,
     **kwargs
 ):
 
     distance = 0
     partial = True
     oldnode = repo[b'.'].node()
     newnode = repo[node].node()
     if matcher is None or matcher.always():
         partial = False
         distance = calcdistance(repo.unfiltered(), oldnode, newnode)
 
     with state_update(
         repo,
         name=b"hg.update",
         oldnode=oldnode,
         newnode=newnode,
         distance=distance,
         partial=partial,
     ):
         return orig(
             repo,
             node,
             branchmerge,
             force,
             ancestor,
             mergeancestor,
             labels,
             matcher,
             **kwargs
         )
 
 
 def repo_has_depth_one_nested_repo(repo):
     for f in repo.wvfs.listdir():
         if os.path.isdir(os.path.join(repo.root, f, b'.hg')):
             msg = b'fsmonitor: sub-repository %r detected, fsmonitor disabled\n'
             repo.ui.debug(msg % f)
             return True
     return False
 
 
 def reposetup(ui, repo):
     # We don't work with largefiles or inotify
     exts = extensions.enabled()
     for ext in _blacklist:
         if ext in exts:
             ui.warn(
                 _(
                     b'The fsmonitor extension is incompatible with the %s '
                     b'extension and has been disabled.\n'
                 )
                 % ext
             )
             return
 
     if repo.local():
         # We don't work with subrepos either.
         #
         # if repo[None].substate can cause a dirstate parse, which is too
         # slow. Instead, look for a file called hgsubstate,
         if repo.wvfs.exists(b'.hgsubstate') or repo.wvfs.exists(b'.hgsub'):
             return
 
         if repo_has_depth_one_nested_repo(repo):
             return
 
         fsmonitorstate = state.state(repo)
         if fsmonitorstate.mode == b'off':
             return
 
         try:
             client = watchmanclient.client(repo.ui, repo.root)
         except Exception as ex:
             _handleunavailable(ui, fsmonitorstate, ex)
             return
 
         repo._fsmonitorstate = fsmonitorstate
         repo._watchmanclient = client
 
         dirstate, cached = localrepo.isfilecached(repo, b'dirstate')
         if cached:
             # at this point since fsmonitorstate wasn't present,
             # repo.dirstate is not a fsmonitordirstate
             makedirstate(repo, dirstate)
 
         class fsmonitorrepo(repo.__class__):
             def status(self, *args, **kwargs):
                 orig = super(fsmonitorrepo, self).status
                 return overridestatus(orig, self, *args, **kwargs)
 
             def wlocknostateupdate(self, *args, **kwargs):
                 return super(fsmonitorrepo, self).wlock(*args, **kwargs)
 
             def wlock(self, *args, **kwargs):
                 l = super(fsmonitorrepo, self).wlock(*args, **kwargs)
                 if not ui.configbool(
                     b"experimental", b"fsmonitor.transaction_notify"
                 ):
                     return l
                 if l.held != 1:
                     return l
                 origrelease = l.releasefn
 
                 def staterelease():
                     if origrelease:
                         origrelease()
                     if l.stateupdate:
                         l.stateupdate.exit()
                         l.stateupdate = None
 
                 try:
                     l.stateupdate = None
                     l.stateupdate = state_update(self, name=b"hg.transaction")
                     l.stateupdate.enter()
                     l.releasefn = staterelease
                 except Exception as e:
                     # Swallow any errors; fire and forget
                     self.ui.log(
                         b'watchman', b'Exception in state update %s\n', e
                     )
                 return l
 
         repo.__class__ = fsmonitorrepo

hgext/githelp.py

@@ -1,1269 +1,1269 b''
 # githelp.py - Try to map Git commands to Mercurial equivalents.
 #
 # Copyright 2013 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 """try mapping git commands to Mercurial commands
 
 Tries to map a given git command to a Mercurial command:
 
   $ hg githelp -- git checkout master
   hg update master
 
 If an unknown command or parameter combination is detected, an error is
 produced.
 """
 
 
 import getopt
 import re
 
 from mercurial.i18n import _
 from mercurial import (
     encoding,
     error,
     fancyopts,
     pycompat,
     registrar,
     scmutil,
 )
 from mercurial.utils import procutil
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 
 def convert(s):
     if s.startswith(b"origin/"):
         return s[7:]
     if b'HEAD' in s:
         s = s.replace(b'HEAD', b'.')
     # HEAD~ in git is .~1 in mercurial
     s = re.sub(b'~$', b'~1', s)
     return s
 
 
 @command(
     b'githelp|git',
     [],
     _(b'hg githelp'),
     helpcategory=command.CATEGORY_HELP,
     helpbasic=True,
 )
 def githelp(ui, repo, *args, **kwargs):
     """suggests the Mercurial equivalent of the given git command
 
     Usage: hg githelp -- <git command>
     """
 
     if len(args) == 0 or (len(args) == 1 and args[0] == b'git'):
         raise error.Abort(
             _(b'missing git command - usage: hg githelp -- <git command>')
         )
 
     if args[0] == b'git':
         args = args[1:]
 
     cmd = args[0]
     if not cmd in gitcommands:
         raise error.Abort(_(b"error: unknown git command %s") % cmd)
 
     ui.pager(b'githelp')
     args = args[1:]
     return gitcommands[cmd](ui, repo, *args, **kwargs)
 
 
 def parseoptions(ui, cmdoptions, args):
     cmdoptions = list(cmdoptions)
     opts = {}
     args = list(args)
     while True:
         try:
             args = fancyopts.fancyopts(list(args), cmdoptions, opts, True)
             break
         except getopt.GetoptError as ex:
             if "requires argument" in ex.msg:
                 raise
             if ('--' + ex.opt) in ex.msg:
                 flag = b'--' + pycompat.bytestr(ex.opt)
             elif ('-' + ex.opt) in ex.msg:
                 flag = b'-' + pycompat.bytestr(ex.opt)
             else:
                 raise error.Abort(
                     _(b"unknown option %s") % pycompat.bytestr(ex.opt)
                 )
             try:
                 args.remove(flag)
             except Exception:
                 msg = _(b"unknown option '%s' packed with other options")
                 hint = _(b"please try passing the option as its own flag: -%s")
                 raise error.Abort(
                     msg % pycompat.bytestr(ex.opt),
                     hint=hint % pycompat.bytestr(ex.opt),
                 )
 
             ui.warn(_(b"ignoring unknown option %s\n") % flag)
 
     args = list([convert(x) for x in args])
     opts = dict(
         [
             (k, convert(v)) if isinstance(v, bytes) else (k, v)
-            for k, v in pycompat.iteritems(opts)
+            for k, v in opts.items()
         ]
     )
 
     return args, opts
 
 
 class Command(object):
     def __init__(self, name):
         self.name = name
         self.args = []
         self.opts = {}
 
     def __bytes__(self):
         cmd = b"hg " + self.name
         if self.opts:
-            for k, values in sorted(pycompat.iteritems(self.opts)):
+            for k, values in sorted(self.opts.items()):
                 for v in values:
                     if v:
                         if isinstance(v, int):
                             fmt = b' %s %d'
                         else:
                             fmt = b' %s %s'
 
                         cmd += fmt % (k, v)
                     else:
                         cmd += b" %s" % (k,)
         if self.args:
             cmd += b" "
             cmd += b" ".join(self.args)
         return cmd
 
     __str__ = encoding.strmethod(__bytes__)
 
     def append(self, value):
         self.args.append(value)
 
     def extend(self, values):
         self.args.extend(values)
 
     def __setitem__(self, key, value):
         values = self.opts.setdefault(key, [])
         values.append(value)
 
     def __and__(self, other):
         return AndCommand(self, other)
 
 
 class AndCommand(object):
     def __init__(self, left, right):
         self.left = left
         self.right = right
 
     def __str__(self):
         return b"%s && %s" % (self.left, self.right)
 
     def __and__(self, other):
         return AndCommand(self, other)
 
 
 def add(ui, repo, *args, **kwargs):
     cmdoptions = [
         (b'A', b'all', None, b''),
         (b'p', b'patch', None, b''),
     ]
     args, opts = parseoptions(ui, cmdoptions, args)
 
     if opts.get(b'patch'):
         ui.status(
             _(
                 b"note: Mercurial will commit when complete, "
                 b"as there is no staging area in Mercurial\n\n"
             )
         )
         cmd = Command(b'commit --interactive')
     else:
         cmd = Command(b"add")
 
         if not opts.get(b'all'):
             cmd.extend(args)
         else:
             ui.status(
                 _(
                     b"note: use hg addremove to remove files that have "
                     b"been deleted\n\n"
                 )
             )
 
     ui.status((bytes(cmd)), b"\n")
 
 
 def am(ui, repo, *args, **kwargs):
     cmdoptions = []
     parseoptions(ui, cmdoptions, args)
     cmd = Command(b'import')
     ui.status(bytes(cmd), b"\n")
 
 
 def apply(ui, repo, *args, **kwargs):
     cmdoptions = [
         (b'p', b'p', int, b''),
         (b'', b'directory', b'', b''),
     ]
     args, opts = parseoptions(ui, cmdoptions, args)
 
     cmd = Command(b'import --no-commit')
     if opts.get(b'p'):
         cmd[b'-p'] = opts.get(b'p')
     if opts.get(b'directory'):
         cmd[b'--prefix'] = opts.get(b'directory')
     cmd.extend(args)
 
     ui.status((bytes(cmd)), b"\n")
 
 
 def bisect(ui, repo, *args, **kwargs):
     ui.status(_(b"see 'hg help bisect' for how to use bisect\n\n"))
 
 
 def blame(ui, repo, *args, **kwargs):
     cmdoptions = []
     args, opts = parseoptions(ui, cmdoptions, args)
     cmd = Command(b'annotate -udl')
     cmd.extend([convert(v) for v in args])
     ui.status((bytes(cmd)), b"\n")
 
 
 def branch(ui, repo, *args, **kwargs):
     cmdoptions = [
         (b'', b'set-upstream', None, b''),
         (b'', b'set-upstream-to', b'', b''),
         (b'd', b'delete', None, b''),
         (b'D', b'delete', None, b''),
         (b'm', b'move', None, b''),
         (b'M', b'move', None, b''),
     ]
     args, opts = parseoptions(ui, cmdoptions, args)
 
     cmd = Command(b"bookmark")
 
     if opts.get(b'set_upstream') or opts.get(b'set_upstream_to'):
         ui.status(_(b"Mercurial has no concept of upstream branches\n"))
         return
     elif opts.get(b'delete'):
         cmd = Command(b"strip")
         for branch in args:
             cmd[b'-B'] = branch
         else:
             cmd[b'-B'] = None
     elif opts.get(b'move'):
         if len(args) > 0:
             if len(args) > 1:
                 old = args.pop(0)
             else:
                 # shell command to output the active bookmark for the active
                 # revision
                 old = b'`hg log -T"{activebookmark}" -r .`'
         else:
             raise error.Abort(_(b'missing newbranch argument'))
         new = args[0]
         cmd[b'-m'] = old
         cmd.append(new)
     else:
         if len(args) > 1:
             cmd[b'-r'] = args[1]
             cmd.append(args[0])
         elif len(args) == 1:
             cmd.append(args[0])
     ui.status((bytes(cmd)), b"\n")
 
 
 def ispath(repo, string):
     """
     The first argument to git checkout can either be a revision or a path. Let's
     generally assume it's a revision, unless it's obviously a path. There are
     too many ways to spell revisions in git for us to reasonably catch all of
     them, so let's be conservative.
     """
     if scmutil.isrevsymbol(repo, string):
         # if it's definitely a revision let's not even check if a file of the
         # same name exists.
         return False
 
     cwd = repo.getcwd()
     if cwd == b'':
         repopath = string
     else:
         repopath = cwd + b'/' + string
 
     exists = repo.wvfs.exists(repopath)
     if exists:
         return True
 
     manifest = repo[b'.'].manifest()
 
     didexist = (repopath in manifest) or manifest.hasdir(repopath)
 
     return didexist
 
 
 def checkout(ui, repo, *args, **kwargs):
     cmdoptions = [
         (b'b', b'branch', b'', b''),
         (b'B', b'branch', b'', b''),
         (b'f', b'force', None, b''),
         (b'p', b'patch', None, b''),
     ]
     paths = []
     if b'--' in args:
         sepindex = args.index(b'--')
         paths.extend(args[sepindex + 1 :])
         args = args[:sepindex]
 
     args, opts = parseoptions(ui, cmdoptions, args)
 
     rev = None
     if args and ispath(repo, args[0]):
         paths = args + paths
     elif args:
         rev = args[0]
         paths = args[1:] + paths
 
     cmd = Command(b'update')
 
     if opts.get(b'force'):
         if paths or rev:
             cmd[b'-C'] = None
 
     if opts.get(b'patch'):
         cmd = Command(b'revert')
         cmd[b'-i'] = None
 
     if opts.get(b'branch'):
         if len(args) == 0:
             cmd = Command(b'bookmark')
             cmd.append(opts.get(b'branch'))
         else:
             cmd.append(args[0])
             bookcmd = Command(b'bookmark')
             bookcmd.append(opts.get(b'branch'))
             cmd = cmd & bookcmd
     # if there is any path argument supplied, use revert instead of update
     elif len(paths) > 0:
         ui.status(_(b"note: use --no-backup to avoid creating .orig files\n\n"))
         cmd = Command(b'revert')
         if opts.get(b'patch'):
             cmd[b'-i'] = None
         if rev:
             cmd[b'-r'] = rev
         cmd.extend(paths)
     elif rev:
         if opts.get(b'patch'):
             cmd[b'-r'] = rev
         else:
             cmd.append(rev)
     elif opts.get(b'force'):
         cmd = Command(b'revert')
         cmd[b'--all'] = None
     else:
         raise error.Abort(_(b"a commit must be specified"))
 
     ui.status((bytes(cmd)), b"\n")
 
 
 def cherrypick(ui, repo, *args, **kwargs):
     cmdoptions = [
         (b'', b'continue', None, b''),
         (b'', b'abort', None, b''),
         (b'e', b'edit', None, b''),
     ]
     args, opts = parseoptions(ui, cmdoptions, args)
 
     cmd = Command(b'graft')
 
     if opts.get(b'edit'):
         cmd[b'--edit'] = None
     if opts.get(b'continue'):
         cmd[b'--continue'] = None
     elif opts.get(b'abort'):
         ui.status(_(b"note: hg graft does not have --abort\n\n"))
         return
     else:
         cmd.extend(args)
 
     ui.status((bytes(cmd)), b"\n")
 
 
 def clean(ui, repo, *args, **kwargs):
     cmdoptions = [
         (b'd', b'd', None, b''),
         (b'f', b'force', None, b''),
         (b'x', b'x', None, b''),
     ]
     args, opts = parseoptions(ui, cmdoptions, args)
 
     cmd = Command(b'purge')
     if opts.get(b'x'):
         cmd[b'--all'] = None
     cmd.extend(args)
 
     ui.status((bytes(cmd)), b"\n")
 
 
 def clone(ui, repo, *args, **kwargs):
     cmdoptions = [
         (b'', b'bare', None, b''),
         (b'n', b'no-checkout', None, b''),
         (b'b', b'branch', b'', b''),
     ]
     args, opts = parseoptions(ui, cmdoptions, args)
 
     if len(args) == 0:
         raise error.Abort(_(b"a repository to clone must be specified"))
 
     cmd = Command(b'clone')
     cmd.append(args[0])
     if len(args) > 1:
         cmd.append(args[1])
 
     if opts.get(b'bare'):
         cmd[b'-U'] = None
         ui.status(
             _(
                 b"note: Mercurial does not have bare clones. "
                 b"-U will clone the repo without checking out a commit\n\n"
             )
         )
     elif opts.get(b'no_checkout'):
         cmd[b'-U'] = None
 
     if opts.get(b'branch'):
         cocmd = Command(b"update")
         cocmd.append(opts.get(b'branch'))
         cmd = cmd & cocmd
 
     ui.status((bytes(cmd)), b"\n")
 
 
 def commit(ui, repo, *args, **kwargs):
     cmdoptions = [
         (b'a', b'all', None, b''),
         (b'm', b'message', b'', b''),
         (b'p', b'patch', None, b''),
         (b'C', b'reuse-message', b'', b''),
         (b'F', b'file', b'', b''),
         (b'', b'author', b'', b''),
         (b'', b'date', b'', b''),
         (b'', b'amend', None, b''),
         (b'', b'no-edit', None, b''),
     ]
     args, opts = parseoptions(ui, cmdoptions, args)
 
     cmd = Command(b'commit')
     if opts.get(b'patch'):
         cmd = Command(b'commit --interactive')
 
     if opts.get(b'amend'):
         if opts.get(b'no_edit'):
             cmd = Command(b'amend')
         else:
             cmd[b'--amend'] = None
 
     if opts.get(b'reuse_message'):
         cmd[b'-M'] = opts.get(b'reuse_message')
 
     if opts.get(b'message'):
         cmd[b'-m'] = b"'%s'" % (opts.get(b'message'),)
 
     if opts.get(b'all'):
         ui.status(
             _(
                 b"note: Mercurial doesn't have a staging area, "
                 b"so there is no --all. -A will add and remove files "
                 b"for you though.\n\n"
             )
         )
 
     if opts.get(b'file'):
         cmd[b'-l'] = opts.get(b'file')
 
     if opts.get(b'author'):
         cmd[b'-u'] = opts.get(b'author')
 
     if opts.get(b'date'):
         cmd[b'-d'] = opts.get(b'date')
 
     cmd.extend(args)
 
     ui.status((bytes(cmd)), b"\n")
 
 
 def deprecated(ui, repo, *args, **kwargs):
     ui.warn(
         _(
             b'this command has been deprecated in the git project, '
             b'thus isn\'t supported by this tool\n\n'
         )
     )
 
 
 def diff(ui, repo, *args, **kwargs):
     cmdoptions = [
         (b'a', b'all', None, b''),
         (b'', b'cached', None, b''),
         (b'R', b'reverse', None, b''),
     ]
     args, opts = parseoptions(ui, cmdoptions, args)
 
     cmd = Command(b'diff')
 
     if opts.get(b'cached'):
         ui.status(
             _(
                 b'note: Mercurial has no concept of a staging area, '
                 b'so --cached does nothing\n\n'
             )
         )
 
     if opts.get(b'reverse'):
         cmd[b'--reverse'] = None
 
     for a in list(args):
         args.remove(a)
         try:
             repo.revs(a)
             cmd[b'-r'] = a
         except Exception:
             cmd.append(a)
 
     ui.status((bytes(cmd)), b"\n")
 
 
 def difftool(ui, repo, *args, **kwargs):
     ui.status(
         _(
             b'Mercurial does not enable external difftool by default. You '
             b'need to enable the extdiff extension in your .hgrc file by adding\n'
             b'extdiff =\n'
             b'to the [extensions] section and then running\n\n'
             b'hg extdiff -p <program>\n\n'
             b'See \'hg help extdiff\' and \'hg help -e extdiff\' for more '
             b'information.\n'
         )
     )
 
 
 def fetch(ui, repo, *args, **kwargs):
     cmdoptions = [
         (b'', b'all', None, b''),
         (b'f', b'force', None, b''),
     ]
     args, opts = parseoptions(ui, cmdoptions, args)
 
     cmd = Command(b'pull')
 
     if len(args) > 0:
         cmd.append(args[0])
         if len(args) > 1:
             ui.status(
                 _(
                     b"note: Mercurial doesn't have refspecs. "
                     b"-r can be used to specify which commits you want to "
                     b"pull. -B can be used to specify which bookmark you "
                     b"want to pull.\n\n"
                 )
             )
             for v in args[1:]:
                 if v in repo._bookmarks:
                     cmd[b'-B'] = v
                 else:
                     cmd[b'-r'] = v
 
     ui.status((bytes(cmd)), b"\n")
 
 
 def grep(ui, repo, *args, **kwargs):
     cmdoptions = []
     args, opts = parseoptions(ui, cmdoptions, args)
 
     cmd = Command(b'grep')
 
     # For basic usage, git grep and hg grep are the same. They both have the
     # pattern first, followed by paths.
     cmd.extend(args)
 
     ui.status((bytes(cmd)), b"\n")
 
 
 def init(ui, repo, *args, **kwargs):
     cmdoptions = []
     args, opts = parseoptions(ui, cmdoptions, args)
 
     cmd = Command(b'init')
 
     if len(args) > 0:
         cmd.append(args[0])
 
     ui.status((bytes(cmd)), b"\n")
 
 
 def log(ui, repo, *args, **kwargs):
     cmdoptions = [
         (b'', b'follow', None, b''),
         (b'', b'decorate', None, b''),
         (b'n', b'number', b'', b''),
         (b'1', b'1', None, b''),
         (b'', b'pretty', b'', b''),
         (b'', b'format', b'', b''),
         (b'', b'oneline', None, b''),
         (b'', b'stat', None, b''),
         (b'', b'graph', None, b''),
         (b'p', b'patch', None, b''),
         (b'G', b'grep-diff', b'', b''),
         (b'S', b'pickaxe-regex', b'', b''),
     ]
     args, opts = parseoptions(ui, cmdoptions, args)
     grep_pat = opts.get(b'grep_diff') or opts.get(b'pickaxe_regex')
     if grep_pat:
         cmd = Command(b'grep')
         cmd[b'--diff'] = grep_pat
         ui.status(b'%s\n' % bytes(cmd))
         return
 
     ui.status(
         _(
             b'note: -v prints the entire commit message like Git does. To '
             b'print just the first line, drop the -v.\n\n'
         )
     )
     ui.status(
         _(
             b"note: see hg help revset for information on how to filter "
             b"log output\n\n"
         )
     )
 
     cmd = Command(b'log')
     cmd[b'-v'] = None
 
     if opts.get(b'number'):
         cmd[b'-l'] = opts.get(b'number')
     if opts.get(b'1'):
         cmd[b'-l'] = b'1'
     if opts.get(b'stat'):
         cmd[b'--stat'] = None
     if opts.get(b'graph'):
         cmd[b'-G'] = None
     if opts.get(b'patch'):
         cmd[b'-p'] = None
 
     if opts.get(b'pretty') or opts.get(b'format') or opts.get(b'oneline'):
         format = opts.get(b'format', b'')
         if b'format:' in format:
             ui.status(
                 _(
                     b"note: --format format:??? equates to Mercurial's "
                     b"--template. See hg help templates for more info.\n\n"
                 )
             )
             cmd[b'--template'] = b'???'
         else:
             ui.status(
                 _(
                     b"note: --pretty/format/oneline equate to Mercurial's "
                     b"--style or --template. See hg help templates for "
                     b"more info.\n\n"
                 )
             )
             cmd[b'--style'] = b'???'
 
     if len(args) > 0:
         if b'..' in args[0]:
             since, until = args[0].split(b'..')
             cmd[b'-r'] = b"'%s::%s'" % (since, until)
             del args[0]
         cmd.extend(args)
 
     ui.status((bytes(cmd)), b"\n")
 
 
 def lsfiles(ui, repo, *args, **kwargs):
     cmdoptions = [
         (b'c', b'cached', None, b''),
         (b'd', b'deleted', None, b''),
         (b'm', b'modified', None, b''),
         (b'o', b'others', None, b''),
         (b'i', b'ignored', None, b''),
         (b's', b'stage', None, b''),
         (b'z', b'_zero', None, b''),
     ]
     args, opts = parseoptions(ui, cmdoptions, args)
 
     if (
         opts.get(b'modified')
         or opts.get(b'deleted')
         or opts.get(b'others')
         or opts.get(b'ignored')
     ):
         cmd = Command(b'status')
         if opts.get(b'deleted'):
             cmd[b'-d'] = None
         if opts.get(b'modified'):
             cmd[b'-m'] = None
         if opts.get(b'others'):
             cmd[b'-o'] = None
         if opts.get(b'ignored'):
             cmd[b'-i'] = None
     else:
         cmd = Command(b'files')
     if opts.get(b'stage'):
         ui.status(
             _(
                 b"note: Mercurial doesn't have a staging area, ignoring "
                 b"--stage\n"
             )
         )
     if opts.get(b'_zero'):
         cmd[b'-0'] = None
     cmd.append(b'.')
     for include in args:
         cmd[b'-I'] = procutil.shellquote(include)
 
     ui.status((bytes(cmd)), b"\n")
 
 
 def merge(ui, repo, *args, **kwargs):
     cmdoptions = []
     args, opts = parseoptions(ui, cmdoptions, args)
 
     cmd = Command(b'merge')
 
     if len(args) > 0:
         cmd.append(args[len(args) - 1])
 
     ui.status((bytes(cmd)), b"\n")
 
 
 def mergebase(ui, repo, *args, **kwargs):
     cmdoptions = []
     args, opts = parseoptions(ui, cmdoptions, args)
 
     if len(args) != 2:
         args = [b'A', b'B']
 
     cmd = Command(
         b"log -T '{node}\\n' -r 'ancestor(%s,%s)'" % (args[0], args[1])
     )
 
     ui.status(
         _(b'note: ancestors() is part of the revset language\n'),
         _(b"(learn more about revsets with 'hg help revsets')\n\n"),
     )
     ui.status((bytes(cmd)), b"\n")
 
 
 def mergetool(ui, repo, *args, **kwargs):
     cmdoptions = []
     args, opts = parseoptions(ui, cmdoptions, args)
 
     cmd = Command(b"resolve")
 
     if len(args) == 0:
         cmd[b'--all'] = None
     cmd.extend(args)
     ui.status((bytes(cmd)), b"\n")
 
 
 def mv(ui, repo, *args, **kwargs):
     cmdoptions = [
         (b'f', b'force', None, b''),
         (b'n', b'dry-run', None, b''),
     ]
     args, opts = parseoptions(ui, cmdoptions, args)
 
     cmd = Command(b'mv')
     cmd.extend(args)
 
     if opts.get(b'force'):
         cmd[b'-f'] = None
     if opts.get(b'dry_run'):
         cmd[b'-n'] = None
 
     ui.status((bytes(cmd)), b"\n")
 
 
 def pull(ui, repo, *args, **kwargs):
     cmdoptions = [
         (b'', b'all', None, b''),
         (b'f', b'force', None, b''),
         (b'r', b'rebase', None, b''),
     ]
     args, opts = parseoptions(ui, cmdoptions, args)
 
     cmd = Command(b'pull')
     cmd[b'--rebase'] = None
 
     if len(args) > 0:
         cmd.append(args[0])
         if len(args) > 1:
             ui.status(
                 _(
                     b"note: Mercurial doesn't have refspecs. "
                     b"-r can be used to specify which commits you want to "
                     b"pull. -B can be used to specify which bookmark you "
                     b"want to pull.\n\n"
                 )
             )
             for v in args[1:]:
                 if v in repo._bookmarks:
                     cmd[b'-B'] = v
                 else:
                     cmd[b'-r'] = v
 
     ui.status((bytes(cmd)), b"\n")
 
 
 def push(ui, repo, *args, **kwargs):
     cmdoptions = [
         (b'', b'all', None, b''),
         (b'f', b'force', None, b''),
     ]
     args, opts = parseoptions(ui, cmdoptions, args)
 
     cmd = Command(b'push')
 
     if len(args) > 0:
         cmd.append(args[0])
         if len(args) > 1:
             ui.status(
                 _(
                     b"note: Mercurial doesn't have refspecs. "
                     b"-r can be used to specify which commits you want "
                     b"to push. -B can be used to specify which bookmark "
                     b"you want to push.\n\n"
                 )
             )
             for v in args[1:]:
                 if v in repo._bookmarks:
                     cmd[b'-B'] = v
                 else:
                     cmd[b'-r'] = v
 
     if opts.get(b'force'):
         cmd[b'-f'] = None
 
     ui.status((bytes(cmd)), b"\n")
 
 
 def rebase(ui, repo, *args, **kwargs):
     cmdoptions = [
         (b'', b'all', None, b''),
         (b'i', b'interactive', None, b''),
         (b'', b'onto', b'', b''),
         (b'', b'abort', None, b''),
         (b'', b'continue', None, b''),
         (b'', b'skip', None, b''),
     ]
     args, opts = parseoptions(ui, cmdoptions, args)
 
     if opts.get(b'interactive'):
         ui.status(
             _(
                 b"note: hg histedit does not perform a rebase. "
                 b"It just edits history.\n\n"
             )
         )
         cmd = Command(b'histedit')
         if len(args) > 0:
             ui.status(
                 _(
                     b"also note: 'hg histedit' will automatically detect"
                     b" your stack, so no second argument is necessary\n\n"
                 )
             )
         ui.status((bytes(cmd)), b"\n")
         return
 
     if opts.get(b'skip'):
         cmd = Command(b'revert --all -r .')
         ui.status((bytes(cmd)), b"\n")
 
     cmd = Command(b'rebase')
 
     if opts.get(b'continue') or opts.get(b'skip'):
         cmd[b'--continue'] = None
     if opts.get(b'abort'):
         cmd[b'--abort'] = None
 
     if opts.get(b'onto'):
         ui.status(
             _(
                 b"note: if you're trying to lift a commit off one branch, "
                 b"try hg rebase -d <destination commit> -s <commit to be "
                 b"lifted>\n\n"
             )
         )
         cmd[b'-d'] = convert(opts.get(b'onto'))
         if len(args) < 2:
             raise error.Abort(_(b"expected format: git rebase --onto X Y Z"))
         cmd[b'-s'] = b"'::%s - ::%s'" % (convert(args[1]), convert(args[0]))
     else:
         if len(args) == 1:
             cmd[b'-d'] = convert(args[0])
         elif len(args) == 2:
             cmd[b'-d'] = convert(args[0])
             cmd[b'-b'] = convert(args[1])
 
     ui.status((bytes(cmd)), b"\n")
 
 
 def reflog(ui, repo, *args, **kwargs):
     cmdoptions = [
         (b'', b'all', None, b''),
     ]
     args, opts = parseoptions(ui, cmdoptions, args)
 
     cmd = Command(b'journal')
     if opts.get(b'all'):
         cmd[b'--all'] = None
     if len(args) > 0:
         cmd.append(args[0])
 
     ui.status(bytes(cmd), b"\n\n")
     ui.status(
         _(
             b"note: in hg commits can be deleted from repo but we always"
             b" have backups\n"
         )
     )
 
 
 def reset(ui, repo, *args, **kwargs):
     cmdoptions = [
         (b'', b'soft', None, b''),
         (b'', b'hard', None, b''),
         (b'', b'mixed', None, b''),
     ]
     args, opts = parseoptions(ui, cmdoptions, args)
 
     commit = convert(args[0] if len(args) > 0 else b'.')
     hard = opts.get(b'hard')
 
     if opts.get(b'mixed'):
         ui.status(
             _(
                 b'note: --mixed has no meaning since Mercurial has no '
                 b'staging area\n\n'
             )
         )
     if opts.get(b'soft'):
         ui.status(
             _(
                 b'note: --soft has no meaning since Mercurial has no '
                 b'staging area\n\n'
             )
         )
 
     cmd = Command(b'update')
     if hard:
         cmd.append(b'--clean')
 
     cmd.append(commit)
 
     ui.status((bytes(cmd)), b"\n")
 
 
 def revert(ui, repo, *args, **kwargs):
     cmdoptions = []
     args, opts = parseoptions(ui, cmdoptions, args)
 
     if len(args) > 1:
         ui.status(
             _(
                 b"note: hg backout doesn't support multiple commits at "
                 b"once\n\n"
             )
         )
 
     cmd = Command(b'backout')
     if args:
         cmd.append(args[0])
 
     ui.status((bytes(cmd)), b"\n")
 
 
 def revparse(ui, repo, *args, **kwargs):
     cmdoptions = [
         (b'', b'show-cdup', None, b''),
         (b'', b'show-toplevel', None, b''),
     ]
     args, opts = parseoptions(ui, cmdoptions, args)
 
     if opts.get(b'show_cdup') or opts.get(b'show_toplevel'):
         cmd = Command(b'root')
         if opts.get(b'show_cdup'):
             ui.status(_(b"note: hg root prints the root of the repository\n\n"))
         ui.status((bytes(cmd)), b"\n")
     else:
         ui.status(_(b"note: see hg help revset for how to refer to commits\n"))
 
 
 def rm(ui, repo, *args, **kwargs):
     cmdoptions = [
         (b'f', b'force', None, b''),
         (b'n', b'dry-run', None, b''),
     ]
     args, opts = parseoptions(ui, cmdoptions, args)
 
     cmd = Command(b'rm')
     cmd.extend(args)
 
     if opts.get(b'force'):
         cmd[b'-f'] = None
     if opts.get(b'dry_run'):
         cmd[b'-n'] = None
 
     ui.status((bytes(cmd)), b"\n")
 
 
 def show(ui, repo, *args, **kwargs):
     cmdoptions = [
         (b'', b'name-status', None, b''),
         (b'', b'pretty', b'', b''),
         (b'U', b'unified', int, b''),
     ]
     args, opts = parseoptions(ui, cmdoptions, args)
 
     if opts.get(b'name_status'):
         if opts.get(b'pretty') == b'format:':
             cmd = Command(b'status')
             cmd[b'--change'] = b'.'
         else:
             cmd = Command(b'log')
             cmd.append(b'--style status')
             cmd.append(b'-r .')
     elif len(args) > 0:
         if ispath(repo, args[0]):
             cmd = Command(b'cat')
         else:
             cmd = Command(b'export')
         cmd.extend(args)
         if opts.get(b'unified'):
             cmd.append(b'--config diff.unified=%d' % (opts[b'unified'],))
     elif opts.get(b'unified'):
         cmd = Command(b'export')
         cmd.append(b'--config diff.unified=%d' % (opts[b'unified'],))
     else:
         cmd = Command(b'export')
 
     ui.status((bytes(cmd)), b"\n")
 
 
 def stash(ui, repo, *args, **kwargs):
     cmdoptions = [
         (b'p', b'patch', None, b''),
     ]
     args, opts = parseoptions(ui, cmdoptions, args)
 
     cmd = Command(b'shelve')
     action = args[0] if len(args) > 0 else None
 
     if action == b'list':
         cmd[b'-l'] = None
         if opts.get(b'patch'):
             cmd[b'-p'] = None
     elif action == b'show':
         if opts.get(b'patch'):
             cmd[b'-p'] = None
         else:
             cmd[b'--stat'] = None
         if len(args) > 1:
             cmd.append(args[1])
     elif action == b'clear':
         cmd[b'--cleanup'] = None
     elif action == b'drop':
         cmd[b'-d'] = None
         if len(args) > 1:
             cmd.append(args[1])
         else:
             cmd.append(b'<shelve name>')
     elif action == b'pop' or action == b'apply':
         cmd = Command(b'unshelve')
         if len(args) > 1:
             cmd.append(args[1])
         if action == b'apply':
             cmd[b'--keep'] = None
     elif action == b'branch' or action == b'create':
         ui.status(
             _(
                 b"note: Mercurial doesn't have equivalents to the "
                 b"git stash branch or create actions\n\n"
             )
         )
         return
     else:
         if len(args) > 0:
             if args[0] != b'save':
                 cmd[b'--name'] = args[0]
             elif len(args) > 1:
                 cmd[b'--name'] = args[1]
 
     ui.status((bytes(cmd)), b"\n")
 
 
 def status(ui, repo, *args, **kwargs):
     cmdoptions = [
         (b'', b'ignored', None, b''),
     ]
     args, opts = parseoptions(ui, cmdoptions, args)
 
     cmd = Command(b'status')
     cmd.extend(args)
 
     if opts.get(b'ignored'):
         cmd[b'-i'] = None
 
     ui.status((bytes(cmd)), b"\n")
 
 
 def svn(ui, repo, *args, **kwargs):
     if not args:
         raise error.Abort(_(b'missing svn command'))
     svncmd = args[0]
     if svncmd not in gitsvncommands:
         raise error.Abort(_(b'unknown git svn command "%s"') % svncmd)
 
     args = args[1:]
     return gitsvncommands[svncmd](ui, repo, *args, **kwargs)
 
 
 def svndcommit(ui, repo, *args, **kwargs):
     cmdoptions = []
     parseoptions(ui, cmdoptions, args)
 
     cmd = Command(b'push')
 
     ui.status((bytes(cmd)), b"\n")
 
 
 def svnfetch(ui, repo, *args, **kwargs):
     cmdoptions = []
     parseoptions(ui, cmdoptions, args)
 
     cmd = Command(b'pull')
     cmd.append(b'default-push')
 
     ui.status((bytes(cmd)), b"\n")
 
 
 def svnfindrev(ui, repo, *args, **kwargs):
     cmdoptions = []
     args, opts = parseoptions(ui, cmdoptions, args)
 
     if not args:
         raise error.Abort(_(b'missing find-rev argument'))
 
     cmd = Command(b'log')
     cmd[b'-r'] = args[0]
 
     ui.status((bytes(cmd)), b"\n")
 
 
 def svnrebase(ui, repo, *args, **kwargs):
     cmdoptions = [
         (b'l', b'local', None, b''),
     ]
     parseoptions(ui, cmdoptions, args)
 
     pullcmd = Command(b'pull')
     pullcmd.append(b'default-push')
     rebasecmd = Command(b'rebase')
     rebasecmd.append(b'tip')
 
     cmd = pullcmd & rebasecmd
 
     ui.status((bytes(cmd)), b"\n")
 
 
 def tag(ui, repo, *args, **kwargs):
     cmdoptions = [
         (b'f', b'force', None, b''),
         (b'l', b'list', None, b''),
         (b'd', b'delete', None, b''),
     ]
     args, opts = parseoptions(ui, cmdoptions, args)
 
     if opts.get(b'list'):
         cmd = Command(b'tags')
     else:
         cmd = Command(b'tag')
 
         if not args:
             raise error.Abort(_(b'missing tag argument'))
 
         cmd.append(args[0])
         if len(args) > 1:
             cmd[b'-r'] = args[1]
 
         if opts.get(b'delete'):
             cmd[b'--remove'] = None
 
         if opts.get(b'force'):
             cmd[b'-f'] = None
 
     ui.status((bytes(cmd)), b"\n")
 
 
 gitcommands = {
     b'add': add,
     b'am': am,
     b'apply': apply,
     b'bisect': bisect,
     b'blame': blame,
     b'branch': branch,
     b'checkout': checkout,
     b'cherry-pick': cherrypick,
     b'clean': clean,
     b'clone': clone,
     b'commit': commit,
     b'diff': diff,
     b'difftool': difftool,
     b'fetch': fetch,
     b'grep': grep,
     b'init': init,
     b'log': log,
     b'ls-files': lsfiles,
     b'merge': merge,
     b'merge-base': mergebase,
     b'mergetool': mergetool,
     b'mv': mv,
     b'pull': pull,
     b'push': push,
     b'rebase': rebase,
     b'reflog': reflog,
     b'reset': reset,
     b'revert': revert,
     b'rev-parse': revparse,
     b'rm': rm,
     b'show': show,
     b'stash': stash,
     b'status': status,
     b'svn': svn,
     b'tag': tag,
     b'whatchanged': deprecated,
 }
 
 gitsvncommands = {
     b'dcommit': svndcommit,
     b'fetch': svnfetch,
     b'find-rev': svnfindrev,
     b'rebase': svnrebase,
 }

hgext/hgk.py

@@ -1,387 +1,385 b''
 # Minimal support for git commands on an hg repository
 #
 # Copyright 2005, 2006 Chris Mason <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''browse the repository in a graphical way
 
 The hgk extension allows browsing the history of a repository in a
 graphical way. It requires Tcl/Tk version 8.4 or later. (Tcl/Tk is not
 distributed with Mercurial.)
 
 hgk consists of two parts: a Tcl script that does the displaying and
 querying of information, and an extension to Mercurial named hgk.py,
 which provides hooks for hgk to get information. hgk can be found in
 the contrib directory, and the extension is shipped in the hgext
 repository, and needs to be enabled.
 
 The :hg:`view` command will launch the hgk Tcl script. For this command
 to work, hgk must be in your search path. Alternately, you can specify
 the path to hgk in your configuration file::
 
   [hgk]
   path = /location/of/hgk
 
 hgk can make use of the extdiff extension to visualize revisions.
 Assuming you had already configured extdiff vdiff command, just add::
 
   [hgk]
   vdiff=vdiff
 
 Revisions context menu will now display additional entries to fire
 vdiff on hovered and selected revisions.
 '''
 
 
 import os
 
 from mercurial.i18n import _
 from mercurial.node import (
     nullrev,
     short,
 )
 from mercurial import (
     commands,
     obsolete,
     patch,
     pycompat,
     registrar,
     scmutil,
 )
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 configitem(
     b'hgk',
     b'path',
     default=b'hgk',
 )
 
 
 @command(
     b'debug-diff-tree',
     [
         (b'p', b'patch', None, _(b'generate patch')),
         (b'r', b'recursive', None, _(b'recursive')),
         (b'P', b'pretty', None, _(b'pretty')),
         (b's', b'stdin', None, _(b'stdin')),
         (b'C', b'copy', None, _(b'detect copies')),
         (b'S', b'search', b"", _(b'search')),
     ],
     b'[OPTION]... NODE1 NODE2 [FILE]...',
     inferrepo=True,
 )
 def difftree(ui, repo, node1=None, node2=None, *files, **opts):
     """diff trees from two commits"""
 
     def __difftree(repo, node1, node2, files=None):
         assert node2 is not None
         if files is None:
             files = []
         mmap = repo[node1].manifest()
         mmap2 = repo[node2].manifest()
         m = scmutil.match(repo[node1], files)
         st = repo.status(node1, node2, m)
         empty = short(repo.nullid)
 
         for f in st.modified:
             # TODO get file permissions
             ui.writenoi18n(
                 b":100664 100664 %s %s M\t%s\t%s\n"
                 % (short(mmap[f]), short(mmap2[f]), f, f)
             )
         for f in st.added:
             ui.writenoi18n(
                 b":000000 100664 %s %s N\t%s\t%s\n"
                 % (empty, short(mmap2[f]), f, f)
             )
         for f in st.removed:
             ui.writenoi18n(
                 b":100664 000000 %s %s D\t%s\t%s\n"
                 % (short(mmap[f]), empty, f, f)
             )
 
     ##
 
     while True:
         if opts['stdin']:
             line = ui.fin.readline()
             if not line:
                 break
             line = line.rstrip(pycompat.oslinesep).split(b' ')
             node1 = line[0]
             if len(line) > 1:
                 node2 = line[1]
             else:
                 node2 = None
         node1 = repo.lookup(node1)
         if node2:
             node2 = repo.lookup(node2)
         else:
             node2 = node1
             node1 = repo.changelog.parents(node1)[0]
         if opts['patch']:
             if opts['pretty']:
                 catcommit(ui, repo, node2, b"")
             m = scmutil.match(repo[node1], files)
             diffopts = patch.difffeatureopts(ui)
             diffopts.git = True
             chunks = patch.diff(repo, node1, node2, match=m, opts=diffopts)
             for chunk in chunks:
                 ui.write(chunk)
         else:
             __difftree(repo, node1, node2, files=files)
         if not opts['stdin']:
             break
 
 
 def catcommit(ui, repo, n, prefix, ctx=None):
     nlprefix = b'\n' + prefix
     if ctx is None:
         ctx = repo[n]
     # use ctx.node() instead ??
     ui.write((b"tree %s\n" % short(ctx.changeset()[0])))
     for p in ctx.parents():
         ui.write((b"parent %s\n" % p))
 
     date = ctx.date()
     description = ctx.description().replace(b"\0", b"")
     ui.write((b"author %s %d %d\n" % (ctx.user(), int(date[0]), date[1])))
 
     if b'committer' in ctx.extra():
         ui.write((b"committer %s\n" % ctx.extra()[b'committer']))
 
     ui.write((b"revision %d\n" % ctx.rev()))
     ui.write((b"branch %s\n" % ctx.branch()))
     if obsolete.isenabled(repo, obsolete.createmarkersopt):
         if ctx.obsolete():
             ui.writenoi18n(b"obsolete\n")
     ui.write((b"phase %s\n\n" % ctx.phasestr()))
 
     if prefix != b"":
         ui.write(
             b"%s%s\n" % (prefix, description.replace(b'\n', nlprefix).strip())
         )
     else:
         ui.write(description + b"\n")
     if prefix:
         ui.write(b'\0')
 
 
 @command(b'debug-merge-base', [], _(b'REV REV'))
 def base(ui, repo, node1, node2):
     """output common ancestor information"""
     node1 = repo.lookup(node1)
     node2 = repo.lookup(node2)
     n = repo.changelog.ancestor(node1, node2)
     ui.write(short(n) + b"\n")
 
 
 @command(
     b'debug-cat-file',
     [(b's', b'stdin', None, _(b'stdin'))],
     _(b'[OPTION]... TYPE FILE'),
     inferrepo=True,
 )
 def catfile(ui, repo, type=None, r=None, **opts):
     """cat a specific revision"""
     # in stdin mode, every line except the commit is prefixed with two
     # spaces.  This way the our caller can find the commit without magic
     # strings
     #
     prefix = b""
     if opts['stdin']:
         line = ui.fin.readline()
         if not line:
             return
         (type, r) = line.rstrip(pycompat.oslinesep).split(b' ')
         prefix = b"    "
     else:
         if not type or not r:
             ui.warn(_(b"cat-file: type or revision not supplied\n"))
             commands.help_(ui, b'cat-file')
 
     while r:
         if type != b"commit":
             ui.warn(_(b"aborting hg cat-file only understands commits\n"))
             return 1
         n = repo.lookup(r)
         catcommit(ui, repo, n, prefix)
         if opts['stdin']:
             line = ui.fin.readline()
             if not line:
                 break
             (type, r) = line.rstrip(pycompat.oslinesep).split(b' ')
         else:
             break
 
 
 # git rev-tree is a confusing thing.  You can supply a number of
 # commit sha1s on the command line, and it walks the commit history
 # telling you which commits are reachable from the supplied ones via
 # a bitmask based on arg position.
 # you can specify a commit to stop at by starting the sha1 with ^
 def revtree(ui, args, repo, full=b"tree", maxnr=0, parents=False):
     def chlogwalk():
         count = len(repo)
         i = count
         l = [0] * 100
         chunk = 100
         while True:
             if chunk > i:
                 chunk = i
                 i = 0
             else:
                 i -= chunk
 
             for x in pycompat.xrange(chunk):
                 if i + x >= count:
                     l[chunk - x :] = [0] * (chunk - x)
                     break
                 if full is not None:
                     if (i + x) in repo:
                         l[x] = repo[i + x]
                         l[x].changeset()  # force reading
                 else:
                     if (i + x) in repo:
                         l[x] = 1
             for x in pycompat.xrange(chunk - 1, -1, -1):
                 if l[x] != 0:
                     yield (i + x, full is not None and l[x] or None)
             if i == 0:
                 break
 
     # calculate and return the reachability bitmask for sha
     def is_reachable(ar, reachable, sha):
         if len(ar) == 0:
             return 1
         mask = 0
         for i in pycompat.xrange(len(ar)):
             if sha in reachable[i]:
                 mask |= 1 << i
 
         return mask
 
     reachable = []
     stop_sha1 = []
     want_sha1 = []
     count = 0
 
     # figure out which commits they are asking for and which ones they
     # want us to stop on
     for i, arg in enumerate(args):
         if arg.startswith(b'^'):
             s = repo.lookup(arg[1:])
             stop_sha1.append(s)
             want_sha1.append(s)
         elif arg != b'HEAD':
             want_sha1.append(repo.lookup(arg))
 
     # calculate the graph for the supplied commits
     for i, n in enumerate(want_sha1):
         reachable.append(set())
         visit = [n]
         reachable[i].add(n)
         while visit:
             n = visit.pop(0)
             if n in stop_sha1:
                 continue
             for p in repo.changelog.parents(n):
                 if p not in reachable[i]:
                     reachable[i].add(p)
                     visit.append(p)
                 if p in stop_sha1:
                     continue
 
     # walk the repository looking for commits that are in our
     # reachability graph
     for i, ctx in chlogwalk():
         if i not in repo:
             continue
         n = repo.changelog.node(i)
         mask = is_reachable(want_sha1, reachable, n)
         if mask:
             parentstr = b""
             if parents:
                 pp = repo.changelog.parents(n)
                 if pp[0] != repo.nullid:
                     parentstr += b" " + short(pp[0])
                 if pp[1] != repo.nullid:
                     parentstr += b" " + short(pp[1])
             if not full:
                 ui.write(b"%s%s\n" % (short(n), parentstr))
             elif full == b"commit":
                 ui.write(b"%s%s\n" % (short(n), parentstr))
                 catcommit(ui, repo, n, b'    ', ctx)
             else:
                 (p1, p2) = repo.changelog.parents(n)
                 (h, h1, h2) = map(short, (n, p1, p2))
                 (i1, i2) = map(repo.changelog.rev, (p1, p2))
 
                 date = ctx.date()[0]
                 ui.write(b"%s %s:%s" % (date, h, mask))
                 mask = is_reachable(want_sha1, reachable, p1)
                 if i1 != nullrev and mask > 0:
                     ui.write(b"%s:%s " % (h1, mask)),
                 mask = is_reachable(want_sha1, reachable, p2)
                 if i2 != nullrev and mask > 0:
                     ui.write(b"%s:%s " % (h2, mask))
                 ui.write(b"\n")
             if maxnr and count >= maxnr:
                 break
             count += 1
 
 
 # git rev-list tries to order things by date, and has the ability to stop
 # at a given commit without walking the whole repo.  TODO add the stop
 # parameter
 @command(
     b'debug-rev-list',
     [
         (b'H', b'header', None, _(b'header')),
         (b't', b'topo-order', None, _(b'topo-order')),
         (b'p', b'parents', None, _(b'parents')),
         (b'n', b'max-count', 0, _(b'max-count')),
     ],
     b'[OPTION]... REV...',
 )
 def revlist(ui, repo, *revs, **opts):
     """print revisions"""
     if opts['header']:
         full = b"commit"
     else:
         full = None
     copy = [x for x in revs]
     revtree(ui, copy, repo, full, opts['max_count'], opts[r'parents'])
 
 
 @command(
     b'view',
     [(b'l', b'limit', b'', _(b'limit number of changes displayed'), _(b'NUM'))],
     _(b'[-l LIMIT] [REVRANGE]'),
     helpcategory=command.CATEGORY_CHANGE_NAVIGATION,
 )
 def view(ui, repo, *etc, **opts):
     """start interactive history viewer"""
     opts = pycompat.byteskwargs(opts)
     os.chdir(repo.root)
-    optstr = b' '.join(
-        [b'--%s %s' % (k, v) for k, v in pycompat.iteritems(opts) if v]
-    )
+    optstr = b' '.join([b'--%s %s' % (k, v) for k, v in opts.items() if v])
     if repo.filtername is None:
         optstr += b'--hidden'
 
     cmd = ui.config(b"hgk", b"path") + b" %s %s" % (optstr, b" ".join(etc))
     ui.debug(b"running %s\n" % cmd)
     ui.system(cmd, blockedtag=b'hgk_view')