py3: use str instead of pycompat.unicode

pycompat.unicode is an alias to str.

Differential Revision: https://phab.mercurial-scm.org/D12340

hgext/convert/common.py

@@ -1,597 +1,597
 # common.py - common code for the convert extension
 #
 #  Copyright 2005-2009 Olivia Mackall <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 import base64
 import datetime
 import errno
 import os
 import pickle
 import re
 import shlex
 import subprocess
 
 from mercurial.i18n import _
 from mercurial.pycompat import open
 from mercurial import (
     encoding,
     error,
     phases,
     pycompat,
     util,
 )
 from mercurial.utils import procutil
 
 propertycache = util.propertycache
 
 
 def _encodeornone(d):
     if d is None:
         return
     return d.encode('latin1')
 
 
 class _shlexpy3proxy(object):
     def __init__(self, l):
         self._l = l
 
     def __iter__(self):
         return (_encodeornone(v) for v in self._l)
 
     def get_token(self):
         return _encodeornone(self._l.get_token())
 
     @property
     def infile(self):
         return self._l.infile or b'<unknown>'
 
     @property
     def lineno(self):
         return self._l.lineno
 
 
 def shlexer(data=None, filepath=None, wordchars=None, whitespace=None):
     if data is None:
         if pycompat.ispy3:
             data = open(filepath, b'r', encoding='latin1')
         else:
             data = open(filepath, b'r')
     else:
         if filepath is not None:
             raise error.ProgrammingError(
                 b'shlexer only accepts data or filepath, not both'
             )
         if pycompat.ispy3:
             data = data.decode('latin1')
     l = shlex.shlex(data, infile=filepath, posix=True)
     if whitespace is not None:
         l.whitespace_split = True
         if pycompat.ispy3:
             l.whitespace += whitespace.decode('latin1')
         else:
             l.whitespace += whitespace
     if wordchars is not None:
         if pycompat.ispy3:
             l.wordchars += wordchars.decode('latin1')
         else:
             l.wordchars += wordchars
     if pycompat.ispy3:
         return _shlexpy3proxy(l)
     return l
 
 
 if pycompat.ispy3:
     base64_encodebytes = base64.encodebytes
     base64_decodebytes = base64.decodebytes
 else:
     base64_encodebytes = base64.encodestring
     base64_decodebytes = base64.decodestring
 
 
 def encodeargs(args):
     def encodearg(s):
         lines = base64_encodebytes(s)
         lines = [l.splitlines()[0] for l in pycompat.iterbytestr(lines)]
         return b''.join(lines)
 
     s = pickle.dumps(args)
     return encodearg(s)
 
 
 def decodeargs(s):
     s = base64_decodebytes(s)
     return pickle.loads(s)
 
 
 class MissingTool(Exception):
     pass
 
 
 def checktool(exe, name=None, abort=True):
     name = name or exe
     if not procutil.findexe(exe):
         if abort:
             exc = error.Abort
         else:
             exc = MissingTool
         raise exc(_(b'cannot find required "%s" tool') % name)
 
 
 class NoRepo(Exception):
     pass
 
 
 SKIPREV = b'SKIP'
 
 
 class commit(object):
     def __init__(
         self,
         author,
         date,
         desc,
         parents,
         branch=None,
         rev=None,
         extra=None,
         sortkey=None,
         saverev=True,
         phase=phases.draft,
         optparents=None,
         ctx=None,
     ):
         self.author = author or b'unknown'
         self.date = date or b'0 0'
         self.desc = desc
         self.parents = parents  # will be converted and used as parents
         self.optparents = optparents or []  # will be used if already converted
         self.branch = branch
         self.rev = rev
         self.extra = extra or {}
         self.sortkey = sortkey
         self.saverev = saverev
         self.phase = phase
         self.ctx = ctx  # for hg to hg conversions
 
 
 class converter_source(object):
     """Conversion source interface"""
 
     def __init__(self, ui, repotype, path=None, revs=None):
         """Initialize conversion source (or raise NoRepo("message")
         exception if path is not a valid repository)"""
         self.ui = ui
         self.path = path
         self.revs = revs
         self.repotype = repotype
 
         self.encoding = b'utf-8'
 
     def checkhexformat(self, revstr, mapname=b'splicemap'):
         """fails if revstr is not a 40 byte hex. mercurial and git both uses
         such format for their revision numbering
         """
         if not re.match(br'[0-9a-fA-F]{40,40}$', revstr):
             raise error.Abort(
                 _(b'%s entry %s is not a valid revision identifier')
                 % (mapname, revstr)
             )
 
     def before(self):
         pass
 
     def after(self):
         pass
 
     def targetfilebelongstosource(self, targetfilename):
         """Returns true if the given targetfile belongs to the source repo. This
         is useful when only a subdirectory of the target belongs to the source
         repo."""
         # For normal full repo converts, this is always True.
         return True
 
     def setrevmap(self, revmap):
         """set the map of already-converted revisions"""
 
     def getheads(self):
         """Return a list of this repository's heads"""
         raise NotImplementedError
 
     def getfile(self, name, rev):
         """Return a pair (data, mode) where data is the file content
         as a string and mode one of '', 'x' or 'l'. rev is the
         identifier returned by a previous call to getchanges().
         Data is None if file is missing/deleted in rev.
         """
         raise NotImplementedError
 
     def getchanges(self, version, full):
         """Returns a tuple of (files, copies, cleanp2).
 
         files is a sorted list of (filename, id) tuples for all files
         changed between version and its first parent returned by
         getcommit(). If full, all files in that revision is returned.
         id is the source revision id of the file.
 
         copies is a dictionary of dest: source
 
         cleanp2 is the set of files filenames that are clean against p2.
         (Files that are clean against p1 are already not in files (unless
         full). This makes it possible to handle p2 clean files similarly.)
         """
         raise NotImplementedError
 
     def getcommit(self, version):
         """Return the commit object for version"""
         raise NotImplementedError
 
     def numcommits(self):
         """Return the number of commits in this source.
 
         If unknown, return None.
         """
         return None
 
     def gettags(self):
         """Return the tags as a dictionary of name: revision
 
         Tag names must be UTF-8 strings.
         """
         raise NotImplementedError
 
     def recode(self, s, encoding=None):
         if not encoding:
             encoding = self.encoding or b'utf-8'
 
-        if isinstance(s, pycompat.unicode):
+        if isinstance(s, str):
             return s.encode("utf-8")
         try:
             return s.decode(pycompat.sysstr(encoding)).encode("utf-8")
         except UnicodeError:
             try:
                 return s.decode("latin-1").encode("utf-8")
             except UnicodeError:
                 return s.decode(pycompat.sysstr(encoding), "replace").encode(
                     "utf-8"
                 )
 
     def getchangedfiles(self, rev, i):
         """Return the files changed by rev compared to parent[i].
 
         i is an index selecting one of the parents of rev.  The return
         value should be the list of files that are different in rev and
         this parent.
 
         If rev has no parents, i is None.
 
         This function is only needed to support --filemap
         """
         raise NotImplementedError
 
     def converted(self, rev, sinkrev):
         '''Notify the source that a revision has been converted.'''
 
     def hasnativeorder(self):
         """Return true if this source has a meaningful, native revision
         order. For instance, Mercurial revisions are store sequentially
         while there is no such global ordering with Darcs.
         """
         return False
 
     def hasnativeclose(self):
         """Return true if this source has ability to close branch."""
         return False
 
     def lookuprev(self, rev):
         """If rev is a meaningful revision reference in source, return
         the referenced identifier in the same format used by getcommit().
         return None otherwise.
         """
         return None
 
     def getbookmarks(self):
         """Return the bookmarks as a dictionary of name: revision
 
         Bookmark names are to be UTF-8 strings.
         """
         return {}
 
     def checkrevformat(self, revstr, mapname=b'splicemap'):
         """revstr is a string that describes a revision in the given
         source control system.  Return true if revstr has correct
         format.
         """
         return True
 
 
 class converter_sink(object):
     """Conversion sink (target) interface"""
 
     def __init__(self, ui, repotype, path):
         """Initialize conversion sink (or raise NoRepo("message")
         exception if path is not a valid repository)
 
         created is a list of paths to remove if a fatal error occurs
         later"""
         self.ui = ui
         self.path = path
         self.created = []
         self.repotype = repotype
 
     def revmapfile(self):
         """Path to a file that will contain lines
         source_rev_id sink_rev_id
         mapping equivalent revision identifiers for each system."""
         raise NotImplementedError
 
     def authorfile(self):
         """Path to a file that will contain lines
         srcauthor=dstauthor
         mapping equivalent authors identifiers for each system."""
         return None
 
     def putcommit(
         self, files, copies, parents, commit, source, revmap, full, cleanp2
     ):
         """Create a revision with all changed files listed in 'files'
         and having listed parents. 'commit' is a commit object
         containing at a minimum the author, date, and message for this
         changeset.  'files' is a list of (path, version) tuples,
         'copies' is a dictionary mapping destinations to sources,
         'source' is the source repository, and 'revmap' is a mapfile
         of source revisions to converted revisions. Only getfile() and
         lookuprev() should be called on 'source'. 'full' means that 'files'
         is complete and all other files should be removed.
         'cleanp2' is a set of the filenames that are unchanged from p2
         (only in the common merge case where there two parents).
 
         Note that the sink repository is not told to update itself to
         a particular revision (or even what that revision would be)
         before it receives the file data.
         """
         raise NotImplementedError
 
     def puttags(self, tags):
         """Put tags into sink.
 
         tags: {tagname: sink_rev_id, ...} where tagname is an UTF-8 string.
         Return a pair (tag_revision, tag_parent_revision), or (None, None)
         if nothing was changed.
         """
         raise NotImplementedError
 
     def setbranch(self, branch, pbranches):
         """Set the current branch name. Called before the first putcommit
         on the branch.
         branch: branch name for subsequent commits
         pbranches: (converted parent revision, parent branch) tuples"""
 
     def setfilemapmode(self, active):
         """Tell the destination that we're using a filemap
 
         Some converter_sources (svn in particular) can claim that a file
         was changed in a revision, even if there was no change.  This method
         tells the destination that we're using a filemap and that it should
         filter empty revisions.
         """
 
     def before(self):
         pass
 
     def after(self):
         pass
 
     def putbookmarks(self, bookmarks):
         """Put bookmarks into sink.
 
         bookmarks: {bookmarkname: sink_rev_id, ...}
         where bookmarkname is an UTF-8 string.
         """
 
     def hascommitfrommap(self, rev):
         """Return False if a rev mentioned in a filemap is known to not be
         present."""
         raise NotImplementedError
 
     def hascommitforsplicemap(self, rev):
         """This method is for the special needs for splicemap handling and not
         for general use. Returns True if the sink contains rev, aborts on some
         special cases."""
         raise NotImplementedError
 
 
 class commandline(object):
     def __init__(self, ui, command):
         self.ui = ui
         self.command = command
 
     def prerun(self):
         pass
 
     def postrun(self):
         pass
 
     def _cmdline(self, cmd, *args, **kwargs):
         kwargs = pycompat.byteskwargs(kwargs)
         cmdline = [self.command, cmd] + list(args)
         for k, v in kwargs.items():
             if len(k) == 1:
                 cmdline.append(b'-' + k)
             else:
                 cmdline.append(b'--' + k.replace(b'_', b'-'))
             try:
                 if len(k) == 1:
                     cmdline.append(b'' + v)
                 else:
                     cmdline[-1] += b'=' + v
             except TypeError:
                 pass
         cmdline = [procutil.shellquote(arg) for arg in cmdline]
         if not self.ui.debugflag:
             cmdline += [b'2>', pycompat.bytestr(os.devnull)]
         cmdline = b' '.join(cmdline)
         return cmdline
 
     def _run(self, cmd, *args, **kwargs):
         def popen(cmdline):
             p = subprocess.Popen(
                 procutil.tonativestr(cmdline),
                 shell=True,
                 bufsize=-1,
                 close_fds=procutil.closefds,
                 stdout=subprocess.PIPE,
             )
             return p
 
         return self._dorun(popen, cmd, *args, **kwargs)
 
     def _run2(self, cmd, *args, **kwargs):
         return self._dorun(procutil.popen2, cmd, *args, **kwargs)
 
     def _run3(self, cmd, *args, **kwargs):
         return self._dorun(procutil.popen3, cmd, *args, **kwargs)
 
     def _dorun(self, openfunc, cmd, *args, **kwargs):
         cmdline = self._cmdline(cmd, *args, **kwargs)
         self.ui.debug(b'running: %s\n' % (cmdline,))
         self.prerun()
         try:
             return openfunc(cmdline)
         finally:
             self.postrun()
 
     def run(self, cmd, *args, **kwargs):
         p = self._run(cmd, *args, **kwargs)
         output = p.communicate()[0]
         self.ui.debug(output)
         return output, p.returncode
 
     def runlines(self, cmd, *args, **kwargs):
         p = self._run(cmd, *args, **kwargs)
         output = p.stdout.readlines()
         p.wait()
         self.ui.debug(b''.join(output))
         return output, p.returncode
 
     def checkexit(self, status, output=b''):
         if status:
             if output:
                 self.ui.warn(_(b'%s error:\n') % self.command)
                 self.ui.warn(output)
             msg = procutil.explainexit(status)
             raise error.Abort(b'%s %s' % (self.command, msg))
 
     def run0(self, cmd, *args, **kwargs):
         output, status = self.run(cmd, *args, **kwargs)
         self.checkexit(status, output)
         return output
 
     def runlines0(self, cmd, *args, **kwargs):
         output, status = self.runlines(cmd, *args, **kwargs)
         self.checkexit(status, b''.join(output))
         return output
 
     @propertycache
     def argmax(self):
         # POSIX requires at least 4096 bytes for ARG_MAX
         argmax = 4096
         try:
             argmax = os.sysconf("SC_ARG_MAX")
         except (AttributeError, ValueError):
             pass
 
         # Windows shells impose their own limits on command line length,
         # down to 2047 bytes for cmd.exe under Windows NT/2k and 2500 bytes
         # for older 4nt.exe. See http://support.microsoft.com/kb/830473 for
         # details about cmd.exe limitations.
 
         # Since ARG_MAX is for command line _and_ environment, lower our limit
         # (and make happy Windows shells while doing this).
         return argmax // 2 - 1
 
     def _limit_arglist(self, arglist, cmd, *args, **kwargs):
         cmdlen = len(self._cmdline(cmd, *args, **kwargs))
         limit = self.argmax - cmdlen
         numbytes = 0
         fl = []
         for fn in arglist:
             b = len(fn) + 3
             if numbytes + b < limit or len(fl) == 0:
                 fl.append(fn)
                 numbytes += b
             else:
                 yield fl
                 fl = [fn]
                 numbytes = b
         if fl:
             yield fl
 
     def xargs(self, arglist, cmd, *args, **kwargs):
         for l in self._limit_arglist(arglist, cmd, *args, **kwargs):
             self.run0(cmd, *(list(args) + l), **kwargs)
 
 
 class mapfile(dict):
     def __init__(self, ui, path):
         super(mapfile, self).__init__()
         self.ui = ui
         self.path = path
         self.fp = None
         self.order = []
         self._read()
 
     def _read(self):
         if not self.path:
             return
         try:
             fp = open(self.path, b'rb')
         except IOError as err:
             if err.errno != errno.ENOENT:
                 raise
             return
         for i, line in enumerate(util.iterfile(fp)):
             line = line.splitlines()[0].rstrip()
             if not line:
                 # Ignore blank lines
                 continue
             try:
                 key, value = line.rsplit(b' ', 1)
             except ValueError:
                 raise error.Abort(
                     _(b'syntax error in %s(%d): key/value pair expected')
                     % (self.path, i + 1)
                 )
             if key not in self:
                 self.order.append(key)
             super(mapfile, self).__setitem__(key, value)
         fp.close()
 
     def __setitem__(self, key, value):
         if self.fp is None:
             try:
                 self.fp = open(self.path, b'ab')
             except IOError as err:
                 raise error.Abort(
                     _(b'could not open map file %r: %s')
                     % (self.path, encoding.strtolocal(err.strerror))
                 )
         self.fp.write(util.tonativeeol(b'%s %s\n' % (key, value)))
         self.fp.flush()
         super(mapfile, self).__setitem__(key, value)
 
     def close(self):
         if self.fp:
             self.fp.close()
             self.fp = None
 
 
 def makedatetimestamp(t):
     """Like dateutil.makedate() but for time t instead of current time"""
     delta = datetime.datetime.utcfromtimestamp(
         t
     ) - datetime.datetime.fromtimestamp(t)
     tz = delta.days * 86400 + delta.seconds
     return t, tz

hgext/convert/convcmd.py

@@ -1,667 +1,667
 # convcmd - convert extension commands definition
 #
 # Copyright 2005-2007 Olivia Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 import collections
 import os
 import shutil
 
 from mercurial.i18n import _
 from mercurial.pycompat import open
 from mercurial import (
     encoding,
     error,
     hg,
     pycompat,
     scmutil,
     util,
 )
 from mercurial.utils import dateutil
 
 from . import (
     bzr,
     common,
     cvs,
     darcs,
     filemap,
     git,
     gnuarch,
     hg as hgconvert,
     monotone,
     p4,
     subversion,
 )
 
 mapfile = common.mapfile
 MissingTool = common.MissingTool
 NoRepo = common.NoRepo
 SKIPREV = common.SKIPREV
 
 bzr_source = bzr.bzr_source
 convert_cvs = cvs.convert_cvs
 convert_git = git.convert_git
 darcs_source = darcs.darcs_source
 gnuarch_source = gnuarch.gnuarch_source
 mercurial_sink = hgconvert.mercurial_sink
 mercurial_source = hgconvert.mercurial_source
 monotone_source = monotone.monotone_source
 p4_source = p4.p4_source
 svn_sink = subversion.svn_sink
 svn_source = subversion.svn_source
 
 orig_encoding = b'ascii'
 
 
 def readauthormap(ui, authorfile, authors=None):
     if authors is None:
         authors = {}
     with open(authorfile, b'rb') as afile:
         for line in afile:
 
             line = line.strip()
             if not line or line.startswith(b'#'):
                 continue
 
             try:
                 srcauthor, dstauthor = line.split(b'=', 1)
             except ValueError:
                 msg = _(b'ignoring bad line in author map file %s: %s\n')
                 ui.warn(msg % (authorfile, line.rstrip()))
                 continue
 
             srcauthor = srcauthor.strip()
             dstauthor = dstauthor.strip()
             if authors.get(srcauthor) in (None, dstauthor):
                 msg = _(b'mapping author %s to %s\n')
                 ui.debug(msg % (srcauthor, dstauthor))
                 authors[srcauthor] = dstauthor
                 continue
 
             m = _(b'overriding mapping for author %s, was %s, will be %s\n')
             ui.status(m % (srcauthor, authors[srcauthor], dstauthor))
     return authors
 
 
 def recode(s):
-    if isinstance(s, pycompat.unicode):
+    if isinstance(s, str):
         return s.encode(pycompat.sysstr(orig_encoding), 'replace')
     else:
         return s.decode('utf-8').encode(
             pycompat.sysstr(orig_encoding), 'replace'
         )
 
 
 def mapbranch(branch, branchmap):
     """
     >>> bmap = {b'default': b'branch1'}
     >>> for i in [b'', None]:
     ...     mapbranch(i, bmap)
     'branch1'
     'branch1'
     >>> bmap = {b'None': b'branch2'}
     >>> for i in [b'', None]:
     ...     mapbranch(i, bmap)
     'branch2'
     'branch2'
     >>> bmap = {b'None': b'branch3', b'default': b'branch4'}
     >>> for i in [b'None', b'', None, b'default', b'branch5']:
     ...     mapbranch(i, bmap)
     'branch3'
     'branch4'
     'branch4'
     'branch4'
     'branch5'
     """
     # If branch is None or empty, this commit is coming from the source
     # repository's default branch and destined for the default branch in the
     # destination repository. For such commits, using a literal "default"
     # in branchmap below allows the user to map "default" to an alternate
     # default branch in the destination repository.
     branch = branchmap.get(branch or b'default', branch)
     # At some point we used "None" literal to denote the default branch,
     # attempt to use that for backward compatibility.
     if not branch:
         branch = branchmap.get(b'None', branch)
     return branch
 
 
 source_converters = [
     (b'cvs', convert_cvs, b'branchsort'),
     (b'git', convert_git, b'branchsort'),
     (b'svn', svn_source, b'branchsort'),
     (b'hg', mercurial_source, b'sourcesort'),
     (b'darcs', darcs_source, b'branchsort'),
     (b'mtn', monotone_source, b'branchsort'),
     (b'gnuarch', gnuarch_source, b'branchsort'),
     (b'bzr', bzr_source, b'branchsort'),
     (b'p4', p4_source, b'branchsort'),
 ]
 
 sink_converters = [
     (b'hg', mercurial_sink),
     (b'svn', svn_sink),
 ]
 
 
 def convertsource(ui, path, type, revs):
     exceptions = []
     if type and type not in [s[0] for s in source_converters]:
         raise error.Abort(_(b'%s: invalid source repository type') % type)
     for name, source, sortmode in source_converters:
         try:
             if not type or name == type:
                 return source(ui, name, path, revs), sortmode
         except (NoRepo, MissingTool) as inst:
             exceptions.append(inst)
     if not ui.quiet:
         for inst in exceptions:
             ui.write(b"%s\n" % pycompat.bytestr(inst.args[0]))
     raise error.Abort(_(b'%s: missing or unsupported repository') % path)
 
 
 def convertsink(ui, path, type):
     if type and type not in [s[0] for s in sink_converters]:
         raise error.Abort(_(b'%s: invalid destination repository type') % type)
     for name, sink in sink_converters:
         try:
             if not type or name == type:
                 return sink(ui, name, path)
         except NoRepo as inst:
             ui.note(_(b"convert: %s\n") % inst)
         except MissingTool as inst:
             raise error.Abort(b'%s\n' % inst)
     raise error.Abort(_(b'%s: unknown repository type') % path)
 
 
 class progresssource(object):
     def __init__(self, ui, source, filecount):
         self.ui = ui
         self.source = source
         self.progress = ui.makeprogress(
             _(b'getting files'), unit=_(b'files'), total=filecount
         )
 
     def getfile(self, file, rev):
         self.progress.increment(item=file)
         return self.source.getfile(file, rev)
 
     def targetfilebelongstosource(self, targetfilename):
         return self.source.targetfilebelongstosource(targetfilename)
 
     def lookuprev(self, rev):
         return self.source.lookuprev(rev)
 
     def close(self):
         self.progress.complete()
 
 
 class converter(object):
     def __init__(self, ui, source, dest, revmapfile, opts):
 
         self.source = source
         self.dest = dest
         self.ui = ui
         self.opts = opts
         self.commitcache = {}
         self.authors = {}
         self.authorfile = None
 
         # Record converted revisions persistently: maps source revision
         # ID to target revision ID (both strings).  (This is how
         # incremental conversions work.)
         self.map = mapfile(ui, revmapfile)
 
         # Read first the dst author map if any
         authorfile = self.dest.authorfile()
         if authorfile and os.path.exists(authorfile):
             self.readauthormap(authorfile)
         # Extend/Override with new author map if necessary
         if opts.get(b'authormap'):
             self.readauthormap(opts.get(b'authormap'))
             self.authorfile = self.dest.authorfile()
 
         self.splicemap = self.parsesplicemap(opts.get(b'splicemap'))
         self.branchmap = mapfile(ui, opts.get(b'branchmap'))
 
     def parsesplicemap(self, path):
         """check and validate the splicemap format and
         return a child/parents dictionary.
         Format checking has two parts.
         1. generic format which is same across all source types
         2. specific format checking which may be different for
            different source type.  This logic is implemented in
            checkrevformat function in source files like
            hg.py, subversion.py etc.
         """
 
         if not path:
             return {}
         m = {}
         try:
             fp = open(path, b'rb')
             for i, line in enumerate(util.iterfile(fp)):
                 line = line.splitlines()[0].rstrip()
                 if not line:
                     # Ignore blank lines
                     continue
                 # split line
                 lex = common.shlexer(data=line, whitespace=b',')
                 line = list(lex)
                 # check number of parents
                 if not (2 <= len(line) <= 3):
                     raise error.Abort(
                         _(
                             b'syntax error in %s(%d): child parent1'
                             b'[,parent2] expected'
                         )
                         % (path, i + 1)
                     )
                 for part in line:
                     self.source.checkrevformat(part)
                 child, p1, p2 = line[0], line[1:2], line[2:]
                 if p1 == p2:
                     m[child] = p1
                 else:
                     m[child] = p1 + p2
         # if file does not exist or error reading, exit
         except IOError:
             raise error.Abort(
                 _(b'splicemap file not found or error reading %s:') % path
             )
         return m
 
     def walktree(self, heads):
         """Return a mapping that identifies the uncommitted parents of every
         uncommitted changeset."""
         visit = list(heads)
         known = set()
         parents = {}
         numcommits = self.source.numcommits()
         progress = self.ui.makeprogress(
             _(b'scanning'), unit=_(b'revisions'), total=numcommits
         )
         while visit:
             n = visit.pop(0)
             if n in known:
                 continue
             if n in self.map:
                 m = self.map[n]
                 if m == SKIPREV or self.dest.hascommitfrommap(m):
                     continue
             known.add(n)
             progress.update(len(known))
             commit = self.cachecommit(n)
             parents[n] = []
             for p in commit.parents:
                 parents[n].append(p)
                 visit.append(p)
         progress.complete()
 
         return parents
 
     def mergesplicemap(self, parents, splicemap):
         """A splicemap redefines child/parent relationships. Check the
         map contains valid revision identifiers and merge the new
         links in the source graph.
         """
         for c in sorted(splicemap):
             if c not in parents:
                 if not self.dest.hascommitforsplicemap(self.map.get(c, c)):
                     # Could be in source but not converted during this run
                     self.ui.warn(
                         _(
                             b'splice map revision %s is not being '
                             b'converted, ignoring\n'
                         )
                         % c
                     )
                 continue
             pc = []
             for p in splicemap[c]:
                 # We do not have to wait for nodes already in dest.
                 if self.dest.hascommitforsplicemap(self.map.get(p, p)):
                     continue
                 # Parent is not in dest and not being converted, not good
                 if p not in parents:
                     raise error.Abort(_(b'unknown splice map parent: %s') % p)
                 pc.append(p)
             parents[c] = pc
 
     def toposort(self, parents, sortmode):
         """Return an ordering such that every uncommitted changeset is
         preceded by all its uncommitted ancestors."""
 
         def mapchildren(parents):
             """Return a (children, roots) tuple where 'children' maps parent
             revision identifiers to children ones, and 'roots' is the list of
             revisions without parents. 'parents' must be a mapping of revision
             identifier to its parents ones.
             """
             visit = collections.deque(sorted(parents))
             seen = set()
             children = {}
             roots = []
 
             while visit:
                 n = visit.popleft()
                 if n in seen:
                     continue
                 seen.add(n)
                 # Ensure that nodes without parents are present in the
                 # 'children' mapping.
                 children.setdefault(n, [])
                 hasparent = False
                 for p in parents[n]:
                     if p not in self.map:
                         visit.append(p)
                         hasparent = True
                     children.setdefault(p, []).append(n)
                 if not hasparent:
                     roots.append(n)
 
             return children, roots
 
         # Sort functions are supposed to take a list of revisions which
         # can be converted immediately and pick one
 
         def makebranchsorter():
             """If the previously converted revision has a child in the
             eligible revisions list, pick it. Return the list head
             otherwise. Branch sort attempts to minimize branch
             switching, which is harmful for Mercurial backend
             compression.
             """
             prev = [None]
 
             def picknext(nodes):
                 next = nodes[0]
                 for n in nodes:
                     if prev[0] in parents[n]:
                         next = n
                         break
                 prev[0] = next
                 return next
 
             return picknext
 
         def makesourcesorter():
             """Source specific sort."""
             keyfn = lambda n: self.commitcache[n].sortkey
 
             def picknext(nodes):
                 return sorted(nodes, key=keyfn)[0]
 
             return picknext
 
         def makeclosesorter():
             """Close order sort."""
             keyfn = lambda n: (
                 b'close' not in self.commitcache[n].extra,
                 self.commitcache[n].sortkey,
             )
 
             def picknext(nodes):
                 return sorted(nodes, key=keyfn)[0]
 
             return picknext
 
         def makedatesorter():
             """Sort revisions by date."""
             dates = {}
 
             def getdate(n):
                 if n not in dates:
                     dates[n] = dateutil.parsedate(self.commitcache[n].date)
                 return dates[n]
 
             def picknext(nodes):
                 return min([(getdate(n), n) for n in nodes])[1]
 
             return picknext
 
         if sortmode == b'branchsort':
             picknext = makebranchsorter()
         elif sortmode == b'datesort':
             picknext = makedatesorter()
         elif sortmode == b'sourcesort':
             picknext = makesourcesorter()
         elif sortmode == b'closesort':
             picknext = makeclosesorter()
         else:
             raise error.Abort(_(b'unknown sort mode: %s') % sortmode)
 
         children, actives = mapchildren(parents)
 
         s = []
         pendings = {}
         while actives:
             n = picknext(actives)
             actives.remove(n)
             s.append(n)
 
             # Update dependents list
             for c in children.get(n, []):
                 if c not in pendings:
                     pendings[c] = [p for p in parents[c] if p not in self.map]
                 try:
                     pendings[c].remove(n)
                 except ValueError:
                     raise error.Abort(
                         _(b'cycle detected between %s and %s')
                         % (recode(c), recode(n))
                     )
                 if not pendings[c]:
                     # Parents are converted, node is eligible
                     actives.insert(0, c)
                     pendings[c] = None
 
         if len(s) != len(parents):
             raise error.Abort(_(b"not all revisions were sorted"))
 
         return s
 
     def writeauthormap(self):
         authorfile = self.authorfile
         if authorfile:
             self.ui.status(_(b'writing author map file %s\n') % authorfile)
             ofile = open(authorfile, b'wb+')
             for author in self.authors:
                 ofile.write(
                     util.tonativeeol(
                         b"%s=%s\n" % (author, self.authors[author])
                     )
                 )
             ofile.close()
 
     def readauthormap(self, authorfile):
         self.authors = readauthormap(self.ui, authorfile, self.authors)
 
     def cachecommit(self, rev):
         commit = self.source.getcommit(rev)
         commit.author = self.authors.get(commit.author, commit.author)
         commit.branch = mapbranch(commit.branch, self.branchmap)
         self.commitcache[rev] = commit
         return commit
 
     def copy(self, rev):
         commit = self.commitcache[rev]
         full = self.opts.get(b'full')
         changes = self.source.getchanges(rev, full)
         if isinstance(changes, bytes):
             if changes == SKIPREV:
                 dest = SKIPREV
             else:
                 dest = self.map[changes]
             self.map[rev] = dest
             return
         files, copies, cleanp2 = changes
         pbranches = []
         if commit.parents:
             for prev in commit.parents:
                 if prev not in self.commitcache:
                     self.cachecommit(prev)
                 pbranches.append(
                     (self.map[prev], self.commitcache[prev].branch)
                 )
         self.dest.setbranch(commit.branch, pbranches)
         try:
             parents = self.splicemap[rev]
             self.ui.status(
                 _(b'spliced in %s as parents of %s\n')
                 % (_(b' and ').join(parents), rev)
             )
             parents = [self.map.get(p, p) for p in parents]
         except KeyError:
             parents = [b[0] for b in pbranches]
             parents.extend(
                 self.map[x] for x in commit.optparents if x in self.map
             )
         if len(pbranches) != 2:
             cleanp2 = set()
         if len(parents) < 3:
             source = progresssource(self.ui, self.source, len(files))
         else:
             # For an octopus merge, we end up traversing the list of
             # changed files N-1 times. This tweak to the number of
             # files makes it so the progress bar doesn't overflow
             # itself.
             source = progresssource(
                 self.ui, self.source, len(files) * (len(parents) - 1)
             )
         newnode = self.dest.putcommit(
             files, copies, parents, commit, source, self.map, full, cleanp2
         )
         source.close()
         self.source.converted(rev, newnode)
         self.map[rev] = newnode
 
     def convert(self, sortmode):
         try:
             self.source.before()
             self.dest.before()
             self.source.setrevmap(self.map)
             self.ui.status(_(b"scanning source...\n"))
             heads = self.source.getheads()
             parents = self.walktree(heads)
             self.mergesplicemap(parents, self.splicemap)
             self.ui.status(_(b"sorting...\n"))
             t = self.toposort(parents, sortmode)
             num = len(t)
             c = None
 
             self.ui.status(_(b"converting...\n"))
             progress = self.ui.makeprogress(
                 _(b'converting'), unit=_(b'revisions'), total=len(t)
             )
             for i, c in enumerate(t):
                 num -= 1
                 desc = self.commitcache[c].desc
                 if b"\n" in desc:
                     desc = desc.splitlines()[0]
                 # convert log message to local encoding without using
                 # tolocal() because the encoding.encoding convert()
                 # uses is 'utf-8'
                 self.ui.status(b"%d %s\n" % (num, recode(desc)))
                 self.ui.note(_(b"source: %s\n") % recode(c))
                 progress.update(i)
                 self.copy(c)
             progress.complete()
 
             if not self.ui.configbool(b'convert', b'skiptags'):
                 tags = self.source.gettags()
                 ctags = {}
                 for k in tags:
                     v = tags[k]
                     if self.map.get(v, SKIPREV) != SKIPREV:
                         ctags[k] = self.map[v]
 
                 if c and ctags:
                     nrev, tagsparent = self.dest.puttags(ctags)
                     if nrev and tagsparent:
                         # write another hash correspondence to override the
                         # previous one so we don't end up with extra tag heads
                         tagsparents = [
                             e for e in self.map.items() if e[1] == tagsparent
                         ]
                         if tagsparents:
                             self.map[tagsparents[0][0]] = nrev
 
             bookmarks = self.source.getbookmarks()
             cbookmarks = {}
             for k in bookmarks:
                 v = bookmarks[k]
                 if self.map.get(v, SKIPREV) != SKIPREV:
                     cbookmarks[k] = self.map[v]
 
             if c and cbookmarks:
                 self.dest.putbookmarks(cbookmarks)
 
             self.writeauthormap()
         finally:
             self.cleanup()
 
     def cleanup(self):
         try:
             self.dest.after()
         finally:
             self.source.after()
         self.map.close()
 
 
 def convert(ui, src, dest=None, revmapfile=None, **opts):
     opts = pycompat.byteskwargs(opts)
     global orig_encoding
     orig_encoding = encoding.encoding
     encoding.encoding = b'UTF-8'
 
     # support --authors as an alias for --authormap
     if not opts.get(b'authormap'):
         opts[b'authormap'] = opts.get(b'authors')
 
     if not dest:
         dest = hg.defaultdest(src) + b"-hg"
         ui.status(_(b"assuming destination %s\n") % dest)
 
     destc = convertsink(ui, dest, opts.get(b'dest_type'))
     destc = scmutil.wrapconvertsink(destc)
 
     try:
         srcc, defaultsort = convertsource(
             ui, src, opts.get(b'source_type'), opts.get(b'rev')
         )
     except Exception:
         for path in destc.created:
             shutil.rmtree(path, True)
         raise
 
     sortmodes = (b'branchsort', b'datesort', b'sourcesort', b'closesort')
     sortmode = [m for m in sortmodes if opts.get(m)]
     if len(sortmode) > 1:
         raise error.Abort(_(b'more than one sort mode specified'))
     if sortmode:
         sortmode = sortmode[0]
     else:
         sortmode = defaultsort
 
     if sortmode == b'sourcesort' and not srcc.hasnativeorder():
         raise error.Abort(
             _(b'--sourcesort is not supported by this data source')
         )
     if sortmode == b'closesort' and not srcc.hasnativeclose():
         raise error.Abort(
             _(b'--closesort is not supported by this data source')
         )
 
     fmap = opts.get(b'filemap')
     if fmap:
         srcc = filemap.filemap_source(ui, srcc, fmap)
         destc.setfilemapmode(True)
 
     if not revmapfile:
         revmapfile = destc.revmapfile()
 
     c = converter(ui, srcc, destc, revmapfile, opts)
     c.convert(sortmode)

hgext/convert/darcs.py

@@ -1,241 +1,241
 # darcs.py - darcs support for the convert extension
 #
 #  Copyright 2007-2009 Olivia Mackall <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 import errno
 import os
 import re
 import shutil
 
 from mercurial.i18n import _
 from mercurial import (
     error,
     pycompat,
     util,
 )
 from mercurial.utils import dateutil
 from . import common
 
 NoRepo = common.NoRepo
 
 # The naming drift of ElementTree is fun!
 
 try:
     import xml.etree.cElementTree.ElementTree as ElementTree
     import xml.etree.cElementTree.XMLParser as XMLParser
 except ImportError:
     try:
         import xml.etree.ElementTree.ElementTree as ElementTree
         import xml.etree.ElementTree.XMLParser as XMLParser
     except ImportError:
         try:
             import elementtree.cElementTree.ElementTree as ElementTree
             import elementtree.cElementTree.XMLParser as XMLParser
         except ImportError:
             try:
                 import elementtree.ElementTree.ElementTree as ElementTree
                 import elementtree.ElementTree.XMLParser as XMLParser
             except ImportError:
                 pass
 
 
 class darcs_source(common.converter_source, common.commandline):
     def __init__(self, ui, repotype, path, revs=None):
         common.converter_source.__init__(self, ui, repotype, path, revs=revs)
         common.commandline.__init__(self, ui, b'darcs')
 
         # check for _darcs, ElementTree so that we can easily skip
         # test-convert-darcs if ElementTree is not around
         if not os.path.exists(os.path.join(path, b'_darcs')):
             raise NoRepo(_(b"%s does not look like a darcs repository") % path)
 
         common.checktool(b'darcs')
         version = self.run0(b'--version').splitlines()[0].strip()
         if version < b'2.1':
             raise error.Abort(
                 _(b'darcs version 2.1 or newer needed (found %r)') % version
             )
 
         if b"ElementTree" not in globals():
             raise error.Abort(_(b"Python ElementTree module is not available"))
 
         self.path = os.path.realpath(path)
 
         self.lastrev = None
         self.changes = {}
         self.parents = {}
         self.tags = {}
 
         # Check darcs repository format
         format = self.format()
         if format:
             if format in (b'darcs-1.0', b'hashed'):
                 raise NoRepo(
                     _(
                         b"%s repository format is unsupported, "
                         b"please upgrade"
                     )
                     % format
                 )
         else:
             self.ui.warn(_(b'failed to detect repository format!'))
 
     def before(self):
         self.tmppath = pycompat.mkdtemp(
             prefix=b'convert-' + os.path.basename(self.path) + b'-'
         )
         output, status = self.run(b'init', repodir=self.tmppath)
         self.checkexit(status)
 
         tree = self.xml(
             b'changes', xml_output=True, summary=True, repodir=self.path
         )
         tagname = None
         child = None
         for elt in tree.findall(b'patch'):
             node = elt.get(b'hash')
             name = elt.findtext(b'name', b'')
             if name.startswith(b'TAG '):
                 tagname = name[4:].strip()
             elif tagname is not None:
                 self.tags[tagname] = node
                 tagname = None
             self.changes[node] = elt
             self.parents[child] = [node]
             child = node
         self.parents[child] = []
 
     def after(self):
         self.ui.debug(b'cleaning up %s\n' % self.tmppath)
         shutil.rmtree(self.tmppath, ignore_errors=True)
 
     def recode(self, s, encoding=None):
-        if isinstance(s, pycompat.unicode):
+        if isinstance(s, str):
             # XMLParser returns unicode objects for anything it can't
             # encode into ASCII. We convert them back to str to get
             # recode's normal conversion behavior.
             s = s.encode('latin-1')
         return super(darcs_source, self).recode(s, encoding)
 
     def xml(self, cmd, **kwargs):
         # NOTE: darcs is currently encoding agnostic and will print
         # patch metadata byte-for-byte, even in the XML changelog.
         etree = ElementTree()
         # While we are decoding the XML as latin-1 to be as liberal as
         # possible, etree will still raise an exception if any
         # non-printable characters are in the XML changelog.
         parser = XMLParser(encoding=b'latin-1')
         p = self._run(cmd, **kwargs)
         etree.parse(p.stdout, parser=parser)
         p.wait()
         self.checkexit(p.returncode)
         return etree.getroot()
 
     def format(self):
         output, status = self.run(b'show', b'repo', repodir=self.path)
         self.checkexit(status)
         m = re.search(r'^\s*Format:\s*(.*)$', output, re.MULTILINE)
         if not m:
             return None
         return b','.join(sorted(f.strip() for f in m.group(1).split(b',')))
 
     def manifest(self):
         man = []
         output, status = self.run(
             b'show', b'files', no_directories=True, repodir=self.tmppath
         )
         self.checkexit(status)
         for line in output.split(b'\n'):
             path = line[2:]
             if path:
                 man.append(path)
         return man
 
     def getheads(self):
         return self.parents[None]
 
     def getcommit(self, rev):
         elt = self.changes[rev]
         dateformat = b'%a %b %d %H:%M:%S %Z %Y'
         date = dateutil.strdate(elt.get(b'local_date'), dateformat)
         desc = elt.findtext(b'name') + b'\n' + elt.findtext(b'comment', b'')
         # etree can return unicode objects for name, comment, and author,
         # so recode() is used to ensure str objects are emitted.
         newdateformat = b'%Y-%m-%d %H:%M:%S %1%2'
         return common.commit(
             author=self.recode(elt.get(b'author')),
             date=dateutil.datestr(date, newdateformat),
             desc=self.recode(desc).strip(),
             parents=self.parents[rev],
         )
 
     def pull(self, rev):
         output, status = self.run(
             b'pull',
             self.path,
             all=True,
             match=b'hash %s' % rev,
             no_test=True,
             no_posthook=True,
             external_merge=b'/bin/false',
             repodir=self.tmppath,
         )
         if status:
             if output.find(b'We have conflicts in') == -1:
                 self.checkexit(status, output)
             output, status = self.run(b'revert', all=True, repodir=self.tmppath)
             self.checkexit(status, output)
 
     def getchanges(self, rev, full):
         if full:
             raise error.Abort(_(b"convert from darcs does not support --full"))
         copies = {}
         changes = []
         man = None
         for elt in self.changes[rev].find(b'summary').getchildren():
             if elt.tag in (b'add_directory', b'remove_directory'):
                 continue
             if elt.tag == b'move':
                 if man is None:
                     man = self.manifest()
                 source, dest = elt.get(b'from'), elt.get(b'to')
                 if source in man:
                     # File move
                     changes.append((source, rev))
                     changes.append((dest, rev))
                     copies[dest] = source
                 else:
                     # Directory move, deduce file moves from manifest
                     source = source + b'/'
                     for f in man:
                         if not f.startswith(source):
                             continue
                         fdest = dest + b'/' + f[len(source) :]
                         changes.append((f, rev))
                         changes.append((fdest, rev))
                         copies[fdest] = f
             else:
                 changes.append((elt.text.strip(), rev))
         self.pull(rev)
         self.lastrev = rev
         return sorted(changes), copies, set()
 
     def getfile(self, name, rev):
         if rev != self.lastrev:
             raise error.Abort(_(b'internal calling inconsistency'))
         path = os.path.join(self.tmppath, name)
         try:
             data = util.readfile(path)
             mode = os.lstat(path).st_mode
         except IOError as inst:
             if inst.errno == errno.ENOENT:
                 return None, None
             raise
         mode = (mode & 0o111) and b'x' or b''
         return data, mode
 
     def gettags(self):
         return self.tags

hgext/lfs/blobstore.py

@@ -1,769 +1,769
 # blobstore.py - local and remote (speaking Git-LFS protocol) blob storages
 #
 # Copyright 2017 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 
 import contextlib
 import errno
 import hashlib
 import json
 import os
 import re
 import socket
 
 from mercurial.i18n import _
 from mercurial.pycompat import getattr
 from mercurial.node import hex
 
 from mercurial import (
     encoding,
     error,
     httpconnection as httpconnectionmod,
     pathutil,
     pycompat,
     url as urlmod,
     util,
     vfs as vfsmod,
     worker,
 )
 
 from mercurial.utils import (
     stringutil,
     urlutil,
 )
 
 from ..largefiles import lfutil
 
 # 64 bytes for SHA256
 _lfsre = re.compile(br'\A[a-f0-9]{64}\Z')
 
 
 class lfsvfs(vfsmod.vfs):
     def join(self, path):
         """split the path at first two characters, like: XX/XXXXX..."""
         if not _lfsre.match(path):
             raise error.ProgrammingError(b'unexpected lfs path: %s' % path)
         return super(lfsvfs, self).join(path[0:2], path[2:])
 
     def walk(self, path=None, onerror=None):
         """Yield (dirpath, [], oids) tuple for blobs under path
 
         Oids only exist in the root of this vfs, so dirpath is always ''.
         """
         root = os.path.normpath(self.base)
         # when dirpath == root, dirpath[prefixlen:] becomes empty
         # because len(dirpath) < prefixlen.
         prefixlen = len(pathutil.normasprefix(root))
         oids = []
 
         for dirpath, dirs, files in os.walk(
             self.reljoin(self.base, path or b''), onerror=onerror
         ):
             dirpath = dirpath[prefixlen:]
 
             # Silently skip unexpected files and directories
             if len(dirpath) == 2:
                 oids.extend(
                     [dirpath + f for f in files if _lfsre.match(dirpath + f)]
                 )
 
         yield (b'', [], oids)
 
 
 class nullvfs(lfsvfs):
     def __init__(self):
         pass
 
     def exists(self, oid):
         return False
 
     def read(self, oid):
         # store.read() calls into here if the blob doesn't exist in its
         # self.vfs.  Raise the same error as a normal vfs when asked to read a
         # file that doesn't exist.  The only difference is the full file path
         # isn't available in the error.
         raise IOError(
             errno.ENOENT,
             pycompat.sysstr(b'%s: No such file or directory' % oid),
         )
 
     def walk(self, path=None, onerror=None):
         return (b'', [], [])
 
     def write(self, oid, data):
         pass
 
 
 class lfsuploadfile(httpconnectionmod.httpsendfile):
     """a file-like object that supports keepalive."""
 
     def __init__(self, ui, filename):
         super(lfsuploadfile, self).__init__(ui, filename, b'rb')
         self.read = self._data.read
 
     def _makeprogress(self):
         return None  # progress is handled by the worker client
 
 
 class local(object):
     """Local blobstore for large file contents.
 
     This blobstore is used both as a cache and as a staging area for large blobs
     to be uploaded to the remote blobstore.
     """
 
     def __init__(self, repo):
         fullpath = repo.svfs.join(b'lfs/objects')
         self.vfs = lfsvfs(fullpath)
 
         if repo.ui.configbool(b'experimental', b'lfs.disableusercache'):
             self.cachevfs = nullvfs()
         else:
             usercache = lfutil._usercachedir(repo.ui, b'lfs')
             self.cachevfs = lfsvfs(usercache)
         self.ui = repo.ui
 
     def open(self, oid):
         """Open a read-only file descriptor to the named blob, in either the
         usercache or the local store."""
         return open(self.path(oid), 'rb')
 
     def path(self, oid):
         """Build the path for the given blob ``oid``.
 
         If the blob exists locally, the path may point to either the usercache
         or the local store.  If it doesn't, it will point to the local store.
         This is meant for situations where existing code that isn't LFS aware
         needs to open a blob.  Generally, prefer the ``open`` method on this
         class.
         """
         # The usercache is the most likely place to hold the file.  Commit will
         # write to both it and the local store, as will anything that downloads
         # the blobs.  However, things like clone without an update won't
         # populate the local store.  For an init + push of a local clone,
         # the usercache is the only place it _could_ be.  If not present, the
         # missing file msg here will indicate the local repo, not the usercache.
         if self.cachevfs.exists(oid):
             return self.cachevfs.join(oid)
 
         return self.vfs.join(oid)
 
     def download(self, oid, src, content_length):
         """Read the blob from the remote source in chunks, verify the content,
         and write to this local blobstore."""
         sha256 = hashlib.sha256()
         size = 0
 
         with self.vfs(oid, b'wb', atomictemp=True) as fp:
             for chunk in util.filechunkiter(src, size=1048576):
                 fp.write(chunk)
                 sha256.update(chunk)
                 size += len(chunk)
 
             # If the server advertised a length longer than what we actually
             # received, then we should expect that the server crashed while
             # producing the response (but the server has no way of telling us
             # that), and we really don't need to try to write the response to
             # the localstore, because it's not going to match the expected.
             if content_length is not None and int(content_length) != size:
                 msg = (
                     b"Response length (%s) does not match Content-Length "
                     b"header (%d): likely server-side crash"
                 )
                 raise LfsRemoteError(_(msg) % (size, int(content_length)))
 
             realoid = hex(sha256.digest())
             if realoid != oid:
                 raise LfsCorruptionError(
                     _(b'corrupt remote lfs object: %s') % oid
                 )
 
         self._linktousercache(oid)
 
     def write(self, oid, data):
         """Write blob to local blobstore.
 
         This should only be called from the filelog during a commit or similar.
         As such, there is no need to verify the data.  Imports from a remote
         store must use ``download()`` instead."""
         with self.vfs(oid, b'wb', atomictemp=True) as fp:
             fp.write(data)
 
         self._linktousercache(oid)
 
     def linkfromusercache(self, oid):
         """Link blobs found in the user cache into this store.
 
         The server module needs to do this when it lets the client know not to
         upload the blob, to ensure it is always available in this store.
         Normally this is done implicitly when the client reads or writes the
         blob, but that doesn't happen when the server tells the client that it
         already has the blob.
         """
         if not isinstance(self.cachevfs, nullvfs) and not self.vfs.exists(oid):
             self.ui.note(_(b'lfs: found %s in the usercache\n') % oid)
             lfutil.link(self.cachevfs.join(oid), self.vfs.join(oid))
 
     def _linktousercache(self, oid):
         # XXX: should we verify the content of the cache, and hardlink back to
         # the local store on success, but truncate, write and link on failure?
         if not self.cachevfs.exists(oid) and not isinstance(
             self.cachevfs, nullvfs
         ):
             self.ui.note(_(b'lfs: adding %s to the usercache\n') % oid)
             lfutil.link(self.vfs.join(oid), self.cachevfs.join(oid))
 
     def read(self, oid, verify=True):
         """Read blob from local blobstore."""
         if not self.vfs.exists(oid):
             blob = self._read(self.cachevfs, oid, verify)
 
             # Even if revlog will verify the content, it needs to be verified
             # now before making the hardlink to avoid propagating corrupt blobs.
             # Don't abort if corruption is detected, because `hg verify` will
             # give more useful info about the corruption- simply don't add the
             # hardlink.
             if verify or hex(hashlib.sha256(blob).digest()) == oid:
                 self.ui.note(_(b'lfs: found %s in the usercache\n') % oid)
                 lfutil.link(self.cachevfs.join(oid), self.vfs.join(oid))
         else:
             self.ui.note(_(b'lfs: found %s in the local lfs store\n') % oid)
             blob = self._read(self.vfs, oid, verify)
         return blob
 
     def _read(self, vfs, oid, verify):
         """Read blob (after verifying) from the given store"""
         blob = vfs.read(oid)
         if verify:
             _verify(oid, blob)
         return blob
 
     def verify(self, oid):
         """Indicate whether or not the hash of the underlying file matches its
         name."""
         sha256 = hashlib.sha256()
 
         with self.open(oid) as fp:
             for chunk in util.filechunkiter(fp, size=1048576):
                 sha256.update(chunk)
 
         return oid == hex(sha256.digest())
 
     def has(self, oid):
         """Returns True if the local blobstore contains the requested blob,
         False otherwise."""
         return self.cachevfs.exists(oid) or self.vfs.exists(oid)
 
 
 def _urlerrorreason(urlerror):
     """Create a friendly message for the given URLError to be used in an
     LfsRemoteError message.
     """
     inst = urlerror
 
     if isinstance(urlerror.reason, Exception):
         inst = urlerror.reason
 
     if util.safehasattr(inst, b'reason'):
         try:  # usually it is in the form (errno, strerror)
             reason = inst.reason.args[1]
         except (AttributeError, IndexError):
             # it might be anything, for example a string
             reason = inst.reason
-        if isinstance(reason, pycompat.unicode):
+        if isinstance(reason, str):
             # SSLError of Python 2.7.9 contains a unicode
             reason = encoding.unitolocal(reason)
         return reason
     elif getattr(inst, "strerror", None):
         return encoding.strtolocal(inst.strerror)
     else:
         return stringutil.forcebytestr(urlerror)
 
 
 class lfsauthhandler(util.urlreq.basehandler):
     handler_order = 480  # Before HTTPDigestAuthHandler (== 490)
 
     def http_error_401(self, req, fp, code, msg, headers):
         """Enforces that any authentication performed is HTTP Basic
         Authentication.  No authentication is also acceptable.
         """
         authreq = headers.get('www-authenticate', None)
         if authreq:
             scheme = authreq.split()[0]
 
             if scheme.lower() != 'basic':
                 msg = _(b'the server must support Basic Authentication')
                 raise util.urlerr.httperror(
                     req.get_full_url(),
                     code,
                     encoding.strfromlocal(msg),
                     headers,
                     fp,
                 )
         return None
 
 
 class _gitlfsremote(object):
     def __init__(self, repo, url):
         ui = repo.ui
         self.ui = ui
         baseurl, authinfo = url.authinfo()
         self.baseurl = baseurl.rstrip(b'/')
         useragent = repo.ui.config(b'experimental', b'lfs.user-agent')
         if not useragent:
             useragent = b'git-lfs/2.3.4 (Mercurial %s)' % util.version()
         self.urlopener = urlmod.opener(ui, authinfo, useragent)
         self.urlopener.add_handler(lfsauthhandler())
         self.retry = ui.configint(b'lfs', b'retry')
 
     def writebatch(self, pointers, fromstore):
         """Batch upload from local to remote blobstore."""
         self._batch(_deduplicate(pointers), fromstore, b'upload')
 
     def readbatch(self, pointers, tostore):
         """Batch download from remote to local blostore."""
         self._batch(_deduplicate(pointers), tostore, b'download')
 
     def _batchrequest(self, pointers, action):
         """Get metadata about objects pointed by pointers for given action
 
         Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]}
         See https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md
         """
         objects = [
             {'oid': pycompat.strurl(p.oid()), 'size': p.size()}
             for p in pointers
         ]
         requestdata = pycompat.bytesurl(
             json.dumps(
                 {
                     'objects': objects,
                     'operation': pycompat.strurl(action),
                 }
             )
         )
         url = b'%s/objects/batch' % self.baseurl
         batchreq = util.urlreq.request(pycompat.strurl(url), data=requestdata)
         batchreq.add_header('Accept', 'application/vnd.git-lfs+json')
         batchreq.add_header('Content-Type', 'application/vnd.git-lfs+json')
         try:
             with contextlib.closing(self.urlopener.open(batchreq)) as rsp:
                 rawjson = rsp.read()
         except util.urlerr.httperror as ex:
             hints = {
                 400: _(
                     b'check that lfs serving is enabled on %s and "%s" is '
                     b'supported'
                 )
                 % (self.baseurl, action),
                 404: _(b'the "lfs.url" config may be used to override %s')
                 % self.baseurl,
             }
             hint = hints.get(ex.code, _(b'api=%s, action=%s') % (url, action))
             raise LfsRemoteError(
                 _(b'LFS HTTP error: %s') % stringutil.forcebytestr(ex),
                 hint=hint,
             )
         except util.urlerr.urlerror as ex:
             hint = (
                 _(b'the "lfs.url" config may be used to override %s')
                 % self.baseurl
             )
             raise LfsRemoteError(
                 _(b'LFS error: %s') % _urlerrorreason(ex), hint=hint
             )
         try:
             response = pycompat.json_loads(rawjson)
         except ValueError:
             raise LfsRemoteError(
                 _(b'LFS server returns invalid JSON: %s')
                 % rawjson.encode("utf-8")
             )
 
         if self.ui.debugflag:
             self.ui.debug(b'Status: %d\n' % rsp.status)
             # lfs-test-server and hg serve return headers in different order
             headers = pycompat.bytestr(rsp.info()).strip()
             self.ui.debug(b'%s\n' % b'\n'.join(sorted(headers.splitlines())))
 
             if 'objects' in response:
                 response['objects'] = sorted(
                     response['objects'], key=lambda p: p['oid']
                 )
             self.ui.debug(
                 b'%s\n'
                 % pycompat.bytesurl(
                     json.dumps(
                         response,
                         indent=2,
                         separators=('', ': '),
                         sort_keys=True,
                     )
                 )
             )
 
         def encodestr(x):
-            if isinstance(x, pycompat.unicode):
+            if isinstance(x, str):
                 return x.encode('utf-8')
             return x
 
         return pycompat.rapply(encodestr, response)
 
     def _checkforservererror(self, pointers, responses, action):
         """Scans errors from objects
 
         Raises LfsRemoteError if any objects have an error"""
         for response in responses:
             # The server should return 404 when objects cannot be found. Some
             # server implementation (ex. lfs-test-server)  does not set "error"
             # but just removes "download" from "actions". Treat that case
             # as the same as 404 error.
             if b'error' not in response:
                 if action == b'download' and action not in response.get(
                     b'actions', []
                 ):
                     code = 404
                 else:
                     continue
             else:
                 # An error dict without a code doesn't make much sense, so
                 # treat as a server error.
                 code = response.get(b'error').get(b'code', 500)
 
             ptrmap = {p.oid(): p for p in pointers}
             p = ptrmap.get(response[b'oid'], None)
             if p:
                 filename = getattr(p, 'filename', b'unknown')
                 errors = {
                     404: b'The object does not exist',
                     410: b'The object was removed by the owner',
                     422: b'Validation error',
                     500: b'Internal server error',
                 }
                 msg = errors.get(code, b'status code %d' % code)
                 raise LfsRemoteError(
                     _(b'LFS server error for "%s": %s') % (filename, msg)
                 )
             else:
                 raise LfsRemoteError(
                     _(b'LFS server error. Unsolicited response for oid %s')
                     % response[b'oid']
                 )
 
     def _extractobjects(self, response, pointers, action):
         """extract objects from response of the batch API
 
         response: parsed JSON object returned by batch API
         return response['objects'] filtered by action
         raise if any object has an error
         """
         # Scan errors from objects - fail early
         objects = response.get(b'objects', [])
         self._checkforservererror(pointers, objects, action)
 
         # Filter objects with given action. Practically, this skips uploading
         # objects which exist in the server.
         filteredobjects = [
             o for o in objects if action in o.get(b'actions', [])
         ]
 
         return filteredobjects
 
     def _basictransfer(self, obj, action, localstore):
         """Download or upload a single object using basic transfer protocol
 
         obj: dict, an object description returned by batch API
         action: string, one of ['upload', 'download']
         localstore: blobstore.local
 
         See https://github.com/git-lfs/git-lfs/blob/master/docs/api/\
         basic-transfers.md
         """
         oid = obj[b'oid']
         href = obj[b'actions'][action].get(b'href')
         headers = obj[b'actions'][action].get(b'header', {}).items()
 
         request = util.urlreq.request(pycompat.strurl(href))
         if action == b'upload':
             # If uploading blobs, read data from local blobstore.
             if not localstore.verify(oid):
                 raise error.Abort(
                     _(b'detected corrupt lfs object: %s') % oid,
                     hint=_(b'run hg verify'),
                 )
 
         for k, v in headers:
             request.add_header(pycompat.strurl(k), pycompat.strurl(v))
 
         try:
             if action == b'upload':
                 request.data = lfsuploadfile(self.ui, localstore.path(oid))
                 request.get_method = lambda: 'PUT'
                 request.add_header('Content-Type', 'application/octet-stream')
                 request.add_header('Content-Length', request.data.length)
 
             with contextlib.closing(self.urlopener.open(request)) as res:
                 contentlength = res.info().get(b"content-length")
                 ui = self.ui  # Shorten debug lines
                 if self.ui.debugflag:
                     ui.debug(b'Status: %d\n' % res.status)
                     # lfs-test-server and hg serve return headers in different
                     # order
                     headers = pycompat.bytestr(res.info()).strip()
                     ui.debug(b'%s\n' % b'\n'.join(sorted(headers.splitlines())))
 
                 if action == b'download':
                     # If downloading blobs, store downloaded data to local
                     # blobstore
                     localstore.download(oid, res, contentlength)
                 else:
                     blocks = []
                     while True:
                         data = res.read(1048576)
                         if not data:
                             break
                         blocks.append(data)
 
                     response = b"".join(blocks)
                     if response:
                         ui.debug(b'lfs %s response: %s' % (action, response))
         except util.urlerr.httperror as ex:
             if self.ui.debugflag:
                 self.ui.debug(
                     b'%s: %s\n' % (oid, ex.read())
                 )  # XXX: also bytes?
             raise LfsRemoteError(
                 _(b'LFS HTTP error: %s (oid=%s, action=%s)')
                 % (stringutil.forcebytestr(ex), oid, action)
             )
         except util.urlerr.urlerror as ex:
             hint = _(b'attempted connection to %s') % pycompat.bytesurl(
                 util.urllibcompat.getfullurl(request)
             )
             raise LfsRemoteError(
                 _(b'LFS error: %s') % _urlerrorreason(ex), hint=hint
             )
         finally:
             if request.data:
                 request.data.close()
 
     def _batch(self, pointers, localstore, action):
         if action not in [b'upload', b'download']:
             raise error.ProgrammingError(b'invalid Git-LFS action: %s' % action)
 
         response = self._batchrequest(pointers, action)
         objects = self._extractobjects(response, pointers, action)
         total = sum(x.get(b'size', 0) for x in objects)
         sizes = {}
         for obj in objects:
             sizes[obj.get(b'oid')] = obj.get(b'size', 0)
         topic = {
             b'upload': _(b'lfs uploading'),
             b'download': _(b'lfs downloading'),
         }[action]
         if len(objects) > 1:
             self.ui.note(
                 _(b'lfs: need to transfer %d objects (%s)\n')
                 % (len(objects), util.bytecount(total))
             )
 
         def transfer(chunk):
             for obj in chunk:
                 objsize = obj.get(b'size', 0)
                 if self.ui.verbose:
                     if action == b'download':
                         msg = _(b'lfs: downloading %s (%s)\n')
                     elif action == b'upload':
                         msg = _(b'lfs: uploading %s (%s)\n')
                     self.ui.note(
                         msg % (obj.get(b'oid'), util.bytecount(objsize))
                     )
                 retry = self.retry
                 while True:
                     try:
                         self._basictransfer(obj, action, localstore)
                         yield 1, obj.get(b'oid')
                         break
                     except socket.error as ex:
                         if retry > 0:
                             self.ui.note(
                                 _(b'lfs: failed: %r (remaining retry %d)\n')
                                 % (stringutil.forcebytestr(ex), retry)
                             )
                             retry -= 1
                             continue
                         raise
 
         # Until https multiplexing gets sorted out
         if self.ui.configbool(b'experimental', b'lfs.worker-enable'):
             oids = worker.worker(
                 self.ui,
                 0.1,
                 transfer,
                 (),
                 sorted(objects, key=lambda o: o.get(b'oid')),
             )
         else:
             oids = transfer(sorted(objects, key=lambda o: o.get(b'oid')))
 
         with self.ui.makeprogress(
             topic, unit=_(b"bytes"), total=total
         ) as progress:
             progress.update(0)
             processed = 0
             blobs = 0
             for _one, oid in oids:
                 processed += sizes[oid]
                 blobs += 1
                 progress.update(processed)
                 self.ui.note(_(b'lfs: processed: %s\n') % oid)
 
         if blobs > 0:
             if action == b'upload':
                 self.ui.status(
                     _(b'lfs: uploaded %d files (%s)\n')
                     % (blobs, util.bytecount(processed))
                 )
             elif action == b'download':
                 self.ui.status(
                     _(b'lfs: downloaded %d files (%s)\n')
                     % (blobs, util.bytecount(processed))
                 )
 
     def __del__(self):
         # copied from mercurial/httppeer.py
         urlopener = getattr(self, 'urlopener', None)
         if urlopener:
             for h in urlopener.handlers:
                 h.close()
                 getattr(h, "close_all", lambda: None)()
 
 
 class _dummyremote(object):
     """Dummy store storing blobs to temp directory."""
 
     def __init__(self, repo, url):
         fullpath = repo.vfs.join(b'lfs', url.path)
         self.vfs = lfsvfs(fullpath)
 
     def writebatch(self, pointers, fromstore):
         for p in _deduplicate(pointers):
             content = fromstore.read(p.oid(), verify=True)
             with self.vfs(p.oid(), b'wb', atomictemp=True) as fp:
                 fp.write(content)
 
     def readbatch(self, pointers, tostore):
         for p in _deduplicate(pointers):
             with self.vfs(p.oid(), b'rb') as fp:
                 tostore.download(p.oid(), fp, None)
 
 
 class _nullremote(object):
     """Null store storing blobs to /dev/null."""
 
     def __init__(self, repo, url):
         pass
 
     def writebatch(self, pointers, fromstore):
         pass
 
     def readbatch(self, pointers, tostore):
         pass
 
 
 class _promptremote(object):
     """Prompt user to set lfs.url when accessed."""
 
     def __init__(self, repo, url):
         pass
 
     def writebatch(self, pointers, fromstore, ui=None):
         self._prompt()
 
     def readbatch(self, pointers, tostore, ui=None):
         self._prompt()
 
     def _prompt(self):
         raise error.Abort(_(b'lfs.url needs to be configured'))
 
 
 _storemap = {
     b'https': _gitlfsremote,
     b'http': _gitlfsremote,
     b'file': _dummyremote,
     b'null': _nullremote,
     None: _promptremote,
 }
 
 
 def _deduplicate(pointers):
     """Remove any duplicate oids that exist in the list"""
     reduced = util.sortdict()
     for p in pointers:
         reduced[p.oid()] = p
     return reduced.values()
 
 
 def _verify(oid, content):
     realoid = hex(hashlib.sha256(content).digest())
     if realoid != oid:
         raise LfsCorruptionError(
             _(b'detected corrupt lfs object: %s') % oid,
             hint=_(b'run hg verify'),
         )
 
 
 def remote(repo, remote=None):
     """remotestore factory. return a store in _storemap depending on config
 
     If ``lfs.url`` is specified, use that remote endpoint.  Otherwise, try to
     infer the endpoint, based on the remote repository using the same path
     adjustments as git.  As an extension, 'http' is supported as well so that
     ``hg serve`` works out of the box.
 
     https://github.com/git-lfs/git-lfs/blob/master/docs/api/server-discovery.md
     """
     lfsurl = repo.ui.config(b'lfs', b'url')
     url = urlutil.url(lfsurl or b'')
     if lfsurl is None:
         if remote:
             path = remote
         elif util.safehasattr(repo, b'_subtoppath'):
             # The pull command sets this during the optional update phase, which
             # tells exactly where the pull originated, whether 'paths.default'
             # or explicit.
             path = repo._subtoppath
         else:
             # TODO: investigate 'paths.remote:lfsurl' style path customization,
             # and fall back to inferring from 'paths.remote' if unspecified.
             path = repo.ui.config(b'paths', b'default') or b''
 
         defaulturl = urlutil.url(path)
 
         # TODO: support local paths as well.
         # TODO: consider the ssh -> https transformation that git applies
         if defaulturl.scheme in (b'http', b'https'):
             if defaulturl.path and defaulturl.path[:-1] != b'/':
                 defaulturl.path += b'/'
             defaulturl.path = (defaulturl.path or b'') + b'.git/info/lfs'
 
             url = urlutil.url(bytes(defaulturl))
             repo.ui.note(_(b'lfs: assuming remote store: %s\n') % url)
 
     scheme = url.scheme
     if scheme not in _storemap:
         raise error.Abort(_(b'lfs: unknown url scheme: %s') % scheme)
     return _storemap[scheme](repo, url)
 
 
 class LfsRemoteError(error.StorageError):
     pass
 
 
 class LfsCorruptionError(error.Abort):
     """Raised when a corrupt blob is detected, aborting an operation
 
     It exists to allow specialized handling on the server side."""

hgext/phabricator.py

@@ -1,2401 +1,2395
 # phabricator.py - simple Phabricator integration
 #
 # Copyright 2017 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 """simple Phabricator integration (EXPERIMENTAL)
 
 This extension provides a ``phabsend`` command which sends a stack of
 changesets to Phabricator, and a ``phabread`` command which prints a stack of
 revisions in a format suitable for :hg:`import`, and a ``phabupdate`` command
 to update statuses in batch.
 
 A "phabstatus" view for :hg:`show` is also provided; it displays status
 information of Phabricator differentials associated with unfinished
 changesets.
 
 By default, Phabricator requires ``Test Plan`` which might prevent some
 changeset from being sent. The requirement could be disabled by changing
 ``differential.require-test-plan-field`` config server side.
 
 Config::
 
     [phabricator]
     # Phabricator URL
     url = https://phab.example.com/
 
     # Repo callsign. If a repo has a URL https://$HOST/diffusion/FOO, then its
     # callsign is "FOO".
     callsign = FOO
 
     # curl command to use. If not set (default), use builtin HTTP library to
     # communicate. If set, use the specified curl command. This could be useful
     # if you need to specify advanced options that is not easily supported by
     # the internal library.
     curlcmd = curl --connect-timeout 2 --retry 3 --silent
 
     # retry failed command N time (default 0). Useful when using the extension
     # over flakly connection.
     #
     # We wait `retry.interval` between each retry, in seconds.
     # (default 1 second).
     retry = 3
     retry.interval = 10
 
     # the retry option can combine well with the http.timeout one.
     #
     # For example to give up on http request after 20 seconds:
     [http]
     timeout=20
 
     [auth]
     example.schemes = https
     example.prefix = phab.example.com
 
     # API token. Get it from https://$HOST/conduit/login/
     example.phabtoken = cli-xxxxxxxxxxxxxxxxxxxxxxxxxxxx
 """
 
 
 import base64
 import contextlib
 import hashlib
 import io
 import itertools
 import json
 import mimetypes
 import operator
 import re
 import time
 
 from mercurial.node import bin, short
 from mercurial.i18n import _
 from mercurial.pycompat import getattr
 from mercurial.thirdparty import attr
 from mercurial import (
     cmdutil,
     context,
     copies,
     encoding,
     error,
     exthelper,
     graphmod,
     httpconnection as httpconnectionmod,
     localrepo,
     logcmdutil,
     match,
     mdiff,
     obsutil,
     parser,
     patch,
     phases,
     pycompat,
     rewriteutil,
     scmutil,
     smartset,
     tags,
     templatefilters,
     templateutil,
     url as urlmod,
     util,
 )
 from mercurial.utils import (
     procutil,
     stringutil,
     urlutil,
 )
 from . import show
 
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 eh = exthelper.exthelper()
 
 cmdtable = eh.cmdtable
 command = eh.command
 configtable = eh.configtable
 templatekeyword = eh.templatekeyword
 uisetup = eh.finaluisetup
 
 # developer config: phabricator.batchsize
 eh.configitem(
     b'phabricator',
     b'batchsize',
     default=12,
 )
 eh.configitem(
     b'phabricator',
     b'callsign',
     default=None,
 )
 eh.configitem(
     b'phabricator',
     b'curlcmd',
     default=None,
 )
 # developer config: phabricator.debug
 eh.configitem(
     b'phabricator',
     b'debug',
     default=False,
 )
 # developer config: phabricator.repophid
 eh.configitem(
     b'phabricator',
     b'repophid',
     default=None,
 )
 eh.configitem(
     b'phabricator',
     b'retry',
     default=0,
 )
 eh.configitem(
     b'phabricator',
     b'retry.interval',
     default=1,
 )
 eh.configitem(
     b'phabricator',
     b'url',
     default=None,
 )
 eh.configitem(
     b'phabsend',
     b'confirm',
     default=False,
 )
 eh.configitem(
     b'phabimport',
     b'secret',
     default=False,
 )
 eh.configitem(
     b'phabimport',
     b'obsolete',
     default=False,
 )
 
 colortable = {
     b'phabricator.action.created': b'green',
     b'phabricator.action.skipped': b'magenta',
     b'phabricator.action.updated': b'magenta',
     b'phabricator.drev': b'bold',
     b'phabricator.status.abandoned': b'magenta dim',
     b'phabricator.status.accepted': b'green bold',
     b'phabricator.status.closed': b'green',
     b'phabricator.status.needsreview': b'yellow',
     b'phabricator.status.needsrevision': b'red',
     b'phabricator.status.changesplanned': b'red',
 }
 
 _VCR_FLAGS = [
     (
         b'',
         b'test-vcr',
         b'',
         _(
             b'Path to a vcr file. If nonexistent, will record a new vcr transcript'
             b', otherwise will mock all http requests using the specified vcr file.'
             b' (ADVANCED)'
         ),
     ),
 ]
 
 
 @eh.wrapfunction(localrepo, "loadhgrc")
 def _loadhgrc(orig, ui, wdirvfs, hgvfs, requirements, *args, **opts):
     """Load ``.arcconfig`` content into a ui instance on repository open."""
     result = False
     arcconfig = {}
 
     try:
         # json.loads only accepts bytes from 3.6+
         rawparams = encoding.unifromlocal(wdirvfs.read(b".arcconfig"))
         # json.loads only returns unicode strings
         arcconfig = pycompat.rapply(
-            lambda x: encoding.unitolocal(x)
-            if isinstance(x, pycompat.unicode)
-            else x,
+            lambda x: encoding.unitolocal(x) if isinstance(x, str) else x,
             pycompat.json_loads(rawparams),
         )
 
         result = True
     except ValueError:
         ui.warn(_(b"invalid JSON in %s\n") % wdirvfs.join(b".arcconfig"))
     except IOError:
         pass
 
     cfg = util.sortdict()
 
     if b"repository.callsign" in arcconfig:
         cfg[(b"phabricator", b"callsign")] = arcconfig[b"repository.callsign"]
 
     if b"phabricator.uri" in arcconfig:
         cfg[(b"phabricator", b"url")] = arcconfig[b"phabricator.uri"]
 
     if cfg:
         ui.applyconfig(cfg, source=wdirvfs.join(b".arcconfig"))
 
     return (
         orig(ui, wdirvfs, hgvfs, requirements, *args, **opts) or result
     )  # Load .hg/hgrc
 
 
 def vcrcommand(name, flags, spec, helpcategory=None, optionalrepo=False):
     fullflags = flags + _VCR_FLAGS
 
     def hgmatcher(r1, r2):
         if r1.uri != r2.uri or r1.method != r2.method:
             return False
         r1params = util.urlreq.parseqs(r1.body)
         r2params = util.urlreq.parseqs(r2.body)
         for key in r1params:
             if key not in r2params:
                 return False
             value = r1params[key][0]
             # we want to compare json payloads without worrying about ordering
             if value.startswith(b'{') and value.endswith(b'}'):
                 r1json = pycompat.json_loads(value)
                 r2json = pycompat.json_loads(r2params[key][0])
                 if r1json != r2json:
                     return False
             elif r2params[key][0] != value:
                 return False
         return True
 
     def sanitiserequest(request):
         request.body = re.sub(
             br'cli-[a-z0-9]+', br'cli-hahayouwish', request.body
         )
         return request
 
     def sanitiseresponse(response):
         if 'set-cookie' in response['headers']:
             del response['headers']['set-cookie']
         return response
 
     def decorate(fn):
         def inner(*args, **kwargs):
             vcr = kwargs.pop('test_vcr')
             if vcr:
                 cassette = pycompat.fsdecode(vcr)
                 import hgdemandimport
 
                 with hgdemandimport.deactivated():
                     import vcr as vcrmod
                     import vcr.stubs as stubs
 
                     vcr = vcrmod.VCR(
                         serializer='json',
                         before_record_request=sanitiserequest,
                         before_record_response=sanitiseresponse,
                         custom_patches=[
                             (
                                 urlmod,
                                 'httpconnection',
                                 stubs.VCRHTTPConnection,
                             ),
                             (
                                 urlmod,
                                 'httpsconnection',
                                 stubs.VCRHTTPSConnection,
                             ),
                         ],
                     )
                     vcr.register_matcher('hgmatcher', hgmatcher)
                     with vcr.use_cassette(cassette, match_on=['hgmatcher']):
                         return fn(*args, **kwargs)
             return fn(*args, **kwargs)
 
         cmd = util.checksignature(inner, depth=2)
         cmd.__name__ = fn.__name__
         cmd.__doc__ = fn.__doc__
 
         return command(
             name,
             fullflags,
             spec,
             helpcategory=helpcategory,
             optionalrepo=optionalrepo,
         )(cmd)
 
     return decorate
 
 
 def _debug(ui, *msg, **opts):
     """write debug output for Phabricator if ``phabricator.debug`` is set
 
     Specifically, this avoids dumping Conduit and HTTP auth chatter that is
     printed with the --debug argument.
     """
     if ui.configbool(b"phabricator", b"debug"):
         flag = ui.debugflag
         try:
             ui.debugflag = True
             ui.write(*msg, **opts)
         finally:
             ui.debugflag = flag
 
 
 def urlencodenested(params):
     """like urlencode, but works with nested parameters.
 
     For example, if params is {'a': ['b', 'c'], 'd': {'e': 'f'}}, it will be
     flattened to {'a[0]': 'b', 'a[1]': 'c', 'd[e]': 'f'} and then passed to
     urlencode. Note: the encoding is consistent with PHP's http_build_query.
     """
     flatparams = util.sortdict()
 
     def process(prefix, obj):
         if isinstance(obj, bool):
             obj = {True: b'true', False: b'false'}[obj]  # Python -> PHP form
         lister = lambda l: [(b'%d' % k, v) for k, v in enumerate(l)]
         items = {list: lister, dict: lambda x: x.items()}.get(type(obj))
         if items is None:
             flatparams[prefix] = obj
         else:
             for k, v in items(obj):
                 if prefix:
                     process(b'%s[%s]' % (prefix, k), v)
                 else:
                     process(k, v)
 
     process(b'', params)
     return urlutil.urlreq.urlencode(flatparams)
 
 
 def readurltoken(ui):
     """return conduit url, token and make sure they exist
 
     Currently read from [auth] config section. In the future, it might
     make sense to read from .arcconfig and .arcrc as well.
     """
     url = ui.config(b'phabricator', b'url')
     if not url:
         raise error.Abort(
             _(b'config %s.%s is required') % (b'phabricator', b'url')
         )
 
     res = httpconnectionmod.readauthforuri(ui, url, urlutil.url(url).user)
     token = None
 
     if res:
         group, auth = res
 
         ui.debug(b"using auth.%s.* for authentication\n" % group)
 
         token = auth.get(b'phabtoken')
 
     if not token:
         raise error.Abort(
             _(b'Can\'t find conduit token associated to %s') % (url,)
         )
 
     return url, token
 
 
 def callconduit(ui, name, params):
     """call Conduit API, params is a dict. return json.loads result, or None"""
     host, token = readurltoken(ui)
     url, authinfo = urlutil.url(b'/'.join([host, b'api', name])).authinfo()
     ui.debug(b'Conduit Call: %s %s\n' % (url, pycompat.byterepr(params)))
     params = params.copy()
     params[b'__conduit__'] = {
         b'token': token,
     }
     rawdata = {
         b'params': templatefilters.json(params),
         b'output': b'json',
         b'__conduit__': 1,
     }
     data = urlencodenested(rawdata)
     curlcmd = ui.config(b'phabricator', b'curlcmd')
     if curlcmd:
         sin, sout = procutil.popen2(
             b'%s -d @- %s' % (curlcmd, procutil.shellquote(url))
         )
         sin.write(data)
         sin.close()
         body = sout.read()
     else:
         urlopener = urlmod.opener(ui, authinfo)
         request = util.urlreq.request(pycompat.strurl(url), data=data)
         max_try = ui.configint(b'phabricator', b'retry') + 1
         timeout = ui.configwith(float, b'http', b'timeout')
         for try_count in range(max_try):
             try:
                 with contextlib.closing(
                     urlopener.open(request, timeout=timeout)
                 ) as rsp:
                     body = rsp.read()
                 break
             except util.urlerr.urlerror as err:
                 if try_count == max_try - 1:
                     raise
                 ui.debug(
                     b'Conduit Request failed (try %d/%d): %r\n'
                     % (try_count + 1, max_try, err)
                 )
                 # failing request might come from overloaded server
                 retry_interval = ui.configint(b'phabricator', b'retry.interval')
                 time.sleep(retry_interval)
     ui.debug(b'Conduit Response: %s\n' % body)
     parsed = pycompat.rapply(
-        lambda x: encoding.unitolocal(x)
-        if isinstance(x, pycompat.unicode)
-        else x,
+        lambda x: encoding.unitolocal(x) if isinstance(x, str) else x,
         # json.loads only accepts bytes from py3.6+
         pycompat.json_loads(encoding.unifromlocal(body)),
     )
     if parsed.get(b'error_code'):
         msg = _(b'Conduit Error (%s): %s') % (
             parsed[b'error_code'],
             parsed[b'error_info'],
         )
         raise error.Abort(msg)
     return parsed[b'result']
 
 
 @vcrcommand(b'debugcallconduit', [], _(b'METHOD'), optionalrepo=True)
 def debugcallconduit(ui, repo, name):
     """call Conduit API
 
     Call parameters are read from stdin as a JSON blob. Result will be written
     to stdout as a JSON blob.
     """
     # json.loads only accepts bytes from 3.6+
     rawparams = encoding.unifromlocal(ui.fin.read())
     # json.loads only returns unicode strings
     params = pycompat.rapply(
-        lambda x: encoding.unitolocal(x)
-        if isinstance(x, pycompat.unicode)
-        else x,
+        lambda x: encoding.unitolocal(x) if isinstance(x, str) else x,
         pycompat.json_loads(rawparams),
     )
     # json.dumps only accepts unicode strings
     result = pycompat.rapply(
         lambda x: encoding.unifromlocal(x) if isinstance(x, bytes) else x,
         callconduit(ui, name, params),
     )
     s = json.dumps(result, sort_keys=True, indent=2, separators=(u',', u': '))
     ui.write(b'%s\n' % encoding.unitolocal(s))
 
 
 def getrepophid(repo):
     """given callsign, return repository PHID or None"""
     # developer config: phabricator.repophid
     repophid = repo.ui.config(b'phabricator', b'repophid')
     if repophid:
         return repophid
     callsign = repo.ui.config(b'phabricator', b'callsign')
     if not callsign:
         return None
     query = callconduit(
         repo.ui,
         b'diffusion.repository.search',
         {b'constraints': {b'callsigns': [callsign]}},
     )
     if len(query[b'data']) == 0:
         return None
     repophid = query[b'data'][0][b'phid']
     repo.ui.setconfig(b'phabricator', b'repophid', repophid)
     return repophid
 
 
 _differentialrevisiontagre = re.compile(br'\AD([1-9][0-9]*)\Z')
 _differentialrevisiondescre = re.compile(
     br'^Differential Revision:\s*(?P<url>(?:.*)D(?P<id>[1-9][0-9]*))$', re.M
 )
 
 
 def getoldnodedrevmap(repo, nodelist):
     """find previous nodes that has been sent to Phabricator
 
     return {node: (oldnode, Differential diff, Differential Revision ID)}
     for node in nodelist with known previous sent versions, or associated
     Differential Revision IDs. ``oldnode`` and ``Differential diff`` could
     be ``None``.
 
     Examines commit messages like "Differential Revision:" to get the
     association information.
 
     If such commit message line is not found, examines all precursors and their
     tags. Tags with format like "D1234" are considered a match and the node
     with that tag, and the number after "D" (ex. 1234) will be returned.
 
     The ``old node``, if not None, is guaranteed to be the last diff of
     corresponding Differential Revision, and exist in the repo.
     """
     unfi = repo.unfiltered()
     has_node = unfi.changelog.index.has_node
 
     result = {}  # {node: (oldnode?, lastdiff?, drev)}
     # ordered for test stability when printing new -> old mapping below
     toconfirm = util.sortdict()  # {node: (force, {precnode}, drev)}
     for node in nodelist:
         ctx = unfi[node]
         # For tags like "D123", put them into "toconfirm" to verify later
         precnodes = list(obsutil.allpredecessors(unfi.obsstore, [node]))
         for n in precnodes:
             if has_node(n):
                 for tag in unfi.nodetags(n):
                     m = _differentialrevisiontagre.match(tag)
                     if m:
                         toconfirm[node] = (0, set(precnodes), int(m.group(1)))
                         break
                 else:
                     continue  # move to next predecessor
                 break  # found a tag, stop
         else:
             # Check commit message
             m = _differentialrevisiondescre.search(ctx.description())
             if m:
                 toconfirm[node] = (1, set(precnodes), int(m.group('id')))
 
     # Double check if tags are genuine by collecting all old nodes from
     # Phabricator, and expect precursors overlap with it.
     if toconfirm:
         drevs = [drev for force, precs, drev in toconfirm.values()]
         alldiffs = callconduit(
             unfi.ui, b'differential.querydiffs', {b'revisionIDs': drevs}
         )
 
         def getnodes(d, precset):
             # Ignore other nodes that were combined into the Differential
             # that aren't predecessors of the current local node.
             return [n for n in getlocalcommits(d) if n in precset]
 
         for newnode, (force, precset, drev) in toconfirm.items():
             diffs = [
                 d for d in alldiffs.values() if int(d[b'revisionID']) == drev
             ]
 
             # local predecessors known by Phabricator
             phprecset = {n for d in diffs for n in getnodes(d, precset)}
 
             # Ignore if precursors (Phabricator and local repo) do not overlap,
             # and force is not set (when commit message says nothing)
             if not force and not phprecset:
                 tagname = b'D%d' % drev
                 tags.tag(
                     repo,
                     tagname,
                     repo.nullid,
                     message=None,
                     user=None,
                     date=None,
                     local=True,
                 )
                 unfi.ui.warn(
                     _(
                         b'D%d: local tag removed - does not match '
                         b'Differential history\n'
                     )
                     % drev
                 )
                 continue
 
             # Find the last node using Phabricator metadata, and make sure it
             # exists in the repo
             oldnode = lastdiff = None
             if diffs:
                 lastdiff = max(diffs, key=lambda d: int(d[b'id']))
                 oldnodes = getnodes(lastdiff, precset)
 
                 _debug(
                     unfi.ui,
                     b"%s mapped to old nodes %s\n"
                     % (
                         short(newnode),
                         stringutil.pprint([short(n) for n in sorted(oldnodes)]),
                     ),
                 )
 
                 # If this commit was the result of `hg fold` after submission,
                 # and now resubmitted with --fold, the easiest thing to do is
                 # to leave the node clear.  This only results in creating a new
                 # diff for the _same_ Differential Revision if this commit is
                 # the first or last in the selected range.  If we picked a node
                 # from the list instead, it would have to be the lowest if at
                 # the beginning of the --fold range, or the highest at the end.
                 # Otherwise, one or more of the nodes wouldn't be considered in
                 # the diff, and the Differential wouldn't be properly updated.
                 # If this commit is the result of `hg split` in the same
                 # scenario, there is a single oldnode here (and multiple
                 # newnodes mapped to it).  That makes it the same as the normal
                 # case, as the edges of the newnode range cleanly maps to one
                 # oldnode each.
                 if len(oldnodes) == 1:
                     oldnode = oldnodes[0]
                 if oldnode and not has_node(oldnode):
                     oldnode = None
 
             result[newnode] = (oldnode, lastdiff, drev)
 
     return result
 
 
 def getdrevmap(repo, revs):
     """Return a dict mapping each rev in `revs` to their Differential Revision
     ID or None.
     """
     result = {}
     for rev in revs:
         result[rev] = None
         ctx = repo[rev]
         # Check commit message
         m = _differentialrevisiondescre.search(ctx.description())
         if m:
             result[rev] = int(m.group('id'))
             continue
         # Check tags
         for tag in repo.nodetags(ctx.node()):
             m = _differentialrevisiontagre.match(tag)
             if m:
                 result[rev] = int(m.group(1))
                 break
 
     return result
 
 
 def getdiff(basectx, ctx, diffopts):
     """plain-text diff without header (user, commit message, etc)"""
     output = util.stringio()
     for chunk, _label in patch.diffui(
         ctx.repo(), basectx.p1().node(), ctx.node(), None, opts=diffopts
     ):
         output.write(chunk)
     return output.getvalue()
 
 
 class DiffChangeType(object):
     ADD = 1
     CHANGE = 2
     DELETE = 3
     MOVE_AWAY = 4
     COPY_AWAY = 5
     MOVE_HERE = 6
     COPY_HERE = 7
     MULTICOPY = 8
 
 
 class DiffFileType(object):
     TEXT = 1
     IMAGE = 2
     BINARY = 3
 
 
 @attr.s
 class phabhunk(dict):
     """Represents a Differential hunk, which is owned by a Differential change"""
 
     oldOffset = attr.ib(default=0)  # camelcase-required
     oldLength = attr.ib(default=0)  # camelcase-required
     newOffset = attr.ib(default=0)  # camelcase-required
     newLength = attr.ib(default=0)  # camelcase-required
     corpus = attr.ib(default='')
     # These get added to the phabchange's equivalents
     addLines = attr.ib(default=0)  # camelcase-required
     delLines = attr.ib(default=0)  # camelcase-required
 
 
 @attr.s
 class phabchange(object):
     """Represents a Differential change, owns Differential hunks and owned by a
     Differential diff.  Each one represents one file in a diff.
     """
 
     currentPath = attr.ib(default=None)  # camelcase-required
     oldPath = attr.ib(default=None)  # camelcase-required
     awayPaths = attr.ib(default=attr.Factory(list))  # camelcase-required
     metadata = attr.ib(default=attr.Factory(dict))
     oldProperties = attr.ib(default=attr.Factory(dict))  # camelcase-required
     newProperties = attr.ib(default=attr.Factory(dict))  # camelcase-required
     type = attr.ib(default=DiffChangeType.CHANGE)
     fileType = attr.ib(default=DiffFileType.TEXT)  # camelcase-required
     commitHash = attr.ib(default=None)  # camelcase-required
     addLines = attr.ib(default=0)  # camelcase-required
     delLines = attr.ib(default=0)  # camelcase-required
     hunks = attr.ib(default=attr.Factory(list))
 
     def copynewmetadatatoold(self):
         for key in list(self.metadata.keys()):
             newkey = key.replace(b'new:', b'old:')
             self.metadata[newkey] = self.metadata[key]
 
     def addoldmode(self, value):
         self.oldProperties[b'unix:filemode'] = value
 
     def addnewmode(self, value):
         self.newProperties[b'unix:filemode'] = value
 
     def addhunk(self, hunk):
         if not isinstance(hunk, phabhunk):
             raise error.Abort(b'phabchange.addhunk only takes phabhunks')
         self.hunks.append(pycompat.byteskwargs(attr.asdict(hunk)))
         # It's useful to include these stats since the Phab web UI shows them,
         # and uses them to estimate how large a change a Revision is. Also used
         # in email subjects for the [+++--] bit.
         self.addLines += hunk.addLines
         self.delLines += hunk.delLines
 
 
 @attr.s
 class phabdiff(object):
     """Represents a Differential diff, owns Differential changes.  Corresponds
     to a commit.
     """
 
     # Doesn't seem to be any reason to send this (output of uname -n)
     sourceMachine = attr.ib(default=b'')  # camelcase-required
     sourcePath = attr.ib(default=b'/')  # camelcase-required
     sourceControlBaseRevision = attr.ib(default=b'0' * 40)  # camelcase-required
     sourceControlPath = attr.ib(default=b'/')  # camelcase-required
     sourceControlSystem = attr.ib(default=b'hg')  # camelcase-required
     branch = attr.ib(default=b'default')
     bookmark = attr.ib(default=None)
     creationMethod = attr.ib(default=b'phabsend')  # camelcase-required
     lintStatus = attr.ib(default=b'none')  # camelcase-required
     unitStatus = attr.ib(default=b'none')  # camelcase-required
     changes = attr.ib(default=attr.Factory(dict))
     repositoryPHID = attr.ib(default=None)  # camelcase-required
 
     def addchange(self, change):
         if not isinstance(change, phabchange):
             raise error.Abort(b'phabdiff.addchange only takes phabchanges')
         self.changes[change.currentPath] = pycompat.byteskwargs(
             attr.asdict(change)
         )
 
 
 def maketext(pchange, basectx, ctx, fname):
     """populate the phabchange for a text file"""
     repo = ctx.repo()
     fmatcher = match.exact([fname])
     diffopts = mdiff.diffopts(git=True, context=32767)
     _pfctx, _fctx, header, fhunks = next(
         patch.diffhunks(repo, basectx.p1(), ctx, fmatcher, opts=diffopts)
     )
 
     for fhunk in fhunks:
         (oldOffset, oldLength, newOffset, newLength), lines = fhunk
         corpus = b''.join(lines[1:])
         shunk = list(header)
         shunk.extend(lines)
         _mf, _mt, addLines, delLines, _hb = patch.diffstatsum(
             patch.diffstatdata(util.iterlines(shunk))
         )
         pchange.addhunk(
             phabhunk(
                 oldOffset,
                 oldLength,
                 newOffset,
                 newLength,
                 corpus,
                 addLines,
                 delLines,
             )
         )
 
 
 def uploadchunks(fctx, fphid):
     """upload large binary files as separate chunks.
     Phab requests chunking over 8MiB, and splits into 4MiB chunks
     """
     ui = fctx.repo().ui
     chunks = callconduit(ui, b'file.querychunks', {b'filePHID': fphid})
     with ui.makeprogress(
         _(b'uploading file chunks'), unit=_(b'chunks'), total=len(chunks)
     ) as progress:
         for chunk in chunks:
             progress.increment()
             if chunk[b'complete']:
                 continue
             bstart = int(chunk[b'byteStart'])
             bend = int(chunk[b'byteEnd'])
             callconduit(
                 ui,
                 b'file.uploadchunk',
                 {
                     b'filePHID': fphid,
                     b'byteStart': bstart,
                     b'data': base64.b64encode(fctx.data()[bstart:bend]),
                     b'dataEncoding': b'base64',
                 },
             )
 
 
 def uploadfile(fctx):
     """upload binary files to Phabricator"""
     repo = fctx.repo()
     ui = repo.ui
     fname = fctx.path()
     size = fctx.size()
     fhash = pycompat.bytestr(hashlib.sha256(fctx.data()).hexdigest())
 
     # an allocate call is required first to see if an upload is even required
     # (Phab might already have it) and to determine if chunking is needed
     allocateparams = {
         b'name': fname,
         b'contentLength': size,
         b'contentHash': fhash,
     }
     filealloc = callconduit(ui, b'file.allocate', allocateparams)
     fphid = filealloc[b'filePHID']
 
     if filealloc[b'upload']:
         ui.write(_(b'uploading %s\n') % bytes(fctx))
         if not fphid:
             uploadparams = {
                 b'name': fname,
                 b'data_base64': base64.b64encode(fctx.data()),
             }
             fphid = callconduit(ui, b'file.upload', uploadparams)
         else:
             uploadchunks(fctx, fphid)
     else:
         ui.debug(b'server already has %s\n' % bytes(fctx))
 
     if not fphid:
         raise error.Abort(b'Upload of %s failed.' % bytes(fctx))
 
     return fphid
 
 
 def addoldbinary(pchange, oldfctx, fctx):
     """add the metadata for the previous version of a binary file to the
     phabchange for the new version
 
     ``oldfctx`` is the previous version of the file; ``fctx`` is the new
     version of the file, or None if the file is being removed.
     """
     if not fctx or fctx.cmp(oldfctx):
         # Files differ, add the old one
         pchange.metadata[b'old:file:size'] = oldfctx.size()
         mimeguess, _enc = mimetypes.guess_type(
             encoding.unifromlocal(oldfctx.path())
         )
         if mimeguess:
             pchange.metadata[b'old:file:mime-type'] = pycompat.bytestr(
                 mimeguess
             )
         fphid = uploadfile(oldfctx)
         pchange.metadata[b'old:binary-phid'] = fphid
     else:
         # If it's left as IMAGE/BINARY web UI might try to display it
         pchange.fileType = DiffFileType.TEXT
         pchange.copynewmetadatatoold()
 
 
 def makebinary(pchange, fctx):
     """populate the phabchange for a binary file"""
     pchange.fileType = DiffFileType.BINARY
     fphid = uploadfile(fctx)
     pchange.metadata[b'new:binary-phid'] = fphid
     pchange.metadata[b'new:file:size'] = fctx.size()
     mimeguess, _enc = mimetypes.guess_type(encoding.unifromlocal(fctx.path()))
     if mimeguess:
         mimeguess = pycompat.bytestr(mimeguess)
         pchange.metadata[b'new:file:mime-type'] = mimeguess
         if mimeguess.startswith(b'image/'):
             pchange.fileType = DiffFileType.IMAGE
 
 
 # Copied from mercurial/patch.py
 gitmode = {b'l': b'120000', b'x': b'100755', b'': b'100644'}
 
 
 def notutf8(fctx):
     """detect non-UTF-8 text files since Phabricator requires them to be marked
     as binary
     """
     try:
         fctx.data().decode('utf-8')
         return False
     except UnicodeDecodeError:
         fctx.repo().ui.write(
             _(b'file %s detected as non-UTF-8, marked as binary\n')
             % fctx.path()
         )
         return True
 
 
 def addremoved(pdiff, basectx, ctx, removed):
     """add removed files to the phabdiff. Shouldn't include moves"""
     for fname in removed:
         pchange = phabchange(
             currentPath=fname, oldPath=fname, type=DiffChangeType.DELETE
         )
         oldfctx = basectx.p1()[fname]
         pchange.addoldmode(gitmode[oldfctx.flags()])
         if not (oldfctx.isbinary() or notutf8(oldfctx)):
             maketext(pchange, basectx, ctx, fname)
 
         pdiff.addchange(pchange)
 
 
 def addmodified(pdiff, basectx, ctx, modified):
     """add modified files to the phabdiff"""
     for fname in modified:
         fctx = ctx[fname]
         oldfctx = basectx.p1()[fname]
         pchange = phabchange(currentPath=fname, oldPath=fname)
         filemode = gitmode[fctx.flags()]
         originalmode = gitmode[oldfctx.flags()]
         if filemode != originalmode:
             pchange.addoldmode(originalmode)
             pchange.addnewmode(filemode)
 
         if (
             fctx.isbinary()
             or notutf8(fctx)
             or oldfctx.isbinary()
             or notutf8(oldfctx)
         ):
             makebinary(pchange, fctx)
             addoldbinary(pchange, oldfctx, fctx)
         else:
             maketext(pchange, basectx, ctx, fname)
 
         pdiff.addchange(pchange)
 
 
 def addadded(pdiff, basectx, ctx, added, removed):
     """add file adds to the phabdiff, both new files and copies/moves"""
     # Keep track of files that've been recorded as moved/copied, so if there are
     # additional copies we can mark them (moves get removed from removed)
     copiedchanges = {}
     movedchanges = {}
 
     copy = {}
     if basectx != ctx:
         copy = copies.pathcopies(basectx.p1(), ctx)
 
     for fname in added:
         fctx = ctx[fname]
         oldfctx = None
         pchange = phabchange(currentPath=fname)
 
         filemode = gitmode[fctx.flags()]
 
         if copy:
             originalfname = copy.get(fname, fname)
         else:
             originalfname = fname
             if fctx.renamed():
                 originalfname = fctx.renamed()[0]
 
         renamed = fname != originalfname
 
         if renamed:
             oldfctx = basectx.p1()[originalfname]
             originalmode = gitmode[oldfctx.flags()]
             pchange.oldPath = originalfname
 
             if originalfname in removed:
                 origpchange = phabchange(
                     currentPath=originalfname,
                     oldPath=originalfname,
                     type=DiffChangeType.MOVE_AWAY,
                     awayPaths=[fname],
                 )
                 movedchanges[originalfname] = origpchange
                 removed.remove(originalfname)
                 pchange.type = DiffChangeType.MOVE_HERE
             elif originalfname in movedchanges:
                 movedchanges[originalfname].type = DiffChangeType.MULTICOPY
                 movedchanges[originalfname].awayPaths.append(fname)
                 pchange.type = DiffChangeType.COPY_HERE
             else:  # pure copy
                 if originalfname not in copiedchanges:
                     origpchange = phabchange(
                         currentPath=originalfname, type=DiffChangeType.COPY_AWAY
                     )
                     copiedchanges[originalfname] = origpchange
                 else:
                     origpchange = copiedchanges[originalfname]
                 origpchange.awayPaths.append(fname)
                 pchange.type = DiffChangeType.COPY_HERE
 
             if filemode != originalmode:
                 pchange.addoldmode(originalmode)
                 pchange.addnewmode(filemode)
         else:  # Brand-new file
             pchange.addnewmode(gitmode[fctx.flags()])
             pchange.type = DiffChangeType.ADD
 
         if (
             fctx.isbinary()
             or notutf8(fctx)
             or (oldfctx and (oldfctx.isbinary() or notutf8(oldfctx)))
         ):
             makebinary(pchange, fctx)
             if renamed:
                 addoldbinary(pchange, oldfctx, fctx)
         else:
             maketext(pchange, basectx, ctx, fname)
 
         pdiff.addchange(pchange)
 
     for _path, copiedchange in copiedchanges.items():
         pdiff.addchange(copiedchange)
     for _path, movedchange in movedchanges.items():
         pdiff.addchange(movedchange)
 
 
 def creatediff(basectx, ctx):
     """create a Differential Diff"""
     repo = ctx.repo()
     repophid = getrepophid(repo)
     # Create a "Differential Diff" via "differential.creatediff" API
     pdiff = phabdiff(
         sourceControlBaseRevision=b'%s' % basectx.p1().hex(),
         branch=b'%s' % ctx.branch(),
     )
     modified, added, removed, _d, _u, _i, _c = basectx.p1().status(ctx)
     # addadded will remove moved files from removed, so addremoved won't get
     # them
     addadded(pdiff, basectx, ctx, added, removed)
     addmodified(pdiff, basectx, ctx, modified)
     addremoved(pdiff, basectx, ctx, removed)
     if repophid:
         pdiff.repositoryPHID = repophid
     diff = callconduit(
         repo.ui,
         b'differential.creatediff',
         pycompat.byteskwargs(attr.asdict(pdiff)),
     )
     if not diff:
         if basectx != ctx:
             msg = _(b'cannot create diff for %s::%s') % (basectx, ctx)
         else:
             msg = _(b'cannot create diff for %s') % ctx
         raise error.Abort(msg)
     return diff
 
 
 def writediffproperties(ctxs, diff):
     """write metadata to diff so patches could be applied losslessly
 
     ``ctxs`` is the list of commits that created the diff, in ascending order.
     The list is generally a single commit, but may be several when using
     ``phabsend --fold``.
     """
     # creatediff returns with a diffid but query returns with an id
     diffid = diff.get(b'diffid', diff.get(b'id'))
     basectx = ctxs[0]
     tipctx = ctxs[-1]
 
     params = {
         b'diff_id': diffid,
         b'name': b'hg:meta',
         b'data': templatefilters.json(
             {
                 b'user': tipctx.user(),
                 b'date': b'%d %d' % tipctx.date(),
                 b'branch': tipctx.branch(),
                 b'node': tipctx.hex(),
                 b'parent': basectx.p1().hex(),
             }
         ),
     }
     callconduit(basectx.repo().ui, b'differential.setdiffproperty', params)
 
     commits = {}
     for ctx in ctxs:
         commits[ctx.hex()] = {
             b'author': stringutil.person(ctx.user()),
             b'authorEmail': stringutil.email(ctx.user()),
             b'time': int(ctx.date()[0]),
             b'commit': ctx.hex(),
             b'parents': [ctx.p1().hex()],
             b'branch': ctx.branch(),
         }
     params = {
         b'diff_id': diffid,
         b'name': b'local:commits',
         b'data': templatefilters.json(commits),
     }
     callconduit(basectx.repo().ui, b'differential.setdiffproperty', params)
 
 
 def createdifferentialrevision(
     ctxs,
     revid=None,
     parentrevphid=None,
     oldbasenode=None,
     oldnode=None,
     olddiff=None,
     actions=None,
     comment=None,
 ):
     """create or update a Differential Revision
 
     If revid is None, create a new Differential Revision, otherwise update
     revid. If parentrevphid is not None, set it as a dependency.
 
     If there is a single commit for the new Differential Revision, ``ctxs`` will
     be a list of that single context.  Otherwise, it is a list that covers the
     range of changes for the differential, where ``ctxs[0]`` is the first change
     to include and ``ctxs[-1]`` is the last.
 
     If oldnode is not None, check if the patch content (without commit message
     and metadata) has changed before creating another diff.  For a Revision with
     a single commit, ``oldbasenode`` and ``oldnode`` have the same value.  For a
     Revision covering multiple commits, ``oldbasenode`` corresponds to
     ``ctxs[0]`` the previous time this Revision was posted, and ``oldnode``
     corresponds to ``ctxs[-1]``.
 
     If actions is not None, they will be appended to the transaction.
     """
     ctx = ctxs[-1]
     basectx = ctxs[0]
 
     repo = ctx.repo()
     if oldnode:
         diffopts = mdiff.diffopts(git=True, context=32767)
         unfi = repo.unfiltered()
         oldctx = unfi[oldnode]
         oldbasectx = unfi[oldbasenode]
         neednewdiff = getdiff(basectx, ctx, diffopts) != getdiff(
             oldbasectx, oldctx, diffopts
         )
     else:
         neednewdiff = True
 
     transactions = []
     if neednewdiff:
         diff = creatediff(basectx, ctx)
         transactions.append({b'type': b'update', b'value': diff[b'phid']})
         if comment:
             transactions.append({b'type': b'comment', b'value': comment})
     else:
         # Even if we don't need to upload a new diff because the patch content
         # does not change. We might still need to update its metadata so
         # pushers could know the correct node metadata.
         assert olddiff
         diff = olddiff
     writediffproperties(ctxs, diff)
 
     # Set the parent Revision every time, so commit re-ordering is picked-up
     if parentrevphid:
         transactions.append(
             {b'type': b'parents.set', b'value': [parentrevphid]}
         )
 
     if actions:
         transactions += actions
 
     # When folding multiple local commits into a single review, arcanist will
     # take the summary line of the first commit as the title, and then
     # concatenate the rest of the remaining messages (including each of their
     # first lines) to the rest of the first commit message (each separated by
     # an empty line), and use that as the summary field.  Do the same here.
     # For commits with only a one line message, there is no summary field, as
     # this gets assigned to the title.
     fields = util.sortdict()  # sorted for stable wire protocol in tests
 
     for i, _ctx in enumerate(ctxs):
         # Parse commit message and update related fields.
         desc = _ctx.description()
         info = callconduit(
             repo.ui, b'differential.parsecommitmessage', {b'corpus': desc}
         )
 
         for k in [b'title', b'summary', b'testPlan']:
             v = info[b'fields'].get(k)
             if not v:
                 continue
 
             if i == 0:
                 # Title, summary and test plan (if present) are taken verbatim
                 # for the first commit.
                 fields[k] = v.rstrip()
                 continue
             elif k == b'title':
                 # Add subsequent titles (i.e. the first line of the commit
                 # message) back to the summary.
                 k = b'summary'
 
             # Append any current field to the existing composite field
             fields[k] = b'\n\n'.join(filter(None, [fields.get(k), v.rstrip()]))
 
     for k, v in fields.items():
         transactions.append({b'type': k, b'value': v})
 
     params = {b'transactions': transactions}
     if revid is not None:
         # Update an existing Differential Revision
         params[b'objectIdentifier'] = revid
 
     revision = callconduit(repo.ui, b'differential.revision.edit', params)
     if not revision:
         if len(ctxs) == 1:
             msg = _(b'cannot create revision for %s') % ctx
         else:
             msg = _(b'cannot create revision for %s::%s') % (basectx, ctx)
         raise error.Abort(msg)
 
     return revision, diff
 
 
 def userphids(ui, names):
     """convert user names to PHIDs"""
     names = [name.lower() for name in names]
     query = {b'constraints': {b'usernames': names}}
     result = callconduit(ui, b'user.search', query)
     # username not found is not an error of the API. So check if we have missed
     # some names here.
     data = result[b'data']
     resolved = {entry[b'fields'][b'username'].lower() for entry in data}
     unresolved = set(names) - resolved
     if unresolved:
         raise error.Abort(
             _(b'unknown username: %s') % b' '.join(sorted(unresolved))
         )
     return [entry[b'phid'] for entry in data]
 
 
 def _print_phabsend_action(ui, ctx, newrevid, action):
     """print the ``action`` that occurred when posting ``ctx`` for review
 
     This is a utility function for the sending phase of ``phabsend``, which
     makes it easier to show a status for all local commits with `--fold``.
     """
     actiondesc = ui.label(
         {
             b'created': _(b'created'),
             b'skipped': _(b'skipped'),
             b'updated': _(b'updated'),
         }[action],
         b'phabricator.action.%s' % action,
     )
     drevdesc = ui.label(b'D%d' % newrevid, b'phabricator.drev')
     summary = cmdutil.format_changeset_summary(ui, ctx, b'phabsend')
     ui.write(_(b'%s - %s - %s\n') % (drevdesc, actiondesc, summary))
 
 
 def _amend_diff_properties(unfi, drevid, newnodes, diff):
     """update the local commit list for the ``diff`` associated with ``drevid``
 
     This is a utility function for the amend phase of ``phabsend``, which
     converts failures to warning messages.
     """
     _debug(
         unfi.ui,
         b"new commits: %s\n" % stringutil.pprint([short(n) for n in newnodes]),
     )
 
     try:
         writediffproperties([unfi[newnode] for newnode in newnodes], diff)
     except util.urlerr.urlerror:
         # If it fails just warn and keep going, otherwise the DREV
         # associations will be lost
         unfi.ui.warnnoi18n(b'Failed to update metadata for D%d\n' % drevid)
 
 
 @vcrcommand(
     b'phabsend',
     [
         (b'r', b'rev', [], _(b'revisions to send'), _(b'REV')),
         (b'', b'amend', True, _(b'update commit messages')),
         (b'', b'reviewer', [], _(b'specify reviewers')),
         (b'', b'blocker', [], _(b'specify blocking reviewers')),
         (
             b'm',
             b'comment',
             b'',
             _(b'add a comment to Revisions with new/updated Diffs'),
         ),
         (b'', b'confirm', None, _(b'ask for confirmation before sending')),
         (b'', b'fold', False, _(b'combine the revisions into one review')),
     ],
     _(b'REV [OPTIONS]'),
     helpcategory=command.CATEGORY_IMPORT_EXPORT,
 )
 def phabsend(ui, repo, *revs, **opts):
     """upload changesets to Phabricator
 
     If there are multiple revisions specified, they will be send as a stack
     with a linear dependencies relationship using the order specified by the
     revset.
 
     For the first time uploading changesets, local tags will be created to
     maintain the association. After the first time, phabsend will check
     obsstore and tags information so it can figure out whether to update an
     existing Differential Revision, or create a new one.
 
     If --amend is set, update commit messages so they have the
     ``Differential Revision`` URL, remove related tags. This is similar to what
     arcanist will do, and is more desired in author-push workflows. Otherwise,
     use local tags to record the ``Differential Revision`` association.
 
     The --confirm option lets you confirm changesets before sending them. You
     can also add following to your configuration file to make it default
     behaviour::
 
         [phabsend]
         confirm = true
 
     By default, a separate review will be created for each commit that is
     selected, and will have the same parent/child relationship in Phabricator.
     If ``--fold`` is set, multiple commits are rolled up into a single review
     as if diffed from the parent of the first revision to the last.  The commit
     messages are concatenated in the summary field on Phabricator.
 
     phabsend will check obsstore and the above association to decide whether to
     update an existing Differential Revision, or create a new one.
     """
     opts = pycompat.byteskwargs(opts)
     revs = list(revs) + opts.get(b'rev', [])
     revs = logcmdutil.revrange(repo, revs)
     revs.sort()  # ascending order to preserve topological parent/child in phab
 
     if not revs:
         raise error.Abort(_(b'phabsend requires at least one changeset'))
     if opts.get(b'amend'):
         cmdutil.checkunfinished(repo)
 
     ctxs = [repo[rev] for rev in revs]
 
     if any(c for c in ctxs if c.obsolete()):
         raise error.Abort(_(b"obsolete commits cannot be posted for review"))
 
     # Ensure the local commits are an unbroken range.  The semantics of the
     # --fold option implies this, and the auto restacking of orphans requires
     # it.  Otherwise A+C in A->B->C will cause B to be orphaned, and C' to
     # get A' as a parent.
     def _fail_nonlinear_revs(revs, revtype):
         badnodes = [repo[r].node() for r in revs]
         raise error.Abort(
             _(b"cannot phabsend multiple %s revisions: %s")
             % (revtype, scmutil.nodesummaries(repo, badnodes)),
             hint=_(b"the revisions must form a linear chain"),
         )
 
     heads = repo.revs(b'heads(%ld)', revs)
     if len(heads) > 1:
         _fail_nonlinear_revs(heads, b"head")
 
     roots = repo.revs(b'roots(%ld)', revs)
     if len(roots) > 1:
         _fail_nonlinear_revs(roots, b"root")
 
     fold = opts.get(b'fold')
     if fold:
         if len(revs) == 1:
             # TODO: just switch to --no-fold instead?
             raise error.Abort(_(b"cannot fold a single revision"))
 
         # There's no clear way to manage multiple commits with a Dxxx tag, so
         # require the amend option.  (We could append "_nnn", but then it
         # becomes jumbled if earlier commits are added to an update.)  It should
         # lock the repo and ensure that the range is editable, but that would
         # make the code pretty convoluted.  The default behavior of `arc` is to
         # create a new review anyway.
         if not opts.get(b"amend"):
             raise error.Abort(_(b"cannot fold with --no-amend"))
 
         # It might be possible to bucketize the revisions by the DREV value, and
         # iterate over those groups when posting, and then again when amending.
         # But for simplicity, require all selected revisions to be for the same
         # DREV (if present).  Adding local revisions to an existing DREV is
         # acceptable.
         drevmatchers = [
             _differentialrevisiondescre.search(ctx.description())
             for ctx in ctxs
         ]
         if len({m.group('url') for m in drevmatchers if m}) > 1:
             raise error.Abort(
                 _(b"cannot fold revisions with different DREV values")
             )
 
     # {newnode: (oldnode, olddiff, olddrev}
     oldmap = getoldnodedrevmap(repo, [repo[r].node() for r in revs])
 
     confirm = ui.configbool(b'phabsend', b'confirm')
     confirm |= bool(opts.get(b'confirm'))
     if confirm:
         confirmed = _confirmbeforesend(repo, revs, oldmap)
         if not confirmed:
             raise error.Abort(_(b'phabsend cancelled'))
 
     actions = []
     reviewers = opts.get(b'reviewer', [])
     blockers = opts.get(b'blocker', [])
     phids = []
     if reviewers:
         phids.extend(userphids(repo.ui, reviewers))
     if blockers:
         phids.extend(
             map(
                 lambda phid: b'blocking(%s)' % phid,
                 userphids(repo.ui, blockers),
             )
         )
     if phids:
         actions.append({b'type': b'reviewers.add', b'value': phids})
 
     drevids = []  # [int]
     diffmap = {}  # {newnode: diff}
 
     # Send patches one by one so we know their Differential Revision PHIDs and
     # can provide dependency relationship
     lastrevphid = None
     for ctx in ctxs:
         if fold:
             ui.debug(b'sending rev %d::%d\n' % (ctx.rev(), ctxs[-1].rev()))
         else:
             ui.debug(b'sending rev %d\n' % ctx.rev())
 
         # Get Differential Revision ID
         oldnode, olddiff, revid = oldmap.get(ctx.node(), (None, None, None))
         oldbasenode, oldbasediff, oldbaserevid = oldnode, olddiff, revid
 
         if fold:
             oldbasenode, oldbasediff, oldbaserevid = oldmap.get(
                 ctxs[-1].node(), (None, None, None)
             )
 
         if oldnode != ctx.node() or opts.get(b'amend'):
             # Create or update Differential Revision
             revision, diff = createdifferentialrevision(
                 ctxs if fold else [ctx],
                 revid,
                 lastrevphid,
                 oldbasenode,
                 oldnode,
                 olddiff,
                 actions,
                 opts.get(b'comment'),
             )
 
             if fold:
                 for ctx in ctxs:
                     diffmap[ctx.node()] = diff
             else:
                 diffmap[ctx.node()] = diff
 
             newrevid = int(revision[b'object'][b'id'])
             newrevphid = revision[b'object'][b'phid']
             if revid:
                 action = b'updated'
             else:
                 action = b'created'
 
             # Create a local tag to note the association, if commit message
             # does not have it already
             if not fold:
                 m = _differentialrevisiondescre.search(ctx.description())
                 if not m or int(m.group('id')) != newrevid:
                     tagname = b'D%d' % newrevid
                     tags.tag(
                         repo,
                         tagname,
                         ctx.node(),
                         message=None,
                         user=None,
                         date=None,
                         local=True,
                     )
         else:
             # Nothing changed. But still set "newrevphid" so the next revision
             # could depend on this one and "newrevid" for the summary line.
             newrevphid = querydrev(repo.ui, b'%d' % revid)[0][b'phid']
             newrevid = revid
             action = b'skipped'
 
         drevids.append(newrevid)
         lastrevphid = newrevphid
 
         if fold:
             for c in ctxs:
                 if oldmap.get(c.node(), (None, None, None))[2]:
                     action = b'updated'
                 else:
                     action = b'created'
                 _print_phabsend_action(ui, c, newrevid, action)
             break
 
         _print_phabsend_action(ui, ctx, newrevid, action)
 
     # Update commit messages and remove tags
     if opts.get(b'amend'):
         unfi = repo.unfiltered()
         drevs = callconduit(ui, b'differential.query', {b'ids': drevids})
         with repo.wlock(), repo.lock(), repo.transaction(b'phabsend'):
             # Eagerly evaluate commits to restabilize before creating new
             # commits.  The selected revisions are excluded because they are
             # automatically restacked as part of the submission process.
             restack = [
                 c
                 for c in repo.set(
                     b"(%ld::) - (%ld) - unstable() - obsolete() - public()",
                     revs,
                     revs,
                 )
             ]
             wnode = unfi[b'.'].node()
             mapping = {}  # {oldnode: [newnode]}
             newnodes = []
 
             drevid = drevids[0]
 
             for i, rev in enumerate(revs):
                 old = unfi[rev]
                 if not fold:
                     drevid = drevids[i]
                 drev = [d for d in drevs if int(d[b'id']) == drevid][0]
 
                 newdesc = get_amended_desc(drev, old, fold)
                 # Make sure commit message contain "Differential Revision"
                 if (
                     old.description() != newdesc
                     or old.p1().node() in mapping
                     or old.p2().node() in mapping
                 ):
                     if old.phase() == phases.public:
                         ui.warn(
                             _(b"warning: not updating public commit %s\n")
                             % scmutil.formatchangeid(old)
                         )
                         continue
                     parents = [
                         mapping.get(old.p1().node(), (old.p1(),))[0],
                         mapping.get(old.p2().node(), (old.p2(),))[0],
                     ]
                     newdesc = rewriteutil.update_hash_refs(
                         repo,
                         newdesc,
                         mapping,
                     )
                     new = context.metadataonlyctx(
                         repo,
                         old,
                         parents=parents,
                         text=newdesc,
                         user=old.user(),
                         date=old.date(),
                         extra=old.extra(),
                     )
 
                     newnode = new.commit()
 
                     mapping[old.node()] = [newnode]
 
                     if fold:
                         # Defer updating the (single) Diff until all nodes are
                         # collected.  No tags were created, so none need to be
                         # removed.
                         newnodes.append(newnode)
                         continue
 
                     _amend_diff_properties(
                         unfi, drevid, [newnode], diffmap[old.node()]
                     )
 
                     # Remove local tags since it's no longer necessary
                     tagname = b'D%d' % drevid
                     if tagname in repo.tags():
                         tags.tag(
                             repo,
                             tagname,
                             repo.nullid,
                             message=None,
                             user=None,
                             date=None,
                             local=True,
                         )
                 elif fold:
                     # When folding multiple commits into one review with
                     # --fold, track even the commits that weren't amended, so
                     # that their association isn't lost if the properties are
                     # rewritten below.
                     newnodes.append(old.node())
 
             # If the submitted commits are public, no amend takes place so
             # there are no newnodes and therefore no diff update to do.
             if fold and newnodes:
                 diff = diffmap[old.node()]
 
                 # The diff object in diffmap doesn't have the local commits
                 # because that could be returned from differential.creatediff,
                 # not differential.querydiffs.  So use the queried diff (if
                 # present), or force the amend (a new revision is being posted.)
                 if not olddiff or set(newnodes) != getlocalcommits(olddiff):
                     _debug(ui, b"updating local commit list for D%d\n" % drevid)
                     _amend_diff_properties(unfi, drevid, newnodes, diff)
                 else:
                     _debug(
                         ui,
                         b"local commit list for D%d is already up-to-date\n"
                         % drevid,
                     )
             elif fold:
                 _debug(ui, b"no newnodes to update\n")
 
             # Restack any children of first-time submissions that were orphaned
             # in the process.  The ctx won't report that it is an orphan until
             # the cleanup takes place below.
             for old in restack:
                 parents = [
                     mapping.get(old.p1().node(), (old.p1(),))[0],
                     mapping.get(old.p2().node(), (old.p2(),))[0],
                 ]
                 new = context.metadataonlyctx(
                     repo,
                     old,
                     parents=parents,
                     text=rewriteutil.update_hash_refs(
                         repo, old.description(), mapping
                     ),
                     user=old.user(),
                     date=old.date(),
                     extra=old.extra(),
                 )
 
                 newnode = new.commit()
 
                 # Don't obsolete unselected descendants of nodes that have not
                 # been changed in this transaction- that results in an error.
                 if newnode != old.node():
                     mapping[old.node()] = [newnode]
                     _debug(
                         ui,
                         b"restabilizing %s as %s\n"
                         % (short(old.node()), short(newnode)),
                     )
                 else:
                     _debug(
                         ui,
                         b"not restabilizing unchanged %s\n" % short(old.node()),
                     )
 
             scmutil.cleanupnodes(repo, mapping, b'phabsend', fixphase=True)
             if wnode in mapping:
                 unfi.setparents(mapping[wnode][0])
 
 
 # Map from "hg:meta" keys to header understood by "hg import". The order is
 # consistent with "hg export" output.
 _metanamemap = util.sortdict(
     [
         (b'user', b'User'),
         (b'date', b'Date'),
         (b'branch', b'Branch'),
         (b'node', b'Node ID'),
         (b'parent', b'Parent '),
     ]
 )
 
 
 def _confirmbeforesend(repo, revs, oldmap):
     url, token = readurltoken(repo.ui)
     ui = repo.ui
     for rev in revs:
         ctx = repo[rev]
         oldnode, olddiff, drevid = oldmap.get(ctx.node(), (None, None, None))
         if drevid:
             drevdesc = ui.label(b'D%d' % drevid, b'phabricator.drev')
         else:
             drevdesc = ui.label(_(b'NEW'), b'phabricator.drev')
 
         ui.write(
             _(b'%s - %s\n')
             % (
                 drevdesc,
                 cmdutil.format_changeset_summary(ui, ctx, b'phabsend'),
             )
         )
 
     if ui.promptchoice(
         _(b'Send the above changes to %s (Y/n)?$$ &Yes $$ &No') % url
     ):
         return False
 
     return True
 
 
 _knownstatusnames = {
     b'accepted',
     b'needsreview',
     b'needsrevision',
     b'closed',
     b'abandoned',
     b'changesplanned',
 }
 
 
 def _getstatusname(drev):
     """get normalized status name from a Differential Revision"""
     return drev[b'statusName'].replace(b' ', b'').lower()
 
 
 # Small language to specify differential revisions. Support symbols: (), :X,
 # +, and -.
 
 _elements = {
     # token-type: binding-strength, primary, prefix, infix, suffix
     b'(': (12, None, (b'group', 1, b')'), None, None),
     b':': (8, None, (b'ancestors', 8), None, None),
     b'&': (5, None, None, (b'and_', 5), None),
     b'+': (4, None, None, (b'add', 4), None),
     b'-': (4, None, None, (b'sub', 4), None),
     b')': (0, None, None, None, None),
     b'symbol': (0, b'symbol', None, None, None),
     b'end': (0, None, None, None, None),
 }
 
 
 def _tokenize(text):
     view = memoryview(text)  # zero-copy slice
     special = b'():+-& '
     pos = 0
     length = len(text)
     while pos < length:
         symbol = b''.join(
             itertools.takewhile(
                 lambda ch: ch not in special, pycompat.iterbytestr(view[pos:])
             )
         )
         if symbol:
             yield (b'symbol', symbol, pos)
             pos += len(symbol)
         else:  # special char, ignore space
             if text[pos : pos + 1] != b' ':
                 yield (text[pos : pos + 1], None, pos)
             pos += 1
     yield (b'end', None, pos)
 
 
 def _parse(text):
     tree, pos = parser.parser(_elements).parse(_tokenize(text))
     if pos != len(text):
         raise error.ParseError(b'invalid token', pos)
     return tree
 
 
 def _parsedrev(symbol):
     """str -> int or None, ex. 'D45' -> 45; '12' -> 12; 'x' -> None"""
     if symbol.startswith(b'D') and symbol[1:].isdigit():
         return int(symbol[1:])
     if symbol.isdigit():
         return int(symbol)
 
 
 def _prefetchdrevs(tree):
     """return ({single-drev-id}, {ancestor-drev-id}) to prefetch"""
     drevs = set()
     ancestordrevs = set()
     op = tree[0]
     if op == b'symbol':
         r = _parsedrev(tree[1])
         if r:
             drevs.add(r)
     elif op == b'ancestors':
         r, a = _prefetchdrevs(tree[1])
         drevs.update(r)
         ancestordrevs.update(r)
         ancestordrevs.update(a)
     else:
         for t in tree[1:]:
             r, a = _prefetchdrevs(t)
             drevs.update(r)
             ancestordrevs.update(a)
     return drevs, ancestordrevs
 
 
 def querydrev(ui, spec):
     """return a list of "Differential Revision" dicts
 
     spec is a string using a simple query language, see docstring in phabread
     for details.
 
     A "Differential Revision dict" looks like:
 
         {
             "activeDiffPHID": "PHID-DIFF-xoqnjkobbm6k4dk6hi72",
             "authorPHID": "PHID-USER-tv3ohwc4v4jeu34otlye",
             "auxiliary": {
               "phabricator:depends-on": [
                 "PHID-DREV-gbapp366kutjebt7agcd"
               ]
               "phabricator:projects": [],
             },
             "branch": "default",
             "ccs": [],
             "commits": [],
             "dateCreated": "1499181406",
             "dateModified": "1499182103",
             "diffs": [
               "3",
               "4",
             ],
             "hashes": [],
             "id": "2",
             "lineCount": "2",
             "phid": "PHID-DREV-672qvysjcczopag46qty",
             "properties": {},
             "repositoryPHID": "PHID-REPO-hub2hx62ieuqeheznasv",
             "reviewers": [],
             "sourcePath": null
             "status": "0",
             "statusName": "Needs Review",
             "summary": "",
             "testPlan": "",
             "title": "example",
             "uri": "https://phab.example.com/D2",
         }
     """
     # TODO: replace differential.query and differential.querydiffs with
     # differential.diff.search because the former (and their output) are
     # frozen, and planned to be deprecated and removed.
 
     def fetch(params):
         """params -> single drev or None"""
         key = (params.get(b'ids') or params.get(b'phids') or [None])[0]
         if key in prefetched:
             return prefetched[key]
         drevs = callconduit(ui, b'differential.query', params)
         # Fill prefetched with the result
         for drev in drevs:
             prefetched[drev[b'phid']] = drev
             prefetched[int(drev[b'id'])] = drev
         if key not in prefetched:
             raise error.Abort(
                 _(b'cannot get Differential Revision %r') % params
             )
         return prefetched[key]
 
     def getstack(topdrevids):
         """given a top, get a stack from the bottom, [id] -> [id]"""
         visited = set()
         result = []
         queue = [{b'ids': [i]} for i in topdrevids]
         while queue:
             params = queue.pop()
             drev = fetch(params)
             if drev[b'id'] in visited:
                 continue
             visited.add(drev[b'id'])
             result.append(int(drev[b'id']))
             auxiliary = drev.get(b'auxiliary', {})
             depends = auxiliary.get(b'phabricator:depends-on', [])
             for phid in depends:
                 queue.append({b'phids': [phid]})
         result.reverse()
         return smartset.baseset(result)
 
     # Initialize prefetch cache
     prefetched = {}  # {id or phid: drev}
 
     tree = _parse(spec)
     drevs, ancestordrevs = _prefetchdrevs(tree)
 
     # developer config: phabricator.batchsize
     batchsize = ui.configint(b'phabricator', b'batchsize')
 
     # Prefetch Differential Revisions in batch
     tofetch = set(drevs)
     for r in ancestordrevs:
         tofetch.update(range(max(1, r - batchsize), r + 1))
     if drevs:
         fetch({b'ids': list(tofetch)})
     validids = sorted(set(getstack(list(ancestordrevs))) | set(drevs))
 
     # Walk through the tree, return smartsets
     def walk(tree):
         op = tree[0]
         if op == b'symbol':
             drev = _parsedrev(tree[1])
             if drev:
                 return smartset.baseset([drev])
             elif tree[1] in _knownstatusnames:
                 drevs = [
                     r
                     for r in validids
                     if _getstatusname(prefetched[r]) == tree[1]
                 ]
                 return smartset.baseset(drevs)
             else:
                 raise error.Abort(_(b'unknown symbol: %s') % tree[1])
         elif op in {b'and_', b'add', b'sub'}:
             assert len(tree) == 3
             return getattr(operator, op)(walk(tree[1]), walk(tree[2]))
         elif op == b'group':
             return walk(tree[1])
         elif op == b'ancestors':
             return getstack(walk(tree[1]))
         else:
             raise error.ProgrammingError(b'illegal tree: %r' % tree)
 
     return [prefetched[r] for r in walk(tree)]
 
 
 def getdescfromdrev(drev):
     """get description (commit message) from "Differential Revision"
 
     This is similar to differential.getcommitmessage API. But we only care
     about limited fields: title, summary, test plan, and URL.
     """
     title = drev[b'title']
     summary = drev[b'summary'].rstrip()
     testplan = drev[b'testPlan'].rstrip()
     if testplan:
         testplan = b'Test Plan:\n%s' % testplan
     uri = b'Differential Revision: %s' % drev[b'uri']
     return b'\n\n'.join(filter(None, [title, summary, testplan, uri]))
 
 
 def get_amended_desc(drev, ctx, folded):
     """similar to ``getdescfromdrev``, but supports a folded series of commits
 
     This is used when determining if an individual commit needs to have its
     message amended after posting it for review.  The determination is made for
     each individual commit, even when they were folded into one review.
     """
     if not folded:
         return getdescfromdrev(drev)
 
     uri = b'Differential Revision: %s' % drev[b'uri']
 
     # Since the commit messages were combined when posting multiple commits
     # with --fold, the fields can't be read from Phabricator here, or *all*
     # affected local revisions will end up with the same commit message after
     # the URI is amended in.  Append in the DREV line, or update it if it
     # exists.  At worst, this means commit message or test plan updates on
     # Phabricator aren't propagated back to the repository, but that seems
     # reasonable for the case where local commits are effectively combined
     # in Phabricator.
     m = _differentialrevisiondescre.search(ctx.description())
     if not m:
         return b'\n\n'.join([ctx.description(), uri])
 
     return _differentialrevisiondescre.sub(uri, ctx.description())
 
 
 def getlocalcommits(diff):
     """get the set of local commits from a diff object
 
     See ``getdiffmeta()`` for an example diff object.
     """
     props = diff.get(b'properties') or {}
     commits = props.get(b'local:commits') or {}
     if len(commits) > 1:
         return {bin(c) for c in commits.keys()}
 
     # Storing the diff metadata predates storing `local:commits`, so continue
     # to use that in the --no-fold case.
     return {bin(getdiffmeta(diff).get(b'node', b'')) or None}
 
 
 def getdiffmeta(diff):
     """get commit metadata (date, node, user, p1) from a diff object
 
     The metadata could be "hg:meta", sent by phabsend, like:
 
         "properties": {
           "hg:meta": {
             "branch": "default",
             "date": "1499571514 25200",
             "node": "98c08acae292b2faf60a279b4189beb6cff1414d",
             "user": "Foo Bar <[e-mail]>",
             "parent": "6d0abad76b30e4724a37ab8721d630394070fe16"
           }
         }
 
     Or converted from "local:commits", sent by "arc", like:
 
         "properties": {
           "local:commits": {
             "98c08acae292b2faf60a279b4189beb6cff1414d": {
               "author": "Foo Bar",
               "authorEmail": "[e-mail]"
               "branch": "default",
               "commit": "98c08acae292b2faf60a279b4189beb6cff1414d",
               "local": "1000",
               "message": "...",
               "parents": ["6d0abad76b30e4724a37ab8721d630394070fe16"],
               "rev": "98c08acae292b2faf60a279b4189beb6cff1414d",
               "summary": "...",
               "tag": "",
               "time": 1499546314,
             }
           }
         }
 
     Note: metadata extracted from "local:commits" will lose time zone
     information.
     """
     props = diff.get(b'properties') or {}
     meta = props.get(b'hg:meta')
     if not meta:
         if props.get(b'local:commits'):
             commit = sorted(props[b'local:commits'].values())[0]
             meta = {}
             if b'author' in commit and b'authorEmail' in commit:
                 meta[b'user'] = b'%s <%s>' % (
                     commit[b'author'],
                     commit[b'authorEmail'],
                 )
             if b'time' in commit:
                 meta[b'date'] = b'%d 0' % int(commit[b'time'])
             if b'branch' in commit:
                 meta[b'branch'] = commit[b'branch']
             node = commit.get(b'commit', commit.get(b'rev'))
             if node:
                 meta[b'node'] = node
             if len(commit.get(b'parents', ())) >= 1:
                 meta[b'parent'] = commit[b'parents'][0]
         else:
             meta = {}
     if b'date' not in meta and b'dateCreated' in diff:
         meta[b'date'] = b'%s 0' % diff[b'dateCreated']
     if b'branch' not in meta and diff.get(b'branch'):
         meta[b'branch'] = diff[b'branch']
     if b'parent' not in meta and diff.get(b'sourceControlBaseRevision'):
         meta[b'parent'] = diff[b'sourceControlBaseRevision']
     return meta
 
 
 def _getdrevs(ui, stack, specs):
     """convert user supplied DREVSPECs into "Differential Revision" dicts
 
     See ``hg help phabread`` for how to specify each DREVSPEC.
     """
     if len(specs) > 0:
 
         def _formatspec(s):
             if stack:
                 s = b':(%s)' % s
             return b'(%s)' % s
 
         spec = b'+'.join(pycompat.maplist(_formatspec, specs))
 
         drevs = querydrev(ui, spec)
         if drevs:
             return drevs
 
     raise error.Abort(_(b"empty DREVSPEC set"))
 
 
 def readpatch(ui, drevs, write):
     """generate plain-text patch readable by 'hg import'
 
     write takes a list of (DREV, bytes), where DREV is the differential number
     (as bytes, without the "D" prefix) and the bytes are the text of a patch
     to be imported. drevs is what "querydrev" returns, results of
     "differential.query".
     """
     # Prefetch hg:meta property for all diffs
     diffids = sorted({max(int(v) for v in drev[b'diffs']) for drev in drevs})
     diffs = callconduit(ui, b'differential.querydiffs', {b'ids': diffids})
 
     patches = []
 
     # Generate patch for each drev
     for drev in drevs:
         ui.note(_(b'reading D%s\n') % drev[b'id'])
 
         diffid = max(int(v) for v in drev[b'diffs'])
         body = callconduit(ui, b'differential.getrawdiff', {b'diffID': diffid})
         desc = getdescfromdrev(drev)
         header = b'# HG changeset patch\n'
 
         # Try to preserve metadata from hg:meta property. Write hg patch
         # headers that can be read by the "import" command. See patchheadermap
         # and extract in mercurial/patch.py for supported headers.
         meta = getdiffmeta(diffs[b'%d' % diffid])
         for k in _metanamemap.keys():
             if k in meta:
                 header += b'# %s %s\n' % (_metanamemap[k], meta[k])
 
         content = b'%s%s\n%s' % (header, desc, body)
         patches.append((drev[b'id'], content))
 
     # Write patches to the supplied callback
     write(patches)
 
 
 @vcrcommand(
     b'phabread',
     [(b'', b'stack', False, _(b'read dependencies'))],
     _(b'DREVSPEC... [OPTIONS]'),
     helpcategory=command.CATEGORY_IMPORT_EXPORT,
     optionalrepo=True,
 )
 def phabread(ui, repo, *specs, **opts):
     """print patches from Phabricator suitable for importing
 
     DREVSPEC could be a Differential Revision identity, like ``D123``, or just
     the number ``123``. It could also have common operators like ``+``, ``-``,
     ``&``, ``(``, ``)`` for complex queries. Prefix ``:`` could be used to
     select a stack.  If multiple DREVSPEC values are given, the result is the
     union of each individually evaluated value.  No attempt is currently made
     to reorder the values to run from parent to child.
 
     ``abandoned``, ``accepted``, ``closed``, ``needsreview``, ``needsrevision``
     could be used to filter patches by status. For performance reason, they
     only represent a subset of non-status selections and cannot be used alone.
 
     For example, ``:D6+8-(2+D4)`` selects a stack up to D6, plus D8 and exclude
     D2 and D4. ``:D9 & needsreview`` selects "Needs Review" revisions in a
     stack up to D9.
 
     If --stack is given, follow dependencies information and read all patches.
     It is equivalent to the ``:`` operator.
     """
     opts = pycompat.byteskwargs(opts)
     drevs = _getdrevs(ui, opts.get(b'stack'), specs)
 
     def _write(patches):
         for drev, content in patches:
             ui.write(content)
 
     readpatch(ui, drevs, _write)
 
 
 @vcrcommand(
     b'phabimport',
     [(b'', b'stack', False, _(b'import dependencies as well'))],
     _(b'DREVSPEC... [OPTIONS]'),
     helpcategory=command.CATEGORY_IMPORT_EXPORT,
 )
 def phabimport(ui, repo, *specs, **opts):
     """import patches from Phabricator for the specified Differential Revisions
 
     The patches are read and applied starting at the parent of the working
     directory.
 
     See ``hg help phabread`` for how to specify DREVSPEC.
     """
     opts = pycompat.byteskwargs(opts)
 
     # --bypass avoids losing exec and symlink bits when importing on Windows,
     # and allows importing with a dirty wdir.  It also aborts instead of leaving
     # rejects.
     opts[b'bypass'] = True
 
     # Mandatory default values, synced with commands.import
     opts[b'strip'] = 1
     opts[b'prefix'] = b''
     # Evolve 9.3.0 assumes this key is present in cmdutil.tryimportone()
     opts[b'obsolete'] = False
 
     if ui.configbool(b'phabimport', b'secret'):
         opts[b'secret'] = True
     if ui.configbool(b'phabimport', b'obsolete'):
         opts[b'obsolete'] = True  # Handled by evolve wrapping tryimportone()
 
     def _write(patches):
         parents = repo[None].parents()
 
         with repo.wlock(), repo.lock(), repo.transaction(b'phabimport'):
             for drev, contents in patches:
                 ui.status(_(b'applying patch from D%s\n') % drev)
 
                 with patch.extract(ui, io.BytesIO(contents)) as patchdata:
                     msg, node, rej = cmdutil.tryimportone(
                         ui,
                         repo,
                         patchdata,
                         parents,
                         opts,
                         [],
                         None,  # Never update wdir to another revision
                     )
 
                     if not node:
                         raise error.Abort(_(b'D%s: no diffs found') % drev)
 
                     ui.note(msg + b'\n')
                     parents = [repo[node]]
 
     drevs = _getdrevs(ui, opts.get(b'stack'), specs)
 
     readpatch(repo.ui, drevs, _write)
 
 
 @vcrcommand(
     b'phabupdate',
     [
         (b'', b'accept', False, _(b'accept revisions')),
         (b'', b'reject', False, _(b'reject revisions')),
         (b'', b'request-review', False, _(b'request review on revisions')),
         (b'', b'abandon', False, _(b'abandon revisions')),
         (b'', b'reclaim', False, _(b'reclaim revisions')),
         (b'', b'close', False, _(b'close revisions')),
         (b'', b'reopen', False, _(b'reopen revisions')),
         (b'', b'plan-changes', False, _(b'plan changes for revisions')),
         (b'', b'resign', False, _(b'resign as a reviewer from revisions')),
         (b'', b'commandeer', False, _(b'commandeer revisions')),
         (b'm', b'comment', b'', _(b'comment on the last revision')),
         (b'r', b'rev', b'', _(b'local revision to update'), _(b'REV')),
     ],
     _(b'[DREVSPEC...| -r REV...] [OPTIONS]'),
     helpcategory=command.CATEGORY_IMPORT_EXPORT,
     optionalrepo=True,
 )
 def phabupdate(ui, repo, *specs, **opts):
     """update Differential Revision in batch
 
     DREVSPEC selects revisions. See :hg:`help phabread` for its usage.
     """
     opts = pycompat.byteskwargs(opts)
     transactions = [
         b'abandon',
         b'accept',
         b'close',
         b'commandeer',
         b'plan-changes',
         b'reclaim',
         b'reject',
         b'reopen',
         b'request-review',
         b'resign',
     ]
     flags = [n for n in transactions if opts.get(n.replace(b'-', b'_'))]
     if len(flags) > 1:
         raise error.Abort(_(b'%s cannot be used together') % b', '.join(flags))
 
     actions = []
     for f in flags:
         actions.append({b'type': f, b'value': True})
 
     revs = opts.get(b'rev')
     if revs:
         if not repo:
             raise error.InputError(_(b'--rev requires a repository'))
 
         if specs:
             raise error.InputError(_(b'cannot specify both DREVSPEC and --rev'))
 
         drevmap = getdrevmap(repo, logcmdutil.revrange(repo, [revs]))
         specs = []
         unknown = []
         for r, d in drevmap.items():
             if d is None:
                 unknown.append(repo[r])
             else:
                 specs.append(b'D%d' % d)
         if unknown:
             raise error.InputError(
                 _(b'selected revisions without a Differential: %s')
                 % scmutil.nodesummaries(repo, unknown)
             )
 
     drevs = _getdrevs(ui, opts.get(b'stack'), specs)
     for i, drev in enumerate(drevs):
         if i + 1 == len(drevs) and opts.get(b'comment'):
             actions.append({b'type': b'comment', b'value': opts[b'comment']})
         if actions:
             params = {
                 b'objectIdentifier': drev[b'phid'],
                 b'transactions': actions,
             }
             callconduit(ui, b'differential.revision.edit', params)
 
 
 @eh.templatekeyword(b'phabreview', requires={b'ctx'})
 def template_review(context, mapping):
     """:phabreview: Object describing the review for this changeset.
     Has attributes `url` and `id`.
     """
     ctx = context.resource(mapping, b'ctx')
     m = _differentialrevisiondescre.search(ctx.description())
     if m:
         return templateutil.hybriddict(
             {
                 b'url': m.group('url'),
                 b'id': b"D%s" % m.group('id'),
             }
         )
     else:
         tags = ctx.repo().nodetags(ctx.node())
         for t in tags:
             if _differentialrevisiontagre.match(t):
                 url = ctx.repo().ui.config(b'phabricator', b'url')
                 if not url.endswith(b'/'):
                     url += b'/'
                 url += t
 
                 return templateutil.hybriddict(
                     {
                         b'url': url,
                         b'id': t,
                     }
                 )
     return None
 
 
 @eh.templatekeyword(b'phabstatus', requires={b'ctx', b'repo', b'ui'})
 def template_status(context, mapping):
     """:phabstatus: String. Status of Phabricator differential."""
     ctx = context.resource(mapping, b'ctx')
     repo = context.resource(mapping, b'repo')
     ui = context.resource(mapping, b'ui')
 
     rev = ctx.rev()
     try:
         drevid = getdrevmap(repo, [rev])[rev]
     except KeyError:
         return None
     drevs = callconduit(ui, b'differential.query', {b'ids': [drevid]})
     for drev in drevs:
         if int(drev[b'id']) == drevid:
             return templateutil.hybriddict(
                 {
                     b'url': drev[b'uri'],
                     b'status': drev[b'statusName'],
                 }
             )
     return None
 
 
 @show.showview(b'phabstatus', csettopic=b'work')
 def phabstatusshowview(ui, repo, displayer):
     """Phabricator differiential status"""
     revs = repo.revs('sort(_underway(), topo)')
     drevmap = getdrevmap(repo, revs)
     unknownrevs, drevids, revsbydrevid = [], set(), {}
     for rev, drevid in drevmap.items():
         if drevid is not None:
             drevids.add(drevid)
             revsbydrevid.setdefault(drevid, set()).add(rev)
         else:
             unknownrevs.append(rev)
 
     drevs = callconduit(ui, b'differential.query', {b'ids': list(drevids)})
     drevsbyrev = {}
     for drev in drevs:
         for rev in revsbydrevid[int(drev[b'id'])]:
             drevsbyrev[rev] = drev
 
     def phabstatus(ctx):
         drev = drevsbyrev[ctx.rev()]
         status = ui.label(
             b'%(statusName)s' % drev,
             b'phabricator.status.%s' % _getstatusname(drev),
         )
         ui.write(b"\n%s %s\n" % (drev[b'uri'], status))
 
     revs -= smartset.baseset(unknownrevs)
     revdag = graphmod.dagwalker(repo, revs)
 
     ui.setconfig(b'experimental', b'graphshorten', True)
     displayer._exthook = phabstatus
     nodelen = show.longestshortest(repo, revs)
     logcmdutil.displaygraph(
         ui,
         repo,
         revdag,
         displayer,
         graphmod.asciiedges,
         props={b'nodelen': nodelen},
     )

hgext/win32mbcs.py

@@ -1,222 +1,222
 # win32mbcs.py -- MBCS filename support for Mercurial
 #
 # Copyright (c) 2008 Shun-ichi Goto <[e-mail]>
 #
 # Version: 0.3
 # Author:  Shun-ichi Goto <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 #
 
 '''allow the use of MBCS paths with problematic encodings
 
 Some MBCS encodings are not good for some path operations (i.e.
 splitting path, case conversion, etc.) with its encoded bytes. We call
 such a encoding (i.e. shift_jis and big5) as "problematic encoding".
 This extension can be used to fix the issue with those encodings by
 wrapping some functions to convert to Unicode string before path
 operation.
 
 This extension is useful for:
 
 - Japanese Windows users using shift_jis encoding.
 - Chinese Windows users using big5 encoding.
 - All users who use a repository with one of problematic encodings on
   case-insensitive file system.
 
 This extension is not needed for:
 
 - Any user who use only ASCII chars in path.
 - Any user who do not use any of problematic encodings.
 
 Note that there are some limitations on using this extension:
 
 - You should use single encoding in one repository.
 - If the repository path ends with 0x5c, .hg/hgrc cannot be read.
 - win32mbcs is not compatible with fixutf8 extension.
 
 By default, win32mbcs uses encoding.encoding decided by Mercurial.
 You can specify the encoding by config option::
 
  [win32mbcs]
  encoding = sjis
 
 It is useful for the users who want to commit with UTF-8 log message.
 '''
 
 import os
 import sys
 
 from mercurial.i18n import _
 from mercurial.pycompat import getattr, setattr
 from mercurial import (
     encoding,
     error,
     pycompat,
     registrar,
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 # Encoding.encoding may be updated by --encoding option.
 # Use a lambda do delay the resolution.
 configitem(
     b'win32mbcs',
     b'encoding',
     default=lambda: encoding.encoding,
 )
 
 _encoding = None  # see extsetup
 
 
 def decode(arg):
     if isinstance(arg, bytes):
         uarg = arg.decode(_encoding)
         if arg == uarg.encode(_encoding):
             return uarg
         raise UnicodeError(b"Not local encoding")
     elif isinstance(arg, tuple):
         return tuple(map(decode, arg))
     elif isinstance(arg, list):
         return map(decode, arg)
     elif isinstance(arg, dict):
         for k, v in arg.items():
             arg[k] = decode(v)
     return arg
 
 
 def encode(arg):
-    if isinstance(arg, pycompat.unicode):
+    if isinstance(arg, str):
         return arg.encode(_encoding)
     elif isinstance(arg, tuple):
         return tuple(map(encode, arg))
     elif isinstance(arg, list):
         return map(encode, arg)
     elif isinstance(arg, dict):
         for k, v in arg.items():
             arg[k] = encode(v)
     return arg
 
 
 def appendsep(s):
     # ensure the path ends with os.sep, appending it if necessary.
     try:
         us = decode(s)
     except UnicodeError:
         us = s
     if us and us[-1] not in b':/\\':
         s += pycompat.ossep
     return s
 
 
 def basewrapper(func, argtype, enc, dec, args, kwds):
     # check check already converted, then call original
     for arg in args:
         if isinstance(arg, argtype):
             return func(*args, **kwds)
 
     try:
         # convert string arguments, call func, then convert back the
         # return value.
         return enc(func(*dec(args), **dec(kwds)))
     except UnicodeError:
         raise error.Abort(
             _(b"[win32mbcs] filename conversion failed with %s encoding\n")
             % _encoding
         )
 
 
 def wrapper(func, args, kwds):
-    return basewrapper(func, pycompat.unicode, encode, decode, args, kwds)
+    return basewrapper(func, str, encode, decode, args, kwds)
 
 
 def reversewrapper(func, args, kwds):
     return basewrapper(func, str, decode, encode, args, kwds)
 
 
 def wrapperforlistdir(func, args, kwds):
     # Ensure 'path' argument ends with os.sep to avoids
     # misinterpreting last 0x5c of MBCS 2nd byte as path separator.
     if args:
         args = list(args)
         args[0] = appendsep(args[0])
     if b'path' in kwds:
         kwds[b'path'] = appendsep(kwds[b'path'])
     return func(*args, **kwds)
 
 
 def wrapname(name, wrapper):
     module, name = name.rsplit(b'.', 1)
     module = sys.modules[module]
     func = getattr(module, name)
 
     def f(*args, **kwds):
         return wrapper(func, args, kwds)
 
     f.__name__ = func.__name__
     setattr(module, name, f)
 
 
 # List of functions to be wrapped.
 # NOTE: os.path.dirname() and os.path.basename() are safe because
 #       they use result of os.path.split()
 funcs = b'''os.path.join os.path.split os.path.splitext
  os.path.normpath os.makedirs mercurial.util.endswithsep
  mercurial.util.splitpath mercurial.util.fscasesensitive
  mercurial.util.fspath mercurial.util.pconvert mercurial.util.normpath
  mercurial.util.checkwinfilename mercurial.util.checkosfilename
  mercurial.util.split'''
 
 # These functions are required to be called with local encoded string
 # because they expects argument is local encoded string and cause
 # problem with unicode string.
 rfuncs = b'''mercurial.encoding.upper mercurial.encoding.lower
  mercurial.util._filenamebytestr'''
 
 # List of Windows specific functions to be wrapped.
 winfuncs = b'''os.path.splitunc'''
 
 # codec and alias names of sjis and big5 to be faked.
 problematic_encodings = b'''big5 big5-tw csbig5 big5hkscs big5-hkscs
  hkscs cp932 932 ms932 mskanji ms-kanji shift_jis csshiftjis shiftjis
  sjis s_jis shift_jis_2004 shiftjis2004 sjis_2004 sjis2004
  shift_jisx0213 shiftjisx0213 sjisx0213 s_jisx0213 950 cp950 ms950 '''
 
 
 def extsetup(ui):
     # TODO: decide use of config section for this extension
     if (not os.path.supports_unicode_filenames) and (
         pycompat.sysplatform != b'cygwin'
     ):
         ui.warn(_(b"[win32mbcs] cannot activate on this platform.\n"))
         return
     # determine encoding for filename
     global _encoding
     _encoding = ui.config(b'win32mbcs', b'encoding')
     # fake is only for relevant environment.
     if _encoding.lower() in problematic_encodings.split():
         for f in funcs.split():
             wrapname(f, wrapper)
         if pycompat.iswindows:
             for f in winfuncs.split():
                 wrapname(f, wrapper)
         wrapname(b"mercurial.util.listdir", wrapperforlistdir)
         wrapname(b"mercurial.windows.listdir", wrapperforlistdir)
         # wrap functions to be called with local byte string arguments
         for f in rfuncs.split():
             wrapname(f, reversewrapper)
         # Check sys.args manually instead of using ui.debug() because
         # command line options is not yet applied when
         # extensions.loadall() is called.
         if b'--debug' in sys.argv:
             ui.writenoi18n(
                 b"[win32mbcs] activated with encoding: %s\n" % _encoding
             )

mercurial/hgweb/__init__.py

@@ -1,123 +1,123
 # hgweb/__init__.py - web interface to a mercurial repository
 #
 # Copyright 21 May 2005 - (c) 2005 Jake Edge <[e-mail]>
 # Copyright 2005 Olivia Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 
 import os
 
 from ..i18n import _
 
 from .. import (
     error,
     pycompat,
 )
 
 from ..utils import procutil
 
 from . import (
     hgweb_mod,
     hgwebdir_mod,
     server,
 )
 
 
 def hgweb(config, name=None, baseui=None):
     """create an hgweb wsgi object
 
     config can be one of:
     - repo object (single repo view)
     - path to repo (single repo view)
     - path to config file (multi-repo view)
     - dict of virtual:real pairs (multi-repo view)
     - list of virtual:real tuples (multi-repo view)
     """
 
-    if isinstance(config, pycompat.unicode):
+    if isinstance(config, str):
         raise error.ProgrammingError(
             b'Mercurial only supports encoded strings: %r' % config
         )
     if (
         (isinstance(config, bytes) and not os.path.isdir(config))
         or isinstance(config, dict)
         or isinstance(config, list)
     ):
         # create a multi-dir interface
         return hgwebdir_mod.hgwebdir(config, baseui=baseui)
     return hgweb_mod.hgweb(config, name=name, baseui=baseui)
 
 
 def hgwebdir(config, baseui=None):
     return hgwebdir_mod.hgwebdir(config, baseui=baseui)
 
 
 class httpservice(object):
     def __init__(self, ui, app, opts):
         self.ui = ui
         self.app = app
         self.opts = opts
 
     def init(self):
         procutil.setsignalhandler()
         self.httpd = server.create_server(self.ui, self.app)
 
         if (
             self.opts[b'port']
             and not self.ui.verbose
             and not self.opts[b'print_url']
         ):
             return
 
         if self.httpd.prefix:
             prefix = self.httpd.prefix.strip(b'/') + b'/'
         else:
             prefix = b''
 
         port = ':%d' % self.httpd.port
         if port == ':80':
             port = ''
 
         bindaddr = self.httpd.addr
         if bindaddr == '0.0.0.0':
             bindaddr = '*'
         elif ':' in bindaddr:  # IPv6
             bindaddr = '[%s]' % bindaddr
 
         fqaddr = self.httpd.fqaddr
         if ':' in fqaddr:
             fqaddr = '[%s]' % fqaddr
 
         url = b'http://%s%s/%s' % (
             pycompat.sysbytes(fqaddr),
             pycompat.sysbytes(port),
             prefix,
         )
         if self.opts[b'print_url']:
             self.ui.write(b'%s\n' % url)
         else:
             if self.opts[b'port']:
                 write = self.ui.status
             else:
                 write = self.ui.write
             write(
                 _(b'listening at %s (bound to %s:%d)\n')
                 % (url, pycompat.sysbytes(bindaddr), self.httpd.port)
             )
         self.ui.flush()  # avoid buffering of status message
 
     def run(self):
         self.httpd.serve_forever()
 
 
 def createapp(baseui, repo, webconf):
     if webconf:
         return hgwebdir_mod.hgwebdir(webconf, baseui=baseui)
     else:
         if not repo:
             raise error.RepoError(
                 _(b"there is no Mercurial repository here (.hg not found)")
             )
         return hgweb_mod.hgweb(repo, baseui=baseui)

mercurial/i18n.py

@@ -1,125 +1,125
 # i18n.py - internationalization support for mercurial
 #
 # Copyright 2005, 2006 Olivia Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 
 import gettext as gettextmod
 import locale
 import os
 import sys
 
 from .pycompat import getattr
 from .utils import resourceutil
 from . import (
     encoding,
     pycompat,
 )
 
 if pycompat.TYPE_CHECKING:
     from typing import (
         Callable,
         List,
     )
 
 
 # modelled after templater.templatepath:
 if getattr(sys, 'frozen', None) is not None:
     module = pycompat.sysexecutable
 else:
     module = pycompat.fsencode(__file__)
 
 _languages = None
 if (
     pycompat.iswindows
     and b'LANGUAGE' not in encoding.environ
     and b'LC_ALL' not in encoding.environ
     and b'LC_MESSAGES' not in encoding.environ
     and b'LANG' not in encoding.environ
 ):
     # Try to detect UI language by "User Interface Language Management" API
     # if no locale variables are set. Note that locale.getdefaultlocale()
     # uses GetLocaleInfo(), which may be different from UI language.
     # (See http://msdn.microsoft.com/en-us/library/dd374098(v=VS.85).aspx )
     try:
         import ctypes
 
         # pytype: disable=module-attr
         langid = ctypes.windll.kernel32.GetUserDefaultUILanguage()
         # pytype: enable=module-attr
 
         _languages = [locale.windows_locale[langid]]
     except (ImportError, AttributeError, KeyError):
         # ctypes not found or unknown langid
         pass
 
 
 datapath = pycompat.fsdecode(resourceutil.datapath)
 localedir = os.path.join(datapath, 'locale')
 t = gettextmod.translation('hg', localedir, _languages, fallback=True)
 try:
     _ugettext = t.ugettext  # pytype: disable=attribute-error
 except AttributeError:
     _ugettext = t.gettext
 
 
 _msgcache = {}  # encoding: {message: translation}
 
 
 def gettext(message):
     # type: (bytes) -> bytes
     """Translate message.
 
     The message is looked up in the catalog to get a Unicode string,
     which is encoded in the local encoding before being returned.
 
     Important: message is restricted to characters in the encoding
     given by sys.getdefaultencoding() which is most likely 'ascii'.
     """
     # If message is None, t.ugettext will return u'None' as the
     # translation whereas our callers expect us to return None.
     if message is None or not _ugettext:
         return message
 
     cache = _msgcache.setdefault(encoding.encoding, {})
     if message not in cache:
-        if type(message) is pycompat.unicode:
+        if type(message) is str:
             # goofy unicode docstrings in test
-            paragraphs = message.split(u'\n\n')  # type: List[pycompat.unicode]
+            paragraphs = message.split(u'\n\n')  # type: List[str]
         else:
             # should be ascii, but we have unicode docstrings in test, which
             # are converted to utf-8 bytes on Python 3.
             paragraphs = [p.decode("utf-8") for p in message.split(b'\n\n')]
         # Be careful not to translate the empty string -- it holds the
         # meta data of the .po file.
         u = u'\n\n'.join([p and _ugettext(p) or u'' for p in paragraphs])
         try:
             # encoding.tolocal cannot be used since it will first try to
             # decode the Unicode string. Calling u.decode(enc) really
             # means u.encode(sys.getdefaultencoding()).decode(enc). Since
             # the Python encoding defaults to 'ascii', this fails if the
             # translated string use non-ASCII characters.
             encodingstr = pycompat.sysstr(encoding.encoding)
             cache[message] = u.encode(encodingstr, "replace")
         except LookupError:
             # An unknown encoding results in a LookupError.
             cache[message] = message
     return cache[message]
 
 
 def _plain():
     if (
         b'HGPLAIN' not in encoding.environ
         and b'HGPLAINEXCEPT' not in encoding.environ
     ):
         return False
     exceptions = encoding.environ.get(b'HGPLAINEXCEPT', b'').strip().split(b',')
     return b'i18n' not in exceptions
 
 
 if _plain():
     _ = lambda message: message  # type: Callable[[bytes], bytes]
 else:
     _ = gettext

mercurial/scmutil.py

@@ -1,2303 +1,2303
 # scmutil.py - Mercurial core utility functions
 #
 #  Copyright Olivia Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 
 import errno
 import glob
 import os
 import posixpath
 import re
 import subprocess
 import weakref
 
 from .i18n import _
 from .node import (
     bin,
     hex,
     nullrev,
     short,
     wdirrev,
 )
 from .pycompat import getattr
 from .thirdparty import attr
 from . import (
     copies as copiesmod,
     encoding,
     error,
     match as matchmod,
     obsolete,
     obsutil,
     pathutil,
     phases,
     policy,
     pycompat,
     requirements as requirementsmod,
     revsetlang,
     similar,
     smartset,
     url,
     util,
     vfs,
 )
 
 from .utils import (
     hashutil,
     procutil,
     stringutil,
 )
 
 if pycompat.iswindows:
     from . import scmwindows as scmplatform
 else:
     from . import scmposix as scmplatform
 
 parsers = policy.importmod('parsers')
 rustrevlog = policy.importrust('revlog')
 
 termsize = scmplatform.termsize
 
 
 @attr.s(slots=True, repr=False)
 class status(object):
     """Struct with a list of files per status.
 
     The 'deleted', 'unknown' and 'ignored' properties are only
     relevant to the working copy.
     """
 
     modified = attr.ib(default=attr.Factory(list))
     added = attr.ib(default=attr.Factory(list))
     removed = attr.ib(default=attr.Factory(list))
     deleted = attr.ib(default=attr.Factory(list))
     unknown = attr.ib(default=attr.Factory(list))
     ignored = attr.ib(default=attr.Factory(list))
     clean = attr.ib(default=attr.Factory(list))
 
     def __iter__(self):
         yield self.modified
         yield self.added
         yield self.removed
         yield self.deleted
         yield self.unknown
         yield self.ignored
         yield self.clean
 
     def __repr__(self):
         return (
             r'<status modified=%s, added=%s, removed=%s, deleted=%s, '
             r'unknown=%s, ignored=%s, clean=%s>'
         ) % tuple(pycompat.sysstr(stringutil.pprint(v)) for v in self)
 
 
 def itersubrepos(ctx1, ctx2):
     """find subrepos in ctx1 or ctx2"""
     # Create a (subpath, ctx) mapping where we prefer subpaths from
     # ctx1. The subpaths from ctx2 are important when the .hgsub file
     # has been modified (in ctx2) but not yet committed (in ctx1).
     subpaths = dict.fromkeys(ctx2.substate, ctx2)
     subpaths.update(dict.fromkeys(ctx1.substate, ctx1))
 
     missing = set()
 
     for subpath in ctx2.substate:
         if subpath not in ctx1.substate:
             del subpaths[subpath]
             missing.add(subpath)
 
     for subpath, ctx in sorted(subpaths.items()):
         yield subpath, ctx.sub(subpath)
 
     # Yield an empty subrepo based on ctx1 for anything only in ctx2.  That way,
     # status and diff will have an accurate result when it does
     # 'sub.{status|diff}(rev2)'.  Otherwise, the ctx2 subrepo is compared
     # against itself.
     for subpath in missing:
         yield subpath, ctx2.nullsub(subpath, ctx1)
 
 
 def nochangesfound(ui, repo, excluded=None):
     """Report no changes for push/pull, excluded is None or a list of
     nodes excluded from the push/pull.
     """
     secretlist = []
     if excluded:
         for n in excluded:
             ctx = repo[n]
             if ctx.phase() >= phases.secret and not ctx.extinct():
                 secretlist.append(n)
 
     if secretlist:
         ui.status(
             _(b"no changes found (ignored %d secret changesets)\n")
             % len(secretlist)
         )
     else:
         ui.status(_(b"no changes found\n"))
 
 
 def callcatch(ui, func):
     """call func() with global exception handling
 
     return func() if no exception happens. otherwise do some error handling
     and return an exit code accordingly. does not handle all exceptions.
     """
     coarse_exit_code = -1
     detailed_exit_code = -1
     try:
         try:
             return func()
         except:  # re-raises
             ui.traceback()
             raise
     # Global exception handling, alphabetically
     # Mercurial-specific first, followed by built-in and library exceptions
     except error.LockHeld as inst:
         detailed_exit_code = 20
         if inst.errno == errno.ETIMEDOUT:
             reason = _(b'timed out waiting for lock held by %r') % (
                 pycompat.bytestr(inst.locker)
             )
         else:
             reason = _(b'lock held by %r') % inst.locker
         ui.error(
             _(b"abort: %s: %s\n")
             % (inst.desc or stringutil.forcebytestr(inst.filename), reason)
         )
         if not inst.locker:
             ui.error(_(b"(lock might be very busy)\n"))
     except error.LockUnavailable as inst:
         detailed_exit_code = 20
         ui.error(
             _(b"abort: could not lock %s: %s\n")
             % (
                 inst.desc or stringutil.forcebytestr(inst.filename),
                 encoding.strtolocal(inst.strerror),
             )
         )
     except error.RepoError as inst:
         if isinstance(inst, error.RepoLookupError):
             detailed_exit_code = 10
         ui.error(_(b"abort: %s\n") % inst)
         if inst.hint:
             ui.error(_(b"(%s)\n") % inst.hint)
     except error.ResponseError as inst:
         ui.error(_(b"abort: %s") % inst.args[0])
         msg = inst.args[1]
         if isinstance(msg, type(u'')):
             msg = pycompat.sysbytes(msg)
         if msg is None:
             ui.error(b"\n")
         elif not isinstance(msg, bytes):
             ui.error(b" %r\n" % (msg,))
         elif not msg:
             ui.error(_(b" empty string\n"))
         else:
             ui.error(b"\n%r\n" % pycompat.bytestr(stringutil.ellipsis(msg)))
     except error.CensoredNodeError as inst:
         ui.error(_(b"abort: file censored %s\n") % inst)
     except error.WdirUnsupported:
         ui.error(_(b"abort: working directory revision cannot be specified\n"))
     except error.Error as inst:
         if inst.detailed_exit_code is not None:
             detailed_exit_code = inst.detailed_exit_code
         if inst.coarse_exit_code is not None:
             coarse_exit_code = inst.coarse_exit_code
         ui.error(inst.format())
     except error.WorkerError as inst:
         # Don't print a message -- the worker already should have
         return inst.status_code
     except ImportError as inst:
         ui.error(_(b"abort: %s\n") % stringutil.forcebytestr(inst))
         m = stringutil.forcebytestr(inst).split()[-1]
         if m in b"mpatch bdiff".split():
             ui.error(_(b"(did you forget to compile extensions?)\n"))
         elif m in b"zlib".split():
             ui.error(_(b"(is your Python install correct?)\n"))
     except util.urlerr.httperror as inst:
         detailed_exit_code = 100
         ui.error(_(b"abort: %s\n") % stringutil.forcebytestr(inst))
     except util.urlerr.urlerror as inst:
         detailed_exit_code = 100
         try:  # usually it is in the form (errno, strerror)
             reason = inst.reason.args[1]
         except (AttributeError, IndexError):
             # it might be anything, for example a string
             reason = inst.reason
-        if isinstance(reason, pycompat.unicode):
+        if isinstance(reason, str):
             # SSLError of Python 2.7.9 contains a unicode
             reason = encoding.unitolocal(reason)
         ui.error(_(b"abort: error: %s\n") % stringutil.forcebytestr(reason))
     except (IOError, OSError) as inst:
         if (
             util.safehasattr(inst, b"args")
             and inst.args
             and inst.args[0] == errno.EPIPE
         ):
             pass
         elif getattr(inst, "strerror", None):  # common IOError or OSError
             if getattr(inst, "filename", None) is not None:
                 ui.error(
                     _(b"abort: %s: '%s'\n")
                     % (
                         encoding.strtolocal(inst.strerror),
                         stringutil.forcebytestr(inst.filename),
                     )
                 )
             else:
                 ui.error(_(b"abort: %s\n") % encoding.strtolocal(inst.strerror))
         else:  # suspicious IOError
             raise
     except MemoryError:
         ui.error(_(b"abort: out of memory\n"))
     except SystemExit as inst:
         # Commands shouldn't sys.exit directly, but give a return code.
         # Just in case catch this and and pass exit code to caller.
         detailed_exit_code = 254
         coarse_exit_code = inst.code
 
     if ui.configbool(b'ui', b'detailed-exit-code'):
         return detailed_exit_code
     else:
         return coarse_exit_code
 
 
 def checknewlabel(repo, lbl, kind):
     # Do not use the "kind" parameter in ui output.
     # It makes strings difficult to translate.
     if lbl in [b'tip', b'.', b'null']:
         raise error.InputError(_(b"the name '%s' is reserved") % lbl)
     for c in (b':', b'\0', b'\n', b'\r'):
         if c in lbl:
             raise error.InputError(
                 _(b"%r cannot be used in a name") % pycompat.bytestr(c)
             )
     try:
         int(lbl)
         raise error.InputError(_(b"cannot use an integer as a name"))
     except ValueError:
         pass
     if lbl.strip() != lbl:
         raise error.InputError(
             _(b"leading or trailing whitespace in name %r") % lbl
         )
 
 
 def checkfilename(f):
     '''Check that the filename f is an acceptable filename for a tracked file'''
     if b'\r' in f or b'\n' in f:
         raise error.InputError(
             _(b"'\\n' and '\\r' disallowed in filenames: %r")
             % pycompat.bytestr(f)
         )
 
 
 def checkportable(ui, f):
     '''Check if filename f is portable and warn or abort depending on config'''
     checkfilename(f)
     abort, warn = checkportabilityalert(ui)
     if abort or warn:
         msg = util.checkwinfilename(f)
         if msg:
             msg = b"%s: %s" % (msg, procutil.shellquote(f))
             if abort:
                 raise error.InputError(msg)
             ui.warn(_(b"warning: %s\n") % msg)
 
 
 def checkportabilityalert(ui):
     """check if the user's config requests nothing, a warning, or abort for
     non-portable filenames"""
     val = ui.config(b'ui', b'portablefilenames')
     lval = val.lower()
     bval = stringutil.parsebool(val)
     abort = pycompat.iswindows or lval == b'abort'
     warn = bval or lval == b'warn'
     if bval is None and not (warn or abort or lval == b'ignore'):
         raise error.ConfigError(
             _(b"ui.portablefilenames value is invalid ('%s')") % val
         )
     return abort, warn
 
 
 class casecollisionauditor(object):
     def __init__(self, ui, abort, dirstate):
         self._ui = ui
         self._abort = abort
         allfiles = b'\0'.join(dirstate)
         self._loweredfiles = set(encoding.lower(allfiles).split(b'\0'))
         self._dirstate = dirstate
         # The purpose of _newfiles is so that we don't complain about
         # case collisions if someone were to call this object with the
         # same filename twice.
         self._newfiles = set()
 
     def __call__(self, f):
         if f in self._newfiles:
             return
         fl = encoding.lower(f)
         if fl in self._loweredfiles and f not in self._dirstate:
             msg = _(b'possible case-folding collision for %s') % f
             if self._abort:
                 raise error.StateError(msg)
             self._ui.warn(_(b"warning: %s\n") % msg)
         self._loweredfiles.add(fl)
         self._newfiles.add(f)
 
 
 def filteredhash(repo, maxrev, needobsolete=False):
     """build hash of filtered revisions in the current repoview.
 
     Multiple caches perform up-to-date validation by checking that the
     tiprev and tipnode stored in the cache file match the current repository.
     However, this is not sufficient for validating repoviews because the set
     of revisions in the view may change without the repository tiprev and
     tipnode changing.
 
     This function hashes all the revs filtered from the view (and, optionally,
     all obsolete revs) up to maxrev and returns that SHA-1 digest.
     """
     cl = repo.changelog
     if needobsolete:
         obsrevs = obsolete.getrevs(repo, b'obsolete')
         if not cl.filteredrevs and not obsrevs:
             return None
         key = (maxrev, hash(cl.filteredrevs), hash(obsrevs))
     else:
         if not cl.filteredrevs:
             return None
         key = maxrev
         obsrevs = frozenset()
 
     result = cl._filteredrevs_hashcache.get(key)
     if not result:
         revs = sorted(r for r in cl.filteredrevs | obsrevs if r <= maxrev)
         if revs:
             s = hashutil.sha1()
             for rev in revs:
                 s.update(b'%d;' % rev)
             result = s.digest()
             cl._filteredrevs_hashcache[key] = result
     return result
 
 
 def walkrepos(path, followsym=False, seen_dirs=None, recurse=False):
     """yield every hg repository under path, always recursively.
     The recurse flag will only control recursion into repo working dirs"""
 
     def errhandler(err):
         if err.filename == path:
             raise err
 
     samestat = getattr(os.path, 'samestat', None)
     if followsym and samestat is not None:
 
         def adddir(dirlst, dirname):
             dirstat = os.stat(dirname)
             match = any(samestat(dirstat, lstdirstat) for lstdirstat in dirlst)
             if not match:
                 dirlst.append(dirstat)
             return not match
 
     else:
         followsym = False
 
     if (seen_dirs is None) and followsym:
         seen_dirs = []
         adddir(seen_dirs, path)
     for root, dirs, files in os.walk(path, topdown=True, onerror=errhandler):
         dirs.sort()
         if b'.hg' in dirs:
             yield root  # found a repository
             qroot = os.path.join(root, b'.hg', b'patches')
             if os.path.isdir(os.path.join(qroot, b'.hg')):
                 yield qroot  # we have a patch queue repo here
             if recurse:
                 # avoid recursing inside the .hg directory
                 dirs.remove(b'.hg')
             else:
                 dirs[:] = []  # don't descend further
         elif followsym:
             newdirs = []
             for d in dirs:
                 fname = os.path.join(root, d)
                 if adddir(seen_dirs, fname):
                     if os.path.islink(fname):
                         for hgname in walkrepos(fname, True, seen_dirs):
                             yield hgname
                     else:
                         newdirs.append(d)
             dirs[:] = newdirs
 
 
 def binnode(ctx):
     """Return binary node id for a given basectx"""
     node = ctx.node()
     if node is None:
         return ctx.repo().nodeconstants.wdirid
     return node
 
 
 def intrev(ctx):
     """Return integer for a given basectx that can be used in comparison or
     arithmetic operation"""
     rev = ctx.rev()
     if rev is None:
         return wdirrev
     return rev
 
 
 def formatchangeid(ctx):
     """Format changectx as '{rev}:{node|formatnode}', which is the default
     template provided by logcmdutil.changesettemplater"""
     repo = ctx.repo()
     return formatrevnode(repo.ui, intrev(ctx), binnode(ctx))
 
 
 def formatrevnode(ui, rev, node):
     """Format given revision and node depending on the current verbosity"""
     if ui.debugflag:
         hexfunc = hex
     else:
         hexfunc = short
     return b'%d:%s' % (rev, hexfunc(node))
 
 
 def resolvehexnodeidprefix(repo, prefix):
     if prefix.startswith(b'x'):
         prefix = prefix[1:]
     try:
         # Uses unfiltered repo because it's faster when prefix is ambiguous/
         # This matches the shortesthexnodeidprefix() function below.
         node = repo.unfiltered().changelog._partialmatch(prefix)
     except error.AmbiguousPrefixLookupError:
         revset = repo.ui.config(
             b'experimental', b'revisions.disambiguatewithin'
         )
         if revset:
             # Clear config to avoid infinite recursion
             configoverrides = {
                 (b'experimental', b'revisions.disambiguatewithin'): None
             }
             with repo.ui.configoverride(configoverrides):
                 revs = repo.anyrevs([revset], user=True)
                 matches = []
                 for rev in revs:
                     node = repo.changelog.node(rev)
                     if hex(node).startswith(prefix):
                         matches.append(node)
                 if len(matches) == 1:
                     return matches[0]
         raise
     if node is None:
         return
     repo.changelog.rev(node)  # make sure node isn't filtered
     return node
 
 
 def mayberevnum(repo, prefix):
     """Checks if the given prefix may be mistaken for a revision number"""
     try:
         i = int(prefix)
         # if we are a pure int, then starting with zero will not be
         # confused as a rev; or, obviously, if the int is larger
         # than the value of the tip rev. We still need to disambiguate if
         # prefix == '0', since that *is* a valid revnum.
         if (prefix != b'0' and prefix[0:1] == b'0') or i >= len(repo):
             return False
         return True
     except ValueError:
         return False
 
 
 def shortesthexnodeidprefix(repo, node, minlength=1, cache=None):
     """Find the shortest unambiguous prefix that matches hexnode.
 
     If "cache" is not None, it must be a dictionary that can be used for
     caching between calls to this method.
     """
     # _partialmatch() of filtered changelog could take O(len(repo)) time,
     # which would be unacceptably slow. so we look for hash collision in
     # unfiltered space, which means some hashes may be slightly longer.
 
     minlength = max(minlength, 1)
 
     def disambiguate(prefix):
         """Disambiguate against revnums."""
         if repo.ui.configbool(b'experimental', b'revisions.prefixhexnode'):
             if mayberevnum(repo, prefix):
                 return b'x' + prefix
             else:
                 return prefix
 
         hexnode = hex(node)
         for length in range(len(prefix), len(hexnode) + 1):
             prefix = hexnode[:length]
             if not mayberevnum(repo, prefix):
                 return prefix
 
     cl = repo.unfiltered().changelog
     revset = repo.ui.config(b'experimental', b'revisions.disambiguatewithin')
     if revset:
         revs = None
         if cache is not None:
             revs = cache.get(b'disambiguationrevset')
         if revs is None:
             revs = repo.anyrevs([revset], user=True)
             if cache is not None:
                 cache[b'disambiguationrevset'] = revs
         if cl.rev(node) in revs:
             hexnode = hex(node)
             nodetree = None
             if cache is not None:
                 nodetree = cache.get(b'disambiguationnodetree')
             if not nodetree:
                 if util.safehasattr(parsers, 'nodetree'):
                     # The CExt is the only implementation to provide a nodetree
                     # class so far.
                     index = cl.index
                     if util.safehasattr(index, 'get_cindex'):
                         # the rust wrapped need to give access to its internal index
                         index = index.get_cindex()
                     nodetree = parsers.nodetree(index, len(revs))
                     for r in revs:
                         nodetree.insert(r)
                     if cache is not None:
                         cache[b'disambiguationnodetree'] = nodetree
             if nodetree is not None:
                 length = max(nodetree.shortest(node), minlength)
                 prefix = hexnode[:length]
                 return disambiguate(prefix)
             for length in range(minlength, len(hexnode) + 1):
                 matches = []
                 prefix = hexnode[:length]
                 for rev in revs:
                     otherhexnode = repo[rev].hex()
                     if prefix == otherhexnode[:length]:
                         matches.append(otherhexnode)
                 if len(matches) == 1:
                     return disambiguate(prefix)
 
     try:
         return disambiguate(cl.shortest(node, minlength))
     except error.LookupError:
         raise error.RepoLookupError()
 
 
 def isrevsymbol(repo, symbol):
     """Checks if a symbol exists in the repo.
 
     See revsymbol() for details. Raises error.AmbiguousPrefixLookupError if the
     symbol is an ambiguous nodeid prefix.
     """
     try:
         revsymbol(repo, symbol)
         return True
     except error.RepoLookupError:
         return False
 
 
 def revsymbol(repo, symbol):
     """Returns a context given a single revision symbol (as string).
 
     This is similar to revsingle(), but accepts only a single revision symbol,
     i.e. things like ".", "tip", "1234", "deadbeef", "my-bookmark" work, but
     not "max(public())".
     """
     if not isinstance(symbol, bytes):
         msg = (
             b"symbol (%s of type %s) was not a string, did you mean "
             b"repo[symbol]?" % (symbol, type(symbol))
         )
         raise error.ProgrammingError(msg)
     try:
         if symbol in (b'.', b'tip', b'null'):
             return repo[symbol]
 
         try:
             r = int(symbol)
             if b'%d' % r != symbol:
                 raise ValueError
             l = len(repo.changelog)
             if r < 0:
                 r += l
             if r < 0 or r >= l and r != wdirrev:
                 raise ValueError
             return repo[r]
         except error.FilteredIndexError:
             raise
         except (ValueError, OverflowError, IndexError):
             pass
 
         if len(symbol) == 2 * repo.nodeconstants.nodelen:
             try:
                 node = bin(symbol)
                 rev = repo.changelog.rev(node)
                 return repo[rev]
             except error.FilteredLookupError:
                 raise
             except (TypeError, LookupError):
                 pass
 
         # look up bookmarks through the name interface
         try:
             node = repo.names.singlenode(repo, symbol)
             rev = repo.changelog.rev(node)
             return repo[rev]
         except KeyError:
             pass
 
         node = resolvehexnodeidprefix(repo, symbol)
         if node is not None:
             rev = repo.changelog.rev(node)
             return repo[rev]
 
         raise error.RepoLookupError(_(b"unknown revision '%s'") % symbol)
 
     except error.WdirUnsupported:
         return repo[None]
     except (
         error.FilteredIndexError,
         error.FilteredLookupError,
         error.FilteredRepoLookupError,
     ):
         raise _filterederror(repo, symbol)
 
 
 def _filterederror(repo, changeid):
     """build an exception to be raised about a filtered changeid
 
     This is extracted in a function to help extensions (eg: evolve) to
     experiment with various message variants."""
     if repo.filtername.startswith(b'visible'):
 
         # Check if the changeset is obsolete
         unfilteredrepo = repo.unfiltered()
         ctx = revsymbol(unfilteredrepo, changeid)
 
         # If the changeset is obsolete, enrich the message with the reason
         # that made this changeset not visible
         if ctx.obsolete():
             msg = obsutil._getfilteredreason(repo, changeid, ctx)
         else:
             msg = _(b"hidden revision '%s'") % changeid
 
         hint = _(b'use --hidden to access hidden revisions')
 
         return error.FilteredRepoLookupError(msg, hint=hint)
     msg = _(b"filtered revision '%s' (not in '%s' subset)")
     msg %= (changeid, repo.filtername)
     return error.FilteredRepoLookupError(msg)
 
 
 def revsingle(repo, revspec, default=b'.', localalias=None):
     if not revspec and revspec != 0:
         return repo[default]
 
     l = revrange(repo, [revspec], localalias=localalias)
     if not l:
         raise error.InputError(_(b'empty revision set'))
     return repo[l.last()]
 
 
 def _pairspec(revspec):
     tree = revsetlang.parse(revspec)
     return tree and tree[0] in (
         b'range',
         b'rangepre',
         b'rangepost',
         b'rangeall',
     )
 
 
 def revpair(repo, revs):
     if not revs:
         return repo[b'.'], repo[None]
 
     l = revrange(repo, revs)
 
     if not l:
         raise error.InputError(_(b'empty revision range'))
 
     first = l.first()
     second = l.last()
 
     if (
         first == second
         and len(revs) >= 2
         and not all(revrange(repo, [r]) for r in revs)
     ):
         raise error.InputError(_(b'empty revision on one side of range'))
 
     # if top-level is range expression, the result must always be a pair
     if first == second and len(revs) == 1 and not _pairspec(revs[0]):
         return repo[first], repo[None]
 
     return repo[first], repo[second]
 
 
 def revrange(repo, specs, localalias=None):
     """Execute 1 to many revsets and return the union.
 
     This is the preferred mechanism for executing revsets using user-specified
     config options, such as revset aliases.
 
     The revsets specified by ``specs`` will be executed via a chained ``OR``
     expression. If ``specs`` is empty, an empty result is returned.
 
     ``specs`` can contain integers, in which case they are assumed to be
     revision numbers.
 
     It is assumed the revsets are already formatted. If you have arguments
     that need to be expanded in the revset, call ``revsetlang.formatspec()``
     and pass the result as an element of ``specs``.
 
     Specifying a single revset is allowed.
 
     Returns a ``smartset.abstractsmartset`` which is a list-like interface over
     integer revisions.
     """
     allspecs = []
     for spec in specs:
         if isinstance(spec, int):
             spec = revsetlang.formatspec(b'%d', spec)
         allspecs.append(spec)
     return repo.anyrevs(allspecs, user=True, localalias=localalias)
 
 
 def increasingwindows(windowsize=8, sizelimit=512):
     while True:
         yield windowsize
         if windowsize < sizelimit:
             windowsize *= 2
 
 
 def walkchangerevs(repo, revs, makefilematcher, prepare):
     """Iterate over files and the revs in a "windowed" way.
 
     Callers most commonly need to iterate backwards over the history
     in which they are interested. Doing so has awful (quadratic-looking)
     performance, so we use iterators in a "windowed" way.
 
     We walk a window of revisions in the desired order.  Within the
     window, we first walk forwards to gather data, then in the desired
     order (usually backwards) to display it.
 
     This function returns an iterator yielding contexts. Before
     yielding each context, the iterator will first call the prepare
     function on each context in the window in forward order."""
 
     if not revs:
         return []
     change = repo.__getitem__
 
     def iterate():
         it = iter(revs)
         stopiteration = False
         for windowsize in increasingwindows():
             nrevs = []
             for i in pycompat.xrange(windowsize):
                 rev = next(it, None)
                 if rev is None:
                     stopiteration = True
                     break
                 nrevs.append(rev)
             for rev in sorted(nrevs):
                 ctx = change(rev)
                 prepare(ctx, makefilematcher(ctx))
             for rev in nrevs:
                 yield change(rev)
 
             if stopiteration:
                 break
 
     return iterate()
 
 
 def meaningfulparents(repo, ctx):
     """Return list of meaningful (or all if debug) parentrevs for rev.
 
     For merges (two non-nullrev revisions) both parents are meaningful.
     Otherwise the first parent revision is considered meaningful if it
     is not the preceding revision.
     """
     parents = ctx.parents()
     if len(parents) > 1:
         return parents
     if repo.ui.debugflag:
         return [parents[0], repo[nullrev]]
     if parents[0].rev() >= intrev(ctx) - 1:
         return []
     return parents
 
 
 def getuipathfn(repo, legacyrelativevalue=False, forcerelativevalue=None):
     """Return a function that produced paths for presenting to the user.
 
     The returned function takes a repo-relative path and produces a path
     that can be presented in the UI.
 
     Depending on the value of ui.relative-paths, either a repo-relative or
     cwd-relative path will be produced.
 
     legacyrelativevalue is the value to use if ui.relative-paths=legacy
 
     If forcerelativevalue is not None, then that value will be used regardless
     of what ui.relative-paths is set to.
     """
     if forcerelativevalue is not None:
         relative = forcerelativevalue
     else:
         config = repo.ui.config(b'ui', b'relative-paths')
         if config == b'legacy':
             relative = legacyrelativevalue
         else:
             relative = stringutil.parsebool(config)
             if relative is None:
                 raise error.ConfigError(
                     _(b"ui.relative-paths is not a boolean ('%s')") % config
                 )
 
     if relative:
         cwd = repo.getcwd()
         if cwd != b'':
             # this branch would work even if cwd == b'' (ie cwd = repo
             # root), but its generality makes the returned function slower
             pathto = repo.pathto
             return lambda f: pathto(f, cwd)
     if repo.ui.configbool(b'ui', b'slash'):
         return lambda f: f
     else:
         return util.localpath
 
 
 def subdiruipathfn(subpath, uipathfn):
     '''Create a new uipathfn that treats the file as relative to subpath.'''
     return lambda f: uipathfn(posixpath.join(subpath, f))
 
 
 def anypats(pats, opts):
     """Checks if any patterns, including --include and --exclude were given.
 
     Some commands (e.g. addremove) use this condition for deciding whether to
     print absolute or relative paths.
     """
     return bool(pats or opts.get(b'include') or opts.get(b'exclude'))
 
 
 def expandpats(pats):
     """Expand bare globs when running on windows.
     On posix we assume it already has already been done by sh."""
     if not util.expandglobs:
         return list(pats)
     ret = []
     for kindpat in pats:
         kind, pat = matchmod._patsplit(kindpat, None)
         if kind is None:
             try:
                 globbed = glob.glob(pat)
             except re.error:
                 globbed = [pat]
             if globbed:
                 ret.extend(globbed)
                 continue
         ret.append(kindpat)
     return ret
 
 
 def matchandpats(
     ctx, pats=(), opts=None, globbed=False, default=b'relpath', badfn=None
 ):
     """Return a matcher and the patterns that were used.
     The matcher will warn about bad matches, unless an alternate badfn callback
     is provided."""
     if opts is None:
         opts = {}
     if not globbed and default == b'relpath':
         pats = expandpats(pats or [])
 
     uipathfn = getuipathfn(ctx.repo(), legacyrelativevalue=True)
 
     def bad(f, msg):
         ctx.repo().ui.warn(b"%s: %s\n" % (uipathfn(f), msg))
 
     if badfn is None:
         badfn = bad
 
     m = ctx.match(
         pats,
         opts.get(b'include'),
         opts.get(b'exclude'),
         default,
         listsubrepos=opts.get(b'subrepos'),
         badfn=badfn,
     )
 
     if m.always():
         pats = []
     return m, pats
 
 
 def match(
     ctx, pats=(), opts=None, globbed=False, default=b'relpath', badfn=None
 ):
     '''Return a matcher that will warn about bad matches.'''
     return matchandpats(ctx, pats, opts, globbed, default, badfn=badfn)[0]
 
 
 def matchall(repo):
     '''Return a matcher that will efficiently match everything.'''
     return matchmod.always()
 
 
 def matchfiles(repo, files, badfn=None):
     '''Return a matcher that will efficiently match exactly these files.'''
     return matchmod.exact(files, badfn=badfn)
 
 
 def parsefollowlinespattern(repo, rev, pat, msg):
     """Return a file name from `pat` pattern suitable for usage in followlines
     logic.
     """
     if not matchmod.patkind(pat):
         return pathutil.canonpath(repo.root, repo.getcwd(), pat)
     else:
         ctx = repo[rev]
         m = matchmod.match(repo.root, repo.getcwd(), [pat], ctx=ctx)
         files = [f for f in ctx if m(f)]
         if len(files) != 1:
             raise error.ParseError(msg)
         return files[0]
 
 
 def getorigvfs(ui, repo):
     """return a vfs suitable to save 'orig' file
 
     return None if no special directory is configured"""
     origbackuppath = ui.config(b'ui', b'origbackuppath')
     if not origbackuppath:
         return None
     return vfs.vfs(repo.wvfs.join(origbackuppath))
 
 
 def backuppath(ui, repo, filepath):
     """customize where working copy backup files (.orig files) are created
 
     Fetch user defined path from config file: [ui] origbackuppath = <path>
     Fall back to default (filepath with .orig suffix) if not specified
 
     filepath is repo-relative
 
     Returns an absolute path
     """
     origvfs = getorigvfs(ui, repo)
     if origvfs is None:
         return repo.wjoin(filepath + b".orig")
 
     origbackupdir = origvfs.dirname(filepath)
     if not origvfs.isdir(origbackupdir) or origvfs.islink(origbackupdir):
         ui.note(_(b'creating directory: %s\n') % origvfs.join(origbackupdir))
 
         # Remove any files that conflict with the backup file's path
         for f in reversed(list(pathutil.finddirs(filepath))):
             if origvfs.isfileorlink(f):
                 ui.note(_(b'removing conflicting file: %s\n') % origvfs.join(f))
                 origvfs.unlink(f)
                 break
 
         origvfs.makedirs(origbackupdir)
 
     if origvfs.isdir(filepath) and not origvfs.islink(filepath):
         ui.note(
             _(b'removing conflicting directory: %s\n') % origvfs.join(filepath)
         )
         origvfs.rmtree(filepath, forcibly=True)
 
     return origvfs.join(filepath)
 
 
 class _containsnode(object):
     """proxy __contains__(node) to container.__contains__ which accepts revs"""
 
     def __init__(self, repo, revcontainer):
         self._torev = repo.changelog.rev
         self._revcontains = revcontainer.__contains__
 
     def __contains__(self, node):
         return self._revcontains(self._torev(node))
 
 
 def cleanupnodes(
     repo,
     replacements,
     operation,
     moves=None,
     metadata=None,
     fixphase=False,
     targetphase=None,
     backup=True,
 ):
     """do common cleanups when old nodes are replaced by new nodes
 
     That includes writing obsmarkers or stripping nodes, and moving bookmarks.
     (we might also want to move working directory parent in the future)
 
     By default, bookmark moves are calculated automatically from 'replacements',
     but 'moves' can be used to override that. Also, 'moves' may include
     additional bookmark moves that should not have associated obsmarkers.
 
     replacements is {oldnode: [newnode]} or a iterable of nodes if they do not
     have replacements. operation is a string, like "rebase".
 
     metadata is dictionary containing metadata to be stored in obsmarker if
     obsolescence is enabled.
     """
     assert fixphase or targetphase is None
     if not replacements and not moves:
         return
 
     # translate mapping's other forms
     if not util.safehasattr(replacements, b'items'):
         replacements = {(n,): () for n in replacements}
     else:
         # upgrading non tuple "source" to tuple ones for BC
         repls = {}
         for key, value in replacements.items():
             if not isinstance(key, tuple):
                 key = (key,)
             repls[key] = value
         replacements = repls
 
     # Unfiltered repo is needed since nodes in replacements might be hidden.
     unfi = repo.unfiltered()
 
     # Calculate bookmark movements
     if moves is None:
         moves = {}
         for oldnodes, newnodes in replacements.items():
             for oldnode in oldnodes:
                 if oldnode in moves:
                     continue
                 if len(newnodes) > 1:
                     # usually a split, take the one with biggest rev number
                     newnode = next(unfi.set(b'max(%ln)', newnodes)).node()
                 elif len(newnodes) == 0:
                     # move bookmark backwards
                     allreplaced = []
                     for rep in replacements:
                         allreplaced.extend(rep)
                     roots = list(
                         unfi.set(b'max((::%n) - %ln)', oldnode, allreplaced)
                     )
                     if roots:
                         newnode = roots[0].node()
                     else:
                         newnode = repo.nullid
                 else:
                     newnode = newnodes[0]
                 moves[oldnode] = newnode
 
     allnewnodes = [n for ns in replacements.values() for n in ns]
     toretract = {}
     toadvance = {}
     if fixphase:
         precursors = {}
         for oldnodes, newnodes in replacements.items():
             for oldnode in oldnodes:
                 for newnode in newnodes:
                     precursors.setdefault(newnode, []).append(oldnode)
 
         allnewnodes.sort(key=lambda n: unfi[n].rev())
         newphases = {}
 
         def phase(ctx):
             return newphases.get(ctx.node(), ctx.phase())
 
         for newnode in allnewnodes:
             ctx = unfi[newnode]
             parentphase = max(phase(p) for p in ctx.parents())
             if targetphase is None:
                 oldphase = max(
                     unfi[oldnode].phase() for oldnode in precursors[newnode]
                 )
                 newphase = max(oldphase, parentphase)
             else:
                 newphase = max(targetphase, parentphase)
             newphases[newnode] = newphase
             if newphase > ctx.phase():
                 toretract.setdefault(newphase, []).append(newnode)
             elif newphase < ctx.phase():
                 toadvance.setdefault(newphase, []).append(newnode)
 
     with repo.transaction(b'cleanup') as tr:
         # Move bookmarks
         bmarks = repo._bookmarks
         bmarkchanges = []
         for oldnode, newnode in moves.items():
             oldbmarks = repo.nodebookmarks(oldnode)
             if not oldbmarks:
                 continue
             from . import bookmarks  # avoid import cycle
 
             repo.ui.debug(
                 b'moving bookmarks %r from %s to %s\n'
                 % (
                     pycompat.rapply(pycompat.maybebytestr, oldbmarks),
                     hex(oldnode),
                     hex(newnode),
                 )
             )
             # Delete divergent bookmarks being parents of related newnodes
             deleterevs = repo.revs(
                 b'parents(roots(%ln & (::%n))) - parents(%n)',
                 allnewnodes,
                 newnode,
                 oldnode,
             )
             deletenodes = _containsnode(repo, deleterevs)
             for name in oldbmarks:
                 bmarkchanges.append((name, newnode))
                 for b in bookmarks.divergent2delete(repo, deletenodes, name):
                     bmarkchanges.append((b, None))
 
         if bmarkchanges:
             bmarks.applychanges(repo, tr, bmarkchanges)
 
         for phase, nodes in toretract.items():
             phases.retractboundary(repo, tr, phase, nodes)
         for phase, nodes in toadvance.items():
             phases.advanceboundary(repo, tr, phase, nodes)
 
         mayusearchived = repo.ui.config(b'experimental', b'cleanup-as-archived')
         # Obsolete or strip nodes
         if obsolete.isenabled(repo, obsolete.createmarkersopt):
             # If a node is already obsoleted, and we want to obsolete it
             # without a successor, skip that obssolete request since it's
             # unnecessary. That's the "if s or not isobs(n)" check below.
             # Also sort the node in topology order, that might be useful for
             # some obsstore logic.
             # NOTE: the sorting might belong to createmarkers.
             torev = unfi.changelog.rev
             sortfunc = lambda ns: torev(ns[0][0])
             rels = []
             for ns, s in sorted(replacements.items(), key=sortfunc):
                 rel = (tuple(unfi[n] for n in ns), tuple(unfi[m] for m in s))
                 rels.append(rel)
             if rels:
                 obsolete.createmarkers(
                     repo, rels, operation=operation, metadata=metadata
                 )
         elif phases.supportinternal(repo) and mayusearchived:
             # this assume we do not have "unstable" nodes above the cleaned ones
             allreplaced = set()
             for ns in replacements.keys():
                 allreplaced.update(ns)
             if backup:
                 from . import repair  # avoid import cycle
 
                 node = min(allreplaced, key=repo.changelog.rev)
                 repair.backupbundle(
                     repo, allreplaced, allreplaced, node, operation
                 )
             phases.retractboundary(repo, tr, phases.archived, allreplaced)
         else:
             from . import repair  # avoid import cycle
 
             tostrip = list(n for ns in replacements for n in ns)
             if tostrip:
                 repair.delayedstrip(
                     repo.ui, repo, tostrip, operation, backup=backup
                 )
 
 
 def addremove(repo, matcher, prefix, uipathfn, opts=None):
     if opts is None:
         opts = {}
     m = matcher
     dry_run = opts.get(b'dry_run')
     try:
         similarity = float(opts.get(b'similarity') or 0)
     except ValueError:
         raise error.InputError(_(b'similarity must be a number'))
     if similarity < 0 or similarity > 100:
         raise error.InputError(_(b'similarity must be between 0 and 100'))
     similarity /= 100.0
 
     ret = 0
 
     wctx = repo[None]
     for subpath in sorted(wctx.substate):
         submatch = matchmod.subdirmatcher(subpath, m)
         if opts.get(b'subrepos') or m.exact(subpath) or any(submatch.files()):
             sub = wctx.sub(subpath)
             subprefix = repo.wvfs.reljoin(prefix, subpath)
             subuipathfn = subdiruipathfn(subpath, uipathfn)
             try:
                 if sub.addremove(submatch, subprefix, subuipathfn, opts):
                     ret = 1
             except error.LookupError:
                 repo.ui.status(
                     _(b"skipping missing subrepository: %s\n")
                     % uipathfn(subpath)
                 )
 
     rejected = []
 
     def badfn(f, msg):
         if f in m.files():
             m.bad(f, msg)
         rejected.append(f)
 
     badmatch = matchmod.badmatch(m, badfn)
     added, unknown, deleted, removed, forgotten = _interestingfiles(
         repo, badmatch
     )
 
     unknownset = set(unknown + forgotten)
     toprint = unknownset.copy()
     toprint.update(deleted)
     for abs in sorted(toprint):
         if repo.ui.verbose or not m.exact(abs):
             if abs in unknownset:
                 status = _(b'adding %s\n') % uipathfn(abs)
                 label = b'ui.addremove.added'
             else:
                 status = _(b'removing %s\n') % uipathfn(abs)
                 label = b'ui.addremove.removed'
             repo.ui.status(status, label=label)
 
     renames = _findrenames(
         repo, m, added + unknown, removed + deleted, similarity, uipathfn
     )
 
     if not dry_run:
         _markchanges(repo, unknown + forgotten, deleted, renames)
 
     for f in rejected:
         if f in m.files():
             return 1
     return ret
 
 
 def marktouched(repo, files, similarity=0.0):
     """Assert that files have somehow been operated upon. files are relative to
     the repo root."""
     m = matchfiles(repo, files, badfn=lambda x, y: rejected.append(x))
     rejected = []
 
     added, unknown, deleted, removed, forgotten = _interestingfiles(repo, m)
 
     if repo.ui.verbose:
         unknownset = set(unknown + forgotten)
         toprint = unknownset.copy()
         toprint.update(deleted)
         for abs in sorted(toprint):
             if abs in unknownset:
                 status = _(b'adding %s\n') % abs
             else:
                 status = _(b'removing %s\n') % abs
             repo.ui.status(status)
 
     # TODO: We should probably have the caller pass in uipathfn and apply it to
     # the messages above too. legacyrelativevalue=True is consistent with how
     # it used to work.
     uipathfn = getuipathfn(repo, legacyrelativevalue=True)
     renames = _findrenames(
         repo, m, added + unknown, removed + deleted, similarity, uipathfn
     )
 
     _markchanges(repo, unknown + forgotten, deleted, renames)
 
     for f in rejected:
         if f in m.files():
             return 1
     return 0
 
 
 def _interestingfiles(repo, matcher):
     """Walk dirstate with matcher, looking for files that addremove would care
     about.
 
     This is different from dirstate.status because it doesn't care about
     whether files are modified or clean."""
     added, unknown, deleted, removed, forgotten = [], [], [], [], []
     audit_path = pathutil.pathauditor(repo.root, cached=True)
 
     ctx = repo[None]
     dirstate = repo.dirstate
     matcher = repo.narrowmatch(matcher, includeexact=True)
     walkresults = dirstate.walk(
         matcher,
         subrepos=sorted(ctx.substate),
         unknown=True,
         ignored=False,
         full=False,
     )
     for abs, st in walkresults.items():
         entry = dirstate.get_entry(abs)
         if (not entry.any_tracked) and audit_path.check(abs):
             unknown.append(abs)
         elif (not entry.removed) and not st:
             deleted.append(abs)
         elif entry.removed and st:
             forgotten.append(abs)
         # for finding renames
         elif entry.removed and not st:
             removed.append(abs)
         elif entry.added:
             added.append(abs)
 
     return added, unknown, deleted, removed, forgotten
 
 
 def _findrenames(repo, matcher, added, removed, similarity, uipathfn):
     '''Find renames from removed files to added ones.'''
     renames = {}
     if similarity > 0:
         for old, new, score in similar.findrenames(
             repo, added, removed, similarity
         ):
             if (
                 repo.ui.verbose
                 or not matcher.exact(old)
                 or not matcher.exact(new)
             ):
                 repo.ui.status(
                     _(
                         b'recording removal of %s as rename to %s '
                         b'(%d%% similar)\n'
                     )
                     % (uipathfn(old), uipathfn(new), score * 100)
                 )
             renames[new] = old
     return renames
 
 
 def _markchanges(repo, unknown, deleted, renames):
     """Marks the files in unknown as added, the files in deleted as removed,
     and the files in renames as copied."""
     wctx = repo[None]
     with repo.wlock():
         wctx.forget(deleted)
         wctx.add(unknown)
         for new, old in renames.items():
             wctx.copy(old, new)
 
 
 def getrenamedfn(repo, endrev=None):
     if copiesmod.usechangesetcentricalgo(repo):
 
         def getrenamed(fn, rev):
             ctx = repo[rev]
             p1copies = ctx.p1copies()
             if fn in p1copies:
                 return p1copies[fn]
             p2copies = ctx.p2copies()
             if fn in p2copies:
                 return p2copies[fn]
             return None
 
         return getrenamed
 
     rcache = {}
     if endrev is None:
         endrev = len(repo)
 
     def getrenamed(fn, rev):
         """looks up all renames for a file (up to endrev) the first
         time the file is given. It indexes on the changerev and only
         parses the manifest if linkrev != changerev.
         Returns rename info for fn at changerev rev."""
         if fn not in rcache:
             rcache[fn] = {}
             fl = repo.file(fn)
             for i in fl:
                 lr = fl.linkrev(i)
                 renamed = fl.renamed(fl.node(i))
                 rcache[fn][lr] = renamed and renamed[0]
                 if lr >= endrev:
                     break
         if rev in rcache[fn]:
             return rcache[fn][rev]
 
         # If linkrev != rev (i.e. rev not found in rcache) fallback to
         # filectx logic.
         try:
             return repo[rev][fn].copysource()
         except error.LookupError:
             return None
 
     return getrenamed
 
 
 def getcopiesfn(repo, endrev=None):
     if copiesmod.usechangesetcentricalgo(repo):
 
         def copiesfn(ctx):
             if ctx.p2copies():
                 allcopies = ctx.p1copies().copy()
                 # There should be no overlap
                 allcopies.update(ctx.p2copies())
                 return sorted(allcopies.items())
             else:
                 return sorted(ctx.p1copies().items())
 
     else:
         getrenamed = getrenamedfn(repo, endrev)
 
         def copiesfn(ctx):
             copies = []
             for fn in ctx.files():
                 rename = getrenamed(fn, ctx.rev())
                 if rename:
                     copies.append((fn, rename))
             return copies
 
     return copiesfn
 
 
 def dirstatecopy(ui, repo, wctx, src, dst, dryrun=False, cwd=None):
     """Update the dirstate to reflect the intent of copying src to dst. For
     different reasons it might not end with dst being marked as copied from src.
     """
     origsrc = repo.dirstate.copied(src) or src
     if dst == origsrc:  # copying back a copy?
         entry = repo.dirstate.get_entry(dst)
         if (entry.added or not entry.tracked) and not dryrun:
             repo.dirstate.set_tracked(dst)
     else:
         if repo.dirstate.get_entry(origsrc).added and origsrc == src:
             if not ui.quiet:
                 ui.warn(
                     _(
                         b"%s has not been committed yet, so no copy "
                         b"data will be stored for %s.\n"
                     )
                     % (repo.pathto(origsrc, cwd), repo.pathto(dst, cwd))
                 )
             if not repo.dirstate.get_entry(dst).tracked and not dryrun:
                 wctx.add([dst])
         elif not dryrun:
             wctx.copy(origsrc, dst)
 
 
 def movedirstate(repo, newctx, match=None):
     """Move the dirstate to newctx and adjust it as necessary.
 
     A matcher can be provided as an optimization. It is probably a bug to pass
     a matcher that doesn't match all the differences between the parent of the
     working copy and newctx.
     """
     oldctx = repo[b'.']
     ds = repo.dirstate
     copies = dict(ds.copies())
     ds.setparents(newctx.node(), repo.nullid)
     s = newctx.status(oldctx, match=match)
 
     for f in s.modified:
         ds.update_file_p1(f, p1_tracked=True)
 
     for f in s.added:
         ds.update_file_p1(f, p1_tracked=False)
 
     for f in s.removed:
         ds.update_file_p1(f, p1_tracked=True)
 
     # Merge old parent and old working dir copies
     oldcopies = copiesmod.pathcopies(newctx, oldctx, match)
     oldcopies.update(copies)
     copies = {dst: oldcopies.get(src, src) for dst, src in oldcopies.items()}
     # Adjust the dirstate copies
     for dst, src in copies.items():
         if src not in newctx or dst in newctx or not ds.get_entry(dst).added:
             src = None
         ds.copy(src, dst)
     repo._quick_access_changeid_invalidate()
 
 
 def filterrequirements(requirements):
     """filters the requirements into two sets:
 
     wcreq: requirements which should be written in .hg/requires
     storereq: which should be written in .hg/store/requires
 
     Returns (wcreq, storereq)
     """
     if requirementsmod.SHARESAFE_REQUIREMENT in requirements:
         wc, store = set(), set()
         for r in requirements:
             if r in requirementsmod.WORKING_DIR_REQUIREMENTS:
                 wc.add(r)
             else:
                 store.add(r)
         return wc, store
     return requirements, None
 
 
 def istreemanifest(repo):
     """returns whether the repository is using treemanifest or not"""
     return requirementsmod.TREEMANIFEST_REQUIREMENT in repo.requirements
 
 
 def writereporequirements(repo, requirements=None):
     """writes requirements for the repo
 
     Requirements are written to .hg/requires and .hg/store/requires based
     on whether share-safe mode is enabled and which requirements are wdir
     requirements and which are store requirements
     """
     if requirements:
         repo.requirements = requirements
     wcreq, storereq = filterrequirements(repo.requirements)
     if wcreq is not None:
         writerequires(repo.vfs, wcreq)
     if storereq is not None:
         writerequires(repo.svfs, storereq)
     elif repo.ui.configbool(b'format', b'usestore'):
         # only remove store requires if we are using store
         repo.svfs.tryunlink(b'requires')
 
 
 def writerequires(opener, requirements):
     with opener(b'requires', b'w', atomictemp=True) as fp:
         for r in sorted(requirements):
             fp.write(b"%s\n" % r)
 
 
 class filecachesubentry(object):
     def __init__(self, path, stat):
         self.path = path
         self.cachestat = None
         self._cacheable = None
 
         if stat:
             self.cachestat = filecachesubentry.stat(self.path)
 
             if self.cachestat:
                 self._cacheable = self.cachestat.cacheable()
             else:
                 # None means we don't know yet
                 self._cacheable = None
 
     def refresh(self):
         if self.cacheable():
             self.cachestat = filecachesubentry.stat(self.path)
 
     def cacheable(self):
         if self._cacheable is not None:
             return self._cacheable
 
         # we don't know yet, assume it is for now
         return True
 
     def changed(self):
         # no point in going further if we can't cache it
         if not self.cacheable():
             return True
 
         newstat = filecachesubentry.stat(self.path)
 
         # we may not know if it's cacheable yet, check again now
         if newstat and self._cacheable is None:
             self._cacheable = newstat.cacheable()
 
             # check again
             if not self._cacheable:
                 return True
 
         if self.cachestat != newstat:
             self.cachestat = newstat
             return True
         else:
             return False
 
     @staticmethod
     def stat(path):
         try:
             return util.cachestat(path)
         except OSError as e:
             if e.errno != errno.ENOENT:
                 raise
 
 
 class filecacheentry(object):
     def __init__(self, paths, stat=True):
         self._entries = []
         for path in paths:
             self._entries.append(filecachesubentry(path, stat))
 
     def changed(self):
         '''true if any entry has changed'''
         for entry in self._entries:
             if entry.changed():
                 return True
         return False
 
     def refresh(self):
         for entry in self._entries:
             entry.refresh()
 
 
 class filecache(object):
     """A property like decorator that tracks files under .hg/ for updates.
 
     On first access, the files defined as arguments are stat()ed and the
     results cached. The decorated function is called. The results are stashed
     away in a ``_filecache`` dict on the object whose method is decorated.
 
     On subsequent access, the cached result is used as it is set to the
     instance dictionary.
 
     On external property set/delete operations, the caller must update the
     corresponding _filecache entry appropriately. Use __class__.<attr>.set()
     instead of directly setting <attr>.
 
     When using the property API, the cached data is always used if available.
     No stat() is performed to check if the file has changed.
 
     Others can muck about with the state of the ``_filecache`` dict. e.g. they
     can populate an entry before the property's getter is called. In this case,
     entries in ``_filecache`` will be used during property operations,
     if available. If the underlying file changes, it is up to external callers
     to reflect this by e.g. calling ``delattr(obj, attr)`` to remove the cached
     method result as well as possibly calling ``del obj._filecache[attr]`` to
     remove the ``filecacheentry``.
     """
 
     def __init__(self, *paths):
         self.paths = paths
 
     def tracked_paths(self, obj):
         return [self.join(obj, path) for path in self.paths]
 
     def join(self, obj, fname):
         """Used to compute the runtime path of a cached file.
 
         Users should subclass filecache and provide their own version of this
         function to call the appropriate join function on 'obj' (an instance
         of the class that its member function was decorated).
         """
         raise NotImplementedError
 
     def __call__(self, func):
         self.func = func
         self.sname = func.__name__
         self.name = pycompat.sysbytes(self.sname)
         return self
 
     def __get__(self, obj, type=None):
         # if accessed on the class, return the descriptor itself.
         if obj is None:
             return self
 
         assert self.sname not in obj.__dict__
 
         entry = obj._filecache.get(self.name)
 
         if entry:
             if entry.changed():
                 entry.obj = self.func(obj)
         else:
             paths = self.tracked_paths(obj)
 
             # We stat -before- creating the object so our cache doesn't lie if
             # a writer modified between the time we read and stat
             entry = filecacheentry(paths, True)
             entry.obj = self.func(obj)
 
             obj._filecache[self.name] = entry
 
         obj.__dict__[self.sname] = entry.obj
         return entry.obj
 
     # don't implement __set__(), which would make __dict__ lookup as slow as
     # function call.
 
     def set(self, obj, value):
         if self.name not in obj._filecache:
             # we add an entry for the missing value because X in __dict__
             # implies X in _filecache
             paths = self.tracked_paths(obj)
             ce = filecacheentry(paths, False)
             obj._filecache[self.name] = ce
         else:
             ce = obj._filecache[self.name]
 
         ce.obj = value  # update cached copy
         obj.__dict__[self.sname] = value  # update copy returned by obj.x
 
 
 def extdatasource(repo, source):
     """Gather a map of rev -> value dict from the specified source
 
     A source spec is treated as a URL, with a special case shell: type
     for parsing the output from a shell command.
 
     The data is parsed as a series of newline-separated records where
     each record is a revision specifier optionally followed by a space
     and a freeform string value. If the revision is known locally, it
     is converted to a rev, otherwise the record is skipped.
 
     Note that both key and value are treated as UTF-8 and converted to
     the local encoding. This allows uniformity between local and
     remote data sources.
     """
 
     spec = repo.ui.config(b"extdata", source)
     if not spec:
         raise error.Abort(_(b"unknown extdata source '%s'") % source)
 
     data = {}
     src = proc = None
     try:
         if spec.startswith(b"shell:"):
             # external commands should be run relative to the repo root
             cmd = spec[6:]
             proc = subprocess.Popen(
                 procutil.tonativestr(cmd),
                 shell=True,
                 bufsize=-1,
                 close_fds=procutil.closefds,
                 stdout=subprocess.PIPE,
                 cwd=procutil.tonativestr(repo.root),
             )
             src = proc.stdout
         else:
             # treat as a URL or file
             src = url.open(repo.ui, spec)
         for l in src:
             if b" " in l:
                 k, v = l.strip().split(b" ", 1)
             else:
                 k, v = l.strip(), b""
 
             k = encoding.tolocal(k)
             try:
                 data[revsingle(repo, k).rev()] = encoding.tolocal(v)
             except (error.LookupError, error.RepoLookupError, error.InputError):
                 pass  # we ignore data for nodes that don't exist locally
     finally:
         if proc:
             try:
                 proc.communicate()
             except ValueError:
                 # This happens if we started iterating src and then
                 # get a parse error on a line. It should be safe to ignore.
                 pass
         if src:
             src.close()
     if proc and proc.returncode != 0:
         raise error.Abort(
             _(b"extdata command '%s' failed: %s")
             % (cmd, procutil.explainexit(proc.returncode))
         )
 
     return data
 
 
 class progress(object):
     def __init__(self, ui, updatebar, topic, unit=b"", total=None):
         self.ui = ui
         self.pos = 0
         self.topic = topic
         self.unit = unit
         self.total = total
         self.debug = ui.configbool(b'progress', b'debug')
         self._updatebar = updatebar
 
     def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_value, exc_tb):
         self.complete()
 
     def update(self, pos, item=b"", total=None):
         assert pos is not None
         if total:
             self.total = total
         self.pos = pos
         self._updatebar(self.topic, self.pos, item, self.unit, self.total)
         if self.debug:
             self._printdebug(item)
 
     def increment(self, step=1, item=b"", total=None):
         self.update(self.pos + step, item, total)
 
     def complete(self):
         self.pos = None
         self.unit = b""
         self.total = None
         self._updatebar(self.topic, self.pos, b"", self.unit, self.total)
 
     def _printdebug(self, item):
         unit = b''
         if self.unit:
             unit = b' ' + self.unit
         if item:
             item = b' ' + item
 
         if self.total:
             pct = 100.0 * self.pos / self.total
             self.ui.debug(
                 b'%s:%s %d/%d%s (%4.2f%%)\n'
                 % (self.topic, item, self.pos, self.total, unit, pct)
             )
         else:
             self.ui.debug(b'%s:%s %d%s\n' % (self.topic, item, self.pos, unit))
 
 
 def gdinitconfig(ui):
     """helper function to know if a repo should be created as general delta"""
     # experimental config: format.generaldelta
     return ui.configbool(b'format', b'generaldelta') or ui.configbool(
         b'format', b'usegeneraldelta'
     )
 
 
 def gddeltaconfig(ui):
     """helper function to know if incoming delta should be optimised"""
     # experimental config: format.generaldelta
     return ui.configbool(b'format', b'generaldelta')
 
 
 class simplekeyvaluefile(object):
     """A simple file with key=value lines
 
     Keys must be alphanumerics and start with a letter, values must not
     contain '\n' characters"""
 
     firstlinekey = b'__firstline'
 
     def __init__(self, vfs, path, keys=None):
         self.vfs = vfs
         self.path = path
 
     def read(self, firstlinenonkeyval=False):
         """Read the contents of a simple key-value file
 
         'firstlinenonkeyval' indicates whether the first line of file should
         be treated as a key-value pair or reuturned fully under the
         __firstline key."""
         lines = self.vfs.readlines(self.path)
         d = {}
         if firstlinenonkeyval:
             if not lines:
                 e = _(b"empty simplekeyvalue file")
                 raise error.CorruptedState(e)
             # we don't want to include '\n' in the __firstline
             d[self.firstlinekey] = lines[0][:-1]
             del lines[0]
 
         try:
             # the 'if line.strip()' part prevents us from failing on empty
             # lines which only contain '\n' therefore are not skipped
             # by 'if line'
             updatedict = dict(
                 line[:-1].split(b'=', 1) for line in lines if line.strip()
             )
             if self.firstlinekey in updatedict:
                 e = _(b"%r can't be used as a key")
                 raise error.CorruptedState(e % self.firstlinekey)
             d.update(updatedict)
         except ValueError as e:
             raise error.CorruptedState(stringutil.forcebytestr(e))
         return d
 
     def write(self, data, firstline=None):
         """Write key=>value mapping to a file
         data is a dict. Keys must be alphanumerical and start with a letter.
         Values must not contain newline characters.
 
         If 'firstline' is not None, it is written to file before
         everything else, as it is, not in a key=value form"""
         lines = []
         if firstline is not None:
             lines.append(b'%s\n' % firstline)
 
         for k, v in data.items():
             if k == self.firstlinekey:
                 e = b"key name '%s' is reserved" % self.firstlinekey
                 raise error.ProgrammingError(e)
             if not k[0:1].isalpha():
                 e = b"keys must start with a letter in a key-value file"
                 raise error.ProgrammingError(e)
             if not k.isalnum():
                 e = b"invalid key name in a simple key-value file"
                 raise error.ProgrammingError(e)
             if b'\n' in v:
                 e = b"invalid value in a simple key-value file"
                 raise error.ProgrammingError(e)
             lines.append(b"%s=%s\n" % (k, v))
         with self.vfs(self.path, mode=b'wb', atomictemp=True) as fp:
             fp.write(b''.join(lines))
 
 
 _reportobsoletedsource = [
     b'debugobsolete',
     b'pull',
     b'push',
     b'serve',
     b'unbundle',
 ]
 
 _reportnewcssource = [
     b'pull',
     b'unbundle',
 ]
 
 
 def prefetchfiles(repo, revmatches):
     """Invokes the registered file prefetch functions, allowing extensions to
     ensure the corresponding files are available locally, before the command
     uses them.
 
     Args:
       revmatches: a list of (revision, match) tuples to indicate the files to
       fetch at each revision. If any of the match elements is None, it matches
       all files.
     """
 
     def _matcher(m):
         if m:
             assert isinstance(m, matchmod.basematcher)
             # The command itself will complain about files that don't exist, so
             # don't duplicate the message.
             return matchmod.badmatch(m, lambda fn, msg: None)
         else:
             return matchall(repo)
 
     revbadmatches = [(rev, _matcher(match)) for (rev, match) in revmatches]
 
     fileprefetchhooks(repo, revbadmatches)
 
 
 # a list of (repo, revs, match) prefetch functions
 fileprefetchhooks = util.hooks()
 
 # A marker that tells the evolve extension to suppress its own reporting
 _reportstroubledchangesets = True
 
 
 def registersummarycallback(repo, otr, txnname=b'', as_validator=False):
     """register a callback to issue a summary after the transaction is closed
 
     If as_validator is true, then the callbacks are registered as transaction
     validators instead
     """
 
     def txmatch(sources):
         return any(txnname.startswith(source) for source in sources)
 
     categories = []
 
     def reportsummary(func):
         """decorator for report callbacks."""
         # The repoview life cycle is shorter than the one of the actual
         # underlying repository. So the filtered object can die before the
         # weakref is used leading to troubles. We keep a reference to the
         # unfiltered object and restore the filtering when retrieving the
         # repository through the weakref.
         filtername = repo.filtername
         reporef = weakref.ref(repo.unfiltered())
 
         def wrapped(tr):
             repo = reporef()
             if filtername:
                 assert repo is not None  # help pytype
                 repo = repo.filtered(filtername)
             func(repo, tr)
 
         newcat = b'%02i-txnreport' % len(categories)
         if as_validator:
             otr.addvalidator(newcat, wrapped)
         else:
             otr.addpostclose(newcat, wrapped)
         categories.append(newcat)
         return wrapped
 
     @reportsummary
     def reportchangegroup(repo, tr):
         cgchangesets = tr.changes.get(b'changegroup-count-changesets', 0)
         cgrevisions = tr.changes.get(b'changegroup-count-revisions', 0)
         cgfiles = tr.changes.get(b'changegroup-count-files', 0)
         cgheads = tr.changes.get(b'changegroup-count-heads', 0)
         if cgchangesets or cgrevisions or cgfiles:
             htext = b""
             if cgheads:
                 htext = _(b" (%+d heads)") % cgheads
             msg = _(b"added %d changesets with %d changes to %d files%s\n")
             if as_validator:
                 msg = _(b"adding %d changesets with %d changes to %d files%s\n")
             assert repo is not None  # help pytype
             repo.ui.status(msg % (cgchangesets, cgrevisions, cgfiles, htext))
 
     if txmatch(_reportobsoletedsource):
 
         @reportsummary
         def reportobsoleted(repo, tr):
             obsoleted = obsutil.getobsoleted(repo, tr)
             newmarkers = len(tr.changes.get(b'obsmarkers', ()))
             if newmarkers:
                 repo.ui.status(_(b'%i new obsolescence markers\n') % newmarkers)
             if obsoleted:
                 msg = _(b'obsoleted %i changesets\n')
                 if as_validator:
                     msg = _(b'obsoleting %i changesets\n')
                 repo.ui.status(msg % len(obsoleted))
 
     if obsolete.isenabled(
         repo, obsolete.createmarkersopt
     ) and repo.ui.configbool(
         b'experimental', b'evolution.report-instabilities'
     ):
         instabilitytypes = [
             (b'orphan', b'orphan'),
             (b'phase-divergent', b'phasedivergent'),
             (b'content-divergent', b'contentdivergent'),
         ]
 
         def getinstabilitycounts(repo):
             filtered = repo.changelog.filteredrevs
             counts = {}
             for instability, revset in instabilitytypes:
                 counts[instability] = len(
                     set(obsolete.getrevs(repo, revset)) - filtered
                 )
             return counts
 
         oldinstabilitycounts = getinstabilitycounts(repo)
 
         @reportsummary
         def reportnewinstabilities(repo, tr):
             newinstabilitycounts = getinstabilitycounts(repo)
             for instability, revset in instabilitytypes:
                 delta = (
                     newinstabilitycounts[instability]
                     - oldinstabilitycounts[instability]
                 )
                 msg = getinstabilitymessage(delta, instability)
                 if msg:
                     repo.ui.warn(msg)
 
     if txmatch(_reportnewcssource):
 
         @reportsummary
         def reportnewcs(repo, tr):
             """Report the range of new revisions pulled/unbundled."""
             origrepolen = tr.changes.get(b'origrepolen', len(repo))
             unfi = repo.unfiltered()
             if origrepolen >= len(unfi):
                 return
 
             # Compute the bounds of new visible revisions' range.
             revs = smartset.spanset(repo, start=origrepolen)
             if revs:
                 minrev, maxrev = repo[revs.min()], repo[revs.max()]
 
                 if minrev == maxrev:
                     revrange = minrev
                 else:
                     revrange = b'%s:%s' % (minrev, maxrev)
                 draft = len(repo.revs(b'%ld and draft()', revs))
                 secret = len(repo.revs(b'%ld and secret()', revs))
                 if not (draft or secret):
                     msg = _(b'new changesets %s\n') % revrange
                 elif draft and secret:
                     msg = _(b'new changesets %s (%d drafts, %d secrets)\n')
                     msg %= (revrange, draft, secret)
                 elif draft:
                     msg = _(b'new changesets %s (%d drafts)\n')
                     msg %= (revrange, draft)
                 elif secret:
                     msg = _(b'new changesets %s (%d secrets)\n')
                     msg %= (revrange, secret)
                 else:
                     errormsg = b'entered unreachable condition'
                     raise error.ProgrammingError(errormsg)
                 repo.ui.status(msg)
 
             # search new changesets directly pulled as obsolete
             duplicates = tr.changes.get(b'revduplicates', ())
             obsadded = unfi.revs(
                 b'(%d: + %ld) and obsolete()', origrepolen, duplicates
             )
             cl = repo.changelog
             extinctadded = [r for r in obsadded if r not in cl]
             if extinctadded:
                 # They are not just obsolete, but obsolete and invisible
                 # we call them "extinct" internally but the terms have not been
                 # exposed to users.
                 msg = b'(%d other changesets obsolete on arrival)\n'
                 repo.ui.status(msg % len(extinctadded))
 
         @reportsummary
         def reportphasechanges(repo, tr):
             """Report statistics of phase changes for changesets pre-existing
             pull/unbundle.
             """
             origrepolen = tr.changes.get(b'origrepolen', len(repo))
             published = []
             for revs, (old, new) in tr.changes.get(b'phases', []):
                 if new != phases.public:
                     continue
                 published.extend(rev for rev in revs if rev < origrepolen)
             if not published:
                 return
             msg = _(b'%d local changesets published\n')
             if as_validator:
                 msg = _(b'%d local changesets will be published\n')
             repo.ui.status(msg % len(published))
 
 
 def getinstabilitymessage(delta, instability):
     """function to return the message to show warning about new instabilities
 
     exists as a separate function so that extension can wrap to show more
     information like how to fix instabilities"""
     if delta > 0:
         return _(b'%i new %s changesets\n') % (delta, instability)
 
 
 def nodesummaries(repo, nodes, maxnumnodes=4):
     if len(nodes) <= maxnumnodes or repo.ui.verbose:
         return b' '.join(short(h) for h in nodes)
     first = b' '.join(short(h) for h in nodes[:maxnumnodes])
     return _(b"%s and %d others") % (first, len(nodes) - maxnumnodes)
 
 
 def enforcesinglehead(repo, tr, desc, accountclosed, filtername):
     """check that no named branch has multiple heads"""
     if desc in (b'strip', b'repair'):
         # skip the logic during strip
         return
     visible = repo.filtered(filtername)
     # possible improvement: we could restrict the check to affected branch
     bm = visible.branchmap()
     for name in bm:
         heads = bm.branchheads(name, closed=accountclosed)
         if len(heads) > 1:
             msg = _(b'rejecting multiple heads on branch "%s"')
             msg %= name
             hint = _(b'%d heads: %s')
             hint %= (len(heads), nodesummaries(repo, heads))
             raise error.Abort(msg, hint=hint)
 
 
 def wrapconvertsink(sink):
     """Allow extensions to wrap the sink returned by convcmd.convertsink()
     before it is used, whether or not the convert extension was formally loaded.
     """
     return sink
 
 
 def unhidehashlikerevs(repo, specs, hiddentype):
     """parse the user specs and unhide changesets whose hash or revision number
     is passed.
 
     hiddentype can be: 1) 'warn': warn while unhiding changesets
                        2) 'nowarn': don't warn while unhiding changesets
 
     returns a repo object with the required changesets unhidden
     """
     if not specs:
         return repo
 
     if not repo.filtername or not repo.ui.configbool(
         b'experimental', b'directaccess'
     ):
         return repo
 
     if repo.filtername not in (b'visible', b'visible-hidden'):
         return repo
 
     symbols = set()
     for spec in specs:
         try:
             tree = revsetlang.parse(spec)
         except error.ParseError:  # will be reported by scmutil.revrange()
             continue
 
         symbols.update(revsetlang.gethashlikesymbols(tree))
 
     if not symbols:
         return repo
 
     revs = _getrevsfromsymbols(repo, symbols)
 
     if not revs:
         return repo
 
     if hiddentype == b'warn':
         unfi = repo.unfiltered()
         revstr = b", ".join([pycompat.bytestr(unfi[l]) for l in revs])
         repo.ui.warn(
             _(
                 b"warning: accessing hidden changesets for write "
                 b"operation: %s\n"
             )
             % revstr
         )
 
     # we have to use new filtername to separate branch/tags cache until we can
     # disbale these cache when revisions are dynamically pinned.
     return repo.filtered(b'visible-hidden', revs)
 
 
 def _getrevsfromsymbols(repo, symbols):
     """parse the list of symbols and returns a set of revision numbers of hidden
     changesets present in symbols"""
     revs = set()
     unfi = repo.unfiltered()
     unficl = unfi.changelog
     cl = repo.changelog
     tiprev = len(unficl)
     allowrevnums = repo.ui.configbool(b'experimental', b'directaccess.revnums')
     for s in symbols:
         try:
             n = int(s)
             if n <= tiprev:
                 if not allowrevnums:
                     continue
                 else:
                     if n not in cl:
                         revs.add(n)
                     continue
         except ValueError:
             pass
 
         try:
             s = resolvehexnodeidprefix(unfi, s)
         except (error.LookupError, error.WdirUnsupported):
             s = None
 
         if s is not None:
             rev = unficl.rev(s)
             if rev not in cl:
                 revs.add(rev)
 
     return revs
 
 
 def bookmarkrevs(repo, mark):
     """Select revisions reachable by a given bookmark
 
     If the bookmarked revision isn't a head, an empty set will be returned.
     """
     return repo.revs(format_bookmark_revspec(mark))
 
 
 def format_bookmark_revspec(mark):
     """Build a revset expression to select revisions reachable by a given
     bookmark"""
     mark = b'literal:' + mark
     return revsetlang.formatspec(
         b"ancestors(bookmark(%s)) - "
         b"ancestors(head() and not bookmark(%s)) - "
         b"ancestors(bookmark() and not bookmark(%s))",
         mark,
         mark,
         mark,
     )

mercurial/templatefilters.py

@@ -1,556 +1,554
 # templatefilters.py - common template expansion filters
 #
 # Copyright 2005-2008 Olivia Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 
 import os
 import re
 import time
 
 from .i18n import _
 from .node import hex
 from . import (
     encoding,
     error,
     pycompat,
     registrar,
     smartset,
     templateutil,
     url,
     util,
 )
 from .utils import (
     cborutil,
     dateutil,
     stringutil,
 )
 
 urlerr = util.urlerr
 urlreq = util.urlreq
 
 # filters are callables like:
 #   fn(obj)
 # with:
 #   obj - object to be filtered (text, date, list and so on)
 filters = {}
 
 templatefilter = registrar.templatefilter(filters)
 
 
 @templatefilter(b'addbreaks', intype=bytes)
 def addbreaks(text):
     """Any text. Add an XHTML "<br />" tag before the end of
     every line except the last.
     """
     return text.replace(b'\n', b'<br/>\n')
 
 
 agescales = [
     (b"year", 3600 * 24 * 365, b'Y'),
     (b"month", 3600 * 24 * 30, b'M'),
     (b"week", 3600 * 24 * 7, b'W'),
     (b"day", 3600 * 24, b'd'),
     (b"hour", 3600, b'h'),
     (b"minute", 60, b'm'),
     (b"second", 1, b's'),
 ]
 
 
 @templatefilter(b'age', intype=templateutil.date)
 def age(date, abbrev=False):
     """Date. Returns a human-readable date/time difference between the
     given date/time and the current date/time.
     """
 
     def plural(t, c):
         if c == 1:
             return t
         return t + b"s"
 
     def fmt(t, c, a):
         if abbrev:
             return b"%d%s" % (c, a)
         return b"%d %s" % (c, plural(t, c))
 
     now = time.time()
     then = date[0]
     future = False
     if then > now:
         future = True
         delta = max(1, int(then - now))
         if delta > agescales[0][1] * 30:
             return b'in the distant future'
     else:
         delta = max(1, int(now - then))
         if delta > agescales[0][1] * 2:
             return dateutil.shortdate(date)
 
     for t, s, a in agescales:
         n = delta // s
         if n >= 2 or s == 1:
             if future:
                 return b'%s from now' % fmt(t, n, a)
             return b'%s ago' % fmt(t, n, a)
 
 
 @templatefilter(b'basename', intype=bytes)
 def basename(path):
     """Any text. Treats the text as a path, and returns the last
     component of the path after splitting by the path separator.
     For example, "foo/bar/baz" becomes "baz" and "foo/bar//" becomes "".
     """
     return os.path.basename(path)
 
 
 def _tocborencodable(obj):
     if isinstance(obj, smartset.abstractsmartset):
         return list(obj)
     return obj
 
 
 @templatefilter(b'cbor')
 def cbor(obj):
     """Any object. Serializes the object to CBOR bytes."""
     # cborutil is stricter about type than json() filter
     obj = pycompat.rapply(_tocborencodable, obj)
     return b''.join(cborutil.streamencode(obj))
 
 
 @templatefilter(b'commondir')
 def commondir(filelist):
     """List of text. Treats each list item as file name with /
     as path separator and returns the longest common directory
     prefix shared by all list items.
     Returns the empty string if no common prefix exists.
 
     The list items are not normalized, i.e. "foo/../bar" is handled as
     file "bar" in the directory "foo/..". Leading slashes are ignored.
 
     For example, ["foo/bar/baz", "foo/baz/bar"] becomes "foo" and
     ["foo/bar", "baz"] becomes "".
     """
 
     def common(a, b):
         if len(a) > len(b):
             a = b[: len(a)]
         elif len(b) > len(a):
             b = b[: len(a)]
         if a == b:
             return a
         for i in pycompat.xrange(len(a)):
             if a[i] != b[i]:
                 return a[:i]
         return a
 
     try:
         if not filelist:
             return b""
         dirlist = [f.lstrip(b'/').split(b'/')[:-1] for f in filelist]
         if len(dirlist) == 1:
             return b'/'.join(dirlist[0])
         a = min(dirlist)
         b = max(dirlist)
         # The common prefix of a and b is shared with all
         # elements of the list since Python sorts lexicographical
         # and [1, x] after [1].
         return b'/'.join(common(a, b))
     except TypeError:
         raise error.ParseError(_(b'argument is not a list of text'))
 
 
 @templatefilter(b'count')
 def count(i):
     """List or text. Returns the length as an integer."""
     try:
         return len(i)
     except TypeError:
         raise error.ParseError(_(b'not countable'))
 
 
 @templatefilter(b'dirname', intype=bytes)
 def dirname(path):
     """Any text. Treats the text as a path, and strips the last
     component of the path after splitting by the path separator.
     """
     return os.path.dirname(path)
 
 
 @templatefilter(b'domain', intype=bytes)
 def domain(author):
     """Any text. Finds the first string that looks like an email
     address, and extracts just the domain component. Example: ``User
     <[e-mail]>`` becomes ``example.com``.
     """
     f = author.find(b'@')
     if f == -1:
         return b''
     author = author[f + 1 :]
     f = author.find(b'>')
     if f >= 0:
         author = author[:f]
     return author
 
 
 @templatefilter(b'email', intype=bytes)
 def email(text):
     """Any text. Extracts the first string that looks like an email
     address. Example: ``User <[e-mail]>`` becomes
     ``[e-mail]``.
     """
     return stringutil.email(text)
 
 
 @templatefilter(b'escape', intype=bytes)
 def escape(text):
     """Any text. Replaces the special XML/XHTML characters "&", "<"
     and ">" with XML entities, and filters out NUL characters.
     """
     return url.escape(text.replace(b'\0', b''), True)
 
 
 para_re = None
 space_re = None
 
 
 def fill(text, width, initindent=b'', hangindent=b''):
     '''fill many paragraphs with optional indentation.'''
     global para_re, space_re
     if para_re is None:
         para_re = re.compile(b'(\n\n|\n\\s*[-*]\\s*)', re.M)
         space_re = re.compile(br'  +')
 
     def findparas():
         start = 0
         while True:
             m = para_re.search(text, start)
             if not m:
                 uctext = encoding.unifromlocal(text[start:])
                 w = len(uctext)
                 while w > 0 and uctext[w - 1].isspace():
                     w -= 1
                 yield (
                     encoding.unitolocal(uctext[:w]),
                     encoding.unitolocal(uctext[w:]),
                 )
                 break
             yield text[start : m.start(0)], m.group(1)
             start = m.end(1)
 
     return b"".join(
         [
             stringutil.wrap(
                 space_re.sub(b' ', stringutil.wrap(para, width)),
                 width,
                 initindent,
                 hangindent,
             )
             + rest
             for para, rest in findparas()
         ]
     )
 
 
 @templatefilter(b'fill68', intype=bytes)
 def fill68(text):
     """Any text. Wraps the text to fit in 68 columns."""
     return fill(text, 68)
 
 
 @templatefilter(b'fill76', intype=bytes)
 def fill76(text):
     """Any text. Wraps the text to fit in 76 columns."""
     return fill(text, 76)
 
 
 @templatefilter(b'firstline', intype=bytes)
 def firstline(text):
     """Any text. Returns the first line of text."""
     try:
         return text.splitlines(True)[0].rstrip(b'\r\n')
     except IndexError:
         return b''
 
 
 @templatefilter(b'hex', intype=bytes)
 def hexfilter(text):
     """Any text. Convert a binary Mercurial node identifier into
     its long hexadecimal representation.
     """
     return hex(text)
 
 
 @templatefilter(b'hgdate', intype=templateutil.date)
 def hgdate(text):
     """Date. Returns the date as a pair of numbers: "1157407993
     25200" (Unix timestamp, timezone offset).
     """
     return b"%d %d" % text
 
 
 @templatefilter(b'isodate', intype=templateutil.date)
 def isodate(text):
     """Date. Returns the date in ISO 8601 format: "2009-08-18 13:00
     +0200".
     """
     return dateutil.datestr(text, b'%Y-%m-%d %H:%M %1%2')
 
 
 @templatefilter(b'isodatesec', intype=templateutil.date)
 def isodatesec(text):
     """Date. Returns the date in ISO 8601 format, including
     seconds: "2009-08-18 13:00:13 +0200". See also the rfc3339date
     filter.
     """
     return dateutil.datestr(text, b'%Y-%m-%d %H:%M:%S %1%2')
 
 
 def indent(text, prefix, firstline=b''):
     '''indent each non-empty line of text after first with prefix.'''
     lines = text.splitlines()
     num_lines = len(lines)
     endswithnewline = text[-1:] == b'\n'
 
     def indenter():
         for i in pycompat.xrange(num_lines):
             l = lines[i]
             if l.strip():
                 yield prefix if i else firstline
             yield l
             if i < num_lines - 1 or endswithnewline:
                 yield b'\n'
 
     return b"".join(indenter())
 
 
 @templatefilter(b'json')
 def json(obj, paranoid=True):
     """Any object. Serializes the object to a JSON formatted text."""
     if obj is None:
         return b'null'
     elif obj is False:
         return b'false'
     elif obj is True:
         return b'true'
     elif isinstance(obj, (int, int, float)):
         return pycompat.bytestr(obj)
     elif isinstance(obj, bytes):
         return b'"%s"' % encoding.jsonescape(obj, paranoid=paranoid)
     elif isinstance(obj, type(u'')):
         raise error.ProgrammingError(
             b'Mercurial only does output with bytes: %r' % obj
         )
     elif util.safehasattr(obj, b'keys'):
         out = [
             b'"%s": %s'
             % (encoding.jsonescape(k, paranoid=paranoid), json(v, paranoid))
             for k, v in sorted(obj.items())
         ]
         return b'{' + b', '.join(out) + b'}'
     elif util.safehasattr(obj, b'__iter__'):
         out = [json(i, paranoid) for i in obj]
         return b'[' + b', '.join(out) + b']'
     raise error.ProgrammingError(b'cannot encode %r' % obj)
 
 
 @templatefilter(b'lower', intype=bytes)
 def lower(text):
     """Any text. Converts the text to lowercase."""
     return encoding.lower(text)
 
 
 @templatefilter(b'nonempty', intype=bytes)
 def nonempty(text):
     """Any text. Returns '(none)' if the string is empty."""
     return text or b"(none)"
 
 
 @templatefilter(b'obfuscate', intype=bytes)
 def obfuscate(text):
     """Any text. Returns the input text rendered as a sequence of
     XML entities.
     """
-    text = pycompat.unicode(
-        text, pycompat.sysstr(encoding.encoding), r'replace'
-    )
+    text = str(text, pycompat.sysstr(encoding.encoding), r'replace')
     return b''.join([b'&#%d;' % ord(c) for c in text])
 
 
 @templatefilter(b'permissions', intype=bytes)
 def permissions(flags):
     if b"l" in flags:
         return b"lrwxrwxrwx"
     if b"x" in flags:
         return b"-rwxr-xr-x"
     return b"-rw-r--r--"
 
 
 @templatefilter(b'person', intype=bytes)
 def person(author):
     """Any text. Returns the name before an email address,
     interpreting it as per RFC 5322.
     """
     return stringutil.person(author)
 
 
 @templatefilter(b'revescape', intype=bytes)
 def revescape(text):
     """Any text. Escapes all "special" characters, except @.
     Forward slashes are escaped twice to prevent web servers from prematurely
     unescaping them. For example, "@foo bar/baz" becomes "@foo%20bar%252Fbaz".
     """
     return urlreq.quote(text, safe=b'/@').replace(b'/', b'%252F')
 
 
 @templatefilter(b'rfc3339date', intype=templateutil.date)
 def rfc3339date(text):
     """Date. Returns a date using the Internet date format
     specified in RFC 3339: "2009-08-18T13:00:13+02:00".
     """
     return dateutil.datestr(text, b"%Y-%m-%dT%H:%M:%S%1:%2")
 
 
 @templatefilter(b'rfc822date', intype=templateutil.date)
 def rfc822date(text):
     """Date. Returns a date using the same format used in email
     headers: "Tue, 18 Aug 2009 13:00:13 +0200".
     """
     return dateutil.datestr(text, b"%a, %d %b %Y %H:%M:%S %1%2")
 
 
 @templatefilter(b'short', intype=bytes)
 def short(text):
     """Changeset hash. Returns the short form of a changeset hash,
     i.e. a 12 hexadecimal digit string.
     """
     return text[:12]
 
 
 @templatefilter(b'shortbisect', intype=bytes)
 def shortbisect(label):
     """Any text. Treats `label` as a bisection status, and
     returns a single-character representing the status (G: good, B: bad,
     S: skipped, U: untested, I: ignored). Returns single space if `text`
     is not a valid bisection status.
     """
     if label:
         return label[0:1].upper()
     return b' '
 
 
 @templatefilter(b'shortdate', intype=templateutil.date)
 def shortdate(text):
     """Date. Returns a date like "2006-09-18"."""
     return dateutil.shortdate(text)
 
 
 @templatefilter(b'slashpath', intype=bytes)
 def slashpath(path):
     """Any text. Replaces the native path separator with slash."""
     return util.pconvert(path)
 
 
 @templatefilter(b'splitlines', intype=bytes)
 def splitlines(text):
     """Any text. Split text into a list of lines."""
     return templateutil.hybridlist(text.splitlines(), name=b'line')
 
 
 @templatefilter(b'stringescape', intype=bytes)
 def stringescape(text):
     return stringutil.escapestr(text)
 
 
 @templatefilter(b'stringify', intype=bytes)
 def stringify(thing):
     """Any type. Turns the value into text by converting values into
     text and concatenating them.
     """
     return thing  # coerced by the intype
 
 
 @templatefilter(b'stripdir', intype=bytes)
 def stripdir(text):
     """Treat the text as path and strip a directory level, if
     possible. For example, "foo" and "foo/bar" becomes "foo".
     """
     dir = os.path.dirname(text)
     if dir == b"":
         return os.path.basename(text)
     else:
         return dir
 
 
 @templatefilter(b'tabindent', intype=bytes)
 def tabindent(text):
     """Any text. Returns the text, with every non-empty line
     except the first starting with a tab character.
     """
     return indent(text, b'\t')
 
 
 @templatefilter(b'upper', intype=bytes)
 def upper(text):
     """Any text. Converts the text to uppercase."""
     return encoding.upper(text)
 
 
 @templatefilter(b'urlescape', intype=bytes)
 def urlescape(text):
     """Any text. Escapes all "special" characters. For example,
     "foo bar" becomes "foo%20bar".
     """
     return urlreq.quote(text)
 
 
 @templatefilter(b'user', intype=bytes)
 def userfilter(text):
     """Any text. Returns a short representation of a user name or email
     address."""
     return stringutil.shortuser(text)
 
 
 @templatefilter(b'emailuser', intype=bytes)
 def emailuser(text):
     """Any text. Returns the user portion of an email address."""
     return stringutil.emailuser(text)
 
 
 @templatefilter(b'utf8', intype=bytes)
 def utf8(text):
     """Any text. Converts from the local character encoding to UTF-8."""
     return encoding.fromlocal(text)
 
 
 @templatefilter(b'xmlescape', intype=bytes)
 def xmlescape(text):
     text = (
         text.replace(b'&', b'&amp;')
         .replace(b'<', b'&lt;')
         .replace(b'>', b'&gt;')
         .replace(b'"', b'&quot;')
         .replace(b"'", b'&#39;')
     )  # &apos; invalid in HTML
     return re.sub(b'[\x00-\x08\x0B\x0C\x0E-\x1F]', b' ', text)
 
 
 def websub(text, websubtable):
     """:websub: Any text. Only applies to hgweb. Applies the regular
     expression replacements defined in the websub section.
     """
     if websubtable:
         for regexp, format in websubtable:
             text = regexp.sub(format, text)
     return text
 
 
 def loadfilter(ui, extname, registrarobj):
     """Load template filter from specified registrarobj"""
     for name, func in registrarobj._table.items():
         filters[name] = func
 
 
 # tell hggettext to extract docstrings from these functions:
 i18nfunctions = filters.values()