py3: convert known-int values to bytes using %d

I decided to grep around for \sstr\( and see what low-hanging fruit
that showed me. This was part of that hunt. That grep pattern still
has some things worth exploring.

Differential Revision: https://phab.mercurial-scm.org/D2441

mercurial/exchange.py

@@ -1,2263 +1,2263 b''
 # exchange.py - utility to exchange data between repos.
 #
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import collections
 import errno
 import hashlib
 
 from .i18n import _
 from .node import (
     bin,
     hex,
     nullid,
 )
 from . import (
     bookmarks as bookmod,
     bundle2,
     changegroup,
     discovery,
     error,
     lock as lockmod,
     logexchange,
     obsolete,
     phases,
     pushkey,
     pycompat,
     scmutil,
     sslutil,
     streamclone,
     url as urlmod,
     util,
 )
 
 urlerr = util.urlerr
 urlreq = util.urlreq
 
 # Maps bundle version human names to changegroup versions.
 _bundlespeccgversions = {'v1': '01',
                          'v2': '02',
                          'packed1': 's1',
                          'bundle2': '02', #legacy
                         }
 
 # Compression engines allowed in version 1. THIS SHOULD NEVER CHANGE.
 _bundlespecv1compengines = {'gzip', 'bzip2', 'none'}
 
 def parsebundlespec(repo, spec, strict=True, externalnames=False):
     """Parse a bundle string specification into parts.
 
     Bundle specifications denote a well-defined bundle/exchange format.
     The content of a given specification should not change over time in
     order to ensure that bundles produced by a newer version of Mercurial are
     readable from an older version.
 
     The string currently has the form:
 
        <compression>-<type>[;<parameter0>[;<parameter1>]]
 
     Where <compression> is one of the supported compression formats
     and <type> is (currently) a version string. A ";" can follow the type and
     all text afterwards is interpreted as URI encoded, ";" delimited key=value
     pairs.
 
     If ``strict`` is True (the default) <compression> is required. Otherwise,
     it is optional.
 
     If ``externalnames`` is False (the default), the human-centric names will
     be converted to their internal representation.
 
     Returns a 3-tuple of (compression, version, parameters). Compression will
     be ``None`` if not in strict mode and a compression isn't defined.
 
     An ``InvalidBundleSpecification`` is raised when the specification is
     not syntactically well formed.
 
     An ``UnsupportedBundleSpecification`` is raised when the compression or
     bundle type/version is not recognized.
 
     Note: this function will likely eventually return a more complex data
     structure, including bundle2 part information.
     """
     def parseparams(s):
         if ';' not in s:
             return s, {}
 
         params = {}
         version, paramstr = s.split(';', 1)
 
         for p in paramstr.split(';'):
             if '=' not in p:
                 raise error.InvalidBundleSpecification(
                     _('invalid bundle specification: '
                       'missing "=" in parameter: %s') % p)
 
             key, value = p.split('=', 1)
             key = urlreq.unquote(key)
             value = urlreq.unquote(value)
             params[key] = value
 
         return version, params
 
 
     if strict and '-' not in spec:
         raise error.InvalidBundleSpecification(
                 _('invalid bundle specification; '
                   'must be prefixed with compression: %s') % spec)
 
     if '-' in spec:
         compression, version = spec.split('-', 1)
 
         if compression not in util.compengines.supportedbundlenames:
             raise error.UnsupportedBundleSpecification(
                     _('%s compression is not supported') % compression)
 
         version, params = parseparams(version)
 
         if version not in _bundlespeccgversions:
             raise error.UnsupportedBundleSpecification(
                     _('%s is not a recognized bundle version') % version)
     else:
         # Value could be just the compression or just the version, in which
         # case some defaults are assumed (but only when not in strict mode).
         assert not strict
 
         spec, params = parseparams(spec)
 
         if spec in util.compengines.supportedbundlenames:
             compression = spec
             version = 'v1'
             # Generaldelta repos require v2.
             if 'generaldelta' in repo.requirements:
                 version = 'v2'
             # Modern compression engines require v2.
             if compression not in _bundlespecv1compengines:
                 version = 'v2'
         elif spec in _bundlespeccgversions:
             if spec == 'packed1':
                 compression = 'none'
             else:
                 compression = 'bzip2'
             version = spec
         else:
             raise error.UnsupportedBundleSpecification(
                     _('%s is not a recognized bundle specification') % spec)
 
     # Bundle version 1 only supports a known set of compression engines.
     if version == 'v1' and compression not in _bundlespecv1compengines:
         raise error.UnsupportedBundleSpecification(
             _('compression engine %s is not supported on v1 bundles') %
             compression)
 
     # The specification for packed1 can optionally declare the data formats
     # required to apply it. If we see this metadata, compare against what the
     # repo supports and error if the bundle isn't compatible.
     if version == 'packed1' and 'requirements' in params:
         requirements = set(params['requirements'].split(','))
         missingreqs = requirements - repo.supportedformats
         if missingreqs:
             raise error.UnsupportedBundleSpecification(
                     _('missing support for repository features: %s') %
                       ', '.join(sorted(missingreqs)))
 
     if not externalnames:
         engine = util.compengines.forbundlename(compression)
         compression = engine.bundletype()[1]
         version = _bundlespeccgversions[version]
     return compression, version, params
 
 def readbundle(ui, fh, fname, vfs=None):
     header = changegroup.readexactly(fh, 4)
 
     alg = None
     if not fname:
         fname = "stream"
         if not header.startswith('HG') and header.startswith('\0'):
             fh = changegroup.headerlessfixup(fh, header)
             header = "HG10"
             alg = 'UN'
     elif vfs:
         fname = vfs.join(fname)
 
     magic, version = header[0:2], header[2:4]
 
     if magic != 'HG':
         raise error.Abort(_('%s: not a Mercurial bundle') % fname)
     if version == '10':
         if alg is None:
             alg = changegroup.readexactly(fh, 2)
         return changegroup.cg1unpacker(fh, alg)
     elif version.startswith('2'):
         return bundle2.getunbundler(ui, fh, magicstring=magic + version)
     elif version == 'S1':
         return streamclone.streamcloneapplier(fh)
     else:
         raise error.Abort(_('%s: unknown bundle version %s') % (fname, version))
 
 def _formatrequirementsspec(requirements):
     return urlreq.quote(','.join(sorted(requirements)))
 
 def _formatrequirementsparams(requirements):
     requirements = _formatrequirementsspec(requirements)
     params = "%s%s" % (urlreq.quote("requirements="), requirements)
     return params
 
 def getbundlespec(ui, fh):
     """Infer the bundlespec from a bundle file handle.
 
     The input file handle is seeked and the original seek position is not
     restored.
     """
     def speccompression(alg):
         try:
             return util.compengines.forbundletype(alg).bundletype()[0]
         except KeyError:
             return None
 
     b = readbundle(ui, fh, None)
     if isinstance(b, changegroup.cg1unpacker):
         alg = b._type
         if alg == '_truncatedBZ':
             alg = 'BZ'
         comp = speccompression(alg)
         if not comp:
             raise error.Abort(_('unknown compression algorithm: %s') % alg)
         return '%s-v1' % comp
     elif isinstance(b, bundle2.unbundle20):
         if 'Compression' in b.params:
             comp = speccompression(b.params['Compression'])
             if not comp:
                 raise error.Abort(_('unknown compression algorithm: %s') % comp)
         else:
             comp = 'none'
 
         version = None
         for part in b.iterparts():
             if part.type == 'changegroup':
                 version = part.params['version']
                 if version in ('01', '02'):
                     version = 'v2'
                 else:
                     raise error.Abort(_('changegroup version %s does not have '
                                         'a known bundlespec') % version,
                                       hint=_('try upgrading your Mercurial '
                                               'client'))
 
         if not version:
             raise error.Abort(_('could not identify changegroup version in '
                                 'bundle'))
 
         return '%s-%s' % (comp, version)
     elif isinstance(b, streamclone.streamcloneapplier):
         requirements = streamclone.readbundle1header(fh)[2]
         return 'none-packed1;%s' % _formatrequirementsparams(requirements)
     else:
         raise error.Abort(_('unknown bundle type: %s') % b)
 
 def _computeoutgoing(repo, heads, common):
     """Computes which revs are outgoing given a set of common
     and a set of heads.
 
     This is a separate function so extensions can have access to
     the logic.
 
     Returns a discovery.outgoing object.
     """
     cl = repo.changelog
     if common:
         hasnode = cl.hasnode
         common = [n for n in common if hasnode(n)]
     else:
         common = [nullid]
     if not heads:
         heads = cl.heads()
     return discovery.outgoing(repo, common, heads)
 
 def _forcebundle1(op):
     """return true if a pull/push must use bundle1
 
     This function is used to allow testing of the older bundle version"""
     ui = op.repo.ui
     forcebundle1 = False
     # The goal is this config is to allow developer to choose the bundle
     # version used during exchanged. This is especially handy during test.
     # Value is a list of bundle version to be picked from, highest version
     # should be used.
     #
     # developer config: devel.legacy.exchange
     exchange = ui.configlist('devel', 'legacy.exchange')
     forcebundle1 = 'bundle2' not in exchange and 'bundle1' in exchange
     return forcebundle1 or not op.remote.capable('bundle2')
 
 class pushoperation(object):
     """A object that represent a single push operation
 
     Its purpose is to carry push related state and very common operations.
 
     A new pushoperation should be created at the beginning of each push and
     discarded afterward.
     """
 
     def __init__(self, repo, remote, force=False, revs=None, newbranch=False,
                  bookmarks=(), pushvars=None):
         # repo we push from
         self.repo = repo
         self.ui = repo.ui
         # repo we push to
         self.remote = remote
         # force option provided
         self.force = force
         # revs to be pushed (None is "all")
         self.revs = revs
         # bookmark explicitly pushed
         self.bookmarks = bookmarks
         # allow push of new branch
         self.newbranch = newbranch
         # step already performed
         # (used to check what steps have been already performed through bundle2)
         self.stepsdone = set()
         # Integer version of the changegroup push result
         # - None means nothing to push
         # - 0 means HTTP error
         # - 1 means we pushed and remote head count is unchanged *or*
         #   we have outgoing changesets but refused to push
         # - other values as described by addchangegroup()
         self.cgresult = None
         # Boolean value for the bookmark push
         self.bkresult = None
         # discover.outgoing object (contains common and outgoing data)
         self.outgoing = None
         # all remote topological heads before the push
         self.remoteheads = None
         # Details of the remote branch pre and post push
         #
         # mapping: {'branch': ([remoteheads],
         #                      [newheads],
         #                      [unsyncedheads],
         #                      [discardedheads])}
         # - branch: the branch name
         # - remoteheads: the list of remote heads known locally
         #                None if the branch is new
         # - newheads: the new remote heads (known locally) with outgoing pushed
         # - unsyncedheads: the list of remote heads unknown locally.
         # - discardedheads: the list of remote heads made obsolete by the push
         self.pushbranchmap = None
         # testable as a boolean indicating if any nodes are missing locally.
         self.incoming = None
         # summary of the remote phase situation
         self.remotephases = None
         # phases changes that must be pushed along side the changesets
         self.outdatedphases = None
         # phases changes that must be pushed if changeset push fails
         self.fallbackoutdatedphases = None
         # outgoing obsmarkers
         self.outobsmarkers = set()
         # outgoing bookmarks
         self.outbookmarks = []
         # transaction manager
         self.trmanager = None
         # map { pushkey partid -> callback handling failure}
         # used to handle exception from mandatory pushkey part failure
         self.pkfailcb = {}
         # an iterable of pushvars or None
         self.pushvars = pushvars
 
     @util.propertycache
     def futureheads(self):
         """future remote heads if the changeset push succeeds"""
         return self.outgoing.missingheads
 
     @util.propertycache
     def fallbackheads(self):
         """future remote heads if the changeset push fails"""
         if self.revs is None:
             # not target to push, all common are relevant
             return self.outgoing.commonheads
         unfi = self.repo.unfiltered()
         # I want cheads = heads(::missingheads and ::commonheads)
         # (missingheads is revs with secret changeset filtered out)
         #
         # This can be expressed as:
         #     cheads = ( (missingheads and ::commonheads)
         #              + (commonheads and ::missingheads))"
         #              )
         #
         # while trying to push we already computed the following:
         #     common = (::commonheads)
         #     missing = ((commonheads::missingheads) - commonheads)
         #
         # We can pick:
         # * missingheads part of common (::commonheads)
         common = self.outgoing.common
         nm = self.repo.changelog.nodemap
         cheads = [node for node in self.revs if nm[node] in common]
         # and
         # * commonheads parents on missing
         revset = unfi.set('%ln and parents(roots(%ln))',
                          self.outgoing.commonheads,
                          self.outgoing.missing)
         cheads.extend(c.node() for c in revset)
         return cheads
 
     @property
     def commonheads(self):
         """set of all common heads after changeset bundle push"""
         if self.cgresult:
             return self.futureheads
         else:
             return self.fallbackheads
 
 # mapping of message used when pushing bookmark
 bookmsgmap = {'update': (_("updating bookmark %s\n"),
                          _('updating bookmark %s failed!\n')),
               'export': (_("exporting bookmark %s\n"),
                          _('exporting bookmark %s failed!\n')),
               'delete': (_("deleting remote bookmark %s\n"),
                          _('deleting remote bookmark %s failed!\n')),
               }
 
 
 def push(repo, remote, force=False, revs=None, newbranch=False, bookmarks=(),
          opargs=None):
     '''Push outgoing changesets (limited by revs) from a local
     repository to remote. Return an integer:
       - None means nothing to push
       - 0 means HTTP error
       - 1 means we pushed and remote head count is unchanged *or*
         we have outgoing changesets but refused to push
       - other values as described by addchangegroup()
     '''
     if opargs is None:
         opargs = {}
     pushop = pushoperation(repo, remote, force, revs, newbranch, bookmarks,
                            **pycompat.strkwargs(opargs))
     if pushop.remote.local():
         missing = (set(pushop.repo.requirements)
                    - pushop.remote.local().supported)
         if missing:
             msg = _("required features are not"
                     " supported in the destination:"
                     " %s") % (', '.join(sorted(missing)))
             raise error.Abort(msg)
 
     if not pushop.remote.canpush():
         raise error.Abort(_("destination does not support push"))
 
     if not pushop.remote.capable('unbundle'):
         raise error.Abort(_('cannot push: destination does not support the '
                             'unbundle wire protocol command'))
 
     # get lock as we might write phase data
     wlock = lock = None
     try:
         # bundle2 push may receive a reply bundle touching bookmarks or other
         # things requiring the wlock. Take it now to ensure proper ordering.
         maypushback = pushop.ui.configbool('experimental', 'bundle2.pushback')
         if (not _forcebundle1(pushop)) and maypushback:
             wlock = pushop.repo.wlock()
         lock = pushop.repo.lock()
         pushop.trmanager = transactionmanager(pushop.repo,
                                               'push-response',
                                               pushop.remote.url())
     except IOError as err:
         if err.errno != errno.EACCES:
             raise
         # source repo cannot be locked.
         # We do not abort the push, but just disable the local phase
         # synchronisation.
         msg = 'cannot lock source repository: %s\n' % err
         pushop.ui.debug(msg)
 
     with wlock or util.nullcontextmanager(), \
             lock or util.nullcontextmanager(), \
             pushop.trmanager or util.nullcontextmanager():
         pushop.repo.checkpush(pushop)
         _pushdiscovery(pushop)
         if not _forcebundle1(pushop):
             _pushbundle2(pushop)
         _pushchangeset(pushop)
         _pushsyncphase(pushop)
         _pushobsolete(pushop)
         _pushbookmark(pushop)
 
     return pushop
 
 # list of steps to perform discovery before push
 pushdiscoveryorder = []
 
 # Mapping between step name and function
 #
 # This exists to help extensions wrap steps if necessary
 pushdiscoverymapping = {}
 
 def pushdiscovery(stepname):
     """decorator for function performing discovery before push
 
     The function is added to the step -> function mapping and appended to the
     list of steps.  Beware that decorated function will be added in order (this
     may matter).
 
     You can only use this decorator for a new step, if you want to wrap a step
     from an extension, change the pushdiscovery dictionary directly."""
     def dec(func):
         assert stepname not in pushdiscoverymapping
         pushdiscoverymapping[stepname] = func
         pushdiscoveryorder.append(stepname)
         return func
     return dec
 
 def _pushdiscovery(pushop):
     """Run all discovery steps"""
     for stepname in pushdiscoveryorder:
         step = pushdiscoverymapping[stepname]
         step(pushop)
 
 @pushdiscovery('changeset')
 def _pushdiscoverychangeset(pushop):
     """discover the changeset that need to be pushed"""
     fci = discovery.findcommonincoming
     if pushop.revs:
         commoninc = fci(pushop.repo, pushop.remote, force=pushop.force,
                         ancestorsof=pushop.revs)
     else:
         commoninc = fci(pushop.repo, pushop.remote, force=pushop.force)
     common, inc, remoteheads = commoninc
     fco = discovery.findcommonoutgoing
     outgoing = fco(pushop.repo, pushop.remote, onlyheads=pushop.revs,
                    commoninc=commoninc, force=pushop.force)
     pushop.outgoing = outgoing
     pushop.remoteheads = remoteheads
     pushop.incoming = inc
 
 @pushdiscovery('phase')
 def _pushdiscoveryphase(pushop):
     """discover the phase that needs to be pushed
 
     (computed for both success and failure case for changesets push)"""
     outgoing = pushop.outgoing
     unfi = pushop.repo.unfiltered()
     remotephases = pushop.remote.listkeys('phases')
     if (pushop.ui.configbool('ui', '_usedassubrepo')
         and remotephases    # server supports phases
         and not pushop.outgoing.missing # no changesets to be pushed
         and remotephases.get('publishing', False)):
         # When:
         # - this is a subrepo push
         # - and remote support phase
         # - and no changeset are to be pushed
         # - and remote is publishing
         # We may be in issue 3781 case!
         # We drop the possible phase synchronisation done by
         # courtesy to publish changesets possibly locally draft
         # on the remote.
         pushop.outdatedphases = []
         pushop.fallbackoutdatedphases = []
         return
 
     pushop.remotephases = phases.remotephasessummary(pushop.repo,
                                                      pushop.fallbackheads,
                                                      remotephases)
     droots = pushop.remotephases.draftroots
 
     extracond = ''
     if not pushop.remotephases.publishing:
         extracond = ' and public()'
     revset = 'heads((%%ln::%%ln) %s)' % extracond
     # Get the list of all revs draft on remote by public here.
     # XXX Beware that revset break if droots is not strictly
     # XXX root we may want to ensure it is but it is costly
     fallback = list(unfi.set(revset, droots, pushop.fallbackheads))
     if not outgoing.missing:
         future = fallback
     else:
         # adds changeset we are going to push as draft
         #
         # should not be necessary for publishing server, but because of an
         # issue fixed in xxxxx we have to do it anyway.
         fdroots = list(unfi.set('roots(%ln  + %ln::)',
                        outgoing.missing, droots))
         fdroots = [f.node() for f in fdroots]
         future = list(unfi.set(revset, fdroots, pushop.futureheads))
     pushop.outdatedphases = future
     pushop.fallbackoutdatedphases = fallback
 
 @pushdiscovery('obsmarker')
 def _pushdiscoveryobsmarkers(pushop):
     if (obsolete.isenabled(pushop.repo, obsolete.exchangeopt)
         and pushop.repo.obsstore
         and 'obsolete' in pushop.remote.listkeys('namespaces')):
         repo = pushop.repo
         # very naive computation, that can be quite expensive on big repo.
         # However: evolution is currently slow on them anyway.
         nodes = (c.node() for c in repo.set('::%ln', pushop.futureheads))
         pushop.outobsmarkers = pushop.repo.obsstore.relevantmarkers(nodes)
 
 @pushdiscovery('bookmarks')
 def _pushdiscoverybookmarks(pushop):
     ui = pushop.ui
     repo = pushop.repo.unfiltered()
     remote = pushop.remote
     ui.debug("checking for updated bookmarks\n")
     ancestors = ()
     if pushop.revs:
         revnums = map(repo.changelog.rev, pushop.revs)
         ancestors = repo.changelog.ancestors(revnums, inclusive=True)
     remotebookmark = remote.listkeys('bookmarks')
 
     explicit = set([repo._bookmarks.expandname(bookmark)
                     for bookmark in pushop.bookmarks])
 
     remotebookmark = bookmod.unhexlifybookmarks(remotebookmark)
     comp = bookmod.comparebookmarks(repo, repo._bookmarks, remotebookmark)
 
     def safehex(x):
         if x is None:
             return x
         return hex(x)
 
     def hexifycompbookmarks(bookmarks):
         for b, scid, dcid in bookmarks:
             yield b, safehex(scid), safehex(dcid)
 
     comp = [hexifycompbookmarks(marks) for marks in comp]
     addsrc, adddst, advsrc, advdst, diverge, differ, invalid, same = comp
 
     for b, scid, dcid in advsrc:
         if b in explicit:
             explicit.remove(b)
         if not ancestors or repo[scid].rev() in ancestors:
             pushop.outbookmarks.append((b, dcid, scid))
     # search added bookmark
     for b, scid, dcid in addsrc:
         if b in explicit:
             explicit.remove(b)
             pushop.outbookmarks.append((b, '', scid))
     # search for overwritten bookmark
     for b, scid, dcid in list(advdst) + list(diverge) + list(differ):
         if b in explicit:
             explicit.remove(b)
             pushop.outbookmarks.append((b, dcid, scid))
     # search for bookmark to delete
     for b, scid, dcid in adddst:
         if b in explicit:
             explicit.remove(b)
             # treat as "deleted locally"
             pushop.outbookmarks.append((b, dcid, ''))
     # identical bookmarks shouldn't get reported
     for b, scid, dcid in same:
         if b in explicit:
             explicit.remove(b)
 
     if explicit:
         explicit = sorted(explicit)
         # we should probably list all of them
         ui.warn(_('bookmark %s does not exist on the local '
                   'or remote repository!\n') % explicit[0])
         pushop.bkresult = 2
 
     pushop.outbookmarks.sort()
 
 def _pushcheckoutgoing(pushop):
     outgoing = pushop.outgoing
     unfi = pushop.repo.unfiltered()
     if not outgoing.missing:
         # nothing to push
         scmutil.nochangesfound(unfi.ui, unfi, outgoing.excluded)
         return False
     # something to push
     if not pushop.force:
         # if repo.obsstore == False --> no obsolete
         # then, save the iteration
         if unfi.obsstore:
             # this message are here for 80 char limit reason
             mso = _("push includes obsolete changeset: %s!")
             mspd = _("push includes phase-divergent changeset: %s!")
             mscd = _("push includes content-divergent changeset: %s!")
             mst = {"orphan": _("push includes orphan changeset: %s!"),
                    "phase-divergent": mspd,
                    "content-divergent": mscd}
             # If we are to push if there is at least one
             # obsolete or unstable changeset in missing, at
             # least one of the missinghead will be obsolete or
             # unstable. So checking heads only is ok
             for node in outgoing.missingheads:
                 ctx = unfi[node]
                 if ctx.obsolete():
                     raise error.Abort(mso % ctx)
                 elif ctx.isunstable():
                     # TODO print more than one instability in the abort
                     # message
                     raise error.Abort(mst[ctx.instabilities()[0]] % ctx)
 
         discovery.checkheads(pushop)
     return True
 
 # List of names of steps to perform for an outgoing bundle2, order matters.
 b2partsgenorder = []
 
 # Mapping between step name and function
 #
 # This exists to help extensions wrap steps if necessary
 b2partsgenmapping = {}
 
 def b2partsgenerator(stepname, idx=None):
     """decorator for function generating bundle2 part
 
     The function is added to the step -> function mapping and appended to the
     list of steps.  Beware that decorated functions will be added in order
     (this may matter).
 
     You can only use this decorator for new steps, if you want to wrap a step
     from an extension, attack the b2partsgenmapping dictionary directly."""
     def dec(func):
         assert stepname not in b2partsgenmapping
         b2partsgenmapping[stepname] = func
         if idx is None:
             b2partsgenorder.append(stepname)
         else:
             b2partsgenorder.insert(idx, stepname)
         return func
     return dec
 
 def _pushb2ctxcheckheads(pushop, bundler):
     """Generate race condition checking parts
 
     Exists as an independent function to aid extensions
     """
     # * 'force' do not check for push race,
     # * if we don't push anything, there are nothing to check.
     if not pushop.force and pushop.outgoing.missingheads:
         allowunrelated = 'related' in bundler.capabilities.get('checkheads', ())
         emptyremote = pushop.pushbranchmap is None
         if not allowunrelated or emptyremote:
             bundler.newpart('check:heads', data=iter(pushop.remoteheads))
         else:
             affected = set()
             for branch, heads in pushop.pushbranchmap.iteritems():
                 remoteheads, newheads, unsyncedheads, discardedheads = heads
                 if remoteheads is not None:
                     remote = set(remoteheads)
                     affected |= set(discardedheads) & remote
                     affected |= remote - set(newheads)
             if affected:
                 data = iter(sorted(affected))
                 bundler.newpart('check:updated-heads', data=data)
 
 def _pushing(pushop):
     """return True if we are pushing anything"""
     return bool(pushop.outgoing.missing
                 or pushop.outdatedphases
                 or pushop.outobsmarkers
                 or pushop.outbookmarks)
 
 @b2partsgenerator('check-bookmarks')
 def _pushb2checkbookmarks(pushop, bundler):
     """insert bookmark move checking"""
     if not _pushing(pushop) or pushop.force:
         return
     b2caps = bundle2.bundle2caps(pushop.remote)
     hasbookmarkcheck = 'bookmarks' in b2caps
     if not (pushop.outbookmarks and hasbookmarkcheck):
         return
     data = []
     for book, old, new in pushop.outbookmarks:
         old = bin(old)
         data.append((book, old))
     checkdata = bookmod.binaryencode(data)
     bundler.newpart('check:bookmarks', data=checkdata)
 
 @b2partsgenerator('check-phases')
 def _pushb2checkphases(pushop, bundler):
     """insert phase move checking"""
     if not _pushing(pushop) or pushop.force:
         return
     b2caps = bundle2.bundle2caps(pushop.remote)
     hasphaseheads = 'heads' in b2caps.get('phases', ())
     if pushop.remotephases is not None and hasphaseheads:
         # check that the remote phase has not changed
         checks = [[] for p in phases.allphases]
         checks[phases.public].extend(pushop.remotephases.publicheads)
         checks[phases.draft].extend(pushop.remotephases.draftroots)
         if any(checks):
             for nodes in checks:
                 nodes.sort()
             checkdata = phases.binaryencode(checks)
             bundler.newpart('check:phases', data=checkdata)
 
 @b2partsgenerator('changeset')
 def _pushb2ctx(pushop, bundler):
     """handle changegroup push through bundle2
 
     addchangegroup result is stored in the ``pushop.cgresult`` attribute.
     """
     if 'changesets' in pushop.stepsdone:
         return
     pushop.stepsdone.add('changesets')
     # Send known heads to the server for race detection.
     if not _pushcheckoutgoing(pushop):
         return
     pushop.repo.prepushoutgoinghooks(pushop)
 
     _pushb2ctxcheckheads(pushop, bundler)
 
     b2caps = bundle2.bundle2caps(pushop.remote)
     version = '01'
     cgversions = b2caps.get('changegroup')
     if cgversions:  # 3.1 and 3.2 ship with an empty value
         cgversions = [v for v in cgversions
                       if v in changegroup.supportedoutgoingversions(
                           pushop.repo)]
         if not cgversions:
             raise ValueError(_('no common changegroup version'))
         version = max(cgversions)
     cgstream = changegroup.makestream(pushop.repo, pushop.outgoing, version,
                                       'push')
     cgpart = bundler.newpart('changegroup', data=cgstream)
     if cgversions:
         cgpart.addparam('version', version)
     if 'treemanifest' in pushop.repo.requirements:
         cgpart.addparam('treemanifest', '1')
     def handlereply(op):
         """extract addchangegroup returns from server reply"""
         cgreplies = op.records.getreplies(cgpart.id)
         assert len(cgreplies['changegroup']) == 1
         pushop.cgresult = cgreplies['changegroup'][0]['return']
     return handlereply
 
 @b2partsgenerator('phase')
 def _pushb2phases(pushop, bundler):
     """handle phase push through bundle2"""
     if 'phases' in pushop.stepsdone:
         return
     b2caps = bundle2.bundle2caps(pushop.remote)
     ui = pushop.repo.ui
 
     legacyphase = 'phases' in ui.configlist('devel', 'legacy.exchange')
     haspushkey = 'pushkey' in b2caps
     hasphaseheads = 'heads' in b2caps.get('phases', ())
 
     if hasphaseheads and not legacyphase:
         return _pushb2phaseheads(pushop, bundler)
     elif haspushkey:
         return _pushb2phasespushkey(pushop, bundler)
 
 def _pushb2phaseheads(pushop, bundler):
     """push phase information through a bundle2 - binary part"""
     pushop.stepsdone.add('phases')
     if pushop.outdatedphases:
         updates = [[] for p in phases.allphases]
         updates[0].extend(h.node() for h in pushop.outdatedphases)
         phasedata = phases.binaryencode(updates)
         bundler.newpart('phase-heads', data=phasedata)
 
 def _pushb2phasespushkey(pushop, bundler):
     """push phase information through a bundle2 - pushkey part"""
     pushop.stepsdone.add('phases')
     part2node = []
 
     def handlefailure(pushop, exc):
         targetid = int(exc.partid)
         for partid, node in part2node:
             if partid == targetid:
                 raise error.Abort(_('updating %s to public failed') % node)
 
     enc = pushkey.encode
     for newremotehead in pushop.outdatedphases:
         part = bundler.newpart('pushkey')
         part.addparam('namespace', enc('phases'))
         part.addparam('key', enc(newremotehead.hex()))
         part.addparam('old', enc('%d' % phases.draft))
         part.addparam('new', enc('%d' % phases.public))
         part2node.append((part.id, newremotehead))
         pushop.pkfailcb[part.id] = handlefailure
 
     def handlereply(op):
         for partid, node in part2node:
             partrep = op.records.getreplies(partid)
             results = partrep['pushkey']
             assert len(results) <= 1
             msg = None
             if not results:
                 msg = _('server ignored update of %s to public!\n') % node
             elif not int(results[0]['return']):
                 msg = _('updating %s to public failed!\n') % node
             if msg is not None:
                 pushop.ui.warn(msg)
     return handlereply
 
 @b2partsgenerator('obsmarkers')
 def _pushb2obsmarkers(pushop, bundler):
     if 'obsmarkers' in pushop.stepsdone:
         return
     remoteversions = bundle2.obsmarkersversion(bundler.capabilities)
     if obsolete.commonversion(remoteversions) is None:
         return
     pushop.stepsdone.add('obsmarkers')
     if pushop.outobsmarkers:
         markers = sorted(pushop.outobsmarkers)
         bundle2.buildobsmarkerspart(bundler, markers)
 
 @b2partsgenerator('bookmarks')
 def _pushb2bookmarks(pushop, bundler):
     """handle bookmark push through bundle2"""
     if 'bookmarks' in pushop.stepsdone:
         return
     b2caps = bundle2.bundle2caps(pushop.remote)
 
     legacy = pushop.repo.ui.configlist('devel', 'legacy.exchange')
     legacybooks = 'bookmarks' in legacy
 
     if not legacybooks and 'bookmarks' in b2caps:
         return _pushb2bookmarkspart(pushop, bundler)
     elif 'pushkey' in b2caps:
         return _pushb2bookmarkspushkey(pushop, bundler)
 
 def _bmaction(old, new):
     """small utility for bookmark pushing"""
     if not old:
         return 'export'
     elif not new:
         return 'delete'
     return 'update'
 
 def _pushb2bookmarkspart(pushop, bundler):
     pushop.stepsdone.add('bookmarks')
     if not pushop.outbookmarks:
         return
 
     allactions = []
     data = []
     for book, old, new in pushop.outbookmarks:
         new = bin(new)
         data.append((book, new))
         allactions.append((book, _bmaction(old, new)))
     checkdata = bookmod.binaryencode(data)
     bundler.newpart('bookmarks', data=checkdata)
 
     def handlereply(op):
         ui = pushop.ui
         # if success
         for book, action in allactions:
             ui.status(bookmsgmap[action][0] % book)
 
     return handlereply
 
 def _pushb2bookmarkspushkey(pushop, bundler):
     pushop.stepsdone.add('bookmarks')
     part2book = []
     enc = pushkey.encode
 
     def handlefailure(pushop, exc):
         targetid = int(exc.partid)
         for partid, book, action in part2book:
             if partid == targetid:
                 raise error.Abort(bookmsgmap[action][1].rstrip() % book)
         # we should not be called for part we did not generated
         assert False
 
     for book, old, new in pushop.outbookmarks:
         part = bundler.newpart('pushkey')
         part.addparam('namespace', enc('bookmarks'))
         part.addparam('key', enc(book))
         part.addparam('old', enc(old))
         part.addparam('new', enc(new))
         action = 'update'
         if not old:
             action = 'export'
         elif not new:
             action = 'delete'
         part2book.append((part.id, book, action))
         pushop.pkfailcb[part.id] = handlefailure
 
     def handlereply(op):
         ui = pushop.ui
         for partid, book, action in part2book:
             partrep = op.records.getreplies(partid)
             results = partrep['pushkey']
             assert len(results) <= 1
             if not results:
                 pushop.ui.warn(_('server ignored bookmark %s update\n') % book)
             else:
                 ret = int(results[0]['return'])
                 if ret:
                     ui.status(bookmsgmap[action][0] % book)
                 else:
                     ui.warn(bookmsgmap[action][1] % book)
                     if pushop.bkresult is not None:
                         pushop.bkresult = 1
     return handlereply
 
 @b2partsgenerator('pushvars', idx=0)
 def _getbundlesendvars(pushop, bundler):
     '''send shellvars via bundle2'''
     pushvars = pushop.pushvars
     if pushvars:
         shellvars = {}
         for raw in pushvars:
             if '=' not in raw:
                 msg = ("unable to parse variable '%s', should follow "
                         "'KEY=VALUE' or 'KEY=' format")
                 raise error.Abort(msg % raw)
             k, v = raw.split('=', 1)
             shellvars[k] = v
 
         part = bundler.newpart('pushvars')
 
         for key, value in shellvars.iteritems():
             part.addparam(key, value, mandatory=False)
 
 def _pushbundle2(pushop):
     """push data to the remote using bundle2
 
     The only currently supported type of data is changegroup but this will
     evolve in the future."""
     bundler = bundle2.bundle20(pushop.ui, bundle2.bundle2caps(pushop.remote))
     pushback = (pushop.trmanager
                 and pushop.ui.configbool('experimental', 'bundle2.pushback'))
 
     # create reply capability
     capsblob = bundle2.encodecaps(bundle2.getrepocaps(pushop.repo,
                                                       allowpushback=pushback,
                                                       role='client'))
     bundler.newpart('replycaps', data=capsblob)
     replyhandlers = []
     for partgenname in b2partsgenorder:
         partgen = b2partsgenmapping[partgenname]
         ret = partgen(pushop, bundler)
         if callable(ret):
             replyhandlers.append(ret)
     # do not push if nothing to push
     if bundler.nbparts <= 1:
         return
     stream = util.chunkbuffer(bundler.getchunks())
     try:
         try:
             reply = pushop.remote.unbundle(
                 stream, ['force'], pushop.remote.url())
         except error.BundleValueError as exc:
             raise error.Abort(_('missing support for %s') % exc)
         try:
             trgetter = None
             if pushback:
                 trgetter = pushop.trmanager.transaction
             op = bundle2.processbundle(pushop.repo, reply, trgetter)
         except error.BundleValueError as exc:
             raise error.Abort(_('missing support for %s') % exc)
         except bundle2.AbortFromPart as exc:
             pushop.ui.status(_('remote: %s\n') % exc)
             if exc.hint is not None:
                 pushop.ui.status(_('remote: %s\n') % ('(%s)' % exc.hint))
             raise error.Abort(_('push failed on remote'))
     except error.PushkeyFailed as exc:
         partid = int(exc.partid)
         if partid not in pushop.pkfailcb:
             raise
         pushop.pkfailcb[partid](pushop, exc)
     for rephand in replyhandlers:
         rephand(op)
 
 def _pushchangeset(pushop):
     """Make the actual push of changeset bundle to remote repo"""
     if 'changesets' in pushop.stepsdone:
         return
     pushop.stepsdone.add('changesets')
     if not _pushcheckoutgoing(pushop):
         return
 
     # Should have verified this in push().
     assert pushop.remote.capable('unbundle')
 
     pushop.repo.prepushoutgoinghooks(pushop)
     outgoing = pushop.outgoing
     # TODO: get bundlecaps from remote
     bundlecaps = None
     # create a changegroup from local
     if pushop.revs is None and not (outgoing.excluded
                             or pushop.repo.changelog.filteredrevs):
         # push everything,
         # use the fast path, no race possible on push
         cg = changegroup.makechangegroup(pushop.repo, outgoing, '01', 'push',
                 fastpath=True, bundlecaps=bundlecaps)
     else:
         cg = changegroup.makechangegroup(pushop.repo, outgoing, '01',
                                         'push', bundlecaps=bundlecaps)
 
     # apply changegroup to remote
     # local repo finds heads on server, finds out what
     # revs it must push. once revs transferred, if server
     # finds it has different heads (someone else won
     # commit/push race), server aborts.
     if pushop.force:
         remoteheads = ['force']
     else:
         remoteheads = pushop.remoteheads
     # ssh: return remote's addchangegroup()
     # http: return remote's addchangegroup() or 0 for error
     pushop.cgresult = pushop.remote.unbundle(cg, remoteheads,
                                         pushop.repo.url())
 
 def _pushsyncphase(pushop):
     """synchronise phase information locally and remotely"""
     cheads = pushop.commonheads
     # even when we don't push, exchanging phase data is useful
     remotephases = pushop.remote.listkeys('phases')
     if (pushop.ui.configbool('ui', '_usedassubrepo')
         and remotephases    # server supports phases
         and pushop.cgresult is None # nothing was pushed
         and remotephases.get('publishing', False)):
         # When:
         # - this is a subrepo push
         # - and remote support phase
         # - and no changeset was pushed
         # - and remote is publishing
         # We may be in issue 3871 case!
         # We drop the possible phase synchronisation done by
         # courtesy to publish changesets possibly locally draft
         # on the remote.
         remotephases = {'publishing': 'True'}
     if not remotephases: # old server or public only reply from non-publishing
         _localphasemove(pushop, cheads)
         # don't push any phase data as there is nothing to push
     else:
         ana = phases.analyzeremotephases(pushop.repo, cheads,
                                          remotephases)
         pheads, droots = ana
         ### Apply remote phase on local
         if remotephases.get('publishing', False):
             _localphasemove(pushop, cheads)
         else: # publish = False
             _localphasemove(pushop, pheads)
             _localphasemove(pushop, cheads, phases.draft)
         ### Apply local phase on remote
 
         if pushop.cgresult:
             if 'phases' in pushop.stepsdone:
                 # phases already pushed though bundle2
                 return
             outdated = pushop.outdatedphases
         else:
             outdated = pushop.fallbackoutdatedphases
 
         pushop.stepsdone.add('phases')
 
         # filter heads already turned public by the push
         outdated = [c for c in outdated if c.node() not in pheads]
         # fallback to independent pushkey command
         for newremotehead in outdated:
             r = pushop.remote.pushkey('phases',
                                       newremotehead.hex(),
-                                      str(phases.draft),
-                                      str(phases.public))
+                                      ('%d' % phases.draft),
+                                      ('%d' % phases.public))
             if not r:
                 pushop.ui.warn(_('updating %s to public failed!\n')
                                % newremotehead)
 
 def _localphasemove(pushop, nodes, phase=phases.public):
     """move <nodes> to <phase> in the local source repo"""
     if pushop.trmanager:
         phases.advanceboundary(pushop.repo,
                                pushop.trmanager.transaction(),
                                phase,
                                nodes)
     else:
         # repo is not locked, do not change any phases!
         # Informs the user that phases should have been moved when
         # applicable.
         actualmoves = [n for n in nodes if phase < pushop.repo[n].phase()]
         phasestr = phases.phasenames[phase]
         if actualmoves:
             pushop.ui.status(_('cannot lock source repo, skipping '
                                'local %s phase update\n') % phasestr)
 
 def _pushobsolete(pushop):
     """utility function to push obsolete markers to a remote"""
     if 'obsmarkers' in pushop.stepsdone:
         return
     repo = pushop.repo
     remote = pushop.remote
     pushop.stepsdone.add('obsmarkers')
     if pushop.outobsmarkers:
         pushop.ui.debug('try to push obsolete markers to remote\n')
         rslts = []
         remotedata = obsolete._pushkeyescape(sorted(pushop.outobsmarkers))
         for key in sorted(remotedata, reverse=True):
             # reverse sort to ensure we end with dump0
             data = remotedata[key]
             rslts.append(remote.pushkey('obsolete', key, '', data))
         if [r for r in rslts if not r]:
             msg = _('failed to push some obsolete markers!\n')
             repo.ui.warn(msg)
 
 def _pushbookmark(pushop):
     """Update bookmark position on remote"""
     if pushop.cgresult == 0 or 'bookmarks' in pushop.stepsdone:
         return
     pushop.stepsdone.add('bookmarks')
     ui = pushop.ui
     remote = pushop.remote
 
     for b, old, new in pushop.outbookmarks:
         action = 'update'
         if not old:
             action = 'export'
         elif not new:
             action = 'delete'
         if remote.pushkey('bookmarks', b, old, new):
             ui.status(bookmsgmap[action][0] % b)
         else:
             ui.warn(bookmsgmap[action][1] % b)
             # discovery can have set the value form invalid entry
             if pushop.bkresult is not None:
                 pushop.bkresult = 1
 
 class pulloperation(object):
     """A object that represent a single pull operation
 
     It purpose is to carry pull related state and very common operation.
 
     A new should be created at the beginning of each pull and discarded
     afterward.
     """
 
     def __init__(self, repo, remote, heads=None, force=False, bookmarks=(),
                  remotebookmarks=None, streamclonerequested=None):
         # repo we pull into
         self.repo = repo
         # repo we pull from
         self.remote = remote
         # revision we try to pull (None is "all")
         self.heads = heads
         # bookmark pulled explicitly
         self.explicitbookmarks = [repo._bookmarks.expandname(bookmark)
                                   for bookmark in bookmarks]
         # do we force pull?
         self.force = force
         # whether a streaming clone was requested
         self.streamclonerequested = streamclonerequested
         # transaction manager
         self.trmanager = None
         # set of common changeset between local and remote before pull
         self.common = None
         # set of pulled head
         self.rheads = None
         # list of missing changeset to fetch remotely
         self.fetch = None
         # remote bookmarks data
         self.remotebookmarks = remotebookmarks
         # result of changegroup pulling (used as return code by pull)
         self.cgresult = None
         # list of step already done
         self.stepsdone = set()
         # Whether we attempted a clone from pre-generated bundles.
         self.clonebundleattempted = False
 
     @util.propertycache
     def pulledsubset(self):
         """heads of the set of changeset target by the pull"""
         # compute target subset
         if self.heads is None:
             # We pulled every thing possible
             # sync on everything common
             c = set(self.common)
             ret = list(self.common)
             for n in self.rheads:
                 if n not in c:
                     ret.append(n)
             return ret
         else:
             # We pulled a specific subset
             # sync on this subset
             return self.heads
 
     @util.propertycache
     def canusebundle2(self):
         return not _forcebundle1(self)
 
     @util.propertycache
     def remotebundle2caps(self):
         return bundle2.bundle2caps(self.remote)
 
     def gettransaction(self):
         # deprecated; talk to trmanager directly
         return self.trmanager.transaction()
 
 class transactionmanager(util.transactional):
     """An object to manage the life cycle of a transaction
 
     It creates the transaction on demand and calls the appropriate hooks when
     closing the transaction."""
     def __init__(self, repo, source, url):
         self.repo = repo
         self.source = source
         self.url = url
         self._tr = None
 
     def transaction(self):
         """Return an open transaction object, constructing if necessary"""
         if not self._tr:
             trname = '%s\n%s' % (self.source, util.hidepassword(self.url))
             self._tr = self.repo.transaction(trname)
             self._tr.hookargs['source'] = self.source
             self._tr.hookargs['url'] = self.url
         return self._tr
 
     def close(self):
         """close transaction if created"""
         if self._tr is not None:
             self._tr.close()
 
     def release(self):
         """release transaction if created"""
         if self._tr is not None:
             self._tr.release()
 
 def pull(repo, remote, heads=None, force=False, bookmarks=(), opargs=None,
          streamclonerequested=None):
     """Fetch repository data from a remote.
 
     This is the main function used to retrieve data from a remote repository.
 
     ``repo`` is the local repository to clone into.
     ``remote`` is a peer instance.
     ``heads`` is an iterable of revisions we want to pull. ``None`` (the
     default) means to pull everything from the remote.
     ``bookmarks`` is an iterable of bookmarks requesting to be pulled. By
     default, all remote bookmarks are pulled.
     ``opargs`` are additional keyword arguments to pass to ``pulloperation``
     initialization.
     ``streamclonerequested`` is a boolean indicating whether a "streaming
     clone" is requested. A "streaming clone" is essentially a raw file copy
     of revlogs from the server. This only works when the local repository is
     empty. The default value of ``None`` means to respect the server
     configuration for preferring stream clones.
 
     Returns the ``pulloperation`` created for this pull.
     """
     if opargs is None:
         opargs = {}
     pullop = pulloperation(repo, remote, heads, force, bookmarks=bookmarks,
                            streamclonerequested=streamclonerequested,
                            **pycompat.strkwargs(opargs))
 
     peerlocal = pullop.remote.local()
     if peerlocal:
         missing = set(peerlocal.requirements) - pullop.repo.supported
         if missing:
             msg = _("required features are not"
                     " supported in the destination:"
                     " %s") % (', '.join(sorted(missing)))
             raise error.Abort(msg)
 
     pullop.trmanager = transactionmanager(repo, 'pull', remote.url())
     with repo.wlock(), repo.lock(), pullop.trmanager:
         # This should ideally be in _pullbundle2(). However, it needs to run
         # before discovery to avoid extra work.
         _maybeapplyclonebundle(pullop)
         streamclone.maybeperformlegacystreamclone(pullop)
         _pulldiscovery(pullop)
         if pullop.canusebundle2:
             _pullbundle2(pullop)
         _pullchangeset(pullop)
         _pullphase(pullop)
         _pullbookmarks(pullop)
         _pullobsolete(pullop)
 
     # storing remotenames
     if repo.ui.configbool('experimental', 'remotenames'):
         logexchange.pullremotenames(repo, remote)
 
     return pullop
 
 # list of steps to perform discovery before pull
 pulldiscoveryorder = []
 
 # Mapping between step name and function
 #
 # This exists to help extensions wrap steps if necessary
 pulldiscoverymapping = {}
 
 def pulldiscovery(stepname):
     """decorator for function performing discovery before pull
 
     The function is added to the step -> function mapping and appended to the
     list of steps.  Beware that decorated function will be added in order (this
     may matter).
 
     You can only use this decorator for a new step, if you want to wrap a step
     from an extension, change the pulldiscovery dictionary directly."""
     def dec(func):
         assert stepname not in pulldiscoverymapping
         pulldiscoverymapping[stepname] = func
         pulldiscoveryorder.append(stepname)
         return func
     return dec
 
 def _pulldiscovery(pullop):
     """Run all discovery steps"""
     for stepname in pulldiscoveryorder:
         step = pulldiscoverymapping[stepname]
         step(pullop)
 
 @pulldiscovery('b1:bookmarks')
 def _pullbookmarkbundle1(pullop):
     """fetch bookmark data in bundle1 case
 
     If not using bundle2, we have to fetch bookmarks before changeset
     discovery to reduce the chance and impact of race conditions."""
     if pullop.remotebookmarks is not None:
         return
     if pullop.canusebundle2 and 'listkeys' in pullop.remotebundle2caps:
         # all known bundle2 servers now support listkeys, but lets be nice with
         # new implementation.
         return
     books = pullop.remote.listkeys('bookmarks')
     pullop.remotebookmarks = bookmod.unhexlifybookmarks(books)
 
 
 @pulldiscovery('changegroup')
 def _pulldiscoverychangegroup(pullop):
     """discovery phase for the pull
 
     Current handle changeset discovery only, will change handle all discovery
     at some point."""
     tmp = discovery.findcommonincoming(pullop.repo,
                                        pullop.remote,
                                        heads=pullop.heads,
                                        force=pullop.force)
     common, fetch, rheads = tmp
     nm = pullop.repo.unfiltered().changelog.nodemap
     if fetch and rheads:
         # If a remote heads is filtered locally, put in back in common.
         #
         # This is a hackish solution to catch most of "common but locally
         # hidden situation".  We do not performs discovery on unfiltered
         # repository because it end up doing a pathological amount of round
         # trip for w huge amount of changeset we do not care about.
         #
         # If a set of such "common but filtered" changeset exist on the server
         # but are not including a remote heads, we'll not be able to detect it,
         scommon = set(common)
         for n in rheads:
             if n in nm:
                 if n not in scommon:
                     common.append(n)
         if set(rheads).issubset(set(common)):
             fetch = []
     pullop.common = common
     pullop.fetch = fetch
     pullop.rheads = rheads
 
 def _pullbundle2(pullop):
     """pull data using bundle2
 
     For now, the only supported data are changegroup."""
     kwargs = {'bundlecaps': caps20to10(pullop.repo, role='client')}
 
     # make ui easier to access
     ui = pullop.repo.ui
 
     # At the moment we don't do stream clones over bundle2. If that is
     # implemented then here's where the check for that will go.
     streaming = streamclone.canperformstreamclone(pullop, bundle2=True)[0]
 
     # declare pull perimeters
     kwargs['common'] = pullop.common
     kwargs['heads'] = pullop.heads or pullop.rheads
 
     if streaming:
         kwargs['cg'] = False
         kwargs['stream'] = True
         pullop.stepsdone.add('changegroup')
         pullop.stepsdone.add('phases')
 
     else:
         # pulling changegroup
         pullop.stepsdone.add('changegroup')
 
         kwargs['cg'] = pullop.fetch
 
         legacyphase = 'phases' in ui.configlist('devel', 'legacy.exchange')
         hasbinaryphase = 'heads' in pullop.remotebundle2caps.get('phases', ())
         if (not legacyphase and hasbinaryphase):
             kwargs['phases'] = True
             pullop.stepsdone.add('phases')
 
         if 'listkeys' in pullop.remotebundle2caps:
             if 'phases' not in pullop.stepsdone:
                 kwargs['listkeys'] = ['phases']
 
     bookmarksrequested = False
     legacybookmark = 'bookmarks' in ui.configlist('devel', 'legacy.exchange')
     hasbinarybook = 'bookmarks' in pullop.remotebundle2caps
 
     if pullop.remotebookmarks is not None:
         pullop.stepsdone.add('request-bookmarks')
 
     if ('request-bookmarks' not in pullop.stepsdone
         and pullop.remotebookmarks is None
         and not legacybookmark and hasbinarybook):
         kwargs['bookmarks'] = True
         bookmarksrequested = True
 
     if 'listkeys' in pullop.remotebundle2caps:
         if 'request-bookmarks' not in pullop.stepsdone:
             # make sure to always includes bookmark data when migrating
             # `hg incoming --bundle` to using this function.
             pullop.stepsdone.add('request-bookmarks')
             kwargs.setdefault('listkeys', []).append('bookmarks')
 
     # If this is a full pull / clone and the server supports the clone bundles
     # feature, tell the server whether we attempted a clone bundle. The
     # presence of this flag indicates the client supports clone bundles. This
     # will enable the server to treat clients that support clone bundles
     # differently from those that don't.
     if (pullop.remote.capable('clonebundles')
         and pullop.heads is None and list(pullop.common) == [nullid]):
         kwargs['cbattempted'] = pullop.clonebundleattempted
 
     if streaming:
         pullop.repo.ui.status(_('streaming all changes\n'))
     elif not pullop.fetch:
         pullop.repo.ui.status(_("no changes found\n"))
         pullop.cgresult = 0
     else:
         if pullop.heads is None and list(pullop.common) == [nullid]:
             pullop.repo.ui.status(_("requesting all changes\n"))
     if obsolete.isenabled(pullop.repo, obsolete.exchangeopt):
         remoteversions = bundle2.obsmarkersversion(pullop.remotebundle2caps)
         if obsolete.commonversion(remoteversions) is not None:
             kwargs['obsmarkers'] = True
             pullop.stepsdone.add('obsmarkers')
     _pullbundle2extraprepare(pullop, kwargs)
     bundle = pullop.remote.getbundle('pull', **pycompat.strkwargs(kwargs))
     try:
         op = bundle2.bundleoperation(pullop.repo, pullop.gettransaction)
         op.modes['bookmarks'] = 'records'
         bundle2.processbundle(pullop.repo, bundle, op=op)
     except bundle2.AbortFromPart as exc:
         pullop.repo.ui.status(_('remote: abort: %s\n') % exc)
         raise error.Abort(_('pull failed on remote'), hint=exc.hint)
     except error.BundleValueError as exc:
         raise error.Abort(_('missing support for %s') % exc)
 
     if pullop.fetch:
         pullop.cgresult = bundle2.combinechangegroupresults(op)
 
     # processing phases change
     for namespace, value in op.records['listkeys']:
         if namespace == 'phases':
             _pullapplyphases(pullop, value)
 
     # processing bookmark update
     if bookmarksrequested:
         books = {}
         for record in op.records['bookmarks']:
             books[record['bookmark']] = record["node"]
         pullop.remotebookmarks = books
     else:
         for namespace, value in op.records['listkeys']:
             if namespace == 'bookmarks':
                 pullop.remotebookmarks = bookmod.unhexlifybookmarks(value)
 
     # bookmark data were either already there or pulled in the bundle
     if pullop.remotebookmarks is not None:
         _pullbookmarks(pullop)
 
 def _pullbundle2extraprepare(pullop, kwargs):
     """hook function so that extensions can extend the getbundle call"""
 
 def _pullchangeset(pullop):
     """pull changeset from unbundle into the local repo"""
     # We delay the open of the transaction as late as possible so we
     # don't open transaction for nothing or you break future useful
     # rollback call
     if 'changegroup' in pullop.stepsdone:
         return
     pullop.stepsdone.add('changegroup')
     if not pullop.fetch:
         pullop.repo.ui.status(_("no changes found\n"))
         pullop.cgresult = 0
         return
     tr = pullop.gettransaction()
     if pullop.heads is None and list(pullop.common) == [nullid]:
         pullop.repo.ui.status(_("requesting all changes\n"))
     elif pullop.heads is None and pullop.remote.capable('changegroupsubset'):
         # issue1320, avoid a race if remote changed after discovery
         pullop.heads = pullop.rheads
 
     if pullop.remote.capable('getbundle'):
         # TODO: get bundlecaps from remote
         cg = pullop.remote.getbundle('pull', common=pullop.common,
                                      heads=pullop.heads or pullop.rheads)
     elif pullop.heads is None:
         cg = pullop.remote.changegroup(pullop.fetch, 'pull')
     elif not pullop.remote.capable('changegroupsubset'):
         raise error.Abort(_("partial pull cannot be done because "
                            "other repository doesn't support "
                            "changegroupsubset."))
     else:
         cg = pullop.remote.changegroupsubset(pullop.fetch, pullop.heads, 'pull')
     bundleop = bundle2.applybundle(pullop.repo, cg, tr, 'pull',
                                    pullop.remote.url())
     pullop.cgresult = bundle2.combinechangegroupresults(bundleop)
 
 def _pullphase(pullop):
     # Get remote phases data from remote
     if 'phases' in pullop.stepsdone:
         return
     remotephases = pullop.remote.listkeys('phases')
     _pullapplyphases(pullop, remotephases)
 
 def _pullapplyphases(pullop, remotephases):
     """apply phase movement from observed remote state"""
     if 'phases' in pullop.stepsdone:
         return
     pullop.stepsdone.add('phases')
     publishing = bool(remotephases.get('publishing', False))
     if remotephases and not publishing:
         # remote is new and non-publishing
         pheads, _dr = phases.analyzeremotephases(pullop.repo,
                                                  pullop.pulledsubset,
                                                  remotephases)
         dheads = pullop.pulledsubset
     else:
         # Remote is old or publishing all common changesets
         # should be seen as public
         pheads = pullop.pulledsubset
         dheads = []
     unfi = pullop.repo.unfiltered()
     phase = unfi._phasecache.phase
     rev = unfi.changelog.nodemap.get
     public = phases.public
     draft = phases.draft
 
     # exclude changesets already public locally and update the others
     pheads = [pn for pn in pheads if phase(unfi, rev(pn)) > public]
     if pheads:
         tr = pullop.gettransaction()
         phases.advanceboundary(pullop.repo, tr, public, pheads)
 
     # exclude changesets already draft locally and update the others
     dheads = [pn for pn in dheads if phase(unfi, rev(pn)) > draft]
     if dheads:
         tr = pullop.gettransaction()
         phases.advanceboundary(pullop.repo, tr, draft, dheads)
 
 def _pullbookmarks(pullop):
     """process the remote bookmark information to update the local one"""
     if 'bookmarks' in pullop.stepsdone:
         return
     pullop.stepsdone.add('bookmarks')
     repo = pullop.repo
     remotebookmarks = pullop.remotebookmarks
     bookmod.updatefromremote(repo.ui, repo, remotebookmarks,
                              pullop.remote.url(),
                              pullop.gettransaction,
                              explicit=pullop.explicitbookmarks)
 
 def _pullobsolete(pullop):
     """utility function to pull obsolete markers from a remote
 
     The `gettransaction` is function that return the pull transaction, creating
     one if necessary. We return the transaction to inform the calling code that
     a new transaction have been created (when applicable).
 
     Exists mostly to allow overriding for experimentation purpose"""
     if 'obsmarkers' in pullop.stepsdone:
         return
     pullop.stepsdone.add('obsmarkers')
     tr = None
     if obsolete.isenabled(pullop.repo, obsolete.exchangeopt):
         pullop.repo.ui.debug('fetching remote obsolete markers\n')
         remoteobs = pullop.remote.listkeys('obsolete')
         if 'dump0' in remoteobs:
             tr = pullop.gettransaction()
             markers = []
             for key in sorted(remoteobs, reverse=True):
                 if key.startswith('dump'):
                     data = util.b85decode(remoteobs[key])
                     version, newmarks = obsolete._readmarkers(data)
                     markers += newmarks
             if markers:
                 pullop.repo.obsstore.add(tr, markers)
             pullop.repo.invalidatevolatilesets()
     return tr
 
 def caps20to10(repo, role):
     """return a set with appropriate options to use bundle20 during getbundle"""
     caps = {'HG20'}
     capsblob = bundle2.encodecaps(bundle2.getrepocaps(repo, role=role))
     caps.add('bundle2=' + urlreq.quote(capsblob))
     return caps
 
 # List of names of steps to perform for a bundle2 for getbundle, order matters.
 getbundle2partsorder = []
 
 # Mapping between step name and function
 #
 # This exists to help extensions wrap steps if necessary
 getbundle2partsmapping = {}
 
 def getbundle2partsgenerator(stepname, idx=None):
     """decorator for function generating bundle2 part for getbundle
 
     The function is added to the step -> function mapping and appended to the
     list of steps.  Beware that decorated functions will be added in order
     (this may matter).
 
     You can only use this decorator for new steps, if you want to wrap a step
     from an extension, attack the getbundle2partsmapping dictionary directly."""
     def dec(func):
         assert stepname not in getbundle2partsmapping
         getbundle2partsmapping[stepname] = func
         if idx is None:
             getbundle2partsorder.append(stepname)
         else:
             getbundle2partsorder.insert(idx, stepname)
         return func
     return dec
 
 def bundle2requested(bundlecaps):
     if bundlecaps is not None:
         return any(cap.startswith('HG2') for cap in bundlecaps)
     return False
 
 def getbundlechunks(repo, source, heads=None, common=None, bundlecaps=None,
                     **kwargs):
     """Return chunks constituting a bundle's raw data.
 
     Could be a bundle HG10 or a bundle HG20 depending on bundlecaps
     passed.
 
     Returns a 2-tuple of a dict with metadata about the generated bundle
     and an iterator over raw chunks (of varying sizes).
     """
     kwargs = pycompat.byteskwargs(kwargs)
     info = {}
     usebundle2 = bundle2requested(bundlecaps)
     # bundle10 case
     if not usebundle2:
         if bundlecaps and not kwargs.get('cg', True):
             raise ValueError(_('request for bundle10 must include changegroup'))
 
         if kwargs:
             raise ValueError(_('unsupported getbundle arguments: %s')
                              % ', '.join(sorted(kwargs.keys())))
         outgoing = _computeoutgoing(repo, heads, common)
         info['bundleversion'] = 1
         return info, changegroup.makestream(repo, outgoing, '01', source,
                                             bundlecaps=bundlecaps)
 
     # bundle20 case
     info['bundleversion'] = 2
     b2caps = {}
     for bcaps in bundlecaps:
         if bcaps.startswith('bundle2='):
             blob = urlreq.unquote(bcaps[len('bundle2='):])
             b2caps.update(bundle2.decodecaps(blob))
     bundler = bundle2.bundle20(repo.ui, b2caps)
 
     kwargs['heads'] = heads
     kwargs['common'] = common
 
     for name in getbundle2partsorder:
         func = getbundle2partsmapping[name]
         func(bundler, repo, source, bundlecaps=bundlecaps, b2caps=b2caps,
              **pycompat.strkwargs(kwargs))
 
     info['prefercompressed'] = bundler.prefercompressed
 
     return info, bundler.getchunks()
 
 @getbundle2partsgenerator('stream2')
 def _getbundlestream2(bundler, repo, source, bundlecaps=None,
                       b2caps=None, heads=None, common=None, **kwargs):
     if not kwargs.get('stream', False):
         return
 
     if not streamclone.allowservergeneration(repo):
         raise error.Abort(_('stream data requested but server does not allow '
                             'this feature'),
                           hint=_('well-behaved clients should not be '
                                  'requesting stream data from servers not '
                                  'advertising it; the client may be buggy'))
 
     # Stream clones don't compress well. And compression undermines a
     # goal of stream clones, which is to be fast. Communicate the desire
     # to avoid compression to consumers of the bundle.
     bundler.prefercompressed = False
 
     filecount, bytecount, it = streamclone.generatev2(repo)
     requirements = _formatrequirementsspec(repo.requirements)
     part = bundler.newpart('stream2', data=it)
     part.addparam('bytecount', '%d' % bytecount, mandatory=True)
     part.addparam('filecount', '%d' % filecount, mandatory=True)
     part.addparam('requirements', requirements, mandatory=True)
 
 @getbundle2partsgenerator('changegroup')
 def _getbundlechangegrouppart(bundler, repo, source, bundlecaps=None,
                               b2caps=None, heads=None, common=None, **kwargs):
     """add a changegroup part to the requested bundle"""
     cgstream = None
     if kwargs.get(r'cg', True):
         # build changegroup bundle here.
         version = '01'
         cgversions = b2caps.get('changegroup')
         if cgversions:  # 3.1 and 3.2 ship with an empty value
             cgversions = [v for v in cgversions
                           if v in changegroup.supportedoutgoingversions(repo)]
             if not cgversions:
                 raise ValueError(_('no common changegroup version'))
             version = max(cgversions)
         outgoing = _computeoutgoing(repo, heads, common)
         if outgoing.missing:
             cgstream = changegroup.makestream(repo, outgoing, version, source,
                                               bundlecaps=bundlecaps)
 
     if cgstream:
         part = bundler.newpart('changegroup', data=cgstream)
         if cgversions:
             part.addparam('version', version)
         part.addparam('nbchanges', '%d' % len(outgoing.missing),
                       mandatory=False)
         if 'treemanifest' in repo.requirements:
             part.addparam('treemanifest', '1')
 
 @getbundle2partsgenerator('bookmarks')
 def _getbundlebookmarkpart(bundler, repo, source, bundlecaps=None,
                               b2caps=None, **kwargs):
     """add a bookmark part to the requested bundle"""
     if not kwargs.get(r'bookmarks', False):
         return
     if 'bookmarks' not in b2caps:
         raise ValueError(_('no common bookmarks exchange method'))
     books  = bookmod.listbinbookmarks(repo)
     data = bookmod.binaryencode(books)
     if data:
         bundler.newpart('bookmarks', data=data)
 
 @getbundle2partsgenerator('listkeys')
 def _getbundlelistkeysparts(bundler, repo, source, bundlecaps=None,
                             b2caps=None, **kwargs):
     """add parts containing listkeys namespaces to the requested bundle"""
     listkeys = kwargs.get(r'listkeys', ())
     for namespace in listkeys:
         part = bundler.newpart('listkeys')
         part.addparam('namespace', namespace)
         keys = repo.listkeys(namespace).items()
         part.data = pushkey.encodekeys(keys)
 
 @getbundle2partsgenerator('obsmarkers')
 def _getbundleobsmarkerpart(bundler, repo, source, bundlecaps=None,
                             b2caps=None, heads=None, **kwargs):
     """add an obsolescence markers part to the requested bundle"""
     if kwargs.get(r'obsmarkers', False):
         if heads is None:
             heads = repo.heads()
         subset = [c.node() for c in repo.set('::%ln', heads)]
         markers = repo.obsstore.relevantmarkers(subset)
         markers = sorted(markers)
         bundle2.buildobsmarkerspart(bundler, markers)
 
 @getbundle2partsgenerator('phases')
 def _getbundlephasespart(bundler, repo, source, bundlecaps=None,
                             b2caps=None, heads=None, **kwargs):
     """add phase heads part to the requested bundle"""
     if kwargs.get(r'phases', False):
         if not 'heads' in b2caps.get('phases'):
             raise ValueError(_('no common phases exchange method'))
         if heads is None:
             heads = repo.heads()
 
         headsbyphase = collections.defaultdict(set)
         if repo.publishing():
             headsbyphase[phases.public] = heads
         else:
             # find the appropriate heads to move
 
             phase = repo._phasecache.phase
             node = repo.changelog.node
             rev = repo.changelog.rev
             for h in heads:
                 headsbyphase[phase(repo, rev(h))].add(h)
             seenphases = list(headsbyphase.keys())
 
             # We do not handle anything but public and draft phase for now)
             if seenphases:
                 assert max(seenphases) <= phases.draft
 
             # if client is pulling non-public changesets, we need to find
             # intermediate public heads.
             draftheads = headsbyphase.get(phases.draft, set())
             if draftheads:
                 publicheads = headsbyphase.get(phases.public, set())
 
                 revset = 'heads(only(%ln, %ln) and public())'
                 extraheads = repo.revs(revset, draftheads, publicheads)
                 for r in extraheads:
                     headsbyphase[phases.public].add(node(r))
 
         # transform data in a format used by the encoding function
         phasemapping = []
         for phase in phases.allphases:
             phasemapping.append(sorted(headsbyphase[phase]))
 
         # generate the actual part
         phasedata = phases.binaryencode(phasemapping)
         bundler.newpart('phase-heads', data=phasedata)
 
 @getbundle2partsgenerator('hgtagsfnodes')
 def _getbundletagsfnodes(bundler, repo, source, bundlecaps=None,
                          b2caps=None, heads=None, common=None,
                          **kwargs):
     """Transfer the .hgtags filenodes mapping.
 
     Only values for heads in this bundle will be transferred.
 
     The part data consists of pairs of 20 byte changeset node and .hgtags
     filenodes raw values.
     """
     # Don't send unless:
     # - changeset are being exchanged,
     # - the client supports it.
     if not (kwargs.get(r'cg', True) and 'hgtagsfnodes' in b2caps):
         return
 
     outgoing = _computeoutgoing(repo, heads, common)
     bundle2.addparttagsfnodescache(repo, bundler, outgoing)
 
 def check_heads(repo, their_heads, context):
     """check if the heads of a repo have been modified
 
     Used by peer for unbundling.
     """
     heads = repo.heads()
     heads_hash = hashlib.sha1(''.join(sorted(heads))).digest()
     if not (their_heads == ['force'] or their_heads == heads or
             their_heads == ['hashed', heads_hash]):
         # someone else committed/pushed/unbundled while we
         # were transferring data
         raise error.PushRaced('repository changed while %s - '
                               'please try again' % context)
 
 def unbundle(repo, cg, heads, source, url):
     """Apply a bundle to a repo.
 
     this function makes sure the repo is locked during the application and have
     mechanism to check that no push race occurred between the creation of the
     bundle and its application.
 
     If the push was raced as PushRaced exception is raised."""
     r = 0
     # need a transaction when processing a bundle2 stream
     # [wlock, lock, tr] - needs to be an array so nested functions can modify it
     lockandtr = [None, None, None]
     recordout = None
     # quick fix for output mismatch with bundle2 in 3.4
     captureoutput = repo.ui.configbool('experimental', 'bundle2-output-capture')
     if url.startswith('remote:http:') or url.startswith('remote:https:'):
         captureoutput = True
     try:
         # note: outside bundle1, 'heads' is expected to be empty and this
         # 'check_heads' call wil be a no-op
         check_heads(repo, heads, 'uploading changes')
         # push can proceed
         if not isinstance(cg, bundle2.unbundle20):
             # legacy case: bundle1 (changegroup 01)
             txnname = "\n".join([source, util.hidepassword(url)])
             with repo.lock(), repo.transaction(txnname) as tr:
                 op = bundle2.applybundle(repo, cg, tr, source, url)
                 r = bundle2.combinechangegroupresults(op)
         else:
             r = None
             try:
                 def gettransaction():
                     if not lockandtr[2]:
                         lockandtr[0] = repo.wlock()
                         lockandtr[1] = repo.lock()
                         lockandtr[2] = repo.transaction(source)
                         lockandtr[2].hookargs['source'] = source
                         lockandtr[2].hookargs['url'] = url
                         lockandtr[2].hookargs['bundle2'] = '1'
                     return lockandtr[2]
 
                 # Do greedy locking by default until we're satisfied with lazy
                 # locking.
                 if not repo.ui.configbool('experimental', 'bundle2lazylocking'):
                     gettransaction()
 
                 op = bundle2.bundleoperation(repo, gettransaction,
                                              captureoutput=captureoutput)
                 try:
                     op = bundle2.processbundle(repo, cg, op=op)
                 finally:
                     r = op.reply
                     if captureoutput and r is not None:
                         repo.ui.pushbuffer(error=True, subproc=True)
                         def recordout(output):
                             r.newpart('output', data=output, mandatory=False)
                 if lockandtr[2] is not None:
                     lockandtr[2].close()
             except BaseException as exc:
                 exc.duringunbundle2 = True
                 if captureoutput and r is not None:
                     parts = exc._bundle2salvagedoutput = r.salvageoutput()
                     def recordout(output):
                         part = bundle2.bundlepart('output', data=output,
                                                   mandatory=False)
                         parts.append(part)
                 raise
     finally:
         lockmod.release(lockandtr[2], lockandtr[1], lockandtr[0])
         if recordout is not None:
             recordout(repo.ui.popbuffer())
     return r
 
 def _maybeapplyclonebundle(pullop):
     """Apply a clone bundle from a remote, if possible."""
 
     repo = pullop.repo
     remote = pullop.remote
 
     if not repo.ui.configbool('ui', 'clonebundles'):
         return
 
     # Only run if local repo is empty.
     if len(repo):
         return
 
     if pullop.heads:
         return
 
     if not remote.capable('clonebundles'):
         return
 
     res = remote._call('clonebundles')
 
     # If we call the wire protocol command, that's good enough to record the
     # attempt.
     pullop.clonebundleattempted = True
 
     entries = parseclonebundlesmanifest(repo, res)
     if not entries:
         repo.ui.note(_('no clone bundles available on remote; '
                        'falling back to regular clone\n'))
         return
 
     entries = filterclonebundleentries(
         repo, entries, streamclonerequested=pullop.streamclonerequested)
 
     if not entries:
         # There is a thundering herd concern here. However, if a server
         # operator doesn't advertise bundles appropriate for its clients,
         # they deserve what's coming. Furthermore, from a client's
         # perspective, no automatic fallback would mean not being able to
         # clone!
         repo.ui.warn(_('no compatible clone bundles available on server; '
                        'falling back to regular clone\n'))
         repo.ui.warn(_('(you may want to report this to the server '
                        'operator)\n'))
         return
 
     entries = sortclonebundleentries(repo.ui, entries)
 
     url = entries[0]['URL']
     repo.ui.status(_('applying clone bundle from %s\n') % url)
     if trypullbundlefromurl(repo.ui, repo, url):
         repo.ui.status(_('finished applying clone bundle\n'))
     # Bundle failed.
     #
     # We abort by default to avoid the thundering herd of
     # clients flooding a server that was expecting expensive
     # clone load to be offloaded.
     elif repo.ui.configbool('ui', 'clonebundlefallback'):
         repo.ui.warn(_('falling back to normal clone\n'))
     else:
         raise error.Abort(_('error applying bundle'),
                           hint=_('if this error persists, consider contacting '
                                  'the server operator or disable clone '
                                  'bundles via '
                                  '"--config ui.clonebundles=false"'))
 
 def parseclonebundlesmanifest(repo, s):
     """Parses the raw text of a clone bundles manifest.
 
     Returns a list of dicts. The dicts have a ``URL`` key corresponding
     to the URL and other keys are the attributes for the entry.
     """
     m = []
     for line in s.splitlines():
         fields = line.split()
         if not fields:
             continue
         attrs = {'URL': fields[0]}
         for rawattr in fields[1:]:
             key, value = rawattr.split('=', 1)
             key = urlreq.unquote(key)
             value = urlreq.unquote(value)
             attrs[key] = value
 
             # Parse BUNDLESPEC into components. This makes client-side
             # preferences easier to specify since you can prefer a single
             # component of the BUNDLESPEC.
             if key == 'BUNDLESPEC':
                 try:
                     comp, version, params = parsebundlespec(repo, value,
                                                             externalnames=True)
                     attrs['COMPRESSION'] = comp
                     attrs['VERSION'] = version
                 except error.InvalidBundleSpecification:
                     pass
                 except error.UnsupportedBundleSpecification:
                     pass
 
         m.append(attrs)
 
     return m
 
 def filterclonebundleentries(repo, entries, streamclonerequested=False):
     """Remove incompatible clone bundle manifest entries.
 
     Accepts a list of entries parsed with ``parseclonebundlesmanifest``
     and returns a new list consisting of only the entries that this client
     should be able to apply.
 
     There is no guarantee we'll be able to apply all returned entries because
     the metadata we use to filter on may be missing or wrong.
     """
     newentries = []
     for entry in entries:
         spec = entry.get('BUNDLESPEC')
         if spec:
             try:
                 comp, version, params = parsebundlespec(repo, spec, strict=True)
 
                 # If a stream clone was requested, filter out non-streamclone
                 # entries.
                 if streamclonerequested and (comp != 'UN' or version != 's1'):
                     repo.ui.debug('filtering %s because not a stream clone\n' %
                                   entry['URL'])
                     continue
 
             except error.InvalidBundleSpecification as e:
                 repo.ui.debug(str(e) + '\n')
                 continue
             except error.UnsupportedBundleSpecification as e:
                 repo.ui.debug('filtering %s because unsupported bundle '
                               'spec: %s\n' % (
                                   entry['URL'], util.forcebytestr(e)))
                 continue
         # If we don't have a spec and requested a stream clone, we don't know
         # what the entry is so don't attempt to apply it.
         elif streamclonerequested:
             repo.ui.debug('filtering %s because cannot determine if a stream '
                           'clone bundle\n' % entry['URL'])
             continue
 
         if 'REQUIRESNI' in entry and not sslutil.hassni:
             repo.ui.debug('filtering %s because SNI not supported\n' %
                           entry['URL'])
             continue
 
         newentries.append(entry)
 
     return newentries
 
 class clonebundleentry(object):
     """Represents an item in a clone bundles manifest.
 
     This rich class is needed to support sorting since sorted() in Python 3
     doesn't support ``cmp`` and our comparison is complex enough that ``key=``
     won't work.
     """
 
     def __init__(self, value, prefers):
         self.value = value
         self.prefers = prefers
 
     def _cmp(self, other):
         for prefkey, prefvalue in self.prefers:
             avalue = self.value.get(prefkey)
             bvalue = other.value.get(prefkey)
 
             # Special case for b missing attribute and a matches exactly.
             if avalue is not None and bvalue is None and avalue == prefvalue:
                 return -1
 
             # Special case for a missing attribute and b matches exactly.
             if bvalue is not None and avalue is None and bvalue == prefvalue:
                 return 1
 
             # We can't compare unless attribute present on both.
             if avalue is None or bvalue is None:
                 continue
 
             # Same values should fall back to next attribute.
             if avalue == bvalue:
                 continue
 
             # Exact matches come first.
             if avalue == prefvalue:
                 return -1
             if bvalue == prefvalue:
                 return 1
 
             # Fall back to next attribute.
             continue
 
         # If we got here we couldn't sort by attributes and prefers. Fall
         # back to index order.
         return 0
 
     def __lt__(self, other):
         return self._cmp(other) < 0
 
     def __gt__(self, other):
         return self._cmp(other) > 0
 
     def __eq__(self, other):
         return self._cmp(other) == 0
 
     def __le__(self, other):
         return self._cmp(other) <= 0
 
     def __ge__(self, other):
         return self._cmp(other) >= 0
 
     def __ne__(self, other):
         return self._cmp(other) != 0
 
 def sortclonebundleentries(ui, entries):
     prefers = ui.configlist('ui', 'clonebundleprefers')
     if not prefers:
         return list(entries)
 
     prefers = [p.split('=', 1) for p in prefers]
 
     items = sorted(clonebundleentry(v, prefers) for v in entries)
     return [i.value for i in items]
 
 def trypullbundlefromurl(ui, repo, url):
     """Attempt to apply a bundle from a URL."""
     with repo.lock(), repo.transaction('bundleurl') as tr:
         try:
             fh = urlmod.open(ui, url)
             cg = readbundle(ui, fh, 'stream')
 
             if isinstance(cg, streamclone.streamcloneapplier):
                 cg.apply(repo)
             else:
                 bundle2.applybundle(repo, cg, tr, 'clonebundles', url)
             return True
         except urlerr.httperror as e:
             ui.warn(_('HTTP error fetching bundle: %s\n') %
                     util.forcebytestr(e))
         except urlerr.urlerror as e:
             ui.warn(_('error fetching bundle: %s\n') % e.reason)
 
         return False

mercurial/obsolete.py

@@ -1,1041 +1,1041 b''
 # obsolete.py - obsolete markers handling
 #
 # Copyright 2012 Pierre-Yves David <[e-mail]>
 #                Logilab SA        <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """Obsolete marker handling
 
 An obsolete marker maps an old changeset to a list of new
 changesets. If the list of new changesets is empty, the old changeset
 is said to be "killed". Otherwise, the old changeset is being
 "replaced" by the new changesets.
 
 Obsolete markers can be used to record and distribute changeset graph
 transformations performed by history rewrite operations, and help
 building new tools to reconcile conflicting rewrite actions. To
 facilitate conflict resolution, markers include various annotations
 besides old and news changeset identifiers, such as creation date or
 author name.
 
 The old obsoleted changeset is called a "predecessor" and possible
 replacements are called "successors". Markers that used changeset X as
 a predecessor are called "successor markers of X" because they hold
 information about the successors of X. Markers that use changeset Y as
 a successors are call "predecessor markers of Y" because they hold
 information about the predecessors of Y.
 
 Examples:
 
 - When changeset A is replaced by changeset A', one marker is stored:
 
     (A, (A',))
 
 - When changesets A and B are folded into a new changeset C, two markers are
   stored:
 
     (A, (C,)) and (B, (C,))
 
 - When changeset A is simply "pruned" from the graph, a marker is created:
 
     (A, ())
 
 - When changeset A is split into B and C, a single marker is used:
 
     (A, (B, C))
 
   We use a single marker to distinguish the "split" case from the "divergence"
   case. If two independent operations rewrite the same changeset A in to A' and
   A'', we have an error case: divergent rewriting. We can detect it because
   two markers will be created independently:
 
   (A, (B,)) and (A, (C,))
 
 Format
 ------
 
 Markers are stored in an append-only file stored in
 '.hg/store/obsstore'.
 
 The file starts with a version header:
 
 - 1 unsigned byte: version number, starting at zero.
 
 The header is followed by the markers. Marker format depend of the version. See
 comment associated with each format for details.
 
 """
 from __future__ import absolute_import
 
 import errno
 import struct
 
 from .i18n import _
 from . import (
     error,
     node,
     obsutil,
     phases,
     policy,
     util,
 )
 
 parsers = policy.importmod(r'parsers')
 
 _pack = struct.pack
 _unpack = struct.unpack
 _calcsize = struct.calcsize
 propertycache = util.propertycache
 
 # the obsolete feature is not mature enough to be enabled by default.
 # you have to rely on third party extension extension to enable this.
 _enabled = False
 
 # Options for obsolescence
 createmarkersopt = 'createmarkers'
 allowunstableopt = 'allowunstable'
 exchangeopt = 'exchange'
 
 def _getoptionvalue(repo, option):
     """Returns True if the given repository has the given obsolete option
     enabled.
     """
     configkey = 'evolution.%s' % option
     newconfig = repo.ui.configbool('experimental', configkey)
 
     # Return the value only if defined
     if newconfig is not None:
         return newconfig
 
     # Fallback on generic option
     try:
         return repo.ui.configbool('experimental', 'evolution')
     except (error.ConfigError, AttributeError):
         # Fallback on old-fashion config
         # inconsistent config: experimental.evolution
         result = set(repo.ui.configlist('experimental', 'evolution'))
 
         if 'all' in result:
             return True
 
         # For migration purposes, temporarily return true if the config hasn't
         # been set but _enabled is true.
         if len(result) == 0 and _enabled:
             return True
 
         # Temporary hack for next check
         newconfig = repo.ui.config('experimental', 'evolution.createmarkers')
         if newconfig:
             result.add('createmarkers')
 
         return option in result
 
 def isenabled(repo, option):
     """Returns True if the given repository has the given obsolete option
     enabled.
     """
     createmarkersvalue = _getoptionvalue(repo, createmarkersopt)
     unstabluevalue = _getoptionvalue(repo, allowunstableopt)
     exchangevalue = _getoptionvalue(repo, exchangeopt)
 
     # createmarkers must be enabled if other options are enabled
     if ((unstabluevalue or exchangevalue) and not createmarkersvalue):
         raise error.Abort(_("'createmarkers' obsolete option must be enabled "
                             "if other obsolete options are enabled"))
 
     return _getoptionvalue(repo, option)
 
 ### obsolescence marker flag
 
 ## bumpedfix flag
 #
 # When a changeset A' succeed to a changeset A which became public, we call A'
 # "bumped" because it's a successors of a public changesets
 #
 # o    A' (bumped)
 # |`:
 # | o  A
 # |/
 # o    Z
 #
 # The way to solve this situation is to create a new changeset Ad as children
 # of A. This changeset have the same content than A'. So the diff from A to A'
 # is the same than the diff from A to Ad. Ad is marked as a successors of A'
 #
 # o   Ad
 # |`:
 # | x A'
 # |'|
 # o | A
 # |/
 # o Z
 #
 # But by transitivity Ad is also a successors of A. To avoid having Ad marked
 # as bumped too, we add the `bumpedfix` flag to the marker. <A', (Ad,)>.
 # This flag mean that the successors express the changes between the public and
 # bumped version and fix the situation, breaking the transitivity of
 # "bumped" here.
 bumpedfix = 1
 usingsha256 = 2
 
 ## Parsing and writing of version "0"
 #
 # The header is followed by the markers. Each marker is made of:
 #
 # - 1 uint8 : number of new changesets "N", can be zero.
 #
 # - 1 uint32: metadata size "M" in bytes.
 #
 # - 1 byte: a bit field. It is reserved for flags used in common
 #   obsolete marker operations, to avoid repeated decoding of metadata
 #   entries.
 #
 # - 20 bytes: obsoleted changeset identifier.
 #
 # - N*20 bytes: new changesets identifiers.
 #
 # - M bytes: metadata as a sequence of nul-terminated strings. Each
 #   string contains a key and a value, separated by a colon ':', without
 #   additional encoding. Keys cannot contain '\0' or ':' and values
 #   cannot contain '\0'.
 _fm0version = 0
 _fm0fixed   = '>BIB20s'
 _fm0node = '20s'
 _fm0fsize = _calcsize(_fm0fixed)
 _fm0fnodesize = _calcsize(_fm0node)
 
 def _fm0readmarkers(data, off, stop):
     # Loop on markers
     while off < stop:
         # read fixed part
         cur = data[off:off + _fm0fsize]
         off += _fm0fsize
         numsuc, mdsize, flags, pre = _unpack(_fm0fixed, cur)
         # read replacement
         sucs = ()
         if numsuc:
             s = (_fm0fnodesize * numsuc)
             cur = data[off:off + s]
             sucs = _unpack(_fm0node * numsuc, cur)
             off += s
         # read metadata
         # (metadata will be decoded on demand)
         metadata = data[off:off + mdsize]
         if len(metadata) != mdsize:
             raise error.Abort(_('parsing obsolete marker: metadata is too '
                                'short, %d bytes expected, got %d')
                              % (mdsize, len(metadata)))
         off += mdsize
         metadata = _fm0decodemeta(metadata)
         try:
             when, offset = metadata.pop('date', '0 0').split(' ')
             date = float(when), int(offset)
         except ValueError:
             date = (0., 0)
         parents = None
         if 'p2' in metadata:
             parents = (metadata.pop('p1', None), metadata.pop('p2', None))
         elif 'p1' in metadata:
             parents = (metadata.pop('p1', None),)
         elif 'p0' in metadata:
             parents = ()
         if parents is not None:
             try:
                 parents = tuple(node.bin(p) for p in parents)
                 # if parent content is not a nodeid, drop the data
                 for p in parents:
                     if len(p) != 20:
                         parents = None
                         break
             except TypeError:
                 # if content cannot be translated to nodeid drop the data.
                 parents = None
 
         metadata = tuple(sorted(metadata.iteritems()))
 
         yield (pre, sucs, flags, metadata, date, parents)
 
 def _fm0encodeonemarker(marker):
     pre, sucs, flags, metadata, date, parents = marker
     if flags & usingsha256:
         raise error.Abort(_('cannot handle sha256 with old obsstore format'))
     metadata = dict(metadata)
     time, tz = date
     metadata['date'] = '%r %i' % (time, tz)
     if parents is not None:
         if not parents:
             # mark that we explicitly recorded no parents
             metadata['p0'] = ''
         for i, p in enumerate(parents, 1):
             metadata['p%i' % i] = node.hex(p)
     metadata = _fm0encodemeta(metadata)
     numsuc = len(sucs)
     format = _fm0fixed + (_fm0node * numsuc)
     data = [numsuc, len(metadata), flags, pre]
     data.extend(sucs)
     return _pack(format, *data) + metadata
 
 def _fm0encodemeta(meta):
     """Return encoded metadata string to string mapping.
 
     Assume no ':' in key and no '\0' in both key and value."""
     for key, value in meta.iteritems():
         if ':' in key or '\0' in key:
             raise ValueError("':' and '\0' are forbidden in metadata key'")
         if '\0' in value:
             raise ValueError("':' is forbidden in metadata value'")
     return '\0'.join(['%s:%s' % (k, meta[k]) for k in sorted(meta)])
 
 def _fm0decodemeta(data):
     """Return string to string dictionary from encoded version."""
     d = {}
     for l in data.split('\0'):
         if l:
             key, value = l.split(':')
             d[key] = value
     return d
 
 ## Parsing and writing of version "1"
 #
 # The header is followed by the markers. Each marker is made of:
 #
 # - uint32: total size of the marker (including this field)
 #
 # - float64: date in seconds since epoch
 #
 # - int16: timezone offset in minutes
 #
 # - uint16: a bit field. It is reserved for flags used in common
 #   obsolete marker operations, to avoid repeated decoding of metadata
 #   entries.
 #
 # - uint8: number of successors "N", can be zero.
 #
 # - uint8: number of parents "P", can be zero.
 #
 #     0: parents data stored but no parent,
 #     1: one parent stored,
 #     2: two parents stored,
 #     3: no parent data stored
 #
 # - uint8: number of metadata entries M
 #
 # - 20 or 32 bytes: predecessor changeset identifier.
 #
 # - N*(20 or 32) bytes: successors changesets identifiers.
 #
 # - P*(20 or 32) bytes: parents of the predecessors changesets.
 #
 # - M*(uint8, uint8): size of all metadata entries (key and value)
 #
 # - remaining bytes: the metadata, each (key, value) pair after the other.
 _fm1version = 1
 _fm1fixed = '>IdhHBBB20s'
 _fm1nodesha1 = '20s'
 _fm1nodesha256 = '32s'
 _fm1nodesha1size = _calcsize(_fm1nodesha1)
 _fm1nodesha256size = _calcsize(_fm1nodesha256)
 _fm1fsize = _calcsize(_fm1fixed)
 _fm1parentnone = 3
 _fm1parentshift = 14
 _fm1parentmask = (_fm1parentnone << _fm1parentshift)
 _fm1metapair = 'BB'
 _fm1metapairsize = _calcsize(_fm1metapair)
 
 def _fm1purereadmarkers(data, off, stop):
     # make some global constants local for performance
     noneflag = _fm1parentnone
     sha2flag = usingsha256
     sha1size = _fm1nodesha1size
     sha2size = _fm1nodesha256size
     sha1fmt = _fm1nodesha1
     sha2fmt = _fm1nodesha256
     metasize = _fm1metapairsize
     metafmt = _fm1metapair
     fsize = _fm1fsize
     unpack = _unpack
 
     # Loop on markers
     ufixed = struct.Struct(_fm1fixed).unpack
 
     while off < stop:
         # read fixed part
         o1 = off + fsize
         t, secs, tz, flags, numsuc, numpar, nummeta, prec = ufixed(data[off:o1])
 
         if flags & sha2flag:
             # FIXME: prec was read as a SHA1, needs to be amended
 
             # read 0 or more successors
             if numsuc == 1:
                 o2 = o1 + sha2size
                 sucs = (data[o1:o2],)
             else:
                 o2 = o1 + sha2size * numsuc
                 sucs = unpack(sha2fmt * numsuc, data[o1:o2])
 
             # read parents
             if numpar == noneflag:
                 o3 = o2
                 parents = None
             elif numpar == 1:
                 o3 = o2 + sha2size
                 parents = (data[o2:o3],)
             else:
                 o3 = o2 + sha2size * numpar
                 parents = unpack(sha2fmt * numpar, data[o2:o3])
         else:
             # read 0 or more successors
             if numsuc == 1:
                 o2 = o1 + sha1size
                 sucs = (data[o1:o2],)
             else:
                 o2 = o1 + sha1size * numsuc
                 sucs = unpack(sha1fmt * numsuc, data[o1:o2])
 
             # read parents
             if numpar == noneflag:
                 o3 = o2
                 parents = None
             elif numpar == 1:
                 o3 = o2 + sha1size
                 parents = (data[o2:o3],)
             else:
                 o3 = o2 + sha1size * numpar
                 parents = unpack(sha1fmt * numpar, data[o2:o3])
 
         # read metadata
         off = o3 + metasize * nummeta
         metapairsize = unpack('>' + (metafmt * nummeta), data[o3:off])
         metadata = []
         for idx in xrange(0, len(metapairsize), 2):
             o1 = off + metapairsize[idx]
             o2 = o1 + metapairsize[idx + 1]
             metadata.append((data[off:o1], data[o1:o2]))
             off = o2
 
         yield (prec, sucs, flags, tuple(metadata), (secs, tz * 60), parents)
 
 def _fm1encodeonemarker(marker):
     pre, sucs, flags, metadata, date, parents = marker
     # determine node size
     _fm1node = _fm1nodesha1
     if flags & usingsha256:
         _fm1node = _fm1nodesha256
     numsuc = len(sucs)
     numextranodes = numsuc
     if parents is None:
         numpar = _fm1parentnone
     else:
         numpar = len(parents)
         numextranodes += numpar
     formatnodes = _fm1node * numextranodes
     formatmeta = _fm1metapair * len(metadata)
     format = _fm1fixed + formatnodes + formatmeta
     # tz is stored in minutes so we divide by 60
     tz = date[1]//60
     data = [None, date[0], tz, flags, numsuc, numpar, len(metadata), pre]
     data.extend(sucs)
     if parents is not None:
         data.extend(parents)
     totalsize = _calcsize(format)
     for key, value in metadata:
         lk = len(key)
         lv = len(value)
         if lk > 255:
             msg = ('obsstore metadata key cannot be longer than 255 bytes'
                    ' (key "%s" is %u bytes)') % (key, lk)
             raise error.ProgrammingError(msg)
         if lv > 255:
             msg = ('obsstore metadata value cannot be longer than 255 bytes'
                    ' (value "%s" for key "%s" is %u bytes)') % (value, key, lv)
             raise error.ProgrammingError(msg)
         data.append(lk)
         data.append(lv)
         totalsize += lk + lv
     data[0] = totalsize
     data = [_pack(format, *data)]
     for key, value in metadata:
         data.append(key)
         data.append(value)
     return ''.join(data)
 
 def _fm1readmarkers(data, off, stop):
     native = getattr(parsers, 'fm1readmarkers', None)
     if not native:
         return _fm1purereadmarkers(data, off, stop)
     return native(data, off, stop)
 
 # mapping to read/write various marker formats
 # <version> -> (decoder, encoder)
 formats = {_fm0version: (_fm0readmarkers, _fm0encodeonemarker),
            _fm1version: (_fm1readmarkers, _fm1encodeonemarker)}
 
 def _readmarkerversion(data):
     return _unpack('>B', data[0:1])[0]
 
 @util.nogc
 def _readmarkers(data, off=None, stop=None):
     """Read and enumerate markers from raw data"""
     diskversion = _readmarkerversion(data)
     if not off:
         off = 1  # skip 1 byte version number
     if stop is None:
         stop = len(data)
     if diskversion not in formats:
         msg = _('parsing obsolete marker: unknown version %r') % diskversion
         raise error.UnknownVersion(msg, version=diskversion)
     return diskversion, formats[diskversion][0](data, off, stop)
 
 def encodeheader(version=_fm0version):
     return _pack('>B', version)
 
 def encodemarkers(markers, addheader=False, version=_fm0version):
     # Kept separate from flushmarkers(), it will be reused for
     # markers exchange.
     encodeone = formats[version][1]
     if addheader:
         yield encodeheader(version)
     for marker in markers:
         yield encodeone(marker)
 
 @util.nogc
 def _addsuccessors(successors, markers):
     for mark in markers:
         successors.setdefault(mark[0], set()).add(mark)
 
 @util.nogc
 def _addpredecessors(predecessors, markers):
     for mark in markers:
         for suc in mark[1]:
             predecessors.setdefault(suc, set()).add(mark)
 
 @util.nogc
 def _addchildren(children, markers):
     for mark in markers:
         parents = mark[5]
         if parents is not None:
             for p in parents:
                 children.setdefault(p, set()).add(mark)
 
 def _checkinvalidmarkers(markers):
     """search for marker with invalid data and raise error if needed
 
     Exist as a separated function to allow the evolve extension for a more
     subtle handling.
     """
     for mark in markers:
         if node.nullid in mark[1]:
             raise error.Abort(_('bad obsolescence marker detected: '
                                'invalid successors nullid'))
 
 class obsstore(object):
     """Store obsolete markers
 
     Markers can be accessed with two mappings:
     - predecessors[x] -> set(markers on predecessors edges of x)
     - successors[x] -> set(markers on successors edges of x)
     - children[x]   -> set(markers on predecessors edges of children(x)
     """
 
     fields = ('prec', 'succs', 'flag', 'meta', 'date', 'parents')
     # prec:    nodeid, predecessors changesets
     # succs:   tuple of nodeid, successor changesets (0-N length)
     # flag:    integer, flag field carrying modifier for the markers (see doc)
     # meta:    binary blob, encoded metadata dictionary
     # date:    (float, int) tuple, date of marker creation
     # parents: (tuple of nodeid) or None, parents of predecessors
     #          None is used when no data has been recorded
 
     def __init__(self, svfs, defaultformat=_fm1version, readonly=False):
         # caches for various obsolescence related cache
         self.caches = {}
         self.svfs = svfs
         self._defaultformat = defaultformat
         self._readonly = readonly
 
     def __iter__(self):
         return iter(self._all)
 
     def __len__(self):
         return len(self._all)
 
     def __nonzero__(self):
         if not self._cached(r'_all'):
             try:
                 return self.svfs.stat('obsstore').st_size > 1
             except OSError as inst:
                 if inst.errno != errno.ENOENT:
                     raise
                 # just build an empty _all list if no obsstore exists, which
                 # avoids further stat() syscalls
         return bool(self._all)
 
     __bool__ = __nonzero__
 
     @property
     def readonly(self):
         """True if marker creation is disabled
 
         Remove me in the future when obsolete marker is always on."""
         return self._readonly
 
     def create(self, transaction, prec, succs=(), flag=0, parents=None,
                date=None, metadata=None, ui=None):
         """obsolete: add a new obsolete marker
 
         * ensuring it is hashable
         * check mandatory metadata
         * encode metadata
 
         If you are a human writing code creating marker you want to use the
         `createmarkers` function in this module instead.
 
         return True if a new marker have been added, False if the markers
         already existed (no op).
         """
         if metadata is None:
             metadata = {}
         if date is None:
             if 'date' in metadata:
                 # as a courtesy for out-of-tree extensions
                 date = util.parsedate(metadata.pop('date'))
             elif ui is not None:
                 date = ui.configdate('devel', 'default-date')
                 if date is None:
                     date = util.makedate()
             else:
                 date = util.makedate()
         if len(prec) != 20:
             raise ValueError(prec)
         for succ in succs:
             if len(succ) != 20:
                 raise ValueError(succ)
         if prec in succs:
             raise ValueError(_('in-marker cycle with %s') % node.hex(prec))
 
         metadata = tuple(sorted(metadata.iteritems()))
 
         marker = (bytes(prec), tuple(succs), int(flag), metadata, date, parents)
         return bool(self.add(transaction, [marker]))
 
     def add(self, transaction, markers):
         """Add new markers to the store
 
         Take care of filtering duplicate.
         Return the number of new marker."""
         if self._readonly:
             raise error.Abort(_('creating obsolete markers is not enabled on '
                               'this repo'))
         known = set()
         getsuccessors = self.successors.get
         new = []
         for m in markers:
             if m not in getsuccessors(m[0], ()) and m not in known:
                 known.add(m)
                 new.append(m)
         if new:
             f = self.svfs('obsstore', 'ab')
             try:
                 offset = f.tell()
                 transaction.add('obsstore', offset)
                 # offset == 0: new file - add the version header
                 data = b''.join(encodemarkers(new, offset == 0, self._version))
                 f.write(data)
             finally:
                 # XXX: f.close() == filecache invalidation == obsstore rebuilt.
                 # call 'filecacheentry.refresh()'  here
                 f.close()
             addedmarkers = transaction.changes.get('obsmarkers')
             if addedmarkers is not None:
                 addedmarkers.update(new)
             self._addmarkers(new, data)
             # new marker *may* have changed several set. invalidate the cache.
             self.caches.clear()
         # records the number of new markers for the transaction hooks
         previous = int(transaction.hookargs.get('new_obsmarkers', '0'))
-        transaction.hookargs['new_obsmarkers'] = str(previous + len(new))
+        transaction.hookargs['new_obsmarkers'] = '%d' % (previous + len(new))
         return len(new)
 
     def mergemarkers(self, transaction, data):
         """merge a binary stream of markers inside the obsstore
 
         Returns the number of new markers added."""
         version, markers = _readmarkers(data)
         return self.add(transaction, markers)
 
     @propertycache
     def _data(self):
         return self.svfs.tryread('obsstore')
 
     @propertycache
     def _version(self):
         if len(self._data) >= 1:
             return _readmarkerversion(self._data)
         else:
             return self._defaultformat
 
     @propertycache
     def _all(self):
         data = self._data
         if not data:
             return []
         self._version, markers = _readmarkers(data)
         markers = list(markers)
         _checkinvalidmarkers(markers)
         return markers
 
     @propertycache
     def successors(self):
         successors = {}
         _addsuccessors(successors, self._all)
         return successors
 
     @propertycache
     def predecessors(self):
         predecessors = {}
         _addpredecessors(predecessors, self._all)
         return predecessors
 
     @propertycache
     def children(self):
         children = {}
         _addchildren(children, self._all)
         return children
 
     def _cached(self, attr):
         return attr in self.__dict__
 
     def _addmarkers(self, markers, rawdata):
         markers = list(markers) # to allow repeated iteration
         self._data = self._data + rawdata
         self._all.extend(markers)
         if self._cached(r'successors'):
             _addsuccessors(self.successors, markers)
         if self._cached(r'predecessors'):
             _addpredecessors(self.predecessors, markers)
         if self._cached(r'children'):
             _addchildren(self.children, markers)
         _checkinvalidmarkers(markers)
 
     def relevantmarkers(self, nodes):
         """return a set of all obsolescence markers relevant to a set of nodes.
 
         "relevant" to a set of nodes mean:
 
         - marker that use this changeset as successor
         - prune marker of direct children on this changeset
         - recursive application of the two rules on predecessors of these
           markers
 
         It is a set so you cannot rely on order."""
 
         pendingnodes = set(nodes)
         seenmarkers = set()
         seennodes = set(pendingnodes)
         precursorsmarkers = self.predecessors
         succsmarkers = self.successors
         children = self.children
         while pendingnodes:
             direct = set()
             for current in pendingnodes:
                 direct.update(precursorsmarkers.get(current, ()))
                 pruned = [m for m in children.get(current, ()) if not m[1]]
                 direct.update(pruned)
                 pruned = [m for m in succsmarkers.get(current, ()) if not m[1]]
                 direct.update(pruned)
             direct -= seenmarkers
             pendingnodes = set([m[0] for m in direct])
             seenmarkers |= direct
             pendingnodes -= seennodes
             seennodes |= pendingnodes
         return seenmarkers
 
 def makestore(ui, repo):
     """Create an obsstore instance from a repo."""
     # read default format for new obsstore.
     # developer config: format.obsstore-version
     defaultformat = ui.configint('format', 'obsstore-version')
     # rely on obsstore class default when possible.
     kwargs = {}
     if defaultformat is not None:
         kwargs[r'defaultformat'] = defaultformat
     readonly = not isenabled(repo, createmarkersopt)
     store = obsstore(repo.svfs, readonly=readonly, **kwargs)
     if store and readonly:
         ui.warn(_('obsolete feature not enabled but %i markers found!\n')
                 % len(list(store)))
     return store
 
 def commonversion(versions):
     """Return the newest version listed in both versions and our local formats.
 
     Returns None if no common version exists.
     """
     versions.sort(reverse=True)
     # search for highest version known on both side
     for v in versions:
         if v in formats:
             return v
     return None
 
 # arbitrary picked to fit into 8K limit from HTTP server
 # you have to take in account:
 # - the version header
 # - the base85 encoding
 _maxpayload = 5300
 
 def _pushkeyescape(markers):
     """encode markers into a dict suitable for pushkey exchange
 
     - binary data is base85 encoded
     - split in chunks smaller than 5300 bytes"""
     keys = {}
     parts = []
     currentlen = _maxpayload * 2  # ensure we create a new part
     for marker in markers:
         nextdata = _fm0encodeonemarker(marker)
         if (len(nextdata) + currentlen > _maxpayload):
             currentpart = []
             currentlen = 0
             parts.append(currentpart)
         currentpart.append(nextdata)
         currentlen += len(nextdata)
     for idx, part in enumerate(reversed(parts)):
         data = ''.join([_pack('>B', _fm0version)] + part)
         keys['dump%i' % idx] = util.b85encode(data)
     return keys
 
 def listmarkers(repo):
     """List markers over pushkey"""
     if not repo.obsstore:
         return {}
     return _pushkeyescape(sorted(repo.obsstore))
 
 def pushmarker(repo, key, old, new):
     """Push markers over pushkey"""
     if not key.startswith('dump'):
         repo.ui.warn(_('unknown key: %r') % key)
         return False
     if old:
         repo.ui.warn(_('unexpected old value for %r') % key)
         return False
     data = util.b85decode(new)
     with repo.lock(), repo.transaction('pushkey: obsolete markers') as tr:
         repo.obsstore.mergemarkers(tr, data)
         repo.invalidatevolatilesets()
         return True
 
 # mapping of 'set-name' -> <function to compute this set>
 cachefuncs = {}
 def cachefor(name):
     """Decorator to register a function as computing the cache for a set"""
     def decorator(func):
         if name in cachefuncs:
             msg = "duplicated registration for volatileset '%s' (existing: %r)"
             raise error.ProgrammingError(msg % (name, cachefuncs[name]))
         cachefuncs[name] = func
         return func
     return decorator
 
 def getrevs(repo, name):
     """Return the set of revision that belong to the <name> set
 
     Such access may compute the set and cache it for future use"""
     repo = repo.unfiltered()
     if not repo.obsstore:
         return frozenset()
     if name not in repo.obsstore.caches:
         repo.obsstore.caches[name] = cachefuncs[name](repo)
     return repo.obsstore.caches[name]
 
 # To be simple we need to invalidate obsolescence cache when:
 #
 # - new changeset is added:
 # - public phase is changed
 # - obsolescence marker are added
 # - strip is used a repo
 def clearobscaches(repo):
     """Remove all obsolescence related cache from a repo
 
     This remove all cache in obsstore is the obsstore already exist on the
     repo.
 
     (We could be smarter here given the exact event that trigger the cache
     clearing)"""
     # only clear cache is there is obsstore data in this repo
     if 'obsstore' in repo._filecache:
         repo.obsstore.caches.clear()
 
 def _mutablerevs(repo):
     """the set of mutable revision in the repository"""
     return repo._phasecache.getrevset(repo, (phases.draft, phases.secret))
 
 @cachefor('obsolete')
 def _computeobsoleteset(repo):
     """the set of obsolete revisions"""
     getnode = repo.changelog.node
     notpublic = _mutablerevs(repo)
     isobs = repo.obsstore.successors.__contains__
     obs = set(r for r in notpublic if isobs(getnode(r)))
     return obs
 
 @cachefor('orphan')
 def _computeorphanset(repo):
     """the set of non obsolete revisions with obsolete parents"""
     pfunc = repo.changelog.parentrevs
     mutable = _mutablerevs(repo)
     obsolete = getrevs(repo, 'obsolete')
     others = mutable - obsolete
     unstable = set()
     for r in sorted(others):
         # A rev is unstable if one of its parent is obsolete or unstable
         # this works since we traverse following growing rev order
         for p in pfunc(r):
             if p in obsolete or p in unstable:
                 unstable.add(r)
                 break
     return unstable
 
 @cachefor('suspended')
 def _computesuspendedset(repo):
     """the set of obsolete parents with non obsolete descendants"""
     suspended = repo.changelog.ancestors(getrevs(repo, 'orphan'))
     return set(r for r in getrevs(repo, 'obsolete') if r in suspended)
 
 @cachefor('extinct')
 def _computeextinctset(repo):
     """the set of obsolete parents without non obsolete descendants"""
     return getrevs(repo, 'obsolete') - getrevs(repo, 'suspended')
 
 @cachefor('phasedivergent')
 def _computephasedivergentset(repo):
     """the set of revs trying to obsolete public revisions"""
     bumped = set()
     # util function (avoid attribute lookup in the loop)
     phase = repo._phasecache.phase # would be faster to grab the full list
     public = phases.public
     cl = repo.changelog
     torev = cl.nodemap.get
     tonode = cl.node
     for rev in repo.revs('(not public()) and (not obsolete())'):
         # We only evaluate mutable, non-obsolete revision
         node = tonode(rev)
         # (future) A cache of predecessors may worth if split is very common
         for pnode in obsutil.allpredecessors(repo.obsstore, [node],
                                    ignoreflags=bumpedfix):
             prev = torev(pnode) # unfiltered! but so is phasecache
             if (prev is not None) and (phase(repo, prev) <= public):
                 # we have a public predecessor
                 bumped.add(rev)
                 break # Next draft!
     return bumped
 
 @cachefor('contentdivergent')
 def _computecontentdivergentset(repo):
     """the set of rev that compete to be the final successors of some revision.
     """
     divergent = set()
     obsstore = repo.obsstore
     newermap = {}
     tonode = repo.changelog.node
     for rev in repo.revs('(not public()) - obsolete()'):
         node = tonode(rev)
         mark = obsstore.predecessors.get(node, ())
         toprocess = set(mark)
         seen = set()
         while toprocess:
             prec = toprocess.pop()[0]
             if prec in seen:
                 continue # emergency cycle hanging prevention
             seen.add(prec)
             if prec not in newermap:
                 obsutil.successorssets(repo, prec, cache=newermap)
             newer = [n for n in newermap[prec] if n]
             if len(newer) > 1:
                 divergent.add(rev)
                 break
             toprocess.update(obsstore.predecessors.get(prec, ()))
     return divergent
 
 
 def createmarkers(repo, relations, flag=0, date=None, metadata=None,
                   operation=None):
     """Add obsolete markers between changesets in a repo
 
     <relations> must be an iterable of (<old>, (<new>, ...)[,{metadata}])
     tuple. `old` and `news` are changectx. metadata is an optional dictionary
     containing metadata for this marker only. It is merged with the global
     metadata specified through the `metadata` argument of this function,
 
     Trying to obsolete a public changeset will raise an exception.
 
     Current user and date are used except if specified otherwise in the
     metadata attribute.
 
     This function operates within a transaction of its own, but does
     not take any lock on the repo.
     """
     # prepare metadata
     if metadata is None:
         metadata = {}
     if 'user' not in metadata:
         develuser = repo.ui.config('devel', 'user.obsmarker')
         if develuser:
             metadata['user'] = develuser
         else:
             metadata['user'] = repo.ui.username()
 
     # Operation metadata handling
     useoperation = repo.ui.configbool('experimental',
         'evolution.track-operation')
     if useoperation and operation:
         metadata['operation'] = operation
 
     # Effect flag metadata handling
     saveeffectflag = repo.ui.configbool('experimental',
                                         'evolution.effect-flags')
 
     with repo.transaction('add-obsolescence-marker') as tr:
         markerargs = []
         for rel in relations:
             prec = rel[0]
             sucs = rel[1]
             localmetadata = metadata.copy()
             if 2 < len(rel):
                 localmetadata.update(rel[2])
 
             if not prec.mutable():
                 raise error.Abort(_("cannot obsolete public changeset: %s")
                                  % prec,
                                  hint="see 'hg help phases' for details")
             nprec = prec.node()
             nsucs = tuple(s.node() for s in sucs)
             npare = None
             if not nsucs:
                 npare = tuple(p.node() for p in prec.parents())
             if nprec in nsucs:
                 raise error.Abort(_("changeset %s cannot obsolete itself")
                                   % prec)
 
             # Effect flag can be different by relation
             if saveeffectflag:
                 # The effect flag is saved in a versioned field name for future
                 # evolution
                 effectflag = obsutil.geteffectflag(rel)
                 localmetadata[obsutil.EFFECTFLAGFIELD] = "%d" % effectflag
 
             # Creating the marker causes the hidden cache to become invalid,
             # which causes recomputation when we ask for prec.parents() above.
             # Resulting in n^2 behavior.  So let's prepare all of the args
             # first, then create the markers.
             markerargs.append((nprec, nsucs, npare, localmetadata))
 
         for args in markerargs:
             nprec, nsucs, npare, localmetadata = args
             repo.obsstore.create(tr, nprec, nsucs, flag, parents=npare,
                                  date=date, metadata=localmetadata,
                                  ui=repo.ui)
             repo.filteredrevcache.clear()

mercurial/patch.py

@@ -1,2903 +1,2903 b''
 # patch.py - patch file parsing routines
 #
 # Copyright 2006 Brendan Cully <[e-mail]>
 # Copyright 2007 Chris Mason <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import, print_function
 
 import collections
 import copy
 import difflib
 import email
 import errno
 import hashlib
 import os
 import posixpath
 import re
 import shutil
 import tempfile
 import zlib
 
 from .i18n import _
 from .node import (
     hex,
     short,
 )
 from . import (
     copies,
     encoding,
     error,
     mail,
     mdiff,
     pathutil,
     policy,
     pycompat,
     scmutil,
     similar,
     util,
     vfs as vfsmod,
 )
 
 diffhelpers = policy.importmod(r'diffhelpers')
 stringio = util.stringio
 
 gitre = re.compile(br'diff --git a/(.*) b/(.*)')
 tabsplitter = re.compile(br'(\t+|[^\t]+)')
 _nonwordre = re.compile(br'([^a-zA-Z0-9_\x80-\xff])')
 
 PatchError = error.PatchError
 
 # public functions
 
 def split(stream):
     '''return an iterator of individual patches from a stream'''
     def isheader(line, inheader):
         if inheader and line[0] in (' ', '\t'):
             # continuation
             return True
         if line[0] in (' ', '-', '+'):
             # diff line - don't check for header pattern in there
             return False
         l = line.split(': ', 1)
         return len(l) == 2 and ' ' not in l[0]
 
     def chunk(lines):
         return stringio(''.join(lines))
 
     def hgsplit(stream, cur):
         inheader = True
 
         for line in stream:
             if not line.strip():
                 inheader = False
             if not inheader and line.startswith('# HG changeset patch'):
                 yield chunk(cur)
                 cur = []
                 inheader = True
 
             cur.append(line)
 
         if cur:
             yield chunk(cur)
 
     def mboxsplit(stream, cur):
         for line in stream:
             if line.startswith('From '):
                 for c in split(chunk(cur[1:])):
                     yield c
                 cur = []
 
             cur.append(line)
 
         if cur:
             for c in split(chunk(cur[1:])):
                 yield c
 
     def mimesplit(stream, cur):
         def msgfp(m):
             fp = stringio()
             g = email.Generator.Generator(fp, mangle_from_=False)
             g.flatten(m)
             fp.seek(0)
             return fp
 
         for line in stream:
             cur.append(line)
         c = chunk(cur)
 
         m = pycompat.emailparser().parse(c)
         if not m.is_multipart():
             yield msgfp(m)
         else:
             ok_types = ('text/plain', 'text/x-diff', 'text/x-patch')
             for part in m.walk():
                 ct = part.get_content_type()
                 if ct not in ok_types:
                     continue
                 yield msgfp(part)
 
     def headersplit(stream, cur):
         inheader = False
 
         for line in stream:
             if not inheader and isheader(line, inheader):
                 yield chunk(cur)
                 cur = []
                 inheader = True
             if inheader and not isheader(line, inheader):
                 inheader = False
 
             cur.append(line)
 
         if cur:
             yield chunk(cur)
 
     def remainder(cur):
         yield chunk(cur)
 
     class fiter(object):
         def __init__(self, fp):
             self.fp = fp
 
         def __iter__(self):
             return self
 
         def next(self):
             l = self.fp.readline()
             if not l:
                 raise StopIteration
             return l
 
         __next__ = next
 
     inheader = False
     cur = []
 
     mimeheaders = ['content-type']
 
     if not util.safehasattr(stream, 'next'):
         # http responses, for example, have readline but not next
         stream = fiter(stream)
 
     for line in stream:
         cur.append(line)
         if line.startswith('# HG changeset patch'):
             return hgsplit(stream, cur)
         elif line.startswith('From '):
             return mboxsplit(stream, cur)
         elif isheader(line, inheader):
             inheader = True
             if line.split(':', 1)[0].lower() in mimeheaders:
                 # let email parser handle this
                 return mimesplit(stream, cur)
         elif line.startswith('--- ') and inheader:
             # No evil headers seen by diff start, split by hand
             return headersplit(stream, cur)
         # Not enough info, keep reading
 
     # if we are here, we have a very plain patch
     return remainder(cur)
 
 ## Some facility for extensible patch parsing:
 # list of pairs ("header to match", "data key")
 patchheadermap = [('Date', 'date'),
                   ('Branch', 'branch'),
                   ('Node ID', 'nodeid'),
                  ]
 
 def extract(ui, fileobj):
     '''extract patch from data read from fileobj.
 
     patch can be a normal patch or contained in an email message.
 
     return a dictionary. Standard keys are:
       - filename,
       - message,
       - user,
       - date,
       - branch,
       - node,
       - p1,
       - p2.
     Any item can be missing from the dictionary. If filename is missing,
     fileobj did not contain a patch. Caller must unlink filename when done.'''
 
     # attempt to detect the start of a patch
     # (this heuristic is borrowed from quilt)
     diffre = re.compile(br'^(?:Index:[ \t]|diff[ \t]-|RCS file: |'
                         br'retrieving revision [0-9]+(\.[0-9]+)*$|'
                         br'---[ \t].*?^\+\+\+[ \t]|'
                         br'\*\*\*[ \t].*?^---[ \t])',
                         re.MULTILINE | re.DOTALL)
 
     data = {}
     fd, tmpname = tempfile.mkstemp(prefix='hg-patch-')
     tmpfp = os.fdopen(fd, pycompat.sysstr('wb'))
     try:
         msg = pycompat.emailparser().parse(fileobj)
 
         subject = msg['Subject'] and mail.headdecode(msg['Subject'])
         data['user'] = msg['From'] and mail.headdecode(msg['From'])
         if not subject and not data['user']:
             # Not an email, restore parsed headers if any
             subject = '\n'.join(': '.join(h) for h in msg.items()) + '\n'
 
         # should try to parse msg['Date']
         parents = []
 
         if subject:
             if subject.startswith('[PATCH'):
                 pend = subject.find(']')
                 if pend >= 0:
                     subject = subject[pend + 1:].lstrip()
             subject = re.sub(br'\n[ \t]+', ' ', subject)
             ui.debug('Subject: %s\n' % subject)
         if data['user']:
             ui.debug('From: %s\n' % data['user'])
         diffs_seen = 0
         ok_types = ('text/plain', 'text/x-diff', 'text/x-patch')
         message = ''
         for part in msg.walk():
             content_type = pycompat.bytestr(part.get_content_type())
             ui.debug('Content-Type: %s\n' % content_type)
             if content_type not in ok_types:
                 continue
             payload = part.get_payload(decode=True)
             m = diffre.search(payload)
             if m:
                 hgpatch = False
                 hgpatchheader = False
                 ignoretext = False
 
                 ui.debug('found patch at byte %d\n' % m.start(0))
                 diffs_seen += 1
                 cfp = stringio()
                 for line in payload[:m.start(0)].splitlines():
                     if line.startswith('# HG changeset patch') and not hgpatch:
                         ui.debug('patch generated by hg export\n')
                         hgpatch = True
                         hgpatchheader = True
                         # drop earlier commit message content
                         cfp.seek(0)
                         cfp.truncate()
                         subject = None
                     elif hgpatchheader:
                         if line.startswith('# User '):
                             data['user'] = line[7:]
                             ui.debug('From: %s\n' % data['user'])
                         elif line.startswith("# Parent "):
                             parents.append(line[9:].lstrip())
                         elif line.startswith("# "):
                             for header, key in patchheadermap:
                                 prefix = '# %s ' % header
                                 if line.startswith(prefix):
                                     data[key] = line[len(prefix):]
                         else:
                             hgpatchheader = False
                     elif line == '---':
                         ignoretext = True
                     if not hgpatchheader and not ignoretext:
                         cfp.write(line)
                         cfp.write('\n')
                 message = cfp.getvalue()
                 if tmpfp:
                     tmpfp.write(payload)
                     if not payload.endswith('\n'):
                         tmpfp.write('\n')
             elif not diffs_seen and message and content_type == 'text/plain':
                 message += '\n' + payload
     except: # re-raises
         tmpfp.close()
         os.unlink(tmpname)
         raise
 
     if subject and not message.startswith(subject):
         message = '%s\n%s' % (subject, message)
     data['message'] = message
     tmpfp.close()
     if parents:
         data['p1'] = parents.pop(0)
         if parents:
             data['p2'] = parents.pop(0)
 
     if diffs_seen:
         data['filename'] = tmpname
     else:
         os.unlink(tmpname)
     return data
 
 class patchmeta(object):
     """Patched file metadata
 
     'op' is the performed operation within ADD, DELETE, RENAME, MODIFY
     or COPY.  'path' is patched file path. 'oldpath' is set to the
     origin file when 'op' is either COPY or RENAME, None otherwise. If
     file mode is changed, 'mode' is a tuple (islink, isexec) where
     'islink' is True if the file is a symlink and 'isexec' is True if
     the file is executable. Otherwise, 'mode' is None.
     """
     def __init__(self, path):
         self.path = path
         self.oldpath = None
         self.mode = None
         self.op = 'MODIFY'
         self.binary = False
 
     def setmode(self, mode):
         islink = mode & 0o20000
         isexec = mode & 0o100
         self.mode = (islink, isexec)
 
     def copy(self):
         other = patchmeta(self.path)
         other.oldpath = self.oldpath
         other.mode = self.mode
         other.op = self.op
         other.binary = self.binary
         return other
 
     def _ispatchinga(self, afile):
         if afile == '/dev/null':
             return self.op == 'ADD'
         return afile == 'a/' + (self.oldpath or self.path)
 
     def _ispatchingb(self, bfile):
         if bfile == '/dev/null':
             return self.op == 'DELETE'
         return bfile == 'b/' + self.path
 
     def ispatching(self, afile, bfile):
         return self._ispatchinga(afile) and self._ispatchingb(bfile)
 
     def __repr__(self):
         return "<patchmeta %s %r>" % (self.op, self.path)
 
 def readgitpatch(lr):
     """extract git-style metadata about patches from <patchname>"""
 
     # Filter patch for git information
     gp = None
     gitpatches = []
     for line in lr:
         line = line.rstrip(' \r\n')
         if line.startswith('diff --git a/'):
             m = gitre.match(line)
             if m:
                 if gp:
                     gitpatches.append(gp)
                 dst = m.group(2)
                 gp = patchmeta(dst)
         elif gp:
             if line.startswith('--- '):
                 gitpatches.append(gp)
                 gp = None
                 continue
             if line.startswith('rename from '):
                 gp.op = 'RENAME'
                 gp.oldpath = line[12:]
             elif line.startswith('rename to '):
                 gp.path = line[10:]
             elif line.startswith('copy from '):
                 gp.op = 'COPY'
                 gp.oldpath = line[10:]
             elif line.startswith('copy to '):
                 gp.path = line[8:]
             elif line.startswith('deleted file'):
                 gp.op = 'DELETE'
             elif line.startswith('new file mode '):
                 gp.op = 'ADD'
                 gp.setmode(int(line[-6:], 8))
             elif line.startswith('new mode '):
                 gp.setmode(int(line[-6:], 8))
             elif line.startswith('GIT binary patch'):
                 gp.binary = True
     if gp:
         gitpatches.append(gp)
 
     return gitpatches
 
 class linereader(object):
     # simple class to allow pushing lines back into the input stream
     def __init__(self, fp):
         self.fp = fp
         self.buf = []
 
     def push(self, line):
         if line is not None:
             self.buf.append(line)
 
     def readline(self):
         if self.buf:
             l = self.buf[0]
             del self.buf[0]
             return l
         return self.fp.readline()
 
     def __iter__(self):
         return iter(self.readline, '')
 
 class abstractbackend(object):
     def __init__(self, ui):
         self.ui = ui
 
     def getfile(self, fname):
         """Return target file data and flags as a (data, (islink,
         isexec)) tuple. Data is None if file is missing/deleted.
         """
         raise NotImplementedError
 
     def setfile(self, fname, data, mode, copysource):
         """Write data to target file fname and set its mode. mode is a
         (islink, isexec) tuple. If data is None, the file content should
         be left unchanged. If the file is modified after being copied,
         copysource is set to the original file name.
         """
         raise NotImplementedError
 
     def unlink(self, fname):
         """Unlink target file."""
         raise NotImplementedError
 
     def writerej(self, fname, failed, total, lines):
         """Write rejected lines for fname. total is the number of hunks
         which failed to apply and total the total number of hunks for this
         files.
         """
 
     def exists(self, fname):
         raise NotImplementedError
 
     def close(self):
         raise NotImplementedError
 
 class fsbackend(abstractbackend):
     def __init__(self, ui, basedir):
         super(fsbackend, self).__init__(ui)
         self.opener = vfsmod.vfs(basedir)
 
     def getfile(self, fname):
         if self.opener.islink(fname):
             return (self.opener.readlink(fname), (True, False))
 
         isexec = False
         try:
             isexec = self.opener.lstat(fname).st_mode & 0o100 != 0
         except OSError as e:
             if e.errno != errno.ENOENT:
                 raise
         try:
             return (self.opener.read(fname), (False, isexec))
         except IOError as e:
             if e.errno != errno.ENOENT:
                 raise
             return None, None
 
     def setfile(self, fname, data, mode, copysource):
         islink, isexec = mode
         if data is None:
             self.opener.setflags(fname, islink, isexec)
             return
         if islink:
             self.opener.symlink(data, fname)
         else:
             self.opener.write(fname, data)
             if isexec:
                 self.opener.setflags(fname, False, True)
 
     def unlink(self, fname):
         self.opener.unlinkpath(fname, ignoremissing=True)
 
     def writerej(self, fname, failed, total, lines):
         fname = fname + ".rej"
         self.ui.warn(
             _("%d out of %d hunks FAILED -- saving rejects to file %s\n") %
             (failed, total, fname))
         fp = self.opener(fname, 'w')
         fp.writelines(lines)
         fp.close()
 
     def exists(self, fname):
         return self.opener.lexists(fname)
 
 class workingbackend(fsbackend):
     def __init__(self, ui, repo, similarity):
         super(workingbackend, self).__init__(ui, repo.root)
         self.repo = repo
         self.similarity = similarity
         self.removed = set()
         self.changed = set()
         self.copied = []
 
     def _checkknown(self, fname):
         if self.repo.dirstate[fname] == '?' and self.exists(fname):
             raise PatchError(_('cannot patch %s: file is not tracked') % fname)
 
     def setfile(self, fname, data, mode, copysource):
         self._checkknown(fname)
         super(workingbackend, self).setfile(fname, data, mode, copysource)
         if copysource is not None:
             self.copied.append((copysource, fname))
         self.changed.add(fname)
 
     def unlink(self, fname):
         self._checkknown(fname)
         super(workingbackend, self).unlink(fname)
         self.removed.add(fname)
         self.changed.add(fname)
 
     def close(self):
         wctx = self.repo[None]
         changed = set(self.changed)
         for src, dst in self.copied:
             scmutil.dirstatecopy(self.ui, self.repo, wctx, src, dst)
         if self.removed:
             wctx.forget(sorted(self.removed))
             for f in self.removed:
                 if f not in self.repo.dirstate:
                     # File was deleted and no longer belongs to the
                     # dirstate, it was probably marked added then
                     # deleted, and should not be considered by
                     # marktouched().
                     changed.discard(f)
         if changed:
             scmutil.marktouched(self.repo, changed, self.similarity)
         return sorted(self.changed)
 
 class filestore(object):
     def __init__(self, maxsize=None):
         self.opener = None
         self.files = {}
         self.created = 0
         self.maxsize = maxsize
         if self.maxsize is None:
             self.maxsize = 4*(2**20)
         self.size = 0
         self.data = {}
 
     def setfile(self, fname, data, mode, copied=None):
         if self.maxsize < 0 or (len(data) + self.size) <= self.maxsize:
             self.data[fname] = (data, mode, copied)
             self.size += len(data)
         else:
             if self.opener is None:
                 root = tempfile.mkdtemp(prefix='hg-patch-')
                 self.opener = vfsmod.vfs(root)
             # Avoid filename issues with these simple names
-            fn = str(self.created)
+            fn = '%d' % self.created
             self.opener.write(fn, data)
             self.created += 1
             self.files[fname] = (fn, mode, copied)
 
     def getfile(self, fname):
         if fname in self.data:
             return self.data[fname]
         if not self.opener or fname not in self.files:
             return None, None, None
         fn, mode, copied = self.files[fname]
         return self.opener.read(fn), mode, copied
 
     def close(self):
         if self.opener:
             shutil.rmtree(self.opener.base)
 
 class repobackend(abstractbackend):
     def __init__(self, ui, repo, ctx, store):
         super(repobackend, self).__init__(ui)
         self.repo = repo
         self.ctx = ctx
         self.store = store
         self.changed = set()
         self.removed = set()
         self.copied = {}
 
     def _checkknown(self, fname):
         if fname not in self.ctx:
             raise PatchError(_('cannot patch %s: file is not tracked') % fname)
 
     def getfile(self, fname):
         try:
             fctx = self.ctx[fname]
         except error.LookupError:
             return None, None
         flags = fctx.flags()
         return fctx.data(), ('l' in flags, 'x' in flags)
 
     def setfile(self, fname, data, mode, copysource):
         if copysource:
             self._checkknown(copysource)
         if data is None:
             data = self.ctx[fname].data()
         self.store.setfile(fname, data, mode, copysource)
         self.changed.add(fname)
         if copysource:
             self.copied[fname] = copysource
 
     def unlink(self, fname):
         self._checkknown(fname)
         self.removed.add(fname)
 
     def exists(self, fname):
         return fname in self.ctx
 
     def close(self):
         return self.changed | self.removed
 
 # @@ -start,len +start,len @@ or @@ -start +start @@ if len is 1
 unidesc = re.compile('@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@')
 contextdesc = re.compile('(?:---|\*\*\*) (\d+)(?:,(\d+))? (?:---|\*\*\*)')
 eolmodes = ['strict', 'crlf', 'lf', 'auto']
 
 class patchfile(object):
     def __init__(self, ui, gp, backend, store, eolmode='strict'):
         self.fname = gp.path
         self.eolmode = eolmode
         self.eol = None
         self.backend = backend
         self.ui = ui
         self.lines = []
         self.exists = False
         self.missing = True
         self.mode = gp.mode
         self.copysource = gp.oldpath
         self.create = gp.op in ('ADD', 'COPY', 'RENAME')
         self.remove = gp.op == 'DELETE'
         if self.copysource is None:
             data, mode = backend.getfile(self.fname)
         else:
             data, mode = store.getfile(self.copysource)[:2]
         if data is not None:
             self.exists = self.copysource is None or backend.exists(self.fname)
             self.missing = False
             if data:
                 self.lines = mdiff.splitnewlines(data)
             if self.mode is None:
                 self.mode = mode
             if self.lines:
                 # Normalize line endings
                 if self.lines[0].endswith('\r\n'):
                     self.eol = '\r\n'
                 elif self.lines[0].endswith('\n'):
                     self.eol = '\n'
                 if eolmode != 'strict':
                     nlines = []
                     for l in self.lines:
                         if l.endswith('\r\n'):
                             l = l[:-2] + '\n'
                         nlines.append(l)
                     self.lines = nlines
         else:
             if self.create:
                 self.missing = False
             if self.mode is None:
                 self.mode = (False, False)
         if self.missing:
             self.ui.warn(_("unable to find '%s' for patching\n") % self.fname)
             self.ui.warn(_("(use '--prefix' to apply patch relative to the "
                            "current directory)\n"))
 
         self.hash = {}
         self.dirty = 0
         self.offset = 0
         self.skew = 0
         self.rej = []
         self.fileprinted = False
         self.printfile(False)
         self.hunks = 0
 
     def writelines(self, fname, lines, mode):
         if self.eolmode == 'auto':
             eol = self.eol
         elif self.eolmode == 'crlf':
             eol = '\r\n'
         else:
             eol = '\n'
 
         if self.eolmode != 'strict' and eol and eol != '\n':
             rawlines = []
             for l in lines:
                 if l and l[-1] == '\n':
                     l = l[:-1] + eol
                 rawlines.append(l)
             lines = rawlines
 
         self.backend.setfile(fname, ''.join(lines), mode, self.copysource)
 
     def printfile(self, warn):
         if self.fileprinted:
             return
         if warn or self.ui.verbose:
             self.fileprinted = True
         s = _("patching file %s\n") % self.fname
         if warn:
             self.ui.warn(s)
         else:
             self.ui.note(s)
 
 
     def findlines(self, l, linenum):
         # looks through the hash and finds candidate lines.  The
         # result is a list of line numbers sorted based on distance
         # from linenum
 
         cand = self.hash.get(l, [])
         if len(cand) > 1:
             # resort our list of potentials forward then back.
             cand.sort(key=lambda x: abs(x - linenum))
         return cand
 
     def write_rej(self):
         # our rejects are a little different from patch(1).  This always
         # creates rejects in the same form as the original patch.  A file
         # header is inserted so that you can run the reject through patch again
         # without having to type the filename.
         if not self.rej:
             return
         base = os.path.basename(self.fname)
         lines = ["--- %s\n+++ %s\n" % (base, base)]
         for x in self.rej:
             for l in x.hunk:
                 lines.append(l)
                 if l[-1:] != '\n':
                     lines.append("\n\ No newline at end of file\n")
         self.backend.writerej(self.fname, len(self.rej), self.hunks, lines)
 
     def apply(self, h):
         if not h.complete():
             raise PatchError(_("bad hunk #%d %s (%d %d %d %d)") %
                             (h.number, h.desc, len(h.a), h.lena, len(h.b),
                             h.lenb))
 
         self.hunks += 1
 
         if self.missing:
             self.rej.append(h)
             return -1
 
         if self.exists and self.create:
             if self.copysource:
                 self.ui.warn(_("cannot create %s: destination already "
                                "exists\n") % self.fname)
             else:
                 self.ui.warn(_("file %s already exists\n") % self.fname)
             self.rej.append(h)
             return -1
 
         if isinstance(h, binhunk):
             if self.remove:
                 self.backend.unlink(self.fname)
             else:
                 l = h.new(self.lines)
                 self.lines[:] = l
                 self.offset += len(l)
                 self.dirty = True
             return 0
 
         horig = h
         if (self.eolmode in ('crlf', 'lf')
             or self.eolmode == 'auto' and self.eol):
             # If new eols are going to be normalized, then normalize
             # hunk data before patching. Otherwise, preserve input
             # line-endings.
             h = h.getnormalized()
 
         # fast case first, no offsets, no fuzz
         old, oldstart, new, newstart = h.fuzzit(0, False)
         oldstart += self.offset
         orig_start = oldstart
         # if there's skew we want to emit the "(offset %d lines)" even
         # when the hunk cleanly applies at start + skew, so skip the
         # fast case code
         if (self.skew == 0 and
             diffhelpers.testhunk(old, self.lines, oldstart) == 0):
             if self.remove:
                 self.backend.unlink(self.fname)
             else:
                 self.lines[oldstart:oldstart + len(old)] = new
                 self.offset += len(new) - len(old)
                 self.dirty = True
             return 0
 
         # ok, we couldn't match the hunk. Lets look for offsets and fuzz it
         self.hash = {}
         for x, s in enumerate(self.lines):
             self.hash.setdefault(s, []).append(x)
 
         for fuzzlen in xrange(self.ui.configint("patch", "fuzz") + 1):
             for toponly in [True, False]:
                 old, oldstart, new, newstart = h.fuzzit(fuzzlen, toponly)
                 oldstart = oldstart + self.offset + self.skew
                 oldstart = min(oldstart, len(self.lines))
                 if old:
                     cand = self.findlines(old[0][1:], oldstart)
                 else:
                     # Only adding lines with no or fuzzed context, just
                     # take the skew in account
                     cand = [oldstart]
 
                 for l in cand:
                     if not old or diffhelpers.testhunk(old, self.lines, l) == 0:
                         self.lines[l : l + len(old)] = new
                         self.offset += len(new) - len(old)
                         self.skew = l - orig_start
                         self.dirty = True
                         offset = l - orig_start - fuzzlen
                         if fuzzlen:
                             msg = _("Hunk #%d succeeded at %d "
                                     "with fuzz %d "
                                     "(offset %d lines).\n")
                             self.printfile(True)
                             self.ui.warn(msg %
                                 (h.number, l + 1, fuzzlen, offset))
                         else:
                             msg = _("Hunk #%d succeeded at %d "
                                     "(offset %d lines).\n")
                             self.ui.note(msg % (h.number, l + 1, offset))
                         return fuzzlen
         self.printfile(True)
         self.ui.warn(_("Hunk #%d FAILED at %d\n") % (h.number, orig_start))
         self.rej.append(horig)
         return -1
 
     def close(self):
         if self.dirty:
             self.writelines(self.fname, self.lines, self.mode)
         self.write_rej()
         return len(self.rej)
 
 class header(object):
     """patch header
     """
     diffgit_re = re.compile('diff --git a/(.*) b/(.*)$')
     diff_re = re.compile('diff -r .* (.*)$')
     allhunks_re = re.compile('(?:index|deleted file) ')
     pretty_re = re.compile('(?:new file|deleted file) ')
     special_re = re.compile('(?:index|deleted|copy|rename) ')
     newfile_re = re.compile('(?:new file)')
 
     def __init__(self, header):
         self.header = header
         self.hunks = []
 
     def binary(self):
         return any(h.startswith('index ') for h in self.header)
 
     def pretty(self, fp):
         for h in self.header:
             if h.startswith('index '):
                 fp.write(_('this modifies a binary file (all or nothing)\n'))
                 break
             if self.pretty_re.match(h):
                 fp.write(h)
                 if self.binary():
                     fp.write(_('this is a binary file\n'))
                 break
             if h.startswith('---'):
                 fp.write(_('%d hunks, %d lines changed\n') %
                          (len(self.hunks),
                           sum([max(h.added, h.removed) for h in self.hunks])))
                 break
             fp.write(h)
 
     def write(self, fp):
         fp.write(''.join(self.header))
 
     def allhunks(self):
         return any(self.allhunks_re.match(h) for h in self.header)
 
     def files(self):
         match = self.diffgit_re.match(self.header[0])
         if match:
             fromfile, tofile = match.groups()
             if fromfile == tofile:
                 return [fromfile]
             return [fromfile, tofile]
         else:
             return self.diff_re.match(self.header[0]).groups()
 
     def filename(self):
         return self.files()[-1]
 
     def __repr__(self):
         return '<header %s>' % (' '.join(map(repr, self.files())))
 
     def isnewfile(self):
         return any(self.newfile_re.match(h) for h in self.header)
 
     def special(self):
         # Special files are shown only at the header level and not at the hunk
         # level for example a file that has been deleted is a special file.
         # The user cannot change the content of the operation, in the case of
         # the deleted file he has to take the deletion or not take it, he
         # cannot take some of it.
         # Newly added files are special if they are empty, they are not special
         # if they have some content as we want to be able to change it
         nocontent = len(self.header) == 2
         emptynewfile = self.isnewfile() and nocontent
         return emptynewfile or \
                 any(self.special_re.match(h) for h in self.header)
 
 class recordhunk(object):
     """patch hunk
 
     XXX shouldn't we merge this with the other hunk class?
     """
 
     def __init__(self, header, fromline, toline, proc, before, hunk, after,
                  maxcontext=None):
         def trimcontext(lines, reverse=False):
             if maxcontext is not None:
                 delta = len(lines) - maxcontext
                 if delta > 0:
                     if reverse:
                         return delta, lines[delta:]
                     else:
                         return delta, lines[:maxcontext]
             return 0, lines
 
         self.header = header
         trimedbefore, self.before = trimcontext(before, True)
         self.fromline = fromline + trimedbefore
         self.toline = toline + trimedbefore
         _trimedafter, self.after = trimcontext(after, False)
         self.proc = proc
         self.hunk = hunk
         self.added, self.removed = self.countchanges(self.hunk)
 
     def __eq__(self, v):
         if not isinstance(v, recordhunk):
             return False
 
         return ((v.hunk == self.hunk) and
                 (v.proc == self.proc) and
                 (self.fromline == v.fromline) and
                 (self.header.files() == v.header.files()))
 
     def __hash__(self):
         return hash((tuple(self.hunk),
             tuple(self.header.files()),
             self.fromline,
             self.proc))
 
     def countchanges(self, hunk):
         """hunk -> (n+,n-)"""
         add = len([h for h in hunk if h.startswith('+')])
         rem = len([h for h in hunk if h.startswith('-')])
         return add, rem
 
     def reversehunk(self):
         """return another recordhunk which is the reverse of the hunk
 
         If this hunk is diff(A, B), the returned hunk is diff(B, A). To do
         that, swap fromline/toline and +/- signs while keep other things
         unchanged.
         """
         m = {'+': '-', '-': '+', '\\': '\\'}
         hunk = ['%s%s' % (m[l[0:1]], l[1:]) for l in self.hunk]
         return recordhunk(self.header, self.toline, self.fromline, self.proc,
                           self.before, hunk, self.after)
 
     def write(self, fp):
         delta = len(self.before) + len(self.after)
         if self.after and self.after[-1] == '\\ No newline at end of file\n':
             delta -= 1
         fromlen = delta + self.removed
         tolen = delta + self.added
         fp.write('@@ -%d,%d +%d,%d @@%s\n' %
                  (self.fromline, fromlen, self.toline, tolen,
                   self.proc and (' ' + self.proc)))
         fp.write(''.join(self.before + self.hunk + self.after))
 
     pretty = write
 
     def filename(self):
         return self.header.filename()
 
     def __repr__(self):
         return '<hunk %r@%d>' % (self.filename(), self.fromline)
 
 def getmessages():
     return {
         'multiple': {
             'apply': _("apply change %d/%d to '%s'?"),
             'discard': _("discard change %d/%d to '%s'?"),
             'record': _("record change %d/%d to '%s'?"),
         },
         'single': {
             'apply': _("apply this change to '%s'?"),
             'discard': _("discard this change to '%s'?"),
             'record': _("record this change to '%s'?"),
         },
         'help': {
             'apply': _('[Ynesfdaq?]'
                          '$$ &Yes, apply this change'
                          '$$ &No, skip this change'
                          '$$ &Edit this change manually'
                          '$$ &Skip remaining changes to this file'
                          '$$ Apply remaining changes to this &file'
                          '$$ &Done, skip remaining changes and files'
                          '$$ Apply &all changes to all remaining files'
                          '$$ &Quit, applying no changes'
                          '$$ &? (display help)'),
             'discard': _('[Ynesfdaq?]'
                          '$$ &Yes, discard this change'
                          '$$ &No, skip this change'
                          '$$ &Edit this change manually'
                          '$$ &Skip remaining changes to this file'
                          '$$ Discard remaining changes to this &file'
                          '$$ &Done, skip remaining changes and files'
                          '$$ Discard &all changes to all remaining files'
                          '$$ &Quit, discarding no changes'
                          '$$ &? (display help)'),
             'record': _('[Ynesfdaq?]'
                         '$$ &Yes, record this change'
                         '$$ &No, skip this change'
                         '$$ &Edit this change manually'
                         '$$ &Skip remaining changes to this file'
                         '$$ Record remaining changes to this &file'
                         '$$ &Done, skip remaining changes and files'
                         '$$ Record &all changes to all remaining files'
                         '$$ &Quit, recording no changes'
                         '$$ &? (display help)'),
         }
     }
 
 def filterpatch(ui, headers, operation=None):
     """Interactively filter patch chunks into applied-only chunks"""
     messages = getmessages()
 
     if operation is None:
         operation = 'record'
 
     def prompt(skipfile, skipall, query, chunk):
         """prompt query, and process base inputs
 
         - y/n for the rest of file
         - y/n for the rest
         - ? (help)
         - q (quit)
 
         Return True/False and possibly updated skipfile and skipall.
         """
         newpatches = None
         if skipall is not None:
             return skipall, skipfile, skipall, newpatches
         if skipfile is not None:
             return skipfile, skipfile, skipall, newpatches
         while True:
             resps = messages['help'][operation]
             r = ui.promptchoice("%s %s" % (query, resps))
             ui.write("\n")
             if r == 8: # ?
                 for c, t in ui.extractchoices(resps)[1]:
                     ui.write('%s - %s\n' % (c, encoding.lower(t)))
                 continue
             elif r == 0: # yes
                 ret = True
             elif r == 1: # no
                 ret = False
             elif r == 2: # Edit patch
                 if chunk is None:
                     ui.write(_('cannot edit patch for whole file'))
                     ui.write("\n")
                     continue
                 if chunk.header.binary():
                     ui.write(_('cannot edit patch for binary file'))
                     ui.write("\n")
                     continue
                 # Patch comment based on the Git one (based on comment at end of
                 # https://mercurial-scm.org/wiki/RecordExtension)
                 phelp = '---' + _("""
 To remove '-' lines, make them ' ' lines (context).
 To remove '+' lines, delete them.
 Lines starting with # will be removed from the patch.
 
 If the patch applies cleanly, the edited hunk will immediately be
 added to the record list. If it does not apply cleanly, a rejects
 file will be generated: you can use that when you try again. If
 all lines of the hunk are removed, then the edit is aborted and
 the hunk is left unchanged.
 """)
                 (patchfd, patchfn) = tempfile.mkstemp(prefix="hg-editor-",
                         suffix=".diff", text=True)
                 ncpatchfp = None
                 try:
                     # Write the initial patch
                     f = os.fdopen(patchfd, pycompat.sysstr("w"))
                     chunk.header.write(f)
                     chunk.write(f)
                     f.write('\n'.join(['# ' + i for i in phelp.splitlines()]))
                     f.close()
                     # Start the editor and wait for it to complete
                     editor = ui.geteditor()
                     ret = ui.system("%s \"%s\"" % (editor, patchfn),
                                     environ={'HGUSER': ui.username()},
                                     blockedtag='filterpatch')
                     if ret != 0:
                         ui.warn(_("editor exited with exit code %d\n") % ret)
                         continue
                     # Remove comment lines
                     patchfp = open(patchfn)
                     ncpatchfp = stringio()
                     for line in util.iterfile(patchfp):
                         if not line.startswith('#'):
                             ncpatchfp.write(line)
                     patchfp.close()
                     ncpatchfp.seek(0)
                     newpatches = parsepatch(ncpatchfp)
                 finally:
                     os.unlink(patchfn)
                     del ncpatchfp
                 # Signal that the chunk shouldn't be applied as-is, but
                 # provide the new patch to be used instead.
                 ret = False
             elif r == 3: # Skip
                 ret = skipfile = False
             elif r == 4: # file (Record remaining)
                 ret = skipfile = True
             elif r == 5: # done, skip remaining
                 ret = skipall = False
             elif r == 6: # all
                 ret = skipall = True
             elif r == 7: # quit
                 raise error.Abort(_('user quit'))
             return ret, skipfile, skipall, newpatches
 
     seen = set()
     applied = {}        # 'filename' -> [] of chunks
     skipfile, skipall = None, None
     pos, total = 1, sum(len(h.hunks) for h in headers)
     for h in headers:
         pos += len(h.hunks)
         skipfile = None
         fixoffset = 0
         hdr = ''.join(h.header)
         if hdr in seen:
             continue
         seen.add(hdr)
         if skipall is None:
             h.pretty(ui)
         msg = (_('examine changes to %s?') %
                _(' and ').join("'%s'" % f for f in h.files()))
         r, skipfile, skipall, np = prompt(skipfile, skipall, msg, None)
         if not r:
             continue
         applied[h.filename()] = [h]
         if h.allhunks():
             applied[h.filename()] += h.hunks
             continue
         for i, chunk in enumerate(h.hunks):
             if skipfile is None and skipall is None:
                 chunk.pretty(ui)
             if total == 1:
                 msg = messages['single'][operation] % chunk.filename()
             else:
                 idx = pos - len(h.hunks) + i
                 msg = messages['multiple'][operation] % (idx, total,
                                                          chunk.filename())
             r, skipfile, skipall, newpatches = prompt(skipfile,
                     skipall, msg, chunk)
             if r:
                 if fixoffset:
                     chunk = copy.copy(chunk)
                     chunk.toline += fixoffset
                 applied[chunk.filename()].append(chunk)
             elif newpatches is not None:
                 for newpatch in newpatches:
                     for newhunk in newpatch.hunks:
                         if fixoffset:
                             newhunk.toline += fixoffset
                         applied[newhunk.filename()].append(newhunk)
             else:
                 fixoffset += chunk.removed - chunk.added
     return (sum([h for h in applied.itervalues()
                if h[0].special() or len(h) > 1], []), {})
 class hunk(object):
     def __init__(self, desc, num, lr, context):
         self.number = num
         self.desc = desc
         self.hunk = [desc]
         self.a = []
         self.b = []
         self.starta = self.lena = None
         self.startb = self.lenb = None
         if lr is not None:
             if context:
                 self.read_context_hunk(lr)
             else:
                 self.read_unified_hunk(lr)
 
     def getnormalized(self):
         """Return a copy with line endings normalized to LF."""
 
         def normalize(lines):
             nlines = []
             for line in lines:
                 if line.endswith('\r\n'):
                     line = line[:-2] + '\n'
                 nlines.append(line)
             return nlines
 
         # Dummy object, it is rebuilt manually
         nh = hunk(self.desc, self.number, None, None)
         nh.number = self.number
         nh.desc = self.desc
         nh.hunk = self.hunk
         nh.a = normalize(self.a)
         nh.b = normalize(self.b)
         nh.starta = self.starta
         nh.startb = self.startb
         nh.lena = self.lena
         nh.lenb = self.lenb
         return nh
 
     def read_unified_hunk(self, lr):
         m = unidesc.match(self.desc)
         if not m:
             raise PatchError(_("bad hunk #%d") % self.number)
         self.starta, self.lena, self.startb, self.lenb = m.groups()
         if self.lena is None:
             self.lena = 1
         else:
             self.lena = int(self.lena)
         if self.lenb is None:
             self.lenb = 1
         else:
             self.lenb = int(self.lenb)
         self.starta = int(self.starta)
         self.startb = int(self.startb)
         diffhelpers.addlines(lr, self.hunk, self.lena, self.lenb, self.a,
                              self.b)
         # if we hit eof before finishing out the hunk, the last line will
         # be zero length.  Lets try to fix it up.
         while len(self.hunk[-1]) == 0:
             del self.hunk[-1]
             del self.a[-1]
             del self.b[-1]
             self.lena -= 1
             self.lenb -= 1
         self._fixnewline(lr)
 
     def read_context_hunk(self, lr):
         self.desc = lr.readline()
         m = contextdesc.match(self.desc)
         if not m:
             raise PatchError(_("bad hunk #%d") % self.number)
         self.starta, aend = m.groups()
         self.starta = int(self.starta)
         if aend is None:
             aend = self.starta
         self.lena = int(aend) - self.starta
         if self.starta:
             self.lena += 1
         for x in xrange(self.lena):
             l = lr.readline()
             if l.startswith('---'):
                 # lines addition, old block is empty
                 lr.push(l)
                 break
             s = l[2:]
             if l.startswith('- ') or l.startswith('! '):
                 u = '-' + s
             elif l.startswith('  '):
                 u = ' ' + s
             else:
                 raise PatchError(_("bad hunk #%d old text line %d") %
                                  (self.number, x))
             self.a.append(u)
             self.hunk.append(u)
 
         l = lr.readline()
         if l.startswith('\ '):
             s = self.a[-1][:-1]
             self.a[-1] = s
             self.hunk[-1] = s
             l = lr.readline()
         m = contextdesc.match(l)
         if not m:
             raise PatchError(_("bad hunk #%d") % self.number)
         self.startb, bend = m.groups()
         self.startb = int(self.startb)
         if bend is None:
             bend = self.startb
         self.lenb = int(bend) - self.startb
         if self.startb:
             self.lenb += 1
         hunki = 1
         for x in xrange(self.lenb):
             l = lr.readline()
             if l.startswith('\ '):
                 # XXX: the only way to hit this is with an invalid line range.
                 # The no-eol marker is not counted in the line range, but I
                 # guess there are diff(1) out there which behave differently.
                 s = self.b[-1][:-1]
                 self.b[-1] = s
                 self.hunk[hunki - 1] = s
                 continue
             if not l:
                 # line deletions, new block is empty and we hit EOF
                 lr.push(l)
                 break
             s = l[2:]
             if l.startswith('+ ') or l.startswith('! '):
                 u = '+' + s
             elif l.startswith('  '):
                 u = ' ' + s
             elif len(self.b) == 0:
                 # line deletions, new block is empty
                 lr.push(l)
                 break
             else:
                 raise PatchError(_("bad hunk #%d old text line %d") %
                                  (self.number, x))
             self.b.append(s)
             while True:
                 if hunki >= len(self.hunk):
                     h = ""
                 else:
                     h = self.hunk[hunki]
                 hunki += 1
                 if h == u:
                     break
                 elif h.startswith('-'):
                     continue
                 else:
                     self.hunk.insert(hunki - 1, u)
                     break
 
         if not self.a:
             # this happens when lines were only added to the hunk
             for x in self.hunk:
                 if x.startswith('-') or x.startswith(' '):
                     self.a.append(x)
         if not self.b:
             # this happens when lines were only deleted from the hunk
             for x in self.hunk:
                 if x.startswith('+') or x.startswith(' '):
                     self.b.append(x[1:])
         # @@ -start,len +start,len @@
         self.desc = "@@ -%d,%d +%d,%d @@\n" % (self.starta, self.lena,
                                              self.startb, self.lenb)
         self.hunk[0] = self.desc
         self._fixnewline(lr)
 
     def _fixnewline(self, lr):
         l = lr.readline()
         if l.startswith('\ '):
             diffhelpers.fix_newline(self.hunk, self.a, self.b)
         else:
             lr.push(l)
 
     def complete(self):
         return len(self.a) == self.lena and len(self.b) == self.lenb
 
     def _fuzzit(self, old, new, fuzz, toponly):
         # this removes context lines from the top and bottom of list 'l'.  It
         # checks the hunk to make sure only context lines are removed, and then
         # returns a new shortened list of lines.
         fuzz = min(fuzz, len(old))
         if fuzz:
             top = 0
             bot = 0
             hlen = len(self.hunk)
             for x in xrange(hlen - 1):
                 # the hunk starts with the @@ line, so use x+1
                 if self.hunk[x + 1][0] == ' ':
                     top += 1
                 else:
                     break
             if not toponly:
                 for x in xrange(hlen - 1):
                     if self.hunk[hlen - bot - 1][0] == ' ':
                         bot += 1
                     else:
                         break
 
             bot = min(fuzz, bot)
             top = min(fuzz, top)
             return old[top:len(old) - bot], new[top:len(new) - bot], top
         return old, new, 0
 
     def fuzzit(self, fuzz, toponly):
         old, new, top = self._fuzzit(self.a, self.b, fuzz, toponly)
         oldstart = self.starta + top
         newstart = self.startb + top
         # zero length hunk ranges already have their start decremented
         if self.lena and oldstart > 0:
             oldstart -= 1
         if self.lenb and newstart > 0:
             newstart -= 1
         return old, oldstart, new, newstart
 
 class binhunk(object):
     'A binary patch file.'
     def __init__(self, lr, fname):
         self.text = None
         self.delta = False
         self.hunk = ['GIT binary patch\n']
         self._fname = fname
         self._read(lr)
 
     def complete(self):
         return self.text is not None
 
     def new(self, lines):
         if self.delta:
             return [applybindelta(self.text, ''.join(lines))]
         return [self.text]
 
     def _read(self, lr):
         def getline(lr, hunk):
             l = lr.readline()
             hunk.append(l)
             return l.rstrip('\r\n')
 
         size = 0
         while True:
             line = getline(lr, self.hunk)
             if not line:
                 raise PatchError(_('could not extract "%s" binary data')
                                  % self._fname)
             if line.startswith('literal '):
                 size = int(line[8:].rstrip())
                 break
             if line.startswith('delta '):
                 size = int(line[6:].rstrip())
                 self.delta = True
                 break
         dec = []
         line = getline(lr, self.hunk)
         while len(line) > 1:
             l = line[0:1]
             if l <= 'Z' and l >= 'A':
                 l = ord(l) - ord('A') + 1
             else:
                 l = ord(l) - ord('a') + 27
             try:
                 dec.append(util.b85decode(line[1:])[:l])
             except ValueError as e:
                 raise PatchError(_('could not decode "%s" binary patch: %s')
                                  % (self._fname, util.forcebytestr(e)))
             line = getline(lr, self.hunk)
         text = zlib.decompress(''.join(dec))
         if len(text) != size:
             raise PatchError(_('"%s" length is %d bytes, should be %d')
                              % (self._fname, len(text), size))
         self.text = text
 
 def parsefilename(str):
     # --- filename \t|space stuff
     s = str[4:].rstrip('\r\n')
     i = s.find('\t')
     if i < 0:
         i = s.find(' ')
         if i < 0:
             return s
     return s[:i]
 
 def reversehunks(hunks):
     '''reverse the signs in the hunks given as argument
 
     This function operates on hunks coming out of patch.filterpatch, that is
     a list of the form: [header1, hunk1, hunk2, header2...]. Example usage:
 
     >>> rawpatch = b"""diff --git a/folder1/g b/folder1/g
     ... --- a/folder1/g
     ... +++ b/folder1/g
     ... @@ -1,7 +1,7 @@
     ... +firstline
     ...  c
     ...  1
     ...  2
     ... + 3
     ... -4
     ...  5
     ...  d
     ... +lastline"""
     >>> hunks = parsepatch([rawpatch])
     >>> hunkscomingfromfilterpatch = []
     >>> for h in hunks:
     ...     hunkscomingfromfilterpatch.append(h)
     ...     hunkscomingfromfilterpatch.extend(h.hunks)
 
     >>> reversedhunks = reversehunks(hunkscomingfromfilterpatch)
     >>> from . import util
     >>> fp = util.stringio()
     >>> for c in reversedhunks:
     ...      c.write(fp)
     >>> fp.seek(0) or None
     >>> reversedpatch = fp.read()
     >>> print(pycompat.sysstr(reversedpatch))
     diff --git a/folder1/g b/folder1/g
     --- a/folder1/g
     +++ b/folder1/g
     @@ -1,4 +1,3 @@
     -firstline
      c
      1
      2
     @@ -2,6 +1,6 @@
      c
      1
      2
     - 3
     +4
      5
      d
     @@ -6,3 +5,2 @@
      5
      d
     -lastline
 
     '''
 
     newhunks = []
     for c in hunks:
         if util.safehasattr(c, 'reversehunk'):
             c = c.reversehunk()
         newhunks.append(c)
     return newhunks
 
 def parsepatch(originalchunks, maxcontext=None):
     """patch -> [] of headers -> [] of hunks
 
     If maxcontext is not None, trim context lines if necessary.
 
     >>> rawpatch = b'''diff --git a/folder1/g b/folder1/g
     ... --- a/folder1/g
     ... +++ b/folder1/g
     ... @@ -1,8 +1,10 @@
     ...  1
     ...  2
     ... -3
     ...  4
     ...  5
     ...  6
     ... +6.1
     ... +6.2
     ...  7
     ...  8
     ... +9'''
     >>> out = util.stringio()
     >>> headers = parsepatch([rawpatch], maxcontext=1)
     >>> for header in headers:
     ...     header.write(out)
     ...     for hunk in header.hunks:
     ...         hunk.write(out)
     >>> print(pycompat.sysstr(out.getvalue()))
     diff --git a/folder1/g b/folder1/g
     --- a/folder1/g
     +++ b/folder1/g
     @@ -2,3 +2,2 @@
      2
     -3
      4
     @@ -6,2 +5,4 @@
      6
     +6.1
     +6.2
      7
     @@ -8,1 +9,2 @@
      8
     +9
     """
     class parser(object):
         """patch parsing state machine"""
         def __init__(self):
             self.fromline = 0
             self.toline = 0
             self.proc = ''
             self.header = None
             self.context = []
             self.before = []
             self.hunk = []
             self.headers = []
 
         def addrange(self, limits):
             fromstart, fromend, tostart, toend, proc = limits
             self.fromline = int(fromstart)
             self.toline = int(tostart)
             self.proc = proc
 
         def addcontext(self, context):
             if self.hunk:
                 h = recordhunk(self.header, self.fromline, self.toline,
                         self.proc, self.before, self.hunk, context, maxcontext)
                 self.header.hunks.append(h)
                 self.fromline += len(self.before) + h.removed
                 self.toline += len(self.before) + h.added
                 self.before = []
                 self.hunk = []
             self.context = context
 
         def addhunk(self, hunk):
             if self.context:
                 self.before = self.context
                 self.context = []
             self.hunk = hunk
 
         def newfile(self, hdr):
             self.addcontext([])
             h = header(hdr)
             self.headers.append(h)
             self.header = h
 
         def addother(self, line):
             pass # 'other' lines are ignored
 
         def finished(self):
             self.addcontext([])
             return self.headers
 
         transitions = {
             'file': {'context': addcontext,
                      'file': newfile,
                      'hunk': addhunk,
                      'range': addrange},
             'context': {'file': newfile,
                         'hunk': addhunk,
                         'range': addrange,
                         'other': addother},
             'hunk': {'context': addcontext,
                      'file': newfile,
                      'range': addrange},
             'range': {'context': addcontext,
                       'hunk': addhunk},
             'other': {'other': addother},
             }
 
     p = parser()
     fp = stringio()
     fp.write(''.join(originalchunks))
     fp.seek(0)
 
     state = 'context'
     for newstate, data in scanpatch(fp):
         try:
             p.transitions[state][newstate](p, data)
         except KeyError:
             raise PatchError('unhandled transition: %s -> %s' %
                                    (state, newstate))
         state = newstate
     del fp
     return p.finished()
 
 def pathtransform(path, strip, prefix):
     '''turn a path from a patch into a path suitable for the repository
 
     prefix, if not empty, is expected to be normalized with a / at the end.
 
     Returns (stripped components, path in repository).
 
     >>> pathtransform(b'a/b/c', 0, b'')
     ('', 'a/b/c')
     >>> pathtransform(b'   a/b/c   ', 0, b'')
     ('', '   a/b/c')
     >>> pathtransform(b'   a/b/c   ', 2, b'')
     ('a/b/', 'c')
     >>> pathtransform(b'a/b/c', 0, b'd/e/')
     ('', 'd/e/a/b/c')
     >>> pathtransform(b'   a//b/c   ', 2, b'd/e/')
     ('a//b/', 'd/e/c')
     >>> pathtransform(b'a/b/c', 3, b'')
     Traceback (most recent call last):
     PatchError: unable to strip away 1 of 3 dirs from a/b/c
     '''
     pathlen = len(path)
     i = 0
     if strip == 0:
         return '', prefix + path.rstrip()
     count = strip
     while count > 0:
         i = path.find('/', i)
         if i == -1:
             raise PatchError(_("unable to strip away %d of %d dirs from %s") %
                              (count, strip, path))
         i += 1
         # consume '//' in the path
         while i < pathlen - 1 and path[i:i + 1] == '/':
             i += 1
         count -= 1
     return path[:i].lstrip(), prefix + path[i:].rstrip()
 
 def makepatchmeta(backend, afile_orig, bfile_orig, hunk, strip, prefix):
     nulla = afile_orig == "/dev/null"
     nullb = bfile_orig == "/dev/null"
     create = nulla and hunk.starta == 0 and hunk.lena == 0
     remove = nullb and hunk.startb == 0 and hunk.lenb == 0
     abase, afile = pathtransform(afile_orig, strip, prefix)
     gooda = not nulla and backend.exists(afile)
     bbase, bfile = pathtransform(bfile_orig, strip, prefix)
     if afile == bfile:
         goodb = gooda
     else:
         goodb = not nullb and backend.exists(bfile)
     missing = not goodb and not gooda and not create
 
     # some diff programs apparently produce patches where the afile is
     # not /dev/null, but afile starts with bfile
     abasedir = afile[:afile.rfind('/') + 1]
     bbasedir = bfile[:bfile.rfind('/') + 1]
     if (missing and abasedir == bbasedir and afile.startswith(bfile)
         and hunk.starta == 0 and hunk.lena == 0):
         create = True
         missing = False
 
     # If afile is "a/b/foo" and bfile is "a/b/foo.orig" we assume the
     # diff is between a file and its backup. In this case, the original
     # file should be patched (see original mpatch code).
     isbackup = (abase == bbase and bfile.startswith(afile))
     fname = None
     if not missing:
         if gooda and goodb:
             if isbackup:
                 fname = afile
             else:
                 fname = bfile
         elif gooda:
             fname = afile
 
     if not fname:
         if not nullb:
             if isbackup:
                 fname = afile
             else:
                 fname = bfile
         elif not nulla:
             fname = afile
         else:
             raise PatchError(_("undefined source and destination files"))
 
     gp = patchmeta(fname)
     if create:
         gp.op = 'ADD'
     elif remove:
         gp.op = 'DELETE'
     return gp
 
 def scanpatch(fp):
     """like patch.iterhunks, but yield different events
 
     - ('file',    [header_lines + fromfile + tofile])
     - ('context', [context_lines])
     - ('hunk',    [hunk_lines])
     - ('range',   (-start,len, +start,len, proc))
     """
     lines_re = re.compile(br'@@ -(\d+),(\d+) \+(\d+),(\d+) @@\s*(.*)')
     lr = linereader(fp)
 
     def scanwhile(first, p):
         """scan lr while predicate holds"""
         lines = [first]
         for line in iter(lr.readline, ''):
             if p(line):
                 lines.append(line)
             else:
                 lr.push(line)
                 break
         return lines
 
     for line in iter(lr.readline, ''):
         if line.startswith('diff --git a/') or line.startswith('diff -r '):
             def notheader(line):
                 s = line.split(None, 1)
                 return not s or s[0] not in ('---', 'diff')
             header = scanwhile(line, notheader)
             fromfile = lr.readline()
             if fromfile.startswith('---'):
                 tofile = lr.readline()
                 header += [fromfile, tofile]
             else:
                 lr.push(fromfile)
             yield 'file', header
         elif line[0:1] == ' ':
             yield 'context', scanwhile(line, lambda l: l[0] in ' \\')
         elif line[0] in '-+':
             yield 'hunk', scanwhile(line, lambda l: l[0] in '-+\\')
         else:
             m = lines_re.match(line)
             if m:
                 yield 'range', m.groups()
             else:
                 yield 'other', line
 
 def scangitpatch(lr, firstline):
     """
     Git patches can emit:
     - rename a to b
     - change b
     - copy a to c
     - change c
 
     We cannot apply this sequence as-is, the renamed 'a' could not be
     found for it would have been renamed already. And we cannot copy
     from 'b' instead because 'b' would have been changed already. So
     we scan the git patch for copy and rename commands so we can
     perform the copies ahead of time.
     """
     pos = 0
     try:
         pos = lr.fp.tell()
         fp = lr.fp
     except IOError:
         fp = stringio(lr.fp.read())
     gitlr = linereader(fp)
     gitlr.push(firstline)
     gitpatches = readgitpatch(gitlr)
     fp.seek(pos)
     return gitpatches
 
 def iterhunks(fp):
     """Read a patch and yield the following events:
     - ("file", afile, bfile, firsthunk): select a new target file.
     - ("hunk", hunk): a new hunk is ready to be applied, follows a
     "file" event.
     - ("git", gitchanges): current diff is in git format, gitchanges
     maps filenames to gitpatch records. Unique event.
     """
     afile = ""
     bfile = ""
     state = None
     hunknum = 0
     emitfile = newfile = False
     gitpatches = None
 
     # our states
     BFILE = 1
     context = None
     lr = linereader(fp)
 
     for x in iter(lr.readline, ''):
         if state == BFILE and (
             (not context and x[0] == '@')
             or (context is not False and x.startswith('***************'))
             or x.startswith('GIT binary patch')):
             gp = None
             if (gitpatches and
                 gitpatches[-1].ispatching(afile, bfile)):
                 gp = gitpatches.pop()
             if x.startswith('GIT binary patch'):
                 h = binhunk(lr, gp.path)
             else:
                 if context is None and x.startswith('***************'):
                     context = True
                 h = hunk(x, hunknum + 1, lr, context)
             hunknum += 1
             if emitfile:
                 emitfile = False
                 yield 'file', (afile, bfile, h, gp and gp.copy() or None)
             yield 'hunk', h
         elif x.startswith('diff --git a/'):
             m = gitre.match(x.rstrip(' \r\n'))
             if not m:
                 continue
             if gitpatches is None:
                 # scan whole input for git metadata
                 gitpatches = scangitpatch(lr, x)
                 yield 'git', [g.copy() for g in gitpatches
                               if g.op in ('COPY', 'RENAME')]
                 gitpatches.reverse()
             afile = 'a/' + m.group(1)
             bfile = 'b/' + m.group(2)
             while gitpatches and not gitpatches[-1].ispatching(afile, bfile):
                 gp = gitpatches.pop()
                 yield 'file', ('a/' + gp.path, 'b/' + gp.path, None, gp.copy())
             if not gitpatches:
                 raise PatchError(_('failed to synchronize metadata for "%s"')
                                  % afile[2:])
             gp = gitpatches[-1]
             newfile = True
         elif x.startswith('---'):
             # check for a unified diff
             l2 = lr.readline()
             if not l2.startswith('+++'):
                 lr.push(l2)
                 continue
             newfile = True
             context = False
             afile = parsefilename(x)
             bfile = parsefilename(l2)
         elif x.startswith('***'):
             # check for a context diff
             l2 = lr.readline()
             if not l2.startswith('---'):
                 lr.push(l2)
                 continue
             l3 = lr.readline()
             lr.push(l3)
             if not l3.startswith("***************"):
                 lr.push(l2)
                 continue
             newfile = True
             context = True
             afile = parsefilename(x)
             bfile = parsefilename(l2)
 
         if newfile:
             newfile = False
             emitfile = True
             state = BFILE
             hunknum = 0
 
     while gitpatches:
         gp = gitpatches.pop()
         yield 'file', ('a/' + gp.path, 'b/' + gp.path, None, gp.copy())
 
 def applybindelta(binchunk, data):
     """Apply a binary delta hunk
     The algorithm used is the algorithm from git's patch-delta.c
     """
     def deltahead(binchunk):
         i = 0
         for c in binchunk:
             i += 1
             if not (ord(c) & 0x80):
                 return i
         return i
     out = ""
     s = deltahead(binchunk)
     binchunk = binchunk[s:]
     s = deltahead(binchunk)
     binchunk = binchunk[s:]
     i = 0
     while i < len(binchunk):
         cmd = ord(binchunk[i])
         i += 1
         if (cmd & 0x80):
             offset = 0
             size = 0
             if (cmd & 0x01):
                 offset = ord(binchunk[i])
                 i += 1
             if (cmd & 0x02):
                 offset |= ord(binchunk[i]) << 8
                 i += 1
             if (cmd & 0x04):
                 offset |= ord(binchunk[i]) << 16
                 i += 1
             if (cmd & 0x08):
                 offset |= ord(binchunk[i]) << 24
                 i += 1
             if (cmd & 0x10):
                 size = ord(binchunk[i])
                 i += 1
             if (cmd & 0x20):
                 size |= ord(binchunk[i]) << 8
                 i += 1
             if (cmd & 0x40):
                 size |= ord(binchunk[i]) << 16
                 i += 1
             if size == 0:
                 size = 0x10000
             offset_end = offset + size
             out += data[offset:offset_end]
         elif cmd != 0:
             offset_end = i + cmd
             out += binchunk[i:offset_end]
             i += cmd
         else:
             raise PatchError(_('unexpected delta opcode 0'))
     return out
 
 def applydiff(ui, fp, backend, store, strip=1, prefix='', eolmode='strict'):
     """Reads a patch from fp and tries to apply it.
 
     Returns 0 for a clean patch, -1 if any rejects were found and 1 if
     there was any fuzz.
 
     If 'eolmode' is 'strict', the patch content and patched file are
     read in binary mode. Otherwise, line endings are ignored when
     patching then normalized according to 'eolmode'.
     """
     return _applydiff(ui, fp, patchfile, backend, store, strip=strip,
                       prefix=prefix, eolmode=eolmode)
 
 def _canonprefix(repo, prefix):
     if prefix:
         prefix = pathutil.canonpath(repo.root, repo.getcwd(), prefix)
         if prefix != '':
             prefix += '/'
     return prefix
 
 def _applydiff(ui, fp, patcher, backend, store, strip=1, prefix='',
                eolmode='strict'):
     prefix = _canonprefix(backend.repo, prefix)
     def pstrip(p):
         return pathtransform(p, strip - 1, prefix)[1]
 
     rejects = 0
     err = 0
     current_file = None
 
     for state, values in iterhunks(fp):
         if state == 'hunk':
             if not current_file:
                 continue
             ret = current_file.apply(values)
             if ret > 0:
                 err = 1
         elif state == 'file':
             if current_file:
                 rejects += current_file.close()
                 current_file = None
             afile, bfile, first_hunk, gp = values
             if gp:
                 gp.path = pstrip(gp.path)
                 if gp.oldpath:
                     gp.oldpath = pstrip(gp.oldpath)
             else:
                 gp = makepatchmeta(backend, afile, bfile, first_hunk, strip,
                                    prefix)
             if gp.op == 'RENAME':
                 backend.unlink(gp.oldpath)
             if not first_hunk:
                 if gp.op == 'DELETE':
                     backend.unlink(gp.path)
                     continue
                 data, mode = None, None
                 if gp.op in ('RENAME', 'COPY'):
                     data, mode = store.getfile(gp.oldpath)[:2]
                     if data is None:
                         # This means that the old path does not exist
                         raise PatchError(_("source file '%s' does not exist")
                                            % gp.oldpath)
                 if gp.mode:
                     mode = gp.mode
                     if gp.op == 'ADD':
                         # Added files without content have no hunk and
                         # must be created
                         data = ''
                 if data or mode:
                     if (gp.op in ('ADD', 'RENAME', 'COPY')
                         and backend.exists(gp.path)):
                         raise PatchError(_("cannot create %s: destination "
                                            "already exists") % gp.path)
                     backend.setfile(gp.path, data, mode, gp.oldpath)
                 continue
             try:
                 current_file = patcher(ui, gp, backend, store,
                                        eolmode=eolmode)
             except PatchError as inst:
                 ui.warn(str(inst) + '\n')
                 current_file = None
                 rejects += 1
                 continue
         elif state == 'git':
             for gp in values:
                 path = pstrip(gp.oldpath)
                 data, mode = backend.getfile(path)
                 if data is None:
                     # The error ignored here will trigger a getfile()
                     # error in a place more appropriate for error
                     # handling, and will not interrupt the patching
                     # process.
                     pass
                 else:
                     store.setfile(path, data, mode)
         else:
             raise error.Abort(_('unsupported parser state: %s') % state)
 
     if current_file:
         rejects += current_file.close()
 
     if rejects:
         return -1
     return err
 
 def _externalpatch(ui, repo, patcher, patchname, strip, files,
                    similarity):
     """use <patcher> to apply <patchname> to the working directory.
     returns whether patch was applied with fuzz factor."""
 
     fuzz = False
     args = []
     cwd = repo.root
     if cwd:
         args.append('-d %s' % util.shellquote(cwd))
     fp = util.popen('%s %s -p%d < %s' % (patcher, ' '.join(args), strip,
                                        util.shellquote(patchname)))
     try:
         for line in util.iterfile(fp):
             line = line.rstrip()
             ui.note(line + '\n')
             if line.startswith('patching file '):
                 pf = util.parsepatchoutput(line)
                 printed_file = False
                 files.add(pf)
             elif line.find('with fuzz') >= 0:
                 fuzz = True
                 if not printed_file:
                     ui.warn(pf + '\n')
                     printed_file = True
                 ui.warn(line + '\n')
             elif line.find('saving rejects to file') >= 0:
                 ui.warn(line + '\n')
             elif line.find('FAILED') >= 0:
                 if not printed_file:
                     ui.warn(pf + '\n')
                     printed_file = True
                 ui.warn(line + '\n')
     finally:
         if files:
             scmutil.marktouched(repo, files, similarity)
     code = fp.close()
     if code:
         raise PatchError(_("patch command failed: %s") %
                          util.explainexit(code)[0])
     return fuzz
 
 def patchbackend(ui, backend, patchobj, strip, prefix, files=None,
                  eolmode='strict'):
     if files is None:
         files = set()
     if eolmode is None:
         eolmode = ui.config('patch', 'eol')
     if eolmode.lower() not in eolmodes:
         raise error.Abort(_('unsupported line endings type: %s') % eolmode)
     eolmode = eolmode.lower()
 
     store = filestore()
     try:
         fp = open(patchobj, 'rb')
     except TypeError:
         fp = patchobj
     try:
         ret = applydiff(ui, fp, backend, store, strip=strip, prefix=prefix,
                         eolmode=eolmode)
     finally:
         if fp != patchobj:
             fp.close()
         files.update(backend.close())
         store.close()
     if ret < 0:
         raise PatchError(_('patch failed to apply'))
     return ret > 0
 
 def internalpatch(ui, repo, patchobj, strip, prefix='', files=None,
                   eolmode='strict', similarity=0):
     """use builtin patch to apply <patchobj> to the working directory.
     returns whether patch was applied with fuzz factor."""
     backend = workingbackend(ui, repo, similarity)
     return patchbackend(ui, backend, patchobj, strip, prefix, files, eolmode)
 
 def patchrepo(ui, repo, ctx, store, patchobj, strip, prefix, files=None,
               eolmode='strict'):
     backend = repobackend(ui, repo, ctx, store)
     return patchbackend(ui, backend, patchobj, strip, prefix, files, eolmode)
 
 def patch(ui, repo, patchname, strip=1, prefix='', files=None, eolmode='strict',
           similarity=0):
     """Apply <patchname> to the working directory.
 
     'eolmode' specifies how end of lines should be handled. It can be:
     - 'strict': inputs are read in binary mode, EOLs are preserved
     - 'crlf': EOLs are ignored when patching and reset to CRLF
     - 'lf': EOLs are ignored when patching and reset to LF
     - None: get it from user settings, default to 'strict'
     'eolmode' is ignored when using an external patcher program.
 
     Returns whether patch was applied with fuzz factor.
     """
     patcher = ui.config('ui', 'patch')
     if files is None:
         files = set()
     if patcher:
         return _externalpatch(ui, repo, patcher, patchname, strip,
                               files, similarity)
     return internalpatch(ui, repo, patchname, strip, prefix, files, eolmode,
                          similarity)
 
 def changedfiles(ui, repo, patchpath, strip=1, prefix=''):
     backend = fsbackend(ui, repo.root)
     prefix = _canonprefix(repo, prefix)
     with open(patchpath, 'rb') as fp:
         changed = set()
         for state, values in iterhunks(fp):
             if state == 'file':
                 afile, bfile, first_hunk, gp = values
                 if gp:
                     gp.path = pathtransform(gp.path, strip - 1, prefix)[1]
                     if gp.oldpath:
                         gp.oldpath = pathtransform(gp.oldpath, strip - 1,
                                                    prefix)[1]
                 else:
                     gp = makepatchmeta(backend, afile, bfile, first_hunk, strip,
                                        prefix)
                 changed.add(gp.path)
                 if gp.op == 'RENAME':
                     changed.add(gp.oldpath)
             elif state not in ('hunk', 'git'):
                 raise error.Abort(_('unsupported parser state: %s') % state)
         return changed
 
 class GitDiffRequired(Exception):
     pass
 
 def diffallopts(ui, opts=None, untrusted=False, section='diff'):
     '''return diffopts with all features supported and parsed'''
     return difffeatureopts(ui, opts=opts, untrusted=untrusted, section=section,
                            git=True, whitespace=True, formatchanging=True)
 
 diffopts = diffallopts
 
 def difffeatureopts(ui, opts=None, untrusted=False, section='diff', git=False,
                     whitespace=False, formatchanging=False):
     '''return diffopts with only opted-in features parsed
 
     Features:
     - git: git-style diffs
     - whitespace: whitespace options like ignoreblanklines and ignorews
     - formatchanging: options that will likely break or cause correctness issues
       with most diff parsers
     '''
     def get(key, name=None, getter=ui.configbool, forceplain=None):
         if opts:
             v = opts.get(key)
             # diffopts flags are either None-default (which is passed
             # through unchanged, so we can identify unset values), or
             # some other falsey default (eg --unified, which defaults
             # to an empty string). We only want to override the config
             # entries from hgrc with command line values if they
             # appear to have been set, which is any truthy value,
             # True, or False.
             if v or isinstance(v, bool):
                 return v
         if forceplain is not None and ui.plain():
             return forceplain
         return getter(section, name or key, untrusted=untrusted)
 
     # core options, expected to be understood by every diff parser
     buildopts = {
         'nodates': get('nodates'),
         'showfunc': get('show_function', 'showfunc'),
         'context': get('unified', getter=ui.config),
     }
     buildopts['worddiff'] = ui.configbool('experimental', 'worddiff')
 
     if git:
         buildopts['git'] = get('git')
 
         # since this is in the experimental section, we need to call
         # ui.configbool directory
         buildopts['showsimilarity'] = ui.configbool('experimental',
                                                     'extendedheader.similarity')
 
         # need to inspect the ui object instead of using get() since we want to
         # test for an int
         hconf = ui.config('experimental', 'extendedheader.index')
         if hconf is not None:
             hlen = None
             try:
                 # the hash config could be an integer (for length of hash) or a
                 # word (e.g. short, full, none)
                 hlen = int(hconf)
                 if hlen < 0 or hlen > 40:
                     msg = _("invalid length for extendedheader.index: '%d'\n")
                     ui.warn(msg % hlen)
             except ValueError:
                 # default value
                 if hconf == 'short' or hconf == '':
                     hlen = 12
                 elif hconf == 'full':
                     hlen = 40
                 elif hconf != 'none':
                     msg = _("invalid value for extendedheader.index: '%s'\n")
                     ui.warn(msg % hconf)
             finally:
                 buildopts['index'] = hlen
 
     if whitespace:
         buildopts['ignorews'] = get('ignore_all_space', 'ignorews')
         buildopts['ignorewsamount'] = get('ignore_space_change',
                                           'ignorewsamount')
         buildopts['ignoreblanklines'] = get('ignore_blank_lines',
                                             'ignoreblanklines')
         buildopts['ignorewseol'] = get('ignore_space_at_eol', 'ignorewseol')
     if formatchanging:
         buildopts['text'] = opts and opts.get('text')
         binary = None if opts is None else opts.get('binary')
         buildopts['nobinary'] = (not binary if binary is not None
                                  else get('nobinary', forceplain=False))
         buildopts['noprefix'] = get('noprefix', forceplain=False)
 
     return mdiff.diffopts(**pycompat.strkwargs(buildopts))
 
 def diff(repo, node1=None, node2=None, match=None, changes=None,
          opts=None, losedatafn=None, prefix='', relroot='', copy=None,
          hunksfilterfn=None):
     '''yields diff of changes to files between two nodes, or node and
     working directory.
 
     if node1 is None, use first dirstate parent instead.
     if node2 is None, compare node1 with working directory.
 
     losedatafn(**kwarg) is a callable run when opts.upgrade=True and
     every time some change cannot be represented with the current
     patch format. Return False to upgrade to git patch format, True to
     accept the loss or raise an exception to abort the diff. It is
     called with the name of current file being diffed as 'fn'. If set
     to None, patches will always be upgraded to git format when
     necessary.
 
     prefix is a filename prefix that is prepended to all filenames on
     display (used for subrepos).
 
     relroot, if not empty, must be normalized with a trailing /. Any match
     patterns that fall outside it will be ignored.
 
     copy, if not empty, should contain mappings {dst@y: src@x} of copy
     information.
 
     hunksfilterfn, if not None, should be a function taking a filectx and
     hunks generator that may yield filtered hunks.
     '''
     for fctx1, fctx2, hdr, hunks in diffhunks(
             repo, node1=node1, node2=node2,
             match=match, changes=changes, opts=opts,
             losedatafn=losedatafn, prefix=prefix, relroot=relroot, copy=copy,
     ):
         if hunksfilterfn is not None:
             # If the file has been removed, fctx2 is None; but this should
             # not occur here since we catch removed files early in
             # logcmdutil.getlinerangerevs() for 'hg log -L'.
             assert fctx2 is not None, \
                 'fctx2 unexpectly None in diff hunks filtering'
             hunks = hunksfilterfn(fctx2, hunks)
         text = ''.join(sum((list(hlines) for hrange, hlines in hunks), []))
         if hdr and (text or len(hdr) > 1):
             yield '\n'.join(hdr) + '\n'
         if text:
             yield text
 
 def diffhunks(repo, node1=None, node2=None, match=None, changes=None,
               opts=None, losedatafn=None, prefix='', relroot='', copy=None):
     """Yield diff of changes to files in the form of (`header`, `hunks`) tuples
     where `header` is a list of diff headers and `hunks` is an iterable of
     (`hunkrange`, `hunklines`) tuples.
 
     See diff() for the meaning of parameters.
     """
 
     if opts is None:
         opts = mdiff.defaultopts
 
     if not node1 and not node2:
         node1 = repo.dirstate.p1()
 
     def lrugetfilectx():
         cache = {}
         order = collections.deque()
         def getfilectx(f, ctx):
             fctx = ctx.filectx(f, filelog=cache.get(f))
             if f not in cache:
                 if len(cache) > 20:
                     del cache[order.popleft()]
                 cache[f] = fctx.filelog()
             else:
                 order.remove(f)
             order.append(f)
             return fctx
         return getfilectx
     getfilectx = lrugetfilectx()
 
     ctx1 = repo[node1]
     ctx2 = repo[node2]
 
     relfiltered = False
     if relroot != '' and match.always():
         # as a special case, create a new matcher with just the relroot
         pats = [relroot]
         match = scmutil.match(ctx2, pats, default='path')
         relfiltered = True
 
     if not changes:
         changes = repo.status(ctx1, ctx2, match=match)
     modified, added, removed = changes[:3]
 
     if not modified and not added and not removed:
         return []
 
     if repo.ui.debugflag:
         hexfunc = hex
     else:
         hexfunc = short
     revs = [hexfunc(node) for node in [ctx1.node(), ctx2.node()] if node]
 
     if copy is None:
         copy = {}
         if opts.git or opts.upgrade:
             copy = copies.pathcopies(ctx1, ctx2, match=match)
 
     if relroot is not None:
         if not relfiltered:
             # XXX this would ideally be done in the matcher, but that is
             # generally meant to 'or' patterns, not 'and' them. In this case we
             # need to 'and' all the patterns from the matcher with relroot.
             def filterrel(l):
                 return [f for f in l if f.startswith(relroot)]
             modified = filterrel(modified)
             added = filterrel(added)
             removed = filterrel(removed)
             relfiltered = True
         # filter out copies where either side isn't inside the relative root
         copy = dict(((dst, src) for (dst, src) in copy.iteritems()
                      if dst.startswith(relroot)
                      and src.startswith(relroot)))
 
     modifiedset = set(modified)
     addedset = set(added)
     removedset = set(removed)
     for f in modified:
         if f not in ctx1:
             # Fix up added, since merged-in additions appear as
             # modifications during merges
             modifiedset.remove(f)
             addedset.add(f)
     for f in removed:
         if f not in ctx1:
             # Merged-in additions that are then removed are reported as removed.
             # They are not in ctx1, so We don't want to show them in the diff.
             removedset.remove(f)
     modified = sorted(modifiedset)
     added = sorted(addedset)
     removed = sorted(removedset)
     for dst, src in list(copy.items()):
         if src not in ctx1:
             # Files merged in during a merge and then copied/renamed are
             # reported as copies. We want to show them in the diff as additions.
             del copy[dst]
 
     def difffn(opts, losedata):
         return trydiff(repo, revs, ctx1, ctx2, modified, added, removed,
                        copy, getfilectx, opts, losedata, prefix, relroot)
     if opts.upgrade and not opts.git:
         try:
             def losedata(fn):
                 if not losedatafn or not losedatafn(fn=fn):
                     raise GitDiffRequired
             # Buffer the whole output until we are sure it can be generated
             return list(difffn(opts.copy(git=False), losedata))
         except GitDiffRequired:
             return difffn(opts.copy(git=True), None)
     else:
         return difffn(opts, None)
 
 def difflabel(func, *args, **kw):
     '''yields 2-tuples of (output, label) based on the output of func()'''
     inlinecolor = False
     if kw.get(r'opts'):
         inlinecolor = kw[r'opts'].worddiff
     headprefixes = [('diff', 'diff.diffline'),
                     ('copy', 'diff.extended'),
                     ('rename', 'diff.extended'),
                     ('old', 'diff.extended'),
                     ('new', 'diff.extended'),
                     ('deleted', 'diff.extended'),
                     ('index', 'diff.extended'),
                     ('similarity', 'diff.extended'),
                     ('---', 'diff.file_a'),
                     ('+++', 'diff.file_b')]
     textprefixes = [('@', 'diff.hunk'),
                     ('-', 'diff.deleted'),
                     ('+', 'diff.inserted')]
     head = False
     for chunk in func(*args, **kw):
         lines = chunk.split('\n')
         matches = {}
         if inlinecolor:
             matches = _findmatches(lines)
         for i, line in enumerate(lines):
             if i != 0:
                 yield ('\n', '')
             if head:
                 if line.startswith('@'):
                     head = False
             else:
                 if line and line[0] not in ' +-@\\':
                     head = True
             stripline = line
             diffline = False
             if not head and line and line[0] in '+-':
                 # highlight tabs and trailing whitespace, but only in
                 # changed lines
                 stripline = line.rstrip()
                 diffline = True
 
             prefixes = textprefixes
             if head:
                 prefixes = headprefixes
             for prefix, label in prefixes:
                 if stripline.startswith(prefix):
                     if diffline:
                         if i in matches:
                             for t, l in _inlinediff(lines[i].rstrip(),
                                                     lines[matches[i]].rstrip(),
                                                     label):
                                 yield (t, l)
                         else:
                             for token in tabsplitter.findall(stripline):
                                 if '\t' == token[0]:
                                     yield (token, 'diff.tab')
                                 else:
                                     yield (token, label)
                     else:
                         yield (stripline, label)
                     break
             else:
                 yield (line, '')
             if line != stripline:
                 yield (line[len(stripline):], 'diff.trailingwhitespace')
 
 def _findmatches(slist):
     '''Look for insertion matches to deletion and returns a dict of
     correspondences.
     '''
     lastmatch = 0
     matches = {}
     for i, line in enumerate(slist):
         if line == '':
             continue
         if line[0] == '-':
             lastmatch = max(lastmatch, i)
             newgroup = False
             for j, newline in enumerate(slist[lastmatch + 1:]):
                 if newline == '':
                     continue
                 if newline[0] == '-' and newgroup: # too far, no match
                     break
                 if newline[0] == '+': # potential match
                     newgroup = True
                     sim = difflib.SequenceMatcher(None, line, newline).ratio()
                     if sim > 0.7:
                         lastmatch = lastmatch + 1 + j
                         matches[i] = lastmatch
                         matches[lastmatch] = i
                         break
     return matches
 
 def _inlinediff(s1, s2, operation):
     '''Perform string diff to highlight specific changes.'''
     operation_skip = '+?' if operation == 'diff.deleted' else '-?'
     if operation == 'diff.deleted':
         s2, s1 = s1, s2
 
     buff = []
     # we never want to higlight the leading +-
     if operation == 'diff.deleted' and s2.startswith('-'):
         label = operation
         token = '-'
         s2 = s2[1:]
         s1 = s1[1:]
     elif operation == 'diff.inserted' and s1.startswith('+'):
         label = operation
         token = '+'
         s2 = s2[1:]
         s1 = s1[1:]
     else:
         raise error.ProgrammingError("Case not expected, operation = %s" %
                                      operation)
 
     s = difflib.ndiff(_nonwordre.split(s2), _nonwordre.split(s1))
     for part in s:
         if part[0] in operation_skip or len(part) == 2:
             continue
         l = operation + '.highlight'
         if part[0] in ' ':
             l = operation
         if part[2:] == '\t':
             l = 'diff.tab'
         if l == label: # contiguous token with same label
             token += part[2:]
             continue
         else:
             buff.append((token, label))
             label = l
             token = part[2:]
     buff.append((token, label))
 
     return buff
 
 def diffui(*args, **kw):
     '''like diff(), but yields 2-tuples of (output, label) for ui.write()'''
     return difflabel(diff, *args, **kw)
 
 def _filepairs(modified, added, removed, copy, opts):
     '''generates tuples (f1, f2, copyop), where f1 is the name of the file
     before and f2 is the the name after. For added files, f1 will be None,
     and for removed files, f2 will be None. copyop may be set to None, 'copy'
     or 'rename' (the latter two only if opts.git is set).'''
     gone = set()
 
     copyto = dict([(v, k) for k, v in copy.items()])
 
     addedset, removedset = set(added), set(removed)
 
     for f in sorted(modified + added + removed):
         copyop = None
         f1, f2 = f, f
         if f in addedset:
             f1 = None
             if f in copy:
                 if opts.git:
                     f1 = copy[f]
                     if f1 in removedset and f1 not in gone:
                         copyop = 'rename'
                         gone.add(f1)
                     else:
                         copyop = 'copy'
         elif f in removedset:
             f2 = None
             if opts.git:
                 # have we already reported a copy above?
                 if (f in copyto and copyto[f] in addedset
                     and copy[copyto[f]] == f):
                     continue
         yield f1, f2, copyop
 
 def trydiff(repo, revs, ctx1, ctx2, modified, added, removed,
             copy, getfilectx, opts, losedatafn, prefix, relroot):
     '''given input data, generate a diff and yield it in blocks
 
     If generating a diff would lose data like flags or binary data and
     losedatafn is not None, it will be called.
 
     relroot is removed and prefix is added to every path in the diff output.
 
     If relroot is not empty, this function expects every path in modified,
     added, removed and copy to start with it.'''
 
     def gitindex(text):
         if not text:
             text = ""
         l = len(text)
         s = hashlib.sha1('blob %d\0' % l)
         s.update(text)
         return hex(s.digest())
 
     if opts.noprefix:
         aprefix = bprefix = ''
     else:
         aprefix = 'a/'
         bprefix = 'b/'
 
     def diffline(f, revs):
         revinfo = ' '.join(["-r %s" % rev for rev in revs])
         return 'diff %s %s' % (revinfo, f)
 
     def isempty(fctx):
         return fctx is None or fctx.size() == 0
 
     date1 = util.datestr(ctx1.date())
     date2 = util.datestr(ctx2.date())
 
     gitmode = {'l': '120000', 'x': '100755', '': '100644'}
 
     if relroot != '' and (repo.ui.configbool('devel', 'all-warnings')
                           or repo.ui.configbool('devel', 'check-relroot')):
         for f in modified + added + removed + list(copy) + list(copy.values()):
             if f is not None and not f.startswith(relroot):
                 raise AssertionError(
                     "file %s doesn't start with relroot %s" % (f, relroot))
 
     for f1, f2, copyop in _filepairs(modified, added, removed, copy, opts):
         content1 = None
         content2 = None
         fctx1 = None
         fctx2 = None
         flag1 = None
         flag2 = None
         if f1:
             fctx1 = getfilectx(f1, ctx1)
             if opts.git or losedatafn:
                 flag1 = ctx1.flags(f1)
         if f2:
             fctx2 = getfilectx(f2, ctx2)
             if opts.git or losedatafn:
                 flag2 = ctx2.flags(f2)
         # if binary is True, output "summary" or "base85", but not "text diff"
         if opts.text:
             binary = False
         else:
             binary = any(f.isbinary() for f in [fctx1, fctx2] if f is not None)
 
         if losedatafn and not opts.git:
             if (binary or
                 # copy/rename
                 f2 in copy or
                 # empty file creation
                 (not f1 and isempty(fctx2)) or
                 # empty file deletion
                 (isempty(fctx1) and not f2) or
                 # create with flags
                 (not f1 and flag2) or
                 # change flags
                 (f1 and f2 and flag1 != flag2)):
                 losedatafn(f2 or f1)
 
         path1 = f1 or f2
         path2 = f2 or f1
         path1 = posixpath.join(prefix, path1[len(relroot):])
         path2 = posixpath.join(prefix, path2[len(relroot):])
         header = []
         if opts.git:
             header.append('diff --git %s%s %s%s' %
                           (aprefix, path1, bprefix, path2))
             if not f1: # added
                 header.append('new file mode %s' % gitmode[flag2])
             elif not f2: # removed
                 header.append('deleted file mode %s' % gitmode[flag1])
             else:  # modified/copied/renamed
                 mode1, mode2 = gitmode[flag1], gitmode[flag2]
                 if mode1 != mode2:
                     header.append('old mode %s' % mode1)
                     header.append('new mode %s' % mode2)
                 if copyop is not None:
                     if opts.showsimilarity:
                         sim = similar.score(ctx1[path1], ctx2[path2]) * 100
                         header.append('similarity index %d%%' % sim)
                     header.append('%s from %s' % (copyop, path1))
                     header.append('%s to %s' % (copyop, path2))
         elif revs and not repo.ui.quiet:
             header.append(diffline(path1, revs))
 
         #  fctx.is  | diffopts                | what to   | is fctx.data()
         #  binary() | text nobinary git index | output?   | outputted?
         # ------------------------------------|----------------------------
         #  yes      | no   no       no  *     | summary   | no
         #  yes      | no   no       yes *     | base85    | yes
         #  yes      | no   yes      no  *     | summary   | no
         #  yes      | no   yes      yes 0     | summary   | no
         #  yes      | no   yes      yes >0    | summary   | semi [1]
         #  yes      | yes  *        *   *     | text diff | yes
         #  no       | *    *        *   *     | text diff | yes
         # [1]: hash(fctx.data()) is outputted. so fctx.data() cannot be faked
         if binary and (not opts.git or (opts.git and opts.nobinary and not
                                         opts.index)):
             # fast path: no binary content will be displayed, content1 and
             # content2 are only used for equivalent test. cmp() could have a
             # fast path.
             if fctx1 is not None:
                 content1 = b'\0'
             if fctx2 is not None:
                 if fctx1 is not None and not fctx1.cmp(fctx2):
                     content2 = b'\0' # not different
                 else:
                     content2 = b'\0\0'
         else:
             # normal path: load contents
             if fctx1 is not None:
                 content1 = fctx1.data()
             if fctx2 is not None:
                 content2 = fctx2.data()
 
         if binary and opts.git and not opts.nobinary:
             text = mdiff.b85diff(content1, content2)
             if text:
                 header.append('index %s..%s' %
                               (gitindex(content1), gitindex(content2)))
             hunks = (None, [text]),
         else:
             if opts.git and opts.index > 0:
                 flag = flag1
                 if flag is None:
                     flag = flag2
                 header.append('index %s..%s %s' %
                               (gitindex(content1)[0:opts.index],
                                gitindex(content2)[0:opts.index],
                                gitmode[flag]))
 
             uheaders, hunks = mdiff.unidiff(content1, date1,
                                             content2, date2,
                                             path1, path2,
                                             binary=binary, opts=opts)
             header.extend(uheaders)
         yield fctx1, fctx2, header, hunks
 
 def diffstatsum(stats):
     maxfile, maxtotal, addtotal, removetotal, binary = 0, 0, 0, 0, False
     for f, a, r, b in stats:
         maxfile = max(maxfile, encoding.colwidth(f))
         maxtotal = max(maxtotal, a + r)
         addtotal += a
         removetotal += r
         binary = binary or b
 
     return maxfile, maxtotal, addtotal, removetotal, binary
 
 def diffstatdata(lines):
     diffre = re.compile('^diff .*-r [a-z0-9]+\s(.*)$')
 
     results = []
     filename, adds, removes, isbinary = None, 0, 0, False
 
     def addresult():
         if filename:
             results.append((filename, adds, removes, isbinary))
 
     # inheader is used to track if a line is in the
     # header portion of the diff.  This helps properly account
     # for lines that start with '--' or '++'
     inheader = False
 
     for line in lines:
         if line.startswith('diff'):
             addresult()
             # starting a new file diff
             # set numbers to 0 and reset inheader
             inheader = True
             adds, removes, isbinary = 0, 0, False
             if line.startswith('diff --git a/'):
                 filename = gitre.search(line).group(2)
             elif line.startswith('diff -r'):
                 # format: "diff -r ... -r ... filename"
                 filename = diffre.search(line).group(1)
         elif line.startswith('@@'):
             inheader = False
         elif line.startswith('+') and not inheader:
             adds += 1
         elif line.startswith('-') and not inheader:
             removes += 1
         elif (line.startswith('GIT binary patch') or
               line.startswith('Binary file')):
             isbinary = True
     addresult()
     return results
 
 def diffstat(lines, width=80):
     output = []
     stats = diffstatdata(lines)
     maxname, maxtotal, totaladds, totalremoves, hasbinary = diffstatsum(stats)
 
     countwidth = len(str(maxtotal))
     if hasbinary and countwidth < 3:
         countwidth = 3
     graphwidth = width - countwidth - maxname - 6
     if graphwidth < 10:
         graphwidth = 10
 
     def scale(i):
         if maxtotal <= graphwidth:
             return i
         # If diffstat runs out of room it doesn't print anything,
         # which isn't very useful, so always print at least one + or -
         # if there were at least some changes.
         return max(i * graphwidth // maxtotal, int(bool(i)))
 
     for filename, adds, removes, isbinary in stats:
         if isbinary:
             count = 'Bin'
         else:
             count = '%d' % (adds + removes)
         pluses = '+' * scale(adds)
         minuses = '-' * scale(removes)
         output.append(' %s%s |  %*s %s%s\n' %
                       (filename, ' ' * (maxname - encoding.colwidth(filename)),
                        countwidth, count, pluses, minuses))
 
     if stats:
         output.append(_(' %d files changed, %d insertions(+), '
                         '%d deletions(-)\n')
                       % (len(stats), totaladds, totalremoves))
 
     return ''.join(output)
 
 def diffstatui(*args, **kw):
     '''like diffstat(), but yields 2-tuples of (output, label) for
     ui.write()
     '''
 
     for line in diffstat(*args, **kw).splitlines():
         if line and line[-1] in '+-':
             name, graph = line.rsplit(' ', 1)
             yield (name + ' ', '')
             m = re.search(br'\++', graph)
             if m:
                 yield (m.group(0), 'diffstat.inserted')
             m = re.search(br'-+', graph)
             if m:
                 yield (m.group(0), 'diffstat.deleted')
         else:
             yield (line, '')
         yield ('\n', '')

mercurial/progress.py

@@ -1,302 +1,302 b''
 # progress.py progress bars related code
 #
 # Copyright (C) 2010 Augie Fackler <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import errno
 import threading
 import time
 
 from .i18n import _
 from . import encoding
 
 def spacejoin(*args):
     return ' '.join(s for s in args if s)
 
 def shouldprint(ui):
     return not (ui.quiet or ui.plain('progress')) and (
         ui._isatty(ui.ferr) or ui.configbool('progress', 'assume-tty'))
 
 def fmtremaining(seconds):
     """format a number of remaining seconds in human readable way
 
     This will properly display seconds, minutes, hours, days if needed"""
     if seconds < 60:
         # i18n: format XX seconds as "XXs"
         return _("%02ds") % (seconds)
     minutes = seconds // 60
     if minutes < 60:
         seconds -= minutes * 60
         # i18n: format X minutes and YY seconds as "XmYYs"
         return _("%dm%02ds") % (minutes, seconds)
     # we're going to ignore seconds in this case
     minutes += 1
     hours = minutes // 60
     minutes -= hours * 60
     if hours < 30:
         # i18n: format X hours and YY minutes as "XhYYm"
         return _("%dh%02dm") % (hours, minutes)
     # we're going to ignore minutes in this case
     hours += 1
     days = hours // 24
     hours -= days * 24
     if days < 15:
         # i18n: format X days and YY hours as "XdYYh"
         return _("%dd%02dh") % (days, hours)
     # we're going to ignore hours in this case
     days += 1
     weeks = days // 7
     days -= weeks * 7
     if weeks < 55:
         # i18n: format X weeks and YY days as "XwYYd"
         return _("%dw%02dd") % (weeks, days)
     # we're going to ignore days and treat a year as 52 weeks
     weeks += 1
     years = weeks // 52
     weeks -= years * 52
     # i18n: format X years and YY weeks as "XyYYw"
     return _("%dy%02dw") % (years, weeks)
 
 # file_write() and file_flush() of Python 2 do not restart on EINTR if
 # the file is attached to a "slow" device (e.g. a terminal) and raise
 # IOError. We cannot know how many bytes would be written by file_write(),
 # but a progress text is known to be short enough to be written by a
 # single write() syscall, so we can just retry file_write() with the whole
 # text. (issue5532)
 #
 # This should be a short-term workaround. We'll need to fix every occurrence
 # of write() to a terminal or pipe.
 def _eintrretry(func, *args):
     while True:
         try:
             return func(*args)
         except IOError as err:
             if err.errno == errno.EINTR:
                 continue
             raise
 
 class progbar(object):
     def __init__(self, ui):
         self.ui = ui
         self._refreshlock = threading.Lock()
         self.resetstate()
 
     def resetstate(self):
         self.topics = []
         self.topicstates = {}
         self.starttimes = {}
         self.startvals = {}
         self.printed = False
         self.lastprint = time.time() + float(self.ui.config(
             'progress', 'delay'))
         self.curtopic = None
         self.lasttopic = None
         self.indetcount = 0
         self.refresh = float(self.ui.config(
             'progress', 'refresh'))
         self.changedelay = max(3 * self.refresh,
                                float(self.ui.config(
                                    'progress', 'changedelay')))
         self.order = self.ui.configlist('progress', 'format')
         self.estimateinterval = self.ui.configwith(
             float, 'progress', 'estimateinterval')
 
     def show(self, now, topic, pos, item, unit, total):
         if not shouldprint(self.ui):
             return
         termwidth = self.width()
         self.printed = True
         head = ''
         needprogress = False
         tail = ''
         for indicator in self.order:
             add = ''
             if indicator == 'topic':
                 add = topic
             elif indicator == 'number':
                 if total:
                     add = b'%*d/%d' % (len(str(total)), pos, total)
                 else:
-                    add = str(pos)
+                    add = b'%d' % pos
             elif indicator.startswith('item') and item:
                 slice = 'end'
                 if '-' in indicator:
                     wid = int(indicator.split('-')[1])
                 elif '+' in indicator:
                     slice = 'beginning'
                     wid = int(indicator.split('+')[1])
                 else:
                     wid = 20
                 if slice == 'end':
                     add = encoding.trim(item, wid, leftside=True)
                 else:
                     add = encoding.trim(item, wid)
                 add += (wid - encoding.colwidth(add)) * ' '
             elif indicator == 'bar':
                 add = ''
                 needprogress = True
             elif indicator == 'unit' and unit:
                 add = unit
             elif indicator == 'estimate':
                 add = self.estimate(topic, pos, total, now)
             elif indicator == 'speed':
                 add = self.speed(topic, pos, unit, now)
             if not needprogress:
                 head = spacejoin(head, add)
             else:
                 tail = spacejoin(tail, add)
         if needprogress:
             used = 0
             if head:
                 used += encoding.colwidth(head) + 1
             if tail:
                 used += encoding.colwidth(tail) + 1
             progwidth = termwidth - used - 3
             if total and pos <= total:
                 amt = pos * progwidth // total
                 bar = '=' * (amt - 1)
                 if amt > 0:
                     bar += '>'
                 bar += ' ' * (progwidth - amt)
             else:
                 progwidth -= 3
                 self.indetcount += 1
                 # mod the count by twice the width so we can make the
                 # cursor bounce between the right and left sides
                 amt = self.indetcount % (2 * progwidth)
                 amt -= progwidth
                 bar = (' ' * int(progwidth - abs(amt)) + '<=>' +
                        ' ' * int(abs(amt)))
             prog = ''.join(('[', bar, ']'))
             out = spacejoin(head, prog, tail)
         else:
             out = spacejoin(head, tail)
         self._writeerr('\r' + encoding.trim(out, termwidth))
         self.lasttopic = topic
         self._flusherr()
 
     def clear(self):
         if not self.printed or not self.lastprint or not shouldprint(self.ui):
             return
         self._writeerr('\r%s\r' % (' ' * self.width()))
         if self.printed:
             # force immediate re-paint of progress bar
             self.lastprint = 0
 
     def complete(self):
         if not shouldprint(self.ui):
             return
         if self.ui.configbool('progress', 'clear-complete'):
             self.clear()
         else:
             self._writeerr('\n')
         self._flusherr()
 
     def _flusherr(self):
         _eintrretry(self.ui.ferr.flush)
 
     def _writeerr(self, msg):
         _eintrretry(self.ui.ferr.write, msg)
 
     def width(self):
         tw = self.ui.termwidth()
         return min(int(self.ui.config('progress', 'width', default=tw)), tw)
 
     def estimate(self, topic, pos, total, now):
         if total is None:
             return ''
         initialpos = self.startvals[topic]
         target = total - initialpos
         delta = pos - initialpos
         if delta > 0:
             elapsed = now - self.starttimes[topic]
             seconds = (elapsed * (target - delta)) // delta + 1
             return fmtremaining(seconds)
         return ''
 
     def speed(self, topic, pos, unit, now):
         initialpos = self.startvals[topic]
         delta = pos - initialpos
         elapsed = now - self.starttimes[topic]
         if elapsed > 0:
             return _('%d %s/sec') % (delta / elapsed, unit)
         return ''
 
     def _oktoprint(self, now):
         '''Check if conditions are met to print - e.g. changedelay elapsed'''
         if (self.lasttopic is None # first time we printed
             # not a topic change
             or self.curtopic == self.lasttopic
             # it's been long enough we should print anyway
             or now - self.lastprint >= self.changedelay):
             return True
         else:
             return False
 
     def _calibrateestimate(self, topic, now, pos):
         '''Adjust starttimes and startvals for topic so ETA works better
 
         If progress is non-linear (ex. get much slower in the last minute),
         it's more friendly to only use a recent time span for ETA and speed
         calculation.
 
             [======================================>       ]
                                              ^^^^^^^
                            estimateinterval, only use this for estimation
         '''
         interval = self.estimateinterval
         if interval <= 0:
             return
         elapsed = now - self.starttimes[topic]
         if elapsed > interval:
             delta = pos - self.startvals[topic]
             newdelta = delta * interval / elapsed
             # If a stall happens temporarily, ETA could change dramatically
             # frequently. This is to avoid such dramatical change and make ETA
             # smoother.
             if newdelta < 0.1:
                 return
             self.startvals[topic] = pos - newdelta
             self.starttimes[topic] = now - interval
 
     def progress(self, topic, pos, item='', unit='', total=None):
         now = time.time()
         self._refreshlock.acquire()
         try:
             if pos is None:
                 self.starttimes.pop(topic, None)
                 self.startvals.pop(topic, None)
                 self.topicstates.pop(topic, None)
                 # reset the progress bar if this is the outermost topic
                 if self.topics and self.topics[0] == topic and self.printed:
                     self.complete()
                     self.resetstate()
                 # truncate the list of topics assuming all topics within
                 # this one are also closed
                 if topic in self.topics:
                     self.topics = self.topics[:self.topics.index(topic)]
                     # reset the last topic to the one we just unwound to,
                     # so that higher-level topics will be stickier than
                     # lower-level topics
                     if self.topics:
                         self.lasttopic = self.topics[-1]
                     else:
                         self.lasttopic = None
             else:
                 if topic not in self.topics:
                     self.starttimes[topic] = now
                     self.startvals[topic] = pos
                     self.topics.append(topic)
                 self.topicstates[topic] = pos, item, unit, total
                 self.curtopic = topic
                 self._calibrateestimate(topic, now, pos)
                 if now - self.lastprint >= self.refresh and self.topics:
                     if self._oktoprint(now):
                         self.lastprint = now
                         self.show(now, topic, *self.topicstates[topic])
         finally:
             self._refreshlock.release()