stringutil: bulk-replace call sites to point to new module

This might conflict with other patches floating around, sorry.

hgext/bugzilla.py

@@ -1,1124 +1,1128
 # bugzilla.py - bugzilla integration for mercurial
 #
 # Copyright 2006 Vadim Gelfer <[e-mail]>
 # Copyright 2011-4 Jim Hague <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''hooks for integrating with the Bugzilla bug tracker
 
 This hook extension adds comments on bugs in Bugzilla when changesets
 that refer to bugs by Bugzilla ID are seen. The comment is formatted using
 the Mercurial template mechanism.
 
 The bug references can optionally include an update for Bugzilla of the
 hours spent working on the bug. Bugs can also be marked fixed.
 
 Four basic modes of access to Bugzilla are provided:
 
 1. Access via the Bugzilla REST-API. Requires bugzilla 5.0 or later.
 
 2. Access via the Bugzilla XMLRPC interface. Requires Bugzilla 3.4 or later.
 
 3. Check data via the Bugzilla XMLRPC interface and submit bug change
    via email to Bugzilla email interface. Requires Bugzilla 3.4 or later.
 
 4. Writing directly to the Bugzilla database. Only Bugzilla installations
    using MySQL are supported. Requires Python MySQLdb.
 
 Writing directly to the database is susceptible to schema changes, and
 relies on a Bugzilla contrib script to send out bug change
 notification emails. This script runs as the user running Mercurial,
 must be run on the host with the Bugzilla install, and requires
 permission to read Bugzilla configuration details and the necessary
 MySQL user and password to have full access rights to the Bugzilla
 database. For these reasons this access mode is now considered
 deprecated, and will not be updated for new Bugzilla versions going
 forward. Only adding comments is supported in this access mode.
 
 Access via XMLRPC needs a Bugzilla username and password to be specified
 in the configuration. Comments are added under that username. Since the
 configuration must be readable by all Mercurial users, it is recommended
 that the rights of that user are restricted in Bugzilla to the minimum
 necessary to add comments. Marking bugs fixed requires Bugzilla 4.0 and later.
 
 Access via XMLRPC/email uses XMLRPC to query Bugzilla, but sends
 email to the Bugzilla email interface to submit comments to bugs.
 The From: address in the email is set to the email address of the Mercurial
 user, so the comment appears to come from the Mercurial user. In the event
 that the Mercurial user email is not recognized by Bugzilla as a Bugzilla
 user, the email associated with the Bugzilla username used to log into
 Bugzilla is used instead as the source of the comment. Marking bugs fixed
 works on all supported Bugzilla versions.
 
 Access via the REST-API needs either a Bugzilla username and password
 or an apikey specified in the configuration. Comments are made under
 the given username or the user associated with the apikey in Bugzilla.
 
 Configuration items common to all access modes:
 
 bugzilla.version
   The access type to use. Values recognized are:
 
   :``restapi``:      Bugzilla REST-API, Bugzilla 5.0 and later.
   :``xmlrpc``:       Bugzilla XMLRPC interface.
   :``xmlrpc+email``: Bugzilla XMLRPC and email interfaces.
   :``3.0``:          MySQL access, Bugzilla 3.0 and later.
   :``2.18``:         MySQL access, Bugzilla 2.18 and up to but not
                      including 3.0.
   :``2.16``:         MySQL access, Bugzilla 2.16 and up to but not
                      including 2.18.
 
 bugzilla.regexp
   Regular expression to match bug IDs for update in changeset commit message.
   It must contain one "()" named group ``<ids>`` containing the bug
   IDs separated by non-digit characters. It may also contain
   a named group ``<hours>`` with a floating-point number giving the
   hours worked on the bug. If no named groups are present, the first
   "()" group is assumed to contain the bug IDs, and work time is not
   updated. The default expression matches ``Bug 1234``, ``Bug no. 1234``,
   ``Bug number 1234``, ``Bugs 1234,5678``, ``Bug 1234 and 5678`` and
   variations thereof, followed by an hours number prefixed by ``h`` or
   ``hours``, e.g. ``hours 1.5``. Matching is case insensitive.
 
 bugzilla.fixregexp
   Regular expression to match bug IDs for marking fixed in changeset
   commit message. This must contain a "()" named group ``<ids>` containing
   the bug IDs separated by non-digit characters. It may also contain
   a named group ``<hours>`` with a floating-point number giving the
   hours worked on the bug. If no named groups are present, the first
   "()" group is assumed to contain the bug IDs, and work time is not
   updated. The default expression matches ``Fixes 1234``, ``Fixes bug 1234``,
   ``Fixes bugs 1234,5678``, ``Fixes 1234 and 5678`` and
   variations thereof, followed by an hours number prefixed by ``h`` or
   ``hours``, e.g. ``hours 1.5``. Matching is case insensitive.
 
 bugzilla.fixstatus
   The status to set a bug to when marking fixed. Default ``RESOLVED``.
 
 bugzilla.fixresolution
   The resolution to set a bug to when marking fixed. Default ``FIXED``.
 
 bugzilla.style
   The style file to use when formatting comments.
 
 bugzilla.template
   Template to use when formatting comments. Overrides style if
   specified. In addition to the usual Mercurial keywords, the
   extension specifies:
 
   :``{bug}``:     The Bugzilla bug ID.
   :``{root}``:    The full pathname of the Mercurial repository.
   :``{webroot}``: Stripped pathname of the Mercurial repository.
   :``{hgweb}``:   Base URL for browsing Mercurial repositories.
 
   Default ``changeset {node|short} in repo {root} refers to bug
   {bug}.\\ndetails:\\n\\t{desc|tabindent}``
 
 bugzilla.strip
   The number of path separator characters to strip from the front of
   the Mercurial repository path (``{root}`` in templates) to produce
   ``{webroot}``. For example, a repository with ``{root}``
   ``/var/local/my-project`` with a strip of 2 gives a value for
   ``{webroot}`` of ``my-project``. Default 0.
 
 web.baseurl
   Base URL for browsing Mercurial repositories. Referenced from
   templates as ``{hgweb}``.
 
 Configuration items common to XMLRPC+email and MySQL access modes:
 
 bugzilla.usermap
   Path of file containing Mercurial committer email to Bugzilla user email
   mappings. If specified, the file should contain one mapping per
   line::
 
     committer = Bugzilla user
 
   See also the ``[usermap]`` section.
 
 The ``[usermap]`` section is used to specify mappings of Mercurial
 committer email to Bugzilla user email. See also ``bugzilla.usermap``.
 Contains entries of the form ``committer = Bugzilla user``.
 
 XMLRPC and REST-API access mode configuration:
 
 bugzilla.bzurl
   The base URL for the Bugzilla installation.
   Default ``http://localhost/bugzilla``.
 
 bugzilla.user
   The username to use to log into Bugzilla via XMLRPC. Default
   ``bugs``.
 
 bugzilla.password
   The password for Bugzilla login.
 
 REST-API access mode uses the options listed above as well as:
 
 bugzilla.apikey
   An apikey generated on the Bugzilla instance for api access.
   Using an apikey removes the need to store the user and password
   options.
 
 XMLRPC+email access mode uses the XMLRPC access mode configuration items,
 and also:
 
 bugzilla.bzemail
   The Bugzilla email address.
 
 In addition, the Mercurial email settings must be configured. See the
 documentation in hgrc(5), sections ``[email]`` and ``[smtp]``.
 
 MySQL access mode configuration:
 
 bugzilla.host
   Hostname of the MySQL server holding the Bugzilla database.
   Default ``localhost``.
 
 bugzilla.db
   Name of the Bugzilla database in MySQL. Default ``bugs``.
 
 bugzilla.user
   Username to use to access MySQL server. Default ``bugs``.
 
 bugzilla.password
   Password to use to access MySQL server.
 
 bugzilla.timeout
   Database connection timeout (seconds). Default 5.
 
 bugzilla.bzuser
   Fallback Bugzilla user name to record comments with, if changeset
   committer cannot be found as a Bugzilla user.
 
 bugzilla.bzdir
    Bugzilla install directory. Used by default notify. Default
    ``/var/www/html/bugzilla``.
 
 bugzilla.notify
   The command to run to get Bugzilla to send bug change notification
   emails. Substitutes from a map with 3 keys, ``bzdir``, ``id`` (bug
   id) and ``user`` (committer bugzilla email). Default depends on
   version; from 2.18 it is "cd %(bzdir)s && perl -T
   contrib/sendbugmail.pl %(id)s %(user)s".
 
 Activating the extension::
 
     [extensions]
     bugzilla =
 
     [hooks]
     # run bugzilla hook on every change pulled or pushed in here
     incoming.bugzilla = python:hgext.bugzilla.hook
 
 Example configurations:
 
 XMLRPC example configuration. This uses the Bugzilla at
 ``http://my-project.org/bugzilla``, logging in as user
 ``[e-mail]`` with password ``plugh``. It is used with a
 collection of Mercurial repositories in ``/var/local/hg/repos/``,
 with a web interface at ``http://my-project.org/hg``. ::
 
     [bugzilla]
     bzurl=http://my-project.org/bugzilla
     user=[e-mail]
     password=plugh
     version=xmlrpc
     template=Changeset {node|short} in {root|basename}.
              {hgweb}/{webroot}/rev/{node|short}\\n
              {desc}\\n
     strip=5
 
     [web]
     baseurl=http://my-project.org/hg
 
 XMLRPC+email example configuration. This uses the Bugzilla at
 ``http://my-project.org/bugzilla``, logging in as user
 ``[e-mail]`` with password ``plugh``. It is used with a
 collection of Mercurial repositories in ``/var/local/hg/repos/``,
 with a web interface at ``http://my-project.org/hg``. Bug comments
 are sent to the Bugzilla email address
 ``[e-mail]``. ::
 
     [bugzilla]
     bzurl=http://my-project.org/bugzilla
     user=[e-mail]
     password=plugh
     version=xmlrpc+email
     bzemail=[e-mail]
     template=Changeset {node|short} in {root|basename}.
              {hgweb}/{webroot}/rev/{node|short}\\n
              {desc}\\n
     strip=5
 
     [web]
     baseurl=http://my-project.org/hg
 
     [usermap]
     [e-mail]=[e-mail]
 
 MySQL example configuration. This has a local Bugzilla 3.2 installation
 in ``/opt/bugzilla-3.2``. The MySQL database is on ``localhost``,
 the Bugzilla database name is ``bugs`` and MySQL is
 accessed with MySQL username ``bugs`` password ``XYZZY``. It is used
 with a collection of Mercurial repositories in ``/var/local/hg/repos/``,
 with a web interface at ``http://my-project.org/hg``. ::
 
     [bugzilla]
     host=localhost
     password=XYZZY
     version=3.0
     bzuser=[e-mail]
     bzdir=/opt/bugzilla-3.2
     template=Changeset {node|short} in {root|basename}.
              {hgweb}/{webroot}/rev/{node|short}\\n
              {desc}\\n
     strip=5
 
     [web]
     baseurl=http://my-project.org/hg
 
     [usermap]
     [e-mail]=[e-mail]
 
 All the above add a comment to the Bugzilla bug record of the form::
 
     Changeset 3b16791d6642 in repository-name.
     http://my-project.org/hg/repository-name/rev/3b16791d6642
 
     Changeset commit comment. Bug 1234.
 '''
 
 from __future__ import absolute_import
 
 import json
 import re
 import time
 
 from mercurial.i18n import _
 from mercurial.node import short
 from mercurial import (
     error,
     logcmdutil,
     mail,
     registrar,
     url,
     util,
 )
+from mercurial.utils import (
+    stringutil,
+)
 
 xmlrpclib = util.xmlrpclib
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 configitem('bugzilla', 'apikey',
     default='',
 )
 configitem('bugzilla', 'bzdir',
     default='/var/www/html/bugzilla',
 )
 configitem('bugzilla', 'bzemail',
     default=None,
 )
 configitem('bugzilla', 'bzurl',
     default='http://localhost/bugzilla/',
 )
 configitem('bugzilla', 'bzuser',
     default=None,
 )
 configitem('bugzilla', 'db',
     default='bugs',
 )
 configitem('bugzilla', 'fixregexp',
     default=(r'fix(?:es)?\s*(?:bugs?\s*)?,?\s*'
              r'(?:nos?\.?|num(?:ber)?s?)?\s*'
              r'(?P<ids>(?:#?\d+\s*(?:,?\s*(?:and)?)?\s*)+)'
              r'\.?\s*(?:h(?:ours?)?\s*(?P<hours>\d*(?:\.\d+)?))?')
 )
 configitem('bugzilla', 'fixresolution',
     default='FIXED',
 )
 configitem('bugzilla', 'fixstatus',
     default='RESOLVED',
 )
 configitem('bugzilla', 'host',
     default='localhost',
 )
 configitem('bugzilla', 'notify',
     default=configitem.dynamicdefault,
 )
 configitem('bugzilla', 'password',
     default=None,
 )
 configitem('bugzilla', 'regexp',
     default=(r'bugs?\s*,?\s*(?:#|nos?\.?|num(?:ber)?s?)?\s*'
              r'(?P<ids>(?:\d+\s*(?:,?\s*(?:and)?)?\s*)+)'
              r'\.?\s*(?:h(?:ours?)?\s*(?P<hours>\d*(?:\.\d+)?))?')
 )
 configitem('bugzilla', 'strip',
     default=0,
 )
 configitem('bugzilla', 'style',
     default=None,
 )
 configitem('bugzilla', 'template',
     default=None,
 )
 configitem('bugzilla', 'timeout',
     default=5,
 )
 configitem('bugzilla', 'user',
     default='bugs',
 )
 configitem('bugzilla', 'usermap',
     default=None,
 )
 configitem('bugzilla', 'version',
     default=None,
 )
 
 class bzaccess(object):
     '''Base class for access to Bugzilla.'''
 
     def __init__(self, ui):
         self.ui = ui
         usermap = self.ui.config('bugzilla', 'usermap')
         if usermap:
             self.ui.readconfig(usermap, sections=['usermap'])
 
     def map_committer(self, user):
         '''map name of committer to Bugzilla user name.'''
         for committer, bzuser in self.ui.configitems('usermap'):
             if committer.lower() == user.lower():
                 return bzuser
         return user
 
     # Methods to be implemented by access classes.
     #
     # 'bugs' is a dict keyed on bug id, where values are a dict holding
     # updates to bug state. Recognized dict keys are:
     #
     # 'hours': Value, float containing work hours to be updated.
     # 'fix':   If key present, bug is to be marked fixed. Value ignored.
 
     def filter_real_bug_ids(self, bugs):
         '''remove bug IDs that do not exist in Bugzilla from bugs.'''
 
     def filter_cset_known_bug_ids(self, node, bugs):
         '''remove bug IDs where node occurs in comment text from bugs.'''
 
     def updatebug(self, bugid, newstate, text, committer):
         '''update the specified bug. Add comment text and set new states.
 
         If possible add the comment as being from the committer of
         the changeset. Otherwise use the default Bugzilla user.
         '''
 
     def notify(self, bugs, committer):
         '''Force sending of Bugzilla notification emails.
 
         Only required if the access method does not trigger notification
         emails automatically.
         '''
 
 # Bugzilla via direct access to MySQL database.
 class bzmysql(bzaccess):
     '''Support for direct MySQL access to Bugzilla.
 
     The earliest Bugzilla version this is tested with is version 2.16.
 
     If your Bugzilla is version 3.4 or above, you are strongly
     recommended to use the XMLRPC access method instead.
     '''
 
     @staticmethod
     def sql_buglist(ids):
         '''return SQL-friendly list of bug ids'''
         return '(' + ','.join(map(str, ids)) + ')'
 
     _MySQLdb = None
 
     def __init__(self, ui):
         try:
             import MySQLdb as mysql
             bzmysql._MySQLdb = mysql
         except ImportError as err:
             raise error.Abort(_('python mysql support not available: %s') % err)
 
         bzaccess.__init__(self, ui)
 
         host = self.ui.config('bugzilla', 'host')
         user = self.ui.config('bugzilla', 'user')
         passwd = self.ui.config('bugzilla', 'password')
         db = self.ui.config('bugzilla', 'db')
         timeout = int(self.ui.config('bugzilla', 'timeout'))
         self.ui.note(_('connecting to %s:%s as %s, password %s\n') %
                      (host, db, user, '*' * len(passwd)))
         self.conn = bzmysql._MySQLdb.connect(host=host,
                                                    user=user, passwd=passwd,
                                                    db=db,
                                                    connect_timeout=timeout)
         self.cursor = self.conn.cursor()
         self.longdesc_id = self.get_longdesc_id()
         self.user_ids = {}
         self.default_notify = "cd %(bzdir)s && ./processmail %(id)s %(user)s"
 
     def run(self, *args, **kwargs):
         '''run a query.'''
         self.ui.note(_('query: %s %s\n') % (args, kwargs))
         try:
             self.cursor.execute(*args, **kwargs)
         except bzmysql._MySQLdb.MySQLError:
             self.ui.note(_('failed query: %s %s\n') % (args, kwargs))
             raise
 
     def get_longdesc_id(self):
         '''get identity of longdesc field'''
         self.run('select fieldid from fielddefs where name = "longdesc"')
         ids = self.cursor.fetchall()
         if len(ids) != 1:
             raise error.Abort(_('unknown database schema'))
         return ids[0][0]
 
     def filter_real_bug_ids(self, bugs):
         '''filter not-existing bugs from set.'''
         self.run('select bug_id from bugs where bug_id in %s' %
                  bzmysql.sql_buglist(bugs.keys()))
         existing = [id for (id,) in self.cursor.fetchall()]
         for id in bugs.keys():
             if id not in existing:
                 self.ui.status(_('bug %d does not exist\n') % id)
                 del bugs[id]
 
     def filter_cset_known_bug_ids(self, node, bugs):
         '''filter bug ids that already refer to this changeset from set.'''
         self.run('''select bug_id from longdescs where
                     bug_id in %s and thetext like "%%%s%%"''' %
                  (bzmysql.sql_buglist(bugs.keys()), short(node)))
         for (id,) in self.cursor.fetchall():
             self.ui.status(_('bug %d already knows about changeset %s\n') %
                            (id, short(node)))
             del bugs[id]
 
     def notify(self, bugs, committer):
         '''tell bugzilla to send mail.'''
         self.ui.status(_('telling bugzilla to send mail:\n'))
         (user, userid) = self.get_bugzilla_user(committer)
         for id in bugs.keys():
             self.ui.status(_('  bug %s\n') % id)
             cmdfmt = self.ui.config('bugzilla', 'notify', self.default_notify)
             bzdir = self.ui.config('bugzilla', 'bzdir')
             try:
                 # Backwards-compatible with old notify string, which
                 # took one string. This will throw with a new format
                 # string.
                 cmd = cmdfmt % id
             except TypeError:
                 cmd = cmdfmt % {'bzdir': bzdir, 'id': id, 'user': user}
             self.ui.note(_('running notify command %s\n') % cmd)
             fp = util.popen('(%s) 2>&1' % cmd)
             out = fp.read()
             ret = fp.close()
             if ret:
                 self.ui.warn(out)
                 raise error.Abort(_('bugzilla notify command %s') %
                                  util.explainexit(ret)[0])
         self.ui.status(_('done\n'))
 
     def get_user_id(self, user):
         '''look up numeric bugzilla user id.'''
         try:
             return self.user_ids[user]
         except KeyError:
             try:
                 userid = int(user)
             except ValueError:
                 self.ui.note(_('looking up user %s\n') % user)
                 self.run('''select userid from profiles
                             where login_name like %s''', user)
                 all = self.cursor.fetchall()
                 if len(all) != 1:
                     raise KeyError(user)
                 userid = int(all[0][0])
             self.user_ids[user] = userid
             return userid
 
     def get_bugzilla_user(self, committer):
         '''See if committer is a registered bugzilla user. Return
         bugzilla username and userid if so. If not, return default
         bugzilla username and userid.'''
         user = self.map_committer(committer)
         try:
             userid = self.get_user_id(user)
         except KeyError:
             try:
                 defaultuser = self.ui.config('bugzilla', 'bzuser')
                 if not defaultuser:
                     raise error.Abort(_('cannot find bugzilla user id for %s') %
                                      user)
                 userid = self.get_user_id(defaultuser)
                 user = defaultuser
             except KeyError:
                 raise error.Abort(_('cannot find bugzilla user id for %s or %s')
                                  % (user, defaultuser))
         return (user, userid)
 
     def updatebug(self, bugid, newstate, text, committer):
         '''update bug state with comment text.
 
         Try adding comment as committer of changeset, otherwise as
         default bugzilla user.'''
         if len(newstate) > 0:
             self.ui.warn(_("Bugzilla/MySQL cannot update bug state\n"))
 
         (user, userid) = self.get_bugzilla_user(committer)
         now = time.strftime(r'%Y-%m-%d %H:%M:%S')
         self.run('''insert into longdescs
                     (bug_id, who, bug_when, thetext)
                     values (%s, %s, %s, %s)''',
                  (bugid, userid, now, text))
         self.run('''insert into bugs_activity (bug_id, who, bug_when, fieldid)
                     values (%s, %s, %s, %s)''',
                  (bugid, userid, now, self.longdesc_id))
         self.conn.commit()
 
 class bzmysql_2_18(bzmysql):
     '''support for bugzilla 2.18 series.'''
 
     def __init__(self, ui):
         bzmysql.__init__(self, ui)
         self.default_notify = \
             "cd %(bzdir)s && perl -T contrib/sendbugmail.pl %(id)s %(user)s"
 
 class bzmysql_3_0(bzmysql_2_18):
     '''support for bugzilla 3.0 series.'''
 
     def __init__(self, ui):
         bzmysql_2_18.__init__(self, ui)
 
     def get_longdesc_id(self):
         '''get identity of longdesc field'''
         self.run('select id from fielddefs where name = "longdesc"')
         ids = self.cursor.fetchall()
         if len(ids) != 1:
             raise error.Abort(_('unknown database schema'))
         return ids[0][0]
 
 # Bugzilla via XMLRPC interface.
 
 class cookietransportrequest(object):
     """A Transport request method that retains cookies over its lifetime.
 
     The regular xmlrpclib transports ignore cookies. Which causes
     a bit of a problem when you need a cookie-based login, as with
     the Bugzilla XMLRPC interface prior to 4.4.3.
 
     So this is a helper for defining a Transport which looks for
     cookies being set in responses and saves them to add to all future
     requests.
     """
 
     # Inspiration drawn from
     # http://blog.godson.in/2010/09/how-to-make-python-xmlrpclib-client.html
     # http://www.itkovian.net/base/transport-class-for-pythons-xml-rpc-lib/
 
     cookies = []
     def send_cookies(self, connection):
         if self.cookies:
             for cookie in self.cookies:
                 connection.putheader("Cookie", cookie)
 
     def request(self, host, handler, request_body, verbose=0):
         self.verbose = verbose
         self.accept_gzip_encoding = False
 
         # issue XML-RPC request
         h = self.make_connection(host)
         if verbose:
             h.set_debuglevel(1)
 
         self.send_request(h, handler, request_body)
         self.send_host(h, host)
         self.send_cookies(h)
         self.send_user_agent(h)
         self.send_content(h, request_body)
 
         # Deal with differences between Python 2.6 and 2.7.
         # In the former h is a HTTP(S). In the latter it's a
         # HTTP(S)Connection. Luckily, the 2.6 implementation of
         # HTTP(S) has an underlying HTTP(S)Connection, so extract
         # that and use it.
         try:
             response = h.getresponse()
         except AttributeError:
             response = h._conn.getresponse()
 
         # Add any cookie definitions to our list.
         for header in response.msg.getallmatchingheaders("Set-Cookie"):
             val = header.split(": ", 1)[1]
             cookie = val.split(";", 1)[0]
             self.cookies.append(cookie)
 
         if response.status != 200:
             raise xmlrpclib.ProtocolError(host + handler, response.status,
                                           response.reason, response.msg.headers)
 
         payload = response.read()
         parser, unmarshaller = self.getparser()
         parser.feed(payload)
         parser.close()
 
         return unmarshaller.close()
 
 # The explicit calls to the underlying xmlrpclib __init__() methods are
 # necessary. The xmlrpclib.Transport classes are old-style classes, and
 # it turns out their __init__() doesn't get called when doing multiple
 # inheritance with a new-style class.
 class cookietransport(cookietransportrequest, xmlrpclib.Transport):
     def __init__(self, use_datetime=0):
         if util.safehasattr(xmlrpclib.Transport, "__init__"):
             xmlrpclib.Transport.__init__(self, use_datetime)
 
 class cookiesafetransport(cookietransportrequest, xmlrpclib.SafeTransport):
     def __init__(self, use_datetime=0):
         if util.safehasattr(xmlrpclib.Transport, "__init__"):
             xmlrpclib.SafeTransport.__init__(self, use_datetime)
 
 class bzxmlrpc(bzaccess):
     """Support for access to Bugzilla via the Bugzilla XMLRPC API.
 
     Requires a minimum Bugzilla version 3.4.
     """
 
     def __init__(self, ui):
         bzaccess.__init__(self, ui)
 
         bzweb = self.ui.config('bugzilla', 'bzurl')
         bzweb = bzweb.rstrip("/") + "/xmlrpc.cgi"
 
         user = self.ui.config('bugzilla', 'user')
         passwd = self.ui.config('bugzilla', 'password')
 
         self.fixstatus = self.ui.config('bugzilla', 'fixstatus')
         self.fixresolution = self.ui.config('bugzilla', 'fixresolution')
 
         self.bzproxy = xmlrpclib.ServerProxy(bzweb, self.transport(bzweb))
         ver = self.bzproxy.Bugzilla.version()['version'].split('.')
         self.bzvermajor = int(ver[0])
         self.bzverminor = int(ver[1])
         login = self.bzproxy.User.login({'login': user, 'password': passwd,
                                          'restrict_login': True})
         self.bztoken = login.get('token', '')
 
     def transport(self, uri):
         if util.urlreq.urlparse(uri, "http")[0] == "https":
             return cookiesafetransport()
         else:
             return cookietransport()
 
     def get_bug_comments(self, id):
         """Return a string with all comment text for a bug."""
         c = self.bzproxy.Bug.comments({'ids': [id],
                                        'include_fields': ['text'],
                                        'token': self.bztoken})
         return ''.join([t['text'] for t in c['bugs'][str(id)]['comments']])
 
     def filter_real_bug_ids(self, bugs):
         probe = self.bzproxy.Bug.get({'ids': sorted(bugs.keys()),
                                       'include_fields': [],
                                       'permissive': True,
                                       'token': self.bztoken,
                                       })
         for badbug in probe['faults']:
             id = badbug['id']
             self.ui.status(_('bug %d does not exist\n') % id)
             del bugs[id]
 
     def filter_cset_known_bug_ids(self, node, bugs):
         for id in sorted(bugs.keys()):
             if self.get_bug_comments(id).find(short(node)) != -1:
                 self.ui.status(_('bug %d already knows about changeset %s\n') %
                                (id, short(node)))
                 del bugs[id]
 
     def updatebug(self, bugid, newstate, text, committer):
         args = {}
         if 'hours' in newstate:
             args['work_time'] = newstate['hours']
 
         if self.bzvermajor >= 4:
             args['ids'] = [bugid]
             args['comment'] = {'body' : text}
             if 'fix' in newstate:
                 args['status'] = self.fixstatus
                 args['resolution'] = self.fixresolution
             args['token'] = self.bztoken
             self.bzproxy.Bug.update(args)
         else:
             if 'fix' in newstate:
                 self.ui.warn(_("Bugzilla/XMLRPC needs Bugzilla 4.0 or later "
                                "to mark bugs fixed\n"))
             args['id'] = bugid
             args['comment'] = text
             self.bzproxy.Bug.add_comment(args)
 
 class bzxmlrpcemail(bzxmlrpc):
     """Read data from Bugzilla via XMLRPC, send updates via email.
 
     Advantages of sending updates via email:
       1. Comments can be added as any user, not just logged in user.
       2. Bug statuses or other fields not accessible via XMLRPC can
          potentially be updated.
 
     There is no XMLRPC function to change bug status before Bugzilla
     4.0, so bugs cannot be marked fixed via XMLRPC before Bugzilla 4.0.
     But bugs can be marked fixed via email from 3.4 onwards.
     """
 
     # The email interface changes subtly between 3.4 and 3.6. In 3.4,
     # in-email fields are specified as '@<fieldname> = <value>'. In
     # 3.6 this becomes '@<fieldname> <value>'. And fieldname @bug_id
     # in 3.4 becomes @id in 3.6. 3.6 and 4.0 both maintain backwards
     # compatibility, but rather than rely on this use the new format for
     # 4.0 onwards.
 
     def __init__(self, ui):
         bzxmlrpc.__init__(self, ui)
 
         self.bzemail = self.ui.config('bugzilla', 'bzemail')
         if not self.bzemail:
             raise error.Abort(_("configuration 'bzemail' missing"))
         mail.validateconfig(self.ui)
 
     def makecommandline(self, fieldname, value):
         if self.bzvermajor >= 4:
             return "@%s %s" % (fieldname, str(value))
         else:
             if fieldname == "id":
                 fieldname = "bug_id"
             return "@%s = %s" % (fieldname, str(value))
 
     def send_bug_modify_email(self, bugid, commands, comment, committer):
         '''send modification message to Bugzilla bug via email.
 
         The message format is documented in the Bugzilla email_in.pl
         specification. commands is a list of command lines, comment is the
         comment text.
 
         To stop users from crafting commit comments with
         Bugzilla commands, specify the bug ID via the message body, rather
         than the subject line, and leave a blank line after it.
         '''
         user = self.map_committer(committer)
         matches = self.bzproxy.User.get({'match': [user],
                                          'token': self.bztoken})
         if not matches['users']:
             user = self.ui.config('bugzilla', 'user')
             matches = self.bzproxy.User.get({'match': [user],
                                              'token': self.bztoken})
             if not matches['users']:
                 raise error.Abort(_("default bugzilla user %s email not found")
                                   % user)
         user = matches['users'][0]['email']
         commands.append(self.makecommandline("id", bugid))
 
         text = "\n".join(commands) + "\n\n" + comment
 
         _charsets = mail._charsets(self.ui)
         user = mail.addressencode(self.ui, user, _charsets)
         bzemail = mail.addressencode(self.ui, self.bzemail, _charsets)
         msg = mail.mimeencode(self.ui, text, _charsets)
         msg['From'] = user
         msg['To'] = bzemail
         msg['Subject'] = mail.headencode(self.ui, "Bug modification", _charsets)
         sendmail = mail.connect(self.ui)
         sendmail(user, bzemail, msg.as_string())
 
     def updatebug(self, bugid, newstate, text, committer):
         cmds = []
         if 'hours' in newstate:
             cmds.append(self.makecommandline("work_time", newstate['hours']))
         if 'fix' in newstate:
             cmds.append(self.makecommandline("bug_status", self.fixstatus))
             cmds.append(self.makecommandline("resolution", self.fixresolution))
         self.send_bug_modify_email(bugid, cmds, text, committer)
 
 class NotFound(LookupError):
     pass
 
 class bzrestapi(bzaccess):
     """Read and write bugzilla data using the REST API available since
     Bugzilla 5.0.
     """
     def __init__(self, ui):
         bzaccess.__init__(self, ui)
         bz = self.ui.config('bugzilla', 'bzurl')
         self.bzroot = '/'.join([bz, 'rest'])
         self.apikey = self.ui.config('bugzilla', 'apikey')
         self.user = self.ui.config('bugzilla', 'user')
         self.passwd = self.ui.config('bugzilla', 'password')
         self.fixstatus = self.ui.config('bugzilla', 'fixstatus')
         self.fixresolution = self.ui.config('bugzilla', 'fixresolution')
 
     def apiurl(self, targets, include_fields=None):
         url = '/'.join([self.bzroot] + [str(t) for t in targets])
         qv = {}
         if self.apikey:
             qv['api_key'] = self.apikey
         elif self.user and self.passwd:
             qv['login'] = self.user
             qv['password'] = self.passwd
         if include_fields:
             qv['include_fields'] = include_fields
         if qv:
             url = '%s?%s' % (url, util.urlreq.urlencode(qv))
         return url
 
     def _fetch(self, burl):
         try:
             resp = url.open(self.ui, burl)
             return json.loads(resp.read())
         except util.urlerr.httperror as inst:
             if inst.code == 401:
                 raise error.Abort(_('authorization failed'))
             if inst.code == 404:
                 raise NotFound()
             else:
                 raise
 
     def _submit(self, burl, data, method='POST'):
         data = json.dumps(data)
         if method == 'PUT':
             class putrequest(util.urlreq.request):
                 def get_method(self):
                     return 'PUT'
             request_type = putrequest
         else:
             request_type = util.urlreq.request
         req = request_type(burl, data,
                            {'Content-Type': 'application/json'})
         try:
             resp = url.opener(self.ui).open(req)
             return json.loads(resp.read())
         except util.urlerr.httperror as inst:
             if inst.code == 401:
                 raise error.Abort(_('authorization failed'))
             if inst.code == 404:
                 raise NotFound()
             else:
                 raise
 
     def filter_real_bug_ids(self, bugs):
         '''remove bug IDs that do not exist in Bugzilla from bugs.'''
         badbugs = set()
         for bugid in bugs:
             burl = self.apiurl(('bug', bugid), include_fields='status')
             try:
                 self._fetch(burl)
             except NotFound:
                 badbugs.add(bugid)
         for bugid in badbugs:
             del bugs[bugid]
 
     def filter_cset_known_bug_ids(self, node, bugs):
         '''remove bug IDs where node occurs in comment text from bugs.'''
         sn = short(node)
         for bugid in bugs.keys():
             burl = self.apiurl(('bug', bugid, 'comment'), include_fields='text')
             result = self._fetch(burl)
             comments = result['bugs'][str(bugid)]['comments']
             if any(sn in c['text'] for c in comments):
                 self.ui.status(_('bug %d already knows about changeset %s\n') %
                                (bugid, sn))
                 del bugs[bugid]
 
     def updatebug(self, bugid, newstate, text, committer):
         '''update the specified bug. Add comment text and set new states.
 
         If possible add the comment as being from the committer of
         the changeset. Otherwise use the default Bugzilla user.
         '''
         bugmod = {}
         if 'hours' in newstate:
             bugmod['work_time'] = newstate['hours']
         if 'fix' in newstate:
             bugmod['status'] = self.fixstatus
             bugmod['resolution'] = self.fixresolution
         if bugmod:
             # if we have to change the bugs state do it here
             bugmod['comment'] = {
                 'comment': text,
                 'is_private': False,
                 'is_markdown': False,
             }
             burl = self.apiurl(('bug', bugid))
             self._submit(burl, bugmod, method='PUT')
             self.ui.debug('updated bug %s\n' % bugid)
         else:
             burl = self.apiurl(('bug', bugid, 'comment'))
             self._submit(burl, {
                 'comment': text,
                 'is_private': False,
                 'is_markdown': False,
             })
             self.ui.debug('added comment to bug %s\n' % bugid)
 
     def notify(self, bugs, committer):
         '''Force sending of Bugzilla notification emails.
 
         Only required if the access method does not trigger notification
         emails automatically.
         '''
         pass
 
 class bugzilla(object):
     # supported versions of bugzilla. different versions have
     # different schemas.
     _versions = {
         '2.16': bzmysql,
         '2.18': bzmysql_2_18,
         '3.0':  bzmysql_3_0,
         'xmlrpc': bzxmlrpc,
         'xmlrpc+email': bzxmlrpcemail,
         'restapi': bzrestapi,
         }
 
     def __init__(self, ui, repo):
         self.ui = ui
         self.repo = repo
 
         bzversion = self.ui.config('bugzilla', 'version')
         try:
             bzclass = bugzilla._versions[bzversion]
         except KeyError:
             raise error.Abort(_('bugzilla version %s not supported') %
                              bzversion)
         self.bzdriver = bzclass(self.ui)
 
         self.bug_re = re.compile(
             self.ui.config('bugzilla', 'regexp'), re.IGNORECASE)
         self.fix_re = re.compile(
             self.ui.config('bugzilla', 'fixregexp'), re.IGNORECASE)
         self.split_re = re.compile(r'\D+')
 
     def find_bugs(self, ctx):
         '''return bugs dictionary created from commit comment.
 
         Extract bug info from changeset comments. Filter out any that are
         not known to Bugzilla, and any that already have a reference to
         the given changeset in their comments.
         '''
         start = 0
         hours = 0.0
         bugs = {}
         bugmatch = self.bug_re.search(ctx.description(), start)
         fixmatch = self.fix_re.search(ctx.description(), start)
         while True:
             bugattribs = {}
             if not bugmatch and not fixmatch:
                 break
             if not bugmatch:
                 m = fixmatch
             elif not fixmatch:
                 m = bugmatch
             else:
                 if bugmatch.start() < fixmatch.start():
                     m = bugmatch
                 else:
                     m = fixmatch
             start = m.end()
             if m is bugmatch:
                 bugmatch = self.bug_re.search(ctx.description(), start)
                 if 'fix' in bugattribs:
                     del bugattribs['fix']
             else:
                 fixmatch = self.fix_re.search(ctx.description(), start)
                 bugattribs['fix'] = None
 
             try:
                 ids = m.group('ids')
             except IndexError:
                 ids = m.group(1)
             try:
                 hours = float(m.group('hours'))
                 bugattribs['hours'] = hours
             except IndexError:
                 pass
             except TypeError:
                 pass
             except ValueError:
                 self.ui.status(_("%s: invalid hours\n") % m.group('hours'))
 
             for id in self.split_re.split(ids):
                 if not id:
                     continue
                 bugs[int(id)] = bugattribs
         if bugs:
             self.bzdriver.filter_real_bug_ids(bugs)
         if bugs:
             self.bzdriver.filter_cset_known_bug_ids(ctx.node(), bugs)
         return bugs
 
     def update(self, bugid, newstate, ctx):
         '''update bugzilla bug with reference to changeset.'''
 
         def webroot(root):
             '''strip leading prefix of repo root and turn into
             url-safe path.'''
             count = int(self.ui.config('bugzilla', 'strip'))
             root = util.pconvert(root)
             while count > 0:
                 c = root.find('/')
                 if c == -1:
                     break
                 root = root[c + 1:]
                 count -= 1
             return root
 
         mapfile = None
         tmpl = self.ui.config('bugzilla', 'template')
         if not tmpl:
             mapfile = self.ui.config('bugzilla', 'style')
         if not mapfile and not tmpl:
             tmpl = _('changeset {node|short} in repo {root} refers '
                      'to bug {bug}.\ndetails:\n\t{desc|tabindent}')
         spec = logcmdutil.templatespec(tmpl, mapfile)
         t = logcmdutil.changesettemplater(self.ui, self.repo, spec)
         self.ui.pushbuffer()
         t.show(ctx, changes=ctx.changeset(),
                bug=str(bugid),
                hgweb=self.ui.config('web', 'baseurl'),
                root=self.repo.root,
                webroot=webroot(self.repo.root))
         data = self.ui.popbuffer()
-        self.bzdriver.updatebug(bugid, newstate, data, util.email(ctx.user()))
+        self.bzdriver.updatebug(bugid, newstate, data,
+                                stringutil.email(ctx.user()))
 
     def notify(self, bugs, committer):
         '''ensure Bugzilla users are notified of bug change.'''
         self.bzdriver.notify(bugs, committer)
 
 def hook(ui, repo, hooktype, node=None, **kwargs):
     '''add comment to bugzilla for each changeset that refers to a
     bugzilla bug id. only add a comment once per bug, so same change
     seen multiple times does not fill bug with duplicate data.'''
     if node is None:
         raise error.Abort(_('hook type %s does not pass a changeset id') %
                          hooktype)
     try:
         bz = bugzilla(ui, repo)
         ctx = repo[node]
         bugs = bz.find_bugs(ctx)
         if bugs:
             for bug in bugs:
                 bz.update(bug, bugs[bug], ctx)
-            bz.notify(bugs, util.email(ctx.user()))
+            bz.notify(bugs, stringutil.email(ctx.user()))
     except Exception as e:
         raise error.Abort(_('Bugzilla error: %s') % e)

hgext/convert/cvsps.py

@@ -1,953 +1,957
 # Mercurial built-in replacement for cvsps.
 #
 # Copyright 2008, Frank Kingswood <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
 
 import os
 import re
 
 from mercurial.i18n import _
 from mercurial import (
     encoding,
     error,
     hook,
     pycompat,
     util,
 )
-from mercurial.utils import dateutil
+from mercurial.utils import (
+    dateutil,
+    stringutil,
+)
 
 pickle = util.pickle
 
 class logentry(object):
     '''Class logentry has the following attributes:
         .author    - author name as CVS knows it
         .branch    - name of branch this revision is on
         .branches  - revision tuple of branches starting at this revision
         .comment   - commit message
         .commitid  - CVS commitid or None
         .date      - the commit date as a (time, tz) tuple
         .dead      - true if file revision is dead
         .file      - Name of file
         .lines     - a tuple (+lines, -lines) or None
         .parent    - Previous revision of this entry
         .rcs       - name of file as returned from CVS
         .revision  - revision number as tuple
         .tags      - list of tags on the file
         .synthetic - is this a synthetic "file ... added on ..." revision?
         .mergepoint - the branch that has been merged from (if present in
                       rlog output) or None
         .branchpoints - the branches that start at the current entry or empty
     '''
     def __init__(self, **entries):
         self.synthetic = False
         self.__dict__.update(entries)
 
     def __repr__(self):
         items = ("%s=%r"%(k, self.__dict__[k]) for k in sorted(self.__dict__))
         return "%s(%s)"%(type(self).__name__, ", ".join(items))
 
 class logerror(Exception):
     pass
 
 def getrepopath(cvspath):
     """Return the repository path from a CVS path.
 
     >>> getrepopath(b'/foo/bar')
     '/foo/bar'
     >>> getrepopath(b'c:/foo/bar')
     '/foo/bar'
     >>> getrepopath(b':pserver:10/foo/bar')
     '/foo/bar'
     >>> getrepopath(b':pserver:10c:/foo/bar')
     '/foo/bar'
     >>> getrepopath(b':pserver:/foo/bar')
     '/foo/bar'
     >>> getrepopath(b':pserver:c:/foo/bar')
     '/foo/bar'
     >>> getrepopath(b':pserver:[e-mail]:/foo/bar')
     '/foo/bar'
     >>> getrepopath(b':pserver:[e-mail]:c:/foo/bar')
     '/foo/bar'
     >>> getrepopath(b'user@server/path/to/repository')
     '/path/to/repository'
     """
     # According to CVS manual, CVS paths are expressed like:
     # [:method:][[user][:password]@]hostname[:[port]]/path/to/repository
     #
     # CVSpath is splitted into parts and then position of the first occurrence
     # of the '/' char after the '@' is located. The solution is the rest of the
     # string after that '/' sign including it
 
     parts = cvspath.split(':')
     atposition = parts[-1].find('@')
     start = 0
 
     if atposition != -1:
         start = atposition
 
     repopath = parts[-1][parts[-1].find('/', start):]
     return repopath
 
 def createlog(ui, directory=None, root="", rlog=True, cache=None):
     '''Collect the CVS rlog'''
 
     # Because we store many duplicate commit log messages, reusing strings
     # saves a lot of memory and pickle storage space.
     _scache = {}
     def scache(s):
         "return a shared version of a string"
         return _scache.setdefault(s, s)
 
     ui.status(_('collecting CVS rlog\n'))
 
     log = []      # list of logentry objects containing the CVS state
 
     # patterns to match in CVS (r)log output, by state of use
     re_00 = re.compile('RCS file: (.+)$')
     re_01 = re.compile('cvs \\[r?log aborted\\]: (.+)$')
     re_02 = re.compile('cvs (r?log|server): (.+)\n$')
     re_03 = re.compile("(Cannot access.+CVSROOT)|"
                        "(can't create temporary directory.+)$")
     re_10 = re.compile('Working file: (.+)$')
     re_20 = re.compile('symbolic names:')
     re_30 = re.compile('\t(.+): ([\\d.]+)$')
     re_31 = re.compile('----------------------------$')
     re_32 = re.compile('======================================='
                        '======================================$')
     re_50 = re.compile('revision ([\\d.]+)(\s+locked by:\s+.+;)?$')
     re_60 = re.compile(r'date:\s+(.+);\s+author:\s+(.+);\s+state:\s+(.+?);'
                        r'(\s+lines:\s+(\+\d+)?\s+(-\d+)?;)?'
                        r'(\s+commitid:\s+([^;]+);)?'
                        r'(.*mergepoint:\s+([^;]+);)?')
     re_70 = re.compile('branches: (.+);$')
 
     file_added_re = re.compile(r'file [^/]+ was (initially )?added on branch')
 
     prefix = ''   # leading path to strip of what we get from CVS
 
     if directory is None:
         # Current working directory
 
         # Get the real directory in the repository
         try:
             prefix = open(os.path.join('CVS','Repository'), 'rb').read().strip()
             directory = prefix
             if prefix == ".":
                 prefix = ""
         except IOError:
             raise logerror(_('not a CVS sandbox'))
 
         if prefix and not prefix.endswith(pycompat.ossep):
             prefix += pycompat.ossep
 
         # Use the Root file in the sandbox, if it exists
         try:
             root = open(os.path.join('CVS','Root'), 'rb').read().strip()
         except IOError:
             pass
 
     if not root:
         root = encoding.environ.get('CVSROOT', '')
 
     # read log cache if one exists
     oldlog = []
     date = None
 
     if cache:
         cachedir = os.path.expanduser('~/.hg.cvsps')
         if not os.path.exists(cachedir):
             os.mkdir(cachedir)
 
         # The cvsps cache pickle needs a uniquified name, based on the
         # repository location. The address may have all sort of nasties
         # in it, slashes, colons and such. So here we take just the
         # alphanumeric characters, concatenated in a way that does not
         # mix up the various components, so that
         #    :pserver:user@server:/path
         # and
         #    /pserver/user/server/path
         # are mapped to different cache file names.
         cachefile = root.split(":") + [directory, "cache"]
         cachefile = ['-'.join(re.findall(br'\w+', s)) for s in cachefile if s]
         cachefile = os.path.join(cachedir,
                                  '.'.join([s for s in cachefile if s]))
 
     if cache == 'update':
         try:
             ui.note(_('reading cvs log cache %s\n') % cachefile)
             oldlog = pickle.load(open(cachefile, 'rb'))
             for e in oldlog:
                 if not (util.safehasattr(e, 'branchpoints') and
                         util.safehasattr(e, 'commitid') and
                         util.safehasattr(e, 'mergepoint')):
                     ui.status(_('ignoring old cache\n'))
                     oldlog = []
                     break
 
             ui.note(_('cache has %d log entries\n') % len(oldlog))
         except Exception as e:
             ui.note(_('error reading cache: %r\n') % e)
 
         if oldlog:
             date = oldlog[-1].date    # last commit date as a (time,tz) tuple
             date = dateutil.datestr(date, '%Y/%m/%d %H:%M:%S %1%2')
 
     # build the CVS commandline
     cmd = ['cvs', '-q']
     if root:
         cmd.append('-d%s' % root)
         p = util.normpath(getrepopath(root))
         if not p.endswith('/'):
             p += '/'
         if prefix:
             # looks like normpath replaces "" by "."
             prefix = p + util.normpath(prefix)
         else:
             prefix = p
     cmd.append(['log', 'rlog'][rlog])
     if date:
         # no space between option and date string
         cmd.append('-d>%s' % date)
     cmd.append(directory)
 
     # state machine begins here
     tags = {}     # dictionary of revisions on current file with their tags
     branchmap = {} # mapping between branch names and revision numbers
     rcsmap = {}
     state = 0
     store = False # set when a new record can be appended
 
     cmd = [util.shellquote(arg) for arg in cmd]
     ui.note(_("running %s\n") % (' '.join(cmd)))
     ui.debug("prefix=%r directory=%r root=%r\n" % (prefix, directory, root))
 
     pfp = util.popen(' '.join(cmd))
     peek = pfp.readline()
     while True:
         line = peek
         if line == '':
             break
         peek = pfp.readline()
         if line.endswith('\n'):
             line = line[:-1]
         #ui.debug('state=%d line=%r\n' % (state, line))
 
         if state == 0:
             # initial state, consume input until we see 'RCS file'
             match = re_00.match(line)
             if match:
                 rcs = match.group(1)
                 tags = {}
                 if rlog:
                     filename = util.normpath(rcs[:-2])
                     if filename.startswith(prefix):
                         filename = filename[len(prefix):]
                     if filename.startswith('/'):
                         filename = filename[1:]
                     if filename.startswith('Attic/'):
                         filename = filename[6:]
                     else:
                         filename = filename.replace('/Attic/', '/')
                     state = 2
                     continue
                 state = 1
                 continue
             match = re_01.match(line)
             if match:
                 raise logerror(match.group(1))
             match = re_02.match(line)
             if match:
                 raise logerror(match.group(2))
             if re_03.match(line):
                 raise logerror(line)
 
         elif state == 1:
             # expect 'Working file' (only when using log instead of rlog)
             match = re_10.match(line)
             assert match, _('RCS file must be followed by working file')
             filename = util.normpath(match.group(1))
             state = 2
 
         elif state == 2:
             # expect 'symbolic names'
             if re_20.match(line):
                 branchmap = {}
                 state = 3
 
         elif state == 3:
             # read the symbolic names and store as tags
             match = re_30.match(line)
             if match:
                 rev = [int(x) for x in match.group(2).split('.')]
 
                 # Convert magic branch number to an odd-numbered one
                 revn = len(rev)
                 if revn > 3 and (revn % 2) == 0 and rev[-2] == 0:
                     rev = rev[:-2] + rev[-1:]
                 rev = tuple(rev)
 
                 if rev not in tags:
                     tags[rev] = []
                 tags[rev].append(match.group(1))
                 branchmap[match.group(1)] = match.group(2)
 
             elif re_31.match(line):
                 state = 5
             elif re_32.match(line):
                 state = 0
 
         elif state == 4:
             # expecting '------' separator before first revision
             if re_31.match(line):
                 state = 5
             else:
                 assert not re_32.match(line), _('must have at least '
                                                 'some revisions')
 
         elif state == 5:
             # expecting revision number and possibly (ignored) lock indication
             # we create the logentry here from values stored in states 0 to 4,
             # as this state is re-entered for subsequent revisions of a file.
             match = re_50.match(line)
             assert match, _('expected revision number')
             e = logentry(rcs=scache(rcs),
                          file=scache(filename),
                          revision=tuple([int(x) for x in
                                          match.group(1).split('.')]),
                          branches=[],
                          parent=None,
                          commitid=None,
                          mergepoint=None,
                          branchpoints=set())
 
             state = 6
 
         elif state == 6:
             # expecting date, author, state, lines changed
             match = re_60.match(line)
             assert match, _('revision must be followed by date line')
             d = match.group(1)
             if d[2] == '/':
                 # Y2K
                 d = '19' + d
 
             if len(d.split()) != 3:
                 # cvs log dates always in GMT
                 d = d + ' UTC'
             e.date = dateutil.parsedate(d, ['%y/%m/%d %H:%M:%S',
                                         '%Y/%m/%d %H:%M:%S',
                                         '%Y-%m-%d %H:%M:%S'])
             e.author = scache(match.group(2))
             e.dead = match.group(3).lower() == 'dead'
 
             if match.group(5):
                 if match.group(6):
                     e.lines = (int(match.group(5)), int(match.group(6)))
                 else:
                     e.lines = (int(match.group(5)), 0)
             elif match.group(6):
                 e.lines = (0, int(match.group(6)))
             else:
                 e.lines = None
 
             if match.group(7): # cvs 1.12 commitid
                 e.commitid = match.group(8)
 
             if match.group(9): # cvsnt mergepoint
                 myrev = match.group(10).split('.')
                 if len(myrev) == 2: # head
                     e.mergepoint = 'HEAD'
                 else:
                     myrev = '.'.join(myrev[:-2] + ['0', myrev[-2]])
                     branches = [b for b in branchmap if branchmap[b] == myrev]
                     assert len(branches) == 1, ('unknown branch: %s'
                                                 % e.mergepoint)
                     e.mergepoint = branches[0]
 
             e.comment = []
             state = 7
 
         elif state == 7:
             # read the revision numbers of branches that start at this revision
             # or store the commit log message otherwise
             m = re_70.match(line)
             if m:
                 e.branches = [tuple([int(y) for y in x.strip().split('.')])
                                 for x in m.group(1).split(';')]
                 state = 8
             elif re_31.match(line) and re_50.match(peek):
                 state = 5
                 store = True
             elif re_32.match(line):
                 state = 0
                 store = True
             else:
                 e.comment.append(line)
 
         elif state == 8:
             # store commit log message
             if re_31.match(line):
                 cpeek = peek
                 if cpeek.endswith('\n'):
                     cpeek = cpeek[:-1]
                 if re_50.match(cpeek):
                     state = 5
                     store = True
                 else:
                     e.comment.append(line)
             elif re_32.match(line):
                 state = 0
                 store = True
             else:
                 e.comment.append(line)
 
         # When a file is added on a branch B1, CVS creates a synthetic
         # dead trunk revision 1.1 so that the branch has a root.
         # Likewise, if you merge such a file to a later branch B2 (one
         # that already existed when the file was added on B1), CVS
         # creates a synthetic dead revision 1.1.x.1 on B2.  Don't drop
         # these revisions now, but mark them synthetic so
         # createchangeset() can take care of them.
         if (store and
               e.dead and
               e.revision[-1] == 1 and      # 1.1 or 1.1.x.1
               len(e.comment) == 1 and
               file_added_re.match(e.comment[0])):
             ui.debug('found synthetic revision in %s: %r\n'
                      % (e.rcs, e.comment[0]))
             e.synthetic = True
 
         if store:
             # clean up the results and save in the log.
             store = False
             e.tags = sorted([scache(x) for x in tags.get(e.revision, [])])
             e.comment = scache('\n'.join(e.comment))
 
             revn = len(e.revision)
             if revn > 3 and (revn % 2) == 0:
                 e.branch = tags.get(e.revision[:-1], [None])[0]
             else:
                 e.branch = None
 
             # find the branches starting from this revision
             branchpoints = set()
             for branch, revision in branchmap.iteritems():
                 revparts = tuple([int(i) for i in revision.split('.')])
                 if len(revparts) < 2: # bad tags
                     continue
                 if revparts[-2] == 0 and revparts[-1] % 2 == 0:
                     # normal branch
                     if revparts[:-2] == e.revision:
                         branchpoints.add(branch)
                 elif revparts == (1, 1, 1): # vendor branch
                     if revparts in e.branches:
                         branchpoints.add(branch)
             e.branchpoints = branchpoints
 
             log.append(e)
 
             rcsmap[e.rcs.replace('/Attic/', '/')] = e.rcs
 
             if len(log) % 100 == 0:
-                ui.status(util.ellipsis('%d %s' % (len(log), e.file), 80)+'\n')
+                ui.status(stringutil.ellipsis('%d %s' % (len(log), e.file), 80)
+                          + '\n')
 
     log.sort(key=lambda x: (x.rcs, x.revision))
 
     # find parent revisions of individual files
     versions = {}
     for e in sorted(oldlog, key=lambda x: (x.rcs, x.revision)):
         rcs = e.rcs.replace('/Attic/', '/')
         if rcs in rcsmap:
             e.rcs = rcsmap[rcs]
         branch = e.revision[:-1]
         versions[(e.rcs, branch)] = e.revision
 
     for e in log:
         branch = e.revision[:-1]
         p = versions.get((e.rcs, branch), None)
         if p is None:
             p = e.revision[:-2]
         e.parent = p
         versions[(e.rcs, branch)] = e.revision
 
     # update the log cache
     if cache:
         if log:
             # join up the old and new logs
             log.sort(key=lambda x: x.date)
 
             if oldlog and oldlog[-1].date >= log[0].date:
                 raise logerror(_('log cache overlaps with new log entries,'
                                  ' re-run without cache.'))
 
             log = oldlog + log
 
             # write the new cachefile
             ui.note(_('writing cvs log cache %s\n') % cachefile)
             pickle.dump(log, open(cachefile, 'wb'))
         else:
             log = oldlog
 
     ui.status(_('%d log entries\n') % len(log))
 
     encodings = ui.configlist('convert', 'cvsps.logencoding')
     if encodings:
         def revstr(r):
             # this is needed, because logentry.revision is a tuple of "int"
             # (e.g. (1, 2) for "1.2")
             return '.'.join(pycompat.maplist(pycompat.bytestr, r))
 
         for entry in log:
             comment = entry.comment
             for e in encodings:
                 try:
                     entry.comment = comment.decode(e).encode('utf-8')
                     if ui.debugflag:
                         ui.debug("transcoding by %s: %s of %s\n" %
                                  (e, revstr(entry.revision), entry.file))
                     break
                 except UnicodeDecodeError:
                     pass # try next encoding
                 except LookupError as inst: # unknown encoding, maybe
                     raise error.Abort(inst,
                                       hint=_('check convert.cvsps.logencoding'
                                              ' configuration'))
             else:
                 raise error.Abort(_("no encoding can transcode"
                                     " CVS log message for %s of %s")
                                   % (revstr(entry.revision), entry.file),
                                   hint=_('check convert.cvsps.logencoding'
                                          ' configuration'))
 
     hook.hook(ui, None, "cvslog", True, log=log)
 
     return log
 
 
 class changeset(object):
     '''Class changeset has the following attributes:
         .id        - integer identifying this changeset (list index)
         .author    - author name as CVS knows it
         .branch    - name of branch this changeset is on, or None
         .comment   - commit message
         .commitid  - CVS commitid or None
         .date      - the commit date as a (time,tz) tuple
         .entries   - list of logentry objects in this changeset
         .parents   - list of one or two parent changesets
         .tags      - list of tags on this changeset
         .synthetic - from synthetic revision "file ... added on branch ..."
         .mergepoint- the branch that has been merged from or None
         .branchpoints- the branches that start at the current entry or empty
     '''
     def __init__(self, **entries):
         self.id = None
         self.synthetic = False
         self.__dict__.update(entries)
 
     def __repr__(self):
         items = ("%s=%r"%(k, self.__dict__[k]) for k in sorted(self.__dict__))
         return "%s(%s)"%(type(self).__name__, ", ".join(items))
 
 def createchangeset(ui, log, fuzz=60, mergefrom=None, mergeto=None):
     '''Convert log into changesets.'''
 
     ui.status(_('creating changesets\n'))
 
     # try to order commitids by date
     mindate = {}
     for e in log:
         if e.commitid:
             mindate[e.commitid] = min(e.date, mindate.get(e.commitid))
 
     # Merge changesets
     log.sort(key=lambda x: (mindate.get(x.commitid), x.commitid, x.comment,
                             x.author, x.branch, x.date, x.branchpoints))
 
     changesets = []
     files = set()
     c = None
     for i, e in enumerate(log):
 
         # Check if log entry belongs to the current changeset or not.
 
         # Since CVS is file-centric, two different file revisions with
         # different branchpoints should be treated as belonging to two
         # different changesets (and the ordering is important and not
         # honoured by cvsps at this point).
         #
         # Consider the following case:
         # foo 1.1 branchpoints: [MYBRANCH]
         # bar 1.1 branchpoints: [MYBRANCH, MYBRANCH2]
         #
         # Here foo is part only of MYBRANCH, but not MYBRANCH2, e.g. a
         # later version of foo may be in MYBRANCH2, so foo should be the
         # first changeset and bar the next and MYBRANCH and MYBRANCH2
         # should both start off of the bar changeset. No provisions are
         # made to ensure that this is, in fact, what happens.
         if not (c and e.branchpoints == c.branchpoints and
                 (# cvs commitids
                  (e.commitid is not None and e.commitid == c.commitid) or
                  (# no commitids, use fuzzy commit detection
                   (e.commitid is None or c.commitid is None) and
                    e.comment == c.comment and
                    e.author == c.author and
                    e.branch == c.branch and
                    ((c.date[0] + c.date[1]) <=
                     (e.date[0] + e.date[1]) <=
                     (c.date[0] + c.date[1]) + fuzz) and
                    e.file not in files))):
             c = changeset(comment=e.comment, author=e.author,
                           branch=e.branch, date=e.date,
                           entries=[], mergepoint=e.mergepoint,
                           branchpoints=e.branchpoints, commitid=e.commitid)
             changesets.append(c)
 
             files = set()
             if len(changesets) % 100 == 0:
                 t = '%d %s' % (len(changesets), repr(e.comment)[1:-1])
-                ui.status(util.ellipsis(t, 80) + '\n')
+                ui.status(stringutil.ellipsis(t, 80) + '\n')
 
         c.entries.append(e)
         files.add(e.file)
         c.date = e.date       # changeset date is date of latest commit in it
 
     # Mark synthetic changesets
 
     for c in changesets:
         # Synthetic revisions always get their own changeset, because
         # the log message includes the filename.  E.g. if you add file3
         # and file4 on a branch, you get four log entries and three
         # changesets:
         #   "File file3 was added on branch ..." (synthetic, 1 entry)
         #   "File file4 was added on branch ..." (synthetic, 1 entry)
         #   "Add file3 and file4 to fix ..."     (real, 2 entries)
         # Hence the check for 1 entry here.
         c.synthetic = len(c.entries) == 1 and c.entries[0].synthetic
 
     # Sort files in each changeset
 
     def entitycompare(l, r):
         'Mimic cvsps sorting order'
         l = l.file.split('/')
         r = r.file.split('/')
         nl = len(l)
         nr = len(r)
         n = min(nl, nr)
         for i in range(n):
             if i + 1 == nl and nl < nr:
                 return -1
             elif i + 1 == nr and nl > nr:
                 return +1
             elif l[i] < r[i]:
                 return -1
             elif l[i] > r[i]:
                 return +1
         return 0
 
     for c in changesets:
         c.entries.sort(entitycompare)
 
     # Sort changesets by date
 
     odd = set()
     def cscmp(l, r):
         d = sum(l.date) - sum(r.date)
         if d:
             return d
 
         # detect vendor branches and initial commits on a branch
         le = {}
         for e in l.entries:
             le[e.rcs] = e.revision
         re = {}
         for e in r.entries:
             re[e.rcs] = e.revision
 
         d = 0
         for e in l.entries:
             if re.get(e.rcs, None) == e.parent:
                 assert not d
                 d = 1
                 break
 
         for e in r.entries:
             if le.get(e.rcs, None) == e.parent:
                 if d:
                     odd.add((l, r))
                 d = -1
                 break
         # By this point, the changesets are sufficiently compared that
         # we don't really care about ordering. However, this leaves
         # some race conditions in the tests, so we compare on the
         # number of files modified, the files contained in each
         # changeset, and the branchpoints in the change to ensure test
         # output remains stable.
 
         # recommended replacement for cmp from
         # https://docs.python.org/3.0/whatsnew/3.0.html
         c = lambda x, y: (x > y) - (x < y)
         # Sort bigger changes first.
         if not d:
             d = c(len(l.entries), len(r.entries))
         # Try sorting by filename in the change.
         if not d:
             d = c([e.file for e in l.entries], [e.file for e in r.entries])
         # Try and put changes without a branch point before ones with
         # a branch point.
         if not d:
             d = c(len(l.branchpoints), len(r.branchpoints))
         return d
 
     changesets.sort(cscmp)
 
     # Collect tags
 
     globaltags = {}
     for c in changesets:
         for e in c.entries:
             for tag in e.tags:
                 # remember which is the latest changeset to have this tag
                 globaltags[tag] = c
 
     for c in changesets:
         tags = set()
         for e in c.entries:
             tags.update(e.tags)
         # remember tags only if this is the latest changeset to have it
         c.tags = sorted(tag for tag in tags if globaltags[tag] is c)
 
     # Find parent changesets, handle {{mergetobranch BRANCHNAME}}
     # by inserting dummy changesets with two parents, and handle
     # {{mergefrombranch BRANCHNAME}} by setting two parents.
 
     if mergeto is None:
         mergeto = r'{{mergetobranch ([-\w]+)}}'
     if mergeto:
         mergeto = re.compile(mergeto)
 
     if mergefrom is None:
         mergefrom = r'{{mergefrombranch ([-\w]+)}}'
     if mergefrom:
         mergefrom = re.compile(mergefrom)
 
     versions = {}    # changeset index where we saw any particular file version
     branches = {}    # changeset index where we saw a branch
     n = len(changesets)
     i = 0
     while i < n:
         c = changesets[i]
 
         for f in c.entries:
             versions[(f.rcs, f.revision)] = i
 
         p = None
         if c.branch in branches:
             p = branches[c.branch]
         else:
             # first changeset on a new branch
             # the parent is a changeset with the branch in its
             # branchpoints such that it is the latest possible
             # commit without any intervening, unrelated commits.
 
             for candidate in xrange(i):
                 if c.branch not in changesets[candidate].branchpoints:
                     if p is not None:
                         break
                     continue
                 p = candidate
 
         c.parents = []
         if p is not None:
             p = changesets[p]
 
             # Ensure no changeset has a synthetic changeset as a parent.
             while p.synthetic:
                 assert len(p.parents) <= 1, \
                        _('synthetic changeset cannot have multiple parents')
                 if p.parents:
                     p = p.parents[0]
                 else:
                     p = None
                     break
 
             if p is not None:
                 c.parents.append(p)
 
         if c.mergepoint:
             if c.mergepoint == 'HEAD':
                 c.mergepoint = None
             c.parents.append(changesets[branches[c.mergepoint]])
 
         if mergefrom:
             m = mergefrom.search(c.comment)
             if m:
                 m = m.group(1)
                 if m == 'HEAD':
                     m = None
                 try:
                     candidate = changesets[branches[m]]
                 except KeyError:
                     ui.warn(_("warning: CVS commit message references "
                               "non-existent branch %r:\n%s\n")
                             % (m, c.comment))
                 if m in branches and c.branch != m and not candidate.synthetic:
                     c.parents.append(candidate)
 
         if mergeto:
             m = mergeto.search(c.comment)
             if m:
                 if m.groups():
                     m = m.group(1)
                     if m == 'HEAD':
                         m = None
                 else:
                     m = None   # if no group found then merge to HEAD
                 if m in branches and c.branch != m:
                     # insert empty changeset for merge
                     cc = changeset(
                         author=c.author, branch=m, date=c.date,
                         comment='convert-repo: CVS merge from branch %s'
                         % c.branch,
                         entries=[], tags=[],
                         parents=[changesets[branches[m]], c])
                     changesets.insert(i + 1, cc)
                     branches[m] = i + 1
 
                     # adjust our loop counters now we have inserted a new entry
                     n += 1
                     i += 2
                     continue
 
         branches[c.branch] = i
         i += 1
 
     # Drop synthetic changesets (safe now that we have ensured no other
     # changesets can have them as parents).
     i = 0
     while i < len(changesets):
         if changesets[i].synthetic:
             del changesets[i]
         else:
             i += 1
 
     # Number changesets
 
     for i, c in enumerate(changesets):
         c.id = i + 1
 
     if odd:
         for l, r in odd:
             if l.id is not None and r.id is not None:
                 ui.warn(_('changeset %d is both before and after %d\n')
                         % (l.id, r.id))
 
     ui.status(_('%d changeset entries\n') % len(changesets))
 
     hook.hook(ui, None, "cvschangesets", True, changesets=changesets)
 
     return changesets
 
 
 def debugcvsps(ui, *args, **opts):
     '''Read CVS rlog for current directory or named path in
     repository, and convert the log to changesets based on matching
     commit log entries and dates.
     '''
     opts = pycompat.byteskwargs(opts)
     if opts["new_cache"]:
         cache = "write"
     elif opts["update_cache"]:
         cache = "update"
     else:
         cache = None
 
     revisions = opts["revisions"]
 
     try:
         if args:
             log = []
             for d in args:
                 log += createlog(ui, d, root=opts["root"], cache=cache)
         else:
             log = createlog(ui, root=opts["root"], cache=cache)
     except logerror as e:
         ui.write("%r\n"%e)
         return
 
     changesets = createchangeset(ui, log, opts["fuzz"])
     del log
 
     # Print changesets (optionally filtered)
 
     off = len(revisions)
     branches = {}    # latest version number in each branch
     ancestors = {}   # parent branch
     for cs in changesets:
 
         if opts["ancestors"]:
             if cs.branch not in branches and cs.parents and cs.parents[0].id:
                 ancestors[cs.branch] = (changesets[cs.parents[0].id - 1].branch,
                                         cs.parents[0].id)
             branches[cs.branch] = cs.id
 
         # limit by branches
         if opts["branches"] and (cs.branch or 'HEAD') not in opts["branches"]:
             continue
 
         if not off:
             # Note: trailing spaces on several lines here are needed to have
             #       bug-for-bug compatibility with cvsps.
             ui.write('---------------------\n')
             ui.write(('PatchSet %d \n' % cs.id))
             ui.write(('Date: %s\n' % dateutil.datestr(cs.date,
                                                  '%Y/%m/%d %H:%M:%S %1%2')))
             ui.write(('Author: %s\n' % cs.author))
             ui.write(('Branch: %s\n' % (cs.branch or 'HEAD')))
             ui.write(('Tag%s: %s \n' % (['', 's'][len(cs.tags) > 1],
                                   ','.join(cs.tags) or '(none)')))
             if cs.branchpoints:
                 ui.write(('Branchpoints: %s \n') %
                          ', '.join(sorted(cs.branchpoints)))
             if opts["parents"] and cs.parents:
                 if len(cs.parents) > 1:
                     ui.write(('Parents: %s\n' %
                              (','.join([str(p.id) for p in cs.parents]))))
                 else:
                     ui.write(('Parent: %d\n' % cs.parents[0].id))
 
             if opts["ancestors"]:
                 b = cs.branch
                 r = []
                 while b:
                     b, c = ancestors[b]
                     r.append('%s:%d:%d' % (b or "HEAD", c, branches[b]))
                 if r:
                     ui.write(('Ancestors: %s\n' % (','.join(r))))
 
             ui.write(('Log:\n'))
             ui.write('%s\n\n' % cs.comment)
             ui.write(('Members: \n'))
             for f in cs.entries:
                 fn = f.file
                 if fn.startswith(opts["prefix"]):
                     fn = fn[len(opts["prefix"]):]
                 ui.write('\t%s:%s->%s%s \n' % (
                         fn, '.'.join([str(x) for x in f.parent]) or 'INITIAL',
                         '.'.join([str(x) for x in f.revision]),
                         ['', '(DEAD)'][f.dead]))
             ui.write('\n')
 
         # have we seen the start tag?
         if revisions and off:
             if revisions[0] == str(cs.id) or \
                 revisions[0] in cs.tags:
                 off = False
 
         # see if we reached the end tag
         if len(revisions) > 1 and not off:
             if revisions[1] == str(cs.id) or \
                 revisions[1] in cs.tags:
                 break

hgext/convert/p4.py

@@ -1,374 +1,377
 # Perforce source for convert extension.
 #
 # Copyright 2009, Frank Kingswood <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
 
 import marshal
 import re
 
 from mercurial.i18n import _
 from mercurial import (
     error,
     util,
 )
-from mercurial.utils import dateutil
+from mercurial.utils import (
+    dateutil,
+    stringutil,
+)
 
 from . import common
 
 def loaditer(f):
     "Yield the dictionary objects generated by p4"
     try:
         while True:
             d = marshal.load(f)
             if not d:
                 break
             yield d
     except EOFError:
         pass
 
 def decodefilename(filename):
     """Perforce escapes special characters @, #, *, or %
     with %40, %23, %2A, or %25 respectively
 
     >>> decodefilename(b'portable-net45%252Bnetcore45%252Bwp8%252BMonoAndroid')
     'portable-net45%2Bnetcore45%2Bwp8%2BMonoAndroid'
     >>> decodefilename(b'//Depot/Directory/%2525/%2523/%23%40.%2A')
     '//Depot/Directory/%25/%23/#@.*'
     """
     replacements = [('%2A', '*'), ('%23', '#'), ('%40', '@'), ('%25', '%')]
     for k, v in replacements:
         filename = filename.replace(k, v)
     return filename
 
 class p4_source(common.converter_source):
     def __init__(self, ui, repotype, path, revs=None):
         # avoid import cycle
         from . import convcmd
 
         super(p4_source, self).__init__(ui, repotype, path, revs=revs)
 
         if "/" in path and not path.startswith('//'):
             raise common.NoRepo(_('%s does not look like a P4 repository') %
                                 path)
 
         common.checktool('p4', abort=False)
 
         self.revmap = {}
         self.encoding = self.ui.config('convert', 'p4.encoding',
                                        convcmd.orig_encoding)
         self.re_type = re.compile(
             "([a-z]+)?(text|binary|symlink|apple|resource|unicode|utf\d+)"
             "(\+\w+)?$")
         self.re_keywords = re.compile(
             r"\$(Id|Header|Date|DateTime|Change|File|Revision|Author)"
             r":[^$\n]*\$")
         self.re_keywords_old = re.compile("\$(Id|Header):[^$\n]*\$")
 
         if revs and len(revs) > 1:
             raise error.Abort(_("p4 source does not support specifying "
                                "multiple revisions"))
 
     def setrevmap(self, revmap):
         """Sets the parsed revmap dictionary.
 
         Revmap stores mappings from a source revision to a target revision.
         It is set in convertcmd.convert and provided by the user as a file
         on the commandline.
 
         Revisions in the map are considered beeing present in the
         repository and ignored during _parse(). This allows for incremental
         imports if a revmap is provided.
         """
         self.revmap = revmap
 
     def _parse_view(self, path):
         "Read changes affecting the path"
         cmd = 'p4 -G changes -s submitted %s' % util.shellquote(path)
         stdout = util.popen(cmd, mode='rb')
         p4changes = {}
         for d in loaditer(stdout):
             c = d.get("change", None)
             if c:
                 p4changes[c] = True
         return p4changes
 
     def _parse(self, ui, path):
         "Prepare list of P4 filenames and revisions to import"
         p4changes = {}
         changeset = {}
         files_map = {}
         copies_map = {}
         localname = {}
         depotname = {}
         heads = []
 
         ui.status(_('reading p4 views\n'))
 
         # read client spec or view
         if "/" in path:
             p4changes.update(self._parse_view(path))
             if path.startswith("//") and path.endswith("/..."):
                 views = {path[:-3]:""}
             else:
                 views = {"//": ""}
         else:
             cmd = 'p4 -G client -o %s' % util.shellquote(path)
             clientspec = marshal.load(util.popen(cmd, mode='rb'))
 
             views = {}
             for client in clientspec:
                 if client.startswith("View"):
                     sview, cview = clientspec[client].split()
                     p4changes.update(self._parse_view(sview))
                     if sview.endswith("...") and cview.endswith("..."):
                         sview = sview[:-3]
                         cview = cview[:-3]
                     cview = cview[2:]
                     cview = cview[cview.find("/") + 1:]
                     views[sview] = cview
 
         # list of changes that affect our source files
         p4changes = p4changes.keys()
         p4changes.sort(key=int)
 
         # list with depot pathnames, longest first
         vieworder = views.keys()
         vieworder.sort(key=len, reverse=True)
 
         # handle revision limiting
         startrev = self.ui.config('convert', 'p4.startrev')
 
         # now read the full changelists to get the list of file revisions
         ui.status(_('collecting p4 changelists\n'))
         lastid = None
         for change in p4changes:
             if startrev and int(change) < int(startrev):
                 continue
             if self.revs and int(change) > int(self.revs[0]):
                 continue
             if change in self.revmap:
                 # Ignore already present revisions, but set the parent pointer.
                 lastid = change
                 continue
 
             if lastid:
                 parents = [lastid]
             else:
                 parents = []
 
             d = self._fetch_revision(change)
             c = self._construct_commit(d, parents)
 
             descarr = c.desc.splitlines(True)
             if len(descarr) > 0:
                 shortdesc = descarr[0].rstrip('\r\n')
             else:
                 shortdesc = '**empty changelist description**'
 
             t = '%s %s' % (c.rev, repr(shortdesc)[1:-1])
-            ui.status(util.ellipsis(t, 80) + '\n')
+            ui.status(stringutil.ellipsis(t, 80) + '\n')
 
             files = []
             copies = {}
             copiedfiles = []
             i = 0
             while ("depotFile%d" % i) in d and ("rev%d" % i) in d:
                 oldname = d["depotFile%d" % i]
                 filename = None
                 for v in vieworder:
                     if oldname.lower().startswith(v.lower()):
                         filename = decodefilename(views[v] + oldname[len(v):])
                         break
                 if filename:
                     files.append((filename, d["rev%d" % i]))
                     depotname[filename] = oldname
                     if (d.get("action%d" % i) == "move/add"):
                         copiedfiles.append(filename)
                     localname[oldname] = filename
                 i += 1
 
             # Collect information about copied files
             for filename in copiedfiles:
                 oldname = depotname[filename]
 
                 flcmd = 'p4 -G filelog %s' \
                       % util.shellquote(oldname)
                 flstdout = util.popen(flcmd, mode='rb')
 
                 copiedfilename = None
                 for d in loaditer(flstdout):
                     copiedoldname = None
 
                     i = 0
                     while ("change%d" % i) in d:
                         if (d["change%d" % i] == change and
                             d["action%d" % i] == "move/add"):
                             j = 0
                             while ("file%d,%d" % (i, j)) in d:
                                 if d["how%d,%d" % (i, j)] == "moved from":
                                     copiedoldname = d["file%d,%d" % (i, j)]
                                     break
                                 j += 1
                         i += 1
 
                     if copiedoldname and copiedoldname in localname:
                         copiedfilename = localname[copiedoldname]
                         break
 
                 if copiedfilename:
                     copies[filename] = copiedfilename
                 else:
                     ui.warn(_("cannot find source for copied file: %s@%s\n")
                             % (filename, change))
 
             changeset[change] = c
             files_map[change] = files
             copies_map[change] = copies
             lastid = change
 
         if lastid and len(changeset) > 0:
             heads = [lastid]
 
         return {
             'changeset': changeset,
             'files': files_map,
             'copies': copies_map,
             'heads': heads,
             'depotname': depotname,
         }
 
     @util.propertycache
     def _parse_once(self):
         return self._parse(self.ui, self.path)
 
     @util.propertycache
     def copies(self):
         return self._parse_once['copies']
 
     @util.propertycache
     def files(self):
         return self._parse_once['files']
 
     @util.propertycache
     def changeset(self):
         return self._parse_once['changeset']
 
     @util.propertycache
     def heads(self):
         return self._parse_once['heads']
 
     @util.propertycache
     def depotname(self):
         return self._parse_once['depotname']
 
     def getheads(self):
         return self.heads
 
     def getfile(self, name, rev):
         cmd = 'p4 -G print %s' \
             % util.shellquote("%s#%s" % (self.depotname[name], rev))
 
         lasterror = None
         while True:
             stdout = util.popen(cmd, mode='rb')
 
             mode = None
             contents = []
             keywords = None
 
             for d in loaditer(stdout):
                 code = d["code"]
                 data = d.get("data")
 
                 if code == "error":
                     # if this is the first time error happened
                     # re-attempt getting the file
                     if not lasterror:
                         lasterror = IOError(d["generic"], data)
                         # this will exit inner-most for-loop
                         break
                     else:
                         raise lasterror
 
                 elif code == "stat":
                     action = d.get("action")
                     if action in ["purge", "delete", "move/delete"]:
                         return None, None
                     p4type = self.re_type.match(d["type"])
                     if p4type:
                         mode = ""
                         flags = ((p4type.group(1) or "")
                                + (p4type.group(3) or ""))
                         if "x" in flags:
                             mode = "x"
                         if p4type.group(2) == "symlink":
                             mode = "l"
                         if "ko" in flags:
                             keywords = self.re_keywords_old
                         elif "k" in flags:
                             keywords = self.re_keywords
 
                 elif code == "text" or code == "binary":
                     contents.append(data)
 
                 lasterror = None
 
             if not lasterror:
                 break
 
         if mode is None:
             return None, None
 
         contents = ''.join(contents)
 
         if keywords:
             contents = keywords.sub("$\\1$", contents)
         if mode == "l" and contents.endswith("\n"):
             contents = contents[:-1]
 
         return contents, mode
 
     def getchanges(self, rev, full):
         if full:
             raise error.Abort(_("convert from p4 does not support --full"))
         return self.files[rev], self.copies[rev], set()
 
     def _construct_commit(self, obj, parents=None):
         """
         Constructs a common.commit object from an unmarshalled
         `p4 describe` output
         """
         desc = self.recode(obj.get("desc", ""))
         date = (int(obj["time"]), 0)     # timezone not set
         if parents is None:
             parents = []
 
         return common.commit(author=self.recode(obj["user"]),
             date=dateutil.datestr(date, '%Y-%m-%d %H:%M:%S %1%2'),
             parents=parents, desc=desc, branch=None, rev=obj['change'],
             extra={"p4": obj['change'], "convert_revision": obj['change']})
 
     def _fetch_revision(self, rev):
         """Return an output of `p4 describe` including author, commit date as
         a dictionary."""
         cmd = "p4 -G describe -s %s" % rev
         stdout = util.popen(cmd, mode='rb')
         return marshal.load(stdout)
 
     def getcommit(self, rev):
         if rev in self.changeset:
             return self.changeset[rev]
         elif rev in self.revmap:
             d = self._fetch_revision(rev)
             return self._construct_commit(d, parents=None)
         raise error.Abort(
             _("cannot find %s in the revmap or parsed changesets") % rev)
 
     def gettags(self):
         return {}
 
     def getchangedfiles(self, rev, i):
         return sorted([x[0] for x in self.files[rev]])

hgext/convert/subversion.py

@@ -1,1357 +1,1360
 # Subversion 1.4/1.5 Python API backend
 #
 # Copyright(C) 2007 Daniel Holth et al
 from __future__ import absolute_import
 
 import os
 import re
 import tempfile
 import xml.dom.minidom
 
 from mercurial.i18n import _
 from mercurial import (
     encoding,
     error,
     pycompat,
     util,
     vfs as vfsmod,
 )
-from mercurial.utils import dateutil
+from mercurial.utils import (
+    dateutil,
+    stringutil,
+)
 
 from . import common
 
 pickle = util.pickle
 stringio = util.stringio
 propertycache = util.propertycache
 urlerr = util.urlerr
 urlreq = util.urlreq
 
 commandline = common.commandline
 commit = common.commit
 converter_sink = common.converter_sink
 converter_source = common.converter_source
 decodeargs = common.decodeargs
 encodeargs = common.encodeargs
 makedatetimestamp = common.makedatetimestamp
 mapfile = common.mapfile
 MissingTool = common.MissingTool
 NoRepo = common.NoRepo
 
 # Subversion stuff. Works best with very recent Python SVN bindings
 # e.g. SVN 1.5 or backports. Thanks to the bzr folks for enhancing
 # these bindings.
 
 try:
     import svn
     import svn.client
     import svn.core
     import svn.ra
     import svn.delta
     from . import transport
     import warnings
     warnings.filterwarnings('ignore',
             module='svn.core',
             category=DeprecationWarning)
     svn.core.SubversionException # trigger import to catch error
 
 except ImportError:
     svn = None
 
 class SvnPathNotFound(Exception):
     pass
 
 def revsplit(rev):
     """Parse a revision string and return (uuid, path, revnum).
     >>> revsplit(b'svn:a2147622-4a9f-4db4-a8d3-13562ff547b2'
     ...          b'/proj%20B/mytrunk/mytrunk@1')
     ('a2147622-4a9f-4db4-a8d3-13562ff547b2', '/proj%20B/mytrunk/mytrunk', 1)
     >>> revsplit(b'svn:8af66a51-67f5-4354-b62c-98d67cc7be1d@1')
     ('', '', 1)
     >>> revsplit(b'@7')
     ('', '', 7)
     >>> revsplit(b'7')
     ('', '', 0)
     >>> revsplit(b'bad')
     ('', '', 0)
     """
     parts = rev.rsplit('@', 1)
     revnum = 0
     if len(parts) > 1:
         revnum = int(parts[1])
     parts = parts[0].split('/', 1)
     uuid = ''
     mod = ''
     if len(parts) > 1 and parts[0].startswith('svn:'):
         uuid = parts[0][4:]
         mod = '/' + parts[1]
     return uuid, mod, revnum
 
 def quote(s):
     # As of svn 1.7, many svn calls expect "canonical" paths. In
     # theory, we should call svn.core.*canonicalize() on all paths
     # before passing them to the API.  Instead, we assume the base url
     # is canonical and copy the behaviour of svn URL encoding function
     # so we can extend it safely with new components. The "safe"
     # characters were taken from the "svn_uri__char_validity" table in
     # libsvn_subr/path.c.
     return urlreq.quote(s, "!$&'()*+,-./:=@_~")
 
 def geturl(path):
     try:
         return svn.client.url_from_path(svn.core.svn_path_canonicalize(path))
     except svn.core.SubversionException:
         # svn.client.url_from_path() fails with local repositories
         pass
     if os.path.isdir(path):
         path = os.path.normpath(os.path.abspath(path))
         if pycompat.iswindows:
             path = '/' + util.normpath(path)
         # Module URL is later compared with the repository URL returned
         # by svn API, which is UTF-8.
         path = encoding.tolocal(path)
         path = 'file://%s' % quote(path)
     return svn.core.svn_path_canonicalize(path)
 
 def optrev(number):
     optrev = svn.core.svn_opt_revision_t()
     optrev.kind = svn.core.svn_opt_revision_number
     optrev.value.number = number
     return optrev
 
 class changedpath(object):
     def __init__(self, p):
         self.copyfrom_path = p.copyfrom_path
         self.copyfrom_rev = p.copyfrom_rev
         self.action = p.action
 
 def get_log_child(fp, url, paths, start, end, limit=0,
                   discover_changed_paths=True, strict_node_history=False):
     protocol = -1
     def receiver(orig_paths, revnum, author, date, message, pool):
         paths = {}
         if orig_paths is not None:
             for k, v in orig_paths.iteritems():
                 paths[k] = changedpath(v)
         pickle.dump((paths, revnum, author, date, message),
                     fp, protocol)
 
     try:
         # Use an ra of our own so that our parent can consume
         # our results without confusing the server.
         t = transport.SvnRaTransport(url=url)
         svn.ra.get_log(t.ra, paths, start, end, limit,
                        discover_changed_paths,
                        strict_node_history,
                        receiver)
     except IOError:
         # Caller may interrupt the iteration
         pickle.dump(None, fp, protocol)
     except Exception as inst:
-        pickle.dump(util.forcebytestr(inst), fp, protocol)
+        pickle.dump(stringutil.forcebytestr(inst), fp, protocol)
     else:
         pickle.dump(None, fp, protocol)
     fp.flush()
     # With large history, cleanup process goes crazy and suddenly
     # consumes *huge* amount of memory. The output file being closed,
     # there is no need for clean termination.
     os._exit(0)
 
 def debugsvnlog(ui, **opts):
     """Fetch SVN log in a subprocess and channel them back to parent to
     avoid memory collection issues.
     """
     if svn is None:
         raise error.Abort(_('debugsvnlog could not load Subversion python '
                            'bindings'))
 
     args = decodeargs(ui.fin.read())
     get_log_child(ui.fout, *args)
 
 class logstream(object):
     """Interruptible revision log iterator."""
     def __init__(self, stdout):
         self._stdout = stdout
 
     def __iter__(self):
         while True:
             try:
                 entry = pickle.load(self._stdout)
             except EOFError:
                 raise error.Abort(_('Mercurial failed to run itself, check'
                                    ' hg executable is in PATH'))
             try:
                 orig_paths, revnum, author, date, message = entry
             except (TypeError, ValueError):
                 if entry is None:
                     break
                 raise error.Abort(_("log stream exception '%s'") % entry)
             yield entry
 
     def close(self):
         if self._stdout:
             self._stdout.close()
             self._stdout = None
 
 class directlogstream(list):
     """Direct revision log iterator.
     This can be used for debugging and development but it will probably leak
     memory and is not suitable for real conversions."""
     def __init__(self, url, paths, start, end, limit=0,
                   discover_changed_paths=True, strict_node_history=False):
 
         def receiver(orig_paths, revnum, author, date, message, pool):
             paths = {}
             if orig_paths is not None:
                 for k, v in orig_paths.iteritems():
                     paths[k] = changedpath(v)
             self.append((paths, revnum, author, date, message))
 
         # Use an ra of our own so that our parent can consume
         # our results without confusing the server.
         t = transport.SvnRaTransport(url=url)
         svn.ra.get_log(t.ra, paths, start, end, limit,
                        discover_changed_paths,
                        strict_node_history,
                        receiver)
 
     def close(self):
         pass
 
 # Check to see if the given path is a local Subversion repo. Verify this by
 # looking for several svn-specific files and directories in the given
 # directory.
 def filecheck(ui, path, proto):
     for x in ('locks', 'hooks', 'format', 'db'):
         if not os.path.exists(os.path.join(path, x)):
             return False
     return True
 
 # Check to see if a given path is the root of an svn repo over http. We verify
 # this by requesting a version-controlled URL we know can't exist and looking
 # for the svn-specific "not found" XML.
 def httpcheck(ui, path, proto):
     try:
         opener = urlreq.buildopener()
         rsp = opener.open('%s://%s/!svn/ver/0/.svn' % (proto, path), 'rb')
         data = rsp.read()
     except urlerr.httperror as inst:
         if inst.code != 404:
             # Except for 404 we cannot know for sure this is not an svn repo
             ui.warn(_('svn: cannot probe remote repository, assume it could '
                       'be a subversion repository. Use --source-type if you '
                       'know better.\n'))
             return True
         data = inst.fp.read()
     except Exception:
         # Could be urlerr.urlerror if the URL is invalid or anything else.
         return False
     return '<m:human-readable errcode="160013">' in data
 
 protomap = {'http': httpcheck,
             'https': httpcheck,
             'file': filecheck,
             }
 def issvnurl(ui, url):
     try:
         proto, path = url.split('://', 1)
         if proto == 'file':
             if (pycompat.iswindows and path[:1] == '/'
                   and path[1:2].isalpha() and path[2:6].lower() == '%3a/'):
                 path = path[:2] + ':/' + path[6:]
             path = urlreq.url2pathname(path)
     except ValueError:
         proto = 'file'
         path = os.path.abspath(url)
     if proto == 'file':
         path = util.pconvert(path)
     check = protomap.get(proto, lambda *args: False)
     while '/' in path:
         if check(ui, path, proto):
             return True
         path = path.rsplit('/', 1)[0]
     return False
 
 # SVN conversion code stolen from bzr-svn and tailor
 #
 # Subversion looks like a versioned filesystem, branches structures
 # are defined by conventions and not enforced by the tool. First,
 # we define the potential branches (modules) as "trunk" and "branches"
 # children directories. Revisions are then identified by their
 # module and revision number (and a repository identifier).
 #
 # The revision graph is really a tree (or a forest). By default, a
 # revision parent is the previous revision in the same module. If the
 # module directory is copied/moved from another module then the
 # revision is the module root and its parent the source revision in
 # the parent module. A revision has at most one parent.
 #
 class svn_source(converter_source):
     def __init__(self, ui, repotype, url, revs=None):
         super(svn_source, self).__init__(ui, repotype, url, revs=revs)
 
         if not (url.startswith('svn://') or url.startswith('svn+ssh://') or
                 (os.path.exists(url) and
                  os.path.exists(os.path.join(url, '.svn'))) or
                 issvnurl(ui, url)):
             raise NoRepo(_("%s does not look like a Subversion repository")
                          % url)
         if svn is None:
             raise MissingTool(_('could not load Subversion python bindings'))
 
         try:
             version = svn.core.SVN_VER_MAJOR, svn.core.SVN_VER_MINOR
             if version < (1, 4):
                 raise MissingTool(_('Subversion python bindings %d.%d found, '
                                     '1.4 or later required') % version)
         except AttributeError:
             raise MissingTool(_('Subversion python bindings are too old, 1.4 '
                                 'or later required'))
 
         self.lastrevs = {}
 
         latest = None
         try:
             # Support file://path@rev syntax. Useful e.g. to convert
             # deleted branches.
             at = url.rfind('@')
             if at >= 0:
                 latest = int(url[at + 1:])
                 url = url[:at]
         except ValueError:
             pass
         self.url = geturl(url)
         self.encoding = 'UTF-8' # Subversion is always nominal UTF-8
         try:
             self.transport = transport.SvnRaTransport(url=self.url)
             self.ra = self.transport.ra
             self.ctx = self.transport.client
             self.baseurl = svn.ra.get_repos_root(self.ra)
             # Module is either empty or a repository path starting with
             # a slash and not ending with a slash.
             self.module = urlreq.unquote(self.url[len(self.baseurl):])
             self.prevmodule = None
             self.rootmodule = self.module
             self.commits = {}
             self.paths = {}
             self.uuid = svn.ra.get_uuid(self.ra)
         except svn.core.SubversionException:
             ui.traceback()
             svnversion = '%d.%d.%d' % (svn.core.SVN_VER_MAJOR,
                                        svn.core.SVN_VER_MINOR,
                                        svn.core.SVN_VER_MICRO)
             raise NoRepo(_("%s does not look like a Subversion repository "
                            "to libsvn version %s")
                          % (self.url, svnversion))
 
         if revs:
             if len(revs) > 1:
                 raise error.Abort(_('subversion source does not support '
                                    'specifying multiple revisions'))
             try:
                 latest = int(revs[0])
             except ValueError:
                 raise error.Abort(_('svn: revision %s is not an integer') %
                                  revs[0])
 
         trunkcfg = self.ui.config('convert', 'svn.trunk')
         if trunkcfg is None:
             trunkcfg = 'trunk'
         self.trunkname = trunkcfg.strip('/')
         self.startrev = self.ui.config('convert', 'svn.startrev')
         try:
             self.startrev = int(self.startrev)
             if self.startrev < 0:
                 self.startrev = 0
         except ValueError:
             raise error.Abort(_('svn: start revision %s is not an integer')
                              % self.startrev)
 
         try:
             self.head = self.latest(self.module, latest)
         except SvnPathNotFound:
             self.head = None
         if not self.head:
             raise error.Abort(_('no revision found in module %s')
                              % self.module)
         self.last_changed = self.revnum(self.head)
 
         self._changescache = (None, None)
 
         if os.path.exists(os.path.join(url, '.svn/entries')):
             self.wc = url
         else:
             self.wc = None
         self.convertfp = None
 
     def setrevmap(self, revmap):
         lastrevs = {}
         for revid in revmap:
             uuid, module, revnum = revsplit(revid)
             lastrevnum = lastrevs.setdefault(module, revnum)
             if revnum > lastrevnum:
                 lastrevs[module] = revnum
         self.lastrevs = lastrevs
 
     def exists(self, path, optrev):
         try:
             svn.client.ls(self.url.rstrip('/') + '/' + quote(path),
                                  optrev, False, self.ctx)
             return True
         except svn.core.SubversionException:
             return False
 
     def getheads(self):
 
         def isdir(path, revnum):
             kind = self._checkpath(path, revnum)
             return kind == svn.core.svn_node_dir
 
         def getcfgpath(name, rev):
             cfgpath = self.ui.config('convert', 'svn.' + name)
             if cfgpath is not None and cfgpath.strip() == '':
                 return None
             path = (cfgpath or name).strip('/')
             if not self.exists(path, rev):
                 if self.module.endswith(path) and name == 'trunk':
                     # we are converting from inside this directory
                     return None
                 if cfgpath:
                     raise error.Abort(_('expected %s to be at %r, but not found'
                                        ) % (name, path))
                 return None
             self.ui.note(_('found %s at %r\n') % (name, path))
             return path
 
         rev = optrev(self.last_changed)
         oldmodule = ''
         trunk = getcfgpath('trunk', rev)
         self.tags = getcfgpath('tags', rev)
         branches = getcfgpath('branches', rev)
 
         # If the project has a trunk or branches, we will extract heads
         # from them. We keep the project root otherwise.
         if trunk:
             oldmodule = self.module or ''
             self.module += '/' + trunk
             self.head = self.latest(self.module, self.last_changed)
             if not self.head:
                 raise error.Abort(_('no revision found in module %s')
                                  % self.module)
 
         # First head in the list is the module's head
         self.heads = [self.head]
         if self.tags is not None:
             self.tags = '%s/%s' % (oldmodule , (self.tags or 'tags'))
 
         # Check if branches bring a few more heads to the list
         if branches:
             rpath = self.url.strip('/')
             branchnames = svn.client.ls(rpath + '/' + quote(branches),
                                         rev, False, self.ctx)
             for branch in sorted(branchnames):
                 module = '%s/%s/%s' % (oldmodule, branches, branch)
                 if not isdir(module, self.last_changed):
                     continue
                 brevid = self.latest(module, self.last_changed)
                 if not brevid:
                     self.ui.note(_('ignoring empty branch %s\n') % branch)
                     continue
                 self.ui.note(_('found branch %s at %d\n') %
                              (branch, self.revnum(brevid)))
                 self.heads.append(brevid)
 
         if self.startrev and self.heads:
             if len(self.heads) > 1:
                 raise error.Abort(_('svn: start revision is not supported '
                                    'with more than one branch'))
             revnum = self.revnum(self.heads[0])
             if revnum < self.startrev:
                 raise error.Abort(
                     _('svn: no revision found after start revision %d')
                                  % self.startrev)
 
         return self.heads
 
     def _getchanges(self, rev, full):
         (paths, parents) = self.paths[rev]
         copies = {}
         if parents:
             files, self.removed, copies = self.expandpaths(rev, paths, parents)
         if full or not parents:
             # Perform a full checkout on roots
             uuid, module, revnum = revsplit(rev)
             entries = svn.client.ls(self.baseurl + quote(module),
                                     optrev(revnum), True, self.ctx)
             files = [n for n, e in entries.iteritems()
                      if e.kind == svn.core.svn_node_file]
             self.removed = set()
 
         files.sort()
         files = zip(files, [rev] * len(files))
         return (files, copies)
 
     def getchanges(self, rev, full):
         # reuse cache from getchangedfiles
         if self._changescache[0] == rev and not full:
             (files, copies) = self._changescache[1]
         else:
             (files, copies) = self._getchanges(rev, full)
             # caller caches the result, so free it here to release memory
             del self.paths[rev]
         return (files, copies, set())
 
     def getchangedfiles(self, rev, i):
         # called from filemap - cache computed values for reuse in getchanges
         (files, copies) = self._getchanges(rev, False)
         self._changescache = (rev, (files, copies))
         return [f[0] for f in files]
 
     def getcommit(self, rev):
         if rev not in self.commits:
             uuid, module, revnum = revsplit(rev)
             self.module = module
             self.reparent(module)
             # We assume that:
             # - requests for revisions after "stop" come from the
             # revision graph backward traversal. Cache all of them
             # down to stop, they will be used eventually.
             # - requests for revisions before "stop" come to get
             # isolated branches parents. Just fetch what is needed.
             stop = self.lastrevs.get(module, 0)
             if revnum < stop:
                 stop = revnum + 1
             self._fetch_revisions(revnum, stop)
             if rev not in self.commits:
                 raise error.Abort(_('svn: revision %s not found') % revnum)
         revcommit = self.commits[rev]
         # caller caches the result, so free it here to release memory
         del self.commits[rev]
         return revcommit
 
     def checkrevformat(self, revstr, mapname='splicemap'):
         """ fails if revision format does not match the correct format"""
         if not re.match(r'svn:[0-9a-f]{8,8}-[0-9a-f]{4,4}-'
                               r'[0-9a-f]{4,4}-[0-9a-f]{4,4}-[0-9a-f]'
                               r'{12,12}(.*)\@[0-9]+$',revstr):
             raise error.Abort(_('%s entry %s is not a valid revision'
                                ' identifier') % (mapname, revstr))
 
     def numcommits(self):
         return int(self.head.rsplit('@', 1)[1]) - self.startrev
 
     def gettags(self):
         tags = {}
         if self.tags is None:
             return tags
 
         # svn tags are just a convention, project branches left in a
         # 'tags' directory. There is no other relationship than
         # ancestry, which is expensive to discover and makes them hard
         # to update incrementally.  Worse, past revisions may be
         # referenced by tags far away in the future, requiring a deep
         # history traversal on every calculation.  Current code
         # performs a single backward traversal, tracking moves within
         # the tags directory (tag renaming) and recording a new tag
         # everytime a project is copied from outside the tags
         # directory. It also lists deleted tags, this behaviour may
         # change in the future.
         pendings = []
         tagspath = self.tags
         start = svn.ra.get_latest_revnum(self.ra)
         stream = self._getlog([self.tags], start, self.startrev)
         try:
             for entry in stream:
                 origpaths, revnum, author, date, message = entry
                 if not origpaths:
                     origpaths = []
                 copies = [(e.copyfrom_path, e.copyfrom_rev, p) for p, e
                           in origpaths.iteritems() if e.copyfrom_path]
                 # Apply moves/copies from more specific to general
                 copies.sort(reverse=True)
 
                 srctagspath = tagspath
                 if copies and copies[-1][2] == tagspath:
                     # Track tags directory moves
                     srctagspath = copies.pop()[0]
 
                 for source, sourcerev, dest in copies:
                     if not dest.startswith(tagspath + '/'):
                         continue
                     for tag in pendings:
                         if tag[0].startswith(dest):
                             tagpath = source + tag[0][len(dest):]
                             tag[:2] = [tagpath, sourcerev]
                             break
                     else:
                         pendings.append([source, sourcerev, dest])
 
                 # Filter out tags with children coming from different
                 # parts of the repository like:
                 # /tags/tag.1 (from /trunk:10)
                 # /tags/tag.1/foo (from /branches/foo:12)
                 # Here/tags/tag.1 discarded as well as its children.
                 # It happens with tools like cvs2svn. Such tags cannot
                 # be represented in mercurial.
                 addeds = dict((p, e.copyfrom_path) for p, e
                               in origpaths.iteritems()
                               if e.action == 'A' and e.copyfrom_path)
                 badroots = set()
                 for destroot in addeds:
                     for source, sourcerev, dest in pendings:
                         if (not dest.startswith(destroot + '/')
                             or source.startswith(addeds[destroot] + '/')):
                             continue
                         badroots.add(destroot)
                         break
 
                 for badroot in badroots:
                     pendings = [p for p in pendings if p[2] != badroot
                                 and not p[2].startswith(badroot + '/')]
 
                 # Tell tag renamings from tag creations
                 renamings = []
                 for source, sourcerev, dest in pendings:
                     tagname = dest.split('/')[-1]
                     if source.startswith(srctagspath):
                         renamings.append([source, sourcerev, tagname])
                         continue
                     if tagname in tags:
                         # Keep the latest tag value
                         continue
                     # From revision may be fake, get one with changes
                     try:
                         tagid = self.latest(source, sourcerev)
                         if tagid and tagname not in tags:
                             tags[tagname] = tagid
                     except SvnPathNotFound:
                         # It happens when we are following directories
                         # we assumed were copied with their parents
                         # but were really created in the tag
                         # directory.
                         pass
                 pendings = renamings
                 tagspath = srctagspath
         finally:
             stream.close()
         return tags
 
     def converted(self, rev, destrev):
         if not self.wc:
             return
         if self.convertfp is None:
             self.convertfp = open(os.path.join(self.wc, '.svn', 'hg-shamap'),
                                   'ab')
         self.convertfp.write(util.tonativeeol('%s %d\n'
                                               % (destrev, self.revnum(rev))))
         self.convertfp.flush()
 
     def revid(self, revnum, module=None):
         return 'svn:%s%s@%s' % (self.uuid, module or self.module, revnum)
 
     def revnum(self, rev):
         return int(rev.split('@')[-1])
 
     def latest(self, path, stop=None):
         """Find the latest revid affecting path, up to stop revision
         number. If stop is None, default to repository latest
         revision. It may return a revision in a different module,
         since a branch may be moved without a change being
         reported. Return None if computed module does not belong to
         rootmodule subtree.
         """
         def findchanges(path, start, stop=None):
             stream = self._getlog([path], start, stop or 1)
             try:
                 for entry in stream:
                     paths, revnum, author, date, message = entry
                     if stop is None and paths:
                         # We do not know the latest changed revision,
                         # keep the first one with changed paths.
                         break
                     if revnum <= stop:
                         break
 
                     for p in paths:
                         if (not path.startswith(p) or
                             not paths[p].copyfrom_path):
                             continue
                         newpath = paths[p].copyfrom_path + path[len(p):]
                         self.ui.debug("branch renamed from %s to %s at %d\n" %
                                       (path, newpath, revnum))
                         path = newpath
                         break
                 if not paths:
                     revnum = None
                 return revnum, path
             finally:
                 stream.close()
 
         if not path.startswith(self.rootmodule):
             # Requests on foreign branches may be forbidden at server level
             self.ui.debug('ignoring foreign branch %r\n' % path)
             return None
 
         if stop is None:
             stop = svn.ra.get_latest_revnum(self.ra)
         try:
             prevmodule = self.reparent('')
             dirent = svn.ra.stat(self.ra, path.strip('/'), stop)
             self.reparent(prevmodule)
         except svn.core.SubversionException:
             dirent = None
         if not dirent:
             raise SvnPathNotFound(_('%s not found up to revision %d')
                                   % (path, stop))
 
         # stat() gives us the previous revision on this line of
         # development, but it might be in *another module*. Fetch the
         # log and detect renames down to the latest revision.
         revnum, realpath = findchanges(path, stop, dirent.created_rev)
         if revnum is None:
             # Tools like svnsync can create empty revision, when
             # synchronizing only a subtree for instance. These empty
             # revisions created_rev still have their original values
             # despite all changes having disappeared and can be
             # returned by ra.stat(), at least when stating the root
             # module. In that case, do not trust created_rev and scan
             # the whole history.
             revnum, realpath = findchanges(path, stop)
             if revnum is None:
                 self.ui.debug('ignoring empty branch %r\n' % realpath)
                 return None
 
         if not realpath.startswith(self.rootmodule):
             self.ui.debug('ignoring foreign branch %r\n' % realpath)
             return None
         return self.revid(revnum, realpath)
 
     def reparent(self, module):
         """Reparent the svn transport and return the previous parent."""
         if self.prevmodule == module:
             return module
         svnurl = self.baseurl + quote(module)
         prevmodule = self.prevmodule
         if prevmodule is None:
             prevmodule = ''
         self.ui.debug("reparent to %s\n" % svnurl)
         svn.ra.reparent(self.ra, svnurl)
         self.prevmodule = module
         return prevmodule
 
     def expandpaths(self, rev, paths, parents):
         changed, removed = set(), set()
         copies = {}
 
         new_module, revnum = revsplit(rev)[1:]
         if new_module != self.module:
             self.module = new_module
             self.reparent(self.module)
 
         for i, (path, ent) in enumerate(paths):
             self.ui.progress(_('scanning paths'), i, item=path,
                              total=len(paths), unit=_('paths'))
             entrypath = self.getrelpath(path)
 
             kind = self._checkpath(entrypath, revnum)
             if kind == svn.core.svn_node_file:
                 changed.add(self.recode(entrypath))
                 if not ent.copyfrom_path or not parents:
                     continue
                 # Copy sources not in parent revisions cannot be
                 # represented, ignore their origin for now
                 pmodule, prevnum = revsplit(parents[0])[1:]
                 if ent.copyfrom_rev < prevnum:
                     continue
                 copyfrom_path = self.getrelpath(ent.copyfrom_path, pmodule)
                 if not copyfrom_path:
                     continue
                 self.ui.debug("copied to %s from %s@%s\n" %
                               (entrypath, copyfrom_path, ent.copyfrom_rev))
                 copies[self.recode(entrypath)] = self.recode(copyfrom_path)
             elif kind == 0: # gone, but had better be a deleted *file*
                 self.ui.debug("gone from %s\n" % ent.copyfrom_rev)
                 pmodule, prevnum = revsplit(parents[0])[1:]
                 parentpath = pmodule + "/" + entrypath
                 fromkind = self._checkpath(entrypath, prevnum, pmodule)
 
                 if fromkind == svn.core.svn_node_file:
                     removed.add(self.recode(entrypath))
                 elif fromkind == svn.core.svn_node_dir:
                     oroot = parentpath.strip('/')
                     nroot = path.strip('/')
                     children = self._iterfiles(oroot, prevnum)
                     for childpath in children:
                         childpath = childpath.replace(oroot, nroot)
                         childpath = self.getrelpath("/" + childpath, pmodule)
                         if childpath:
                             removed.add(self.recode(childpath))
                 else:
                     self.ui.debug('unknown path in revision %d: %s\n' % \
                                   (revnum, path))
             elif kind == svn.core.svn_node_dir:
                 if ent.action == 'M':
                     # If the directory just had a prop change,
                     # then we shouldn't need to look for its children.
                     continue
                 if ent.action == 'R' and parents:
                     # If a directory is replacing a file, mark the previous
                     # file as deleted
                     pmodule, prevnum = revsplit(parents[0])[1:]
                     pkind = self._checkpath(entrypath, prevnum, pmodule)
                     if pkind == svn.core.svn_node_file:
                         removed.add(self.recode(entrypath))
                     elif pkind == svn.core.svn_node_dir:
                         # We do not know what files were kept or removed,
                         # mark them all as changed.
                         for childpath in self._iterfiles(pmodule, prevnum):
                             childpath = self.getrelpath("/" + childpath)
                             if childpath:
                                 changed.add(self.recode(childpath))
 
                 for childpath in self._iterfiles(path, revnum):
                     childpath = self.getrelpath("/" + childpath)
                     if childpath:
                         changed.add(self.recode(childpath))
 
                 # Handle directory copies
                 if not ent.copyfrom_path or not parents:
                     continue
                 # Copy sources not in parent revisions cannot be
                 # represented, ignore their origin for now
                 pmodule, prevnum = revsplit(parents[0])[1:]
                 if ent.copyfrom_rev < prevnum:
                     continue
                 copyfrompath = self.getrelpath(ent.copyfrom_path, pmodule)
                 if not copyfrompath:
                     continue
                 self.ui.debug("mark %s came from %s:%d\n"
                               % (path, copyfrompath, ent.copyfrom_rev))
                 children = self._iterfiles(ent.copyfrom_path, ent.copyfrom_rev)
                 for childpath in children:
                     childpath = self.getrelpath("/" + childpath, pmodule)
                     if not childpath:
                         continue
                     copytopath = path + childpath[len(copyfrompath):]
                     copytopath = self.getrelpath(copytopath)
                     copies[self.recode(copytopath)] = self.recode(childpath)
 
         self.ui.progress(_('scanning paths'), None)
         changed.update(removed)
         return (list(changed), removed, copies)
 
     def _fetch_revisions(self, from_revnum, to_revnum):
         if from_revnum < to_revnum:
             from_revnum, to_revnum = to_revnum, from_revnum
 
         self.child_cset = None
 
         def parselogentry(orig_paths, revnum, author, date, message):
             """Return the parsed commit object or None, and True if
             the revision is a branch root.
             """
             self.ui.debug("parsing revision %d (%d changes)\n" %
                           (revnum, len(orig_paths)))
 
             branched = False
             rev = self.revid(revnum)
             # branch log might return entries for a parent we already have
 
             if rev in self.commits or revnum < to_revnum:
                 return None, branched
 
             parents = []
             # check whether this revision is the start of a branch or part
             # of a branch renaming
             orig_paths = sorted(orig_paths.iteritems())
             root_paths = [(p, e) for p, e in orig_paths
                           if self.module.startswith(p)]
             if root_paths:
                 path, ent = root_paths[-1]
                 if ent.copyfrom_path:
                     branched = True
                     newpath = ent.copyfrom_path + self.module[len(path):]
                     # ent.copyfrom_rev may not be the actual last revision
                     previd = self.latest(newpath, ent.copyfrom_rev)
                     if previd is not None:
                         prevmodule, prevnum = revsplit(previd)[1:]
                         if prevnum >= self.startrev:
                             parents = [previd]
                             self.ui.note(
                                 _('found parent of branch %s at %d: %s\n') %
                                 (self.module, prevnum, prevmodule))
                 else:
                     self.ui.debug("no copyfrom path, don't know what to do.\n")
 
             paths = []
             # filter out unrelated paths
             for path, ent in orig_paths:
                 if self.getrelpath(path) is None:
                     continue
                 paths.append((path, ent))
 
             # Example SVN datetime. Includes microseconds.
             # ISO-8601 conformant
             # '2007-01-04T17:35:00.902377Z'
             date = dateutil.parsedate(date[:19] + " UTC", ["%Y-%m-%dT%H:%M:%S"])
             if self.ui.configbool('convert', 'localtimezone'):
                 date = makedatetimestamp(date[0])
 
             if message:
                 log = self.recode(message)
             else:
                 log = ''
 
             if author:
                 author = self.recode(author)
             else:
                 author = ''
 
             try:
                 branch = self.module.split("/")[-1]
                 if branch == self.trunkname:
                     branch = None
             except IndexError:
                 branch = None
 
             cset = commit(author=author,
                           date=dateutil.datestr(date, '%Y-%m-%d %H:%M:%S %1%2'),
                           desc=log,
                           parents=parents,
                           branch=branch,
                           rev=rev)
 
             self.commits[rev] = cset
             # The parents list is *shared* among self.paths and the
             # commit object. Both will be updated below.
             self.paths[rev] = (paths, cset.parents)
             if self.child_cset and not self.child_cset.parents:
                 self.child_cset.parents[:] = [rev]
             self.child_cset = cset
             return cset, branched
 
         self.ui.note(_('fetching revision log for "%s" from %d to %d\n') %
                      (self.module, from_revnum, to_revnum))
 
         try:
             firstcset = None
             lastonbranch = False
             stream = self._getlog([self.module], from_revnum, to_revnum)
             try:
                 for entry in stream:
                     paths, revnum, author, date, message = entry
                     if revnum < self.startrev:
                         lastonbranch = True
                         break
                     if not paths:
                         self.ui.debug('revision %d has no entries\n' % revnum)
                         # If we ever leave the loop on an empty
                         # revision, do not try to get a parent branch
                         lastonbranch = lastonbranch or revnum == 0
                         continue
                     cset, lastonbranch = parselogentry(paths, revnum, author,
                                                        date, message)
                     if cset:
                         firstcset = cset
                     if lastonbranch:
                         break
             finally:
                 stream.close()
 
             if not lastonbranch and firstcset and not firstcset.parents:
                 # The first revision of the sequence (the last fetched one)
                 # has invalid parents if not a branch root. Find the parent
                 # revision now, if any.
                 try:
                     firstrevnum = self.revnum(firstcset.rev)
                     if firstrevnum > 1:
                         latest = self.latest(self.module, firstrevnum - 1)
                         if latest:
                             firstcset.parents.append(latest)
                 except SvnPathNotFound:
                     pass
         except svn.core.SubversionException as xxx_todo_changeme:
             (inst, num) = xxx_todo_changeme.args
             if num == svn.core.SVN_ERR_FS_NO_SUCH_REVISION:
                 raise error.Abort(_('svn: branch has no revision %s')
                                  % to_revnum)
             raise
 
     def getfile(self, file, rev):
         # TODO: ra.get_file transmits the whole file instead of diffs.
         if file in self.removed:
             return None, None
         mode = ''
         try:
             new_module, revnum = revsplit(rev)[1:]
             if self.module != new_module:
                 self.module = new_module
                 self.reparent(self.module)
             io = stringio()
             info = svn.ra.get_file(self.ra, file, revnum, io)
             data = io.getvalue()
             # ra.get_file() seems to keep a reference on the input buffer
             # preventing collection. Release it explicitly.
             io.close()
             if isinstance(info, list):
                 info = info[-1]
             mode = ("svn:executable" in info) and 'x' or ''
             mode = ("svn:special" in info) and 'l' or mode
         except svn.core.SubversionException as e:
             notfound = (svn.core.SVN_ERR_FS_NOT_FOUND,
                 svn.core.SVN_ERR_RA_DAV_PATH_NOT_FOUND)
             if e.apr_err in notfound: # File not found
                 return None, None
             raise
         if mode == 'l':
             link_prefix = "link "
             if data.startswith(link_prefix):
                 data = data[len(link_prefix):]
         return data, mode
 
     def _iterfiles(self, path, revnum):
         """Enumerate all files in path at revnum, recursively."""
         path = path.strip('/')
         pool = svn.core.Pool()
         rpath = '/'.join([self.baseurl, quote(path)]).strip('/')
         entries = svn.client.ls(rpath, optrev(revnum), True, self.ctx, pool)
         if path:
             path += '/'
         return ((path + p) for p, e in entries.iteritems()
                 if e.kind == svn.core.svn_node_file)
 
     def getrelpath(self, path, module=None):
         if module is None:
             module = self.module
         # Given the repository url of this wc, say
         #   "http://server/plone/CMFPlone/branches/Plone-2_0-branch"
         # extract the "entry" portion (a relative path) from what
         # svn log --xml says, i.e.
         #   "/CMFPlone/branches/Plone-2_0-branch/tests/PloneTestCase.py"
         # that is to say "tests/PloneTestCase.py"
         if path.startswith(module):
             relative = path.rstrip('/')[len(module):]
             if relative.startswith('/'):
                 return relative[1:]
             elif relative == '':
                 return relative
 
         # The path is outside our tracked tree...
         self.ui.debug('%r is not under %r, ignoring\n' % (path, module))
         return None
 
     def _checkpath(self, path, revnum, module=None):
         if module is not None:
             prevmodule = self.reparent('')
             path = module + '/' + path
         try:
             # ra.check_path does not like leading slashes very much, it leads
             # to PROPFIND subversion errors
             return svn.ra.check_path(self.ra, path.strip('/'), revnum)
         finally:
             if module is not None:
                 self.reparent(prevmodule)
 
     def _getlog(self, paths, start, end, limit=0, discover_changed_paths=True,
                 strict_node_history=False):
         # Normalize path names, svn >= 1.5 only wants paths relative to
         # supplied URL
         relpaths = []
         for p in paths:
             if not p.startswith('/'):
                 p = self.module + '/' + p
             relpaths.append(p.strip('/'))
         args = [self.baseurl, relpaths, start, end, limit,
                 discover_changed_paths, strict_node_history]
         # developer config: convert.svn.debugsvnlog
         if not self.ui.configbool('convert', 'svn.debugsvnlog'):
             return directlogstream(*args)
         arg = encodeargs(args)
         hgexe = util.hgexecutable()
         cmd = '%s debugsvnlog' % util.shellquote(hgexe)
         stdin, stdout = util.popen2(util.quotecommand(cmd))
         stdin.write(arg)
         try:
             stdin.close()
         except IOError:
             raise error.Abort(_('Mercurial failed to run itself, check'
                                ' hg executable is in PATH'))
         return logstream(stdout)
 
 pre_revprop_change = '''#!/bin/sh
 
 REPOS="$1"
 REV="$2"
 USER="$3"
 PROPNAME="$4"
 ACTION="$5"
 
 if [ "$ACTION" = "M" -a "$PROPNAME" = "svn:log" ]; then exit 0; fi
 if [ "$ACTION" = "A" -a "$PROPNAME" = "hg:convert-branch" ]; then exit 0; fi
 if [ "$ACTION" = "A" -a "$PROPNAME" = "hg:convert-rev" ]; then exit 0; fi
 
 echo "Changing prohibited revision property" >&2
 exit 1
 '''
 
 class svn_sink(converter_sink, commandline):
     commit_re = re.compile(r'Committed revision (\d+).', re.M)
     uuid_re = re.compile(r'Repository UUID:\s*(\S+)', re.M)
 
     def prerun(self):
         if self.wc:
             os.chdir(self.wc)
 
     def postrun(self):
         if self.wc:
             os.chdir(self.cwd)
 
     def join(self, name):
         return os.path.join(self.wc, '.svn', name)
 
     def revmapfile(self):
         return self.join('hg-shamap')
 
     def authorfile(self):
         return self.join('hg-authormap')
 
     def __init__(self, ui, repotype, path):
 
         converter_sink.__init__(self, ui, repotype, path)
         commandline.__init__(self, ui, 'svn')
         self.delete = []
         self.setexec = []
         self.delexec = []
         self.copies = []
         self.wc = None
         self.cwd = pycompat.getcwd()
 
         created = False
         if os.path.isfile(os.path.join(path, '.svn', 'entries')):
             self.wc = os.path.realpath(path)
             self.run0('update')
         else:
             if not re.search(br'^(file|http|https|svn|svn\+ssh)\://', path):
                 path = os.path.realpath(path)
                 if os.path.isdir(os.path.dirname(path)):
                     if not os.path.exists(os.path.join(path, 'db', 'fs-type')):
                         ui.status(_('initializing svn repository %r\n') %
                                   os.path.basename(path))
                         commandline(ui, 'svnadmin').run0('create', path)
                         created = path
                     path = util.normpath(path)
                     if not path.startswith('/'):
                         path = '/' + path
                     path = 'file://' + path
 
             wcpath = os.path.join(pycompat.getcwd(), os.path.basename(path) +
                                 '-wc')
             ui.status(_('initializing svn working copy %r\n')
                       % os.path.basename(wcpath))
             self.run0('checkout', path, wcpath)
 
             self.wc = wcpath
         self.opener = vfsmod.vfs(self.wc)
         self.wopener = vfsmod.vfs(self.wc)
         self.childmap = mapfile(ui, self.join('hg-childmap'))
         if util.checkexec(self.wc):
             self.is_exec = util.isexec
         else:
             self.is_exec = None
 
         if created:
             hook = os.path.join(created, 'hooks', 'pre-revprop-change')
             fp = open(hook, 'wb')
             fp.write(pre_revprop_change)
             fp.close()
             util.setflags(hook, False, True)
 
         output = self.run0('info')
         self.uuid = self.uuid_re.search(output).group(1).strip()
 
     def wjoin(self, *names):
         return os.path.join(self.wc, *names)
 
     @propertycache
     def manifest(self):
         # As of svn 1.7, the "add" command fails when receiving
         # already tracked entries, so we have to track and filter them
         # ourselves.
         m = set()
         output = self.run0('ls', recursive=True, xml=True)
         doc = xml.dom.minidom.parseString(output)
         for e in doc.getElementsByTagName('entry'):
             for n in e.childNodes:
                 if n.nodeType != n.ELEMENT_NODE or n.tagName != 'name':
                     continue
                 name = ''.join(c.data for c in n.childNodes
                                if c.nodeType == c.TEXT_NODE)
                 # Entries are compared with names coming from
                 # mercurial, so bytes with undefined encoding. Our
                 # best bet is to assume they are in local
                 # encoding. They will be passed to command line calls
                 # later anyway, so they better be.
                 m.add(encoding.unitolocal(name))
                 break
         return m
 
     def putfile(self, filename, flags, data):
         if 'l' in flags:
             self.wopener.symlink(data, filename)
         else:
             try:
                 if os.path.islink(self.wjoin(filename)):
                     os.unlink(filename)
             except OSError:
                 pass
             self.wopener.write(filename, data)
 
             if self.is_exec:
                 if self.is_exec(self.wjoin(filename)):
                     if 'x' not in flags:
                         self.delexec.append(filename)
                 else:
                     if 'x' in flags:
                         self.setexec.append(filename)
                 util.setflags(self.wjoin(filename), False, 'x' in flags)
 
     def _copyfile(self, source, dest):
         # SVN's copy command pukes if the destination file exists, but
         # our copyfile method expects to record a copy that has
         # already occurred.  Cross the semantic gap.
         wdest = self.wjoin(dest)
         exists = os.path.lexists(wdest)
         if exists:
             fd, tempname = tempfile.mkstemp(
                 prefix='hg-copy-', dir=os.path.dirname(wdest))
             os.close(fd)
             os.unlink(tempname)
             os.rename(wdest, tempname)
         try:
             self.run0('copy', source, dest)
         finally:
             self.manifest.add(dest)
             if exists:
                 try:
                     os.unlink(wdest)
                 except OSError:
                     pass
                 os.rename(tempname, wdest)
 
     def dirs_of(self, files):
         dirs = set()
         for f in files:
             if os.path.isdir(self.wjoin(f)):
                 dirs.add(f)
             i = len(f)
             for i in iter(lambda: f.rfind('/', 0, i), -1):
                 dirs.add(f[:i])
         return dirs
 
     def add_dirs(self, files):
         add_dirs = [d for d in sorted(self.dirs_of(files))
                     if d not in self.manifest]
         if add_dirs:
             self.manifest.update(add_dirs)
             self.xargs(add_dirs, 'add', non_recursive=True, quiet=True)
         return add_dirs
 
     def add_files(self, files):
         files = [f for f in files if f not in self.manifest]
         if files:
             self.manifest.update(files)
             self.xargs(files, 'add', quiet=True)
         return files
 
     def addchild(self, parent, child):
         self.childmap[parent] = child
 
     def revid(self, rev):
         return u"svn:%s@%s" % (self.uuid, rev)
 
     def putcommit(self, files, copies, parents, commit, source, revmap, full,
                   cleanp2):
         for parent in parents:
             try:
                 return self.revid(self.childmap[parent])
             except KeyError:
                 pass
 
         # Apply changes to working copy
         for f, v in files:
             data, mode = source.getfile(f, v)
             if data is None:
                 self.delete.append(f)
             else:
                 self.putfile(f, mode, data)
                 if f in copies:
                     self.copies.append([copies[f], f])
         if full:
             self.delete.extend(sorted(self.manifest.difference(files)))
         files = [f[0] for f in files]
 
         entries = set(self.delete)
         files = frozenset(files)
         entries.update(self.add_dirs(files.difference(entries)))
         if self.copies:
             for s, d in self.copies:
                 self._copyfile(s, d)
             self.copies = []
         if self.delete:
             self.xargs(self.delete, 'delete')
             for f in self.delete:
                 self.manifest.remove(f)
             self.delete = []
         entries.update(self.add_files(files.difference(entries)))
         if self.delexec:
             self.xargs(self.delexec, 'propdel', 'svn:executable')
             self.delexec = []
         if self.setexec:
             self.xargs(self.setexec, 'propset', 'svn:executable', '*')
             self.setexec = []
 
         fd, messagefile = tempfile.mkstemp(prefix='hg-convert-')
         fp = os.fdopen(fd, r'wb')
         fp.write(util.tonativeeol(commit.desc))
         fp.close()
         try:
             output = self.run0('commit',
-                               username=util.shortuser(commit.author),
+                               username=stringutil.shortuser(commit.author),
                                file=messagefile,
                                encoding='utf-8')
             try:
                 rev = self.commit_re.search(output).group(1)
             except AttributeError:
                 if parents and not files:
                     return parents[0]
                 self.ui.warn(_('unexpected svn output:\n'))
                 self.ui.warn(output)
                 raise error.Abort(_('unable to cope with svn output'))
             if commit.rev:
                 self.run('propset', 'hg:convert-rev', commit.rev,
                          revprop=True, revision=rev)
             if commit.branch and commit.branch != 'default':
                 self.run('propset', 'hg:convert-branch', commit.branch,
                          revprop=True, revision=rev)
             for parent in parents:
                 self.addchild(parent, rev)
             return self.revid(rev)
         finally:
             os.unlink(messagefile)
 
     def puttags(self, tags):
         self.ui.warn(_('writing Subversion tags is not yet implemented\n'))
         return None, None
 
     def hascommitfrommap(self, rev):
         # We trust that revisions referenced in a map still is present
         # TODO: implement something better if necessary and feasible
         return True
 
     def hascommitforsplicemap(self, rev):
         # This is not correct as one can convert to an existing subversion
         # repository and childmap would not list all revisions. Too bad.
         if rev in self.childmap:
             return True
         raise error.Abort(_('splice map revision %s not found in subversion '
                            'child map (revision lookups are not implemented)')
                          % rev)

hgext/eol.py

@@ -1,416 +1,419
 """automatically manage newlines in repository files
 
 This extension allows you to manage the type of line endings (CRLF or
 LF) that are used in the repository and in the local working
 directory. That way you can get CRLF line endings on Windows and LF on
 Unix/Mac, thereby letting everybody use their OS native line endings.
 
 The extension reads its configuration from a versioned ``.hgeol``
 configuration file found in the root of the working directory. The
 ``.hgeol`` file use the same syntax as all other Mercurial
 configuration files. It uses two sections, ``[patterns]`` and
 ``[repository]``.
 
 The ``[patterns]`` section specifies how line endings should be
 converted between the working directory and the repository. The format is
 specified by a file pattern. The first match is used, so put more
 specific patterns first. The available line endings are ``LF``,
 ``CRLF``, and ``BIN``.
 
 Files with the declared format of ``CRLF`` or ``LF`` are always
 checked out and stored in the repository in that format and files
 declared to be binary (``BIN``) are left unchanged. Additionally,
 ``native`` is an alias for checking out in the platform's default line
 ending: ``LF`` on Unix (including Mac OS X) and ``CRLF`` on
 Windows. Note that ``BIN`` (do nothing to line endings) is Mercurial's
 default behavior; it is only needed if you need to override a later,
 more general pattern.
 
 The optional ``[repository]`` section specifies the line endings to
 use for files stored in the repository. It has a single setting,
 ``native``, which determines the storage line endings for files
 declared as ``native`` in the ``[patterns]`` section. It can be set to
 ``LF`` or ``CRLF``. The default is ``LF``. For example, this means
 that on Windows, files configured as ``native`` (``CRLF`` by default)
 will be converted to ``LF`` when stored in the repository. Files
 declared as ``LF``, ``CRLF``, or ``BIN`` in the ``[patterns]`` section
 are always stored as-is in the repository.
 
 Example versioned ``.hgeol`` file::
 
   [patterns]
   **.py = native
   **.vcproj = CRLF
   **.txt = native
   Makefile = LF
   **.jpg = BIN
 
   [repository]
   native = LF
 
 .. note::
 
    The rules will first apply when files are touched in the working
    directory, e.g. by updating to null and back to tip to touch all files.
 
 The extension uses an optional ``[eol]`` section read from both the
 normal Mercurial configuration files and the ``.hgeol`` file, with the
 latter overriding the former. You can use that section to control the
 overall behavior. There are three settings:
 
 - ``eol.native`` (default ``os.linesep``) can be set to ``LF`` or
   ``CRLF`` to override the default interpretation of ``native`` for
   checkout. This can be used with :hg:`archive` on Unix, say, to
   generate an archive where files have line endings for Windows.
 
 - ``eol.only-consistent`` (default True) can be set to False to make
   the extension convert files with inconsistent EOLs. Inconsistent
   means that there is both ``CRLF`` and ``LF`` present in the file.
   Such files are normally not touched under the assumption that they
   have mixed EOLs on purpose.
 
 - ``eol.fix-trailing-newline`` (default False) can be set to True to
   ensure that converted files end with a EOL character (either ``\\n``
   or ``\\r\\n`` as per the configured patterns).
 
 The extension provides ``cleverencode:`` and ``cleverdecode:`` filters
 like the deprecated win32text extension does. This means that you can
 disable win32text and enable eol and your filters will still work. You
 only need to these filters until you have prepared a ``.hgeol`` file.
 
 The ``win32text.forbid*`` hooks provided by the win32text extension
 have been unified into a single hook named ``eol.checkheadshook``. The
 hook will lookup the expected line endings from the ``.hgeol`` file,
 which means you must migrate to a ``.hgeol`` file first before using
 the hook. ``eol.checkheadshook`` only checks heads, intermediate
 invalid revisions will be pushed. To forbid them completely, use the
 ``eol.checkallhook`` hook. These hooks are best used as
 ``pretxnchangegroup`` hooks.
 
 See :hg:`help patterns` for more information about the glob patterns
 used.
 """
 
 from __future__ import absolute_import
 
 import os
 import re
 from mercurial.i18n import _
 from mercurial import (
     config,
     error as errormod,
     extensions,
     match,
     pycompat,
     registrar,
     util,
 )
+from mercurial.utils import (
+    stringutil,
+)
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 configitem('eol', 'fix-trailing-newline',
     default=False,
 )
 configitem('eol', 'native',
     default=pycompat.oslinesep,
 )
 configitem('eol', 'only-consistent',
     default=True,
 )
 
 # Matches a lone LF, i.e., one that is not part of CRLF.
 singlelf = re.compile('(^|[^\r])\n')
 
 def inconsistenteol(data):
     return '\r\n' in data and singlelf.search(data)
 
 def tolf(s, params, ui, **kwargs):
     """Filter to convert to LF EOLs."""
-    if util.binary(s):
+    if stringutil.binary(s):
         return s
     if ui.configbool('eol', 'only-consistent') and inconsistenteol(s):
         return s
     if (ui.configbool('eol', 'fix-trailing-newline')
         and s and s[-1] != '\n'):
         s = s + '\n'
     return util.tolf(s)
 
 def tocrlf(s, params, ui, **kwargs):
     """Filter to convert to CRLF EOLs."""
-    if util.binary(s):
+    if stringutil.binary(s):
         return s
     if ui.configbool('eol', 'only-consistent') and inconsistenteol(s):
         return s
     if (ui.configbool('eol', 'fix-trailing-newline')
         and s and s[-1] != '\n'):
         s = s + '\n'
     return util.tocrlf(s)
 
 def isbinary(s, params):
     """Filter to do nothing with the file."""
     return s
 
 filters = {
     'to-lf': tolf,
     'to-crlf': tocrlf,
     'is-binary': isbinary,
     # The following provide backwards compatibility with win32text
     'cleverencode:': tolf,
     'cleverdecode:': tocrlf
 }
 
 class eolfile(object):
     def __init__(self, ui, root, data):
         self._decode = {'LF': 'to-lf', 'CRLF': 'to-crlf', 'BIN': 'is-binary'}
         self._encode = {'LF': 'to-lf', 'CRLF': 'to-crlf', 'BIN': 'is-binary'}
 
         self.cfg = config.config()
         # Our files should not be touched. The pattern must be
         # inserted first override a '** = native' pattern.
         self.cfg.set('patterns', '.hg*', 'BIN', 'eol')
         # We can then parse the user's patterns.
         self.cfg.parse('.hgeol', data)
 
         isrepolf = self.cfg.get('repository', 'native') != 'CRLF'
         self._encode['NATIVE'] = isrepolf and 'to-lf' or 'to-crlf'
         iswdlf = ui.config('eol', 'native') in ('LF', '\n')
         self._decode['NATIVE'] = iswdlf and 'to-lf' or 'to-crlf'
 
         include = []
         exclude = []
         self.patterns = []
         for pattern, style in self.cfg.items('patterns'):
             key = style.upper()
             if key == 'BIN':
                 exclude.append(pattern)
             else:
                 include.append(pattern)
             m = match.match(root, '', [pattern])
             self.patterns.append((pattern, key, m))
         # This will match the files for which we need to care
         # about inconsistent newlines.
         self.match = match.match(root, '', [], include, exclude)
 
     def copytoui(self, ui):
         for pattern, key, m in self.patterns:
             try:
                 ui.setconfig('decode', pattern, self._decode[key], 'eol')
                 ui.setconfig('encode', pattern, self._encode[key], 'eol')
             except KeyError:
                 ui.warn(_("ignoring unknown EOL style '%s' from %s\n")
                         % (key, self.cfg.source('patterns', pattern)))
         # eol.only-consistent can be specified in ~/.hgrc or .hgeol
         for k, v in self.cfg.items('eol'):
             ui.setconfig('eol', k, v, 'eol')
 
     def checkrev(self, repo, ctx, files):
         failed = []
         for f in (files or ctx.files()):
             if f not in ctx:
                 continue
             for pattern, key, m in self.patterns:
                 if not m(f):
                     continue
                 target = self._encode[key]
                 data = ctx[f].data()
                 if (target == "to-lf" and "\r\n" in data
                     or target == "to-crlf" and singlelf.search(data)):
                     failed.append((f, target, bytes(ctx)))
                 break
         return failed
 
 def parseeol(ui, repo, nodes):
     try:
         for node in nodes:
             try:
                 if node is None:
                     # Cannot use workingctx.data() since it would load
                     # and cache the filters before we configure them.
                     data = repo.wvfs('.hgeol').read()
                 else:
                     data = repo[node]['.hgeol'].data()
                 return eolfile(ui, repo.root, data)
             except (IOError, LookupError):
                 pass
     except errormod.ParseError as inst:
         ui.warn(_("warning: ignoring .hgeol file due to parse error "
                   "at %s: %s\n") % (inst.args[1], inst.args[0]))
     return None
 
 def ensureenabled(ui):
     """make sure the extension is enabled when used as hook
 
     When eol is used through hooks, the extension is never formally loaded and
     enabled. This has some side effect, for example the config declaration is
     never loaded. This function ensure the extension is enabled when running
     hooks.
     """
     if 'eol' in ui._knownconfig:
         return
     ui.setconfig('extensions', 'eol', '', source='internal')
     extensions.loadall(ui, ['eol'])
 
 def _checkhook(ui, repo, node, headsonly):
     # Get revisions to check and touched files at the same time
     ensureenabled(ui)
     files = set()
     revs = set()
     for rev in xrange(repo[node].rev(), len(repo)):
         revs.add(rev)
         if headsonly:
             ctx = repo[rev]
             files.update(ctx.files())
             for pctx in ctx.parents():
                 revs.discard(pctx.rev())
     failed = []
     for rev in revs:
         ctx = repo[rev]
         eol = parseeol(ui, repo, [ctx.node()])
         if eol:
             failed.extend(eol.checkrev(repo, ctx, files))
 
     if failed:
         eols = {'to-lf': 'CRLF', 'to-crlf': 'LF'}
         msgs = []
         for f, target, node in sorted(failed):
             msgs.append(_("  %s in %s should not have %s line endings") %
                         (f, node, eols[target]))
         raise errormod.Abort(_("end-of-line check failed:\n") + "\n".join(msgs))
 
 def checkallhook(ui, repo, node, hooktype, **kwargs):
     """verify that files have expected EOLs"""
     _checkhook(ui, repo, node, False)
 
 def checkheadshook(ui, repo, node, hooktype, **kwargs):
     """verify that files have expected EOLs"""
     _checkhook(ui, repo, node, True)
 
 # "checkheadshook" used to be called "hook"
 hook = checkheadshook
 
 def preupdate(ui, repo, hooktype, parent1, parent2):
     repo.loadeol([parent1])
     return False
 
 def uisetup(ui):
     ui.setconfig('hooks', 'preupdate.eol', preupdate, 'eol')
 
 def extsetup(ui):
     try:
         extensions.find('win32text')
         ui.warn(_("the eol extension is incompatible with the "
                   "win32text extension\n"))
     except KeyError:
         pass
 
 
 def reposetup(ui, repo):
     uisetup(repo.ui)
 
     if not repo.local():
         return
     for name, fn in filters.iteritems():
         repo.adddatafilter(name, fn)
 
     ui.setconfig('patch', 'eol', 'auto', 'eol')
 
     class eolrepo(repo.__class__):
 
         def loadeol(self, nodes):
             eol = parseeol(self.ui, self, nodes)
             if eol is None:
                 return None
             eol.copytoui(self.ui)
             return eol.match
 
         def _hgcleardirstate(self):
             self._eolmatch = self.loadeol([None, 'tip'])
             if not self._eolmatch:
                 self._eolmatch = util.never
                 return
 
             oldeol = None
             try:
                 cachemtime = os.path.getmtime(self.vfs.join("eol.cache"))
             except OSError:
                 cachemtime = 0
             else:
                 olddata = self.vfs.read("eol.cache")
                 if olddata:
                     oldeol = eolfile(self.ui, self.root, olddata)
 
             try:
                 eolmtime = os.path.getmtime(self.wjoin(".hgeol"))
             except OSError:
                 eolmtime = 0
 
             if eolmtime > cachemtime:
                 self.ui.debug("eol: detected change in .hgeol\n")
 
                 hgeoldata = self.wvfs.read('.hgeol')
                 neweol = eolfile(self.ui, self.root, hgeoldata)
 
                 wlock = None
                 try:
                     wlock = self.wlock()
                     for f in self.dirstate:
                         if self.dirstate[f] != 'n':
                             continue
                         if oldeol is not None:
                             if not oldeol.match(f) and not neweol.match(f):
                                 continue
                             oldkey = None
                             for pattern, key, m in oldeol.patterns:
                                 if m(f):
                                     oldkey = key
                                     break
                             newkey = None
                             for pattern, key, m in neweol.patterns:
                                 if m(f):
                                     newkey = key
                                     break
                             if oldkey == newkey:
                                 continue
                         # all normal files need to be looked at again since
                         # the new .hgeol file specify a different filter
                         self.dirstate.normallookup(f)
                     # Write the cache to update mtime and cache .hgeol
                     with self.vfs("eol.cache", "w") as f:
                         f.write(hgeoldata)
                 except errormod.LockUnavailable:
                     # If we cannot lock the repository and clear the
                     # dirstate, then a commit might not see all files
                     # as modified. But if we cannot lock the
                     # repository, then we can also not make a commit,
                     # so ignore the error.
                     pass
                 finally:
                     if wlock is not None:
                         wlock.release()
 
         def commitctx(self, ctx, error=False):
             for f in sorted(ctx.added() + ctx.modified()):
                 if not self._eolmatch(f):
                     continue
                 fctx = ctx[f]
                 if fctx is None:
                     continue
                 data = fctx.data()
-                if util.binary(data):
+                if stringutil.binary(data):
                     # We should not abort here, since the user should
                     # be able to say "** = native" to automatically
                     # have all non-binary files taken care of.
                     continue
                 if inconsistenteol(data):
                     raise errormod.Abort(_("inconsistent newline style "
                                            "in %s\n") % f)
             return super(eolrepo, self).commitctx(ctx, error)
     repo.__class__ = eolrepo
     repo._hgcleardirstate()

hgext/extdiff.py

@@ -1,418 +1,421
 # extdiff.py - external diff program support for mercurial
 #
 # Copyright 2006 Vadim Gelfer <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''command to allow external programs to compare revisions
 
 The extdiff Mercurial extension allows you to use external programs
 to compare revisions, or revision with working directory. The external
 diff programs are called with a configurable set of options and two
 non-option arguments: paths to directories containing snapshots of
 files to compare.
 
 The extdiff extension also allows you to configure new diff commands, so
 you do not need to type :hg:`extdiff -p kdiff3` always. ::
 
   [extdiff]
   # add new command that runs GNU diff(1) in 'context diff' mode
   cdiff = gdiff -Nprc5
   ## or the old way:
   #cmd.cdiff = gdiff
   #opts.cdiff = -Nprc5
 
   # add new command called meld, runs meld (no need to name twice).  If
   # the meld executable is not available, the meld tool in [merge-tools]
   # will be used, if available
   meld =
 
   # add new command called vimdiff, runs gvimdiff with DirDiff plugin
   # (see http://www.vim.org/scripts/script.php?script_id=102) Non
   # English user, be sure to put "let g:DirDiffDynamicDiffText = 1" in
   # your .vimrc
   vimdiff = gvim -f "+next" \\
             "+execute 'DirDiff' fnameescape(argv(0)) fnameescape(argv(1))"
 
 Tool arguments can include variables that are expanded at runtime::
 
   $parent1, $plabel1 - filename, descriptive label of first parent
   $child,   $clabel  - filename, descriptive label of child revision
   $parent2, $plabel2 - filename, descriptive label of second parent
   $root              - repository root
   $parent is an alias for $parent1.
 
 The extdiff extension will look in your [diff-tools] and [merge-tools]
 sections for diff tool arguments, when none are specified in [extdiff].
 
 ::
 
   [extdiff]
   kdiff3 =
 
   [diff-tools]
   kdiff3.diffargs=--L1 '$plabel1' --L2 '$clabel' $parent $child
 
 You can use -I/-X and list of file or directory names like normal
 :hg:`diff` command. The extdiff extension makes snapshots of only
 needed files, so running the external diff program will actually be
 pretty fast (at least faster than having to compare the entire tree).
 '''
 
 from __future__ import absolute_import
 
 import os
 import re
 import shutil
 import stat
 import tempfile
 from mercurial.i18n import _
 from mercurial.node import (
     nullid,
     short,
 )
 from mercurial import (
     archival,
     cmdutil,
     error,
     filemerge,
     pycompat,
     registrar,
     scmutil,
     util,
 )
+from mercurial.utils import (
+    stringutil,
+)
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 configitem('extdiff', br'opts\..*',
     default='',
     generic=True,
 )
 
 configitem('diff-tools', br'.*\.diffargs$',
     default=None,
     generic=True,
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
 def snapshot(ui, repo, files, node, tmproot, listsubrepos):
     '''snapshot files as of some revision
     if not using snapshot, -I/-X does not work and recursive diff
     in tools like kdiff3 and meld displays too many files.'''
     dirname = os.path.basename(repo.root)
     if dirname == "":
         dirname = "root"
     if node is not None:
         dirname = '%s.%s' % (dirname, short(node))
     base = os.path.join(tmproot, dirname)
     os.mkdir(base)
     fnsandstat = []
 
     if node is not None:
         ui.note(_('making snapshot of %d files from rev %s\n') %
                 (len(files), short(node)))
     else:
         ui.note(_('making snapshot of %d files from working directory\n') %
             (len(files)))
 
     if files:
         repo.ui.setconfig("ui", "archivemeta", False)
 
         archival.archive(repo, base, node, 'files',
                          matchfn=scmutil.matchfiles(repo, files),
                          subrepos=listsubrepos)
 
         for fn in sorted(files):
             wfn = util.pconvert(fn)
             ui.note('  %s\n' % wfn)
 
             if node is None:
                 dest = os.path.join(base, wfn)
 
                 fnsandstat.append((dest, repo.wjoin(fn), os.lstat(dest)))
     return dirname, fnsandstat
 
 def dodiff(ui, repo, cmdline, pats, opts):
     '''Do the actual diff:
 
     - copy to a temp structure if diffing 2 internal revisions
     - copy to a temp structure if diffing working revision with
       another one and more than 1 file is changed
     - just invoke the diff for a single file in the working dir
     '''
 
     revs = opts.get('rev')
     change = opts.get('change')
     do3way = '$parent2' in cmdline
 
     if revs and change:
         msg = _('cannot specify --rev and --change at the same time')
         raise error.Abort(msg)
     elif change:
         node2 = scmutil.revsingle(repo, change, None).node()
         node1a, node1b = repo.changelog.parents(node2)
     else:
         node1a, node2 = scmutil.revpair(repo, revs)
         if not revs:
             node1b = repo.dirstate.p2()
         else:
             node1b = nullid
 
     # Disable 3-way merge if there is only one parent
     if do3way:
         if node1b == nullid:
             do3way = False
 
     subrepos=opts.get('subrepos')
 
     matcher = scmutil.match(repo[node2], pats, opts)
 
     if opts.get('patch'):
         if subrepos:
             raise error.Abort(_('--patch cannot be used with --subrepos'))
         if node2 is None:
             raise error.Abort(_('--patch requires two revisions'))
     else:
         mod_a, add_a, rem_a = map(set, repo.status(node1a, node2, matcher,
                                                    listsubrepos=subrepos)[:3])
         if do3way:
             mod_b, add_b, rem_b = map(set,
                                       repo.status(node1b, node2, matcher,
                                                   listsubrepos=subrepos)[:3])
         else:
             mod_b, add_b, rem_b = set(), set(), set()
         modadd = mod_a | add_a | mod_b | add_b
         common = modadd | rem_a | rem_b
         if not common:
             return 0
 
     tmproot = tempfile.mkdtemp(prefix='extdiff.')
     try:
         if not opts.get('patch'):
             # Always make a copy of node1a (and node1b, if applicable)
             dir1a_files = mod_a | rem_a | ((mod_b | add_b) - add_a)
             dir1a = snapshot(ui, repo, dir1a_files, node1a, tmproot,
                              subrepos)[0]
             rev1a = '@%d' % repo[node1a].rev()
             if do3way:
                 dir1b_files = mod_b | rem_b | ((mod_a | add_a) - add_b)
                 dir1b = snapshot(ui, repo, dir1b_files, node1b, tmproot,
                                  subrepos)[0]
                 rev1b = '@%d' % repo[node1b].rev()
             else:
                 dir1b = None
                 rev1b = ''
 
             fnsandstat = []
 
             # If node2 in not the wc or there is >1 change, copy it
             dir2root = ''
             rev2 = ''
             if node2:
                 dir2 = snapshot(ui, repo, modadd, node2, tmproot, subrepos)[0]
                 rev2 = '@%d' % repo[node2].rev()
             elif len(common) > 1:
                 #we only actually need to get the files to copy back to
                 #the working dir in this case (because the other cases
                 #are: diffing 2 revisions or single file -- in which case
                 #the file is already directly passed to the diff tool).
                 dir2, fnsandstat = snapshot(ui, repo, modadd, None, tmproot,
                                             subrepos)
             else:
                 # This lets the diff tool open the changed file directly
                 dir2 = ''
                 dir2root = repo.root
 
             label1a = rev1a
             label1b = rev1b
             label2 = rev2
 
             # If only one change, diff the files instead of the directories
             # Handle bogus modifies correctly by checking if the files exist
             if len(common) == 1:
                 common_file = util.localpath(common.pop())
                 dir1a = os.path.join(tmproot, dir1a, common_file)
                 label1a = common_file + rev1a
                 if not os.path.isfile(dir1a):
                     dir1a = os.devnull
                 if do3way:
                     dir1b = os.path.join(tmproot, dir1b, common_file)
                     label1b = common_file + rev1b
                     if not os.path.isfile(dir1b):
                         dir1b = os.devnull
                 dir2 = os.path.join(dir2root, dir2, common_file)
                 label2 = common_file + rev2
         else:
             template = 'hg-%h.patch'
             cmdutil.export(repo, [repo[node1a].rev(), repo[node2].rev()],
                            fntemplate=repo.vfs.reljoin(tmproot, template),
                            match=matcher)
             label1a = cmdutil.makefilename(repo[node1a], template)
             label2 = cmdutil.makefilename(repo[node2], template)
             dir1a = repo.vfs.reljoin(tmproot, label1a)
             dir2 = repo.vfs.reljoin(tmproot, label2)
             dir1b = None
             label1b = None
             fnsandstat = []
 
         # Function to quote file/dir names in the argument string.
         # When not operating in 3-way mode, an empty string is
         # returned for parent2
         replace = {'parent': dir1a, 'parent1': dir1a, 'parent2': dir1b,
                    'plabel1': label1a, 'plabel2': label1b,
                    'clabel': label2, 'child': dir2,
                    'root': repo.root}
         def quote(match):
             pre = match.group(2)
             key = match.group(3)
             if not do3way and key == 'parent2':
                 return pre
             return pre + util.shellquote(replace[key])
 
         # Match parent2 first, so 'parent1?' will match both parent1 and parent
         regex = (br'''(['"]?)([^\s'"$]*)'''
                  br'\$(parent2|parent1?|child|plabel1|plabel2|clabel|root)\1')
         if not do3way and not re.search(regex, cmdline):
             cmdline += ' $parent1 $child'
         cmdline = re.sub(regex, quote, cmdline)
 
         ui.debug('running %r in %s\n' % (pycompat.bytestr(cmdline), tmproot))
         ui.system(cmdline, cwd=tmproot, blockedtag='extdiff')
 
         for copy_fn, working_fn, st in fnsandstat:
             cpstat = os.lstat(copy_fn)
             # Some tools copy the file and attributes, so mtime may not detect
             # all changes.  A size check will detect more cases, but not all.
             # The only certain way to detect every case is to diff all files,
             # which could be expensive.
             # copyfile() carries over the permission, so the mode check could
             # be in an 'elif' branch, but for the case where the file has
             # changed without affecting mtime or size.
             if (cpstat[stat.ST_MTIME] != st[stat.ST_MTIME]
                 or cpstat.st_size != st.st_size
                 or (cpstat.st_mode & 0o100) != (st.st_mode & 0o100)):
                 ui.debug('file changed while diffing. '
                          'Overwriting: %s (src: %s)\n' % (working_fn, copy_fn))
                 util.copyfile(copy_fn, working_fn)
 
         return 1
     finally:
         ui.note(_('cleaning up temp directory\n'))
         shutil.rmtree(tmproot)
 
 extdiffopts = [
     ('o', 'option', [],
      _('pass option to comparison program'), _('OPT')),
     ('r', 'rev', [], _('revision'), _('REV')),
     ('c', 'change', '', _('change made by revision'), _('REV')),
     ('', 'patch', None, _('compare patches for two revisions'))
     ] + cmdutil.walkopts + cmdutil.subrepoopts
 
 @command('extdiff',
     [('p', 'program', '', _('comparison program to run'), _('CMD')),
      ] + extdiffopts,
     _('hg extdiff [OPT]... [FILE]...'),
     inferrepo=True)
 def extdiff(ui, repo, *pats, **opts):
     '''use external program to diff repository (or selected files)
 
     Show differences between revisions for the specified files, using
     an external program. The default program used is diff, with
     default options "-Npru".
 
     To select a different program, use the -p/--program option. The
     program will be passed the names of two directories to compare. To
     pass additional options to the program, use -o/--option. These
     will be passed before the names of the directories to compare.
 
     When two revision arguments are given, then changes are shown
     between those revisions. If only one revision is specified then
     that revision is compared to the working directory, and, when no
     revisions are specified, the working directory files are compared
     to its parent.'''
     opts = pycompat.byteskwargs(opts)
     program = opts.get('program')
     option = opts.get('option')
     if not program:
         program = 'diff'
         option = option or ['-Npru']
     cmdline = ' '.join(map(util.shellquote, [program] + option))
     return dodiff(ui, repo, cmdline, pats, opts)
 
 class savedcmd(object):
     """use external program to diff repository (or selected files)
 
     Show differences between revisions for the specified files, using
     the following program::
 
         %(path)s
 
     When two revision arguments are given, then changes are shown
     between those revisions. If only one revision is specified then
     that revision is compared to the working directory, and, when no
     revisions are specified, the working directory files are compared
     to its parent.
     """
 
     def __init__(self, path, cmdline):
         # We can't pass non-ASCII through docstrings (and path is
         # in an unknown encoding anyway)
-        docpath = util.escapestr(path)
-        self.__doc__ %= {r'path': pycompat.sysstr(util.uirepr(docpath))}
+        docpath = stringutil.escapestr(path)
+        self.__doc__ %= {r'path': pycompat.sysstr(stringutil.uirepr(docpath))}
         self._cmdline = cmdline
 
     def __call__(self, ui, repo, *pats, **opts):
         opts = pycompat.byteskwargs(opts)
         options = ' '.join(map(util.shellquote, opts['option']))
         if options:
             options = ' ' + options
         return dodiff(ui, repo, self._cmdline + options, pats, opts)
 
 def uisetup(ui):
     for cmd, path in ui.configitems('extdiff'):
         path = util.expandpath(path)
         if cmd.startswith('cmd.'):
             cmd = cmd[4:]
             if not path:
                 path = util.findexe(cmd)
                 if path is None:
                     path = filemerge.findexternaltool(ui, cmd) or cmd
             diffopts = ui.config('extdiff', 'opts.' + cmd)
             cmdline = util.shellquote(path)
             if diffopts:
                 cmdline += ' ' + diffopts
         elif cmd.startswith('opts.'):
             continue
         else:
             if path:
                 # case "cmd = path opts"
                 cmdline = path
                 diffopts = len(pycompat.shlexsplit(cmdline)) > 1
             else:
                 # case "cmd ="
                 path = util.findexe(cmd)
                 if path is None:
                     path = filemerge.findexternaltool(ui, cmd) or cmd
                 cmdline = util.shellquote(path)
                 diffopts = False
         # look for diff arguments in [diff-tools] then [merge-tools]
         if not diffopts:
             args = ui.config('diff-tools', cmd+'.diffargs') or \
                    ui.config('merge-tools', cmd+'.diffargs')
             if args:
                 cmdline += ' ' + args
         command(cmd, extdiffopts[:], _('hg %s [OPTION]... [FILE]...') % cmd,
                 inferrepo=True)(savedcmd(path, cmdline))
 
 # tell hggettext to extract docstrings from these functions:
 i18nfunctions = [savedcmd]

hgext/highlight/highlight.py

@@ -1,93 +1,96
 # highlight.py - highlight extension implementation file
 #
 #  Copyright 2007-2009 Adam Hupp <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 #
 # The original module was split in an interface and an implementation
 # file to defer pygments loading and speedup extension setup.
 
 from __future__ import absolute_import
 
 from mercurial import demandimport
 demandimport.ignore.extend(['pkgutil', 'pkg_resources', '__main__'])
 
 from mercurial import (
     encoding,
-    util,
+)
+
+from mercurial.utils import (
+    stringutil,
 )
 
 with demandimport.deactivated():
     import pygments
     import pygments.formatters
     import pygments.lexers
     import pygments.plugin
     import pygments.util
 
     for unused in pygments.plugin.find_plugin_lexers():
         pass
 
 highlight = pygments.highlight
 ClassNotFound = pygments.util.ClassNotFound
 guess_lexer = pygments.lexers.guess_lexer
 guess_lexer_for_filename = pygments.lexers.guess_lexer_for_filename
 TextLexer = pygments.lexers.TextLexer
 HtmlFormatter = pygments.formatters.HtmlFormatter
 
 SYNTAX_CSS = ('\n<link rel="stylesheet" href="{url}highlightcss" '
               'type="text/css" />')
 
 def pygmentize(field, fctx, style, tmpl, guessfilenameonly=False):
 
     # append a <link ...> to the syntax highlighting css
     old_header = tmpl.load('header')
     if SYNTAX_CSS not in old_header:
         new_header = old_header + SYNTAX_CSS
         tmpl.cache['header'] = new_header
 
     text = fctx.data()
-    if util.binary(text):
+    if stringutil.binary(text):
         return
 
     # str.splitlines() != unicode.splitlines() because "reasons"
     for c in "\x0c\x1c\x1d\x1e":
         if c in text:
             text = text.replace(c, '')
 
     # Pygments is best used with Unicode strings:
     # <http://pygments.org/docs/unicode/>
     text = text.decode(encoding.encoding, 'replace')
 
     # To get multi-line strings right, we can't format line-by-line
     try:
         lexer = guess_lexer_for_filename(fctx.path(), text[:1024],
                                          stripnl=False)
     except (ClassNotFound, ValueError):
         # guess_lexer will return a lexer if *any* lexer matches. There is
         # no way to specify a minimum match score. This can give a high rate of
         # false positives on files with an unknown filename pattern.
         if guessfilenameonly:
             return
 
         try:
             lexer = guess_lexer(text[:1024], stripnl=False)
         except (ClassNotFound, ValueError):
             # Don't highlight unknown files
             return
 
     # Don't highlight text files
     if isinstance(lexer, TextLexer):
         return
 
     formatter = HtmlFormatter(nowrap=True, style=style)
 
     colorized = highlight(text, lexer, formatter)
     coloriter = (s.encode(encoding.encoding, 'replace')
                  for s in colorized.splitlines())
 
     tmpl.filters['colorize'] = lambda x: next(coloriter)
 
     oldl = tmpl.cache[field]
     newl = oldl.replace('line|escape', 'line|colorize')
     tmpl.cache[field] = newl

hgext/histedit.py

@@ -1,1651 +1,1654
 # histedit.py - interactive history editing for mercurial
 #
 # Copyright 2009 Augie Fackler <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 """interactive history editing
 
 With this extension installed, Mercurial gains one new command: histedit. Usage
 is as follows, assuming the following history::
 
  @  3[tip]   7c2fd3b9020c   2009-04-27 18:04 -0500   durin42
  |    Add delta
  |
  o  2   030b686bedc4   2009-04-27 18:04 -0500   durin42
  |    Add gamma
  |
  o  1   c561b4e977df   2009-04-27 18:04 -0500   durin42
  |    Add beta
  |
  o  0   d8d2fcd0e319   2009-04-27 18:04 -0500   durin42
       Add alpha
 
 If you were to run ``hg histedit c561b4e977df``, you would see the following
 file open in your editor::
 
  pick c561b4e977df Add beta
  pick 030b686bedc4 Add gamma
  pick 7c2fd3b9020c Add delta
 
  # Edit history between c561b4e977df and 7c2fd3b9020c
  #
  # Commits are listed from least to most recent
  #
  # Commands:
  #  p, pick = use commit
  #  e, edit = use commit, but stop for amending
  #  f, fold = use commit, but combine it with the one above
  #  r, roll = like fold, but discard this commit's description and date
  #  d, drop = remove commit from history
  #  m, mess = edit commit message without changing commit content
  #  b, base = checkout changeset and apply further changesets from there
  #
 
 In this file, lines beginning with ``#`` are ignored. You must specify a rule
 for each revision in your history. For example, if you had meant to add gamma
 before beta, and then wanted to add delta in the same revision as beta, you
 would reorganize the file to look like this::
 
  pick 030b686bedc4 Add gamma
  pick c561b4e977df Add beta
  fold 7c2fd3b9020c Add delta
 
  # Edit history between c561b4e977df and 7c2fd3b9020c
  #
  # Commits are listed from least to most recent
  #
  # Commands:
  #  p, pick = use commit
  #  e, edit = use commit, but stop for amending
  #  f, fold = use commit, but combine it with the one above
  #  r, roll = like fold, but discard this commit's description and date
  #  d, drop = remove commit from history
  #  m, mess = edit commit message without changing commit content
  #  b, base = checkout changeset and apply further changesets from there
  #
 
 At which point you close the editor and ``histedit`` starts working. When you
 specify a ``fold`` operation, ``histedit`` will open an editor when it folds
 those revisions together, offering you a chance to clean up the commit message::
 
  Add beta
  ***
  Add delta
 
 Edit the commit message to your liking, then close the editor. The date used
 for the commit will be the later of the two commits' dates. For this example,
 let's assume that the commit message was changed to ``Add beta and delta.``
 After histedit has run and had a chance to remove any old or temporary
 revisions it needed, the history looks like this::
 
  @  2[tip]   989b4d060121   2009-04-27 18:04 -0500   durin42
  |    Add beta and delta.
  |
  o  1   081603921c3f   2009-04-27 18:04 -0500   durin42
  |    Add gamma
  |
  o  0   d8d2fcd0e319   2009-04-27 18:04 -0500   durin42
       Add alpha
 
 Note that ``histedit`` does *not* remove any revisions (even its own temporary
 ones) until after it has completed all the editing operations, so it will
 probably perform several strip operations when it's done. For the above example,
 it had to run strip twice. Strip can be slow depending on a variety of factors,
 so you might need to be a little patient. You can choose to keep the original
 revisions by passing the ``--keep`` flag.
 
 The ``edit`` operation will drop you back to a command prompt,
 allowing you to edit files freely, or even use ``hg record`` to commit
 some changes as a separate commit. When you're done, any remaining
 uncommitted changes will be committed as well. When done, run ``hg
 histedit --continue`` to finish this step. If there are uncommitted
 changes, you'll be prompted for a new commit message, but the default
 commit message will be the original message for the ``edit`` ed
 revision, and the date of the original commit will be preserved.
 
 The ``message`` operation will give you a chance to revise a commit
 message without changing the contents. It's a shortcut for doing
 ``edit`` immediately followed by `hg histedit --continue``.
 
 If ``histedit`` encounters a conflict when moving a revision (while
 handling ``pick`` or ``fold``), it'll stop in a similar manner to
 ``edit`` with the difference that it won't prompt you for a commit
 message when done. If you decide at this point that you don't like how
 much work it will be to rearrange history, or that you made a mistake,
 you can use ``hg histedit --abort`` to abandon the new changes you
 have made and return to the state before you attempted to edit your
 history.
 
 If we clone the histedit-ed example repository above and add four more
 changes, such that we have the following history::
 
    @  6[tip]   038383181893   2009-04-27 18:04 -0500   stefan
    |    Add theta
    |
    o  5   140988835471   2009-04-27 18:04 -0500   stefan
    |    Add eta
    |
    o  4   122930637314   2009-04-27 18:04 -0500   stefan
    |    Add zeta
    |
    o  3   836302820282   2009-04-27 18:04 -0500   stefan
    |    Add epsilon
    |
    o  2   989b4d060121   2009-04-27 18:04 -0500   durin42
    |    Add beta and delta.
    |
    o  1   081603921c3f   2009-04-27 18:04 -0500   durin42
    |    Add gamma
    |
    o  0   d8d2fcd0e319   2009-04-27 18:04 -0500   durin42
         Add alpha
 
 If you run ``hg histedit --outgoing`` on the clone then it is the same
 as running ``hg histedit 836302820282``. If you need plan to push to a
 repository that Mercurial does not detect to be related to the source
 repo, you can add a ``--force`` option.
 
 Config
 ------
 
 Histedit rule lines are truncated to 80 characters by default. You
 can customize this behavior by setting a different length in your
 configuration file::
 
   [histedit]
   linelen = 120      # truncate rule lines at 120 characters
 
 ``hg histedit`` attempts to automatically choose an appropriate base
 revision to use. To change which base revision is used, define a
 revset in your configuration file::
 
   [histedit]
   defaultrev = only(.) & draft()
 
 By default each edited revision needs to be present in histedit commands.
 To remove revision you need to use ``drop`` operation. You can configure
 the drop to be implicit for missing commits by adding::
 
   [histedit]
   dropmissing = True
 
 By default, histedit will close the transaction after each action. For
 performance purposes, you can configure histedit to use a single transaction
 across the entire histedit. WARNING: This setting introduces a significant risk
 of losing the work you've done in a histedit if the histedit aborts
 unexpectedly::
 
   [histedit]
   singletransaction = True
 
 """
 
 from __future__ import absolute_import
 
 import errno
 import os
 
 from mercurial.i18n import _
 from mercurial import (
     bundle2,
     cmdutil,
     context,
     copies,
     destutil,
     discovery,
     error,
     exchange,
     extensions,
     hg,
     lock,
     merge as mergemod,
     mergeutil,
     node,
     obsolete,
     pycompat,
     registrar,
     repair,
     scmutil,
     util,
 )
+from mercurial.utils import (
+    stringutil,
+)
 
 pickle = util.pickle
 release = lock.release
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 configitem('experimental', 'histedit.autoverb',
     default=False,
 )
 configitem('histedit', 'defaultrev',
     default=None,
 )
 configitem('histedit', 'dropmissing',
     default=False,
 )
 configitem('histedit', 'linelen',
     default=80,
 )
 configitem('histedit', 'singletransaction',
     default=False,
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
 actiontable = {}
 primaryactions = set()
 secondaryactions = set()
 tertiaryactions = set()
 internalactions = set()
 
 def geteditcomment(ui, first, last):
     """ construct the editor comment
     The comment includes::
      - an intro
      - sorted primary commands
      - sorted short commands
      - sorted long commands
      - additional hints
 
     Commands are only included once.
     """
     intro = _("""Edit history between %s and %s
 
 Commits are listed from least to most recent
 
 You can reorder changesets by reordering the lines
 
 Commands:
 """)
     actions = []
     def addverb(v):
         a = actiontable[v]
         lines = a.message.split("\n")
         if len(a.verbs):
             v = ', '.join(sorted(a.verbs, key=lambda v: len(v)))
         actions.append(" %s = %s" % (v, lines[0]))
         actions.extend(['  %s' for l in lines[1:]])
 
     for v in (
          sorted(primaryactions) +
          sorted(secondaryactions) +
          sorted(tertiaryactions)
         ):
         addverb(v)
     actions.append('')
 
     hints = []
     if ui.configbool('histedit', 'dropmissing'):
         hints.append("Deleting a changeset from the list "
                      "will DISCARD it from the edited history!")
 
     lines = (intro % (first, last)).split('\n') + actions + hints
 
     return ''.join(['# %s\n' % l if l else '#\n' for l in lines])
 
 class histeditstate(object):
     def __init__(self, repo, parentctxnode=None, actions=None, keep=None,
             topmost=None, replacements=None, lock=None, wlock=None):
         self.repo = repo
         self.actions = actions
         self.keep = keep
         self.topmost = topmost
         self.parentctxnode = parentctxnode
         self.lock = lock
         self.wlock = wlock
         self.backupfile = None
         if replacements is None:
             self.replacements = []
         else:
             self.replacements = replacements
 
     def read(self):
         """Load histedit state from disk and set fields appropriately."""
         try:
             state = self.repo.vfs.read('histedit-state')
         except IOError as err:
             if err.errno != errno.ENOENT:
                 raise
             cmdutil.wrongtooltocontinue(self.repo, _('histedit'))
 
         if state.startswith('v1\n'):
             data = self._load()
             parentctxnode, rules, keep, topmost, replacements, backupfile = data
         else:
             data = pickle.loads(state)
             parentctxnode, rules, keep, topmost, replacements = data
             backupfile = None
 
         self.parentctxnode = parentctxnode
         rules = "\n".join(["%s %s" % (verb, rest) for [verb, rest] in rules])
         actions = parserules(rules, self)
         self.actions = actions
         self.keep = keep
         self.topmost = topmost
         self.replacements = replacements
         self.backupfile = backupfile
 
     def write(self, tr=None):
         if tr:
             tr.addfilegenerator('histedit-state', ('histedit-state',),
                                 self._write, location='plain')
         else:
             with self.repo.vfs("histedit-state", "w") as f:
                 self._write(f)
 
     def _write(self, fp):
         fp.write('v1\n')
         fp.write('%s\n' % node.hex(self.parentctxnode))
         fp.write('%s\n' % node.hex(self.topmost))
         fp.write('%s\n' % ('True' if self.keep else 'False'))
         fp.write('%d\n' % len(self.actions))
         for action in self.actions:
             fp.write('%s\n' % action.tostate())
         fp.write('%d\n' % len(self.replacements))
         for replacement in self.replacements:
             fp.write('%s%s\n' % (node.hex(replacement[0]), ''.join(node.hex(r)
                 for r in replacement[1])))
         backupfile = self.backupfile
         if not backupfile:
             backupfile = ''
         fp.write('%s\n' % backupfile)
 
     def _load(self):
         fp = self.repo.vfs('histedit-state', 'r')
         lines = [l[:-1] for l in fp.readlines()]
 
         index = 0
         lines[index] # version number
         index += 1
 
         parentctxnode = node.bin(lines[index])
         index += 1
 
         topmost = node.bin(lines[index])
         index += 1
 
         keep = lines[index] == 'True'
         index += 1
 
         # Rules
         rules = []
         rulelen = int(lines[index])
         index += 1
         for i in xrange(rulelen):
             ruleaction = lines[index]
             index += 1
             rule = lines[index]
             index += 1
             rules.append((ruleaction, rule))
 
         # Replacements
         replacements = []
         replacementlen = int(lines[index])
         index += 1
         for i in xrange(replacementlen):
             replacement = lines[index]
             original = node.bin(replacement[:40])
             succ = [node.bin(replacement[i:i + 40]) for i in
                     range(40, len(replacement), 40)]
             replacements.append((original, succ))
             index += 1
 
         backupfile = lines[index]
         index += 1
 
         fp.close()
 
         return parentctxnode, rules, keep, topmost, replacements, backupfile
 
     def clear(self):
         if self.inprogress():
             self.repo.vfs.unlink('histedit-state')
 
     def inprogress(self):
         return self.repo.vfs.exists('histedit-state')
 
 
 class histeditaction(object):
     def __init__(self, state, node):
         self.state = state
         self.repo = state.repo
         self.node = node
 
     @classmethod
     def fromrule(cls, state, rule):
         """Parses the given rule, returning an instance of the histeditaction.
         """
         rulehash = rule.strip().split(' ', 1)[0]
         try:
             rev = node.bin(rulehash)
         except TypeError:
             raise error.ParseError("invalid changeset %s" % rulehash)
         return cls(state, rev)
 
     def verify(self, prev, expected, seen):
         """ Verifies semantic correctness of the rule"""
         repo = self.repo
         ha = node.hex(self.node)
         try:
             self.node = repo[ha].node()
         except error.RepoError:
             raise error.ParseError(_('unknown changeset %s listed')
                               % ha[:12])
         if self.node is not None:
             self._verifynodeconstraints(prev, expected, seen)
 
     def _verifynodeconstraints(self, prev, expected, seen):
         # by default command need a node in the edited list
         if self.node not in expected:
             raise error.ParseError(_('%s "%s" changeset was not a candidate')
                                    % (self.verb, node.short(self.node)),
                                    hint=_('only use listed changesets'))
         # and only one command per node
         if self.node in seen:
             raise error.ParseError(_('duplicated command for changeset %s') %
                                    node.short(self.node))
 
     def torule(self):
         """build a histedit rule line for an action
 
         by default lines are in the form:
         <hash> <rev> <summary>
         """
         ctx = self.repo[self.node]
         summary = _getsummary(ctx)
         line = '%s %s %d %s' % (self.verb, ctx, ctx.rev(), summary)
         # trim to 75 columns by default so it's not stupidly wide in my editor
         # (the 5 more are left for verb)
         maxlen = self.repo.ui.configint('histedit', 'linelen')
         maxlen = max(maxlen, 22) # avoid truncating hash
-        return util.ellipsis(line, maxlen)
+        return stringutil.ellipsis(line, maxlen)
 
     def tostate(self):
         """Print an action in format used by histedit state files
            (the first line is a verb, the remainder is the second)
         """
         return "%s\n%s" % (self.verb, node.hex(self.node))
 
     def run(self):
         """Runs the action. The default behavior is simply apply the action's
         rulectx onto the current parentctx."""
         self.applychange()
         self.continuedirty()
         return self.continueclean()
 
     def applychange(self):
         """Applies the changes from this action's rulectx onto the current
         parentctx, but does not commit them."""
         repo = self.repo
         rulectx = repo[self.node]
         repo.ui.pushbuffer(error=True, labeled=True)
         hg.update(repo, self.state.parentctxnode, quietempty=True)
         stats = applychanges(repo.ui, repo, rulectx, {})
         repo.dirstate.setbranch(rulectx.branch())
         if stats and stats[3] > 0:
             buf = repo.ui.popbuffer()
             repo.ui.write(buf)
             raise error.InterventionRequired(
                 _('Fix up the change (%s %s)') %
                 (self.verb, node.short(self.node)),
                 hint=_('hg histedit --continue to resume'))
         else:
             repo.ui.popbuffer()
 
     def continuedirty(self):
         """Continues the action when changes have been applied to the working
         copy. The default behavior is to commit the dirty changes."""
         repo = self.repo
         rulectx = repo[self.node]
 
         editor = self.commiteditor()
         commit = commitfuncfor(repo, rulectx)
 
         commit(text=rulectx.description(), user=rulectx.user(),
                date=rulectx.date(), extra=rulectx.extra(), editor=editor)
 
     def commiteditor(self):
         """The editor to be used to edit the commit message."""
         return False
 
     def continueclean(self):
         """Continues the action when the working copy is clean. The default
         behavior is to accept the current commit as the new version of the
         rulectx."""
         ctx = self.repo['.']
         if ctx.node() == self.state.parentctxnode:
             self.repo.ui.warn(_('%s: skipping changeset (no changes)\n') %
                               node.short(self.node))
             return ctx, [(self.node, tuple())]
         if ctx.node() == self.node:
             # Nothing changed
             return ctx, []
         return ctx, [(self.node, (ctx.node(),))]
 
 def commitfuncfor(repo, src):
     """Build a commit function for the replacement of <src>
 
     This function ensure we apply the same treatment to all changesets.
 
     - Add a 'histedit_source' entry in extra.
 
     Note that fold has its own separated logic because its handling is a bit
     different and not easily factored out of the fold method.
     """
     phasemin = src.phase()
     def commitfunc(**kwargs):
         overrides = {('phases', 'new-commit'): phasemin}
         with repo.ui.configoverride(overrides, 'histedit'):
             extra = kwargs.get(r'extra', {}).copy()
             extra['histedit_source'] = src.hex()
             kwargs[r'extra'] = extra
             return repo.commit(**kwargs)
     return commitfunc
 
 def applychanges(ui, repo, ctx, opts):
     """Merge changeset from ctx (only) in the current working directory"""
     wcpar = repo.dirstate.parents()[0]
     if ctx.p1().node() == wcpar:
         # edits are "in place" we do not need to make any merge,
         # just applies changes on parent for editing
         cmdutil.revert(ui, repo, ctx, (wcpar, node.nullid), all=True)
         stats = None
     else:
         try:
             # ui.forcemerge is an internal variable, do not document
             repo.ui.setconfig('ui', 'forcemerge', opts.get('tool', ''),
                               'histedit')
             stats = mergemod.graft(repo, ctx, ctx.p1(), ['local', 'histedit'])
         finally:
             repo.ui.setconfig('ui', 'forcemerge', '', 'histedit')
     return stats
 
 def collapse(repo, firstctx, lastctx, commitopts, skipprompt=False):
     """collapse the set of revisions from first to last as new one.
 
     Expected commit options are:
         - message
         - date
         - username
     Commit message is edited in all cases.
 
     This function works in memory."""
     ctxs = list(repo.set('%d::%d', firstctx.rev(), lastctx.rev()))
     if not ctxs:
         return None
     for c in ctxs:
         if not c.mutable():
             raise error.ParseError(
                 _("cannot fold into public change %s") % node.short(c.node()))
     base = firstctx.parents()[0]
 
     # commit a new version of the old changeset, including the update
     # collect all files which might be affected
     files = set()
     for ctx in ctxs:
         files.update(ctx.files())
 
     # Recompute copies (avoid recording a -> b -> a)
     copied = copies.pathcopies(base, lastctx)
 
     # prune files which were reverted by the updates
     files = [f for f in files if not cmdutil.samefile(f, lastctx, base)]
     # commit version of these files as defined by head
     headmf = lastctx.manifest()
     def filectxfn(repo, ctx, path):
         if path in headmf:
             fctx = lastctx[path]
             flags = fctx.flags()
             mctx = context.memfilectx(repo, ctx,
                                       fctx.path(), fctx.data(),
                                       islink='l' in flags,
                                       isexec='x' in flags,
                                       copied=copied.get(path))
             return mctx
         return None
 
     if commitopts.get('message'):
         message = commitopts['message']
     else:
         message = firstctx.description()
     user = commitopts.get('user')
     date = commitopts.get('date')
     extra = commitopts.get('extra')
 
     parents = (firstctx.p1().node(), firstctx.p2().node())
     editor = None
     if not skipprompt:
         editor = cmdutil.getcommiteditor(edit=True, editform='histedit.fold')
     new = context.memctx(repo,
                          parents=parents,
                          text=message,
                          files=files,
                          filectxfn=filectxfn,
                          user=user,
                          date=date,
                          extra=extra,
                          editor=editor)
     return repo.commitctx(new)
 
 def _isdirtywc(repo):
     return repo[None].dirty(missing=True)
 
 def abortdirty():
     raise error.Abort(_('working copy has pending changes'),
         hint=_('amend, commit, or revert them and run histedit '
             '--continue, or abort with histedit --abort'))
 
 def action(verbs, message, priority=False, internal=False):
     def wrap(cls):
         assert not priority or not internal
         verb = verbs[0]
         if priority:
             primaryactions.add(verb)
         elif internal:
             internalactions.add(verb)
         elif len(verbs) > 1:
             secondaryactions.add(verb)
         else:
             tertiaryactions.add(verb)
 
         cls.verb = verb
         cls.verbs = verbs
         cls.message = message
         for verb in verbs:
             actiontable[verb] = cls
         return cls
     return wrap
 
 @action(['pick', 'p'],
         _('use commit'),
         priority=True)
 class pick(histeditaction):
     def run(self):
         rulectx = self.repo[self.node]
         if rulectx.parents()[0].node() == self.state.parentctxnode:
             self.repo.ui.debug('node %s unchanged\n' % node.short(self.node))
             return rulectx, []
 
         return super(pick, self).run()
 
 @action(['edit', 'e'],
         _('use commit, but stop for amending'),
         priority=True)
 class edit(histeditaction):
     def run(self):
         repo = self.repo
         rulectx = repo[self.node]
         hg.update(repo, self.state.parentctxnode, quietempty=True)
         applychanges(repo.ui, repo, rulectx, {})
         raise error.InterventionRequired(
             _('Editing (%s), you may commit or record as needed now.')
             % node.short(self.node),
             hint=_('hg histedit --continue to resume'))
 
     def commiteditor(self):
         return cmdutil.getcommiteditor(edit=True, editform='histedit.edit')
 
 @action(['fold', 'f'],
         _('use commit, but combine it with the one above'))
 class fold(histeditaction):
     def verify(self, prev, expected, seen):
         """ Verifies semantic correctness of the fold rule"""
         super(fold, self).verify(prev, expected, seen)
         repo = self.repo
         if not prev:
             c = repo[self.node].parents()[0]
         elif not prev.verb in ('pick', 'base'):
             return
         else:
             c = repo[prev.node]
         if not c.mutable():
             raise error.ParseError(
                 _("cannot fold into public change %s") % node.short(c.node()))
 
 
     def continuedirty(self):
         repo = self.repo
         rulectx = repo[self.node]
 
         commit = commitfuncfor(repo, rulectx)
         commit(text='fold-temp-revision %s' % node.short(self.node),
                user=rulectx.user(), date=rulectx.date(),
                extra=rulectx.extra())
 
     def continueclean(self):
         repo = self.repo
         ctx = repo['.']
         rulectx = repo[self.node]
         parentctxnode = self.state.parentctxnode
         if ctx.node() == parentctxnode:
             repo.ui.warn(_('%s: empty changeset\n') %
                               node.short(self.node))
             return ctx, [(self.node, (parentctxnode,))]
 
         parentctx = repo[parentctxnode]
         newcommits = set(c.node() for c in repo.set('(%d::. - %d)',
                                                     parentctx.rev(),
                                                     parentctx.rev()))
         if not newcommits:
             repo.ui.warn(_('%s: cannot fold - working copy is not a '
                            'descendant of previous commit %s\n') %
                            (node.short(self.node), node.short(parentctxnode)))
             return ctx, [(self.node, (ctx.node(),))]
 
         middlecommits = newcommits.copy()
         middlecommits.discard(ctx.node())
 
         return self.finishfold(repo.ui, repo, parentctx, rulectx, ctx.node(),
                                middlecommits)
 
     def skipprompt(self):
         """Returns true if the rule should skip the message editor.
 
         For example, 'fold' wants to show an editor, but 'rollup'
         doesn't want to.
         """
         return False
 
     def mergedescs(self):
         """Returns true if the rule should merge messages of multiple changes.
 
         This exists mainly so that 'rollup' rules can be a subclass of
         'fold'.
         """
         return True
 
     def firstdate(self):
         """Returns true if the rule should preserve the date of the first
         change.
 
         This exists mainly so that 'rollup' rules can be a subclass of
         'fold'.
         """
         return False
 
     def finishfold(self, ui, repo, ctx, oldctx, newnode, internalchanges):
         parent = ctx.parents()[0].node()
         repo.ui.pushbuffer()
         hg.update(repo, parent)
         repo.ui.popbuffer()
         ### prepare new commit data
         commitopts = {}
         commitopts['user'] = ctx.user()
         # commit message
         if not self.mergedescs():
             newmessage = ctx.description()
         else:
             newmessage = '\n***\n'.join(
                 [ctx.description()] +
                 [repo[r].description() for r in internalchanges] +
                 [oldctx.description()]) + '\n'
         commitopts['message'] = newmessage
         # date
         if self.firstdate():
             commitopts['date'] = ctx.date()
         else:
             commitopts['date'] = max(ctx.date(), oldctx.date())
         extra = ctx.extra().copy()
         # histedit_source
         # note: ctx is likely a temporary commit but that the best we can do
         #       here. This is sufficient to solve issue3681 anyway.
         extra['histedit_source'] = '%s,%s' % (ctx.hex(), oldctx.hex())
         commitopts['extra'] = extra
         phasemin = max(ctx.phase(), oldctx.phase())
         overrides = {('phases', 'new-commit'): phasemin}
         with repo.ui.configoverride(overrides, 'histedit'):
             n = collapse(repo, ctx, repo[newnode], commitopts,
                          skipprompt=self.skipprompt())
         if n is None:
             return ctx, []
         repo.ui.pushbuffer()
         hg.update(repo, n)
         repo.ui.popbuffer()
         replacements = [(oldctx.node(), (newnode,)),
                         (ctx.node(), (n,)),
                         (newnode, (n,)),
                        ]
         for ich in internalchanges:
             replacements.append((ich, (n,)))
         return repo[n], replacements
 
 @action(['base', 'b'],
         _('checkout changeset and apply further changesets from there'))
 class base(histeditaction):
 
     def run(self):
         if self.repo['.'].node() != self.node:
             mergemod.update(self.repo, self.node, False, True)
             #                                     branchmerge, force)
         return self.continueclean()
 
     def continuedirty(self):
         abortdirty()
 
     def continueclean(self):
         basectx = self.repo['.']
         return basectx, []
 
     def _verifynodeconstraints(self, prev, expected, seen):
         # base can only be use with a node not in the edited set
         if self.node in expected:
             msg = _('%s "%s" changeset was an edited list candidate')
             raise error.ParseError(
                 msg % (self.verb, node.short(self.node)),
                 hint=_('base must only use unlisted changesets'))
 
 @action(['_multifold'],
         _(
     """fold subclass used for when multiple folds happen in a row
 
     We only want to fire the editor for the folded message once when
     (say) four changes are folded down into a single change. This is
     similar to rollup, but we should preserve both messages so that
     when the last fold operation runs we can show the user all the
     commit messages in their editor.
     """),
         internal=True)
 class _multifold(fold):
     def skipprompt(self):
         return True
 
 @action(["roll", "r"],
         _("like fold, but discard this commit's description and date"))
 class rollup(fold):
     def mergedescs(self):
         return False
 
     def skipprompt(self):
         return True
 
     def firstdate(self):
         return True
 
 @action(["drop", "d"],
         _('remove commit from history'))
 class drop(histeditaction):
     def run(self):
         parentctx = self.repo[self.state.parentctxnode]
         return parentctx, [(self.node, tuple())]
 
 @action(["mess", "m"],
         _('edit commit message without changing commit content'),
         priority=True)
 class message(histeditaction):
     def commiteditor(self):
         return cmdutil.getcommiteditor(edit=True, editform='histedit.mess')
 
 def findoutgoing(ui, repo, remote=None, force=False, opts=None):
     """utility function to find the first outgoing changeset
 
     Used by initialization code"""
     if opts is None:
         opts = {}
     dest = ui.expandpath(remote or 'default-push', remote or 'default')
     dest, revs = hg.parseurl(dest, None)[:2]
     ui.status(_('comparing with %s\n') % util.hidepassword(dest))
 
     revs, checkout = hg.addbranchrevs(repo, repo, revs, None)
     other = hg.peer(repo, opts, dest)
 
     if revs:
         revs = [repo.lookup(rev) for rev in revs]
 
     outgoing = discovery.findcommonoutgoing(repo, other, revs, force=force)
     if not outgoing.missing:
         raise error.Abort(_('no outgoing ancestors'))
     roots = list(repo.revs("roots(%ln)", outgoing.missing))
     if 1 < len(roots):
         msg = _('there are ambiguous outgoing revisions')
         hint = _("see 'hg help histedit' for more detail")
         raise error.Abort(msg, hint=hint)
     return repo.lookup(roots[0])
 
 @command('histedit',
     [('', 'commands', '',
       _('read history edits from the specified file'), _('FILE')),
      ('c', 'continue', False, _('continue an edit already in progress')),
      ('', 'edit-plan', False, _('edit remaining actions list')),
      ('k', 'keep', False,
       _("don't strip old nodes after edit is complete")),
      ('', 'abort', False, _('abort an edit in progress')),
      ('o', 'outgoing', False, _('changesets not found in destination')),
      ('f', 'force', False,
       _('force outgoing even for unrelated repositories')),
      ('r', 'rev', [], _('first revision to be edited'), _('REV'))] +
     cmdutil.formatteropts,
      _("[OPTIONS] ([ANCESTOR] | --outgoing [URL])"))
 def histedit(ui, repo, *freeargs, **opts):
     """interactively edit changeset history
 
     This command lets you edit a linear series of changesets (up to
     and including the working directory, which should be clean).
     You can:
 
     - `pick` to [re]order a changeset
 
     - `drop` to omit changeset
 
     - `mess` to reword the changeset commit message
 
     - `fold` to combine it with the preceding changeset (using the later date)
 
     - `roll` like fold, but discarding this commit's description and date
 
     - `edit` to edit this changeset (preserving date)
 
     - `base` to checkout changeset and apply further changesets from there
 
     There are a number of ways to select the root changeset:
 
     - Specify ANCESTOR directly
 
     - Use --outgoing -- it will be the first linear changeset not
       included in destination. (See :hg:`help config.paths.default-push`)
 
     - Otherwise, the value from the "histedit.defaultrev" config option
       is used as a revset to select the base revision when ANCESTOR is not
       specified. The first revision returned by the revset is used. By
       default, this selects the editable history that is unique to the
       ancestry of the working directory.
 
     .. container:: verbose
 
        If you use --outgoing, this command will abort if there are ambiguous
        outgoing revisions. For example, if there are multiple branches
        containing outgoing revisions.
 
        Use "min(outgoing() and ::.)" or similar revset specification
        instead of --outgoing to specify edit target revision exactly in
        such ambiguous situation. See :hg:`help revsets` for detail about
        selecting revisions.
 
     .. container:: verbose
 
        Examples:
 
          - A number of changes have been made.
            Revision 3 is no longer needed.
 
            Start history editing from revision 3::
 
              hg histedit -r 3
 
            An editor opens, containing the list of revisions,
            with specific actions specified::
 
              pick 5339bf82f0ca 3 Zworgle the foobar
              pick 8ef592ce7cc4 4 Bedazzle the zerlog
              pick 0a9639fcda9d 5 Morgify the cromulancy
 
            Additional information about the possible actions
            to take appears below the list of revisions.
 
            To remove revision 3 from the history,
            its action (at the beginning of the relevant line)
            is changed to 'drop'::
 
              drop 5339bf82f0ca 3 Zworgle the foobar
              pick 8ef592ce7cc4 4 Bedazzle the zerlog
              pick 0a9639fcda9d 5 Morgify the cromulancy
 
          - A number of changes have been made.
            Revision 2 and 4 need to be swapped.
 
            Start history editing from revision 2::
 
              hg histedit -r 2
 
            An editor opens, containing the list of revisions,
            with specific actions specified::
 
              pick 252a1af424ad 2 Blorb a morgwazzle
              pick 5339bf82f0ca 3 Zworgle the foobar
              pick 8ef592ce7cc4 4 Bedazzle the zerlog
 
            To swap revision 2 and 4, its lines are swapped
            in the editor::
 
              pick 8ef592ce7cc4 4 Bedazzle the zerlog
              pick 5339bf82f0ca 3 Zworgle the foobar
              pick 252a1af424ad 2 Blorb a morgwazzle
 
     Returns 0 on success, 1 if user intervention is required (not only
     for intentional "edit" command, but also for resolving unexpected
     conflicts).
     """
     state = histeditstate(repo)
     try:
         state.wlock = repo.wlock()
         state.lock = repo.lock()
         _histedit(ui, repo, state, *freeargs, **opts)
     finally:
         release(state.lock, state.wlock)
 
 goalcontinue = 'continue'
 goalabort = 'abort'
 goaleditplan = 'edit-plan'
 goalnew = 'new'
 
 def _getgoal(opts):
     if opts.get('continue'):
         return goalcontinue
     if opts.get('abort'):
         return goalabort
     if opts.get('edit_plan'):
         return goaleditplan
     return goalnew
 
 def _readfile(ui, path):
     if path == '-':
         with ui.timeblockedsection('histedit'):
             return ui.fin.read()
     else:
         with open(path, 'rb') as f:
             return f.read()
 
 def _validateargs(ui, repo, state, freeargs, opts, goal, rules, revs):
     # TODO only abort if we try to histedit mq patches, not just
     # blanket if mq patches are applied somewhere
     mq = getattr(repo, 'mq', None)
     if mq and mq.applied:
         raise error.Abort(_('source has mq patches applied'))
 
     # basic argument incompatibility processing
     outg = opts.get('outgoing')
     editplan = opts.get('edit_plan')
     abort = opts.get('abort')
     force = opts.get('force')
     if force and not outg:
         raise error.Abort(_('--force only allowed with --outgoing'))
     if goal == 'continue':
         if any((outg, abort, revs, freeargs, rules, editplan)):
             raise error.Abort(_('no arguments allowed with --continue'))
     elif goal == 'abort':
         if any((outg, revs, freeargs, rules, editplan)):
             raise error.Abort(_('no arguments allowed with --abort'))
     elif goal == 'edit-plan':
         if any((outg, revs, freeargs)):
             raise error.Abort(_('only --commands argument allowed with '
                                '--edit-plan'))
     else:
         if os.path.exists(os.path.join(repo.path, 'histedit-state')):
             raise error.Abort(_('history edit already in progress, try '
                                '--continue or --abort'))
         if outg:
             if revs:
                 raise error.Abort(_('no revisions allowed with --outgoing'))
             if len(freeargs) > 1:
                 raise error.Abort(
                     _('only one repo argument allowed with --outgoing'))
         else:
             revs.extend(freeargs)
             if len(revs) == 0:
                 defaultrev = destutil.desthistedit(ui, repo)
                 if defaultrev is not None:
                     revs.append(defaultrev)
 
             if len(revs) != 1:
                 raise error.Abort(
                     _('histedit requires exactly one ancestor revision'))
 
 def _histedit(ui, repo, state, *freeargs, **opts):
     opts = pycompat.byteskwargs(opts)
     fm = ui.formatter('histedit', opts)
     fm.startitem()
     goal = _getgoal(opts)
     revs = opts.get('rev', [])
     rules = opts.get('commands', '')
     state.keep = opts.get('keep', False)
 
     _validateargs(ui, repo, state, freeargs, opts, goal, rules, revs)
 
     # rebuild state
     if goal == goalcontinue:
         state.read()
         state = bootstrapcontinue(ui, state, opts)
     elif goal == goaleditplan:
         _edithisteditplan(ui, repo, state, rules)
         return
     elif goal == goalabort:
         _aborthistedit(ui, repo, state)
         return
     else:
         # goal == goalnew
         _newhistedit(ui, repo, state, revs, freeargs, opts)
 
     _continuehistedit(ui, repo, state)
     _finishhistedit(ui, repo, state, fm)
     fm.end()
 
 def _continuehistedit(ui, repo, state):
     """This function runs after either:
     - bootstrapcontinue (if the goal is 'continue')
     - _newhistedit (if the goal is 'new')
     """
     # preprocess rules so that we can hide inner folds from the user
     # and only show one editor
     actions = state.actions[:]
     for idx, (action, nextact) in enumerate(
             zip(actions, actions[1:] + [None])):
         if action.verb == 'fold' and nextact and nextact.verb == 'fold':
             state.actions[idx].__class__ = _multifold
 
     # Force an initial state file write, so the user can run --abort/continue
     # even if there's an exception before the first transaction serialize.
     state.write()
 
     total = len(state.actions)
     pos = 0
     tr = None
     # Don't use singletransaction by default since it rolls the entire
     # transaction back if an unexpected exception happens (like a
     # pretxncommit hook throws, or the user aborts the commit msg editor).
     if ui.configbool("histedit", "singletransaction"):
         # Don't use a 'with' for the transaction, since actions may close
         # and reopen a transaction. For example, if the action executes an
         # external process it may choose to commit the transaction first.
         tr = repo.transaction('histedit')
     with util.acceptintervention(tr):
         while state.actions:
             state.write(tr=tr)
             actobj = state.actions[0]
             pos += 1
             ui.progress(_("editing"), pos, actobj.torule(),
                         _('changes'), total)
             ui.debug('histedit: processing %s %s\n' % (actobj.verb,\
                                                        actobj.torule()))
             parentctx, replacement_ = actobj.run()
             state.parentctxnode = parentctx.node()
             state.replacements.extend(replacement_)
             state.actions.pop(0)
 
     state.write()
     ui.progress(_("editing"), None)
 
 def _finishhistedit(ui, repo, state, fm):
     """This action runs when histedit is finishing its session"""
     repo.ui.pushbuffer()
     hg.update(repo, state.parentctxnode, quietempty=True)
     repo.ui.popbuffer()
 
     mapping, tmpnodes, created, ntm = processreplacement(state)
     if mapping:
         for prec, succs in mapping.iteritems():
             if not succs:
                 ui.debug('histedit: %s is dropped\n' % node.short(prec))
             else:
                 ui.debug('histedit: %s is replaced by %s\n' % (
                     node.short(prec), node.short(succs[0])))
                 if len(succs) > 1:
                     m = 'histedit:                            %s'
                     for n in succs[1:]:
                         ui.debug(m % node.short(n))
 
     if not state.keep:
         if mapping:
             movetopmostbookmarks(repo, state.topmost, ntm)
             # TODO update mq state
     else:
         mapping = {}
 
     for n in tmpnodes:
         mapping[n] = ()
 
     # remove entries about unknown nodes
     nodemap = repo.unfiltered().changelog.nodemap
     mapping = {k: v for k, v in mapping.items()
                if k in nodemap and all(n in nodemap for n in v)}
     scmutil.cleanupnodes(repo, mapping, 'histedit')
     hf = fm.hexfunc
     fl = fm.formatlist
     fd = fm.formatdict
     nodechanges = fd({hf(oldn): fl([hf(n) for n in newn], name='node')
                       for oldn, newn in mapping.iteritems()},
                      key="oldnode", value="newnodes")
     fm.data(nodechanges=nodechanges)
 
     state.clear()
     if os.path.exists(repo.sjoin('undo')):
         os.unlink(repo.sjoin('undo'))
     if repo.vfs.exists('histedit-last-edit.txt'):
         repo.vfs.unlink('histedit-last-edit.txt')
 
 def _aborthistedit(ui, repo, state):
     try:
         state.read()
         __, leafs, tmpnodes, __ = processreplacement(state)
         ui.debug('restore wc to old parent %s\n'
                 % node.short(state.topmost))
 
         # Recover our old commits if necessary
         if not state.topmost in repo and state.backupfile:
             backupfile = repo.vfs.join(state.backupfile)
             f = hg.openpath(ui, backupfile)
             gen = exchange.readbundle(ui, f, backupfile)
             with repo.transaction('histedit.abort') as tr:
                 bundle2.applybundle(repo, gen, tr, source='histedit',
                                     url='bundle:' + backupfile)
 
             os.remove(backupfile)
 
         # check whether we should update away
         if repo.unfiltered().revs('parents() and (%n  or %ln::)',
                                 state.parentctxnode, leafs | tmpnodes):
             hg.clean(repo, state.topmost, show_stats=True, quietempty=True)
         cleanupnode(ui, repo, tmpnodes)
         cleanupnode(ui, repo, leafs)
     except Exception:
         if state.inprogress():
             ui.warn(_('warning: encountered an exception during histedit '
                 '--abort; the repository may not have been completely '
                 'cleaned up\n'))
         raise
     finally:
             state.clear()
 
 def _edithisteditplan(ui, repo, state, rules):
     state.read()
     if not rules:
         comment = geteditcomment(ui,
                                  node.short(state.parentctxnode),
                                  node.short(state.topmost))
         rules = ruleeditor(repo, ui, state.actions, comment)
     else:
         rules = _readfile(ui, rules)
     actions = parserules(rules, state)
     ctxs = [repo[act.node] \
             for act in state.actions if act.node]
     warnverifyactions(ui, repo, actions, state, ctxs)
     state.actions = actions
     state.write()
 
 def _newhistedit(ui, repo, state, revs, freeargs, opts):
     outg = opts.get('outgoing')
     rules = opts.get('commands', '')
     force = opts.get('force')
 
     cmdutil.checkunfinished(repo)
     cmdutil.bailifchanged(repo)
 
     topmost, empty = repo.dirstate.parents()
     if outg:
         if freeargs:
             remote = freeargs[0]
         else:
             remote = None
         root = findoutgoing(ui, repo, remote, force, opts)
     else:
         rr = list(repo.set('roots(%ld)', scmutil.revrange(repo, revs)))
         if len(rr) != 1:
             raise error.Abort(_('The specified revisions must have '
                 'exactly one common root'))
         root = rr[0].node()
 
     revs = between(repo, root, topmost, state.keep)
     if not revs:
         raise error.Abort(_('%s is not an ancestor of working directory') %
                          node.short(root))
 
     ctxs = [repo[r] for r in revs]
     if not rules:
         comment = geteditcomment(ui, node.short(root), node.short(topmost))
         actions = [pick(state, r) for r in revs]
         rules = ruleeditor(repo, ui, actions, comment)
     else:
         rules = _readfile(ui, rules)
     actions = parserules(rules, state)
     warnverifyactions(ui, repo, actions, state, ctxs)
 
     parentctxnode = repo[root].parents()[0].node()
 
     state.parentctxnode = parentctxnode
     state.actions = actions
     state.topmost = topmost
     state.replacements = []
 
     ui.log("histedit", "%d actions to histedit", len(actions),
            histedit_num_actions=len(actions))
 
     # Create a backup so we can always abort completely.
     backupfile = None
     if not obsolete.isenabled(repo, obsolete.createmarkersopt):
         backupfile = repair.backupbundle(repo, [parentctxnode],
                                          [topmost], root, 'histedit')
     state.backupfile = backupfile
 
 def _getsummary(ctx):
     # a common pattern is to extract the summary but default to the empty
     # string
     summary = ctx.description() or ''
     if summary:
         summary = summary.splitlines()[0]
     return summary
 
 def bootstrapcontinue(ui, state, opts):
     repo = state.repo
 
     ms = mergemod.mergestate.read(repo)
     mergeutil.checkunresolved(ms)
 
     if state.actions:
         actobj = state.actions.pop(0)
 
         if _isdirtywc(repo):
             actobj.continuedirty()
             if _isdirtywc(repo):
                 abortdirty()
 
         parentctx, replacements = actobj.continueclean()
 
         state.parentctxnode = parentctx.node()
         state.replacements.extend(replacements)
 
     return state
 
 def between(repo, old, new, keep):
     """select and validate the set of revision to edit
 
     When keep is false, the specified set can't have children."""
     revs = repo.revs('%n::%n', old, new)
     if revs and not keep:
         if (not obsolete.isenabled(repo, obsolete.allowunstableopt) and
             repo.revs('(%ld::) - (%ld)', revs, revs)):
             raise error.Abort(_('can only histedit a changeset together '
                                 'with all its descendants'))
         if repo.revs('(%ld) and merge()', revs):
             raise error.Abort(_('cannot edit history that contains merges'))
         root = repo[revs.first()]  # list is already sorted by repo.revs()
         if not root.mutable():
             raise error.Abort(_('cannot edit public changeset: %s') % root,
                              hint=_("see 'hg help phases' for details"))
     return pycompat.maplist(repo.changelog.node, revs)
 
 def ruleeditor(repo, ui, actions, editcomment=""):
     """open an editor to edit rules
 
     rules are in the format [ [act, ctx], ...] like in state.rules
     """
     if repo.ui.configbool("experimental", "histedit.autoverb"):
         newact = util.sortdict()
         for act in actions:
             ctx = repo[act.node]
             summary = _getsummary(ctx)
             fword = summary.split(' ', 1)[0].lower()
             added = False
 
             # if it doesn't end with the special character '!' just skip this
             if fword.endswith('!'):
                 fword = fword[:-1]
                 if fword in primaryactions | secondaryactions | tertiaryactions:
                     act.verb = fword
                     # get the target summary
                     tsum = summary[len(fword) + 1:].lstrip()
                     # safe but slow: reverse iterate over the actions so we
                     # don't clash on two commits having the same summary
                     for na, l in reversed(list(newact.iteritems())):
                         actx = repo[na.node]
                         asum = _getsummary(actx)
                         if asum == tsum:
                             added = True
                             l.append(act)
                             break
 
             if not added:
                 newact[act] = []
 
         # copy over and flatten the new list
         actions = []
         for na, l in newact.iteritems():
             actions.append(na)
             actions += l
 
     rules = '\n'.join([act.torule() for act in actions])
     rules += '\n\n'
     rules += editcomment
     rules = ui.edit(rules, ui.username(), {'prefix': 'histedit'},
                     repopath=repo.path, action='histedit')
 
     # Save edit rules in .hg/histedit-last-edit.txt in case
     # the user needs to ask for help after something
     # surprising happens.
     with repo.vfs('histedit-last-edit.txt', 'wb') as f:
         f.write(rules)
 
     return rules
 
 def parserules(rules, state):
     """Read the histedit rules string and return list of action objects """
     rules = [l for l in (r.strip() for r in rules.splitlines())
                 if l and not l.startswith('#')]
     actions = []
     for r in rules:
         if ' ' not in r:
             raise error.ParseError(_('malformed line "%s"') % r)
         verb, rest = r.split(' ', 1)
 
         if verb not in actiontable:
             raise error.ParseError(_('unknown action "%s"') % verb)
 
         action = actiontable[verb].fromrule(state, rest)
         actions.append(action)
     return actions
 
 def warnverifyactions(ui, repo, actions, state, ctxs):
     try:
         verifyactions(actions, state, ctxs)
     except error.ParseError:
         if repo.vfs.exists('histedit-last-edit.txt'):
             ui.warn(_('warning: histedit rules saved '
                       'to: .hg/histedit-last-edit.txt\n'))
         raise
 
 def verifyactions(actions, state, ctxs):
     """Verify that there exists exactly one action per given changeset and
     other constraints.
 
     Will abort if there are to many or too few rules, a malformed rule,
     or a rule on a changeset outside of the user-given range.
     """
     expected = set(c.node() for c in ctxs)
     seen = set()
     prev = None
 
     if actions and actions[0].verb in ['roll', 'fold']:
         raise error.ParseError(_('first changeset cannot use verb "%s"') %
                                actions[0].verb)
 
     for action in actions:
         action.verify(prev, expected, seen)
         prev = action
         if action.node is not None:
             seen.add(action.node)
     missing = sorted(expected - seen)  # sort to stabilize output
 
     if state.repo.ui.configbool('histedit', 'dropmissing'):
         if len(actions) == 0:
             raise error.ParseError(_('no rules provided'),
                     hint=_('use strip extension to remove commits'))
 
         drops = [drop(state, n) for n in missing]
         # put the in the beginning so they execute immediately and
         # don't show in the edit-plan in the future
         actions[:0] = drops
     elif missing:
         raise error.ParseError(_('missing rules for changeset %s') %
                 node.short(missing[0]),
                 hint=_('use "drop %s" to discard, see also: '
                        "'hg help -e histedit.config'")
                        % node.short(missing[0]))
 
 def adjustreplacementsfrommarkers(repo, oldreplacements):
     """Adjust replacements from obsolescence markers
 
     Replacements structure is originally generated based on
     histedit's state and does not account for changes that are
     not recorded there. This function fixes that by adding
     data read from obsolescence markers"""
     if not obsolete.isenabled(repo, obsolete.createmarkersopt):
         return oldreplacements
 
     unfi = repo.unfiltered()
     nm = unfi.changelog.nodemap
     obsstore = repo.obsstore
     newreplacements = list(oldreplacements)
     oldsuccs = [r[1] for r in oldreplacements]
     # successors that have already been added to succstocheck once
     seensuccs = set().union(*oldsuccs) # create a set from an iterable of tuples
     succstocheck = list(seensuccs)
     while succstocheck:
         n = succstocheck.pop()
         missing = nm.get(n) is None
         markers = obsstore.successors.get(n, ())
         if missing and not markers:
             # dead end, mark it as such
             newreplacements.append((n, ()))
         for marker in markers:
             nsuccs = marker[1]
             newreplacements.append((n, nsuccs))
             for nsucc in nsuccs:
                 if nsucc not in seensuccs:
                     seensuccs.add(nsucc)
                     succstocheck.append(nsucc)
 
     return newreplacements
 
 def processreplacement(state):
     """process the list of replacements to return
 
     1) the final mapping between original and created nodes
     2) the list of temporary node created by histedit
     3) the list of new commit created by histedit"""
     replacements = adjustreplacementsfrommarkers(state.repo, state.replacements)
     allsuccs = set()
     replaced = set()
     fullmapping = {}
     # initialize basic set
     # fullmapping records all operations recorded in replacement
     for rep in replacements:
         allsuccs.update(rep[1])
         replaced.add(rep[0])
         fullmapping.setdefault(rep[0], set()).update(rep[1])
     new = allsuccs - replaced
     tmpnodes = allsuccs & replaced
     # Reduce content fullmapping into direct relation between original nodes
     # and final node created during history edition
     # Dropped changeset are replaced by an empty list
     toproceed = set(fullmapping)
     final = {}
     while toproceed:
         for x in list(toproceed):
             succs = fullmapping[x]
             for s in list(succs):
                 if s in toproceed:
                     # non final node with unknown closure
                     # We can't process this now
                     break
                 elif s in final:
                     # non final node, replace with closure
                     succs.remove(s)
                     succs.update(final[s])
             else:
                 final[x] = succs
                 toproceed.remove(x)
     # remove tmpnodes from final mapping
     for n in tmpnodes:
         del final[n]
     # we expect all changes involved in final to exist in the repo
     # turn `final` into list (topologically sorted)
     nm = state.repo.changelog.nodemap
     for prec, succs in final.items():
         final[prec] = sorted(succs, key=nm.get)
 
     # computed topmost element (necessary for bookmark)
     if new:
         newtopmost = sorted(new, key=state.repo.changelog.rev)[-1]
     elif not final:
         # Nothing rewritten at all. we won't need `newtopmost`
         # It is the same as `oldtopmost` and `processreplacement` know it
         newtopmost = None
     else:
         # every body died. The newtopmost is the parent of the root.
         r = state.repo.changelog.rev
         newtopmost = state.repo[sorted(final, key=r)[0]].p1().node()
 
     return final, tmpnodes, new, newtopmost
 
 def movetopmostbookmarks(repo, oldtopmost, newtopmost):
     """Move bookmark from oldtopmost to newly created topmost
 
     This is arguably a feature and we may only want that for the active
     bookmark. But the behavior is kept compatible with the old version for now.
     """
     if not oldtopmost or not newtopmost:
         return
     oldbmarks = repo.nodebookmarks(oldtopmost)
     if oldbmarks:
         with repo.lock(), repo.transaction('histedit') as tr:
             marks = repo._bookmarks
             changes = []
             for name in oldbmarks:
                 changes.append((name, newtopmost))
             marks.applychanges(repo, tr, changes)
 
 def cleanupnode(ui, repo, nodes):
     """strip a group of nodes from the repository
 
     The set of node to strip may contains unknown nodes."""
     with repo.lock():
         # do not let filtering get in the way of the cleanse
         # we should probably get rid of obsolescence marker created during the
         # histedit, but we currently do not have such information.
         repo = repo.unfiltered()
         # Find all nodes that need to be stripped
         # (we use %lr instead of %ln to silently ignore unknown items)
         nm = repo.changelog.nodemap
         nodes = sorted(n for n in nodes if n in nm)
         roots = [c.node() for c in repo.set("roots(%ln)", nodes)]
         if roots:
             repair.strip(ui, repo, roots)
 
 def stripwrapper(orig, ui, repo, nodelist, *args, **kwargs):
     if isinstance(nodelist, str):
         nodelist = [nodelist]
     if os.path.exists(os.path.join(repo.path, 'histedit-state')):
         state = histeditstate(repo)
         state.read()
         histedit_nodes = {action.node for action
                           in state.actions if action.node}
         common_nodes = histedit_nodes & set(nodelist)
         if common_nodes:
             raise error.Abort(_("histedit in progress, can't strip %s")
                              % ', '.join(node.short(x) for x in common_nodes))
     return orig(ui, repo, nodelist, *args, **kwargs)
 
 extensions.wrapfunction(repair, 'strip', stripwrapper)
 
 def summaryhook(ui, repo):
     if not os.path.exists(repo.vfs.join('histedit-state')):
         return
     state = histeditstate(repo)
     state.read()
     if state.actions:
         # i18n: column positioning for "hg summary"
         ui.write(_('hist:   %s (histedit --continue)\n') %
                  (ui.label(_('%d remaining'), 'histedit.remaining') %
                   len(state.actions)))
 
 def extsetup(ui):
     cmdutil.summaryhooks.add('histedit', summaryhook)
     cmdutil.unfinishedstates.append(
         ['histedit-state', False, True, _('histedit in progress'),
          _("use 'hg histedit --continue' or 'hg histedit --abort'")])
     cmdutil.afterresolvedstates.append(
         ['histedit-state', _('hg histedit --continue')])

hgext/journal.py

@@ -1,520 +1,523
 # journal.py
 #
 # Copyright 2014-2016 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 """track previous positions of bookmarks (EXPERIMENTAL)
 
 This extension adds a new command: `hg journal`, which shows you where
 bookmarks were previously located.
 
 """
 
 from __future__ import absolute_import
 
 import collections
 import errno
 import os
 import weakref
 
 from mercurial.i18n import _
 
 from mercurial import (
     bookmarks,
     cmdutil,
     dispatch,
     encoding,
     error,
     extensions,
     hg,
     localrepo,
     lock,
     logcmdutil,
     node,
     pycompat,
     registrar,
     util,
 )
-from mercurial.utils import dateutil
+from mercurial.utils import (
+    dateutil,
+    stringutil,
+)
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
 # storage format version; increment when the format changes
 storageversion = 0
 
 # namespaces
 bookmarktype = 'bookmark'
 wdirparenttype = 'wdirparent'
 # In a shared repository, what shared feature name is used
 # to indicate this namespace is shared with the source?
 sharednamespaces = {
     bookmarktype: hg.sharedbookmarks,
 }
 
 # Journal recording, register hooks and storage object
 def extsetup(ui):
     extensions.wrapfunction(dispatch, 'runcommand', runcommand)
     extensions.wrapfunction(bookmarks.bmstore, '_write', recordbookmarks)
     extensions.wrapfilecache(
         localrepo.localrepository, 'dirstate', wrapdirstate)
     extensions.wrapfunction(hg, 'postshare', wrappostshare)
     extensions.wrapfunction(hg, 'copystore', unsharejournal)
 
 def reposetup(ui, repo):
     if repo.local():
         repo.journal = journalstorage(repo)
         repo._wlockfreeprefix.add('namejournal')
 
         dirstate, cached = localrepo.isfilecached(repo, 'dirstate')
         if cached:
             # already instantiated dirstate isn't yet marked as
             # "journal"-ing, even though repo.dirstate() was already
             # wrapped by own wrapdirstate()
             _setupdirstate(repo, dirstate)
 
 def runcommand(orig, lui, repo, cmd, fullargs, *args):
     """Track the command line options for recording in the journal"""
     journalstorage.recordcommand(*fullargs)
     return orig(lui, repo, cmd, fullargs, *args)
 
 def _setupdirstate(repo, dirstate):
     dirstate.journalstorage = repo.journal
     dirstate.addparentchangecallback('journal', recorddirstateparents)
 
 # hooks to record dirstate changes
 def wrapdirstate(orig, repo):
     """Make journal storage available to the dirstate object"""
     dirstate = orig(repo)
     if util.safehasattr(repo, 'journal'):
         _setupdirstate(repo, dirstate)
     return dirstate
 
 def recorddirstateparents(dirstate, old, new):
     """Records all dirstate parent changes in the journal."""
     old = list(old)
     new = list(new)
     if util.safehasattr(dirstate, 'journalstorage'):
         # only record two hashes if there was a merge
         oldhashes = old[:1] if old[1] == node.nullid else old
         newhashes = new[:1] if new[1] == node.nullid else new
         dirstate.journalstorage.record(
             wdirparenttype, '.', oldhashes, newhashes)
 
 # hooks to record bookmark changes (both local and remote)
 def recordbookmarks(orig, store, fp):
     """Records all bookmark changes in the journal."""
     repo = store._repo
     if util.safehasattr(repo, 'journal'):
         oldmarks = bookmarks.bmstore(repo)
         for mark, value in store.iteritems():
             oldvalue = oldmarks.get(mark, node.nullid)
             if value != oldvalue:
                 repo.journal.record(bookmarktype, mark, oldvalue, value)
     return orig(store, fp)
 
 # shared repository support
 def _readsharedfeatures(repo):
     """A set of shared features for this repository"""
     try:
         return set(repo.vfs.read('shared').splitlines())
     except IOError as inst:
         if inst.errno != errno.ENOENT:
             raise
         return set()
 
 def _mergeentriesiter(*iterables, **kwargs):
     """Given a set of sorted iterables, yield the next entry in merged order
 
     Note that by default entries go from most recent to oldest.
     """
     order = kwargs.pop(r'order', max)
     iterables = [iter(it) for it in iterables]
     # this tracks still active iterables; iterables are deleted as they are
     # exhausted, which is why this is a dictionary and why each entry also
     # stores the key. Entries are mutable so we can store the next value each
     # time.
     iterable_map = {}
     for key, it in enumerate(iterables):
         try:
             iterable_map[key] = [next(it), key, it]
         except StopIteration:
             # empty entry, can be ignored
             pass
 
     while iterable_map:
         value, key, it = order(iterable_map.itervalues())
         yield value
         try:
             iterable_map[key][0] = next(it)
         except StopIteration:
             # this iterable is empty, remove it from consideration
             del iterable_map[key]
 
 def wrappostshare(orig, sourcerepo, destrepo, **kwargs):
     """Mark this shared working copy as sharing journal information"""
     with destrepo.wlock():
         orig(sourcerepo, destrepo, **kwargs)
         with destrepo.vfs('shared', 'a') as fp:
             fp.write('journal\n')
 
 def unsharejournal(orig, ui, repo, repopath):
     """Copy shared journal entries into this repo when unsharing"""
     if (repo.path == repopath and repo.shared() and
             util.safehasattr(repo, 'journal')):
         sharedrepo = hg.sharedreposource(repo)
         sharedfeatures = _readsharedfeatures(repo)
         if sharedrepo and sharedfeatures > {'journal'}:
             # there is a shared repository and there are shared journal entries
             # to copy. move shared date over from source to destination but
             # move the local file first
             if repo.vfs.exists('namejournal'):
                 journalpath = repo.vfs.join('namejournal')
                 util.rename(journalpath, journalpath + '.bak')
             storage = repo.journal
             local = storage._open(
                 repo.vfs, filename='namejournal.bak', _newestfirst=False)
             shared = (
                 e for e in storage._open(sharedrepo.vfs, _newestfirst=False)
                 if sharednamespaces.get(e.namespace) in sharedfeatures)
             for entry in _mergeentriesiter(local, shared, order=min):
                 storage._write(repo.vfs, entry)
 
     return orig(ui, repo, repopath)
 
 class journalentry(collections.namedtuple(
         u'journalentry',
         u'timestamp user command namespace name oldhashes newhashes')):
     """Individual journal entry
 
     * timestamp: a mercurial (time, timezone) tuple
     * user: the username that ran the command
     * namespace: the entry namespace, an opaque string
     * name: the name of the changed item, opaque string with meaning in the
       namespace
     * command: the hg command that triggered this record
     * oldhashes: a tuple of one or more binary hashes for the old location
     * newhashes: a tuple of one or more binary hashes for the new location
 
     Handles serialisation from and to the storage format. Fields are
     separated by newlines, hashes are written out in hex separated by commas,
     timestamp and timezone are separated by a space.
 
     """
     @classmethod
     def fromstorage(cls, line):
         (time, user, command, namespace, name,
          oldhashes, newhashes) = line.split('\n')
         timestamp, tz = time.split()
         timestamp, tz = float(timestamp), int(tz)
         oldhashes = tuple(node.bin(hash) for hash in oldhashes.split(','))
         newhashes = tuple(node.bin(hash) for hash in newhashes.split(','))
         return cls(
             (timestamp, tz), user, command, namespace, name,
             oldhashes, newhashes)
 
     def __bytes__(self):
         """bytes representation for storage"""
         time = ' '.join(map(str, self.timestamp))
         oldhashes = ','.join([node.hex(hash) for hash in self.oldhashes])
         newhashes = ','.join([node.hex(hash) for hash in self.newhashes])
         return '\n'.join((
             time, self.user, self.command, self.namespace, self.name,
             oldhashes, newhashes))
 
     __str__ = encoding.strmethod(__bytes__)
 
 class journalstorage(object):
     """Storage for journal entries
 
     Entries are divided over two files; one with entries that pertain to the
     local working copy *only*, and one with entries that are shared across
     multiple working copies when shared using the share extension.
 
     Entries are stored with NUL bytes as separators. See the journalentry
     class for the per-entry structure.
 
     The file format starts with an integer version, delimited by a NUL.
 
     This storage uses a dedicated lock; this makes it easier to avoid issues
     with adding entries that added when the regular wlock is unlocked (e.g.
     the dirstate).
 
     """
     _currentcommand = ()
     _lockref = None
 
     def __init__(self, repo):
         self.user = util.getuser()
         self.ui = repo.ui
         self.vfs = repo.vfs
 
         # is this working copy using a shared storage?
         self.sharedfeatures = self.sharedvfs = None
         if repo.shared():
             features = _readsharedfeatures(repo)
             sharedrepo = hg.sharedreposource(repo)
             if sharedrepo is not None and 'journal' in features:
                 self.sharedvfs = sharedrepo.vfs
                 self.sharedfeatures = features
 
     # track the current command for recording in journal entries
     @property
     def command(self):
         commandstr = ' '.join(
             map(util.shellquote, journalstorage._currentcommand))
         if '\n' in commandstr:
             # truncate multi-line commands
             commandstr = commandstr.partition('\n')[0] + ' ...'
         return commandstr
 
     @classmethod
     def recordcommand(cls, *fullargs):
         """Set the current hg arguments, stored with recorded entries"""
         # Set the current command on the class because we may have started
         # with a non-local repo (cloning for example).
         cls._currentcommand = fullargs
 
     def _currentlock(self, lockref):
         """Returns the lock if it's held, or None if it's not.
 
         (This is copied from the localrepo class)
         """
         if lockref is None:
             return None
         l = lockref()
         if l is None or not l.held:
             return None
         return l
 
     def jlock(self, vfs):
         """Create a lock for the journal file"""
         if self._currentlock(self._lockref) is not None:
             raise error.Abort(_('journal lock does not support nesting'))
         desc = _('journal of %s') % vfs.base
         try:
             l = lock.lock(vfs, 'namejournal.lock', 0, desc=desc)
         except error.LockHeld as inst:
             self.ui.warn(
                 _("waiting for lock on %s held by %r\n") % (desc, inst.locker))
             # default to 600 seconds timeout
             l = lock.lock(
                 vfs, 'namejournal.lock',
                 self.ui.configint("ui", "timeout"), desc=desc)
             self.ui.warn(_("got lock after %s seconds\n") % l.delay)
         self._lockref = weakref.ref(l)
         return l
 
     def record(self, namespace, name, oldhashes, newhashes):
         """Record a new journal entry
 
         * namespace: an opaque string; this can be used to filter on the type
           of recorded entries.
         * name: the name defining this entry; for bookmarks, this is the
           bookmark name. Can be filtered on when retrieving entries.
         * oldhashes and newhashes: each a single binary hash, or a list of
           binary hashes. These represent the old and new position of the named
           item.
 
         """
         if not isinstance(oldhashes, list):
             oldhashes = [oldhashes]
         if not isinstance(newhashes, list):
             newhashes = [newhashes]
 
         entry = journalentry(
             dateutil.makedate(), self.user, self.command, namespace, name,
             oldhashes, newhashes)
 
         vfs = self.vfs
         if self.sharedvfs is not None:
             # write to the shared repository if this feature is being
             # shared between working copies.
             if sharednamespaces.get(namespace) in self.sharedfeatures:
                 vfs = self.sharedvfs
 
         self._write(vfs, entry)
 
     def _write(self, vfs, entry):
         with self.jlock(vfs):
             version = None
             # open file in amend mode to ensure it is created if missing
             with vfs('namejournal', mode='a+b') as f:
                 f.seek(0, os.SEEK_SET)
                 # Read just enough bytes to get a version number (up to 2
                 # digits plus separator)
                 version = f.read(3).partition('\0')[0]
                 if version and version != "%d" % storageversion:
                     # different version of the storage. Exit early (and not
                     # write anything) if this is not a version we can handle or
                     # the file is corrupt. In future, perhaps rotate the file
                     # instead?
                     self.ui.warn(
                         _("unsupported journal file version '%s'\n") % version)
                     return
                 if not version:
                     # empty file, write version first
                     f.write(("%d" % storageversion) + '\0')
                 f.seek(0, os.SEEK_END)
                 f.write(bytes(entry) + '\0')
 
     def filtered(self, namespace=None, name=None):
         """Yield all journal entries with the given namespace or name
 
         Both the namespace and the name are optional; if neither is given all
         entries in the journal are produced.
 
         Matching supports regular expressions by using the `re:` prefix
         (use `literal:` to match names or namespaces that start with `re:`)
 
         """
         if namespace is not None:
-            namespace = util.stringmatcher(namespace)[-1]
+            namespace = stringutil.stringmatcher(namespace)[-1]
         if name is not None:
-            name = util.stringmatcher(name)[-1]
+            name = stringutil.stringmatcher(name)[-1]
         for entry in self:
             if namespace is not None and not namespace(entry.namespace):
                 continue
             if name is not None and not name(entry.name):
                 continue
             yield entry
 
     def __iter__(self):
         """Iterate over the storage
 
         Yields journalentry instances for each contained journal record.
 
         """
         local = self._open(self.vfs)
 
         if self.sharedvfs is None:
             return local
 
         # iterate over both local and shared entries, but only those
         # shared entries that are among the currently shared features
         shared = (
             e for e in self._open(self.sharedvfs)
             if sharednamespaces.get(e.namespace) in self.sharedfeatures)
         return _mergeentriesiter(local, shared)
 
     def _open(self, vfs, filename='namejournal', _newestfirst=True):
         if not vfs.exists(filename):
             return
 
         with vfs(filename) as f:
             raw = f.read()
 
         lines = raw.split('\0')
         version = lines and lines[0]
         if version != "%d" % storageversion:
             version = version or _('not available')
             raise error.Abort(_("unknown journal file version '%s'") % version)
 
         # Skip the first line, it's a version number. Normally we iterate over
         # these in reverse order to list newest first; only when copying across
         # a shared storage do we forgo reversing.
         lines = lines[1:]
         if _newestfirst:
             lines = reversed(lines)
         for line in lines:
             if not line:
                 continue
             yield journalentry.fromstorage(line)
 
 # journal reading
 # log options that don't make sense for journal
 _ignoreopts = ('no-merges', 'graph')
 @command(
     'journal', [
         ('', 'all', None, 'show history for all names'),
         ('c', 'commits', None, 'show commit metadata'),
     ] + [opt for opt in cmdutil.logopts if opt[1] not in _ignoreopts],
     '[OPTION]... [BOOKMARKNAME]')
 def journal(ui, repo, *args, **opts):
     """show the previous position of bookmarks and the working copy
 
     The journal is used to see the previous commits that bookmarks and the
     working copy pointed to. By default the previous locations for the working
     copy.  Passing a bookmark name will show all the previous positions of
     that bookmark. Use the --all switch to show previous locations for all
     bookmarks and the working copy; each line will then include the bookmark
     name, or '.' for the working copy, as well.
 
     If `name` starts with `re:`, the remainder of the name is treated as
     a regular expression. To match a name that actually starts with `re:`,
     use the prefix `literal:`.
 
     By default hg journal only shows the commit hash and the command that was
     running at that time. -v/--verbose will show the prior hash, the user, and
     the time at which it happened.
 
     Use -c/--commits to output log information on each commit hash; at this
     point you can use the usual `--patch`, `--git`, `--stat` and `--template`
     switches to alter the log output for these.
 
     `hg journal -T json` can be used to produce machine readable output.
 
     """
     opts = pycompat.byteskwargs(opts)
     name = '.'
     if opts.get('all'):
         if args:
             raise error.Abort(
                 _("You can't combine --all and filtering on a name"))
         name = None
     if args:
         name = args[0]
 
     fm = ui.formatter('journal', opts)
 
     if opts.get("template") != "json":
         if name is None:
             displayname = _('the working copy and bookmarks')
         else:
             displayname = "'%s'" % name
         ui.status(_("previous locations of %s:\n") % displayname)
 
     limit = logcmdutil.getlimit(opts)
     entry = None
     ui.pager('journal')
     for count, entry in enumerate(repo.journal.filtered(name=name)):
         if count == limit:
             break
         newhashesstr = fm.formatlist(map(fm.hexfunc, entry.newhashes),
                                      name='node', sep=',')
         oldhashesstr = fm.formatlist(map(fm.hexfunc, entry.oldhashes),
                                      name='node', sep=',')
 
         fm.startitem()
         fm.condwrite(ui.verbose, 'oldhashes', '%s -> ', oldhashesstr)
         fm.write('newhashes', '%s', newhashesstr)
         fm.condwrite(ui.verbose, 'user', ' %-8s', entry.user)
         fm.condwrite(
             opts.get('all') or name.startswith('re:'),
             'name', '  %-8s', entry.name)
 
         timestring = fm.formatdate(entry.timestamp, '%Y-%m-%d %H:%M %1%2')
         fm.condwrite(ui.verbose, 'date', ' %s', timestring)
         fm.write('command', '  %s\n', entry.command)
 
         if opts.get("commits"):
             displayer = logcmdutil.changesetdisplayer(ui, repo, opts)
             for hash in entry.newhashes:
                 try:
                     ctx = repo[hash]
                     displayer.show(ctx)
                 except error.RepoLookupError as e:
                     fm.write('repolookuperror', "%s\n\n", pycompat.bytestr(e))
             displayer.close()
 
     fm.end()
 
     if entry is None:
         ui.status(_("no recorded locations\n"))

hgext/keyword.py

@@ -1,811 +1,815
 # keyword.py - $Keyword$ expansion for Mercurial
 #
 # Copyright 2007-2015 Christian Ebert <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 #
 # $Id$
 #
 # Keyword expansion hack against the grain of a Distributed SCM
 #
 # There are many good reasons why this is not needed in a distributed
 # SCM, still it may be useful in very small projects based on single
 # files (like LaTeX packages), that are mostly addressed to an
 # audience not running a version control system.
 #
 # For in-depth discussion refer to
 # <https://mercurial-scm.org/wiki/KeywordPlan>.
 #
 # Keyword expansion is based on Mercurial's changeset template mappings.
 #
 # Binary files are not touched.
 #
 # Files to act upon/ignore are specified in the [keyword] section.
 # Customized keyword template mappings in the [keywordmaps] section.
 #
 # Run 'hg help keyword' and 'hg kwdemo' to get info on configuration.
 
 '''expand keywords in tracked files
 
 This extension expands RCS/CVS-like or self-customized $Keywords$ in
 tracked text files selected by your configuration.
 
 Keywords are only expanded in local repositories and not stored in the
 change history. The mechanism can be regarded as a convenience for the
 current user or for archive distribution.
 
 Keywords expand to the changeset data pertaining to the latest change
 relative to the working directory parent of each file.
 
 Configuration is done in the [keyword], [keywordset] and [keywordmaps]
 sections of hgrc files.
 
 Example::
 
     [keyword]
     # expand keywords in every python file except those matching "x*"
     **.py =
     x*    = ignore
 
     [keywordset]
     # prefer svn- over cvs-like default keywordmaps
     svn = True
 
 .. note::
 
    The more specific you are in your filename patterns the less you
    lose speed in huge repositories.
 
 For [keywordmaps] template mapping and expansion demonstration and
 control run :hg:`kwdemo`. See :hg:`help templates` for a list of
 available templates and filters.
 
 Three additional date template filters are provided:
 
 :``utcdate``:    "2006/09/18 15:13:13"
 :``svnutcdate``: "2006-09-18 15:13:13Z"
 :``svnisodate``: "2006-09-18 08:13:13 -700 (Mon, 18 Sep 2006)"
 
 The default template mappings (view with :hg:`kwdemo -d`) can be
 replaced with customized keywords and templates. Again, run
 :hg:`kwdemo` to control the results of your configuration changes.
 
 Before changing/disabling active keywords, you must run :hg:`kwshrink`
 to avoid storing expanded keywords in the change history.
 
 To force expansion after enabling it, or a configuration change, run
 :hg:`kwexpand`.
 
 Expansions spanning more than one line and incremental expansions,
 like CVS' $Log$, are not supported. A keyword template map "Log =
 {desc}" expands to the first line of the changeset description.
 '''
 
 
 from __future__ import absolute_import
 
 import os
 import re
 import tempfile
 import weakref
 
 from mercurial.i18n import _
 from mercurial.hgweb import webcommands
 
 from mercurial import (
     cmdutil,
     context,
     dispatch,
     error,
     extensions,
     filelog,
     localrepo,
     logcmdutil,
     match,
     patch,
     pathutil,
     pycompat,
     registrar,
     scmutil,
     templatefilters,
     util,
 )
-from mercurial.utils import dateutil
+from mercurial.utils import (
+    dateutil,
+    stringutil,
+)
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
 # hg commands that do not act on keywords
 nokwcommands = ('add addremove annotate bundle export grep incoming init log'
                 ' outgoing push tip verify convert email glog')
 
 # webcommands that do not act on keywords
 nokwwebcommands = ('annotate changeset rev filediff diff comparison')
 
 # hg commands that trigger expansion only when writing to working dir,
 # not when reading filelog, and unexpand when reading from working dir
 restricted = ('merge kwexpand kwshrink record qrecord resolve transplant'
               ' unshelve rebase graft backout histedit fetch')
 
 # names of extensions using dorecord
 recordextensions = 'record'
 
 colortable = {
     'kwfiles.enabled': 'green bold',
     'kwfiles.deleted': 'cyan bold underline',
     'kwfiles.enabledunknown': 'green',
     'kwfiles.ignored': 'bold',
     'kwfiles.ignoredunknown': 'none'
 }
 
 templatefilter = registrar.templatefilter()
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 configitem('keywordset', 'svn',
     default=False,
 )
 # date like in cvs' $Date
 @templatefilter('utcdate')
 def utcdate(text):
     '''Date. Returns a UTC-date in this format: "2009/08/18 11:00:13".
     '''
     dateformat = '%Y/%m/%d %H:%M:%S'
     return dateutil.datestr((dateutil.parsedate(text)[0], 0), dateformat)
 # date like in svn's $Date
 @templatefilter('svnisodate')
 def svnisodate(text):
     '''Date. Returns a date in this format: "2009-08-18 13:00:13
     +0200 (Tue, 18 Aug 2009)".
     '''
     return dateutil.datestr(text, '%Y-%m-%d %H:%M:%S %1%2 (%a, %d %b %Y)')
 # date like in svn's $Id
 @templatefilter('svnutcdate')
 def svnutcdate(text):
     '''Date. Returns a UTC-date in this format: "2009-08-18
     11:00:13Z".
     '''
     dateformat = '%Y-%m-%d %H:%M:%SZ'
     return dateutil.datestr((dateutil.parsedate(text)[0], 0), dateformat)
 
 # make keyword tools accessible
 kwtools = {'hgcmd': ''}
 
 def _defaultkwmaps(ui):
     '''Returns default keywordmaps according to keywordset configuration.'''
     templates = {
         'Revision': '{node|short}',
         'Author': '{author|user}',
     }
     kwsets = ({
         'Date': '{date|utcdate}',
         'RCSfile': '{file|basename},v',
         'RCSFile': '{file|basename},v', # kept for backwards compatibility
                                         # with hg-keyword
         'Source': '{root}/{file},v',
         'Id': '{file|basename},v {node|short} {date|utcdate} {author|user}',
         'Header': '{root}/{file},v {node|short} {date|utcdate} {author|user}',
     }, {
         'Date': '{date|svnisodate}',
         'Id': '{file|basename},v {node|short} {date|svnutcdate} {author|user}',
         'LastChangedRevision': '{node|short}',
         'LastChangedBy': '{author|user}',
         'LastChangedDate': '{date|svnisodate}',
     })
     templates.update(kwsets[ui.configbool('keywordset', 'svn')])
     return templates
 
 def _shrinktext(text, subfunc):
     '''Helper for keyword expansion removal in text.
     Depending on subfunc also returns number of substitutions.'''
     return subfunc(r'$\1$', text)
 
 def _preselect(wstatus, changed):
     '''Retrieves modified and added files from a working directory state
     and returns the subset of each contained in given changed files
     retrieved from a change context.'''
     modified = [f for f in wstatus.modified if f in changed]
     added = [f for f in wstatus.added if f in changed]
     return modified, added
 
 
 class kwtemplater(object):
     '''
     Sets up keyword templates, corresponding keyword regex, and
     provides keyword substitution functions.
     '''
 
     def __init__(self, ui, repo, inc, exc):
         self.ui = ui
         self._repo = weakref.ref(repo)
         self.match = match.match(repo.root, '', [], inc, exc)
         self.restrict = kwtools['hgcmd'] in restricted.split()
         self.postcommit = False
 
         kwmaps = self.ui.configitems('keywordmaps')
         if kwmaps: # override default templates
             self.templates = dict(kwmaps)
         else:
             self.templates = _defaultkwmaps(self.ui)
 
     @property
     def repo(self):
         return self._repo()
 
     @util.propertycache
     def escape(self):
         '''Returns bar-separated and escaped keywords.'''
         return '|'.join(map(re.escape, self.templates.keys()))
 
     @util.propertycache
     def rekw(self):
         '''Returns regex for unexpanded keywords.'''
         return re.compile(r'\$(%s)\$' % self.escape)
 
     @util.propertycache
     def rekwexp(self):
         '''Returns regex for expanded keywords.'''
         return re.compile(r'\$(%s): [^$\n\r]*? \$' % self.escape)
 
     def substitute(self, data, path, ctx, subfunc):
         '''Replaces keywords in data with expanded template.'''
         def kwsub(mobj):
             kw = mobj.group(1)
             ct = logcmdutil.maketemplater(self.ui, self.repo,
                                           self.templates[kw])
             self.ui.pushbuffer()
             ct.show(ctx, root=self.repo.root, file=path)
             ekw = templatefilters.firstline(self.ui.popbuffer())
             return '$%s: %s $' % (kw, ekw)
         return subfunc(kwsub, data)
 
     def linkctx(self, path, fileid):
         '''Similar to filelog.linkrev, but returns a changectx.'''
         return self.repo.filectx(path, fileid=fileid).changectx()
 
     def expand(self, path, node, data):
         '''Returns data with keywords expanded.'''
-        if not self.restrict and self.match(path) and not util.binary(data):
+        if (not self.restrict and self.match(path)
+            and not stringutil.binary(data)):
             ctx = self.linkctx(path, node)
             return self.substitute(data, path, ctx, self.rekw.sub)
         return data
 
     def iskwfile(self, cand, ctx):
         '''Returns subset of candidates which are configured for keyword
         expansion but are not symbolic links.'''
         return [f for f in cand if self.match(f) and 'l' not in ctx.flags(f)]
 
     def overwrite(self, ctx, candidates, lookup, expand, rekw=False):
         '''Overwrites selected files expanding/shrinking keywords.'''
         if self.restrict or lookup or self.postcommit: # exclude kw_copy
             candidates = self.iskwfile(candidates, ctx)
         if not candidates:
             return
         kwcmd = self.restrict and lookup # kwexpand/kwshrink
         if self.restrict or expand and lookup:
             mf = ctx.manifest()
         if self.restrict or rekw:
             re_kw = self.rekw
         else:
             re_kw = self.rekwexp
         if expand:
             msg = _('overwriting %s expanding keywords\n')
         else:
             msg = _('overwriting %s shrinking keywords\n')
         for f in candidates:
             if self.restrict:
                 data = self.repo.file(f).read(mf[f])
             else:
                 data = self.repo.wread(f)
-            if util.binary(data):
+            if stringutil.binary(data):
                 continue
             if expand:
                 parents = ctx.parents()
                 if lookup:
                     ctx = self.linkctx(f, mf[f])
                 elif self.restrict and len(parents) > 1:
                     # merge commit
                     # in case of conflict f is in modified state during
                     # merge, even if f does not differ from f in parent
                     for p in parents:
                         if f in p and not p[f].cmp(ctx[f]):
                             ctx = p[f].changectx()
                             break
                 data, found = self.substitute(data, f, ctx, re_kw.subn)
             elif self.restrict:
                 found = re_kw.search(data)
             else:
                 data, found = _shrinktext(data, re_kw.subn)
             if found:
                 self.ui.note(msg % f)
                 fp = self.repo.wvfs(f, "wb", atomictemp=True)
                 fp.write(data)
                 fp.close()
                 if kwcmd:
                     self.repo.dirstate.normal(f)
                 elif self.postcommit:
                     self.repo.dirstate.normallookup(f)
 
     def shrink(self, fname, text):
         '''Returns text with all keyword substitutions removed.'''
-        if self.match(fname) and not util.binary(text):
+        if self.match(fname) and not stringutil.binary(text):
             return _shrinktext(text, self.rekwexp.sub)
         return text
 
     def shrinklines(self, fname, lines):
         '''Returns lines with keyword substitutions removed.'''
         if self.match(fname):
             text = ''.join(lines)
-            if not util.binary(text):
+            if not stringutil.binary(text):
                 return _shrinktext(text, self.rekwexp.sub).splitlines(True)
         return lines
 
     def wread(self, fname, data):
         '''If in restricted mode returns data read from wdir with
         keyword substitutions removed.'''
         if self.restrict:
             return self.shrink(fname, data)
         return data
 
 class kwfilelog(filelog.filelog):
     '''
     Subclass of filelog to hook into its read, add, cmp methods.
     Keywords are "stored" unexpanded, and processed on reading.
     '''
     def __init__(self, opener, kwt, path):
         super(kwfilelog, self).__init__(opener, path)
         self.kwt = kwt
         self.path = path
 
     def read(self, node):
         '''Expands keywords when reading filelog.'''
         data = super(kwfilelog, self).read(node)
         if self.renamed(node):
             return data
         return self.kwt.expand(self.path, node, data)
 
     def add(self, text, meta, tr, link, p1=None, p2=None):
         '''Removes keyword substitutions when adding to filelog.'''
         text = self.kwt.shrink(self.path, text)
         return super(kwfilelog, self).add(text, meta, tr, link, p1, p2)
 
     def cmp(self, node, text):
         '''Removes keyword substitutions for comparison.'''
         text = self.kwt.shrink(self.path, text)
         return super(kwfilelog, self).cmp(node, text)
 
 def _status(ui, repo, wctx, kwt, *pats, **opts):
     '''Bails out if [keyword] configuration is not active.
     Returns status of working directory.'''
     if kwt:
         opts = pycompat.byteskwargs(opts)
         return repo.status(match=scmutil.match(wctx, pats, opts), clean=True,
                            unknown=opts.get('unknown') or opts.get('all'))
     if ui.configitems('keyword'):
         raise error.Abort(_('[keyword] patterns cannot match'))
     raise error.Abort(_('no [keyword] patterns configured'))
 
 def _kwfwrite(ui, repo, expand, *pats, **opts):
     '''Selects files and passes them to kwtemplater.overwrite.'''
     wctx = repo[None]
     if len(wctx.parents()) > 1:
         raise error.Abort(_('outstanding uncommitted merge'))
     kwt = getattr(repo, '_keywordkwt', None)
     with repo.wlock():
         status = _status(ui, repo, wctx, kwt, *pats, **opts)
         if status.modified or status.added or status.removed or status.deleted:
             raise error.Abort(_('outstanding uncommitted changes'))
         kwt.overwrite(wctx, status.clean, True, expand)
 
 @command('kwdemo',
          [('d', 'default', None, _('show default keyword template maps')),
           ('f', 'rcfile', '',
            _('read maps from rcfile'), _('FILE'))],
          _('hg kwdemo [-d] [-f RCFILE] [TEMPLATEMAP]...'),
          optionalrepo=True)
 def demo(ui, repo, *args, **opts):
     '''print [keywordmaps] configuration and an expansion example
 
     Show current, custom, or default keyword template maps and their
     expansions.
 
     Extend the current configuration by specifying maps as arguments
     and using -f/--rcfile to source an external hgrc file.
 
     Use -d/--default to disable current configuration.
 
     See :hg:`help templates` for information on templates and filters.
     '''
     def demoitems(section, items):
         ui.write('[%s]\n' % section)
         for k, v in sorted(items):
             ui.write('%s = %s\n' % (k, v))
 
     fn = 'demo.txt'
     tmpdir = tempfile.mkdtemp('', 'kwdemo.')
     ui.note(_('creating temporary repository at %s\n') % tmpdir)
     if repo is None:
         baseui = ui
     else:
         baseui = repo.baseui
     repo = localrepo.localrepository(baseui, tmpdir, True)
     ui.setconfig('keyword', fn, '', 'keyword')
     svn = ui.configbool('keywordset', 'svn')
     # explicitly set keywordset for demo output
     ui.setconfig('keywordset', 'svn', svn, 'keyword')
 
     uikwmaps = ui.configitems('keywordmaps')
     if args or opts.get(r'rcfile'):
         ui.status(_('\n\tconfiguration using custom keyword template maps\n'))
         if uikwmaps:
             ui.status(_('\textending current template maps\n'))
         if opts.get(r'default') or not uikwmaps:
             if svn:
                 ui.status(_('\toverriding default svn keywordset\n'))
             else:
                 ui.status(_('\toverriding default cvs keywordset\n'))
         if opts.get(r'rcfile'):
             ui.readconfig(opts.get('rcfile'))
         if args:
             # simulate hgrc parsing
             rcmaps = '[keywordmaps]\n%s\n' % '\n'.join(args)
             repo.vfs.write('hgrc', rcmaps)
             ui.readconfig(repo.vfs.join('hgrc'))
         kwmaps = dict(ui.configitems('keywordmaps'))
     elif opts.get(r'default'):
         if svn:
             ui.status(_('\n\tconfiguration using default svn keywordset\n'))
         else:
             ui.status(_('\n\tconfiguration using default cvs keywordset\n'))
         kwmaps = _defaultkwmaps(ui)
         if uikwmaps:
             ui.status(_('\tdisabling current template maps\n'))
             for k, v in kwmaps.iteritems():
                 ui.setconfig('keywordmaps', k, v, 'keyword')
     else:
         ui.status(_('\n\tconfiguration using current keyword template maps\n'))
         if uikwmaps:
             kwmaps = dict(uikwmaps)
         else:
             kwmaps = _defaultkwmaps(ui)
 
     uisetup(ui)
     reposetup(ui, repo)
     ui.write(('[extensions]\nkeyword =\n'))
     demoitems('keyword', ui.configitems('keyword'))
     demoitems('keywordset', ui.configitems('keywordset'))
     demoitems('keywordmaps', kwmaps.iteritems())
     keywords = '$' + '$\n$'.join(sorted(kwmaps.keys())) + '$\n'
     repo.wvfs.write(fn, keywords)
     repo[None].add([fn])
     ui.note(_('\nkeywords written to %s:\n') % fn)
     ui.note(keywords)
     with repo.wlock():
         repo.dirstate.setbranch('demobranch')
     for name, cmd in ui.configitems('hooks'):
         if name.split('.', 1)[0].find('commit') > -1:
             repo.ui.setconfig('hooks', name, '', 'keyword')
     msg = _('hg keyword configuration and expansion example')
     ui.note(("hg ci -m '%s'\n" % msg))
     repo.commit(text=msg)
     ui.status(_('\n\tkeywords expanded\n'))
     ui.write(repo.wread(fn))
     repo.wvfs.rmtree(repo.root)
 
 @command('kwexpand',
     cmdutil.walkopts,
     _('hg kwexpand [OPTION]... [FILE]...'),
     inferrepo=True)
 def expand(ui, repo, *pats, **opts):
     '''expand keywords in the working directory
 
     Run after (re)enabling keyword expansion.
 
     kwexpand refuses to run if given files contain local changes.
     '''
     # 3rd argument sets expansion to True
     _kwfwrite(ui, repo, True, *pats, **opts)
 
 @command('kwfiles',
          [('A', 'all', None, _('show keyword status flags of all files')),
           ('i', 'ignore', None, _('show files excluded from expansion')),
           ('u', 'unknown', None, _('only show unknown (not tracked) files')),
          ] + cmdutil.walkopts,
          _('hg kwfiles [OPTION]... [FILE]...'),
          inferrepo=True)
 def files(ui, repo, *pats, **opts):
     '''show files configured for keyword expansion
 
     List which files in the working directory are matched by the
     [keyword] configuration patterns.
 
     Useful to prevent inadvertent keyword expansion and to speed up
     execution by including only files that are actual candidates for
     expansion.
 
     See :hg:`help keyword` on how to construct patterns both for
     inclusion and exclusion of files.
 
     With -A/--all and -v/--verbose the codes used to show the status
     of files are::
 
       K = keyword expansion candidate
       k = keyword expansion candidate (not tracked)
       I = ignored
       i = ignored (not tracked)
     '''
     kwt = getattr(repo, '_keywordkwt', None)
     wctx = repo[None]
     status = _status(ui, repo, wctx, kwt, *pats, **opts)
     if pats:
         cwd = repo.getcwd()
     else:
         cwd = ''
     files = []
     opts = pycompat.byteskwargs(opts)
     if not opts.get('unknown') or opts.get('all'):
         files = sorted(status.modified + status.added + status.clean)
     kwfiles = kwt.iskwfile(files, wctx)
     kwdeleted = kwt.iskwfile(status.deleted, wctx)
     kwunknown = kwt.iskwfile(status.unknown, wctx)
     if not opts.get('ignore') or opts.get('all'):
         showfiles = kwfiles, kwdeleted, kwunknown
     else:
         showfiles = [], [], []
     if opts.get('all') or opts.get('ignore'):
         showfiles += ([f for f in files if f not in kwfiles],
                       [f for f in status.unknown if f not in kwunknown])
     kwlabels = 'enabled deleted enabledunknown ignored ignoredunknown'.split()
     kwstates = zip(kwlabels, 'K!kIi', showfiles)
     fm = ui.formatter('kwfiles', opts)
     fmt = '%.0s%s\n'
     if opts.get('all') or ui.verbose:
         fmt = '%s %s\n'
     for kwstate, char, filenames in kwstates:
         label = 'kwfiles.' + kwstate
         for f in filenames:
             fm.startitem()
             fm.write('kwstatus path', fmt, char,
                      repo.pathto(f, cwd), label=label)
     fm.end()
 
 @command('kwshrink',
     cmdutil.walkopts,
     _('hg kwshrink [OPTION]... [FILE]...'),
     inferrepo=True)
 def shrink(ui, repo, *pats, **opts):
     '''revert expanded keywords in the working directory
 
     Must be run before changing/disabling active keywords.
 
     kwshrink refuses to run if given files contain local changes.
     '''
     # 3rd argument sets expansion to False
     _kwfwrite(ui, repo, False, *pats, **opts)
 
 # monkeypatches
 
 def kwpatchfile_init(orig, self, ui, gp, backend, store, eolmode=None):
     '''Monkeypatch/wrap patch.patchfile.__init__ to avoid
     rejects or conflicts due to expanded keywords in working dir.'''
     orig(self, ui, gp, backend, store, eolmode)
     kwt = getattr(getattr(backend, 'repo', None), '_keywordkwt', None)
     if kwt:
         # shrink keywords read from working dir
         self.lines = kwt.shrinklines(self.fname, self.lines)
 
 def kwdiff(orig, repo, *args, **kwargs):
     '''Monkeypatch patch.diff to avoid expansion.'''
     kwt = getattr(repo, '_keywordkwt', None)
     if kwt:
         restrict = kwt.restrict
         kwt.restrict = True
     try:
         for chunk in orig(repo, *args, **kwargs):
             yield chunk
     finally:
         if kwt:
             kwt.restrict = restrict
 
 def kwweb_skip(orig, web):
     '''Wraps webcommands.x turning off keyword expansion.'''
     kwt = getattr(web.repo, '_keywordkwt', None)
     if kwt:
         origmatch = kwt.match
         kwt.match = util.never
     try:
         for chunk in orig(web):
             yield chunk
     finally:
         if kwt:
             kwt.match = origmatch
 
 def kw_amend(orig, ui, repo, old, extra, pats, opts):
     '''Wraps cmdutil.amend expanding keywords after amend.'''
     kwt = getattr(repo, '_keywordkwt', None)
     if kwt is None:
         return orig(ui, repo, old, extra, pats, opts)
     with repo.wlock():
         kwt.postcommit = True
         newid = orig(ui, repo, old, extra, pats, opts)
         if newid != old.node():
             ctx = repo[newid]
             kwt.restrict = True
             kwt.overwrite(ctx, ctx.files(), False, True)
             kwt.restrict = False
         return newid
 
 def kw_copy(orig, ui, repo, pats, opts, rename=False):
     '''Wraps cmdutil.copy so that copy/rename destinations do not
     contain expanded keywords.
     Note that the source of a regular file destination may also be a
     symlink:
     hg cp sym x                -> x is symlink
     cp sym x; hg cp -A sym x   -> x is file (maybe expanded keywords)
     For the latter we have to follow the symlink to find out whether its
     target is configured for expansion and we therefore must unexpand the
     keywords in the destination.'''
     kwt = getattr(repo, '_keywordkwt', None)
     if kwt is None:
         return orig(ui, repo, pats, opts, rename)
     with repo.wlock():
         orig(ui, repo, pats, opts, rename)
         if opts.get('dry_run'):
             return
         wctx = repo[None]
         cwd = repo.getcwd()
 
         def haskwsource(dest):
             '''Returns true if dest is a regular file and configured for
             expansion or a symlink which points to a file configured for
             expansion. '''
             source = repo.dirstate.copied(dest)
             if 'l' in wctx.flags(source):
                 source = pathutil.canonpath(repo.root, cwd,
                                            os.path.realpath(source))
             return kwt.match(source)
 
         candidates = [f for f in repo.dirstate.copies() if
                       'l' not in wctx.flags(f) and haskwsource(f)]
         kwt.overwrite(wctx, candidates, False, False)
 
 def kw_dorecord(orig, ui, repo, commitfunc, *pats, **opts):
     '''Wraps record.dorecord expanding keywords after recording.'''
     kwt = getattr(repo, '_keywordkwt', None)
     if kwt is None:
         return orig(ui, repo, commitfunc, *pats, **opts)
     with repo.wlock():
         # record returns 0 even when nothing has changed
         # therefore compare nodes before and after
         kwt.postcommit = True
         ctx = repo['.']
         wstatus = ctx.status()
         ret = orig(ui, repo, commitfunc, *pats, **opts)
         recctx = repo['.']
         if ctx != recctx:
             modified, added = _preselect(wstatus, recctx.files())
             kwt.restrict = False
             kwt.overwrite(recctx, modified, False, True)
             kwt.overwrite(recctx, added, False, True, True)
             kwt.restrict = True
         return ret
 
 def kwfilectx_cmp(orig, self, fctx):
     if fctx._customcmp:
         return fctx.cmp(self)
     kwt = getattr(self._repo, '_keywordkwt', None)
     if kwt is None:
         return orig(self, fctx)
     # keyword affects data size, comparing wdir and filelog size does
     # not make sense
     if (fctx._filenode is None and
         (self._repo._encodefilterpats or
          kwt.match(fctx.path()) and 'l' not in fctx.flags() or
          self.size() - 4 == fctx.size()) or
         self.size() == fctx.size()):
         return self._filelog.cmp(self._filenode, fctx.data())
     return True
 
 def uisetup(ui):
     ''' Monkeypatches dispatch._parse to retrieve user command.
     Overrides file method to return kwfilelog instead of filelog
     if file matches user configuration.
     Wraps commit to overwrite configured files with updated
     keyword substitutions.
     Monkeypatches patch and webcommands.'''
 
     def kwdispatch_parse(orig, ui, args):
         '''Monkeypatch dispatch._parse to obtain running hg command.'''
         cmd, func, args, options, cmdoptions = orig(ui, args)
         kwtools['hgcmd'] = cmd
         return cmd, func, args, options, cmdoptions
 
     extensions.wrapfunction(dispatch, '_parse', kwdispatch_parse)
 
     extensions.wrapfunction(context.filectx, 'cmp', kwfilectx_cmp)
     extensions.wrapfunction(patch.patchfile, '__init__', kwpatchfile_init)
     extensions.wrapfunction(patch, 'diff', kwdiff)
     extensions.wrapfunction(cmdutil, 'amend', kw_amend)
     extensions.wrapfunction(cmdutil, 'copy', kw_copy)
     extensions.wrapfunction(cmdutil, 'dorecord', kw_dorecord)
     for c in nokwwebcommands.split():
         extensions.wrapfunction(webcommands, c, kwweb_skip)
 
 def reposetup(ui, repo):
     '''Sets up repo as kwrepo for keyword substitution.'''
 
     try:
         if (not repo.local() or kwtools['hgcmd'] in nokwcommands.split()
             or '.hg' in util.splitpath(repo.root)
             or repo._url.startswith('bundle:')):
             return
     except AttributeError:
         pass
 
     inc, exc = [], ['.hg*']
     for pat, opt in ui.configitems('keyword'):
         if opt != 'ignore':
             inc.append(pat)
         else:
             exc.append(pat)
     if not inc:
         return
 
     kwt = kwtemplater(ui, repo, inc, exc)
 
     class kwrepo(repo.__class__):
         def file(self, f):
             if f[0] == '/':
                 f = f[1:]
             return kwfilelog(self.svfs, kwt, f)
 
         def wread(self, filename):
             data = super(kwrepo, self).wread(filename)
             return kwt.wread(filename, data)
 
         def commit(self, *args, **opts):
             # use custom commitctx for user commands
             # other extensions can still wrap repo.commitctx directly
             self.commitctx = self.kwcommitctx
             try:
                 return super(kwrepo, self).commit(*args, **opts)
             finally:
                 del self.commitctx
 
         def kwcommitctx(self, ctx, error=False):
             n = super(kwrepo, self).commitctx(ctx, error)
             # no lock needed, only called from repo.commit() which already locks
             if not kwt.postcommit:
                 restrict = kwt.restrict
                 kwt.restrict = True
                 kwt.overwrite(self[n], sorted(ctx.added() + ctx.modified()),
                               False, True)
                 kwt.restrict = restrict
             return n
 
         def rollback(self, dryrun=False, force=False):
             with self.wlock():
                 origrestrict = kwt.restrict
                 try:
                     if not dryrun:
                         changed = self['.'].files()
                     ret = super(kwrepo, self).rollback(dryrun, force)
                     if not dryrun:
                         ctx = self['.']
                         modified, added = _preselect(ctx.status(), changed)
                         kwt.restrict = False
                         kwt.overwrite(ctx, modified, True, True)
                         kwt.overwrite(ctx, added, True, False)
                     return ret
                 finally:
                     kwt.restrict = origrestrict
 
     repo.__class__ = kwrepo
     repo._keywordkwt = kwt

hgext/largefiles/remotestore.py

@@ -1,128 +1,132
 # Copyright 2010-2011 Fog Creek Software
 # Copyright 2010-2011 Unity Technologies
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''remote largefile store; the base class for wirestore'''
 from __future__ import absolute_import
 
 from mercurial.i18n import _
 
 from mercurial import (
     error,
     util,
 )
 
+from mercurial.utils import (
+    stringutil,
+)
+
 from . import (
     basestore,
     lfutil,
     localstore,
 )
 
 urlerr = util.urlerr
 urlreq = util.urlreq
 
 class remotestore(basestore.basestore):
     '''a largefile store accessed over a network'''
     def __init__(self, ui, repo, url):
         super(remotestore, self).__init__(ui, repo, url)
         self._lstore = None
         if repo is not None:
             self._lstore = localstore.localstore(self.ui, self.repo, self.repo)
 
     def put(self, source, hash):
         if self.sendfile(source, hash):
             raise error.Abort(
                 _('remotestore: could not put %s to remote store %s')
                 % (source, util.hidepassword(self.url)))
         self.ui.debug(
             _('remotestore: put %s to remote store %s\n')
             % (source, util.hidepassword(self.url)))
 
     def exists(self, hashes):
         return dict((h, s == 0) for (h, s) in # dict-from-generator
                     self._stat(hashes).iteritems())
 
     def sendfile(self, filename, hash):
         self.ui.debug('remotestore: sendfile(%s, %s)\n' % (filename, hash))
         try:
             with lfutil.httpsendfile(self.ui, filename) as fd:
                 return self._put(hash, fd)
         except IOError as e:
             raise error.Abort(
                 _('remotestore: could not open file %s: %s')
-                % (filename, util.forcebytestr(e)))
+                % (filename, stringutil.forcebytestr(e)))
 
     def _getfile(self, tmpfile, filename, hash):
         try:
             chunks = self._get(hash)
         except urlerr.httperror as e:
             # 401s get converted to error.Aborts; everything else is fine being
             # turned into a StoreError
             raise basestore.StoreError(filename, hash, self.url,
-                                       util.forcebytestr(e))
+                                       stringutil.forcebytestr(e))
         except urlerr.urlerror as e:
             # This usually indicates a connection problem, so don't
             # keep trying with the other files... they will probably
             # all fail too.
             raise error.Abort('%s: %s' %
                              (util.hidepassword(self.url), e.reason))
         except IOError as e:
             raise basestore.StoreError(filename, hash, self.url,
-                                       util.forcebytestr(e))
+                                       stringutil.forcebytestr(e))
 
         return lfutil.copyandhash(chunks, tmpfile)
 
     def _hashesavailablelocally(self, hashes):
         existslocallymap = self._lstore.exists(hashes)
         localhashes = [hash for hash in hashes if existslocallymap[hash]]
         return localhashes
 
     def _verifyfiles(self, contents, filestocheck):
         failed = False
         expectedhashes = [expectedhash
                           for cset, filename, expectedhash in filestocheck]
         localhashes = self._hashesavailablelocally(expectedhashes)
         stats = self._stat([expectedhash for expectedhash in expectedhashes
                             if expectedhash not in localhashes])
 
         for cset, filename, expectedhash in filestocheck:
             if expectedhash in localhashes:
                 filetocheck = (cset, filename, expectedhash)
                 verifyresult = self._lstore._verifyfiles(contents,
                                                          [filetocheck])
                 if verifyresult:
                     failed = True
             else:
                 stat = stats[expectedhash]
                 if stat:
                     if stat == 1:
                         self.ui.warn(
                             _('changeset %s: %s: contents differ\n')
                             % (cset, filename))
                         failed = True
                     elif stat == 2:
                         self.ui.warn(
                             _('changeset %s: %s missing\n')
                             % (cset, filename))
                         failed = True
                     else:
                         raise RuntimeError('verify failed: unexpected response '
                                            'from statlfile (%r)' % stat)
         return failed
 
     def _put(self, hash, fd):
         '''Put file with the given hash in the remote store.'''
         raise NotImplementedError('abstract method')
 
     def _get(self, hash):
         '''Get a iterator for content with the given hash.'''
         raise NotImplementedError('abstract method')
 
     def _stat(self, hashes):
         '''Get information about availability of files specified by
         hashes in the remote store. Return dictionary mapping hashes
         to return code where 0 means that file is available, other
         values if not.'''
         raise NotImplementedError('abstract method')

hgext/lfs/wrapper.py

@@ -1,391 +1,395
 # wrapper.py - methods wrapping core mercurial logic
 #
 # Copyright 2017 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import hashlib
 
 from mercurial.i18n import _
 from mercurial.node import bin, hex, nullid, short
 
 from mercurial import (
     error,
     filelog,
     revlog,
     util,
 )
 
+from mercurial.utils import (
+    stringutil,
+)
+
 from ..largefiles import lfutil
 
 from . import (
     blobstore,
     pointer,
 )
 
 def supportedoutgoingversions(orig, repo):
     versions = orig(repo)
     if 'lfs' in repo.requirements:
         versions.discard('01')
         versions.discard('02')
     versions.add('03')
     return versions
 
 def allsupportedversions(orig, ui):
     versions = orig(ui)
     versions.add('03')
     return versions
 
 def _capabilities(orig, repo, proto):
     '''Wrap server command to announce lfs server capability'''
     caps = orig(repo, proto)
     # XXX: change to 'lfs=serve' when separate git server isn't required?
     caps.append('lfs')
     return caps
 
 def bypasscheckhash(self, text):
     return False
 
 def readfromstore(self, text):
     """Read filelog content from local blobstore transform for flagprocessor.
 
     Default tranform for flagprocessor, returning contents from blobstore.
     Returns a 2-typle (text, validatehash) where validatehash is True as the
     contents of the blobstore should be checked using checkhash.
     """
     p = pointer.deserialize(text)
     oid = p.oid()
     store = self.opener.lfslocalblobstore
     if not store.has(oid):
         p.filename = self.filename
         self.opener.lfsremoteblobstore.readbatch([p], store)
 
     # The caller will validate the content
     text = store.read(oid, verify=False)
 
     # pack hg filelog metadata
     hgmeta = {}
     for k in p.keys():
         if k.startswith('x-hg-'):
             name = k[len('x-hg-'):]
             hgmeta[name] = p[k]
     if hgmeta or text.startswith('\1\n'):
         text = filelog.packmeta(hgmeta, text)
 
     return (text, True)
 
 def writetostore(self, text):
     # hg filelog metadata (includes rename, etc)
     hgmeta, offset = filelog.parsemeta(text)
     if offset and offset > 0:
         # lfs blob does not contain hg filelog metadata
         text = text[offset:]
 
     # git-lfs only supports sha256
     oid = hex(hashlib.sha256(text).digest())
     self.opener.lfslocalblobstore.write(oid, text)
 
     # replace contents with metadata
     longoid = 'sha256:%s' % oid
     metadata = pointer.gitlfspointer(oid=longoid, size='%d' % len(text))
 
     # by default, we expect the content to be binary. however, LFS could also
     # be used for non-binary content. add a special entry for non-binary data.
     # this will be used by filectx.isbinary().
-    if not util.binary(text):
+    if not stringutil.binary(text):
         # not hg filelog metadata (affecting commit hash), no "x-hg-" prefix
         metadata['x-is-binary'] = '0'
 
     # translate hg filelog metadata to lfs metadata with "x-hg-" prefix
     if hgmeta is not None:
         for k, v in hgmeta.iteritems():
             metadata['x-hg-%s' % k] = v
 
     rawtext = metadata.serialize()
     return (rawtext, False)
 
 def _islfs(rlog, node=None, rev=None):
     if rev is None:
         if node is None:
             # both None - likely working copy content where node is not ready
             return False
         rev = rlog.rev(node)
     else:
         node = rlog.node(rev)
     if node == nullid:
         return False
     flags = rlog.flags(rev)
     return bool(flags & revlog.REVIDX_EXTSTORED)
 
 def filelogaddrevision(orig, self, text, transaction, link, p1, p2,
                        cachedelta=None, node=None,
                        flags=revlog.REVIDX_DEFAULT_FLAGS, **kwds):
     textlen = len(text)
     # exclude hg rename meta from file size
     meta, offset = filelog.parsemeta(text)
     if offset:
         textlen -= offset
 
     lfstrack = self.opener.options['lfstrack']
 
     if lfstrack(self.filename, textlen):
         flags |= revlog.REVIDX_EXTSTORED
 
     return orig(self, text, transaction, link, p1, p2, cachedelta=cachedelta,
                 node=node, flags=flags, **kwds)
 
 def filelogrenamed(orig, self, node):
     if _islfs(self, node):
         rawtext = self.revision(node, raw=True)
         if not rawtext:
             return False
         metadata = pointer.deserialize(rawtext)
         if 'x-hg-copy' in metadata and 'x-hg-copyrev' in metadata:
             return metadata['x-hg-copy'], bin(metadata['x-hg-copyrev'])
         else:
             return False
     return orig(self, node)
 
 def filelogsize(orig, self, rev):
     if _islfs(self, rev=rev):
         # fast path: use lfs metadata to answer size
         rawtext = self.revision(rev, raw=True)
         metadata = pointer.deserialize(rawtext)
         return int(metadata['size'])
     return orig(self, rev)
 
 def filectxcmp(orig, self, fctx):
     """returns True if text is different than fctx"""
     # some fctx (ex. hg-git) is not based on basefilectx and do not have islfs
     if self.islfs() and getattr(fctx, 'islfs', lambda: False)():
         # fast path: check LFS oid
         p1 = pointer.deserialize(self.rawdata())
         p2 = pointer.deserialize(fctx.rawdata())
         return p1.oid() != p2.oid()
     return orig(self, fctx)
 
 def filectxisbinary(orig, self):
     if self.islfs():
         # fast path: use lfs metadata to answer isbinary
         metadata = pointer.deserialize(self.rawdata())
         # if lfs metadata says nothing, assume it's binary by default
         return bool(int(metadata.get('x-is-binary', 1)))
     return orig(self)
 
 def filectxislfs(self):
     return _islfs(self.filelog(), self.filenode())
 
 def _updatecatformatter(orig, fm, ctx, matcher, path, decode):
     orig(fm, ctx, matcher, path, decode)
     fm.data(rawdata=ctx[path].rawdata())
 
 def convertsink(orig, sink):
     sink = orig(sink)
     if sink.repotype == 'hg':
         class lfssink(sink.__class__):
             def putcommit(self, files, copies, parents, commit, source, revmap,
                           full, cleanp2):
                 pc = super(lfssink, self).putcommit
                 node = pc(files, copies, parents, commit, source, revmap, full,
                           cleanp2)
 
                 if 'lfs' not in self.repo.requirements:
                     ctx = self.repo[node]
 
                     # The file list may contain removed files, so check for
                     # membership before assuming it is in the context.
                     if any(f in ctx and ctx[f].islfs() for f, n in files):
                         self.repo.requirements.add('lfs')
                         self.repo._writerequirements()
 
                         # Permanently enable lfs locally
                         self.repo.vfs.append(
                             'hgrc', util.tonativeeol('\n[extensions]\nlfs=\n'))
 
                 return node
 
         sink.__class__ = lfssink
 
     return sink
 
 def vfsinit(orig, self, othervfs):
     orig(self, othervfs)
     # copy lfs related options
     for k, v in othervfs.options.items():
         if k.startswith('lfs'):
             self.options[k] = v
     # also copy lfs blobstores. note: this can run before reposetup, so lfs
     # blobstore attributes are not always ready at this time.
     for name in ['lfslocalblobstore', 'lfsremoteblobstore']:
         if util.safehasattr(othervfs, name):
             setattr(self, name, getattr(othervfs, name))
 
 def hgclone(orig, ui, opts, *args, **kwargs):
     result = orig(ui, opts, *args, **kwargs)
 
     if result is not None:
         sourcerepo, destrepo = result
         repo = destrepo.local()
 
         # When cloning to a remote repo (like through SSH), no repo is available
         # from the peer.  Therefore the hgrc can't be updated.
         if not repo:
             return result
 
         # If lfs is required for this repo, permanently enable it locally
         if 'lfs' in repo.requirements:
             repo.vfs.append('hgrc',
                             util.tonativeeol('\n[extensions]\nlfs=\n'))
 
     return result
 
 def hgpostshare(orig, sourcerepo, destrepo, bookmarks=True, defaultpath=None):
     orig(sourcerepo, destrepo, bookmarks, defaultpath)
 
     # If lfs is required for this repo, permanently enable it locally
     if 'lfs' in destrepo.requirements:
         destrepo.vfs.append('hgrc', util.tonativeeol('\n[extensions]\nlfs=\n'))
 
 def _prefetchfiles(repo, ctx, files):
     """Ensure that required LFS blobs are present, fetching them as a group if
     needed."""
     pointers = []
     localstore = repo.svfs.lfslocalblobstore
 
     for f in files:
         p = pointerfromctx(ctx, f)
         if p and not localstore.has(p.oid()):
             p.filename = f
             pointers.append(p)
 
     if pointers:
         repo.svfs.lfsremoteblobstore.readbatch(pointers, localstore)
 
 def _canskipupload(repo):
     # if remotestore is a null store, upload is a no-op and can be skipped
     return isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote)
 
 def candownload(repo):
     # if remotestore is a null store, downloads will lead to nothing
     return not isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote)
 
 def uploadblobsfromrevs(repo, revs):
     '''upload lfs blobs introduced by revs
 
     Note: also used by other extensions e. g. infinitepush. avoid renaming.
     '''
     if _canskipupload(repo):
         return
     pointers = extractpointers(repo, revs)
     uploadblobs(repo, pointers)
 
 def prepush(pushop):
     """Prepush hook.
 
     Read through the revisions to push, looking for filelog entries that can be
     deserialized into metadata so that we can block the push on their upload to
     the remote blobstore.
     """
     return uploadblobsfromrevs(pushop.repo, pushop.outgoing.missing)
 
 def push(orig, repo, remote, *args, **kwargs):
     """bail on push if the extension isn't enabled on remote when needed"""
     if 'lfs' in repo.requirements:
         # If the remote peer is for a local repo, the requirement tests in the
         # base class method enforce lfs support.  Otherwise, some revisions in
         # this repo use lfs, and the remote repo needs the extension loaded.
         if not remote.local() and not remote.capable('lfs'):
             # This is a copy of the message in exchange.push() when requirements
             # are missing between local repos.
             m = _("required features are not supported in the destination: %s")
             raise error.Abort(m % 'lfs',
                               hint=_('enable the lfs extension on the server'))
     return orig(repo, remote, *args, **kwargs)
 
 def writenewbundle(orig, ui, repo, source, filename, bundletype, outgoing,
                    *args, **kwargs):
     """upload LFS blobs added by outgoing revisions on 'hg bundle'"""
     uploadblobsfromrevs(repo, outgoing.missing)
     return orig(ui, repo, source, filename, bundletype, outgoing, *args,
                 **kwargs)
 
 def extractpointers(repo, revs):
     """return a list of lfs pointers added by given revs"""
     repo.ui.debug('lfs: computing set of blobs to upload\n')
     pointers = {}
     for r in revs:
         ctx = repo[r]
         for p in pointersfromctx(ctx).values():
             pointers[p.oid()] = p
     return sorted(pointers.values())
 
 def pointerfromctx(ctx, f, removed=False):
     """return a pointer for the named file from the given changectx, or None if
     the file isn't LFS.
 
     Optionally, the pointer for a file deleted from the context can be returned.
     Since no such pointer is actually stored, and to distinguish from a non LFS
     file, this pointer is represented by an empty dict.
     """
     _ctx = ctx
     if f not in ctx:
         if not removed:
             return None
         if f in ctx.p1():
             _ctx = ctx.p1()
         elif f in ctx.p2():
             _ctx = ctx.p2()
         else:
             return None
     fctx = _ctx[f]
     if not _islfs(fctx.filelog(), fctx.filenode()):
         return None
     try:
         p = pointer.deserialize(fctx.rawdata())
         if ctx == _ctx:
             return p
         return {}
     except pointer.InvalidPointer as ex:
         raise error.Abort(_('lfs: corrupted pointer (%s@%s): %s\n')
                           % (f, short(_ctx.node()), ex))
 
 def pointersfromctx(ctx, removed=False):
     """return a dict {path: pointer} for given single changectx.
 
     If ``removed`` == True and the LFS file was removed from ``ctx``, the value
     stored for the path is an empty dict.
     """
     result = {}
     for f in ctx.files():
         p = pointerfromctx(ctx, f, removed=removed)
         if p is not None:
             result[f] = p
     return result
 
 def uploadblobs(repo, pointers):
     """upload given pointers from local blobstore"""
     if not pointers:
         return
 
     remoteblob = repo.svfs.lfsremoteblobstore
     remoteblob.writebatch(pointers, repo.svfs.lfslocalblobstore)
 
 def upgradefinishdatamigration(orig, ui, srcrepo, dstrepo, requirements):
     orig(ui, srcrepo, dstrepo, requirements)
 
     srclfsvfs = srcrepo.svfs.lfslocalblobstore.vfs
     dstlfsvfs = dstrepo.svfs.lfslocalblobstore.vfs
 
     for dirpath, dirs, files in srclfsvfs.walk():
         for oid in files:
             ui.write(_('copying lfs blob %s\n') % oid)
             lfutil.link(srclfsvfs.join(oid), dstlfsvfs.join(oid))
 
 def upgraderequirements(orig, repo):
     reqs = orig(repo)
     if 'lfs' in repo.requirements:
         reqs.add('lfs')
     return reqs

hgext/mq.py

@@ -1,3656 +1,3659
 # mq.py - patch queues for mercurial
 #
 # Copyright 2005, 2006 Chris Mason <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''manage a stack of patches
 
 This extension lets you work with a stack of patches in a Mercurial
 repository. It manages two stacks of patches - all known patches, and
 applied patches (subset of known patches).
 
 Known patches are represented as patch files in the .hg/patches
 directory. Applied patches are both patch files and changesets.
 
 Common tasks (use :hg:`help COMMAND` for more details)::
 
   create new patch                          qnew
   import existing patch                     qimport
 
   print patch series                        qseries
   print applied patches                     qapplied
 
   add known patch to applied stack          qpush
   remove patch from applied stack           qpop
   refresh contents of top applied patch     qrefresh
 
 By default, mq will automatically use git patches when required to
 avoid losing file mode changes, copy records, binary files or empty
 files creations or deletions. This behavior can be configured with::
 
   [mq]
   git = auto/keep/yes/no
 
 If set to 'keep', mq will obey the [diff] section configuration while
 preserving existing git patches upon qrefresh. If set to 'yes' or
 'no', mq will override the [diff] section and always generate git or
 regular patches, possibly losing data in the second case.
 
 It may be desirable for mq changesets to be kept in the secret phase (see
 :hg:`help phases`), which can be enabled with the following setting::
 
   [mq]
   secret = True
 
 You will by default be managing a patch queue named "patches". You can
 create other, independent patch queues with the :hg:`qqueue` command.
 
 If the working directory contains uncommitted files, qpush, qpop and
 qgoto abort immediately. If -f/--force is used, the changes are
 discarded. Setting::
 
   [mq]
   keepchanges = True
 
 make them behave as if --keep-changes were passed, and non-conflicting
 local changes will be tolerated and preserved. If incompatible options
 such as -f/--force or --exact are passed, this setting is ignored.
 
 This extension used to provide a strip command. This command now lives
 in the strip extension.
 '''
 
 from __future__ import absolute_import, print_function
 
 import errno
 import os
 import re
 import shutil
 from mercurial.i18n import _
 from mercurial.node import (
     bin,
     hex,
     nullid,
     nullrev,
     short,
 )
 from mercurial import (
     cmdutil,
     commands,
     dirstateguard,
     encoding,
     error,
     extensions,
     hg,
     localrepo,
     lock as lockmod,
     logcmdutil,
     patch as patchmod,
     phases,
     pycompat,
     registrar,
     revsetlang,
     scmutil,
     smartset,
     subrepoutil,
     util,
     vfs as vfsmod,
 )
-from mercurial.utils import dateutil
+from mercurial.utils import (
+    dateutil,
+    stringutil,
+)
 
 release = lockmod.release
 seriesopts = [('s', 'summary', None, _('print first line of patch header'))]
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 configitem('mq', 'git',
     default='auto',
 )
 configitem('mq', 'keepchanges',
     default=False,
 )
 configitem('mq', 'plain',
     default=False,
 )
 configitem('mq', 'secret',
     default=False,
 )
 
 # force load strip extension formerly included in mq and import some utility
 try:
     stripext = extensions.find('strip')
 except KeyError:
     # note: load is lazy so we could avoid the try-except,
     # but I (marmoute) prefer this explicit code.
     class dummyui(object):
         def debug(self, msg):
             pass
     stripext = extensions.load(dummyui(), 'strip', '')
 
 strip = stripext.strip
 checksubstate = stripext.checksubstate
 checklocalchanges = stripext.checklocalchanges
 
 
 # Patch names looks like unix-file names.
 # They must be joinable with queue directory and result in the patch path.
 normname = util.normpath
 
 class statusentry(object):
     def __init__(self, node, name):
         self.node, self.name = node, name
 
     def __bytes__(self):
         return hex(self.node) + ':' + self.name
 
     __str__ = encoding.strmethod(__bytes__)
     __repr__ = encoding.strmethod(__bytes__)
 
 # The order of the headers in 'hg export' HG patches:
 HGHEADERS = [
 #   '# HG changeset patch',
     '# User ',
     '# Date ',
     '#      ',
     '# Branch ',
     '# Node ID ',
     '# Parent  ', # can occur twice for merges - but that is not relevant for mq
     ]
 # The order of headers in plain 'mail style' patches:
 PLAINHEADERS = {
     'from': 0,
     'date': 1,
     'subject': 2,
     }
 
 def inserthgheader(lines, header, value):
     """Assuming lines contains a HG patch header, add a header line with value.
     >>> try: inserthgheader([], b'# Date ', b'z')
     ... except ValueError as inst: print("oops")
     oops
     >>> inserthgheader([b'# HG changeset patch'], b'# Date ', b'z')
     ['# HG changeset patch', '# Date z']
     >>> inserthgheader([b'# HG changeset patch', b''], b'# Date ', b'z')
     ['# HG changeset patch', '# Date z', '']
     >>> inserthgheader([b'# HG changeset patch', b'# User y'], b'# Date ', b'z')
     ['# HG changeset patch', '# User y', '# Date z']
     >>> inserthgheader([b'# HG changeset patch', b'# Date x', b'# User y'],
     ...                b'# User ', b'z')
     ['# HG changeset patch', '# Date x', '# User z']
     >>> inserthgheader([b'# HG changeset patch', b'# Date y'], b'# Date ', b'z')
     ['# HG changeset patch', '# Date z']
     >>> inserthgheader([b'# HG changeset patch', b'', b'# Date y'],
     ...                b'# Date ', b'z')
     ['# HG changeset patch', '# Date z', '', '# Date y']
     >>> inserthgheader([b'# HG changeset patch', b'# Parent  y'],
     ...                b'# Date ', b'z')
     ['# HG changeset patch', '# Date z', '# Parent  y']
     """
     start = lines.index('# HG changeset patch') + 1
     newindex = HGHEADERS.index(header)
     bestpos = len(lines)
     for i in range(start, len(lines)):
         line = lines[i]
         if not line.startswith('# '):
             bestpos = min(bestpos, i)
             break
         for lineindex, h in enumerate(HGHEADERS):
             if line.startswith(h):
                 if lineindex == newindex:
                     lines[i] = header + value
                     return lines
                 if lineindex > newindex:
                     bestpos = min(bestpos, i)
                 break # next line
     lines.insert(bestpos, header + value)
     return lines
 
 def insertplainheader(lines, header, value):
     """For lines containing a plain patch header, add a header line with value.
     >>> insertplainheader([], b'Date', b'z')
     ['Date: z']
     >>> insertplainheader([b''], b'Date', b'z')
     ['Date: z', '']
     >>> insertplainheader([b'x'], b'Date', b'z')
     ['Date: z', '', 'x']
     >>> insertplainheader([b'From: y', b'x'], b'Date', b'z')
     ['From: y', 'Date: z', '', 'x']
     >>> insertplainheader([b' date : x', b' from : y', b''], b'From', b'z')
     [' date : x', 'From: z', '']
     >>> insertplainheader([b'', b'Date: y'], b'Date', b'z')
     ['Date: z', '', 'Date: y']
     >>> insertplainheader([b'foo: bar', b'DATE: z', b'x'], b'From', b'y')
     ['From: y', 'foo: bar', 'DATE: z', '', 'x']
     """
     newprio = PLAINHEADERS[header.lower()]
     bestpos = len(lines)
     for i, line in enumerate(lines):
         if ':' in line:
             lheader = line.split(':', 1)[0].strip().lower()
             lprio = PLAINHEADERS.get(lheader, newprio + 1)
             if lprio == newprio:
                 lines[i] = '%s: %s' % (header, value)
                 return lines
             if lprio > newprio and i < bestpos:
                 bestpos = i
         else:
             if line:
                 lines.insert(i, '')
             if i < bestpos:
                 bestpos = i
             break
     lines.insert(bestpos, '%s: %s' % (header, value))
     return lines
 
 class patchheader(object):
     def __init__(self, pf, plainmode=False):
         def eatdiff(lines):
             while lines:
                 l = lines[-1]
                 if (l.startswith("diff -") or
                     l.startswith("Index:") or
                     l.startswith("===========")):
                     del lines[-1]
                 else:
                     break
         def eatempty(lines):
             while lines:
                 if not lines[-1].strip():
                     del lines[-1]
                 else:
                     break
 
         message = []
         comments = []
         user = None
         date = None
         parent = None
         format = None
         subject = None
         branch = None
         nodeid = None
         diffstart = 0
 
         for line in open(pf, 'rb'):
             line = line.rstrip()
             if (line.startswith('diff --git')
                 or (diffstart and line.startswith('+++ '))):
                 diffstart = 2
                 break
             diffstart = 0 # reset
             if line.startswith("--- "):
                 diffstart = 1
                 continue
             elif format == "hgpatch":
                 # parse values when importing the result of an hg export
                 if line.startswith("# User "):
                     user = line[7:]
                 elif line.startswith("# Date "):
                     date = line[7:]
                 elif line.startswith("# Parent "):
                     parent = line[9:].lstrip() # handle double trailing space
                 elif line.startswith("# Branch "):
                     branch = line[9:]
                 elif line.startswith("# Node ID "):
                     nodeid = line[10:]
                 elif not line.startswith("# ") and line:
                     message.append(line)
                     format = None
             elif line == '# HG changeset patch':
                 message = []
                 format = "hgpatch"
             elif (format != "tagdone" and (line.startswith("Subject: ") or
                                            line.startswith("subject: "))):
                 subject = line[9:]
                 format = "tag"
             elif (format != "tagdone" and (line.startswith("From: ") or
                                            line.startswith("from: "))):
                 user = line[6:]
                 format = "tag"
             elif (format != "tagdone" and (line.startswith("Date: ") or
                                            line.startswith("date: "))):
                 date = line[6:]
                 format = "tag"
             elif format == "tag" and line == "":
                 # when looking for tags (subject: from: etc) they
                 # end once you find a blank line in the source
                 format = "tagdone"
             elif message or line:
                 message.append(line)
             comments.append(line)
 
         eatdiff(message)
         eatdiff(comments)
         # Remember the exact starting line of the patch diffs before consuming
         # empty lines, for external use by TortoiseHg and others
         self.diffstartline = len(comments)
         eatempty(message)
         eatempty(comments)
 
         # make sure message isn't empty
         if format and format.startswith("tag") and subject:
             message.insert(0, subject)
 
         self.message = message
         self.comments = comments
         self.user = user
         self.date = date
         self.parent = parent
         # nodeid and branch are for external use by TortoiseHg and others
         self.nodeid = nodeid
         self.branch = branch
         self.haspatch = diffstart > 1
         self.plainmode = (plainmode or
                           '# HG changeset patch' not in self.comments and
                           any(c.startswith('Date: ') or
                                    c.startswith('From: ')
                                    for c in self.comments))
 
     def setuser(self, user):
         try:
             inserthgheader(self.comments, '# User ', user)
         except ValueError:
             if self.plainmode:
                 insertplainheader(self.comments, 'From', user)
             else:
                 tmp = ['# HG changeset patch', '# User ' + user]
                 self.comments = tmp + self.comments
         self.user = user
 
     def setdate(self, date):
         try:
             inserthgheader(self.comments, '# Date ', date)
         except ValueError:
             if self.plainmode:
                 insertplainheader(self.comments, 'Date', date)
             else:
                 tmp = ['# HG changeset patch', '# Date ' + date]
                 self.comments = tmp + self.comments
         self.date = date
 
     def setparent(self, parent):
         try:
             inserthgheader(self.comments, '# Parent  ', parent)
         except ValueError:
             if not self.plainmode:
                 tmp = ['# HG changeset patch', '# Parent  ' + parent]
                 self.comments = tmp + self.comments
         self.parent = parent
 
     def setmessage(self, message):
         if self.comments:
             self._delmsg()
         self.message = [message]
         if message:
             if self.plainmode and self.comments and self.comments[-1]:
                 self.comments.append('')
             self.comments.append(message)
 
     def __bytes__(self):
         s = '\n'.join(self.comments).rstrip()
         if not s:
             return ''
         return s + '\n\n'
 
     __str__ = encoding.strmethod(__bytes__)
 
     def _delmsg(self):
         '''Remove existing message, keeping the rest of the comments fields.
         If comments contains 'subject: ', message will prepend
         the field and a blank line.'''
         if self.message:
             subj = 'subject: ' + self.message[0].lower()
             for i in xrange(len(self.comments)):
                 if subj == self.comments[i].lower():
                     del self.comments[i]
                     self.message = self.message[2:]
                     break
         ci = 0
         for mi in self.message:
             while mi != self.comments[ci]:
                 ci += 1
             del self.comments[ci]
 
 def newcommit(repo, phase, *args, **kwargs):
     """helper dedicated to ensure a commit respect mq.secret setting
 
     It should be used instead of repo.commit inside the mq source for operation
     creating new changeset.
     """
     repo = repo.unfiltered()
     if phase is None:
         if repo.ui.configbool('mq', 'secret'):
             phase = phases.secret
     overrides = {('ui', 'allowemptycommit'): True}
     if phase is not None:
         overrides[('phases', 'new-commit')] = phase
     with repo.ui.configoverride(overrides, 'mq'):
         repo.ui.setconfig('ui', 'allowemptycommit', True)
         return repo.commit(*args, **kwargs)
 
 class AbortNoCleanup(error.Abort):
     pass
 
 class queue(object):
     def __init__(self, ui, baseui, path, patchdir=None):
         self.basepath = path
         try:
             with open(os.path.join(path, 'patches.queue'), r'rb') as fh:
                 cur = fh.read().rstrip()
 
             if not cur:
                 curpath = os.path.join(path, 'patches')
             else:
                 curpath = os.path.join(path, 'patches-' + cur)
         except IOError:
             curpath = os.path.join(path, 'patches')
         self.path = patchdir or curpath
         self.opener = vfsmod.vfs(self.path)
         self.ui = ui
         self.baseui = baseui
         self.applieddirty = False
         self.seriesdirty = False
         self.added = []
         self.seriespath = "series"
         self.statuspath = "status"
         self.guardspath = "guards"
         self.activeguards = None
         self.guardsdirty = False
         # Handle mq.git as a bool with extended values
         gitmode = ui.config('mq', 'git').lower()
-        boolmode = util.parsebool(gitmode)
+        boolmode = stringutil.parsebool(gitmode)
         if boolmode is not None:
             if boolmode:
                 gitmode = 'yes'
             else:
                 gitmode = 'no'
         self.gitmode = gitmode
         # deprecated config: mq.plain
         self.plainmode = ui.configbool('mq', 'plain')
         self.checkapplied = True
 
     @util.propertycache
     def applied(self):
         def parselines(lines):
             for l in lines:
                 entry = l.split(':', 1)
                 if len(entry) > 1:
                     n, name = entry
                     yield statusentry(bin(n), name)
                 elif l.strip():
                     self.ui.warn(_('malformated mq status line: %s\n') % entry)
                 # else we ignore empty lines
         try:
             lines = self.opener.read(self.statuspath).splitlines()
             return list(parselines(lines))
         except IOError as e:
             if e.errno == errno.ENOENT:
                 return []
             raise
 
     @util.propertycache
     def fullseries(self):
         try:
             return self.opener.read(self.seriespath).splitlines()
         except IOError as e:
             if e.errno == errno.ENOENT:
                 return []
             raise
 
     @util.propertycache
     def series(self):
         self.parseseries()
         return self.series
 
     @util.propertycache
     def seriesguards(self):
         self.parseseries()
         return self.seriesguards
 
     def invalidate(self):
         for a in 'applied fullseries series seriesguards'.split():
             if a in self.__dict__:
                 delattr(self, a)
         self.applieddirty = False
         self.seriesdirty = False
         self.guardsdirty = False
         self.activeguards = None
 
     def diffopts(self, opts=None, patchfn=None, plain=False):
         """Return diff options tweaked for this mq use, possibly upgrading to
         git format, and possibly plain and without lossy options."""
         diffopts = patchmod.difffeatureopts(self.ui, opts,
             git=True, whitespace=not plain, formatchanging=not plain)
         if self.gitmode == 'auto':
             diffopts.upgrade = True
         elif self.gitmode == 'keep':
             pass
         elif self.gitmode in ('yes', 'no'):
             diffopts.git = self.gitmode == 'yes'
         else:
             raise error.Abort(_('mq.git option can be auto/keep/yes/no'
                                ' got %s') % self.gitmode)
         if patchfn:
             diffopts = self.patchopts(diffopts, patchfn)
         return diffopts
 
     def patchopts(self, diffopts, *patches):
         """Return a copy of input diff options with git set to true if
         referenced patch is a git patch and should be preserved as such.
         """
         diffopts = diffopts.copy()
         if not diffopts.git and self.gitmode == 'keep':
             for patchfn in patches:
                 patchf = self.opener(patchfn, 'r')
                 # if the patch was a git patch, refresh it as a git patch
                 diffopts.git = any(line.startswith('diff --git')
                                    for line in patchf)
                 patchf.close()
         return diffopts
 
     def join(self, *p):
         return os.path.join(self.path, *p)
 
     def findseries(self, patch):
         def matchpatch(l):
             l = l.split('#', 1)[0]
             return l.strip() == patch
         for index, l in enumerate(self.fullseries):
             if matchpatch(l):
                 return index
         return None
 
     guard_re = re.compile(br'\s?#([-+][^-+# \t\r\n\f][^# \t\r\n\f]*)')
 
     def parseseries(self):
         self.series = []
         self.seriesguards = []
         for l in self.fullseries:
             h = l.find('#')
             if h == -1:
                 patch = l
                 comment = ''
             elif h == 0:
                 continue
             else:
                 patch = l[:h]
                 comment = l[h:]
             patch = patch.strip()
             if patch:
                 if patch in self.series:
                     raise error.Abort(_('%s appears more than once in %s') %
                                      (patch, self.join(self.seriespath)))
                 self.series.append(patch)
                 self.seriesguards.append(self.guard_re.findall(comment))
 
     def checkguard(self, guard):
         if not guard:
             return _('guard cannot be an empty string')
         bad_chars = '# \t\r\n\f'
         first = guard[0]
         if first in '-+':
             return (_('guard %r starts with invalid character: %r') %
                       (guard, first))
         for c in bad_chars:
             if c in guard:
                 return _('invalid character in guard %r: %r') % (guard, c)
 
     def setactive(self, guards):
         for guard in guards:
             bad = self.checkguard(guard)
             if bad:
                 raise error.Abort(bad)
         guards = sorted(set(guards))
         self.ui.debug('active guards: %s\n' % ' '.join(guards))
         self.activeguards = guards
         self.guardsdirty = True
 
     def active(self):
         if self.activeguards is None:
             self.activeguards = []
             try:
                 guards = self.opener.read(self.guardspath).split()
             except IOError as err:
                 if err.errno != errno.ENOENT:
                     raise
                 guards = []
             for i, guard in enumerate(guards):
                 bad = self.checkguard(guard)
                 if bad:
                     self.ui.warn('%s:%d: %s\n' %
                                  (self.join(self.guardspath), i + 1, bad))
                 else:
                     self.activeguards.append(guard)
         return self.activeguards
 
     def setguards(self, idx, guards):
         for g in guards:
             if len(g) < 2:
                 raise error.Abort(_('guard %r too short') % g)
             if g[0] not in '-+':
                 raise error.Abort(_('guard %r starts with invalid char') % g)
             bad = self.checkguard(g[1:])
             if bad:
                 raise error.Abort(bad)
         drop = self.guard_re.sub('', self.fullseries[idx])
         self.fullseries[idx] = drop + ''.join([' #' + g for g in guards])
         self.parseseries()
         self.seriesdirty = True
 
     def pushable(self, idx):
         if isinstance(idx, bytes):
             idx = self.series.index(idx)
         patchguards = self.seriesguards[idx]
         if not patchguards:
             return True, None
         guards = self.active()
         exactneg = [g for g in patchguards if g[0] == '-' and g[1:] in guards]
         if exactneg:
             return False, repr(exactneg[0])
         pos = [g for g in patchguards if g[0] == '+']
         exactpos = [g for g in pos if g[1:] in guards]
         if pos:
             if exactpos:
                 return True, repr(exactpos[0])
             return False, ' '.join(map(repr, pos))
         return True, ''
 
     def explainpushable(self, idx, all_patches=False):
         if all_patches:
             write = self.ui.write
         else:
             write = self.ui.warn
 
         if all_patches or self.ui.verbose:
             if isinstance(idx, str):
                 idx = self.series.index(idx)
             pushable, why = self.pushable(idx)
             if all_patches and pushable:
                 if why is None:
                     write(_('allowing %s - no guards in effect\n') %
                           self.series[idx])
                 else:
                     if not why:
                         write(_('allowing %s - no matching negative guards\n') %
                               self.series[idx])
                     else:
                         write(_('allowing %s - guarded by %s\n') %
                               (self.series[idx], why))
             if not pushable:
                 if why:
                     write(_('skipping %s - guarded by %s\n') %
                           (self.series[idx], why))
                 else:
                     write(_('skipping %s - no matching guards\n') %
                           self.series[idx])
 
     def savedirty(self):
         def writelist(items, path):
             fp = self.opener(path, 'wb')
             for i in items:
                 fp.write("%s\n" % i)
             fp.close()
         if self.applieddirty:
             writelist(map(bytes, self.applied), self.statuspath)
             self.applieddirty = False
         if self.seriesdirty:
             writelist(self.fullseries, self.seriespath)
             self.seriesdirty = False
         if self.guardsdirty:
             writelist(self.activeguards, self.guardspath)
             self.guardsdirty = False
         if self.added:
             qrepo = self.qrepo()
             if qrepo:
                 qrepo[None].add(f for f in self.added if f not in qrepo[None])
             self.added = []
 
     def removeundo(self, repo):
         undo = repo.sjoin('undo')
         if not os.path.exists(undo):
             return
         try:
             os.unlink(undo)
         except OSError as inst:
             self.ui.warn(_('error removing undo: %s\n') %
-                         util.forcebytestr(inst))
+                         stringutil.forcebytestr(inst))
 
     def backup(self, repo, files, copy=False):
         # backup local changes in --force case
         for f in sorted(files):
             absf = repo.wjoin(f)
             if os.path.lexists(absf):
                 self.ui.note(_('saving current version of %s as %s\n') %
                              (f, scmutil.origpath(self.ui, repo, f)))
 
                 absorig = scmutil.origpath(self.ui, repo, absf)
                 if copy:
                     util.copyfile(absf, absorig)
                 else:
                     util.rename(absf, absorig)
 
     def printdiff(self, repo, diffopts, node1, node2=None, files=None,
                   fp=None, changes=None, opts=None):
         if opts is None:
             opts = {}
         stat = opts.get('stat')
         m = scmutil.match(repo[node1], files, opts)
         logcmdutil.diffordiffstat(self.ui, repo, diffopts, node1, node2, m,
                                   changes, stat, fp)
 
     def mergeone(self, repo, mergeq, head, patch, rev, diffopts):
         # first try just applying the patch
         (err, n) = self.apply(repo, [patch], update_status=False,
                               strict=True, merge=rev)
 
         if err == 0:
             return (err, n)
 
         if n is None:
             raise error.Abort(_("apply failed for patch %s") % patch)
 
         self.ui.warn(_("patch didn't work out, merging %s\n") % patch)
 
         # apply failed, strip away that rev and merge.
         hg.clean(repo, head)
         strip(self.ui, repo, [n], update=False, backup=False)
 
         ctx = repo[rev]
         ret = hg.merge(repo, rev)
         if ret:
             raise error.Abort(_("update returned %d") % ret)
         n = newcommit(repo, None, ctx.description(), ctx.user(), force=True)
         if n is None:
             raise error.Abort(_("repo commit failed"))
         try:
             ph = patchheader(mergeq.join(patch), self.plainmode)
         except Exception:
             raise error.Abort(_("unable to read %s") % patch)
 
         diffopts = self.patchopts(diffopts, patch)
         patchf = self.opener(patch, "w")
         comments = bytes(ph)
         if comments:
             patchf.write(comments)
         self.printdiff(repo, diffopts, head, n, fp=patchf)
         patchf.close()
         self.removeundo(repo)
         return (0, n)
 
     def qparents(self, repo, rev=None):
         """return the mq handled parent or p1
 
         In some case where mq get himself in being the parent of a merge the
         appropriate parent may be p2.
         (eg: an in progress merge started with mq disabled)
 
         If no parent are managed by mq, p1 is returned.
         """
         if rev is None:
             (p1, p2) = repo.dirstate.parents()
             if p2 == nullid:
                 return p1
             if not self.applied:
                 return None
             return self.applied[-1].node
         p1, p2 = repo.changelog.parents(rev)
         if p2 != nullid and p2 in [x.node for x in self.applied]:
             return p2
         return p1
 
     def mergepatch(self, repo, mergeq, series, diffopts):
         if not self.applied:
             # each of the patches merged in will have two parents.  This
             # can confuse the qrefresh, qdiff, and strip code because it
             # needs to know which parent is actually in the patch queue.
             # so, we insert a merge marker with only one parent.  This way
             # the first patch in the queue is never a merge patch
             #
             pname = ".hg.patches.merge.marker"
             n = newcommit(repo, None, '[mq]: merge marker', force=True)
             self.removeundo(repo)
             self.applied.append(statusentry(n, pname))
             self.applieddirty = True
 
         head = self.qparents(repo)
 
         for patch in series:
             patch = mergeq.lookup(patch, strict=True)
             if not patch:
                 self.ui.warn(_("patch %s does not exist\n") % patch)
                 return (1, None)
             pushable, reason = self.pushable(patch)
             if not pushable:
                 self.explainpushable(patch, all_patches=True)
                 continue
             info = mergeq.isapplied(patch)
             if not info:
                 self.ui.warn(_("patch %s is not applied\n") % patch)
                 return (1, None)
             rev = info[1]
             err, head = self.mergeone(repo, mergeq, head, patch, rev, diffopts)
             if head:
                 self.applied.append(statusentry(head, patch))
                 self.applieddirty = True
             if err:
                 return (err, head)
         self.savedirty()
         return (0, head)
 
     def patch(self, repo, patchfile):
         '''Apply patchfile  to the working directory.
         patchfile: name of patch file'''
         files = set()
         try:
             fuzz = patchmod.patch(self.ui, repo, patchfile, strip=1,
                                   files=files, eolmode=None)
             return (True, list(files), fuzz)
         except Exception as inst:
-            self.ui.note(util.forcebytestr(inst) + '\n')
+            self.ui.note(stringutil.forcebytestr(inst) + '\n')
             if not self.ui.verbose:
                 self.ui.warn(_("patch failed, unable to continue (try -v)\n"))
             self.ui.traceback()
             return (False, list(files), False)
 
     def apply(self, repo, series, list=False, update_status=True,
               strict=False, patchdir=None, merge=None, all_files=None,
               tobackup=None, keepchanges=False):
         wlock = lock = tr = None
         try:
             wlock = repo.wlock()
             lock = repo.lock()
             tr = repo.transaction("qpush")
             try:
                 ret = self._apply(repo, series, list, update_status,
                                   strict, patchdir, merge, all_files=all_files,
                                   tobackup=tobackup, keepchanges=keepchanges)
                 tr.close()
                 self.savedirty()
                 return ret
             except AbortNoCleanup:
                 tr.close()
                 self.savedirty()
                 raise
             except: # re-raises
                 try:
                     tr.abort()
                 finally:
                     self.invalidate()
                 raise
         finally:
             release(tr, lock, wlock)
             self.removeundo(repo)
 
     def _apply(self, repo, series, list=False, update_status=True,
                strict=False, patchdir=None, merge=None, all_files=None,
                tobackup=None, keepchanges=False):
         """returns (error, hash)
 
         error = 1 for unable to read, 2 for patch failed, 3 for patch
         fuzz. tobackup is None or a set of files to backup before they
         are modified by a patch.
         """
         # TODO unify with commands.py
         if not patchdir:
             patchdir = self.path
         err = 0
         n = None
         for patchname in series:
             pushable, reason = self.pushable(patchname)
             if not pushable:
                 self.explainpushable(patchname, all_patches=True)
                 continue
             self.ui.status(_("applying %s\n") % patchname)
             pf = os.path.join(patchdir, patchname)
 
             try:
                 ph = patchheader(self.join(patchname), self.plainmode)
             except IOError:
                 self.ui.warn(_("unable to read %s\n") % patchname)
                 err = 1
                 break
 
             message = ph.message
             if not message:
                 # The commit message should not be translated
                 message = "imported patch %s\n" % patchname
             else:
                 if list:
                     # The commit message should not be translated
                     message.append("\nimported patch %s" % patchname)
                 message = '\n'.join(message)
 
             if ph.haspatch:
                 if tobackup:
                     touched = patchmod.changedfiles(self.ui, repo, pf)
                     touched = set(touched) & tobackup
                     if touched and keepchanges:
                         raise AbortNoCleanup(
                             _("conflicting local changes found"),
                             hint=_("did you forget to qrefresh?"))
                     self.backup(repo, touched, copy=True)
                     tobackup = tobackup - touched
                 (patcherr, files, fuzz) = self.patch(repo, pf)
                 if all_files is not None:
                     all_files.update(files)
                 patcherr = not patcherr
             else:
                 self.ui.warn(_("patch %s is empty\n") % patchname)
                 patcherr, files, fuzz = 0, [], 0
 
             if merge and files:
                 # Mark as removed/merged and update dirstate parent info
                 removed = []
                 merged = []
                 for f in files:
                     if os.path.lexists(repo.wjoin(f)):
                         merged.append(f)
                     else:
                         removed.append(f)
                 with repo.dirstate.parentchange():
                     for f in removed:
                         repo.dirstate.remove(f)
                     for f in merged:
                         repo.dirstate.merge(f)
                     p1, p2 = repo.dirstate.parents()
                     repo.setparents(p1, merge)
 
             if all_files and '.hgsubstate' in all_files:
                 wctx = repo[None]
                 pctx = repo['.']
                 overwrite = False
                 mergedsubstate = subrepoutil.submerge(repo, pctx, wctx, wctx,
                                                       overwrite)
                 files += mergedsubstate.keys()
 
             match = scmutil.matchfiles(repo, files or [])
             oldtip = repo['tip']
             n = newcommit(repo, None, message, ph.user, ph.date, match=match,
                           force=True)
             if repo['tip'] == oldtip:
                 raise error.Abort(_("qpush exactly duplicates child changeset"))
             if n is None:
                 raise error.Abort(_("repository commit failed"))
 
             if update_status:
                 self.applied.append(statusentry(n, patchname))
 
             if patcherr:
                 self.ui.warn(_("patch failed, rejects left in working "
                                "directory\n"))
                 err = 2
                 break
 
             if fuzz and strict:
                 self.ui.warn(_("fuzz found when applying patch, stopping\n"))
                 err = 3
                 break
         return (err, n)
 
     def _cleanup(self, patches, numrevs, keep=False):
         if not keep:
             r = self.qrepo()
             if r:
                 r[None].forget(patches)
             for p in patches:
                 try:
                     os.unlink(self.join(p))
                 except OSError as inst:
                     if inst.errno != errno.ENOENT:
                         raise
 
         qfinished = []
         if numrevs:
             qfinished = self.applied[:numrevs]
             del self.applied[:numrevs]
             self.applieddirty = True
 
         unknown = []
 
         for (i, p) in sorted([(self.findseries(p), p) for p in patches],
                              reverse=True):
             if i is not None:
                 del self.fullseries[i]
             else:
                 unknown.append(p)
 
         if unknown:
             if numrevs:
                 rev  = dict((entry.name, entry.node) for entry in qfinished)
                 for p in unknown:
                     msg = _('revision %s refers to unknown patches: %s\n')
                     self.ui.warn(msg % (short(rev[p]), p))
             else:
                 msg = _('unknown patches: %s\n')
                 raise error.Abort(''.join(msg % p for p in unknown))
 
         self.parseseries()
         self.seriesdirty = True
         return [entry.node for entry in qfinished]
 
     def _revpatches(self, repo, revs):
         firstrev = repo[self.applied[0].node].rev()
         patches = []
         for i, rev in enumerate(revs):
 
             if rev < firstrev:
                 raise error.Abort(_('revision %d is not managed') % rev)
 
             ctx = repo[rev]
             base = self.applied[i].node
             if ctx.node() != base:
                 msg = _('cannot delete revision %d above applied patches')
                 raise error.Abort(msg % rev)
 
             patch = self.applied[i].name
             for fmt in ('[mq]: %s', 'imported patch %s'):
                 if ctx.description() == fmt % patch:
                     msg = _('patch %s finalized without changeset message\n')
                     repo.ui.status(msg % patch)
                     break
 
             patches.append(patch)
         return patches
 
     def finish(self, repo, revs):
         # Manually trigger phase computation to ensure phasedefaults is
         # executed before we remove the patches.
         repo._phasecache
         patches = self._revpatches(repo, sorted(revs))
         qfinished = self._cleanup(patches, len(patches))
         if qfinished and repo.ui.configbool('mq', 'secret'):
             # only use this logic when the secret option is added
             oldqbase = repo[qfinished[0]]
             tphase = phases.newcommitphase(repo.ui)
             if oldqbase.phase() > tphase and oldqbase.p1().phase() <= tphase:
                 with repo.transaction('qfinish') as tr:
                     phases.advanceboundary(repo, tr, tphase, qfinished)
 
     def delete(self, repo, patches, opts):
         if not patches and not opts.get('rev'):
             raise error.Abort(_('qdelete requires at least one revision or '
                                'patch name'))
 
         realpatches = []
         for patch in patches:
             patch = self.lookup(patch, strict=True)
             info = self.isapplied(patch)
             if info:
                 raise error.Abort(_("cannot delete applied patch %s") % patch)
             if patch not in self.series:
                 raise error.Abort(_("patch %s not in series file") % patch)
             if patch not in realpatches:
                 realpatches.append(patch)
 
         numrevs = 0
         if opts.get('rev'):
             if not self.applied:
                 raise error.Abort(_('no patches applied'))
             revs = scmutil.revrange(repo, opts.get('rev'))
             revs.sort()
             revpatches = self._revpatches(repo, revs)
             realpatches += revpatches
             numrevs = len(revpatches)
 
         self._cleanup(realpatches, numrevs, opts.get('keep'))
 
     def checktoppatch(self, repo):
         '''check that working directory is at qtip'''
         if self.applied:
             top = self.applied[-1].node
             patch = self.applied[-1].name
             if repo.dirstate.p1() != top:
                 raise error.Abort(_("working directory revision is not qtip"))
             return top, patch
         return None, None
 
     def putsubstate2changes(self, substatestate, changes):
         for files in changes[:3]:
             if '.hgsubstate' in files:
                 return # already listed up
         # not yet listed up
         if substatestate in 'a?':
             changes[1].append('.hgsubstate')
         elif substatestate in 'r':
             changes[2].append('.hgsubstate')
         else: # modified
             changes[0].append('.hgsubstate')
 
     def checklocalchanges(self, repo, force=False, refresh=True):
         excsuffix = ''
         if refresh:
             excsuffix = ', qrefresh first'
             # plain versions for i18n tool to detect them
             _("local changes found, qrefresh first")
             _("local changed subrepos found, qrefresh first")
         return checklocalchanges(repo, force, excsuffix)
 
     _reserved = ('series', 'status', 'guards', '.', '..')
     def checkreservedname(self, name):
         if name in self._reserved:
             raise error.Abort(_('"%s" cannot be used as the name of a patch')
                              % name)
         if name != name.strip():
             # whitespace is stripped by parseseries()
             raise error.Abort(_('patch name cannot begin or end with '
                                 'whitespace'))
         for prefix in ('.hg', '.mq'):
             if name.startswith(prefix):
                 raise error.Abort(_('patch name cannot begin with "%s"')
                                  % prefix)
         for c in ('#', ':', '\r', '\n'):
             if c in name:
                 raise error.Abort(_('%r cannot be used in the name of a patch')
                                  % c)
 
     def checkpatchname(self, name, force=False):
         self.checkreservedname(name)
         if not force and os.path.exists(self.join(name)):
             if os.path.isdir(self.join(name)):
                 raise error.Abort(_('"%s" already exists as a directory')
                                  % name)
             else:
                 raise error.Abort(_('patch "%s" already exists') % name)
 
     def makepatchname(self, title, fallbackname):
         """Return a suitable filename for title, adding a suffix to make
         it unique in the existing list"""
         namebase = re.sub('[\s\W_]+', '_', title.lower()).strip('_')
         namebase = namebase[:75] # avoid too long name (issue5117)
         if namebase:
             try:
                 self.checkreservedname(namebase)
             except error.Abort:
                 namebase = fallbackname
         else:
             namebase = fallbackname
         name = namebase
         i = 0
         while True:
             if name not in self.fullseries:
                 try:
                     self.checkpatchname(name)
                     break
                 except error.Abort:
                     pass
             i += 1
             name = '%s__%d' % (namebase, i)
         return name
 
     def checkkeepchanges(self, keepchanges, force):
         if force and keepchanges:
             raise error.Abort(_('cannot use both --force and --keep-changes'))
 
     def new(self, repo, patchfn, *pats, **opts):
         """options:
            msg: a string or a no-argument function returning a string
         """
         opts = pycompat.byteskwargs(opts)
         msg = opts.get('msg')
         edit = opts.get('edit')
         editform = opts.get('editform', 'mq.qnew')
         user = opts.get('user')
         date = opts.get('date')
         if date:
             date = dateutil.parsedate(date)
         diffopts = self.diffopts({'git': opts.get('git')}, plain=True)
         if opts.get('checkname', True):
             self.checkpatchname(patchfn)
         inclsubs = checksubstate(repo)
         if inclsubs:
             substatestate = repo.dirstate['.hgsubstate']
         if opts.get('include') or opts.get('exclude') or pats:
             # detect missing files in pats
             def badfn(f, msg):
                 if f != '.hgsubstate': # .hgsubstate is auto-created
                     raise error.Abort('%s: %s' % (f, msg))
             match = scmutil.match(repo[None], pats, opts, badfn=badfn)
             changes = repo.status(match=match)
         else:
             changes = self.checklocalchanges(repo, force=True)
         commitfiles = list(inclsubs)
         for files in changes[:3]:
             commitfiles.extend(files)
         match = scmutil.matchfiles(repo, commitfiles)
         if len(repo[None].parents()) > 1:
             raise error.Abort(_('cannot manage merge changesets'))
         self.checktoppatch(repo)
         insert = self.fullseriesend()
         with repo.wlock():
             try:
                 # if patch file write fails, abort early
                 p = self.opener(patchfn, "w")
             except IOError as e:
                 raise error.Abort(_('cannot write patch "%s": %s')
                                  % (patchfn, encoding.strtolocal(e.strerror)))
             try:
                 defaultmsg = "[mq]: %s" % patchfn
                 editor = cmdutil.getcommiteditor(editform=editform)
                 if edit:
                     def finishdesc(desc):
                         if desc.rstrip():
                             return desc
                         else:
                             return defaultmsg
                     # i18n: this message is shown in editor with "HG: " prefix
                     extramsg = _('Leave message empty to use default message.')
                     editor = cmdutil.getcommiteditor(finishdesc=finishdesc,
                                                      extramsg=extramsg,
                                                      editform=editform)
                     commitmsg = msg
                 else:
                     commitmsg = msg or defaultmsg
 
                 n = newcommit(repo, None, commitmsg, user, date, match=match,
                               force=True, editor=editor)
                 if n is None:
                     raise error.Abort(_("repo commit failed"))
                 try:
                     self.fullseries[insert:insert] = [patchfn]
                     self.applied.append(statusentry(n, patchfn))
                     self.parseseries()
                     self.seriesdirty = True
                     self.applieddirty = True
                     nctx = repo[n]
                     ph = patchheader(self.join(patchfn), self.plainmode)
                     if user:
                         ph.setuser(user)
                     if date:
                         ph.setdate('%d %d' % date)
                     ph.setparent(hex(nctx.p1().node()))
                     msg = nctx.description().strip()
                     if msg == defaultmsg.strip():
                         msg = ''
                     ph.setmessage(msg)
                     p.write(bytes(ph))
                     if commitfiles:
                         parent = self.qparents(repo, n)
                         if inclsubs:
                             self.putsubstate2changes(substatestate, changes)
                         chunks = patchmod.diff(repo, node1=parent, node2=n,
                                                changes=changes, opts=diffopts)
                         for chunk in chunks:
                             p.write(chunk)
                     p.close()
                     r = self.qrepo()
                     if r:
                         r[None].add([patchfn])
                 except: # re-raises
                     repo.rollback()
                     raise
             except Exception:
                 patchpath = self.join(patchfn)
                 try:
                     os.unlink(patchpath)
                 except OSError:
                     self.ui.warn(_('error unlinking %s\n') % patchpath)
                 raise
             self.removeundo(repo)
 
     def isapplied(self, patch):
         """returns (index, rev, patch)"""
         for i, a in enumerate(self.applied):
             if a.name == patch:
                 return (i, a.node, a.name)
         return None
 
     # if the exact patch name does not exist, we try a few
     # variations.  If strict is passed, we try only #1
     #
     # 1) a number (as string) to indicate an offset in the series file
     # 2) a unique substring of the patch name was given
     # 3) patchname[-+]num to indicate an offset in the series file
     def lookup(self, patch, strict=False):
         def partialname(s):
             if s in self.series:
                 return s
             matches = [x for x in self.series if s in x]
             if len(matches) > 1:
                 self.ui.warn(_('patch name "%s" is ambiguous:\n') % s)
                 for m in matches:
                     self.ui.warn('  %s\n' % m)
                 return None
             if matches:
                 return matches[0]
             if self.series and self.applied:
                 if s == 'qtip':
                     return self.series[self.seriesend(True) - 1]
                 if s == 'qbase':
                     return self.series[0]
             return None
 
         if patch in self.series:
             return patch
 
         if not os.path.isfile(self.join(patch)):
             try:
                 sno = int(patch)
             except (ValueError, OverflowError):
                 pass
             else:
                 if -len(self.series) <= sno < len(self.series):
                     return self.series[sno]
 
             if not strict:
                 res = partialname(patch)
                 if res:
                     return res
                 minus = patch.rfind('-')
                 if minus >= 0:
                     res = partialname(patch[:minus])
                     if res:
                         i = self.series.index(res)
                         try:
                             off = int(patch[minus + 1:] or 1)
                         except (ValueError, OverflowError):
                             pass
                         else:
                             if i - off >= 0:
                                 return self.series[i - off]
                 plus = patch.rfind('+')
                 if plus >= 0:
                     res = partialname(patch[:plus])
                     if res:
                         i = self.series.index(res)
                         try:
                             off = int(patch[plus + 1:] or 1)
                         except (ValueError, OverflowError):
                             pass
                         else:
                             if i + off < len(self.series):
                                 return self.series[i + off]
         raise error.Abort(_("patch %s not in series") % patch)
 
     def push(self, repo, patch=None, force=False, list=False, mergeq=None,
              all=False, move=False, exact=False, nobackup=False,
              keepchanges=False):
         self.checkkeepchanges(keepchanges, force)
         diffopts = self.diffopts()
         with repo.wlock():
             heads = []
             for hs in repo.branchmap().itervalues():
                 heads.extend(hs)
             if not heads:
                 heads = [nullid]
             if repo.dirstate.p1() not in heads and not exact:
                 self.ui.status(_("(working directory not at a head)\n"))
 
             if not self.series:
                 self.ui.warn(_('no patches in series\n'))
                 return 0
 
             # Suppose our series file is: A B C and the current 'top'
             # patch is B. qpush C should be performed (moving forward)
             # qpush B is a NOP (no change) qpush A is an error (can't
             # go backwards with qpush)
             if patch:
                 patch = self.lookup(patch)
                 info = self.isapplied(patch)
                 if info and info[0] >= len(self.applied) - 1:
                     self.ui.warn(
                         _('qpush: %s is already at the top\n') % patch)
                     return 0
 
                 pushable, reason = self.pushable(patch)
                 if pushable:
                     if self.series.index(patch) < self.seriesend():
                         raise error.Abort(
                             _("cannot push to a previous patch: %s") % patch)
                 else:
                     if reason:
                         reason = _('guarded by %s') % reason
                     else:
                         reason = _('no matching guards')
                     self.ui.warn(_("cannot push '%s' - %s\n") % (patch, reason))
                     return 1
             elif all:
                 patch = self.series[-1]
                 if self.isapplied(patch):
                     self.ui.warn(_('all patches are currently applied\n'))
                     return 0
 
             # Following the above example, starting at 'top' of B:
             # qpush should be performed (pushes C), but a subsequent
             # qpush without an argument is an error (nothing to
             # apply). This allows a loop of "...while hg qpush..." to
             # work as it detects an error when done
             start = self.seriesend()
             if start == len(self.series):
                 self.ui.warn(_('patch series already fully applied\n'))
                 return 1
             if not force and not keepchanges:
                 self.checklocalchanges(repo, refresh=self.applied)
 
             if exact:
                 if keepchanges:
                     raise error.Abort(
                         _("cannot use --exact and --keep-changes together"))
                 if move:
                     raise error.Abort(_('cannot use --exact and --move '
                                        'together'))
                 if self.applied:
                     raise error.Abort(_('cannot push --exact with applied '
                                        'patches'))
                 root = self.series[start]
                 target = patchheader(self.join(root), self.plainmode).parent
                 if not target:
                     raise error.Abort(
                         _("%s does not have a parent recorded") % root)
                 if not repo[target] == repo['.']:
                     hg.update(repo, target)
 
             if move:
                 if not patch:
                     raise error.Abort(_("please specify the patch to move"))
                 for fullstart, rpn in enumerate(self.fullseries):
                     # strip markers for patch guards
                     if self.guard_re.split(rpn, 1)[0] == self.series[start]:
                         break
                 for i, rpn in enumerate(self.fullseries[fullstart:]):
                     # strip markers for patch guards
                     if self.guard_re.split(rpn, 1)[0] == patch:
                         break
                 index = fullstart + i
                 assert index < len(self.fullseries)
                 fullpatch = self.fullseries[index]
                 del self.fullseries[index]
                 self.fullseries.insert(fullstart, fullpatch)
                 self.parseseries()
                 self.seriesdirty = True
 
             self.applieddirty = True
             if start > 0:
                 self.checktoppatch(repo)
             if not patch:
                 patch = self.series[start]
                 end = start + 1
             else:
                 end = self.series.index(patch, start) + 1
 
             tobackup = set()
             if (not nobackup and force) or keepchanges:
                 status = self.checklocalchanges(repo, force=True)
                 if keepchanges:
                     tobackup.update(status.modified + status.added +
                                     status.removed + status.deleted)
                 else:
                     tobackup.update(status.modified + status.added)
 
             s = self.series[start:end]
             all_files = set()
             try:
                 if mergeq:
                     ret = self.mergepatch(repo, mergeq, s, diffopts)
                 else:
                     ret = self.apply(repo, s, list, all_files=all_files,
                                      tobackup=tobackup, keepchanges=keepchanges)
             except AbortNoCleanup:
                 raise
             except: # re-raises
                 self.ui.warn(_('cleaning up working directory...\n'))
                 cmdutil.revert(self.ui, repo, repo['.'],
                                repo.dirstate.parents(), no_backup=True)
                 # only remove unknown files that we know we touched or
                 # created while patching
                 for f in all_files:
                     if f not in repo.dirstate:
                         repo.wvfs.unlinkpath(f, ignoremissing=True)
                 self.ui.warn(_('done\n'))
                 raise
 
             if not self.applied:
                 return ret[0]
             top = self.applied[-1].name
             if ret[0] and ret[0] > 1:
                 msg = _("errors during apply, please fix and qrefresh %s\n")
                 self.ui.write(msg % top)
             else:
                 self.ui.write(_("now at: %s\n") % top)
             return ret[0]
 
     def pop(self, repo, patch=None, force=False, update=True, all=False,
             nobackup=False, keepchanges=False):
         self.checkkeepchanges(keepchanges, force)
         with repo.wlock():
             if patch:
                 # index, rev, patch
                 info = self.isapplied(patch)
                 if not info:
                     patch = self.lookup(patch)
                 info = self.isapplied(patch)
                 if not info:
                     raise error.Abort(_("patch %s is not applied") % patch)
 
             if not self.applied:
                 # Allow qpop -a to work repeatedly,
                 # but not qpop without an argument
                 self.ui.warn(_("no patches applied\n"))
                 return not all
 
             if all:
                 start = 0
             elif patch:
                 start = info[0] + 1
             else:
                 start = len(self.applied) - 1
 
             if start >= len(self.applied):
                 self.ui.warn(_("qpop: %s is already at the top\n") % patch)
                 return
 
             if not update:
                 parents = repo.dirstate.parents()
                 rr = [x.node for x in self.applied]
                 for p in parents:
                     if p in rr:
                         self.ui.warn(_("qpop: forcing dirstate update\n"))
                         update = True
             else:
                 parents = [p.node() for p in repo[None].parents()]
                 update = any(entry.node in parents
                              for entry in self.applied[start:])
 
             tobackup = set()
             if update:
                 s = self.checklocalchanges(repo, force=force or keepchanges)
                 if force:
                     if not nobackup:
                         tobackup.update(s.modified + s.added)
                 elif keepchanges:
                     tobackup.update(s.modified + s.added +
                                     s.removed + s.deleted)
 
             self.applieddirty = True
             end = len(self.applied)
             rev = self.applied[start].node
 
             try:
                 heads = repo.changelog.heads(rev)
             except error.LookupError:
                 node = short(rev)
                 raise error.Abort(_('trying to pop unknown node %s') % node)
 
             if heads != [self.applied[-1].node]:
                 raise error.Abort(_("popping would remove a revision not "
                                    "managed by this patch queue"))
             if not repo[self.applied[-1].node].mutable():
                 raise error.Abort(
                     _("popping would remove a public revision"),
                     hint=_("see 'hg help phases' for details"))
 
             # we know there are no local changes, so we can make a simplified
             # form of hg.update.
             if update:
                 qp = self.qparents(repo, rev)
                 ctx = repo[qp]
                 m, a, r, d = repo.status(qp, '.')[:4]
                 if d:
                     raise error.Abort(_("deletions found between repo revs"))
 
                 tobackup = set(a + m + r) & tobackup
                 if keepchanges and tobackup:
                     raise error.Abort(_("local changes found, qrefresh first"))
                 self.backup(repo, tobackup)
                 with repo.dirstate.parentchange():
                     for f in a:
                         repo.wvfs.unlinkpath(f, ignoremissing=True)
                         repo.dirstate.drop(f)
                     for f in m + r:
                         fctx = ctx[f]
                         repo.wwrite(f, fctx.data(), fctx.flags())
                         repo.dirstate.normal(f)
                     repo.setparents(qp, nullid)
             for patch in reversed(self.applied[start:end]):
                 self.ui.status(_("popping %s\n") % patch.name)
             del self.applied[start:end]
             strip(self.ui, repo, [rev], update=False, backup=False)
             for s, state in repo['.'].substate.items():
                 repo['.'].sub(s).get(state)
             if self.applied:
                 self.ui.write(_("now at: %s\n") % self.applied[-1].name)
             else:
                 self.ui.write(_("patch queue now empty\n"))
 
     def diff(self, repo, pats, opts):
         top, patch = self.checktoppatch(repo)
         if not top:
             self.ui.write(_("no patches applied\n"))
             return
         qp = self.qparents(repo, top)
         if opts.get('reverse'):
             node1, node2 = None, qp
         else:
             node1, node2 = qp, None
         diffopts = self.diffopts(opts, patch)
         self.printdiff(repo, diffopts, node1, node2, files=pats, opts=opts)
 
     def refresh(self, repo, pats=None, **opts):
         opts = pycompat.byteskwargs(opts)
         if not self.applied:
             self.ui.write(_("no patches applied\n"))
             return 1
         msg = opts.get('msg', '').rstrip()
         edit = opts.get('edit')
         editform = opts.get('editform', 'mq.qrefresh')
         newuser = opts.get('user')
         newdate = opts.get('date')
         if newdate:
             newdate = '%d %d' % dateutil.parsedate(newdate)
         wlock = repo.wlock()
 
         try:
             self.checktoppatch(repo)
             (top, patchfn) = (self.applied[-1].node, self.applied[-1].name)
             if repo.changelog.heads(top) != [top]:
                 raise error.Abort(_("cannot qrefresh a revision with children"))
             if not repo[top].mutable():
                 raise error.Abort(_("cannot qrefresh public revision"),
                                  hint=_("see 'hg help phases' for details"))
 
             cparents = repo.changelog.parents(top)
             patchparent = self.qparents(repo, top)
 
             inclsubs = checksubstate(repo, hex(patchparent))
             if inclsubs:
                 substatestate = repo.dirstate['.hgsubstate']
 
             ph = patchheader(self.join(patchfn), self.plainmode)
             diffopts = self.diffopts({'git': opts.get('git')}, patchfn,
                                      plain=True)
             if newuser:
                 ph.setuser(newuser)
             if newdate:
                 ph.setdate(newdate)
             ph.setparent(hex(patchparent))
 
             # only commit new patch when write is complete
             patchf = self.opener(patchfn, 'w', atomictemp=True)
 
             # update the dirstate in place, strip off the qtip commit
             # and then commit.
             #
             # this should really read:
             #   mm, dd, aa = repo.status(top, patchparent)[:3]
             # but we do it backwards to take advantage of manifest/changelog
             # caching against the next repo.status call
             mm, aa, dd = repo.status(patchparent, top)[:3]
             changes = repo.changelog.read(top)
             man = repo.manifestlog[changes[0]].read()
             aaa = aa[:]
             match1 = scmutil.match(repo[None], pats, opts)
             # in short mode, we only diff the files included in the
             # patch already plus specified files
             if opts.get('short'):
                 # if amending a patch, we start with existing
                 # files plus specified files - unfiltered
                 match = scmutil.matchfiles(repo, mm + aa + dd + match1.files())
                 # filter with include/exclude options
                 match1 = scmutil.match(repo[None], opts=opts)
             else:
                 match = scmutil.matchall(repo)
             m, a, r, d = repo.status(match=match)[:4]
             mm = set(mm)
             aa = set(aa)
             dd = set(dd)
 
             # we might end up with files that were added between
             # qtip and the dirstate parent, but then changed in the
             # local dirstate. in this case, we want them to only
             # show up in the added section
             for x in m:
                 if x not in aa:
                     mm.add(x)
             # we might end up with files added by the local dirstate that
             # were deleted by the patch.  In this case, they should only
             # show up in the changed section.
             for x in a:
                 if x in dd:
                     dd.remove(x)
                     mm.add(x)
                 else:
                     aa.add(x)
             # make sure any files deleted in the local dirstate
             # are not in the add or change column of the patch
             forget = []
             for x in d + r:
                 if x in aa:
                     aa.remove(x)
                     forget.append(x)
                     continue
                 else:
                     mm.discard(x)
                 dd.add(x)
 
             m = list(mm)
             r = list(dd)
             a = list(aa)
 
             # create 'match' that includes the files to be recommitted.
             # apply match1 via repo.status to ensure correct case handling.
             cm, ca, cr, cd = repo.status(patchparent, match=match1)[:4]
             allmatches = set(cm + ca + cr + cd)
             refreshchanges = [x.intersection(allmatches) for x in (mm, aa, dd)]
 
             files = set(inclsubs)
             for x in refreshchanges:
                 files.update(x)
             match = scmutil.matchfiles(repo, files)
 
             bmlist = repo[top].bookmarks()
 
             dsguard = None
             try:
                 dsguard = dirstateguard.dirstateguard(repo, 'mq.refresh')
                 if diffopts.git or diffopts.upgrade:
                     copies = {}
                     for dst in a:
                         src = repo.dirstate.copied(dst)
                         # during qfold, the source file for copies may
                         # be removed. Treat this as a simple add.
                         if src is not None and src in repo.dirstate:
                             copies.setdefault(src, []).append(dst)
                         repo.dirstate.add(dst)
                     # remember the copies between patchparent and qtip
                     for dst in aaa:
                         f = repo.file(dst)
                         src = f.renamed(man[dst])
                         if src:
                             copies.setdefault(src[0], []).extend(
                                 copies.get(dst, []))
                             if dst in a:
                                 copies[src[0]].append(dst)
                         # we can't copy a file created by the patch itself
                         if dst in copies:
                             del copies[dst]
                     for src, dsts in copies.iteritems():
                         for dst in dsts:
                             repo.dirstate.copy(src, dst)
                 else:
                     for dst in a:
                         repo.dirstate.add(dst)
                     # Drop useless copy information
                     for f in list(repo.dirstate.copies()):
                         repo.dirstate.copy(None, f)
                 for f in r:
                     repo.dirstate.remove(f)
                 # if the patch excludes a modified file, mark that
                 # file with mtime=0 so status can see it.
                 mm = []
                 for i in xrange(len(m) - 1, -1, -1):
                     if not match1(m[i]):
                         mm.append(m[i])
                         del m[i]
                 for f in m:
                     repo.dirstate.normal(f)
                 for f in mm:
                     repo.dirstate.normallookup(f)
                 for f in forget:
                     repo.dirstate.drop(f)
 
                 user = ph.user or changes[1]
 
                 oldphase = repo[top].phase()
 
                 # assumes strip can roll itself back if interrupted
                 repo.setparents(*cparents)
                 self.applied.pop()
                 self.applieddirty = True
                 strip(self.ui, repo, [top], update=False, backup=False)
                 dsguard.close()
             finally:
                 release(dsguard)
 
             try:
                 # might be nice to attempt to roll back strip after this
 
                 defaultmsg = "[mq]: %s" % patchfn
                 editor = cmdutil.getcommiteditor(editform=editform)
                 if edit:
                     def finishdesc(desc):
                         if desc.rstrip():
                             ph.setmessage(desc)
                             return desc
                         return defaultmsg
                     # i18n: this message is shown in editor with "HG: " prefix
                     extramsg = _('Leave message empty to use default message.')
                     editor = cmdutil.getcommiteditor(finishdesc=finishdesc,
                                                      extramsg=extramsg,
                                                      editform=editform)
                     message = msg or "\n".join(ph.message)
                 elif not msg:
                     if not ph.message:
                         message = defaultmsg
                     else:
                         message = "\n".join(ph.message)
                 else:
                     message = msg
                     ph.setmessage(msg)
 
                 # Ensure we create a new changeset in the same phase than
                 # the old one.
                 lock = tr = None
                 try:
                     lock = repo.lock()
                     tr = repo.transaction('mq')
                     n = newcommit(repo, oldphase, message, user, ph.date,
                               match=match, force=True, editor=editor)
                     # only write patch after a successful commit
                     c = [list(x) for x in refreshchanges]
                     if inclsubs:
                         self.putsubstate2changes(substatestate, c)
                     chunks = patchmod.diff(repo, patchparent,
                                            changes=c, opts=diffopts)
                     comments = bytes(ph)
                     if comments:
                         patchf.write(comments)
                     for chunk in chunks:
                         patchf.write(chunk)
                     patchf.close()
 
                     marks = repo._bookmarks
                     marks.applychanges(repo, tr, [(bm, n) for bm in bmlist])
                     tr.close()
 
                     self.applied.append(statusentry(n, patchfn))
                 finally:
                     lockmod.release(tr, lock)
             except: # re-raises
                 ctx = repo[cparents[0]]
                 repo.dirstate.rebuild(ctx.node(), ctx.manifest())
                 self.savedirty()
                 self.ui.warn(_('qrefresh interrupted while patch was popped! '
                                '(revert --all, qpush to recover)\n'))
                 raise
         finally:
             wlock.release()
             self.removeundo(repo)
 
     def init(self, repo, create=False):
         if not create and os.path.isdir(self.path):
             raise error.Abort(_("patch queue directory already exists"))
         try:
             os.mkdir(self.path)
         except OSError as inst:
             if inst.errno != errno.EEXIST or not create:
                 raise
         if create:
             return self.qrepo(create=True)
 
     def unapplied(self, repo, patch=None):
         if patch and patch not in self.series:
             raise error.Abort(_("patch %s is not in series file") % patch)
         if not patch:
             start = self.seriesend()
         else:
             start = self.series.index(patch) + 1
         unapplied = []
         for i in xrange(start, len(self.series)):
             pushable, reason = self.pushable(i)
             if pushable:
                 unapplied.append((i, self.series[i]))
             self.explainpushable(i)
         return unapplied
 
     def qseries(self, repo, missing=None, start=0, length=None, status=None,
                 summary=False):
         def displayname(pfx, patchname, state):
             if pfx:
                 self.ui.write(pfx)
             if summary:
                 ph = patchheader(self.join(patchname), self.plainmode)
                 if ph.message:
                     msg = ph.message[0]
                 else:
                     msg = ''
 
                 if self.ui.formatted():
                     width = self.ui.termwidth() - len(pfx) - len(patchname) - 2
                     if width > 0:
-                        msg = util.ellipsis(msg, width)
+                        msg = stringutil.ellipsis(msg, width)
                     else:
                         msg = ''
                 self.ui.write(patchname, label='qseries.' + state)
                 self.ui.write(': ')
                 self.ui.write(msg, label='qseries.message.' + state)
             else:
                 self.ui.write(patchname, label='qseries.' + state)
             self.ui.write('\n')
 
         applied = set([p.name for p in self.applied])
         if length is None:
             length = len(self.series) - start
         if not missing:
             if self.ui.verbose:
                 idxwidth = len("%d" % (start + length - 1))
             for i in xrange(start, start + length):
                 patch = self.series[i]
                 if patch in applied:
                     char, state = 'A', 'applied'
                 elif self.pushable(i)[0]:
                     char, state = 'U', 'unapplied'
                 else:
                     char, state = 'G', 'guarded'
                 pfx = ''
                 if self.ui.verbose:
                     pfx = '%*d %s ' % (idxwidth, i, char)
                 elif status and status != char:
                     continue
                 displayname(pfx, patch, state)
         else:
             msng_list = []
             for root, dirs, files in os.walk(self.path):
                 d = root[len(self.path) + 1:]
                 for f in files:
                     fl = os.path.join(d, f)
                     if (fl not in self.series and
                         fl not in (self.statuspath, self.seriespath,
                                    self.guardspath)
                         and not fl.startswith('.')):
                         msng_list.append(fl)
             for x in sorted(msng_list):
                 pfx = self.ui.verbose and ('D ') or ''
                 displayname(pfx, x, 'missing')
 
     def issaveline(self, l):
         if l.name == '.hg.patches.save.line':
             return True
 
     def qrepo(self, create=False):
         ui = self.baseui.copy()
         # copy back attributes set by ui.pager()
         if self.ui.pageractive and not ui.pageractive:
             ui.pageractive = self.ui.pageractive
             # internal config: ui.formatted
             ui.setconfig('ui', 'formatted',
                          self.ui.config('ui', 'formatted'), 'mqpager')
             ui.setconfig('ui', 'interactive',
                          self.ui.config('ui', 'interactive'), 'mqpager')
         if create or os.path.isdir(self.join(".hg")):
             return hg.repository(ui, path=self.path, create=create)
 
     def restore(self, repo, rev, delete=None, qupdate=None):
         desc = repo[rev].description().strip()
         lines = desc.splitlines()
         i = 0
         datastart = None
         series = []
         applied = []
         qpp = None
         for i, line in enumerate(lines):
             if line == 'Patch Data:':
                 datastart = i + 1
             elif line.startswith('Dirstate:'):
                 l = line.rstrip()
                 l = l[10:].split(' ')
                 qpp = [bin(x) for x in l]
             elif datastart is not None:
                 l = line.rstrip()
                 n, name = l.split(':', 1)
                 if n:
                     applied.append(statusentry(bin(n), name))
                 else:
                     series.append(l)
         if datastart is None:
             self.ui.warn(_("no saved patch data found\n"))
             return 1
         self.ui.warn(_("restoring status: %s\n") % lines[0])
         self.fullseries = series
         self.applied = applied
         self.parseseries()
         self.seriesdirty = True
         self.applieddirty = True
         heads = repo.changelog.heads()
         if delete:
             if rev not in heads:
                 self.ui.warn(_("save entry has children, leaving it alone\n"))
             else:
                 self.ui.warn(_("removing save entry %s\n") % short(rev))
                 pp = repo.dirstate.parents()
                 if rev in pp:
                     update = True
                 else:
                     update = False
                 strip(self.ui, repo, [rev], update=update, backup=False)
         if qpp:
             self.ui.warn(_("saved queue repository parents: %s %s\n") %
                          (short(qpp[0]), short(qpp[1])))
             if qupdate:
                 self.ui.status(_("updating queue directory\n"))
                 r = self.qrepo()
                 if not r:
                     self.ui.warn(_("unable to load queue repository\n"))
                     return 1
                 hg.clean(r, qpp[0])
 
     def save(self, repo, msg=None):
         if not self.applied:
             self.ui.warn(_("save: no patches applied, exiting\n"))
             return 1
         if self.issaveline(self.applied[-1]):
             self.ui.warn(_("status is already saved\n"))
             return 1
 
         if not msg:
             msg = _("hg patches saved state")
         else:
             msg = "hg patches: " + msg.rstrip('\r\n')
         r = self.qrepo()
         if r:
             pp = r.dirstate.parents()
             msg += "\nDirstate: %s %s" % (hex(pp[0]), hex(pp[1]))
         msg += "\n\nPatch Data:\n"
         msg += ''.join('%s\n' % x for x in self.applied)
         msg += ''.join(':%s\n' % x for x in self.fullseries)
         n = repo.commit(msg, force=True)
         if not n:
             self.ui.warn(_("repo commit failed\n"))
             return 1
         self.applied.append(statusentry(n, '.hg.patches.save.line'))
         self.applieddirty = True
         self.removeundo(repo)
 
     def fullseriesend(self):
         if self.applied:
             p = self.applied[-1].name
             end = self.findseries(p)
             if end is None:
                 return len(self.fullseries)
             return end + 1
         return 0
 
     def seriesend(self, all_patches=False):
         """If all_patches is False, return the index of the next pushable patch
         in the series, or the series length. If all_patches is True, return the
         index of the first patch past the last applied one.
         """
         end = 0
         def nextpatch(start):
             if all_patches or start >= len(self.series):
                 return start
             for i in xrange(start, len(self.series)):
                 p, reason = self.pushable(i)
                 if p:
                     return i
                 self.explainpushable(i)
             return len(self.series)
         if self.applied:
             p = self.applied[-1].name
             try:
                 end = self.series.index(p)
             except ValueError:
                 return 0
             return nextpatch(end + 1)
         return nextpatch(end)
 
     def appliedname(self, index):
         pname = self.applied[index].name
         if not self.ui.verbose:
             p = pname
         else:
             p = ("%d" % self.series.index(pname)) + " " + pname
         return p
 
     def qimport(self, repo, files, patchname=None, rev=None, existing=None,
                 force=None, git=False):
         def checkseries(patchname):
             if patchname in self.series:
                 raise error.Abort(_('patch %s is already in the series file')
                                  % patchname)
 
         if rev:
             if files:
                 raise error.Abort(_('option "-r" not valid when importing '
                                    'files'))
             rev = scmutil.revrange(repo, rev)
             rev.sort(reverse=True)
         elif not files:
             raise error.Abort(_('no files or revisions specified'))
         if (len(files) > 1 or len(rev) > 1) and patchname:
             raise error.Abort(_('option "-n" not valid when importing multiple '
                                'patches'))
         imported = []
         if rev:
             # If mq patches are applied, we can only import revisions
             # that form a linear path to qbase.
             # Otherwise, they should form a linear path to a head.
             heads = repo.changelog.heads(repo.changelog.node(rev.first()))
             if len(heads) > 1:
                 raise error.Abort(_('revision %d is the root of more than one '
                                    'branch') % rev.last())
             if self.applied:
                 base = repo.changelog.node(rev.first())
                 if base in [n.node for n in self.applied]:
                     raise error.Abort(_('revision %d is already managed')
                                      % rev.first())
                 if heads != [self.applied[-1].node]:
                     raise error.Abort(_('revision %d is not the parent of '
                                        'the queue') % rev.first())
                 base = repo.changelog.rev(self.applied[0].node)
                 lastparent = repo.changelog.parentrevs(base)[0]
             else:
                 if heads != [repo.changelog.node(rev.first())]:
                     raise error.Abort(_('revision %d has unmanaged children')
                                      % rev.first())
                 lastparent = None
 
             diffopts = self.diffopts({'git': git})
             with repo.transaction('qimport') as tr:
                 for r in rev:
                     if not repo[r].mutable():
                         raise error.Abort(_('revision %d is not mutable') % r,
                                          hint=_("see 'hg help phases' "
                                                 'for details'))
                     p1, p2 = repo.changelog.parentrevs(r)
                     n = repo.changelog.node(r)
                     if p2 != nullrev:
                         raise error.Abort(_('cannot import merge revision %d')
                                          % r)
                     if lastparent and lastparent != r:
                         raise error.Abort(_('revision %d is not the parent of '
                                            '%d')
                                          % (r, lastparent))
                     lastparent = p1
 
                     if not patchname:
                         patchname = self.makepatchname(
                             repo[r].description().split('\n', 1)[0],
                             '%d.diff' % r)
                     checkseries(patchname)
                     self.checkpatchname(patchname, force)
                     self.fullseries.insert(0, patchname)
 
                     patchf = self.opener(patchname, "w")
                     cmdutil.export(repo, [n], fp=patchf, opts=diffopts)
                     patchf.close()
 
                     se = statusentry(n, patchname)
                     self.applied.insert(0, se)
 
                     self.added.append(patchname)
                     imported.append(patchname)
                     patchname = None
                     if rev and repo.ui.configbool('mq', 'secret'):
                         # if we added anything with --rev, move the secret root
                         phases.retractboundary(repo, tr, phases.secret, [n])
                     self.parseseries()
                     self.applieddirty = True
                     self.seriesdirty = True
 
         for i, filename in enumerate(files):
             if existing:
                 if filename == '-':
                     raise error.Abort(_('-e is incompatible with import from -')
                                      )
                 filename = normname(filename)
                 self.checkreservedname(filename)
                 if util.url(filename).islocal():
                     originpath = self.join(filename)
                     if not os.path.isfile(originpath):
                         raise error.Abort(
                             _("patch %s does not exist") % filename)
 
                 if patchname:
                     self.checkpatchname(patchname, force)
 
                     self.ui.write(_('renaming %s to %s\n')
                                         % (filename, patchname))
                     util.rename(originpath, self.join(patchname))
                 else:
                     patchname = filename
 
             else:
                 if filename == '-' and not patchname:
                     raise error.Abort(_('need --name to import a patch from -'))
                 elif not patchname:
                     patchname = normname(os.path.basename(filename.rstrip('/')))
                 self.checkpatchname(patchname, force)
                 try:
                     if filename == '-':
                         text = self.ui.fin.read()
                     else:
                         fp = hg.openpath(self.ui, filename)
                         text = fp.read()
                         fp.close()
                 except (OSError, IOError):
                     raise error.Abort(_("unable to read file %s") % filename)
                 patchf = self.opener(patchname, "w")
                 patchf.write(text)
                 patchf.close()
             if not force:
                 checkseries(patchname)
             if patchname not in self.series:
                 index = self.fullseriesend() + i
                 self.fullseries[index:index] = [patchname]
             self.parseseries()
             self.seriesdirty = True
             self.ui.warn(_("adding %s to series file\n") % patchname)
             self.added.append(patchname)
             imported.append(patchname)
             patchname = None
 
         self.removeundo(repo)
         return imported
 
 def fixkeepchangesopts(ui, opts):
     if (not ui.configbool('mq', 'keepchanges') or opts.get('force')
         or opts.get('exact')):
         return opts
     opts = dict(opts)
     opts['keep_changes'] = True
     return opts
 
 @command("qdelete|qremove|qrm",
          [('k', 'keep', None, _('keep patch file')),
           ('r', 'rev', [],
            _('stop managing a revision (DEPRECATED)'), _('REV'))],
          _('hg qdelete [-k] [PATCH]...'))
 def delete(ui, repo, *patches, **opts):
     """remove patches from queue
 
     The patches must not be applied, and at least one patch is required. Exact
     patch identifiers must be given. With -k/--keep, the patch files are
     preserved in the patch directory.
 
     To stop managing a patch and move it into permanent history,
     use the :hg:`qfinish` command."""
     q = repo.mq
     q.delete(repo, patches, pycompat.byteskwargs(opts))
     q.savedirty()
     return 0
 
 @command("qapplied",
          [('1', 'last', None, _('show only the preceding applied patch'))
           ] + seriesopts,
          _('hg qapplied [-1] [-s] [PATCH]'))
 def applied(ui, repo, patch=None, **opts):
     """print the patches already applied
 
     Returns 0 on success."""
 
     q = repo.mq
     opts = pycompat.byteskwargs(opts)
 
     if patch:
         if patch not in q.series:
             raise error.Abort(_("patch %s is not in series file") % patch)
         end = q.series.index(patch) + 1
     else:
         end = q.seriesend(True)
 
     if opts.get('last') and not end:
         ui.write(_("no patches applied\n"))
         return 1
     elif opts.get('last') and end == 1:
         ui.write(_("only one patch applied\n"))
         return 1
     elif opts.get('last'):
         start = end - 2
         end = 1
     else:
         start = 0
 
     q.qseries(repo, length=end, start=start, status='A',
               summary=opts.get('summary'))
 
 
 @command("qunapplied",
          [('1', 'first', None, _('show only the first patch'))] + seriesopts,
          _('hg qunapplied [-1] [-s] [PATCH]'))
 def unapplied(ui, repo, patch=None, **opts):
     """print the patches not yet applied
 
     Returns 0 on success."""
 
     q = repo.mq
     opts = pycompat.byteskwargs(opts)
     if patch:
         if patch not in q.series:
             raise error.Abort(_("patch %s is not in series file") % patch)
         start = q.series.index(patch) + 1
     else:
         start = q.seriesend(True)
 
     if start == len(q.series) and opts.get('first'):
         ui.write(_("all patches applied\n"))
         return 1
 
     if opts.get('first'):
         length = 1
     else:
         length = None
     q.qseries(repo, start=start, length=length, status='U',
               summary=opts.get('summary'))
 
 @command("qimport",
          [('e', 'existing', None, _('import file in patch directory')),
           ('n', 'name', '',
            _('name of patch file'), _('NAME')),
           ('f', 'force', None, _('overwrite existing files')),
           ('r', 'rev', [],
            _('place existing revisions under mq control'), _('REV')),
           ('g', 'git', None, _('use git extended diff format')),
           ('P', 'push', None, _('qpush after importing'))],
          _('hg qimport [-e] [-n NAME] [-f] [-g] [-P] [-r REV]... [FILE]...'))
 def qimport(ui, repo, *filename, **opts):
     """import a patch or existing changeset
 
     The patch is inserted into the series after the last applied
     patch. If no patches have been applied, qimport prepends the patch
     to the series.
 
     The patch will have the same name as its source file unless you
     give it a new one with -n/--name.
 
     You can register an existing patch inside the patch directory with
     the -e/--existing flag.
 
     With -f/--force, an existing patch of the same name will be
     overwritten.
 
     An existing changeset may be placed under mq control with -r/--rev
     (e.g. qimport --rev . -n patch will place the current revision
     under mq control). With -g/--git, patches imported with --rev will
     use the git diff format. See the diffs help topic for information
     on why this is important for preserving rename/copy information
     and permission changes. Use :hg:`qfinish` to remove changesets
     from mq control.
 
     To import a patch from standard input, pass - as the patch file.
     When importing from standard input, a patch name must be specified
     using the --name flag.
 
     To import an existing patch while renaming it::
 
       hg qimport -e existing-patch -n new-name
 
     Returns 0 if import succeeded.
     """
     opts = pycompat.byteskwargs(opts)
     with repo.lock(): # cause this may move phase
         q = repo.mq
         try:
             imported = q.qimport(
                 repo, filename, patchname=opts.get('name'),
                 existing=opts.get('existing'), force=opts.get('force'),
                 rev=opts.get('rev'), git=opts.get('git'))
         finally:
             q.savedirty()
 
     if imported and opts.get('push') and not opts.get('rev'):
         return q.push(repo, imported[-1])
     return 0
 
 def qinit(ui, repo, create):
     """initialize a new queue repository
 
     This command also creates a series file for ordering patches, and
     an mq-specific .hgignore file in the queue repository, to exclude
     the status and guards files (these contain mostly transient state).
 
     Returns 0 if initialization succeeded."""
     q = repo.mq
     r = q.init(repo, create)
     q.savedirty()
     if r:
         if not os.path.exists(r.wjoin('.hgignore')):
             fp = r.wvfs('.hgignore', 'w')
             fp.write('^\\.hg\n')
             fp.write('^\\.mq\n')
             fp.write('syntax: glob\n')
             fp.write('status\n')
             fp.write('guards\n')
             fp.close()
         if not os.path.exists(r.wjoin('series')):
             r.wvfs('series', 'w').close()
         r[None].add(['.hgignore', 'series'])
         commands.add(ui, r)
     return 0
 
 @command("^qinit",
          [('c', 'create-repo', None, _('create queue repository'))],
          _('hg qinit [-c]'))
 def init(ui, repo, **opts):
     """init a new queue repository (DEPRECATED)
 
     The queue repository is unversioned by default. If
     -c/--create-repo is specified, qinit will create a separate nested
     repository for patches (qinit -c may also be run later to convert
     an unversioned patch repository into a versioned one). You can use
     qcommit to commit changes to this queue repository.
 
     This command is deprecated. Without -c, it's implied by other relevant
     commands. With -c, use :hg:`init --mq` instead."""
     return qinit(ui, repo, create=opts.get(r'create_repo'))
 
 @command("qclone",
          [('', 'pull', None, _('use pull protocol to copy metadata')),
           ('U', 'noupdate', None,
            _('do not update the new working directories')),
           ('', 'uncompressed', None,
            _('use uncompressed transfer (fast over LAN)')),
           ('p', 'patches', '',
            _('location of source patch repository'), _('REPO')),
          ] + cmdutil.remoteopts,
          _('hg qclone [OPTION]... SOURCE [DEST]'),
          norepo=True)
 def clone(ui, source, dest=None, **opts):
     '''clone main and patch repository at same time
 
     If source is local, destination will have no patches applied. If
     source is remote, this command can not check if patches are
     applied in source, so cannot guarantee that patches are not
     applied in destination. If you clone remote repository, be sure
     before that it has no patches applied.
 
     Source patch repository is looked for in <src>/.hg/patches by
     default. Use -p <url> to change.
 
     The patch directory must be a nested Mercurial repository, as
     would be created by :hg:`init --mq`.
 
     Return 0 on success.
     '''
     opts = pycompat.byteskwargs(opts)
     def patchdir(repo):
         """compute a patch repo url from a repo object"""
         url = repo.url()
         if url.endswith('/'):
             url = url[:-1]
         return url + '/.hg/patches'
 
     # main repo (destination and sources)
     if dest is None:
         dest = hg.defaultdest(source)
     sr = hg.peer(ui, opts, ui.expandpath(source))
 
     # patches repo (source only)
     if opts.get('patches'):
         patchespath = ui.expandpath(opts.get('patches'))
     else:
         patchespath = patchdir(sr)
     try:
         hg.peer(ui, opts, patchespath)
     except error.RepoError:
         raise error.Abort(_('versioned patch repository not found'
                            ' (see init --mq)'))
     qbase, destrev = None, None
     if sr.local():
         repo = sr.local()
         if repo.mq.applied and repo[qbase].phase() != phases.secret:
             qbase = repo.mq.applied[0].node
             if not hg.islocal(dest):
                 heads = set(repo.heads())
                 destrev = list(heads.difference(repo.heads(qbase)))
                 destrev.append(repo.changelog.parents(qbase)[0])
     elif sr.capable('lookup'):
         try:
             qbase = sr.lookup('qbase')
         except error.RepoError:
             pass
 
     ui.note(_('cloning main repository\n'))
     sr, dr = hg.clone(ui, opts, sr.url(), dest,
                       pull=opts.get('pull'),
                       rev=destrev,
                       update=False,
                       stream=opts.get('uncompressed'))
 
     ui.note(_('cloning patch repository\n'))
     hg.clone(ui, opts, opts.get('patches') or patchdir(sr), patchdir(dr),
              pull=opts.get('pull'), update=not opts.get('noupdate'),
              stream=opts.get('uncompressed'))
 
     if dr.local():
         repo = dr.local()
         if qbase:
             ui.note(_('stripping applied patches from destination '
                       'repository\n'))
             strip(ui, repo, [qbase], update=False, backup=None)
         if not opts.get('noupdate'):
             ui.note(_('updating destination repository\n'))
             hg.update(repo, repo.changelog.tip())
 
 @command("qcommit|qci",
          commands.table["^commit|ci"][1],
          _('hg qcommit [OPTION]... [FILE]...'),
          inferrepo=True)
 def commit(ui, repo, *pats, **opts):
     """commit changes in the queue repository (DEPRECATED)
 
     This command is deprecated; use :hg:`commit --mq` instead."""
     q = repo.mq
     r = q.qrepo()
     if not r:
         raise error.Abort('no queue repository')
     commands.commit(r.ui, r, *pats, **opts)
 
 @command("qseries",
          [('m', 'missing', None, _('print patches not in series')),
          ] + seriesopts,
           _('hg qseries [-ms]'))
 def series(ui, repo, **opts):
     """print the entire series file
 
     Returns 0 on success."""
     repo.mq.qseries(repo, missing=opts.get(r'missing'),
                     summary=opts.get(r'summary'))
     return 0
 
 @command("qtop", seriesopts, _('hg qtop [-s]'))
 def top(ui, repo, **opts):
     """print the name of the current patch
 
     Returns 0 on success."""
     q = repo.mq
     if q.applied:
         t = q.seriesend(True)
     else:
         t = 0
 
     if t:
         q.qseries(repo, start=t - 1, length=1, status='A',
                   summary=opts.get(r'summary'))
     else:
         ui.write(_("no patches applied\n"))
         return 1
 
 @command("qnext", seriesopts, _('hg qnext [-s]'))
 def next(ui, repo, **opts):
     """print the name of the next pushable patch
 
     Returns 0 on success."""
     q = repo.mq
     end = q.seriesend()
     if end == len(q.series):
         ui.write(_("all patches applied\n"))
         return 1
     q.qseries(repo, start=end, length=1, summary=opts.get(r'summary'))
 
 @command("qprev", seriesopts, _('hg qprev [-s]'))
 def prev(ui, repo, **opts):
     """print the name of the preceding applied patch
 
     Returns 0 on success."""
     q = repo.mq
     l = len(q.applied)
     if l == 1:
         ui.write(_("only one patch applied\n"))
         return 1
     if not l:
         ui.write(_("no patches applied\n"))
         return 1
     idx = q.series.index(q.applied[-2].name)
     q.qseries(repo, start=idx, length=1, status='A',
               summary=opts.get(r'summary'))
 
 def setupheaderopts(ui, opts):
     if not opts.get('user') and opts.get('currentuser'):
         opts['user'] = ui.username()
     if not opts.get('date') and opts.get('currentdate'):
         opts['date'] = "%d %d" % dateutil.makedate()
 
 @command("^qnew",
          [('e', 'edit', None, _('invoke editor on commit messages')),
           ('f', 'force', None, _('import uncommitted changes (DEPRECATED)')),
           ('g', 'git', None, _('use git extended diff format')),
           ('U', 'currentuser', None, _('add "From: <current user>" to patch')),
           ('u', 'user', '',
            _('add "From: <USER>" to patch'), _('USER')),
           ('D', 'currentdate', None, _('add "Date: <current date>" to patch')),
           ('d', 'date', '',
            _('add "Date: <DATE>" to patch'), _('DATE'))
           ] + cmdutil.walkopts + cmdutil.commitopts,
          _('hg qnew [-e] [-m TEXT] [-l FILE] PATCH [FILE]...'),
          inferrepo=True)
 def new(ui, repo, patch, *args, **opts):
     """create a new patch
 
     qnew creates a new patch on top of the currently-applied patch (if
     any). The patch will be initialized with any outstanding changes
     in the working directory. You may also use -I/--include,
     -X/--exclude, and/or a list of files after the patch name to add
     only changes to matching files to the new patch, leaving the rest
     as uncommitted modifications.
 
     -u/--user and -d/--date can be used to set the (given) user and
     date, respectively. -U/--currentuser and -D/--currentdate set user
     to current user and date to current date.
 
     -e/--edit, -m/--message or -l/--logfile set the patch header as
     well as the commit message. If none is specified, the header is
     empty and the commit message is '[mq]: PATCH'.
 
     Use the -g/--git option to keep the patch in the git extended diff
     format. Read the diffs help topic for more information on why this
     is important for preserving permission changes and copy/rename
     information.
 
     Returns 0 on successful creation of a new patch.
     """
     opts = pycompat.byteskwargs(opts)
     msg = cmdutil.logmessage(ui, opts)
     q = repo.mq
     opts['msg'] = msg
     setupheaderopts(ui, opts)
     q.new(repo, patch, *args, **pycompat.strkwargs(opts))
     q.savedirty()
     return 0
 
 @command("^qrefresh",
          [('e', 'edit', None, _('invoke editor on commit messages')),
           ('g', 'git', None, _('use git extended diff format')),
           ('s', 'short', None,
            _('refresh only files already in the patch and specified files')),
           ('U', 'currentuser', None,
            _('add/update author field in patch with current user')),
           ('u', 'user', '',
            _('add/update author field in patch with given user'), _('USER')),
           ('D', 'currentdate', None,
            _('add/update date field in patch with current date')),
           ('d', 'date', '',
            _('add/update date field in patch with given date'), _('DATE'))
           ] + cmdutil.walkopts + cmdutil.commitopts,
          _('hg qrefresh [-I] [-X] [-e] [-m TEXT] [-l FILE] [-s] [FILE]...'),
          inferrepo=True)
 def refresh(ui, repo, *pats, **opts):
     """update the current patch
 
     If any file patterns are provided, the refreshed patch will
     contain only the modifications that match those patterns; the
     remaining modifications will remain in the working directory.
 
     If -s/--short is specified, files currently included in the patch
     will be refreshed just like matched files and remain in the patch.
 
     If -e/--edit is specified, Mercurial will start your configured editor for
     you to enter a message. In case qrefresh fails, you will find a backup of
     your message in ``.hg/last-message.txt``.
 
     hg add/remove/copy/rename work as usual, though you might want to
     use git-style patches (-g/--git or [diff] git=1) to track copies
     and renames. See the diffs help topic for more information on the
     git diff format.
 
     Returns 0 on success.
     """
     opts = pycompat.byteskwargs(opts)
     q = repo.mq
     message = cmdutil.logmessage(ui, opts)
     setupheaderopts(ui, opts)
     with repo.wlock():
         ret = q.refresh(repo, pats, msg=message, **pycompat.strkwargs(opts))
         q.savedirty()
         return ret
 
 @command("^qdiff",
          cmdutil.diffopts + cmdutil.diffopts2 + cmdutil.walkopts,
          _('hg qdiff [OPTION]... [FILE]...'),
          inferrepo=True)
 def diff(ui, repo, *pats, **opts):
     """diff of the current patch and subsequent modifications
 
     Shows a diff which includes the current patch as well as any
     changes which have been made in the working directory since the
     last refresh (thus showing what the current patch would become
     after a qrefresh).
 
     Use :hg:`diff` if you only want to see the changes made since the
     last qrefresh, or :hg:`export qtip` if you want to see changes
     made by the current patch without including changes made since the
     qrefresh.
 
     Returns 0 on success.
     """
     ui.pager('qdiff')
     repo.mq.diff(repo, pats, pycompat.byteskwargs(opts))
     return 0
 
 @command('qfold',
          [('e', 'edit', None, _('invoke editor on commit messages')),
           ('k', 'keep', None, _('keep folded patch files')),
          ] + cmdutil.commitopts,
          _('hg qfold [-e] [-k] [-m TEXT] [-l FILE] PATCH...'))
 def fold(ui, repo, *files, **opts):
     """fold the named patches into the current patch
 
     Patches must not yet be applied. Each patch will be successively
     applied to the current patch in the order given. If all the
     patches apply successfully, the current patch will be refreshed
     with the new cumulative patch, and the folded patches will be
     deleted. With -k/--keep, the folded patch files will not be
     removed afterwards.
 
     The header for each folded patch will be concatenated with the
     current patch header, separated by a line of ``* * *``.
 
     Returns 0 on success."""
     opts = pycompat.byteskwargs(opts)
     q = repo.mq
     if not files:
         raise error.Abort(_('qfold requires at least one patch name'))
     if not q.checktoppatch(repo)[0]:
         raise error.Abort(_('no patches applied'))
     q.checklocalchanges(repo)
 
     message = cmdutil.logmessage(ui, opts)
 
     parent = q.lookup('qtip')
     patches = []
     messages = []
     for f in files:
         p = q.lookup(f)
         if p in patches or p == parent:
             ui.warn(_('skipping already folded patch %s\n') % p)
         if q.isapplied(p):
             raise error.Abort(_('qfold cannot fold already applied patch %s')
                              % p)
         patches.append(p)
 
     for p in patches:
         if not message:
             ph = patchheader(q.join(p), q.plainmode)
             if ph.message:
                 messages.append(ph.message)
         pf = q.join(p)
         (patchsuccess, files, fuzz) = q.patch(repo, pf)
         if not patchsuccess:
             raise error.Abort(_('error folding patch %s') % p)
 
     if not message:
         ph = patchheader(q.join(parent), q.plainmode)
         message = ph.message
         for msg in messages:
             if msg:
                 if message:
                     message.append('* * *')
                 message.extend(msg)
         message = '\n'.join(message)
 
     diffopts = q.patchopts(q.diffopts(), *patches)
     with repo.wlock():
         q.refresh(repo, msg=message, git=diffopts.git, edit=opts.get('edit'),
                   editform='mq.qfold')
         q.delete(repo, patches, opts)
         q.savedirty()
 
 @command("qgoto",
          [('', 'keep-changes', None,
            _('tolerate non-conflicting local changes')),
           ('f', 'force', None, _('overwrite any local changes')),
           ('', 'no-backup', None, _('do not save backup copies of files'))],
          _('hg qgoto [OPTION]... PATCH'))
 def goto(ui, repo, patch, **opts):
     '''push or pop patches until named patch is at top of stack
 
     Returns 0 on success.'''
     opts = pycompat.byteskwargs(opts)
     opts = fixkeepchangesopts(ui, opts)
     q = repo.mq
     patch = q.lookup(patch)
     nobackup = opts.get('no_backup')
     keepchanges = opts.get('keep_changes')
     if q.isapplied(patch):
         ret = q.pop(repo, patch, force=opts.get('force'), nobackup=nobackup,
                     keepchanges=keepchanges)
     else:
         ret = q.push(repo, patch, force=opts.get('force'), nobackup=nobackup,
                      keepchanges=keepchanges)
     q.savedirty()
     return ret
 
 @command("qguard",
          [('l', 'list', None, _('list all patches and guards')),
           ('n', 'none', None, _('drop all guards'))],
          _('hg qguard [-l] [-n] [PATCH] [-- [+GUARD]... [-GUARD]...]'))
 def guard(ui, repo, *args, **opts):
     '''set or print guards for a patch
 
     Guards control whether a patch can be pushed. A patch with no
     guards is always pushed. A patch with a positive guard ("+foo") is
     pushed only if the :hg:`qselect` command has activated it. A patch with
     a negative guard ("-foo") is never pushed if the :hg:`qselect` command
     has activated it.
 
     With no arguments, print the currently active guards.
     With arguments, set guards for the named patch.
 
     .. note::
 
        Specifying negative guards now requires '--'.
 
     To set guards on another patch::
 
       hg qguard other.patch -- +2.6.17 -stable
 
     Returns 0 on success.
     '''
     def status(idx):
         guards = q.seriesguards[idx] or ['unguarded']
         if q.series[idx] in applied:
             state = 'applied'
         elif q.pushable(idx)[0]:
             state = 'unapplied'
         else:
             state = 'guarded'
         label = 'qguard.patch qguard.%s qseries.%s' % (state, state)
         ui.write('%s: ' % ui.label(q.series[idx], label))
 
         for i, guard in enumerate(guards):
             if guard.startswith('+'):
                 ui.write(guard, label='qguard.positive')
             elif guard.startswith('-'):
                 ui.write(guard, label='qguard.negative')
             else:
                 ui.write(guard, label='qguard.unguarded')
             if i != len(guards) - 1:
                 ui.write(' ')
         ui.write('\n')
     q = repo.mq
     applied = set(p.name for p in q.applied)
     patch = None
     args = list(args)
     if opts.get(r'list'):
         if args or opts.get('none'):
             raise error.Abort(_('cannot mix -l/--list with options or '
                                'arguments'))
         for i in xrange(len(q.series)):
             status(i)
         return
     if not args or args[0][0:1] in '-+':
         if not q.applied:
             raise error.Abort(_('no patches applied'))
         patch = q.applied[-1].name
     if patch is None and args[0][0:1] not in '-+':
         patch = args.pop(0)
     if patch is None:
         raise error.Abort(_('no patch to work with'))
     if args or opts.get('none'):
         idx = q.findseries(patch)
         if idx is None:
             raise error.Abort(_('no patch named %s') % patch)
         q.setguards(idx, args)
         q.savedirty()
     else:
         status(q.series.index(q.lookup(patch)))
 
 @command("qheader", [], _('hg qheader [PATCH]'))
 def header(ui, repo, patch=None):
     """print the header of the topmost or specified patch
 
     Returns 0 on success."""
     q = repo.mq
 
     if patch:
         patch = q.lookup(patch)
     else:
         if not q.applied:
             ui.write(_('no patches applied\n'))
             return 1
         patch = q.lookup('qtip')
     ph = patchheader(q.join(patch), q.plainmode)
 
     ui.write('\n'.join(ph.message) + '\n')
 
 def lastsavename(path):
     (directory, base) = os.path.split(path)
     names = os.listdir(directory)
     namere = re.compile("%s.([0-9]+)" % base)
     maxindex = None
     maxname = None
     for f in names:
         m = namere.match(f)
         if m:
             index = int(m.group(1))
             if maxindex is None or index > maxindex:
                 maxindex = index
                 maxname = f
     if maxname:
         return (os.path.join(directory, maxname), maxindex)
     return (None, None)
 
 def savename(path):
     (last, index) = lastsavename(path)
     if last is None:
         index = 0
     newpath = path + ".%d" % (index + 1)
     return newpath
 
 @command("^qpush",
          [('', 'keep-changes', None,
            _('tolerate non-conflicting local changes')),
           ('f', 'force', None, _('apply on top of local changes')),
           ('e', 'exact', None,
            _('apply the target patch to its recorded parent')),
           ('l', 'list', None, _('list patch name in commit text')),
           ('a', 'all', None, _('apply all patches')),
           ('m', 'merge', None, _('merge from another queue (DEPRECATED)')),
           ('n', 'name', '',
            _('merge queue name (DEPRECATED)'), _('NAME')),
           ('', 'move', None,
            _('reorder patch series and apply only the patch')),
           ('', 'no-backup', None, _('do not save backup copies of files'))],
          _('hg qpush [-f] [-l] [-a] [--move] [PATCH | INDEX]'))
 def push(ui, repo, patch=None, **opts):
     """push the next patch onto the stack
 
     By default, abort if the working directory contains uncommitted
     changes. With --keep-changes, abort only if the uncommitted files
     overlap with patched files. With -f/--force, backup and patch over
     uncommitted changes.
 
     Return 0 on success.
     """
     q = repo.mq
     mergeq = None
 
     opts = pycompat.byteskwargs(opts)
     opts = fixkeepchangesopts(ui, opts)
     if opts.get('merge'):
         if opts.get('name'):
             newpath = repo.vfs.join(opts.get('name'))
         else:
             newpath, i = lastsavename(q.path)
         if not newpath:
             ui.warn(_("no saved queues found, please use -n\n"))
             return 1
         mergeq = queue(ui, repo.baseui, repo.path, newpath)
         ui.warn(_("merging with queue at: %s\n") % mergeq.path)
     ret = q.push(repo, patch, force=opts.get('force'), list=opts.get('list'),
                  mergeq=mergeq, all=opts.get('all'), move=opts.get('move'),
                  exact=opts.get('exact'), nobackup=opts.get('no_backup'),
                  keepchanges=opts.get('keep_changes'))
     return ret
 
 @command("^qpop",
          [('a', 'all', None, _('pop all patches')),
           ('n', 'name', '',
            _('queue name to pop (DEPRECATED)'), _('NAME')),
           ('', 'keep-changes', None,
            _('tolerate non-conflicting local changes')),
           ('f', 'force', None, _('forget any local changes to patched files')),
           ('', 'no-backup', None, _('do not save backup copies of files'))],
          _('hg qpop [-a] [-f] [PATCH | INDEX]'))
 def pop(ui, repo, patch=None, **opts):
     """pop the current patch off the stack
 
     Without argument, pops off the top of the patch stack. If given a
     patch name, keeps popping off patches until the named patch is at
     the top of the stack.
 
     By default, abort if the working directory contains uncommitted
     changes. With --keep-changes, abort only if the uncommitted files
     overlap with patched files. With -f/--force, backup and discard
     changes made to such files.
 
     Return 0 on success.
     """
     opts = pycompat.byteskwargs(opts)
     opts = fixkeepchangesopts(ui, opts)
     localupdate = True
     if opts.get('name'):
         q = queue(ui, repo.baseui, repo.path, repo.vfs.join(opts.get('name')))
         ui.warn(_('using patch queue: %s\n') % q.path)
         localupdate = False
     else:
         q = repo.mq
     ret = q.pop(repo, patch, force=opts.get('force'), update=localupdate,
                 all=opts.get('all'), nobackup=opts.get('no_backup'),
                 keepchanges=opts.get('keep_changes'))
     q.savedirty()
     return ret
 
 @command("qrename|qmv", [], _('hg qrename PATCH1 [PATCH2]'))
 def rename(ui, repo, patch, name=None, **opts):
     """rename a patch
 
     With one argument, renames the current patch to PATCH1.
     With two arguments, renames PATCH1 to PATCH2.
 
     Returns 0 on success."""
     q = repo.mq
     if not name:
         name = patch
         patch = None
 
     if patch:
         patch = q.lookup(patch)
     else:
         if not q.applied:
             ui.write(_('no patches applied\n'))
             return
         patch = q.lookup('qtip')
     absdest = q.join(name)
     if os.path.isdir(absdest):
         name = normname(os.path.join(name, os.path.basename(patch)))
         absdest = q.join(name)
     q.checkpatchname(name)
 
     ui.note(_('renaming %s to %s\n') % (patch, name))
     i = q.findseries(patch)
     guards = q.guard_re.findall(q.fullseries[i])
     q.fullseries[i] = name + ''.join([' #' + g for g in guards])
     q.parseseries()
     q.seriesdirty = True
 
     info = q.isapplied(patch)
     if info:
         q.applied[info[0]] = statusentry(info[1], name)
     q.applieddirty = True
 
     destdir = os.path.dirname(absdest)
     if not os.path.isdir(destdir):
         os.makedirs(destdir)
     util.rename(q.join(patch), absdest)
     r = q.qrepo()
     if r and patch in r.dirstate:
         wctx = r[None]
         with r.wlock():
             if r.dirstate[patch] == 'a':
                 r.dirstate.drop(patch)
                 r.dirstate.add(name)
             else:
                 wctx.copy(patch, name)
                 wctx.forget([patch])
 
     q.savedirty()
 
 @command("qrestore",
          [('d', 'delete', None, _('delete save entry')),
           ('u', 'update', None, _('update queue working directory'))],
          _('hg qrestore [-d] [-u] REV'))
 def restore(ui, repo, rev, **opts):
     """restore the queue state saved by a revision (DEPRECATED)
 
     This command is deprecated, use :hg:`rebase` instead."""
     rev = repo.lookup(rev)
     q = repo.mq
     q.restore(repo, rev, delete=opts.get(r'delete'),
               qupdate=opts.get(r'update'))
     q.savedirty()
     return 0
 
 @command("qsave",
          [('c', 'copy', None, _('copy patch directory')),
           ('n', 'name', '',
            _('copy directory name'), _('NAME')),
           ('e', 'empty', None, _('clear queue status file')),
           ('f', 'force', None, _('force copy'))] + cmdutil.commitopts,
          _('hg qsave [-m TEXT] [-l FILE] [-c] [-n NAME] [-e] [-f]'))
 def save(ui, repo, **opts):
     """save current queue state (DEPRECATED)
 
     This command is deprecated, use :hg:`rebase` instead."""
     q = repo.mq
     opts = pycompat.byteskwargs(opts)
     message = cmdutil.logmessage(ui, opts)
     ret = q.save(repo, msg=message)
     if ret:
         return ret
     q.savedirty() # save to .hg/patches before copying
     if opts.get('copy'):
         path = q.path
         if opts.get('name'):
             newpath = os.path.join(q.basepath, opts.get('name'))
             if os.path.exists(newpath):
                 if not os.path.isdir(newpath):
                     raise error.Abort(_('destination %s exists and is not '
                                        'a directory') % newpath)
                 if not opts.get('force'):
                     raise error.Abort(_('destination %s exists, '
                                        'use -f to force') % newpath)
         else:
             newpath = savename(path)
         ui.warn(_("copy %s to %s\n") % (path, newpath))
         util.copyfiles(path, newpath)
     if opts.get('empty'):
         del q.applied[:]
         q.applieddirty = True
         q.savedirty()
     return 0
 
 
 @command("qselect",
          [('n', 'none', None, _('disable all guards')),
           ('s', 'series', None, _('list all guards in series file')),
           ('', 'pop', None, _('pop to before first guarded applied patch')),
           ('', 'reapply', None, _('pop, then reapply patches'))],
          _('hg qselect [OPTION]... [GUARD]...'))
 def select(ui, repo, *args, **opts):
     '''set or print guarded patches to push
 
     Use the :hg:`qguard` command to set or print guards on patch, then use
     qselect to tell mq which guards to use. A patch will be pushed if
     it has no guards or any positive guards match the currently
     selected guard, but will not be pushed if any negative guards
     match the current guard. For example::
 
         qguard foo.patch -- -stable    (negative guard)
         qguard bar.patch    +stable    (positive guard)
         qselect stable
 
     This activates the "stable" guard. mq will skip foo.patch (because
     it has a negative match) but push bar.patch (because it has a
     positive match).
 
     With no arguments, prints the currently active guards.
     With one argument, sets the active guard.
 
     Use -n/--none to deactivate guards (no other arguments needed).
     When no guards are active, patches with positive guards are
     skipped and patches with negative guards are pushed.
 
     qselect can change the guards on applied patches. It does not pop
     guarded patches by default. Use --pop to pop back to the last
     applied patch that is not guarded. Use --reapply (which implies
     --pop) to push back to the current patch afterwards, but skip
     guarded patches.
 
     Use -s/--series to print a list of all guards in the series file
     (no other arguments needed). Use -v for more information.
 
     Returns 0 on success.'''
 
     q = repo.mq
     opts = pycompat.byteskwargs(opts)
     guards = q.active()
     pushable = lambda i: q.pushable(q.applied[i].name)[0]
     if args or opts.get('none'):
         old_unapplied = q.unapplied(repo)
         old_guarded = [i for i in xrange(len(q.applied)) if not pushable(i)]
         q.setactive(args)
         q.savedirty()
         if not args:
             ui.status(_('guards deactivated\n'))
         if not opts.get('pop') and not opts.get('reapply'):
             unapplied = q.unapplied(repo)
             guarded = [i for i in xrange(len(q.applied)) if not pushable(i)]
             if len(unapplied) != len(old_unapplied):
                 ui.status(_('number of unguarded, unapplied patches has '
                             'changed from %d to %d\n') %
                           (len(old_unapplied), len(unapplied)))
             if len(guarded) != len(old_guarded):
                 ui.status(_('number of guarded, applied patches has changed '
                             'from %d to %d\n') %
                           (len(old_guarded), len(guarded)))
     elif opts.get('series'):
         guards = {}
         noguards = 0
         for gs in q.seriesguards:
             if not gs:
                 noguards += 1
             for g in gs:
                 guards.setdefault(g, 0)
                 guards[g] += 1
         if ui.verbose:
             guards['NONE'] = noguards
         guards = list(guards.items())
         guards.sort(key=lambda x: x[0][1:])
         if guards:
             ui.note(_('guards in series file:\n'))
             for guard, count in guards:
                 ui.note('%2d  ' % count)
                 ui.write(guard, '\n')
         else:
             ui.note(_('no guards in series file\n'))
     else:
         if guards:
             ui.note(_('active guards:\n'))
             for g in guards:
                 ui.write(g, '\n')
         else:
             ui.write(_('no active guards\n'))
     reapply = opts.get('reapply') and q.applied and q.applied[-1].name
     popped = False
     if opts.get('pop') or opts.get('reapply'):
         for i in xrange(len(q.applied)):
             if not pushable(i):
                 ui.status(_('popping guarded patches\n'))
                 popped = True
                 if i == 0:
                     q.pop(repo, all=True)
                 else:
                     q.pop(repo, q.applied[i - 1].name)
                 break
     if popped:
         try:
             if reapply:
                 ui.status(_('reapplying unguarded patches\n'))
                 q.push(repo, reapply)
         finally:
             q.savedirty()
 
 @command("qfinish",
          [('a', 'applied', None, _('finish all applied changesets'))],
          _('hg qfinish [-a] [REV]...'))
 def finish(ui, repo, *revrange, **opts):
     """move applied patches into repository history
 
     Finishes the specified revisions (corresponding to applied
     patches) by moving them out of mq control into regular repository
     history.
 
     Accepts a revision range or the -a/--applied option. If --applied
     is specified, all applied mq revisions are removed from mq
     control. Otherwise, the given revisions must be at the base of the
     stack of applied patches.
 
     This can be especially useful if your changes have been applied to
     an upstream repository, or if you are about to push your changes
     to upstream.
 
     Returns 0 on success.
     """
     if not opts.get(r'applied') and not revrange:
         raise error.Abort(_('no revisions specified'))
     elif opts.get(r'applied'):
         revrange = ('qbase::qtip',) + revrange
 
     q = repo.mq
     if not q.applied:
         ui.status(_('no patches applied\n'))
         return 0
 
     revs = scmutil.revrange(repo, revrange)
     if repo['.'].rev() in revs and repo[None].files():
         ui.warn(_('warning: uncommitted changes in the working directory\n'))
     # queue.finish may changes phases but leave the responsibility to lock the
     # repo to the caller to avoid deadlock with wlock. This command code is
     # responsibility for this locking.
     with repo.lock():
         q.finish(repo, revs)
         q.savedirty()
     return 0
 
 @command("qqueue",
          [('l', 'list', False, _('list all available queues')),
           ('', 'active', False, _('print name of active queue')),
           ('c', 'create', False, _('create new queue')),
           ('', 'rename', False, _('rename active queue')),
           ('', 'delete', False, _('delete reference to queue')),
           ('', 'purge', False, _('delete queue, and remove patch dir')),
          ],
          _('[OPTION] [QUEUE]'))
 def qqueue(ui, repo, name=None, **opts):
     '''manage multiple patch queues
 
     Supports switching between different patch queues, as well as creating
     new patch queues and deleting existing ones.
 
     Omitting a queue name or specifying -l/--list will show you the registered
     queues - by default the "normal" patches queue is registered. The currently
     active queue will be marked with "(active)". Specifying --active will print
     only the name of the active queue.
 
     To create a new queue, use -c/--create. The queue is automatically made
     active, except in the case where there are applied patches from the
     currently active queue in the repository. Then the queue will only be
     created and switching will fail.
 
     To delete an existing queue, use --delete. You cannot delete the currently
     active queue.
 
     Returns 0 on success.
     '''
     q = repo.mq
     _defaultqueue = 'patches'
     _allqueues = 'patches.queues'
     _activequeue = 'patches.queue'
 
     def _getcurrent():
         cur = os.path.basename(q.path)
         if cur.startswith('patches-'):
             cur = cur[8:]
         return cur
 
     def _noqueues():
         try:
             fh = repo.vfs(_allqueues, 'r')
             fh.close()
         except IOError:
             return True
 
         return False
 
     def _getqueues():
         current = _getcurrent()
 
         try:
             fh = repo.vfs(_allqueues, 'r')
             queues = [queue.strip() for queue in fh if queue.strip()]
             fh.close()
             if current not in queues:
                 queues.append(current)
         except IOError:
             queues = [_defaultqueue]
 
         return sorted(queues)
 
     def _setactive(name):
         if q.applied:
             raise error.Abort(_('new queue created, but cannot make active '
                                'as patches are applied'))
         _setactivenocheck(name)
 
     def _setactivenocheck(name):
         fh = repo.vfs(_activequeue, 'w')
         if name != 'patches':
             fh.write(name)
         fh.close()
 
     def _addqueue(name):
         fh = repo.vfs(_allqueues, 'a')
         fh.write('%s\n' % (name,))
         fh.close()
 
     def _queuedir(name):
         if name == 'patches':
             return repo.vfs.join('patches')
         else:
             return repo.vfs.join('patches-' + name)
 
     def _validname(name):
         for n in name:
             if n in ':\\/.':
                 return False
         return True
 
     def _delete(name):
         if name not in existing:
             raise error.Abort(_('cannot delete queue that does not exist'))
 
         current = _getcurrent()
 
         if name == current:
             raise error.Abort(_('cannot delete currently active queue'))
 
         fh = repo.vfs('patches.queues.new', 'w')
         for queue in existing:
             if queue == name:
                 continue
             fh.write('%s\n' % (queue,))
         fh.close()
         repo.vfs.rename('patches.queues.new', _allqueues)
 
     opts = pycompat.byteskwargs(opts)
     if not name or opts.get('list') or opts.get('active'):
         current = _getcurrent()
         if opts.get('active'):
             ui.write('%s\n' % (current,))
             return
         for queue in _getqueues():
             ui.write('%s' % (queue,))
             if queue == current and not ui.quiet:
                 ui.write(_(' (active)\n'))
             else:
                 ui.write('\n')
         return
 
     if not _validname(name):
         raise error.Abort(
                 _('invalid queue name, may not contain the characters ":\\/."'))
 
     with repo.wlock():
         existing = _getqueues()
 
         if opts.get('create'):
             if name in existing:
                 raise error.Abort(_('queue "%s" already exists') % name)
             if _noqueues():
                 _addqueue(_defaultqueue)
             _addqueue(name)
             _setactive(name)
         elif opts.get('rename'):
             current = _getcurrent()
             if name == current:
                 raise error.Abort(_('can\'t rename "%s" to its current name')
                                   % name)
             if name in existing:
                 raise error.Abort(_('queue "%s" already exists') % name)
 
             olddir = _queuedir(current)
             newdir = _queuedir(name)
 
             if os.path.exists(newdir):
                 raise error.Abort(_('non-queue directory "%s" already exists') %
                         newdir)
 
             fh = repo.vfs('patches.queues.new', 'w')
             for queue in existing:
                 if queue == current:
                     fh.write('%s\n' % (name,))
                     if os.path.exists(olddir):
                         util.rename(olddir, newdir)
                 else:
                     fh.write('%s\n' % (queue,))
             fh.close()
             repo.vfs.rename('patches.queues.new', _allqueues)
             _setactivenocheck(name)
         elif opts.get('delete'):
             _delete(name)
         elif opts.get('purge'):
             if name in existing:
                 _delete(name)
             qdir = _queuedir(name)
             if os.path.exists(qdir):
                 shutil.rmtree(qdir)
         else:
             if name not in existing:
                 raise error.Abort(_('use --create to create a new queue'))
             _setactive(name)
 
 def mqphasedefaults(repo, roots):
     """callback used to set mq changeset as secret when no phase data exists"""
     if repo.mq.applied:
         if repo.ui.configbool('mq', 'secret'):
             mqphase = phases.secret
         else:
             mqphase = phases.draft
         qbase = repo[repo.mq.applied[0].node]
         roots[mqphase].add(qbase.node())
     return roots
 
 def reposetup(ui, repo):
     class mqrepo(repo.__class__):
         @localrepo.unfilteredpropertycache
         def mq(self):
             return queue(self.ui, self.baseui, self.path)
 
         def invalidateall(self):
             super(mqrepo, self).invalidateall()
             if localrepo.hasunfilteredcache(self, 'mq'):
                 # recreate mq in case queue path was changed
                 delattr(self.unfiltered(), 'mq')
 
         def abortifwdirpatched(self, errmsg, force=False):
             if self.mq.applied and self.mq.checkapplied and not force:
                 parents = self.dirstate.parents()
                 patches = [s.node for s in self.mq.applied]
                 if parents[0] in patches or parents[1] in patches:
                     raise error.Abort(errmsg)
 
         def commit(self, text="", user=None, date=None, match=None,
                    force=False, editor=False, extra=None):
             if extra is None:
                 extra = {}
             self.abortifwdirpatched(
                 _('cannot commit over an applied mq patch'),
                 force)
 
             return super(mqrepo, self).commit(text, user, date, match, force,
                                               editor, extra)
 
         def checkpush(self, pushop):
             if self.mq.applied and self.mq.checkapplied and not pushop.force:
                 outapplied = [e.node for e in self.mq.applied]
                 if pushop.revs:
                     # Assume applied patches have no non-patch descendants and
                     # are not on remote already. Filtering any changeset not
                     # pushed.
                     heads = set(pushop.revs)
                     for node in reversed(outapplied):
                         if node in heads:
                             break
                         else:
                             outapplied.pop()
                 # looking for pushed and shared changeset
                 for node in outapplied:
                     if self[node].phase() < phases.secret:
                         raise error.Abort(_('source has mq patches applied'))
                 # no non-secret patches pushed
             super(mqrepo, self).checkpush(pushop)
 
         def _findtags(self):
             '''augment tags from base class with patch tags'''
             result = super(mqrepo, self)._findtags()
 
             q = self.mq
             if not q.applied:
                 return result
 
             mqtags = [(patch.node, patch.name) for patch in q.applied]
 
             try:
                 # for now ignore filtering business
                 self.unfiltered().changelog.rev(mqtags[-1][0])
             except error.LookupError:
                 self.ui.warn(_('mq status file refers to unknown node %s\n')
                              % short(mqtags[-1][0]))
                 return result
 
             # do not add fake tags for filtered revisions
             included = self.changelog.hasnode
             mqtags = [mqt for mqt in mqtags if included(mqt[0])]
             if not mqtags:
                 return result
 
             mqtags.append((mqtags[-1][0], 'qtip'))
             mqtags.append((mqtags[0][0], 'qbase'))
             mqtags.append((self.changelog.parents(mqtags[0][0])[0], 'qparent'))
             tags = result[0]
             for patch in mqtags:
                 if patch[1] in tags:
                     self.ui.warn(_('tag %s overrides mq patch of the same '
                                    'name\n') % patch[1])
                 else:
                     tags[patch[1]] = patch[0]
 
             return result
 
     if repo.local():
         repo.__class__ = mqrepo
 
         repo._phasedefaults.append(mqphasedefaults)
 
 def mqimport(orig, ui, repo, *args, **kwargs):
     if (util.safehasattr(repo, 'abortifwdirpatched')
         and not kwargs.get(r'no_commit', False)):
         repo.abortifwdirpatched(_('cannot import over an applied patch'),
                                    kwargs.get(r'force'))
     return orig(ui, repo, *args, **kwargs)
 
 def mqinit(orig, ui, *args, **kwargs):
     mq = kwargs.pop(r'mq', None)
 
     if not mq:
         return orig(ui, *args, **kwargs)
 
     if args:
         repopath = args[0]
         if not hg.islocal(repopath):
             raise error.Abort(_('only a local queue repository '
                                'may be initialized'))
     else:
         repopath = cmdutil.findrepo(pycompat.getcwd())
         if not repopath:
             raise error.Abort(_('there is no Mercurial repository here '
                                '(.hg not found)'))
     repo = hg.repository(ui, repopath)
     return qinit(ui, repo, True)
 
 def mqcommand(orig, ui, repo, *args, **kwargs):
     """Add --mq option to operate on patch repository instead of main"""
 
     # some commands do not like getting unknown options
     mq = kwargs.pop(r'mq', None)
 
     if not mq:
         return orig(ui, repo, *args, **kwargs)
 
     q = repo.mq
     r = q.qrepo()
     if not r:
         raise error.Abort(_('no queue repository'))
     return orig(r.ui, r, *args, **kwargs)
 
 def summaryhook(ui, repo):
     q = repo.mq
     m = []
     a, u = len(q.applied), len(q.unapplied(repo))
     if a:
         m.append(ui.label(_("%d applied"), 'qseries.applied') % a)
     if u:
         m.append(ui.label(_("%d unapplied"), 'qseries.unapplied') % u)
     if m:
         # i18n: column positioning for "hg summary"
         ui.write(_("mq:     %s\n") % ', '.join(m))
     else:
         # i18n: column positioning for "hg summary"
         ui.note(_("mq:     (empty queue)\n"))
 
 revsetpredicate = registrar.revsetpredicate()
 
 @revsetpredicate('mq()')
 def revsetmq(repo, subset, x):
     """Changesets managed by MQ.
     """
     revsetlang.getargs(x, 0, 0, _("mq takes no arguments"))
     applied = set([repo[r.node].rev() for r in repo.mq.applied])
     return smartset.baseset([r for r in subset if r in applied])
 
 # tell hggettext to extract docstrings from these functions:
 i18nfunctions = [revsetmq]
 
 def extsetup(ui):
     # Ensure mq wrappers are called first, regardless of extension load order by
     # NOT wrapping in uisetup() and instead deferring to init stage two here.
     mqopt = [('', 'mq', None, _("operate on patch repository"))]
 
     extensions.wrapcommand(commands.table, 'import', mqimport)
     cmdutil.summaryhooks.add('mq', summaryhook)
 
     entry = extensions.wrapcommand(commands.table, 'init', mqinit)
     entry[1].extend(mqopt)
 
     def dotable(cmdtable):
         for cmd, entry in cmdtable.iteritems():
             cmd = cmdutil.parsealiases(cmd)[0]
             func = entry[0]
             if func.norepo:
                 continue
             entry = extensions.wrapcommand(cmdtable, cmd, mqcommand)
             entry[1].extend(mqopt)
 
     dotable(commands.table)
 
     for extname, extmodule in extensions.extensions():
         if extmodule.__file__ != __file__:
             dotable(getattr(extmodule, 'cmdtable', {}))
 
 colortable = {'qguard.negative': 'red',
               'qguard.positive': 'yellow',
               'qguard.unguarded': 'green',
               'qseries.applied': 'blue bold underline',
               'qseries.guarded': 'black bold',
               'qseries.missing': 'red bold',
               'qseries.unapplied': 'black bold'}

hgext/narrow/narrowbundle2.py

@@ -1,502 +1,505
 # narrowbundle2.py - bundle2 extensions for narrow repository support
 #
 # Copyright 2017 Google, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import collections
 import errno
 import struct
 
 from mercurial.i18n import _
 from mercurial.node import (
     bin,
     nullid,
     nullrev,
 )
 from mercurial import (
     bundle2,
     changegroup,
     dagutil,
     error,
     exchange,
     extensions,
     narrowspec,
     repair,
     util,
     wireproto,
 )
+from mercurial.utils import (
+    stringutil,
+)
 
 NARROWCAP = 'narrow'
 _NARROWACL_SECTION = 'narrowhgacl'
 _CHANGESPECPART = NARROWCAP + ':changespec'
 _SPECPART = NARROWCAP + ':spec'
 _SPECPART_INCLUDE = 'include'
 _SPECPART_EXCLUDE = 'exclude'
 _KILLNODESIGNAL = 'KILL'
 _DONESIGNAL = 'DONE'
 _ELIDEDCSHEADER = '>20s20s20sl' # cset id, p1, p2, len(text)
 _ELIDEDMFHEADER = '>20s20s20s20sl' # manifest id, p1, p2, link id, len(text)
 _CSHEADERSIZE = struct.calcsize(_ELIDEDCSHEADER)
 _MFHEADERSIZE = struct.calcsize(_ELIDEDMFHEADER)
 
 # When advertising capabilities, always include narrow clone support.
 def getrepocaps_narrow(orig, repo, **kwargs):
     caps = orig(repo, **kwargs)
     caps[NARROWCAP] = ['v0']
     return caps
 
 def _computeellipsis(repo, common, heads, known, match, depth=None):
     """Compute the shape of a narrowed DAG.
 
     Args:
       repo: The repository we're transferring.
       common: The roots of the DAG range we're transferring.
               May be just [nullid], which means all ancestors of heads.
       heads: The heads of the DAG range we're transferring.
       match: The narrowmatcher that allows us to identify relevant changes.
       depth: If not None, only consider nodes to be full nodes if they are at
              most depth changesets away from one of heads.
 
     Returns:
       A tuple of (visitnodes, relevant_nodes, ellipsisroots) where:
 
         visitnodes: The list of nodes (either full or ellipsis) which
                     need to be sent to the client.
         relevant_nodes: The set of changelog nodes which change a file inside
                  the narrowspec. The client needs these as non-ellipsis nodes.
         ellipsisroots: A dict of {rev: parents} that is used in
                        narrowchangegroup to produce ellipsis nodes with the
                        correct parents.
     """
     cl = repo.changelog
     mfl = repo.manifestlog
 
     cldag = dagutil.revlogdag(cl)
     # dagutil does not like nullid/nullrev
     commonrevs = cldag.internalizeall(common - set([nullid])) | set([nullrev])
     headsrevs = cldag.internalizeall(heads)
     if depth:
         revdepth = {h: 0 for h in headsrevs}
 
     ellipsisheads = collections.defaultdict(set)
     ellipsisroots = collections.defaultdict(set)
 
     def addroot(head, curchange):
         """Add a root to an ellipsis head, splitting heads with 3 roots."""
         ellipsisroots[head].add(curchange)
         # Recursively split ellipsis heads with 3 roots by finding the
         # roots' youngest common descendant which is an elided merge commit.
         # That descendant takes 2 of the 3 roots as its own, and becomes a
         # root of the head.
         while len(ellipsisroots[head]) > 2:
             child, roots = splithead(head)
             splitroots(head, child, roots)
             head = child  # Recurse in case we just added a 3rd root
 
     def splitroots(head, child, roots):
         ellipsisroots[head].difference_update(roots)
         ellipsisroots[head].add(child)
         ellipsisroots[child].update(roots)
         ellipsisroots[child].discard(child)
 
     def splithead(head):
         r1, r2, r3 = sorted(ellipsisroots[head])
         for nr1, nr2 in ((r2, r3), (r1, r3), (r1, r2)):
             mid = repo.revs('sort(merge() & %d::%d & %d::%d, -rev)',
                             nr1, head, nr2, head)
             for j in mid:
                 if j == nr2:
                     return nr2, (nr1, nr2)
                 if j not in ellipsisroots or len(ellipsisroots[j]) < 2:
                     return j, (nr1, nr2)
         raise error.Abort('Failed to split up ellipsis node! head: %d, '
                           'roots: %d %d %d' % (head, r1, r2, r3))
 
     missing = list(cl.findmissingrevs(common=commonrevs, heads=headsrevs))
     visit = reversed(missing)
     relevant_nodes = set()
     visitnodes = [cl.node(m) for m in missing]
     required = set(headsrevs) | known
     for rev in visit:
         clrev = cl.changelogrevision(rev)
         ps = cldag.parents(rev)
         if depth is not None:
             curdepth = revdepth[rev]
             for p in ps:
                 revdepth[p] = min(curdepth + 1, revdepth.get(p, depth + 1))
         needed = False
         shallow_enough = depth is None or revdepth[rev] <= depth
         if shallow_enough:
             curmf = mfl[clrev.manifest].read()
             if ps:
                 # We choose to not trust the changed files list in
                 # changesets because it's not always correct. TODO: could
                 # we trust it for the non-merge case?
                 p1mf = mfl[cl.changelogrevision(ps[0]).manifest].read()
                 needed = bool(curmf.diff(p1mf, match))
                 if not needed and len(ps) > 1:
                     # For merge changes, the list of changed files is not
                     # helpful, since we need to emit the merge if a file
                     # in the narrow spec has changed on either side of the
                     # merge. As a result, we do a manifest diff to check.
                     p2mf = mfl[cl.changelogrevision(ps[1]).manifest].read()
                     needed = bool(curmf.diff(p2mf, match))
             else:
                 # For a root node, we need to include the node if any
                 # files in the node match the narrowspec.
                 needed = any(curmf.walk(match))
 
         if needed:
             for head in ellipsisheads[rev]:
                 addroot(head, rev)
             for p in ps:
                 required.add(p)
             relevant_nodes.add(cl.node(rev))
         else:
             if not ps:
                 ps = [nullrev]
             if rev in required:
                 for head in ellipsisheads[rev]:
                     addroot(head, rev)
                 for p in ps:
                     ellipsisheads[p].add(rev)
             else:
                 for p in ps:
                     ellipsisheads[p] |= ellipsisheads[rev]
 
     # add common changesets as roots of their reachable ellipsis heads
     for c in commonrevs:
         for head in ellipsisheads[c]:
             addroot(head, c)
     return visitnodes, relevant_nodes, ellipsisroots
 
 def _packellipsischangegroup(repo, common, match, relevant_nodes,
                              ellipsisroots, visitnodes, depth, source, version):
     if version in ('01', '02'):
         raise error.Abort(
             'ellipsis nodes require at least cg3 on client and server, '
             'but negotiated version %s' % version)
     # We wrap cg1packer.revchunk, using a side channel to pass
     # relevant_nodes into that area. Then if linknode isn't in the
     # set, we know we have an ellipsis node and we should defer
     # sending that node's data. We override close() to detect
     # pending ellipsis nodes and flush them.
     packer = changegroup.getbundler(version, repo)
     # Let the packer have access to the narrow matcher so it can
     # omit filelogs and dirlogs as needed
     packer._narrow_matcher = lambda : match
     # Give the packer the list of nodes which should not be
     # ellipsis nodes. We store this rather than the set of nodes
     # that should be an ellipsis because for very large histories
     # we expect this to be significantly smaller.
     packer.full_nodes = relevant_nodes
     # Maps ellipsis revs to their roots at the changelog level.
     packer.precomputed_ellipsis = ellipsisroots
     # Maps CL revs to per-revlog revisions. Cleared in close() at
     # the end of each group.
     packer.clrev_to_localrev = {}
     packer.next_clrev_to_localrev = {}
     # Maps changelog nodes to changelog revs. Filled in once
     # during changelog stage and then left unmodified.
     packer.clnode_to_rev = {}
     packer.changelog_done = False
     # If true, informs the packer that it is serving shallow content and might
     # need to pack file contents not introduced by the changes being packed.
     packer.is_shallow = depth is not None
 
     return packer.generate(common, visitnodes, False, source)
 
 # Serve a changegroup for a client with a narrow clone.
 def getbundlechangegrouppart_narrow(bundler, repo, source,
                                     bundlecaps=None, b2caps=None, heads=None,
                                     common=None, **kwargs):
     cgversions = b2caps.get('changegroup')
     if cgversions:  # 3.1 and 3.2 ship with an empty value
         cgversions = [v for v in cgversions
                       if v in changegroup.supportedoutgoingversions(repo)]
         if not cgversions:
             raise ValueError(_('no common changegroup version'))
         version = max(cgversions)
     else:
         raise ValueError(_("server does not advertise changegroup version,"
                            " can't negotiate support for ellipsis nodes"))
 
     include = sorted(filter(bool, kwargs.get(r'includepats', [])))
     exclude = sorted(filter(bool, kwargs.get(r'excludepats', [])))
     newmatch = narrowspec.match(repo.root, include=include, exclude=exclude)
     if not repo.ui.configbool("experimental", "narrowservebrokenellipses"):
         outgoing = exchange._computeoutgoing(repo, heads, common)
         if not outgoing.missing:
             return
         def wrappedgetbundler(orig, *args, **kwargs):
             bundler = orig(*args, **kwargs)
             bundler._narrow_matcher = lambda : newmatch
             return bundler
         with extensions.wrappedfunction(changegroup, 'getbundler',
                                         wrappedgetbundler):
             cg = changegroup.makestream(repo, outgoing, version, source)
         part = bundler.newpart('changegroup', data=cg)
         part.addparam('version', version)
         if 'treemanifest' in repo.requirements:
             part.addparam('treemanifest', '1')
 
         if include or exclude:
             narrowspecpart = bundler.newpart(_SPECPART)
             if include:
                 narrowspecpart.addparam(
                     _SPECPART_INCLUDE, '\n'.join(include), mandatory=True)
             if exclude:
                 narrowspecpart.addparam(
                     _SPECPART_EXCLUDE, '\n'.join(exclude), mandatory=True)
 
         return
 
     depth = kwargs.get(r'depth', None)
     if depth is not None:
         depth = int(depth)
         if depth < 1:
             raise error.Abort(_('depth must be positive, got %d') % depth)
 
     heads = set(heads or repo.heads())
     common = set(common or [nullid])
     oldinclude = sorted(filter(bool, kwargs.get(r'oldincludepats', [])))
     oldexclude = sorted(filter(bool, kwargs.get(r'oldexcludepats', [])))
     known = {bin(n) for n in kwargs.get(r'known', [])}
     if known and (oldinclude != include or oldexclude != exclude):
         # Steps:
         # 1. Send kill for "$known & ::common"
         #
         # 2. Send changegroup for ::common
         #
         # 3. Proceed.
         #
         # In the future, we can send kills for only the specific
         # nodes we know should go away or change shape, and then
         # send a data stream that tells the client something like this:
         #
         # a) apply this changegroup
         # b) apply nodes XXX, YYY, ZZZ that you already have
         # c) goto a
         #
         # until they've built up the full new state.
         # Convert to revnums and intersect with "common". The client should
         # have made it a subset of "common" already, but let's be safe.
         known = set(repo.revs("%ln & ::%ln", known, common))
         # TODO: we could send only roots() of this set, and the
         # list of nodes in common, and the client could work out
         # what to strip, instead of us explicitly sending every
         # single node.
         deadrevs = known
         def genkills():
             for r in deadrevs:
                 yield _KILLNODESIGNAL
                 yield repo.changelog.node(r)
             yield _DONESIGNAL
         bundler.newpart(_CHANGESPECPART, data=genkills())
         newvisit, newfull, newellipsis = _computeellipsis(
             repo, set(), common, known, newmatch)
         if newvisit:
             cg = _packellipsischangegroup(
                 repo, common, newmatch, newfull, newellipsis,
                 newvisit, depth, source, version)
             part = bundler.newpart('changegroup', data=cg)
             part.addparam('version', version)
             if 'treemanifest' in repo.requirements:
                 part.addparam('treemanifest', '1')
 
     visitnodes, relevant_nodes, ellipsisroots = _computeellipsis(
         repo, common, heads, set(), newmatch, depth=depth)
 
     repo.ui.debug('Found %d relevant revs\n' % len(relevant_nodes))
     if visitnodes:
         cg = _packellipsischangegroup(
             repo, common, newmatch, relevant_nodes, ellipsisroots,
             visitnodes, depth, source, version)
         part = bundler.newpart('changegroup', data=cg)
         part.addparam('version', version)
         if 'treemanifest' in repo.requirements:
             part.addparam('treemanifest', '1')
 
 def applyacl_narrow(repo, kwargs):
     ui = repo.ui
     username = ui.shortuser(ui.environ.get('REMOTE_USER') or ui.username())
     user_includes = ui.configlist(
         _NARROWACL_SECTION, username + '.includes',
         ui.configlist(_NARROWACL_SECTION, 'default.includes'))
     user_excludes = ui.configlist(
         _NARROWACL_SECTION, username + '.excludes',
         ui.configlist(_NARROWACL_SECTION, 'default.excludes'))
     if not user_includes:
         raise error.Abort(_("{} configuration for user {} is empty")
                           .format(_NARROWACL_SECTION, username))
 
     user_includes = [
         'path:.' if p == '*' else 'path:' + p for p in user_includes]
     user_excludes = [
         'path:.' if p == '*' else 'path:' + p for p in user_excludes]
 
     req_includes = set(kwargs.get(r'includepats', []))
     req_excludes = set(kwargs.get(r'excludepats', []))
 
     req_includes, req_excludes, invalid_includes = narrowspec.restrictpatterns(
         req_includes, req_excludes, user_includes, user_excludes)
 
     if invalid_includes:
         raise error.Abort(
             _("The following includes are not accessible for {}: {}")
             .format(username, invalid_includes))
 
     new_args = {}
     new_args.update(kwargs)
     new_args['includepats'] = req_includes
     if req_excludes:
         new_args['excludepats'] = req_excludes
     return new_args
 
 @bundle2.parthandler(_SPECPART, (_SPECPART_INCLUDE, _SPECPART_EXCLUDE))
 def _handlechangespec_2(op, inpart):
     includepats = set(inpart.params.get(_SPECPART_INCLUDE, '').splitlines())
     excludepats = set(inpart.params.get(_SPECPART_EXCLUDE, '').splitlines())
     if not changegroup.NARROW_REQUIREMENT in op.repo.requirements:
         op.repo.requirements.add(changegroup.NARROW_REQUIREMENT)
         op.repo._writerequirements()
     op.repo.setnarrowpats(includepats, excludepats)
 
 @bundle2.parthandler(_CHANGESPECPART)
 def _handlechangespec(op, inpart):
     repo = op.repo
     cl = repo.changelog
 
     # changesets which need to be stripped entirely. either they're no longer
     # needed in the new narrow spec, or the server is sending a replacement
     # in the changegroup part.
     clkills = set()
 
     # A changespec part contains all the updates to ellipsis nodes
     # that will happen as a result of widening or narrowing a
     # repo. All the changes that this block encounters are ellipsis
     # nodes or flags to kill an existing ellipsis.
     chunksignal = changegroup.readexactly(inpart, 4)
     while chunksignal != _DONESIGNAL:
         if chunksignal == _KILLNODESIGNAL:
             # a node used to be an ellipsis but isn't anymore
             ck = changegroup.readexactly(inpart, 20)
             if cl.hasnode(ck):
                 clkills.add(ck)
         else:
             raise error.Abort(
                 _('unexpected changespec node chunk type: %s') % chunksignal)
         chunksignal = changegroup.readexactly(inpart, 4)
 
     if clkills:
         # preserve bookmarks that repair.strip() would otherwise strip
         bmstore = repo._bookmarks
         class dummybmstore(dict):
             def applychanges(self, repo, tr, changes):
                 pass
             def recordchange(self, tr): # legacy version
                 pass
         repo._bookmarks = dummybmstore()
         chgrpfile = repair.strip(op.ui, repo, list(clkills), backup=True,
                                  topic='widen')
         repo._bookmarks = bmstore
         if chgrpfile:
             # presence of _widen_bundle attribute activates widen handler later
             op._widen_bundle = chgrpfile
     # Set the new narrowspec if we're widening. The setnewnarrowpats() method
     # will currently always be there when using the core+narrowhg server, but
     # other servers may include a changespec part even when not widening (e.g.
     # because we're deepening a shallow repo).
     if util.safehasattr(repo, 'setnewnarrowpats'):
         repo.setnewnarrowpats()
 
 def handlechangegroup_widen(op, inpart):
     """Changegroup exchange handler which restores temporarily-stripped nodes"""
     # We saved a bundle with stripped node data we must now restore.
     # This approach is based on mercurial/repair.py@6ee26a53c111.
     repo = op.repo
     ui = op.ui
 
     chgrpfile = op._widen_bundle
     del op._widen_bundle
     vfs = repo.vfs
 
     ui.note(_("adding branch\n"))
     f = vfs.open(chgrpfile, "rb")
     try:
         gen = exchange.readbundle(ui, f, chgrpfile, vfs)
         if not ui.verbose:
             # silence internal shuffling chatter
             ui.pushbuffer()
         if isinstance(gen, bundle2.unbundle20):
             with repo.transaction('strip') as tr:
                 bundle2.processbundle(repo, gen, lambda: tr)
         else:
             gen.apply(repo, 'strip', 'bundle:' + vfs.join(chgrpfile), True)
         if not ui.verbose:
             ui.popbuffer()
     finally:
         f.close()
 
     # remove undo files
     for undovfs, undofile in repo.undofiles():
         try:
             undovfs.unlink(undofile)
         except OSError as e:
             if e.errno != errno.ENOENT:
                 ui.warn(_('error removing %s: %s\n') %
-                        (undovfs.join(undofile), util.forcebytestr(e)))
+                        (undovfs.join(undofile), stringutil.forcebytestr(e)))
 
     # Remove partial backup only if there were no exceptions
     vfs.unlink(chgrpfile)
 
 def setup():
     """Enable narrow repo support in bundle2-related extension points."""
     extensions.wrapfunction(bundle2, 'getrepocaps', getrepocaps_narrow)
 
     wireproto.gboptsmap['narrow'] = 'boolean'
     wireproto.gboptsmap['depth'] = 'plain'
     wireproto.gboptsmap['oldincludepats'] = 'csv'
     wireproto.gboptsmap['oldexcludepats'] = 'csv'
     wireproto.gboptsmap['includepats'] = 'csv'
     wireproto.gboptsmap['excludepats'] = 'csv'
     wireproto.gboptsmap['known'] = 'csv'
 
     # Extend changegroup serving to handle requests from narrow clients.
     origcgfn = exchange.getbundle2partsmapping['changegroup']
     def wrappedcgfn(*args, **kwargs):
         repo = args[1]
         if repo.ui.has_section(_NARROWACL_SECTION):
             getbundlechangegrouppart_narrow(
                 *args, **applyacl_narrow(repo, kwargs))
         elif kwargs.get(r'narrow', False):
             getbundlechangegrouppart_narrow(*args, **kwargs)
         else:
             origcgfn(*args, **kwargs)
     exchange.getbundle2partsmapping['changegroup'] = wrappedcgfn
 
     # disable rev branch cache exchange when serving a narrow bundle
     # (currently incompatible with that part)
     origrbcfn = exchange.getbundle2partsmapping['cache:rev-branch-cache']
     def wrappedcgfn(*args, **kwargs):
         repo = args[1]
         if repo.ui.has_section(_NARROWACL_SECTION):
             return
         elif kwargs.get(r'narrow', False):
             return
         else:
             origrbcfn(*args, **kwargs)
     exchange.getbundle2partsmapping['cache:rev-branch-cache'] = wrappedcgfn
 
     # Extend changegroup receiver so client can fixup after widen requests.
     origcghandler = bundle2.parthandlermapping['changegroup']
     def wrappedcghandler(op, inpart):
         origcghandler(op, inpart)
         if util.safehasattr(op, '_widen_bundle'):
             handlechangegroup_widen(op, inpart)
     wrappedcghandler.params = origcghandler.params
     bundle2.parthandlermapping['changegroup'] = wrappedcghandler

hgext/notify.py

@@ -1,485 +1,488
 # notify.py - email notifications for mercurial
 #
 # Copyright 2006 Vadim Gelfer <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''hooks for sending email push notifications
 
 This extension implements hooks to send email notifications when
 changesets are sent from or received by the local repository.
 
 First, enable the extension as explained in :hg:`help extensions`, and
 register the hook you want to run. ``incoming`` and ``changegroup`` hooks
 are run when changesets are received, while ``outgoing`` hooks are for
 changesets sent to another repository::
 
   [hooks]
   # one email for each incoming changeset
   incoming.notify = python:hgext.notify.hook
   # one email for all incoming changesets
   changegroup.notify = python:hgext.notify.hook
 
   # one email for all outgoing changesets
   outgoing.notify = python:hgext.notify.hook
 
 This registers the hooks. To enable notification, subscribers must
 be assigned to repositories. The ``[usersubs]`` section maps multiple
 repositories to a given recipient. The ``[reposubs]`` section maps
 multiple recipients to a single repository::
 
   [usersubs]
   # key is subscriber email, value is a comma-separated list of repo patterns
   user@host = pattern
 
   [reposubs]
   # key is repo pattern, value is a comma-separated list of subscriber emails
   pattern = user@host
 
 A ``pattern`` is a ``glob`` matching the absolute path to a repository,
 optionally combined with a revset expression. A revset expression, if
 present, is separated from the glob by a hash. Example::
 
   [reposubs]
   */widgets#branch(release) = [e-mail]
 
 This sends to ``[e-mail]`` whenever a changeset on the ``release``
 branch triggers a notification in any repository ending in ``widgets``.
 
 In order to place them under direct user management, ``[usersubs]`` and
 ``[reposubs]`` sections may be placed in a separate ``hgrc`` file and
 incorporated by reference::
 
   [notify]
   config = /path/to/subscriptionsfile
 
 Notifications will not be sent until the ``notify.test`` value is set
 to ``False``; see below.
 
 Notifications content can be tweaked with the following configuration entries:
 
 notify.test
   If ``True``, print messages to stdout instead of sending them. Default: True.
 
 notify.sources
   Space-separated list of change sources. Notifications are activated only
   when a changeset's source is in this list. Sources may be:
 
   :``serve``: changesets received via http or ssh
   :``pull``: changesets received via ``hg pull``
   :``unbundle``: changesets received via ``hg unbundle``
   :``push``: changesets sent or received via ``hg push``
   :``bundle``: changesets sent via ``hg unbundle``
 
   Default: serve.
 
 notify.strip
   Number of leading slashes to strip from url paths. By default, notifications
   reference repositories with their absolute path. ``notify.strip`` lets you
   turn them into relative paths. For example, ``notify.strip=3`` will change
   ``/long/path/repository`` into ``repository``. Default: 0.
 
 notify.domain
   Default email domain for sender or recipients with no explicit domain.
 
 notify.style
   Style file to use when formatting emails.
 
 notify.template
   Template to use when formatting emails.
 
 notify.incoming
   Template to use when run as an incoming hook, overriding ``notify.template``.
 
 notify.outgoing
   Template to use when run as an outgoing hook, overriding ``notify.template``.
 
 notify.changegroup
   Template to use when running as a changegroup hook, overriding
   ``notify.template``.
 
 notify.maxdiff
   Maximum number of diff lines to include in notification email. Set to 0
   to disable the diff, or -1 to include all of it. Default: 300.
 
 notify.maxsubject
   Maximum number of characters in email's subject line. Default: 67.
 
 notify.diffstat
   Set to True to include a diffstat before diff content. Default: True.
 
 notify.merge
   If True, send notifications for merge changesets. Default: True.
 
 notify.mbox
   If set, append mails to this mbox file instead of sending. Default: None.
 
 notify.fromauthor
   If set, use the committer of the first changeset in a changegroup for
   the "From" field of the notification mail. If not set, take the user
   from the pushing repo.  Default: False.
 
 If set, the following entries will also be used to customize the
 notifications:
 
 email.from
   Email ``From`` address to use if none can be found in the generated
   email content.
 
 web.baseurl
   Root repository URL to combine with repository paths when making
   references. See also ``notify.strip``.
 
 '''
 from __future__ import absolute_import
 
 import email
 import email.parser as emailparser
 import fnmatch
 import socket
 import time
 
 from mercurial.i18n import _
 from mercurial import (
     error,
     logcmdutil,
     mail,
     patch,
     registrar,
     util,
 )
-from mercurial.utils import dateutil
+from mercurial.utils import (
+    dateutil,
+    stringutil,
+)
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 configitem('notify', 'changegroup',
     default=None,
 )
 configitem('notify', 'config',
     default=None,
 )
 configitem('notify', 'diffstat',
     default=True,
 )
 configitem('notify', 'domain',
     default=None,
 )
 configitem('notify', 'fromauthor',
     default=None,
 )
 configitem('notify', 'incoming',
     default=None,
 )
 configitem('notify', 'maxdiff',
     default=300,
 )
 configitem('notify', 'maxsubject',
     default=67,
 )
 configitem('notify', 'mbox',
     default=None,
 )
 configitem('notify', 'merge',
     default=True,
 )
 configitem('notify', 'outgoing',
     default=None,
 )
 configitem('notify', 'sources',
     default='serve',
 )
 configitem('notify', 'strip',
     default=0,
 )
 configitem('notify', 'style',
     default=None,
 )
 configitem('notify', 'template',
     default=None,
 )
 configitem('notify', 'test',
     default=True,
 )
 
 # template for single changeset can include email headers.
 single_template = '''
 Subject: changeset in {webroot}: {desc|firstline|strip}
 From: {author}
 
 changeset {node|short} in {root}
 details: {baseurl}{webroot}?cmd=changeset;node={node|short}
 description:
 \t{desc|tabindent|strip}
 '''.lstrip()
 
 # template for multiple changesets should not contain email headers,
 # because only first set of headers will be used and result will look
 # strange.
 multiple_template = '''
 changeset {node|short} in {root}
 details: {baseurl}{webroot}?cmd=changeset;node={node|short}
 summary: {desc|firstline}
 '''
 
 deftemplates = {
     'changegroup': multiple_template,
 }
 
 class notifier(object):
     '''email notification class.'''
 
     def __init__(self, ui, repo, hooktype):
         self.ui = ui
         cfg = self.ui.config('notify', 'config')
         if cfg:
             self.ui.readconfig(cfg, sections=['usersubs', 'reposubs'])
         self.repo = repo
         self.stripcount = int(self.ui.config('notify', 'strip'))
         self.root = self.strip(self.repo.root)
         self.domain = self.ui.config('notify', 'domain')
         self.mbox = self.ui.config('notify', 'mbox')
         self.test = self.ui.configbool('notify', 'test')
         self.charsets = mail._charsets(self.ui)
         self.subs = self.subscribers()
         self.merge = self.ui.configbool('notify', 'merge')
 
         mapfile = None
         template = (self.ui.config('notify', hooktype) or
                     self.ui.config('notify', 'template'))
         if not template:
             mapfile = self.ui.config('notify', 'style')
         if not mapfile and not template:
             template = deftemplates.get(hooktype) or single_template
         spec = logcmdutil.templatespec(template, mapfile)
         self.t = logcmdutil.changesettemplater(self.ui, self.repo, spec)
 
     def strip(self, path):
         '''strip leading slashes from local path, turn into web-safe path.'''
 
         path = util.pconvert(path)
         count = self.stripcount
         while count > 0:
             c = path.find('/')
             if c == -1:
                 break
             path = path[c + 1:]
             count -= 1
         return path
 
     def fixmail(self, addr):
         '''try to clean up email addresses.'''
 
-        addr = util.email(addr.strip())
+        addr = stringutil.email(addr.strip())
         if self.domain:
             a = addr.find('@localhost')
             if a != -1:
                 addr = addr[:a]
             if '@' not in addr:
                 return addr + '@' + self.domain
         return addr
 
     def subscribers(self):
         '''return list of email addresses of subscribers to this repo.'''
         subs = set()
         for user, pats in self.ui.configitems('usersubs'):
             for pat in pats.split(','):
                 if '#' in pat:
                     pat, revs = pat.split('#', 1)
                 else:
                     revs = None
                 if fnmatch.fnmatch(self.repo.root, pat.strip()):
                     subs.add((self.fixmail(user), revs))
         for pat, users in self.ui.configitems('reposubs'):
             if '#' in pat:
                 pat, revs = pat.split('#', 1)
             else:
                 revs = None
             if fnmatch.fnmatch(self.repo.root, pat):
                 for user in users.split(','):
                     subs.add((self.fixmail(user), revs))
         return [(mail.addressencode(self.ui, s, self.charsets, self.test), r)
                 for s, r in sorted(subs)]
 
     def node(self, ctx, **props):
         '''format one changeset, unless it is a suppressed merge.'''
         if not self.merge and len(ctx.parents()) > 1:
             return False
         self.t.show(ctx, changes=ctx.changeset(),
                     baseurl=self.ui.config('web', 'baseurl'),
                     root=self.repo.root, webroot=self.root, **props)
         return True
 
     def skipsource(self, source):
         '''true if incoming changes from this source should be skipped.'''
         ok_sources = self.ui.config('notify', 'sources').split()
         return source not in ok_sources
 
     def send(self, ctx, count, data):
         '''send message.'''
 
         # Select subscribers by revset
         subs = set()
         for sub, spec in self.subs:
             if spec is None:
                 subs.add(sub)
                 continue
             revs = self.repo.revs('%r and %d:', spec, ctx.rev())
             if len(revs):
                 subs.add(sub)
                 continue
         if len(subs) == 0:
             self.ui.debug('notify: no subscribers to selected repo '
                           'and revset\n')
             return
 
         p = emailparser.Parser()
         try:
             msg = p.parsestr(data)
         except email.Errors.MessageParseError as inst:
             raise error.Abort(inst)
 
         # store sender and subject
         sender, subject = msg['From'], msg['Subject']
         del msg['From'], msg['Subject']
 
         if not msg.is_multipart():
             # create fresh mime message from scratch
             # (multipart templates must take care of this themselves)
             headers = msg.items()
             payload = msg.get_payload()
             # for notification prefer readability over data precision
             msg = mail.mimeencode(self.ui, payload, self.charsets, self.test)
             # reinstate custom headers
             for k, v in headers:
                 msg[k] = v
 
         msg['Date'] = dateutil.datestr(format="%a, %d %b %Y %H:%M:%S %1%2")
 
         # try to make subject line exist and be useful
         if not subject:
             if count > 1:
                 subject = _('%s: %d new changesets') % (self.root, count)
             else:
                 s = ctx.description().lstrip().split('\n', 1)[0].rstrip()
                 subject = '%s: %s' % (self.root, s)
         maxsubject = int(self.ui.config('notify', 'maxsubject'))
         if maxsubject:
-            subject = util.ellipsis(subject, maxsubject)
+            subject = stringutil.ellipsis(subject, maxsubject)
         msg['Subject'] = mail.headencode(self.ui, subject,
                                          self.charsets, self.test)
 
         # try to make message have proper sender
         if not sender:
             sender = self.ui.config('email', 'from') or self.ui.username()
         if '@' not in sender or '@localhost' in sender:
             sender = self.fixmail(sender)
         msg['From'] = mail.addressencode(self.ui, sender,
                                          self.charsets, self.test)
 
         msg['X-Hg-Notification'] = 'changeset %s' % ctx
         if not msg['Message-Id']:
             msg['Message-Id'] = ('<hg.%s.%s.%s@%s>' %
                                  (ctx, int(time.time()),
                                   hash(self.repo.root), socket.getfqdn()))
         msg['To'] = ', '.join(sorted(subs))
 
         msgtext = msg.as_string()
         if self.test:
             self.ui.write(msgtext)
             if not msgtext.endswith('\n'):
                 self.ui.write('\n')
         else:
             self.ui.status(_('notify: sending %d subscribers %d changes\n') %
                            (len(subs), count))
-            mail.sendmail(self.ui, util.email(msg['From']),
+            mail.sendmail(self.ui, stringutil.email(msg['From']),
                           subs, msgtext, mbox=self.mbox)
 
     def diff(self, ctx, ref=None):
 
         maxdiff = int(self.ui.config('notify', 'maxdiff'))
         prev = ctx.p1().node()
         if ref:
             ref = ref.node()
         else:
             ref = ctx.node()
         chunks = patch.diff(self.repo, prev, ref,
                             opts=patch.diffallopts(self.ui))
         difflines = ''.join(chunks).splitlines()
 
         if self.ui.configbool('notify', 'diffstat'):
             s = patch.diffstat(difflines)
             # s may be nil, don't include the header if it is
             if s:
                 self.ui.write(_('\ndiffstat:\n\n%s') % s)
 
         if maxdiff == 0:
             return
         elif maxdiff > 0 and len(difflines) > maxdiff:
             msg = _('\ndiffs (truncated from %d to %d lines):\n\n')
             self.ui.write(msg % (len(difflines), maxdiff))
             difflines = difflines[:maxdiff]
         elif difflines:
             self.ui.write(_('\ndiffs (%d lines):\n\n') % len(difflines))
 
         self.ui.write("\n".join(difflines))
 
 def hook(ui, repo, hooktype, node=None, source=None, **kwargs):
     '''send email notifications to interested subscribers.
 
     if used as changegroup hook, send one email for all changesets in
     changegroup. else send one email per changeset.'''
 
     n = notifier(ui, repo, hooktype)
     ctx = repo[node]
 
     if not n.subs:
         ui.debug('notify: no subscribers to repository %s\n' % n.root)
         return
     if n.skipsource(source):
         ui.debug('notify: changes have source "%s" - skipping\n' % source)
         return
 
     ui.pushbuffer()
     data = ''
     count = 0
     author = ''
     if hooktype == 'changegroup' or hooktype == 'outgoing':
         start, end = ctx.rev(), len(repo)
         for rev in xrange(start, end):
             if n.node(repo[rev]):
                 count += 1
                 if not author:
                     author = repo[rev].user()
             else:
                 data += ui.popbuffer()
                 ui.note(_('notify: suppressing notification for merge %d:%s\n')
                         % (rev, repo[rev].hex()[:12]))
                 ui.pushbuffer()
         if count:
             n.diff(ctx, repo['tip'])
     else:
         if not n.node(ctx):
             ui.popbuffer()
             ui.note(_('notify: suppressing notification for merge %d:%s\n') %
                     (ctx.rev(), ctx.hex()[:12]))
             return
         count += 1
         n.diff(ctx)
         if not author:
             author = ctx.user()
 
     data += ui.popbuffer()
     fromauthor = ui.config('notify', 'fromauthor')
     if author and fromauthor:
         data = '\n'.join(['From: %s' % author, data])
 
     if count:
         n.send(ctx, count, data)

hgext/relink.py

@@ -1,195 +1,198
 # Mercurial extension to provide 'hg relink' command
 #
 # Copyright (C) 2007 Brendan Cully <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """recreates hardlinks between repository clones"""
 from __future__ import absolute_import
 
 import os
 import stat
 
 from mercurial.i18n import _
 from mercurial import (
     error,
     hg,
     registrar,
     util,
 )
+from mercurial.utils import (
+    stringutil,
+)
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
 @command('relink', [], _('[ORIGIN]'))
 def relink(ui, repo, origin=None, **opts):
     """recreate hardlinks between two repositories
 
     When repositories are cloned locally, their data files will be
     hardlinked so that they only use the space of a single repository.
 
     Unfortunately, subsequent pulls into either repository will break
     hardlinks for any files touched by the new changesets, even if
     both repositories end up pulling the same changes.
 
     Similarly, passing --rev to "hg clone" will fail to use any
     hardlinks, falling back to a complete copy of the source
     repository.
 
     This command lets you recreate those hardlinks and reclaim that
     wasted space.
 
     This repository will be relinked to share space with ORIGIN, which
     must be on the same local disk. If ORIGIN is omitted, looks for
     "default-relink", then "default", in [paths].
 
     Do not attempt any read operations on this repository while the
     command is running. (Both repositories will be locked against
     writes.)
     """
     if (not util.safehasattr(util, 'samefile') or
         not util.safehasattr(util, 'samedevice')):
         raise error.Abort(_('hardlinks are not supported on this system'))
     src = hg.repository(repo.baseui, ui.expandpath(origin or 'default-relink',
                                           origin or 'default'))
     ui.status(_('relinking %s to %s\n') % (src.store.path, repo.store.path))
     if repo.root == src.root:
         ui.status(_('there is nothing to relink\n'))
         return
 
     if not util.samedevice(src.store.path, repo.store.path):
         # No point in continuing
         raise error.Abort(_('source and destination are on different devices'))
 
     locallock = repo.lock()
     try:
         remotelock = src.lock()
         try:
             candidates = sorted(collect(src, ui))
             targets = prune(candidates, src.store.path, repo.store.path, ui)
             do_relink(src.store.path, repo.store.path, targets, ui)
         finally:
             remotelock.release()
     finally:
         locallock.release()
 
 def collect(src, ui):
     seplen = len(os.path.sep)
     candidates = []
     live = len(src['tip'].manifest())
     # Your average repository has some files which were deleted before
     # the tip revision. We account for that by assuming that there are
     # 3 tracked files for every 2 live files as of the tip version of
     # the repository.
     #
     # mozilla-central as of 2010-06-10 had a ratio of just over 7:5.
     total = live * 3 // 2
     src = src.store.path
     pos = 0
     ui.status(_("tip has %d files, estimated total number of files: %d\n")
               % (live, total))
     for dirpath, dirnames, filenames in os.walk(src):
         dirnames.sort()
         relpath = dirpath[len(src) + seplen:]
         for filename in sorted(filenames):
             if filename[-2:] not in ('.d', '.i'):
                 continue
             st = os.stat(os.path.join(dirpath, filename))
             if not stat.S_ISREG(st.st_mode):
                 continue
             pos += 1
             candidates.append((os.path.join(relpath, filename), st))
             ui.progress(_('collecting'), pos, filename, _('files'), total)
 
     ui.progress(_('collecting'), None)
     ui.status(_('collected %d candidate storage files\n') % len(candidates))
     return candidates
 
 def prune(candidates, src, dst, ui):
     def linkfilter(src, dst, st):
         try:
             ts = os.stat(dst)
         except OSError:
             # Destination doesn't have this file?
             return False
         if util.samefile(src, dst):
             return False
         if not util.samedevice(src, dst):
             # No point in continuing
             raise error.Abort(
                 _('source and destination are on different devices'))
         if st.st_size != ts.st_size:
             return False
         return st
 
     targets = []
     total = len(candidates)
     pos = 0
     for fn, st in candidates:
         pos += 1
         srcpath = os.path.join(src, fn)
         tgt = os.path.join(dst, fn)
         ts = linkfilter(srcpath, tgt, st)
         if not ts:
             ui.debug('not linkable: %s\n' % fn)
             continue
         targets.append((fn, ts.st_size))
         ui.progress(_('pruning'), pos, fn, _('files'), total)
 
     ui.progress(_('pruning'), None)
     ui.status(_('pruned down to %d probably relinkable files\n') % len(targets))
     return targets
 
 def do_relink(src, dst, files, ui):
     def relinkfile(src, dst):
         bak = dst + '.bak'
         os.rename(dst, bak)
         try:
             util.oslink(src, dst)
         except OSError:
             os.rename(bak, dst)
             raise
         os.remove(bak)
 
     CHUNKLEN = 65536
     relinked = 0
     savedbytes = 0
 
     pos = 0
     total = len(files)
     for f, sz in files:
         pos += 1
         source = os.path.join(src, f)
         tgt = os.path.join(dst, f)
         # Binary mode, so that read() works correctly, especially on Windows
         sfp = open(source, 'rb')
         dfp = open(tgt, 'rb')
         sin = sfp.read(CHUNKLEN)
         while sin:
             din = dfp.read(CHUNKLEN)
             if sin != din:
                 break
             sin = sfp.read(CHUNKLEN)
         sfp.close()
         dfp.close()
         if sin:
             ui.debug('not linkable: %s\n' % f)
             continue
         try:
             relinkfile(source, tgt)
             ui.progress(_('relinking'), pos, f, _('files'), total)
             relinked += 1
             savedbytes += sz
         except OSError as inst:
-            ui.warn('%s: %s\n' % (tgt, util.forcebytestr(inst)))
+            ui.warn('%s: %s\n' % (tgt, stringutil.forcebytestr(inst)))
 
     ui.progress(_('relinking'), None)
 
     ui.status(_('relinked %d files (%s reclaimed)\n') %
               (relinked, util.bytecount(savedbytes)))