merge default into stable for 2.0 code freeze

contrib/debugcmdserver.py

@@ -0,0 +1,47
+#!/usr/bin/env python
+#
+# Dumps output generated by Mercurial's command server in a formatted style to a
+# given file or stderr if '-' is specified. Output is also written in its raw
+# format to stdout.
+#
+# $ ./hg serve --cmds pipe | ./contrib/debugcmdserver.py -
+# o, 52   -> 'capabilities: getencoding runcommand\nencoding: UTF-8'
+
+import sys, struct
+
+if len(sys.argv) != 2:
+    print 'usage: debugcmdserver.py FILE'
+    sys.exit(1)
+
+outputfmt = '>cI'
+outputfmtsize = struct.calcsize(outputfmt)
+
+if sys.argv[1] == '-':
+    log = sys.stderr
+else:
+    log = open(sys.argv[1], 'a')
+
+def read(size):
+    data = sys.stdin.read(size)
+    if not data:
+        raise EOFError()
+    sys.stdout.write(data)
+    sys.stdout.flush()
+    return data
+
+try:
+    while True:
+        header = read(outputfmtsize)
+        channel, length = struct.unpack(outputfmt, header)
+        log.write('%s, %-4d' % (channel, length))
+        if channel in 'IL':
+            log.write(' -> waiting for input\n')
+        else:
+            data = read(length)
+            log.write(' -> %r\n' % data)
+        log.flush()
+except EOFError:
+    pass
+finally:
+    if log != sys.stderr:
+        log.close()

hgext/largefiles/CONTRIBUTORS

@@ -0,0 +1,4
+Greg Ward, author of the original bfiles extension
+Na'Tosha Bard of Unity Technologies
+Fog Creek Software
+Special thanks to the University of Toronto and the UCOSP program

hgext/largefiles/__init__.py

@@ -0,0 +1,94
+# Copyright 2009-2010 Gregory P. Ward
+# Copyright 2009-2010 Intelerad Medical Systems Incorporated
+# Copyright 2010-2011 Fog Creek Software
+# Copyright 2010-2011 Unity Technologies
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+'''track large binary files
+
+Large binary files tend to be not very compressible, not very
+diffable, and not at all mergeable. Such files are not handled
+efficiently by Mercurial's storage format (revlog), which is based on
+compressed binary deltas; storing large binary files as regular
+Mercurial files wastes bandwidth and disk space and increases
+Mercurial's memory usage. The largefiles extension addresses these
+problems by adding a centralized client-server layer on top of
+Mercurial: largefiles live in a *central store* out on the network
+somewhere, and you only fetch the revisions that you need when you
+need them.
+
+largefiles works by maintaining a "standin file" in .hglf/ for each
+largefile. The standins are small (41 bytes: an SHA-1 hash plus
+newline) and are tracked by Mercurial. Largefile revisions are
+identified by the SHA-1 hash of their contents, which is written to
+the standin. largefiles uses that revision ID to get/put largefile
+revisions from/to the central store. This saves both disk space and
+bandwidth, since you don't need to retrieve all historical revisions
+of large files when you clone or pull.
+
+To start a new repository or add new large binary files, just add
+--large to your ``hg add`` command. For example::
+
+  $ dd if=/dev/urandom of=randomdata count=2000
+  $ hg add --large randomdata
+  $ hg commit -m 'add randomdata as a largefile'
+
+When you push a changeset that adds/modifies largefiles to a remote
+repository, its largefile revisions will be uploaded along with it.
+Note that the remote Mercurial must also have the largefiles extension
+enabled for this to work.
+
+When you pull a changeset that affects largefiles from a remote
+repository, Mercurial behaves as normal. However, when you update to
+such a revision, any largefiles needed by that revision are downloaded
+and cached (if they have never been downloaded before). This means
+that network access may be required to update to changesets you have
+not previously updated to.
+
+If you already have large files tracked by Mercurial without the
+largefiles extension, you will need to convert your repository in
+order to benefit from largefiles. This is done with the 'hg lfconvert'
+command::
+
+  $ hg lfconvert --size 10 oldrepo newrepo
+
+In repositories that already have largefiles in them, any new file
+over 10MB will automatically be added as a largefile. To change this
+threshhold, set ``largefiles.size`` in your Mercurial config file to
+the minimum size in megabytes to track as a largefile, or use the
+--lfsize option to the add command (also in megabytes)::
+
+  [largefiles]
+  size = 2           XXX wouldn't minsize be a better name?
+
+  $ hg add --lfsize 2
+
+The ``largefiles.patterns`` config option allows you to specify a list
+of filename patterns (see ``hg help patterns``) that should always be
+tracked as largefiles::
+
+  [largefiles]
+  patterns =
+    *.jpg
+    re:.*\.(png|bmp)$
+    library.zip
+    content/audio/*
+
+Files that match one of these patterns will be added as largefiles
+regardless of their size.
+'''
+
+from mercurial import commands
+
+import lfcommands
+import reposetup
+import uisetup
+
+reposetup = reposetup.reposetup
+uisetup = uisetup.uisetup
+
+commands.norepo += " lfconvert"
+
+cmdtable = lfcommands.cmdtable

hgext/largefiles/basestore.py

@@ -0,0 +1,202
+# Copyright 2009-2010 Gregory P. Ward
+# Copyright 2009-2010 Intelerad Medical Systems Incorporated
+# Copyright 2010-2011 Fog Creek Software
+# Copyright 2010-2011 Unity Technologies
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+'''base class for store implementations and store-related utility code'''
+
+import os
+import tempfile
+import binascii
+import re
+
+from mercurial import util, node, hg
+from mercurial.i18n import _
+
+import lfutil
+
+class StoreError(Exception):
+    '''Raised when there is a problem getting files from or putting
+    files to a central store.'''
+    def __init__(self, filename, hash, url, detail):
+        self.filename = filename
+        self.hash = hash
+        self.url = url
+        self.detail = detail
+
+    def longmessage(self):
+        if self.url:
+            return ('%s: %s\n'
+                    '(failed URL: %s)\n'
+                    % (self.filename, self.detail, self.url))
+        else:
+            return ('%s: %s\n'
+                    '(no default or default-push path set in hgrc)\n'
+                    % (self.filename, self.detail))
+
+    def __str__(self):
+        return "%s: %s" % (self.url, self.detail)
+
+class basestore(object):
+    def __init__(self, ui, repo, url):
+        self.ui = ui
+        self.repo = repo
+        self.url = url
+
+    def put(self, source, hash):
+        '''Put source file into the store under <filename>/<hash>.'''
+        raise NotImplementedError('abstract method')
+
+    def exists(self, hash):
+        '''Check to see if the store contains the given hash.'''
+        raise NotImplementedError('abstract method')
+
+    def get(self, files):
+        '''Get the specified largefiles from the store and write to local
+        files under repo.root.  files is a list of (filename, hash)
+        tuples.  Return (success, missing), lists of files successfuly
+        downloaded and those not found in the store.  success is a list
+        of (filename, hash) tuples; missing is a list of filenames that
+        we could not get.  (The detailed error message will already have
+        been presented to the user, so missing is just supplied as a
+        summary.)'''
+        success = []
+        missing = []
+        ui = self.ui
+
+        at = 0
+        for filename, hash in files:
+            ui.progress(_('getting largefiles'), at, unit='lfile',
+                total=len(files))
+            at += 1
+            ui.note(_('getting %s:%s\n') % (filename, hash))
+
+            cachefilename = lfutil.cachepath(self.repo, hash)
+            cachedir = os.path.dirname(cachefilename)
+
+            # No need to pass mode='wb' to fdopen(), since mkstemp() already
+            # opened the file in binary mode.
+            (tmpfd, tmpfilename) = tempfile.mkstemp(
+                dir=cachedir, prefix=os.path.basename(filename))
+            tmpfile = os.fdopen(tmpfd, 'w')
+
+            try:
+                hhash = binascii.hexlify(self._getfile(tmpfile, filename, hash))
+            except StoreError, err:
+                ui.warn(err.longmessage())
+                hhash = ""
+
+            if hhash != hash:
+                if hhash != "":
+                    ui.warn(_('%s: data corruption (expected %s, got %s)\n')
+                            % (filename, hash, hhash))
+                tmpfile.close() # no-op if it's already closed
+                os.remove(tmpfilename)
+                missing.append(filename)
+                continue
+
+            if os.path.exists(cachefilename): # Windows
+                os.remove(cachefilename)
+            os.rename(tmpfilename, cachefilename)
+            lfutil.linktosystemcache(self.repo, hash)
+            success.append((filename, hhash))
+
+        ui.progress(_('getting largefiles'), None)
+        return (success, missing)
+
+    def verify(self, revs, contents=False):
+        '''Verify the existence (and, optionally, contents) of every big
+        file revision referenced by every changeset in revs.
+        Return 0 if all is well, non-zero on any errors.'''
+        write = self.ui.write
+        failed = False
+
+        write(_('searching %d changesets for largefiles\n') % len(revs))
+        verified = set()                # set of (filename, filenode) tuples
+
+        for rev in revs:
+            cctx = self.repo[rev]
+            cset = "%d:%s" % (cctx.rev(), node.short(cctx.node()))
+
+            failed = lfutil.any_(self._verifyfile(
+                cctx, cset, contents, standin, verified) for standin in cctx)
+
+        num_revs = len(verified)
+        num_lfiles = len(set([fname for (fname, fnode) in verified]))
+        if contents:
+            write(_('verified contents of %d revisions of %d largefiles\n')
+                  % (num_revs, num_lfiles))
+        else:
+            write(_('verified existence of %d revisions of %d largefiles\n')
+                  % (num_revs, num_lfiles))
+
+        return int(failed)
+
+    def _getfile(self, tmpfile, filename, hash):
+        '''Fetch one revision of one file from the store and write it
+        to tmpfile.  Compute the hash of the file on-the-fly as it
+        downloads and return the binary hash.  Close tmpfile.  Raise
+        StoreError if unable to download the file (e.g. it does not
+        exist in the store).'''
+        raise NotImplementedError('abstract method')
+
+    def _verifyfile(self, cctx, cset, contents, standin, verified):
+        '''Perform the actual verification of a file in the store.
+        '''
+        raise NotImplementedError('abstract method')
+
+import localstore, wirestore
+
+_storeprovider = {
+    'file':  [localstore.localstore],
+    'http':  [wirestore.wirestore],
+    'https': [wirestore.wirestore],
+    'ssh': [wirestore.wirestore],
+    }
+
+_scheme_re = re.compile(r'^([a-zA-Z0-9+-.]+)://')
+
+# During clone this function is passed the src's ui object
+# but it needs the dest's ui object so it can read out of
+# the config file. Use repo.ui instead.
+def _openstore(repo, remote=None, put=False):
+    ui = repo.ui
+
+    if not remote:
+        path = (getattr(repo, 'lfpullsource', None) or
+                ui.expandpath('default-push', 'default'))
+
+        # ui.expandpath() leaves 'default-push' and 'default' alone if
+        # they cannot be expanded: fallback to the empty string,
+        # meaning the current directory.
+        if path == 'default-push' or path == 'default':
+            path = ''
+            remote = repo
+        else:
+            remote = hg.peer(repo, {}, path)
+
+    # The path could be a scheme so use Mercurial's normal functionality
+    # to resolve the scheme to a repository and use its path
+    path = util.safehasattr(remote, 'url') and remote.url() or remote.path
+
+    match = _scheme_re.match(path)
+    if not match:                       # regular filesystem path
+        scheme = 'file'
+    else:
+        scheme = match.group(1)
+
+    try:
+        storeproviders = _storeprovider[scheme]
+    except KeyError:
+        raise util.Abort(_('unsupported URL scheme %r') % scheme)
+
+    for class_obj in storeproviders:
+        try:
+            return class_obj(ui, repo, remote)
+        except lfutil.storeprotonotcapable:
+            pass
+
+    raise util.Abort(_('%s does not appear to be a largefile store'), path)

hgext/largefiles/design.txt

@@ -0,0 +1,49
+= largefiles - manage large binary files =
+This extension is based off of Greg Ward's bfiles extension which can be found
+at http://mercurial.selenic.com/wiki/BfilesExtension.
+
+== The largefile store ==
+
+largefile stores are, in the typical use case, centralized servers that have
+every past revision of a given binary file.  Each largefile is identified by
+its sha1 hash, and all interactions with the store take one of the following
+forms.
+
+-Download a bfile with this hash
+-Upload a bfile with this hash
+-Check if the store has a bfile with this hash
+
+largefiles stores can take one of two forms:
+
+-Directories on a network file share
+-Mercurial wireproto servers, either via ssh or http (hgweb)
+
+== The Local Repository ==
+
+The local repository has a largefile cache in .hg/largefiles which holds a
+subset of the largefiles needed. On a clone only the largefiles at tip are
+downloaded. When largefiles are downloaded from the central store, a copy is
+saved in this store.
+
+== The Global Cache ==
+
+largefiles in a local repository cache are hardlinked to files in the global
+cache. Before a file is downloaded we check if it is in the global cache.
+
+== Implementation Details ==
+
+Each largefile has a standin which is in .hglf. The standin is tracked by
+Mercurial.  The standin contains the SHA1 hash of the largefile. When a
+largefile is added/removed/copied/renamed/etc the same operation is applied to
+the standin. Thus the history of the standin is the history of the largefile.
+
+For performance reasons, the contents of a standin are only updated before a
+commit.  Standins are added/removed/copied/renamed from add/remove/copy/rename
+Mercurial commands but their contents will not be updated. The contents of a
+standin will always be the hash of the largefile as of the last commit. To
+support some commands (revert) some standins are temporarily updated but will
+be changed back after the command is finished.
+
+A Mercurial dirstate object tracks the state of the largefiles. The dirstate
+uses the last modified time and current size to detect if a file has changed
+(without reading the entire contents of the file).

hgext/largefiles/lfcommands.py

@@ -0,0 +1,481
+# Copyright 2009-2010 Gregory P. Ward
+# Copyright 2009-2010 Intelerad Medical Systems Incorporated
+# Copyright 2010-2011 Fog Creek Software
+# Copyright 2010-2011 Unity Technologies
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+'''High-level command function for lfconvert, plus the cmdtable.'''
+
+import os
+import shutil
+
+from mercurial import util, match as match_, hg, node, context, error
+from mercurial.i18n import _
+
+import lfutil
+import basestore
+
+# -- Commands ----------------------------------------------------------
+
+def lfconvert(ui, src, dest, *pats, **opts):
+    '''convert a normal repository to a largefiles repository
+
+    Convert repository SOURCE to a new repository DEST, identical to
+    SOURCE except that certain files will be converted as largefiles:
+    specifically, any file that matches any PATTERN *or* whose size is
+    above the minimum size threshold is converted as a largefile. The
+    size used to determine whether or not to track a file as a
+    largefile is the size of the first version of the file. The
+    minimum size can be specified either with --size or in
+    configuration as ``largefiles.size``.
+
+    After running this command you will need to make sure that
+    largefiles is enabled anywhere you intend to push the new
+    repository.
+
+    Use --tonormal to convert largefiles back to normal files; after
+    this, the DEST repository can be used without largefiles at all.'''
+
+    if opts['tonormal']:
+        tolfile = False
+    else:
+        tolfile = True
+        size = lfutil.getminsize(ui, True, opts.get('size'), default=None)
+    try:
+        rsrc = hg.repository(ui, src)
+        if not rsrc.local():
+            raise util.Abort(_('%s is not a local Mercurial repo') % src)
+    except error.RepoError, err:
+        ui.traceback()
+        raise util.Abort(err.args[0])
+    if os.path.exists(dest):
+        if not os.path.isdir(dest):
+            raise util.Abort(_('destination %s already exists') % dest)
+        elif os.listdir(dest):
+            raise util.Abort(_('destination %s is not empty') % dest)
+    try:
+        ui.status(_('initializing destination %s\n') % dest)
+        rdst = hg.repository(ui, dest, create=True)
+        if not rdst.local():
+            raise util.Abort(_('%s is not a local Mercurial repo') % dest)
+    except error.RepoError:
+        ui.traceback()
+        raise util.Abort(_('%s is not a repo') % dest)
+
+    success = False
+    try:
+        # Lock destination to prevent modification while it is converted to.
+        # Don't need to lock src because we are just reading from its history
+        # which can't change.
+        dst_lock = rdst.lock()
+
+        # Get a list of all changesets in the source.  The easy way to do this
+        # is to simply walk the changelog, using changelog.nodesbewteen().
+        # Take a look at mercurial/revlog.py:639 for more details.
+        # Use a generator instead of a list to decrease memory usage
+        ctxs = (rsrc[ctx] for ctx in rsrc.changelog.nodesbetween(None,
+            rsrc.heads())[0])
+        revmap = {node.nullid: node.nullid}
+        if tolfile:
+            lfiles = set()
+            normalfiles = set()
+            if not pats:
+                pats = ui.config(lfutil.longname, 'patterns', default=())
+                if pats:
+                    pats = pats.split(' ')
+            if pats:
+                matcher = match_.match(rsrc.root, '', list(pats))
+            else:
+                matcher = None
+
+            lfiletohash = {}
+            for ctx in ctxs:
+                ui.progress(_('converting revisions'), ctx.rev(),
+                    unit=_('revision'), total=rsrc['tip'].rev())
+                _lfconvert_addchangeset(rsrc, rdst, ctx, revmap,
+                    lfiles, normalfiles, matcher, size, lfiletohash)
+            ui.progress(_('converting revisions'), None)
+
+            if os.path.exists(rdst.wjoin(lfutil.shortname)):
+                shutil.rmtree(rdst.wjoin(lfutil.shortname))
+
+            for f in lfiletohash.keys():
+                if os.path.isfile(rdst.wjoin(f)):
+                    os.unlink(rdst.wjoin(f))
+                try:
+                    os.removedirs(os.path.dirname(rdst.wjoin(f)))
+                except OSError:
+                    pass
+
+        else:
+            for ctx in ctxs:
+                ui.progress(_('converting revisions'), ctx.rev(),
+                    unit=_('revision'), total=rsrc['tip'].rev())
+                _addchangeset(ui, rsrc, rdst, ctx, revmap)
+
+            ui.progress(_('converting revisions'), None)
+        success = True
+    finally:
+        if not success:
+            # we failed, remove the new directory
+            shutil.rmtree(rdst.root)
+        dst_lock.release()
+
+def _addchangeset(ui, rsrc, rdst, ctx, revmap):
+ # Convert src parents to dst parents
+    parents = []
+    for p in ctx.parents():
+        parents.append(revmap[p.node()])
+    while len(parents) < 2:
+        parents.append(node.nullid)
+
+    # Generate list of changed files
+    files = set(ctx.files())
+    if node.nullid not in parents:
+        mc = ctx.manifest()
+        mp1 = ctx.parents()[0].manifest()
+        mp2 = ctx.parents()[1].manifest()
+        files |= (set(mp1) | set(mp2)) - set(mc)
+        for f in mc:
+            if mc[f] != mp1.get(f, None) or mc[f] != mp2.get(f, None):
+                files.add(f)
+
+    def getfilectx(repo, memctx, f):
+        if lfutil.standin(f) in files:
+            # if the file isn't in the manifest then it was removed
+            # or renamed, raise IOError to indicate this
+            try:
+                fctx = ctx.filectx(lfutil.standin(f))
+            except error.LookupError:
+                raise IOError()
+            renamed = fctx.renamed()
+            if renamed:
+                renamed = lfutil.splitstandin(renamed[0])
+
+            hash = fctx.data().strip()
+            path = lfutil.findfile(rsrc, hash)
+            ### TODO: What if the file is not cached?
+            data = ''
+            fd = None
+            try:
+                fd = open(path, 'rb')
+                data = fd.read()
+            finally:
+                if fd:
+                    fd.close()
+            return context.memfilectx(f, data, 'l' in fctx.flags(),
+                                      'x' in fctx.flags(), renamed)
+        else:
+            try:
+                fctx = ctx.filectx(f)
+            except error.LookupError:
+                raise IOError()
+            renamed = fctx.renamed()
+            if renamed:
+                renamed = renamed[0]
+            data = fctx.data()
+            if f == '.hgtags':
+                newdata = []
+                for line in data.splitlines():
+                    id, name = line.split(' ', 1)
+                    newdata.append('%s %s\n' % (node.hex(revmap[node.bin(id)]),
+                        name))
+                data = ''.join(newdata)
+            return context.memfilectx(f, data, 'l' in fctx.flags(),
+                                      'x' in fctx.flags(), renamed)
+
+    dstfiles = []
+    for file in files:
+        if lfutil.isstandin(file):
+            dstfiles.append(lfutil.splitstandin(file))
+        else:
+            dstfiles.append(file)
+    # Commit
+    mctx = context.memctx(rdst, parents, ctx.description(), dstfiles,
+                          getfilectx, ctx.user(), ctx.date(), ctx.extra())
+    ret = rdst.commitctx(mctx)
+    rdst.dirstate.setparents(ret)
+    revmap[ctx.node()] = rdst.changelog.tip()
+
+def _lfconvert_addchangeset(rsrc, rdst, ctx, revmap, lfiles, normalfiles,
+        matcher, size, lfiletohash):
+    # Convert src parents to dst parents
+    parents = []
+    for p in ctx.parents():
+        parents.append(revmap[p.node()])
+    while len(parents) < 2:
+        parents.append(node.nullid)
+
+    # Generate list of changed files
+    files = set(ctx.files())
+    if node.nullid not in parents:
+        mc = ctx.manifest()
+        mp1 = ctx.parents()[0].manifest()
+        mp2 = ctx.parents()[1].manifest()
+        files |= (set(mp1) | set(mp2)) - set(mc)
+        for f in mc:
+            if mc[f] != mp1.get(f, None) or mc[f] != mp2.get(f, None):
+                files.add(f)
+
+    dstfiles = []
+    for f in files:
+        if f not in lfiles and f not in normalfiles:
+            islfile = _islfile(f, ctx, matcher, size)
+            # If this file was renamed or copied then copy
+            # the lfileness of its predecessor
+            if f in ctx.manifest():
+                fctx = ctx.filectx(f)
+                renamed = fctx.renamed()
+                renamedlfile = renamed and renamed[0] in lfiles
+                islfile |= renamedlfile
+                if 'l' in fctx.flags():
+                    if renamedlfile:
+                        raise util.Abort(
+                            _('Renamed/copied largefile %s becomes symlink')
+                            % f)
+                    islfile = False
+            if islfile:
+                lfiles.add(f)
+            else:
+                normalfiles.add(f)
+
+        if f in lfiles:
+            dstfiles.append(lfutil.standin(f))
+            # largefile in manifest if it has not been removed/renamed
+            if f in ctx.manifest():
+                if 'l' in ctx.filectx(f).flags():
+                    if renamed and renamed[0] in lfiles:
+                        raise util.Abort(_('largefile %s becomes symlink') % f)
+
+                # largefile was modified, update standins
+                fullpath = rdst.wjoin(f)
+                lfutil.createdir(os.path.dirname(fullpath))
+                m = util.sha1('')
+                m.update(ctx[f].data())
+                hash = m.hexdigest()
+                if f not in lfiletohash or lfiletohash[f] != hash:
+                    try:
+                        fd = open(fullpath, 'wb')
+                        fd.write(ctx[f].data())
+                    finally:
+                        if fd:
+                            fd.close()
+                    executable = 'x' in ctx[f].flags()
+                    os.chmod(fullpath, lfutil.getmode(executable))
+                    lfutil.writestandin(rdst, lfutil.standin(f), hash,
+                        executable)
+                    lfiletohash[f] = hash
+        else:
+            # normal file
+            dstfiles.append(f)
+
+    def getfilectx(repo, memctx, f):
+        if lfutil.isstandin(f):
+            # if the file isn't in the manifest then it was removed
+            # or renamed, raise IOError to indicate this
+            srcfname = lfutil.splitstandin(f)
+            try:
+                fctx = ctx.filectx(srcfname)
+            except error.LookupError:
+                raise IOError()
+            renamed = fctx.renamed()
+            if renamed:
+                # standin is always a largefile because largefile-ness
+                # doesn't change after rename or copy
+                renamed = lfutil.standin(renamed[0])
+
+            return context.memfilectx(f, lfiletohash[srcfname], 'l' in
+                fctx.flags(), 'x' in fctx.flags(), renamed)
+        else:
+            try:
+                fctx = ctx.filectx(f)
+            except error.LookupError:
+                raise IOError()
+            renamed = fctx.renamed()
+            if renamed:
+                renamed = renamed[0]
+
+            data = fctx.data()
+            if f == '.hgtags':
+                newdata = []
+                for line in data.splitlines():
+                    id, name = line.split(' ', 1)
+                    newdata.append('%s %s\n' % (node.hex(revmap[node.bin(id)]),
+                        name))
+                data = ''.join(newdata)
+            return context.memfilectx(f, data, 'l' in fctx.flags(),
+                                      'x' in fctx.flags(), renamed)
+
+    # Commit
+    mctx = context.memctx(rdst, parents, ctx.description(), dstfiles,
+                          getfilectx, ctx.user(), ctx.date(), ctx.extra())
+    ret = rdst.commitctx(mctx)
+    rdst.dirstate.setparents(ret)
+    revmap[ctx.node()] = rdst.changelog.tip()
+
+def _islfile(file, ctx, matcher, size):
+    '''Return true if file should be considered a largefile, i.e.
+    matcher matches it or it is larger than size.'''
+    # never store special .hg* files as largefiles
+    if file == '.hgtags' or file == '.hgignore' or file == '.hgsigs':
+        return False
+    if matcher and matcher(file):
+        return True
+    try:
+        return ctx.filectx(file).size() >= size * 1024 * 1024
+    except error.LookupError:
+        return False
+
+def uploadlfiles(ui, rsrc, rdst, files):
+    '''upload largefiles to the central store'''
+
+    # Don't upload locally. All largefiles are in the system wide cache
+    # so the other repo can just get them from there.
+    if not files or rdst.local():
+        return
+
+    store = basestore._openstore(rsrc, rdst, put=True)
+
+    at = 0
+    files = filter(lambda h: not store.exists(h), files)
+    for hash in files:
+        ui.progress(_('uploading largefiles'), at, unit='largefile',
+                    total=len(files))
+        source = lfutil.findfile(rsrc, hash)
+        if not source:
+            raise util.Abort(_('largefile %s missing from store'
+                               ' (needs to be uploaded)') % hash)
+        # XXX check for errors here
+        store.put(source, hash)
+        at += 1
+    ui.progress(_('uploading largefiles'), None)
+
+def verifylfiles(ui, repo, all=False, contents=False):
+    '''Verify that every big file revision in the current changeset
+    exists in the central store.  With --contents, also verify that
+    the contents of each big file revision are correct (SHA-1 hash
+    matches the revision ID).  With --all, check every changeset in
+    this repository.'''
+    if all:
+        # Pass a list to the function rather than an iterator because we know a
+        # list will work.
+        revs = range(len(repo))
+    else:
+        revs = ['.']
+
+    store = basestore._openstore(repo)
+    return store.verify(revs, contents=contents)
+
+def cachelfiles(ui, repo, node):
+    '''cachelfiles ensures that all largefiles needed by the specified revision
+    are present in the repository's largefile cache.
+
+    returns a tuple (cached, missing).  cached is the list of files downloaded
+    by this operation; missing is the list of files that were needed but could
+    not be found.'''
+    lfiles = lfutil.listlfiles(repo, node)
+    toget = []
+
+    for lfile in lfiles:
+        expectedhash = repo[node][lfutil.standin(lfile)].data().strip()
+        # if it exists and its hash matches, it might have been locally
+        # modified before updating and the user chose 'local'.  in this case,
+        # it will not be in any store, so don't look for it.
+        if ((not os.path.exists(repo.wjoin(lfile)) or
+             expectedhash != lfutil.hashfile(repo.wjoin(lfile))) and
+            not lfutil.findfile(repo, expectedhash)):
+            toget.append((lfile, expectedhash))
+
+    if toget:
+        store = basestore._openstore(repo)
+        ret = store.get(toget)
+        return ret
+
+    return ([], [])
+
+def updatelfiles(ui, repo, filelist=None, printmessage=True):
+    wlock = repo.wlock()
+    try:
+        lfdirstate = lfutil.openlfdirstate(ui, repo)
+        lfiles = set(lfutil.listlfiles(repo)) | set(lfdirstate)
+
+        if filelist is not None:
+            lfiles = [f for f in lfiles if f in filelist]
+
+        printed = False
+        if printmessage and lfiles:
+            ui.status(_('getting changed largefiles\n'))
+            printed = True
+            cachelfiles(ui, repo, '.')
+
+        updated, removed = 0, 0
+        for i in map(lambda f: _updatelfile(repo, lfdirstate, f), lfiles):
+            # increment the appropriate counter according to _updatelfile's
+            # return value
+            updated += i > 0 and i or 0
+            removed -= i < 0 and i or 0
+            if printmessage and (removed or updated) and not printed:
+                ui.status(_('getting changed largefiles\n'))
+                printed = True
+
+        lfdirstate.write()
+        if printed and printmessage:
+            ui.status(_('%d largefiles updated, %d removed\n') % (updated,
+                removed))
+    finally:
+        wlock.release()
+
+def _updatelfile(repo, lfdirstate, lfile):
+    '''updates a single largefile and copies the state of its standin from
+    the repository's dirstate to its state in the lfdirstate.
+
+    returns 1 if the file was modified, -1 if the file was removed, 0 if the
+    file was unchanged, and None if the needed largefile was missing from the
+    cache.'''
+    ret = 0
+    abslfile = repo.wjoin(lfile)
+    absstandin = repo.wjoin(lfutil.standin(lfile))
+    if os.path.exists(absstandin):
+        if os.path.exists(absstandin+'.orig'):
+            shutil.copyfile(abslfile, abslfile+'.orig')
+        expecthash = lfutil.readstandin(repo, lfile)
+        if (expecthash != '' and
+            (not os.path.exists(abslfile) or
+             expecthash != lfutil.hashfile(abslfile))):
+            if not lfutil.copyfromcache(repo, expecthash, lfile):
+                return None # don't try to set the mode or update the dirstate
+            ret = 1
+        mode = os.stat(absstandin).st_mode
+        if mode != os.stat(abslfile).st_mode:
+            os.chmod(abslfile, mode)
+            ret = 1
+    else:
+        if os.path.exists(abslfile):
+            os.unlink(abslfile)
+            ret = -1
+    state = repo.dirstate[lfutil.standin(lfile)]
+    if state == 'n':
+        lfdirstate.normal(lfile)
+    elif state == 'r':
+        lfdirstate.remove(lfile)
+    elif state == 'a':
+        lfdirstate.add(lfile)
+    elif state == '?':
+        lfdirstate.drop(lfile)
+    return ret
+
+# -- hg commands declarations ------------------------------------------------
+
+cmdtable = {
+    'lfconvert': (lfconvert,
+                  [('s', 'size', '',
+                    _('minimum size (MB) for files to be converted '
+                      'as largefiles'),
+                    'SIZE'),
+                  ('', 'tonormal', False,
+                   _('convert from a largefiles repo to a normal repo')),
+                  ],
+                  _('hg lfconvert SOURCE DEST [FILE ...]')),
+    }

hgext/largefiles/lfutil.py

@@ -0,0 +1,448
+# Copyright 2009-2010 Gregory P. Ward
+# Copyright 2009-2010 Intelerad Medical Systems Incorporated
+# Copyright 2010-2011 Fog Creek Software
+# Copyright 2010-2011 Unity Technologies
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+'''largefiles utility code: must not import other modules in this package.'''
+
+import os
+import errno
+import shutil
+import stat
+import hashlib
+
+from mercurial import dirstate, httpconnection, match as match_, util, scmutil
+from mercurial.i18n import _
+
+shortname = '.hglf'
+longname = 'largefiles'
+
+
+# -- Portability wrappers ----------------------------------------------
+
+def dirstate_walk(dirstate, matcher, unknown=False, ignored=False):
+    return dirstate.walk(matcher, [], unknown, ignored)
+
+def repo_add(repo, list):
+    add = repo[None].add
+    return add(list)
+
+def repo_remove(repo, list, unlink=False):
+    def remove(list, unlink):
+        wlock = repo.wlock()
+        try:
+            if unlink:
+                for f in list:
+                    try:
+                        util.unlinkpath(repo.wjoin(f))
+                    except OSError, inst:
+                        if inst.errno != errno.ENOENT:
+                            raise
+            repo[None].forget(list)
+        finally:
+            wlock.release()
+    return remove(list, unlink=unlink)
+
+def repo_forget(repo, list):
+    forget = repo[None].forget
+    return forget(list)
+
+def findoutgoing(repo, remote, force):
+    from mercurial import discovery
+    common, _anyinc, _heads = discovery.findcommonincoming(repo,
+        remote, force=force)
+    return repo.changelog.findmissing(common)
+
+# -- Private worker functions ------------------------------------------
+
+def getminsize(ui, assumelfiles, opt, default=10):
+    lfsize = opt
+    if not lfsize and assumelfiles:
+        lfsize = ui.config(longname, 'size', default=default)
+    if lfsize:
+        try:
+            lfsize = float(lfsize)
+        except ValueError:
+            raise util.Abort(_('largefiles: size must be number (not %s)\n')
+                             % lfsize)
+    if lfsize is None:
+        raise util.Abort(_('minimum size for largefiles must be specified'))
+    return lfsize
+
+def link(src, dest):
+    try:
+        util.oslink(src, dest)
+    except OSError:
+        # if hardlinks fail, fallback on copy
+        shutil.copyfile(src, dest)
+        os.chmod(dest, os.stat(src).st_mode)
+
+def systemcachepath(ui, hash):
+    path = ui.config(longname, 'systemcache', None)
+    if path:
+        path = os.path.join(path, hash)
+    else:
+        if os.name == 'nt':
+            appdata = os.getenv('LOCALAPPDATA', os.getenv('APPDATA'))
+            path = os.path.join(appdata, longname, hash)
+        elif os.name == 'posix':
+            path = os.path.join(os.getenv('HOME'), '.' + longname, hash)
+        else:
+            raise util.Abort(_('unknown operating system: %s\n') % os.name)
+    return path
+
+def insystemcache(ui, hash):
+    return os.path.exists(systemcachepath(ui, hash))
+
+def findfile(repo, hash):
+    if incache(repo, hash):
+        repo.ui.note(_('Found %s in cache\n') % hash)
+        return cachepath(repo, hash)
+    if insystemcache(repo.ui, hash):
+        repo.ui.note(_('Found %s in system cache\n') % hash)
+        return systemcachepath(repo.ui, hash)
+    return None
+
+class largefiles_dirstate(dirstate.dirstate):
+    def __getitem__(self, key):
+        return super(largefiles_dirstate, self).__getitem__(unixpath(key))
+    def normal(self, f):
+        return super(largefiles_dirstate, self).normal(unixpath(f))
+    def remove(self, f):
+        return super(largefiles_dirstate, self).remove(unixpath(f))
+    def add(self, f):
+        return super(largefiles_dirstate, self).add(unixpath(f))
+    def drop(self, f):
+        return super(largefiles_dirstate, self).drop(unixpath(f))
+    def forget(self, f):
+        return super(largefiles_dirstate, self).forget(unixpath(f))
+
+def openlfdirstate(ui, repo):
+    '''
+    Return a dirstate object that tracks largefiles: i.e. its root is
+    the repo root, but it is saved in .hg/largefiles/dirstate.
+    '''
+    admin = repo.join(longname)
+    opener = scmutil.opener(admin)
+    if util.safehasattr(repo.dirstate, '_validate'):
+        lfdirstate = largefiles_dirstate(opener, ui, repo.root,
+            repo.dirstate._validate)
+    else:
+        lfdirstate = largefiles_dirstate(opener, ui, repo.root)
+
+    # If the largefiles dirstate does not exist, populate and create
+    # it. This ensures that we create it on the first meaningful
+    # largefiles operation in a new clone. It also gives us an easy
+    # way to forcibly rebuild largefiles state:
+    #   rm .hg/largefiles/dirstate && hg status
+    # Or even, if things are really messed up:
+    #   rm -rf .hg/largefiles && hg status
+    if not os.path.exists(os.path.join(admin, 'dirstate')):
+        util.makedirs(admin)
+        matcher = getstandinmatcher(repo)
+        for standin in dirstate_walk(repo.dirstate, matcher):
+            lfile = splitstandin(standin)
+            hash = readstandin(repo, lfile)
+            lfdirstate.normallookup(lfile)
+            try:
+                if hash == hashfile(lfile):
+                    lfdirstate.normal(lfile)
+            except IOError, err:
+                if err.errno != errno.ENOENT:
+                    raise
+
+        lfdirstate.write()
+
+    return lfdirstate
+
+def lfdirstate_status(lfdirstate, repo, rev):
+    wlock = repo.wlock()
+    try:
+        match = match_.always(repo.root, repo.getcwd())
+        s = lfdirstate.status(match, [], False, False, False)
+        unsure, modified, added, removed, missing, unknown, ignored, clean = s
+        for lfile in unsure:
+            if repo[rev][standin(lfile)].data().strip() != \
+                    hashfile(repo.wjoin(lfile)):
+                modified.append(lfile)
+            else:
+                clean.append(lfile)
+                lfdirstate.normal(lfile)
+        lfdirstate.write()
+    finally:
+        wlock.release()
+    return (modified, added, removed, missing, unknown, ignored, clean)
+
+def listlfiles(repo, rev=None, matcher=None):
+    '''return a list of largefiles in the working copy or the
+    specified changeset'''
+
+    if matcher is None:
+        matcher = getstandinmatcher(repo)
+
+    # ignore unknown files in working directory
+    return [splitstandin(f)
+            for f in repo[rev].walk(matcher)
+            if rev is not None or repo.dirstate[f] != '?']
+
+def incache(repo, hash):
+    return os.path.exists(cachepath(repo, hash))
+
+def createdir(dir):
+    if not os.path.exists(dir):
+        os.makedirs(dir)
+
+def cachepath(repo, hash):
+    return repo.join(os.path.join(longname, hash))
+
+def copyfromcache(repo, hash, filename):
+    '''Copy the specified largefile from the repo or system cache to
+    filename in the repository. Return true on success or false if the
+    file was not found in either cache (which should not happened:
+    this is meant to be called only after ensuring that the needed
+    largefile exists in the cache).'''
+    path = findfile(repo, hash)
+    if path is None:
+        return False
+    util.makedirs(os.path.dirname(repo.wjoin(filename)))
+    shutil.copy(path, repo.wjoin(filename))
+    return True
+
+def copytocache(repo, rev, file, uploaded=False):
+    hash = readstandin(repo, file)
+    if incache(repo, hash):
+        return
+    copytocacheabsolute(repo, repo.wjoin(file), hash)
+
+def copytocacheabsolute(repo, file, hash):
+    createdir(os.path.dirname(cachepath(repo, hash)))
+    if insystemcache(repo.ui, hash):
+        link(systemcachepath(repo.ui, hash), cachepath(repo, hash))
+    else:
+        shutil.copyfile(file, cachepath(repo, hash))
+        os.chmod(cachepath(repo, hash), os.stat(file).st_mode)
+        linktosystemcache(repo, hash)
+
+def linktosystemcache(repo, hash):
+    createdir(os.path.dirname(systemcachepath(repo.ui, hash)))
+    link(cachepath(repo, hash), systemcachepath(repo.ui, hash))
+
+def getstandinmatcher(repo, pats=[], opts={}):
+    '''Return a match object that applies pats to the standin directory'''
+    standindir = repo.pathto(shortname)
+    if pats:
+        # patterns supplied: search standin directory relative to current dir
+        cwd = repo.getcwd()
+        if os.path.isabs(cwd):
+            # cwd is an absolute path for hg -R <reponame>
+            # work relative to the repository root in this case
+            cwd = ''
+        pats = [os.path.join(standindir, cwd, pat) for pat in pats]
+    elif os.path.isdir(standindir):
+        # no patterns: relative to repo root
+        pats = [standindir]
+    else:
+        # no patterns and no standin dir: return matcher that matches nothing
+        match = match_.match(repo.root, None, [], exact=True)
+        match.matchfn = lambda f: False
+        return match
+    return getmatcher(repo, pats, opts, showbad=False)
+
+def getmatcher(repo, pats=[], opts={}, showbad=True):
+    '''Wrapper around scmutil.match() that adds showbad: if false,
+    neuter the match object's bad() method so it does not print any
+    warnings about missing files or directories.'''
+    match = scmutil.match(repo[None], pats, opts)
+
+    if not showbad:
+        match.bad = lambda f, msg: None
+    return match
+
+def composestandinmatcher(repo, rmatcher):
+    '''Return a matcher that accepts standins corresponding to the
+    files accepted by rmatcher. Pass the list of files in the matcher
+    as the paths specified by the user.'''
+    smatcher = getstandinmatcher(repo, rmatcher.files())
+    isstandin = smatcher.matchfn
+    def composed_matchfn(f):
+        return isstandin(f) and rmatcher.matchfn(splitstandin(f))
+    smatcher.matchfn = composed_matchfn
+
+    return smatcher
+
+def standin(filename):
+    '''Return the repo-relative path to the standin for the specified big
+    file.'''
+    # Notes:
+    # 1) Most callers want an absolute path, but _create_standin() needs
+    #    it repo-relative so lfadd() can pass it to repo_add().  So leave
+    #    it up to the caller to use repo.wjoin() to get an absolute path.
+    # 2) Join with '/' because that's what dirstate always uses, even on
+    #    Windows. Change existing separator to '/' first in case we are
+    #    passed filenames from an external source (like the command line).
+    return shortname + '/' + filename.replace(os.sep, '/')
+
+def isstandin(filename):
+    '''Return true if filename is a big file standin. filename must be
+    in Mercurial's internal form (slash-separated).'''
+    return filename.startswith(shortname + '/')
+
+def splitstandin(filename):
+    # Split on / because that's what dirstate always uses, even on Windows.
+    # Change local separator to / first just in case we are passed filenames
+    # from an external source (like the command line).
+    bits = filename.replace(os.sep, '/').split('/', 1)
+    if len(bits) == 2 and bits[0] == shortname:
+        return bits[1]
+    else:
+        return None
+
+def updatestandin(repo, standin):
+    file = repo.wjoin(splitstandin(standin))
+    if os.path.exists(file):
+        hash = hashfile(file)
+        executable = getexecutable(file)
+        writestandin(repo, standin, hash, executable)
+
+def readstandin(repo, filename, node=None):
+    '''read hex hash from standin for filename at given node, or working
+    directory if no node is given'''
+    return repo[node][standin(filename)].data().strip()
+
+def writestandin(repo, standin, hash, executable):
+    '''write hash to <repo.root>/<standin>'''
+    writehash(hash, repo.wjoin(standin), executable)
+
+def copyandhash(instream, outfile):
+    '''Read bytes from instream (iterable) and write them to outfile,
+    computing the SHA-1 hash of the data along the way.  Close outfile
+    when done and return the binary hash.'''
+    hasher = util.sha1('')
+    for data in instream:
+        hasher.update(data)
+        outfile.write(data)
+
+    # Blecch: closing a file that somebody else opened is rude and
+    # wrong. But it's so darn convenient and practical! After all,
+    # outfile was opened just to copy and hash.
+    outfile.close()
+
+    return hasher.digest()
+
+def hashrepofile(repo, file):
+    return hashfile(repo.wjoin(file))
+
+def hashfile(file):
+    if not os.path.exists(file):
+        return ''
+    hasher = util.sha1('')
+    fd = open(file, 'rb')
+    for data in blockstream(fd):
+        hasher.update(data)
+    fd.close()
+    return hasher.hexdigest()
+
+class limitreader(object):
+    def __init__(self, f, limit):
+        self.f = f
+        self.limit = limit
+
+    def read(self, length):
+        if self.limit == 0:
+            return ''
+        length = length > self.limit and self.limit or length
+        self.limit -= length
+        return self.f.read(length)
+
+    def close(self):
+        pass
+
+def blockstream(infile, blocksize=128 * 1024):
+    """Generator that yields blocks of data from infile and closes infile."""
+    while True:
+        data = infile.read(blocksize)
+        if not data:
+            break
+        yield data
+    # same blecch as copyandhash() above
+    infile.close()
+
+def readhash(filename):
+    rfile = open(filename, 'rb')
+    hash = rfile.read(40)
+    rfile.close()
+    if len(hash) < 40:
+        raise util.Abort(_('bad hash in \'%s\' (only %d bytes long)')
+                         % (filename, len(hash)))
+    return hash
+
+def writehash(hash, filename, executable):
+    util.makedirs(os.path.dirname(filename))
+    if os.path.exists(filename):
+        os.unlink(filename)
+    wfile = open(filename, 'wb')
+
+    try:
+        wfile.write(hash)
+        wfile.write('\n')
+    finally:
+        wfile.close()
+    if os.path.exists(filename):
+        os.chmod(filename, getmode(executable))
+
+def getexecutable(filename):
+    mode = os.stat(filename).st_mode
+    return ((mode & stat.S_IXUSR) and
+            (mode & stat.S_IXGRP) and
+            (mode & stat.S_IXOTH))
+
+def getmode(executable):
+    if executable:
+        return 0755
+    else:
+        return 0644
+
+def urljoin(first, second, *arg):
+    def join(left, right):
+        if not left.endswith('/'):
+            left += '/'
+        if right.startswith('/'):
+            right = right[1:]
+        return left + right
+
+    url = join(first, second)
+    for a in arg:
+        url = join(url, a)
+    return url
+
+def hexsha1(data):
+    """hexsha1 returns the hex-encoded sha1 sum of the data in the file-like
+    object data"""
+    h = hashlib.sha1()
+    for chunk in util.filechunkiter(data):
+        h.update(chunk)
+    return h.hexdigest()
+
+def httpsendfile(ui, filename):
+    return httpconnection.httpsendfile(ui, filename, 'rb')
+
+def unixpath(path):
+    '''Return a version of path normalized for use with the lfdirstate.'''
+    return os.path.normpath(path).replace(os.sep, '/')
+
+def islfilesrepo(repo):
+    return ('largefiles' in repo.requirements and
+            any_(shortname + '/' in f[0] for f in repo.store.datafiles()))
+
+def any_(gen):
+    for x in gen:
+        if x:
+            return True
+    return False
+
+class storeprotonotcapable(BaseException):
+    def __init__(self, storetypes):
+        self.storetypes = storetypes

hgext/largefiles/localstore.py

@@ -0,0 +1,71
+# Copyright 2009-2010 Gregory P. Ward
+# Copyright 2009-2010 Intelerad Medical Systems Incorporated
+# Copyright 2010-2011 Fog Creek Software
+# Copyright 2010-2011 Unity Technologies
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+'''store class for local filesystem'''
+
+import os
+
+from mercurial import util
+from mercurial.i18n import _
+
+import lfutil
+import basestore
+
+class localstore(basestore.basestore):
+    '''Because there is a system-wide cache, the local store always
+    uses that cache. Since the cache is updated elsewhere, we can
+    just read from it here as if it were the store.'''
+
+    def __init__(self, ui, repo, remote):
+        url = os.path.join(remote.path, '.hg', lfutil.longname)
+        super(localstore, self).__init__(ui, repo, util.expandpath(url))
+
+    def put(self, source, filename, hash):
+        '''Any file that is put must already be in the system-wide
+        cache so do nothing.'''
+        return
+
+    def exists(self, hash):
+        return lfutil.insystemcache(self.repo.ui, hash)
+
+    def _getfile(self, tmpfile, filename, hash):
+        if lfutil.insystemcache(self.ui, hash):
+            return lfutil.systemcachepath(self.ui, hash)
+        raise basestore.StoreError(filename, hash, '',
+            _("Can't get file locally"))
+
+    def _verifyfile(self, cctx, cset, contents, standin, verified):
+        filename = lfutil.splitstandin(standin)
+        if not filename:
+            return False
+        fctx = cctx[standin]
+        key = (filename, fctx.filenode())
+        if key in verified:
+            return False
+
+        expecthash = fctx.data()[0:40]
+        verified.add(key)
+        if not lfutil.insystemcache(self.ui, expecthash):
+            self.ui.warn(
+                _('changeset %s: %s missing\n'
+                  '  (looked for hash %s)\n')
+                % (cset, filename, expecthash))
+            return True                 # failed
+
+        if contents:
+            storepath = lfutil.systemcachepath(self.ui, expecthash)
+            actualhash = lfutil.hashfile(storepath)
+            if actualhash != expecthash:
+                self.ui.warn(
+                    _('changeset %s: %s: contents differ\n'
+                      '  (%s:\n'
+                      '  expected hash %s,\n'
+                      '  but got %s)\n')
+                    % (cset, filename, storepath, expecthash, actualhash))
+                return True             # failed
+        return False

hgext/largefiles/overrides.py

@@ -0,0 +1,830
+# Copyright 2009-2010 Gregory P. Ward
+# Copyright 2009-2010 Intelerad Medical Systems Incorporated
+# Copyright 2010-2011 Fog Creek Software
+# Copyright 2010-2011 Unity Technologies
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+'''Overridden Mercurial commands and functions for the largefiles extension'''
+
+import os
+import copy
+
+from mercurial import hg, commands, util, cmdutil, match as match_, node, \
+        archival, error, merge
+from mercurial.i18n import _
+from mercurial.node import hex
+from hgext import rebase
+import lfutil
+
+try:
+    from mercurial import scmutil
+except ImportError:
+    pass
+
+import lfutil
+import lfcommands
+
+def installnormalfilesmatchfn(manifest):
+    '''overrides scmutil.match so that the matcher it returns will ignore all
+    largefiles'''
+    oldmatch = None # for the closure
+    def override_match(repo, pats=[], opts={}, globbed=False,
+            default='relpath'):
+        match = oldmatch(repo, pats, opts, globbed, default)
+        m = copy.copy(match)
+        notlfile = lambda f: not (lfutil.isstandin(f) or lfutil.standin(f) in
+                manifest)
+        m._files = filter(notlfile, m._files)
+        m._fmap = set(m._files)
+        orig_matchfn = m.matchfn
+        m.matchfn = lambda f: notlfile(f) and orig_matchfn(f) or None
+        return m
+    oldmatch = installmatchfn(override_match)
+
+def installmatchfn(f):
+    oldmatch = scmutil.match
+    setattr(f, 'oldmatch', oldmatch)
+    scmutil.match = f
+    return oldmatch
+
+def restorematchfn():
+    '''restores scmutil.match to what it was before installnormalfilesmatchfn
+    was called.  no-op if scmutil.match is its original function.
+
+    Note that n calls to installnormalfilesmatchfn will require n calls to
+    restore matchfn to reverse'''
+    scmutil.match = getattr(scmutil.match, 'oldmatch', scmutil.match)
+
+# -- Wrappers: modify existing commands --------------------------------
+
+# Add works by going through the files that the user wanted to add and
+# checking if they should be added as largefiles. Then it makes a new
+# matcher which matches only the normal files and runs the original
+# version of add.
+def override_add(orig, ui, repo, *pats, **opts):
+    large = opts.pop('large', None)
+    lfsize = lfutil.getminsize(
+        ui, lfutil.islfilesrepo(repo), opts.pop('lfsize', None))
+
+    lfmatcher = None
+    if os.path.exists(repo.wjoin(lfutil.shortname)):
+        lfpats = ui.configlist(lfutil.longname, 'patterns', default=[])
+        if lfpats:
+            lfmatcher = match_.match(repo.root, '', list(lfpats))
+
+    lfnames = []
+    m = scmutil.match(repo[None], pats, opts)
+    m.bad = lambda x, y: None
+    wctx = repo[None]
+    for f in repo.walk(m):
+        exact = m.exact(f)
+        lfile = lfutil.standin(f) in wctx
+        nfile = f in wctx
+        exists = lfile or nfile
+
+        # Don't warn the user when they attempt to add a normal tracked file.
+        # The normal add code will do that for us.
+        if exact and exists:
+            if lfile:
+                ui.warn(_('%s already a largefile\n') % f)
+            continue
+
+        if exact or not exists:
+            abovemin = (lfsize and
+                        os.path.getsize(repo.wjoin(f)) >= lfsize * 1024 * 1024)
+            if large or abovemin or (lfmatcher and lfmatcher(f)):
+                lfnames.append(f)
+                if ui.verbose or not exact:
+                    ui.status(_('adding %s as a largefile\n') % m.rel(f))
+
+    bad = []
+    standins = []
+
+    # Need to lock, otherwise there could be a race condition between
+    # when standins are created and added to the repo.
+    wlock = repo.wlock()
+    try:
+        if not opts.get('dry_run'):
+            lfdirstate = lfutil.openlfdirstate(ui, repo)
+            for f in lfnames:
+                standinname = lfutil.standin(f)
+                lfutil.writestandin(repo, standinname, hash='',
+                    executable=lfutil.getexecutable(repo.wjoin(f)))
+                standins.append(standinname)
+                if lfdirstate[f] == 'r':
+                    lfdirstate.normallookup(f)
+                else:
+                    lfdirstate.add(f)
+            lfdirstate.write()
+            bad += [lfutil.splitstandin(f)
+                    for f in lfutil.repo_add(repo, standins)
+                    if f in m.files()]
+    finally:
+        wlock.release()
+
+    installnormalfilesmatchfn(repo[None].manifest())
+    result = orig(ui, repo, *pats, **opts)
+    restorematchfn()
+
+    return (result == 1 or bad) and 1 or 0
+
+def override_remove(orig, ui, repo, *pats, **opts):
+    manifest = repo[None].manifest()
+    installnormalfilesmatchfn(manifest)
+    orig(ui, repo, *pats, **opts)
+    restorematchfn()
+
+    after, force = opts.get('after'), opts.get('force')
+    if not pats and not after:
+        raise util.Abort(_('no files specified'))
+    m = scmutil.match(repo[None], pats, opts)
+    try:
+        repo.lfstatus = True
+        s = repo.status(match=m, clean=True)
+    finally:
+        repo.lfstatus = False
+    modified, added, deleted, clean = [[f for f in list
+                                        if lfutil.standin(f) in manifest]
+                                       for list in [s[0], s[1], s[3], s[6]]]
+
+    def warn(files, reason):
+        for f in files:
+            ui.warn(_('not removing %s: file %s (use -f to force removal)\n')
+                    % (m.rel(f), reason))
+
+    if force:
+        remove, forget = modified + deleted + clean, added
+    elif after:
+        remove, forget = deleted, []
+        warn(modified + added + clean, _('still exists'))
+    else:
+        remove, forget = deleted + clean, []
+        warn(modified, _('is modified'))
+        warn(added, _('has been marked for add'))
+
+    for f in sorted(remove + forget):
+        if ui.verbose or not m.exact(f):
+            ui.status(_('removing %s\n') % m.rel(f))
+
+    # Need to lock because standin files are deleted then removed from the
+    # repository and we could race inbetween.
+    wlock = repo.wlock()
+    try:
+        lfdirstate = lfutil.openlfdirstate(ui, repo)
+        for f in remove:
+            if not after:
+                os.unlink(repo.wjoin(f))
+                currentdir = os.path.split(f)[0]
+                while currentdir and not os.listdir(repo.wjoin(currentdir)):
+                    os.rmdir(repo.wjoin(currentdir))
+                    currentdir = os.path.split(currentdir)[0]
+            lfdirstate.remove(f)
+        lfdirstate.write()
+
+        forget = [lfutil.standin(f) for f in forget]
+        remove = [lfutil.standin(f) for f in remove]
+        lfutil.repo_forget(repo, forget)
+        lfutil.repo_remove(repo, remove, unlink=True)
+    finally:
+        wlock.release()
+
+def override_status(orig, ui, repo, *pats, **opts):
+    try:
+        repo.lfstatus = True
+        return orig(ui, repo, *pats, **opts)
+    finally:
+        repo.lfstatus = False
+
+def override_log(orig, ui, repo, *pats, **opts):
+    try:
+        repo.lfstatus = True
+        orig(ui, repo, *pats, **opts)
+    finally:
+        repo.lfstatus = False
+
+def override_verify(orig, ui, repo, *pats, **opts):
+    large = opts.pop('large', False)
+    all = opts.pop('lfa', False)
+    contents = opts.pop('lfc', False)
+
+    result = orig(ui, repo, *pats, **opts)
+    if large:
+        result = result or lfcommands.verifylfiles(ui, repo, all, contents)
+    return result
+
+# Override needs to refresh standins so that update's normal merge
+# will go through properly. Then the other update hook (overriding repo.update)
+# will get the new files. Filemerge is also overriden so that the merge
+# will merge standins correctly.
+def override_update(orig, ui, repo, *pats, **opts):
+    lfdirstate = lfutil.openlfdirstate(ui, repo)
+    s = lfdirstate.status(match_.always(repo.root, repo.getcwd()), [], False,
+        False, False)
+    (unsure, modified, added, removed, missing, unknown, ignored, clean) = s
+
+    # Need to lock between the standins getting updated and their
+    # largefiles getting updated
+    wlock = repo.wlock()
+    try:
+        if opts['check']:
+            mod = len(modified) > 0
+            for lfile in unsure:
+                standin = lfutil.standin(lfile)
+                if repo['.'][standin].data().strip() != \
+                        lfutil.hashfile(repo.wjoin(lfile)):
+                    mod = True
+                else:
+                    lfdirstate.normal(lfile)
+            lfdirstate.write()
+            if mod:
+                raise util.Abort(_('uncommitted local changes'))
+        # XXX handle removed differently
+        if not opts['clean']:
+            for lfile in unsure + modified + added:
+                lfutil.updatestandin(repo, lfutil.standin(lfile))
+    finally:
+        wlock.release()
+    return orig(ui, repo, *pats, **opts)
+
+# Override filemerge to prompt the user about how they wish to merge
+# largefiles. This will handle identical edits, and copy/rename +
+# edit without prompting the user.
+def override_filemerge(origfn, repo, mynode, orig, fcd, fco, fca):
+    # Use better variable names here. Because this is a wrapper we cannot
+    # change the variable names in the function declaration.
+    fcdest, fcother, fcancestor = fcd, fco, fca
+    if not lfutil.isstandin(orig):
+        return origfn(repo, mynode, orig, fcdest, fcother, fcancestor)
+    else:
+        if not fcother.cmp(fcdest): # files identical?
+            return None
+
+        # backwards, use working dir parent as ancestor
+        if fcancestor == fcother:
+            fcancestor = fcdest.parents()[0]
+
+        if orig != fcother.path():
+            repo.ui.status(_('merging %s and %s to %s\n')
+                           % (lfutil.splitstandin(orig),
+                              lfutil.splitstandin(fcother.path()),
+                              lfutil.splitstandin(fcdest.path())))
+        else:
+            repo.ui.status(_('merging %s\n')
+                           % lfutil.splitstandin(fcdest.path()))
+
+        if fcancestor.path() != fcother.path() and fcother.data() == \
+                fcancestor.data():
+            return 0
+        if fcancestor.path() != fcdest.path() and fcdest.data() == \
+                fcancestor.data():
+            repo.wwrite(fcdest.path(), fcother.data(), fcother.flags())
+            return 0
+
+        if repo.ui.promptchoice(_('largefile %s has a merge conflict\n'
+                             'keep (l)ocal or take (o)ther?') %
+                             lfutil.splitstandin(orig),
+                             (_('&Local'), _('&Other')), 0) == 0:
+            return 0
+        else:
+            repo.wwrite(fcdest.path(), fcother.data(), fcother.flags())
+            return 0
+
+# Copy first changes the matchers to match standins instead of
+# largefiles.  Then it overrides util.copyfile in that function it
+# checks if the destination largefile already exists. It also keeps a
+# list of copied files so that the largefiles can be copied and the
+# dirstate updated.
+def override_copy(orig, ui, repo, pats, opts, rename=False):
+    # doesn't remove largefile on rename
+    if len(pats) < 2:
+        # this isn't legal, let the original function deal with it
+        return orig(ui, repo, pats, opts, rename)
+
+    def makestandin(relpath):
+        path = scmutil.canonpath(repo.root, repo.getcwd(), relpath)
+        return os.path.join(os.path.relpath('.', repo.getcwd()),
+            lfutil.standin(path))
+
+    fullpats = scmutil.expandpats(pats)
+    dest = fullpats[-1]
+
+    if os.path.isdir(dest):
+        if not os.path.isdir(makestandin(dest)):
+            os.makedirs(makestandin(dest))
+    # This could copy both lfiles and normal files in one command,
+    # but we don't want to do that. First replace their matcher to
+    # only match normal files and run it, then replace it to just
+    # match largefiles and run it again.
+    nonormalfiles = False
+    nolfiles = False
+    try:
+        installnormalfilesmatchfn(repo[None].manifest())
+        result = orig(ui, repo, pats, opts, rename)
+    except util.Abort, e:
+        if str(e) != 'no files to copy':
+            raise e
+        else:
+            nonormalfiles = True
+        result = 0
+    finally:
+        restorematchfn()
+
+    # The first rename can cause our current working directory to be removed.
+    # In that case there is nothing left to copy/rename so just quit.
+    try:
+        repo.getcwd()
+    except OSError:
+        return result
+
+    try:
+        # When we call orig below it creates the standins but we don't add them
+        # to the dir state until later so lock during that time.
+        wlock = repo.wlock()
+
+        manifest = repo[None].manifest()
+        oldmatch = None # for the closure
+        def override_match(repo, pats=[], opts={}, globbed=False,
+                default='relpath'):
+            newpats = []
+            # The patterns were previously mangled to add the standin
+            # directory; we need to remove that now
+            for pat in pats:
+                if match_.patkind(pat) is None and lfutil.shortname in pat:
+                    newpats.append(pat.replace(lfutil.shortname, ''))
+                else:
+                    newpats.append(pat)
+            match = oldmatch(repo, newpats, opts, globbed, default)
+            m = copy.copy(match)
+            lfile = lambda f: lfutil.standin(f) in manifest
+            m._files = [lfutil.standin(f) for f in m._files if lfile(f)]
+            m._fmap = set(m._files)
+            orig_matchfn = m.matchfn
+            m.matchfn = lambda f: (lfutil.isstandin(f) and
+                                   lfile(lfutil.splitstandin(f)) and
+                                   orig_matchfn(lfutil.splitstandin(f)) or
+                                   None)
+            return m
+        oldmatch = installmatchfn(override_match)
+        listpats = []
+        for pat in pats:
+            if match_.patkind(pat) is not None:
+                listpats.append(pat)
+            else:
+                listpats.append(makestandin(pat))
+
+        try:
+            origcopyfile = util.copyfile
+            copiedfiles = []
+            def override_copyfile(src, dest):
+                if lfutil.shortname in src and lfutil.shortname in dest:
+                    destlfile = dest.replace(lfutil.shortname, '')
+                    if not opts['force'] and os.path.exists(destlfile):
+                        raise IOError('',
+                            _('destination largefile already exists'))
+                copiedfiles.append((src, dest))
+                origcopyfile(src, dest)
+
+            util.copyfile = override_copyfile
+            result += orig(ui, repo, listpats, opts, rename)
+        finally:
+            util.copyfile = origcopyfile
+
+        lfdirstate = lfutil.openlfdirstate(ui, repo)
+        for (src, dest) in copiedfiles:
+            if lfutil.shortname in src and lfutil.shortname in dest:
+                srclfile = src.replace(lfutil.shortname, '')
+                destlfile = dest.replace(lfutil.shortname, '')
+                destlfiledir = os.path.dirname(destlfile) or '.'
+                if not os.path.isdir(destlfiledir):
+                    os.makedirs(destlfiledir)
+                if rename:
+                    os.rename(srclfile, destlfile)
+                    lfdirstate.remove(os.path.relpath(srclfile,
+                        repo.root))
+                else:
+                    util.copyfile(srclfile, destlfile)
+                lfdirstate.add(os.path.relpath(destlfile,
+                    repo.root))
+        lfdirstate.write()
+    except util.Abort, e:
+        if str(e) != 'no files to copy':
+            raise e
+        else:
+            nolfiles = True
+    finally:
+        restorematchfn()
+        wlock.release()
+
+    if nolfiles and nonormalfiles:
+        raise util.Abort(_('no files to copy'))
+
+    return result
+
+# When the user calls revert, we have to be careful to not revert any
+# changes to other largefiles accidentally. This means we have to keep
+# track of the largefiles that are being reverted so we only pull down
+# the necessary largefiles.
+#
+# Standins are only updated (to match the hash of largefiles) before
+# commits. Update the standins then run the original revert, changing
+# the matcher to hit standins instead of largefiles. Based on the
+# resulting standins update the largefiles. Then return the standins
+# to their proper state
+def override_revert(orig, ui, repo, *pats, **opts):
+    # Because we put the standins in a bad state (by updating them)
+    # and then return them to a correct state we need to lock to
+    # prevent others from changing them in their incorrect state.
+    wlock = repo.wlock()
+    try:
+        lfdirstate = lfutil.openlfdirstate(ui, repo)
+        (modified, added, removed, missing, unknown, ignored, clean) = \
+            lfutil.lfdirstate_status(lfdirstate, repo, repo['.'].rev())
+        for lfile in modified:
+            lfutil.updatestandin(repo, lfutil.standin(lfile))
+
+        try:
+            ctx = repo[opts.get('rev')]
+            oldmatch = None # for the closure
+            def override_match(ctxorrepo, pats=[], opts={}, globbed=False,
+                    default='relpath'):
+                if util.safehasattr(ctxorrepo, 'match'):
+                    ctx0 = ctxorrepo
+                else:
+                    ctx0 = ctxorrepo[None]
+                match = oldmatch(ctxorrepo, pats, opts, globbed, default)
+                m = copy.copy(match)
+                def tostandin(f):
+                    if lfutil.standin(f) in ctx0 or lfutil.standin(f) in ctx:
+                        return lfutil.standin(f)
+                    elif lfutil.standin(f) in repo[None]:
+                        return None
+                    return f
+                m._files = [tostandin(f) for f in m._files]
+                m._files = [f for f in m._files if f is not None]
+                m._fmap = set(m._files)
+                orig_matchfn = m.matchfn
+                def matchfn(f):
+                    if lfutil.isstandin(f):
+                        # We need to keep track of what largefiles are being
+                        # matched so we know which ones to update later --
+                        # otherwise we accidentally revert changes to other
+                        # largefiles. This is repo-specific, so duckpunch the
+                        # repo object to keep the list of largefiles for us
+                        # later.
+                        if orig_matchfn(lfutil.splitstandin(f)) and \
+                                (f in repo[None] or f in ctx):
+                            lfileslist = getattr(repo, '_lfilestoupdate', [])
+                            lfileslist.append(lfutil.splitstandin(f))
+                            repo._lfilestoupdate = lfileslist
+                            return True
+                        else:
+                            return False
+                    return orig_matchfn(f)
+                m.matchfn = matchfn
+                return m
+            oldmatch = installmatchfn(override_match)
+            scmutil.match
+            matches = override_match(repo[None], pats, opts)
+            orig(ui, repo, *pats, **opts)
+        finally:
+            restorematchfn()
+        lfileslist = getattr(repo, '_lfilestoupdate', [])
+        lfcommands.updatelfiles(ui, repo, filelist=lfileslist,
+                                printmessage=False)
+
+        # empty out the largefiles list so we start fresh next time
+        repo._lfilestoupdate = []
+        for lfile in modified:
+            if lfile in lfileslist:
+                if os.path.exists(repo.wjoin(lfutil.standin(lfile))) and lfile\
+                        in repo['.']:
+                    lfutil.writestandin(repo, lfutil.standin(lfile),
+                        repo['.'][lfile].data().strip(),
+                        'x' in repo['.'][lfile].flags())
+        lfdirstate = lfutil.openlfdirstate(ui, repo)
+        for lfile in added:
+            standin = lfutil.standin(lfile)
+            if standin not in ctx and (standin in matches or opts.get('all')):
+                if lfile in lfdirstate:
+                    lfdirstate.drop(lfile)
+                util.unlinkpath(repo.wjoin(standin))
+        lfdirstate.write()
+    finally:
+        wlock.release()
+
+def hg_update(orig, repo, node):
+    result = orig(repo, node)
+    # XXX check if it worked first
+    lfcommands.updatelfiles(repo.ui, repo)
+    return result
+
+def hg_clean(orig, repo, node, show_stats=True):
+    result = orig(repo, node, show_stats)
+    lfcommands.updatelfiles(repo.ui, repo)
+    return result
+
+def hg_merge(orig, repo, node, force=None, remind=True):
+    result = orig(repo, node, force, remind)
+    lfcommands.updatelfiles(repo.ui, repo)
+    return result
+
+# When we rebase a repository with remotely changed largefiles, we need to
+# take some extra care so that the largefiles are correctly updated in the
+# working copy
+def override_pull(orig, ui, repo, source=None, **opts):
+    if opts.get('rebase', False):
+        repo._isrebasing = True
+        try:
+            if opts.get('update'):
+                 del opts['update']
+                 ui.debug('--update and --rebase are not compatible, ignoring '
+                          'the update flag\n')
+            del opts['rebase']
+            cmdutil.bailifchanged(repo)
+            revsprepull = len(repo)
+            origpostincoming = commands.postincoming
+            def _dummy(*args, **kwargs):
+                pass
+            commands.postincoming = _dummy
+            repo.lfpullsource = source
+            if not source:
+                source = 'default'
+            try:
+                result = commands.pull(ui, repo, source, **opts)
+            finally:
+                commands.postincoming = origpostincoming
+            revspostpull = len(repo)
+            if revspostpull > revsprepull:
+                result = result or rebase.rebase(ui, repo)
+        finally:
+            repo._isrebasing = False
+    else:
+        repo.lfpullsource = source
+        if not source:
+            source = 'default'
+        result = orig(ui, repo, source, **opts)
+    return result
+
+def override_rebase(orig, ui, repo, **opts):
+    repo._isrebasing = True
+    try:
+        orig(ui, repo, **opts)
+    finally:
+        repo._isrebasing = False
+
+def override_archive(orig, repo, dest, node, kind, decode=True, matchfn=None,
+            prefix=None, mtime=None, subrepos=None):
+    # No need to lock because we are only reading history and
+    # largefile caches, neither of which are modified.
+    lfcommands.cachelfiles(repo.ui, repo, node)
+
+    if kind not in archival.archivers:
+        raise util.Abort(_("unknown archive type '%s'") % kind)
+
+    ctx = repo[node]
+
+    if kind == 'files':
+        if prefix:
+            raise util.Abort(
+                _('cannot give prefix when archiving to files'))
+    else:
+        prefix = archival.tidyprefix(dest, kind, prefix)
+
+    def write(name, mode, islink, getdata):
+        if matchfn and not matchfn(name):
+            return
+        data = getdata()
+        if decode:
+            data = repo.wwritedata(name, data)
+        archiver.addfile(prefix + name, mode, islink, data)
+
+    archiver = archival.archivers[kind](dest, mtime or ctx.date()[0])
+
+    if repo.ui.configbool("ui", "archivemeta", True):
+        def metadata():
+            base = 'repo: %s\nnode: %s\nbranch: %s\n' % (
+                hex(repo.changelog.node(0)), hex(node), ctx.branch())
+
+            tags = ''.join('tag: %s\n' % t for t in ctx.tags()
+                           if repo.tagtype(t) == 'global')
+            if not tags:
+                repo.ui.pushbuffer()
+                opts = {'template': '{latesttag}\n{latesttagdistance}',
+                        'style': '', 'patch': None, 'git': None}
+                cmdutil.show_changeset(repo.ui, repo, opts).show(ctx)
+                ltags, dist = repo.ui.popbuffer().split('\n')
+                tags = ''.join('latesttag: %s\n' % t for t in ltags.split(':'))
+                tags += 'latesttagdistance: %s\n' % dist
+
+            return base + tags
+
+        write('.hg_archival.txt', 0644, False, metadata)
+
+    for f in ctx:
+        ff = ctx.flags(f)
+        getdata = ctx[f].data
+        if lfutil.isstandin(f):
+            path = lfutil.findfile(repo, getdata().strip())
+            f = lfutil.splitstandin(f)
+
+            def getdatafn():
+                try:
+                    fd = open(path, 'rb')
+                    return fd.read()
+                finally:
+                    fd.close()
+
+            getdata = getdatafn
+        write(f, 'x' in ff and 0755 or 0644, 'l' in ff, getdata)
+
+    if subrepos:
+        for subpath in ctx.substate:
+            sub = ctx.sub(subpath)
+            try:
+                sub.archive(repo.ui, archiver, prefix)
+            except TypeError:
+                sub.archive(archiver, prefix)
+
+    archiver.done()
+
+# If a largefile is modified, the change is not reflected in its
+# standin until a commit. cmdutil.bailifchanged() raises an exception
+# if the repo has uncommitted changes. Wrap it to also check if
+# largefiles were changed. This is used by bisect and backout.
+def override_bailifchanged(orig, repo):
+    orig(repo)
+    repo.lfstatus = True
+    modified, added, removed, deleted = repo.status()[:4]
+    repo.lfstatus = False
+    if modified or added or removed or deleted:
+        raise util.Abort(_('outstanding uncommitted changes'))
+
+# Fetch doesn't use cmdutil.bail_if_changed so override it to add the check
+def override_fetch(orig, ui, repo, *pats, **opts):
+    repo.lfstatus = True
+    modified, added, removed, deleted = repo.status()[:4]
+    repo.lfstatus = False
+    if modified or added or removed or deleted:
+        raise util.Abort(_('outstanding uncommitted changes'))
+    return orig(ui, repo, *pats, **opts)
+
+def override_forget(orig, ui, repo, *pats, **opts):
+    installnormalfilesmatchfn(repo[None].manifest())
+    orig(ui, repo, *pats, **opts)
+    restorematchfn()
+    m = scmutil.match(repo[None], pats, opts)
+
+    try:
+        repo.lfstatus = True
+        s = repo.status(match=m, clean=True)
+    finally:
+        repo.lfstatus = False
+    forget = sorted(s[0] + s[1] + s[3] + s[6])
+    forget = [f for f in forget if lfutil.standin(f) in repo[None].manifest()]
+
+    for f in forget:
+        if lfutil.standin(f) not in repo.dirstate and not \
+                os.path.isdir(m.rel(lfutil.standin(f))):
+            ui.warn(_('not removing %s: file is already untracked\n')
+                    % m.rel(f))
+
+    for f in forget:
+        if ui.verbose or not m.exact(f):
+            ui.status(_('removing %s\n') % m.rel(f))
+
+    # Need to lock because standin files are deleted then removed from the
+    # repository and we could race inbetween.
+    wlock = repo.wlock()
+    try:
+        lfdirstate = lfutil.openlfdirstate(ui, repo)
+        for f in forget:
+            if lfdirstate[f] == 'a':
+                lfdirstate.drop(f)
+            else:
+                lfdirstate.remove(f)
+        lfdirstate.write()
+        lfutil.repo_remove(repo, [lfutil.standin(f) for f in forget],
+            unlink=True)
+    finally:
+        wlock.release()
+
+def getoutgoinglfiles(ui, repo, dest=None, **opts):
+    dest = ui.expandpath(dest or 'default-push', dest or 'default')
+    dest, branches = hg.parseurl(dest, opts.get('branch'))
+    revs, checkout = hg.addbranchrevs(repo, repo, branches, opts.get('rev'))
+    if revs:
+        revs = [repo.lookup(rev) for rev in revs]
+
+    remoteui = hg.remoteui
+
+    try:
+        remote = hg.repository(remoteui(repo, opts), dest)
+    except error.RepoError:
+        return None
+    o = lfutil.findoutgoing(repo, remote, False)
+    if not o:
+        return None
+    o = repo.changelog.nodesbetween(o, revs)[0]
+    if opts.get('newest_first'):
+        o.reverse()
+
+    toupload = set()
+    for n in o:
+        parents = [p for p in repo.changelog.parents(n) if p != node.nullid]
+        ctx = repo[n]
+        files = set(ctx.files())
+        if len(parents) == 2:
+            mc = ctx.manifest()
+            mp1 = ctx.parents()[0].manifest()
+            mp2 = ctx.parents()[1].manifest()
+            for f in mp1:
+                if f not in mc:
+                        files.add(f)
+            for f in mp2:
+                if f not in mc:
+                    files.add(f)
+            for f in mc:
+                if mc[f] != mp1.get(f, None) or mc[f] != mp2.get(f, None):
+                    files.add(f)
+        toupload = toupload.union(
+            set([f for f in files if lfutil.isstandin(f) and f in ctx]))
+    return toupload
+
+def override_outgoing(orig, ui, repo, dest=None, **opts):
+    orig(ui, repo, dest, **opts)
+
+    if opts.pop('large', None):
+        toupload = getoutgoinglfiles(ui, repo, dest, **opts)
+        if toupload is None:
+            ui.status(_('largefiles: No remote repo\n'))
+        else:
+            ui.status(_('largefiles to upload:\n'))
+            for file in toupload:
+                ui.status(lfutil.splitstandin(file) + '\n')
+            ui.status('\n')
+
+def override_summary(orig, ui, repo, *pats, **opts):
+    orig(ui, repo, *pats, **opts)
+
+    if opts.pop('large', None):
+        toupload = getoutgoinglfiles(ui, repo, None, **opts)
+        if toupload is None:
+            ui.status(_('largefiles: No remote repo\n'))
+        else:
+            ui.status(_('largefiles: %d to upload\n') % len(toupload))
+
+def override_addremove(orig, ui, repo, *pats, **opts):
+    # Check if the parent or child has largefiles; if so, disallow
+    # addremove. If there is a symlink in the manifest then getting
+    # the manifest throws an exception: catch it and let addremove
+    # deal with it.
+    try:
+        manifesttip = set(repo['tip'].manifest())
+    except util.Abort:
+        manifesttip = set()
+    try:
+        manifestworking = set(repo[None].manifest())
+    except util.Abort:
+        manifestworking = set()
+
+    # Manifests are only iterable so turn them into sets then union
+    for file in manifesttip.union(manifestworking):
+        if file.startswith(lfutil.shortname):
+            raise util.Abort(
+                _('addremove cannot be run on a repo with largefiles'))
+
+    return orig(ui, repo, *pats, **opts)
+
+# Calling purge with --all will cause the largefiles to be deleted.
+# Override repo.status to prevent this from happening.
+def override_purge(orig, ui, repo, *dirs, **opts):
+    oldstatus = repo.status
+    def override_status(node1='.', node2=None, match=None, ignored=False,
+                        clean=False, unknown=False, listsubrepos=False):
+        r = oldstatus(node1, node2, match, ignored, clean, unknown,
+                      listsubrepos)
+        lfdirstate = lfutil.openlfdirstate(ui, repo)
+        modified, added, removed, deleted, unknown, ignored, clean = r
+        unknown = [f for f in unknown if lfdirstate[f] == '?']
+        ignored = [f for f in ignored if lfdirstate[f] == '?']
+        return modified, added, removed, deleted, unknown, ignored, clean
+    repo.status = override_status
+    orig(ui, repo, *dirs, **opts)
+    repo.status = oldstatus
+
+def override_rollback(orig, ui, repo, **opts):
+    result = orig(ui, repo, **opts)
+    merge.update(repo, node=None, branchmerge=False, force=True,
+        partial=lfutil.isstandin)
+    lfdirstate = lfutil.openlfdirstate(ui, repo)
+    lfiles = lfutil.listlfiles(repo)
+    oldlfiles = lfutil.listlfiles(repo, repo[None].parents()[0].rev())
+    for file in lfiles:
+        if file in oldlfiles:
+            lfdirstate.normallookup(file)
+        else:
+            lfdirstate.add(file)
+    lfdirstate.write()
+    return result

hgext/largefiles/proto.py

@@ -0,0 +1,160
+# Copyright 2011 Fog Creek Software
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+import os
+import tempfile
+import urllib2
+
+from mercurial import error, httprepo, util, wireproto
+from mercurial.i18n import _
+
+import lfutil
+
+LARGEFILES_REQUIRED_MSG = ('\nThis repository uses the largefiles extension.'
+                           '\n\nPlease enable it in your Mercurial config '
+                           'file.\n')
+
+def putlfile(repo, proto, sha):
+    '''Put a largefile into a repository's local cache and into the
+    system cache.'''
+    f = None
+    proto.redirect()
+    try:
+        try:
+            f = tempfile.NamedTemporaryFile(mode='wb+', prefix='hg-putlfile-')
+            proto.getfile(f)
+            f.seek(0)
+            if sha != lfutil.hexsha1(f):
+                return wireproto.pushres(1)
+            lfutil.copytocacheabsolute(repo, f.name, sha)
+        except IOError:
+            repo.ui.warn(
+                _('error: could not put received data into largefile store'))
+            return wireproto.pushres(1)
+    finally:
+        if f:
+            f.close()
+
+    return wireproto.pushres(0)
+
+def getlfile(repo, proto, sha):
+    '''Retrieve a largefile from the repository-local cache or system
+    cache.'''
+    filename = lfutil.findfile(repo, sha)
+    if not filename:
+        raise util.Abort(_('requested largefile %s not present in cache') % sha)
+    f = open(filename, 'rb')
+    length = os.fstat(f.fileno())[6]
+
+    # Since we can't set an HTTP content-length header here, and
+    # Mercurial core provides no way to give the length of a streamres
+    # (and reading the entire file into RAM would be ill-advised), we
+    # just send the length on the first line of the response, like the
+    # ssh proto does for string responses.
+    def generator():
+        yield '%d\n' % length
+        for chunk in f:
+            yield chunk
+    return wireproto.streamres(generator())
+
+def statlfile(repo, proto, sha):
+    '''Return '2\n' if the largefile is missing, '1\n' if it has a
+    mismatched checksum, or '0\n' if it is in good condition'''
+    filename = lfutil.findfile(repo, sha)
+    if not filename:
+        return '2\n'
+    fd = None
+    try:
+        fd = open(filename, 'rb')
+        return lfutil.hexsha1(fd) == sha and '0\n' or '1\n'
+    finally:
+        if fd:
+            fd.close()
+
+def wirereposetup(ui, repo):
+    class lfileswirerepository(repo.__class__):
+        def putlfile(self, sha, fd):
+            # unfortunately, httprepository._callpush tries to convert its
+            # input file-like into a bundle before sending it, so we can't use
+            # it ...
+            if issubclass(self.__class__, httprepo.httprepository):
+                try:
+                    return int(self._call('putlfile', data=fd, sha=sha,
+                        headers={'content-type':'application/mercurial-0.1'}))
+                except (ValueError, urllib2.HTTPError):
+                    return 1
+            # ... but we can't use sshrepository._call because the data=
+            # argument won't get sent, and _callpush does exactly what we want
+            # in this case: send the data straight through
+            else:
+                try:
+                    ret, output = self._callpush("putlfile", fd, sha=sha)
+                    if ret == "":
+                        raise error.ResponseError(_('putlfile failed:'),
+                                output)
+                    return int(ret)
+                except IOError:
+                    return 1
+                except ValueError:
+                    raise error.ResponseError(
+                        _('putlfile failed (unexpected response):'), ret)
+
+        def getlfile(self, sha):
+            stream = self._callstream("getlfile", sha=sha)
+            length = stream.readline()
+            try:
+                length = int(length)
+            except ValueError:
+                self._abort(error.ResponseError(_("unexpected response:"),
+                                                length))
+            return (length, stream)
+
+        def statlfile(self, sha):
+            try:
+                return int(self._call("statlfile", sha=sha))
+            except (ValueError, urllib2.HTTPError):
+                # If the server returns anything but an integer followed by a
+                # newline, newline, it's not speaking our language; if we get
+                # an HTTP error, we can't be sure the largefile is present;
+                # either way, consider it missing.
+                return 2
+
+    repo.__class__ = lfileswirerepository
+
+# advertise the largefiles=serve capability
+def capabilities(repo, proto):
+    return capabilities_orig(repo, proto) + ' largefiles=serve'
+
+# duplicate what Mercurial's new out-of-band errors mechanism does, because
+# clients old and new alike both handle it well
+def webproto_refuseclient(self, message):
+    self.req.header([('Content-Type', 'application/hg-error')])
+    return message
+
+def sshproto_refuseclient(self, message):
+    self.ui.write_err('%s\n-\n' % message)
+    self.fout.write('\n')
+    self.fout.flush()
+
+    return ''
+
+def heads(repo, proto):
+    if lfutil.islfilesrepo(repo):
+        return wireproto.ooberror(LARGEFILES_REQUIRED_MSG)
+    return wireproto.heads(repo, proto)
+
+def sshrepo_callstream(self, cmd, **args):
+    if cmd == 'heads' and self.capable('largefiles'):
+        cmd = 'lheads'
+    if cmd == 'batch' and self.capable('largefiles'):
+        args['cmds'] = args['cmds'].replace('heads ', 'lheads ')
+    return ssh_oldcallstream(self, cmd, **args)
+
+def httprepo_callstream(self, cmd, **args):
+    if cmd == 'heads' and self.capable('largefiles'):
+        cmd = 'lheads'
+    if cmd == 'batch' and self.capable('largefiles'):
+        args['cmds'] = args['cmds'].replace('heads ', 'lheads ')
+    return http_oldcallstream(self, cmd, **args)

hgext/largefiles/remotestore.py

@@ -0,0 +1,106
+# Copyright 2010-2011 Fog Creek Software
+# Copyright 2010-2011 Unity Technologies
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+'''remote largefile store; the base class for servestore'''
+
+import urllib2
+
+from mercurial import util
+from mercurial.i18n import _
+
+import lfutil
+import basestore
+
+class remotestore(basestore.basestore):
+    '''a largefile store accessed over a network'''
+    def __init__(self, ui, repo, url):
+        super(remotestore, self).__init__(ui, repo, url)
+
+    def put(self, source, hash):
+        if self._verify(hash):
+            return
+        if self.sendfile(source, hash):
+            raise util.Abort(
+                _('remotestore: could not put %s to remote store %s')
+                % (source, self.url))
+        self.ui.debug(
+            _('remotestore: put %s to remote store %s') % (source, self.url))
+
+    def exists(self, hash):
+        return self._verify(hash)
+
+    def sendfile(self, filename, hash):
+        self.ui.debug('remotestore: sendfile(%s, %s)\n' % (filename, hash))
+        fd = None
+        try:
+            try:
+                fd = lfutil.httpsendfile(self.ui, filename)
+            except IOError, e:
+                raise util.Abort(
+                    _('remotestore: could not open file %s: %s')
+                    % (filename, str(e)))
+            return self._put(hash, fd)
+        finally:
+            if fd:
+                fd.close()
+
+    def _getfile(self, tmpfile, filename, hash):
+        # quit if the largefile isn't there
+        stat = self._stat(hash)
+        if stat == 1:
+            raise util.Abort(_('remotestore: largefile %s is invalid') % hash)
+        elif stat == 2:
+            raise util.Abort(_('remotestore: largefile %s is missing') % hash)
+
+        try:
+            length, infile = self._get(hash)
+        except urllib2.HTTPError, e:
+            # 401s get converted to util.Aborts; everything else is fine being
+            # turned into a StoreError
+            raise basestore.StoreError(filename, hash, self.url, str(e))
+        except urllib2.URLError, e:
+            # This usually indicates a connection problem, so don't
+            # keep trying with the other files... they will probably
+            # all fail too.
+            raise util.Abort('%s: %s' % (self.url, e.reason))
+        except IOError, e:
+            raise basestore.StoreError(filename, hash, self.url, str(e))
+
+        # Mercurial does not close its SSH connections after writing a stream
+        if length is not None:
+            infile = lfutil.limitreader(infile, length)
+        return lfutil.copyandhash(lfutil.blockstream(infile), tmpfile)
+
+    def _verify(self, hash):
+        return not self._stat(hash)
+
+    def _verifyfile(self, cctx, cset, contents, standin, verified):
+        filename = lfutil.splitstandin(standin)
+        if not filename:
+            return False
+        fctx = cctx[standin]
+        key = (filename, fctx.filenode())
+        if key in verified:
+            return False
+
+        verified.add(key)
+
+        stat = self._stat(hash)
+        if not stat:
+            return False
+        elif stat == 1:
+            self.ui.warn(
+                _('changeset %s: %s: contents differ\n')
+                % (cset, filename))
+            return True # failed
+        elif stat == 2:
+            self.ui.warn(
+                _('changeset %s: %s missing\n')
+                % (cset, filename))
+            return True # failed
+        else:
+            raise RuntimeError('verify failed: unexpected response from '
+                               'statlfile (%r)' % stat)

hgext/largefiles/reposetup.py

@@ -0,0 +1,416
+# Copyright 2009-2010 Gregory P. Ward
+# Copyright 2009-2010 Intelerad Medical Systems Incorporated
+# Copyright 2010-2011 Fog Creek Software
+# Copyright 2010-2011 Unity Technologies
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+'''setup for largefiles repositories: reposetup'''
+import copy
+import types
+import os
+import re
+
+from mercurial import context, error, manifest, match as match_, \
+        node, util
+from mercurial.i18n import _
+
+import lfcommands
+import proto
+import lfutil
+
+def reposetup(ui, repo):
+    # wire repositories should be given new wireproto functions but not the
+    # other largefiles modifications
+    if not repo.local():
+        return proto.wirereposetup(ui, repo)
+
+    for name in ('status', 'commitctx', 'commit', 'push'):
+        method = getattr(repo, name)
+        #if not (isinstance(method, types.MethodType) and
+        #        method.im_func is repo.__class__.commitctx.im_func):
+        if (isinstance(method, types.FunctionType) and
+            method.func_name == 'wrap'):
+            ui.warn(_('largefiles: repo method %r appears to have already been'
+                    ' wrapped by another extension: '
+                    'largefiles may behave incorrectly\n')
+                    % name)
+
+    class lfiles_repo(repo.__class__):
+        lfstatus = False
+        def status_nolfiles(self, *args, **kwargs):
+            return super(lfiles_repo, self).status(*args, **kwargs)
+
+        # When lfstatus is set, return a context that gives the names
+        # of largefiles instead of their corresponding standins and
+        # identifies the largefiles as always binary, regardless of
+        # their actual contents.
+        def __getitem__(self, changeid):
+            ctx = super(lfiles_repo, self).__getitem__(changeid)
+            if self.lfstatus:
+                class lfiles_manifestdict(manifest.manifestdict):
+                    def __contains__(self, filename):
+                        if super(lfiles_manifestdict,
+                                self).__contains__(filename):
+                            return True
+                        return super(lfiles_manifestdict,
+                            self).__contains__(lfutil.shortname+'/' + filename)
+                class lfiles_ctx(ctx.__class__):
+                    def files(self):
+                        filenames = super(lfiles_ctx, self).files()
+                        return [re.sub('^\\'+lfutil.shortname+'/', '',
+                                       filename) for filename in filenames]
+                    def manifest(self):
+                        man1 = super(lfiles_ctx, self).manifest()
+                        man1.__class__ = lfiles_manifestdict
+                        return man1
+                    def filectx(self, path, fileid=None, filelog=None):
+                        try:
+                            result = super(lfiles_ctx, self).filectx(path,
+                                fileid, filelog)
+                        except error.LookupError:
+                            # Adding a null character will cause Mercurial to
+                            # identify this as a binary file.
+                            result = super(lfiles_ctx, self).filectx(
+                                lfutil.shortname + '/' + path, fileid,
+                                filelog)
+                            olddata = result.data
+                            result.data = lambda: olddata() + '\0'
+                        return result
+                ctx.__class__ = lfiles_ctx
+            return ctx
+
+        # Figure out the status of big files and insert them into the
+        # appropriate list in the result. Also removes standin files
+        # from the listing. Revert to the original status if
+        # self.lfstatus is False.
+        def status(self, node1='.', node2=None, match=None, ignored=False,
+                clean=False, unknown=False, listsubrepos=False):
+            listignored, listclean, listunknown = ignored, clean, unknown
+            if not self.lfstatus:
+                try:
+                    return super(lfiles_repo, self).status(node1, node2, match,
+                        listignored, listclean, listunknown, listsubrepos)
+                except TypeError:
+                    return super(lfiles_repo, self).status(node1, node2, match,
+                        listignored, listclean, listunknown)
+            else:
+                # some calls in this function rely on the old version of status
+                self.lfstatus = False
+                if isinstance(node1, context.changectx):
+                    ctx1 = node1
+                else:
+                    ctx1 = repo[node1]
+                if isinstance(node2, context.changectx):
+                    ctx2 = node2
+                else:
+                    ctx2 = repo[node2]
+                working = ctx2.rev() is None
+                parentworking = working and ctx1 == self['.']
+
+                def inctx(file, ctx):
+                    try:
+                        if ctx.rev() is None:
+                            return file in ctx.manifest()
+                        ctx[file]
+                        return True
+                    except KeyError:
+                        return False
+
+                if match is None:
+                    match = match_.always(self.root, self.getcwd())
+
+                # Create a copy of match that matches standins instead
+                # of largefiles.
+                def tostandin(file):
+                    if inctx(lfutil.standin(file), ctx2):
+                        return lfutil.standin(file)
+                    return file
+
+                m = copy.copy(match)
+                m._files = [tostandin(f) for f in m._files]
+
+                # get ignored, clean, and unknown but remove them
+                # later if they were not asked for
+                try:
+                    result = super(lfiles_repo, self).status(node1, node2, m,
+                        True, True, True, listsubrepos)
+                except TypeError:
+                    result = super(lfiles_repo, self).status(node1, node2, m,
+                        True, True, True)
+                if working:
+                    # hold the wlock while we read largefiles and
+                    # update the lfdirstate
+                    wlock = repo.wlock()
+                    try:
+                        # Any non-largefiles that were explicitly listed must be
+                        # taken out or lfdirstate.status will report an error.
+                        # The status of these files was already computed using
+                        # super's status.
+                        lfdirstate = lfutil.openlfdirstate(ui, self)
+                        match._files = [f for f in match._files if f in
+                            lfdirstate]
+                        s = lfdirstate.status(match, [], listignored,
+                                listclean, listunknown)
+                        (unsure, modified, added, removed, missing, unknown,
+                                ignored, clean) = s
+                        if parentworking:
+                            for lfile in unsure:
+                                if ctx1[lfutil.standin(lfile)].data().strip() \
+                                        != lfutil.hashfile(self.wjoin(lfile)):
+                                    modified.append(lfile)
+                                else:
+                                    clean.append(lfile)
+                                    lfdirstate.normal(lfile)
+                            lfdirstate.write()
+                        else:
+                            tocheck = unsure + modified + added + clean
+                            modified, added, clean = [], [], []
+
+                            for lfile in tocheck:
+                                standin = lfutil.standin(lfile)
+                                if inctx(standin, ctx1):
+                                    if ctx1[standin].data().strip() != \
+                                            lfutil.hashfile(self.wjoin(lfile)):
+                                        modified.append(lfile)
+                                    else:
+                                        clean.append(lfile)
+                                else:
+                                    added.append(lfile)
+                    finally:
+                        wlock.release()
+
+                    for standin in ctx1.manifest():
+                        if not lfutil.isstandin(standin):
+                            continue
+                        lfile = lfutil.splitstandin(standin)
+                        if not match(lfile):
+                            continue
+                        if lfile not in lfdirstate:
+                            removed.append(lfile)
+                    # Handle unknown and ignored differently
+                    lfiles = (modified, added, removed, missing, [], [], clean)
+                    result = list(result)
+                    # Unknown files
+                    result[4] = [f for f in unknown
+                                 if (repo.dirstate[f] == '?' and
+                                     not lfutil.isstandin(f))]
+                    # Ignored files must be ignored by both the dirstate and
+                    # lfdirstate
+                    result[5] = set(ignored).intersection(set(result[5]))
+                    # combine normal files and largefiles
+                    normals = [[fn for fn in filelist
+                                if not lfutil.isstandin(fn)]
+                               for filelist in result]
+                    result = [sorted(list1 + list2)
+                              for (list1, list2) in zip(normals, lfiles)]
+                else:
+                    def toname(f):
+                        if lfutil.isstandin(f):
+                            return lfutil.splitstandin(f)
+                        return f
+                    result = [[toname(f) for f in items] for items in result]
+
+                if not listunknown:
+                    result[4] = []
+                if not listignored:
+                    result[5] = []
+                if not listclean:
+                    result[6] = []
+                self.lfstatus = True
+                return result
+
+        # As part of committing, copy all of the largefiles into the
+        # cache.
+        def commitctx(self, *args, **kwargs):
+            node = super(lfiles_repo, self).commitctx(*args, **kwargs)
+            ctx = self[node]
+            for filename in ctx.files():
+                if lfutil.isstandin(filename) and filename in ctx.manifest():
+                    realfile = lfutil.splitstandin(filename)
+                    lfutil.copytocache(self, ctx.node(), realfile)
+
+            return node
+
+        # Before commit, largefile standins have not had their
+        # contents updated to reflect the hash of their largefile.
+        # Do that here.
+        def commit(self, text="", user=None, date=None, match=None,
+                force=False, editor=False, extra={}):
+            orig = super(lfiles_repo, self).commit
+
+            wlock = repo.wlock()
+            try:
+                if getattr(repo, "_isrebasing", False):
+                    # We have to take the time to pull down the new
+                    # largefiles now. Otherwise if we are rebasing,
+                    # any largefiles that were modified in the
+                    # destination changesets get overwritten, either
+                    # by the rebase or in the first commit after the
+                    # rebase.
+                    lfcommands.updatelfiles(repo.ui, repo)
+                # Case 1: user calls commit with no specific files or
+                # include/exclude patterns: refresh and commit all files that
+                # are "dirty".
+                if ((match is None) or
+                    (not match.anypats() and not match.files())):
+                    # Spend a bit of time here to get a list of files we know
+                    # are modified so we can compare only against those.
+                    # It can cost a lot of time (several seconds)
+                    # otherwise to update all standins if the largefiles are
+                    # large.
+                    lfdirstate = lfutil.openlfdirstate(ui, self)
+                    dirtymatch = match_.always(repo.root, repo.getcwd())
+                    s = lfdirstate.status(dirtymatch, [], False, False, False)
+                    modifiedfiles = []
+                    for i in s:
+                        modifiedfiles.extend(i)
+                    lfiles = lfutil.listlfiles(self)
+                    # this only loops through largefiles that exist (not
+                    # removed/renamed)
+                    for lfile in lfiles:
+                        if lfile in modifiedfiles:
+                            if os.path.exists(self.wjoin(lfutil.standin(lfile))):
+                                # this handles the case where a rebase is being
+                                # performed and the working copy is not updated
+                                # yet.
+                                if os.path.exists(self.wjoin(lfile)):
+                                    lfutil.updatestandin(self,
+                                        lfutil.standin(lfile))
+                                    lfdirstate.normal(lfile)
+                    for lfile in lfdirstate:
+                        if lfile in modifiedfiles:
+                            if not os.path.exists(
+                                    repo.wjoin(lfutil.standin(lfile))):
+                                lfdirstate.drop(lfile)
+                    lfdirstate.write()
+
+                    return orig(text=text, user=user, date=date, match=match,
+                                    force=force, editor=editor, extra=extra)
+
+                for f in match.files():
+                    if lfutil.isstandin(f):
+                        raise util.Abort(
+                            _('file "%s" is a largefile standin') % f,
+                            hint=('commit the largefile itself instead'))
+
+                # Case 2: user calls commit with specified patterns: refresh
+                # any matching big files.
+                smatcher = lfutil.composestandinmatcher(self, match)
+                standins = lfutil.dirstate_walk(self.dirstate, smatcher)
+
+                # No matching big files: get out of the way and pass control to
+                # the usual commit() method.
+                if not standins:
+                    return orig(text=text, user=user, date=date, match=match,
+                                    force=force, editor=editor, extra=extra)
+
+                # Refresh all matching big files.  It's possible that the
+                # commit will end up failing, in which case the big files will
+                # stay refreshed.  No harm done: the user modified them and
+                # asked to commit them, so sooner or later we're going to
+                # refresh the standins.  Might as well leave them refreshed.
+                lfdirstate = lfutil.openlfdirstate(ui, self)
+                for standin in standins:
+                    lfile = lfutil.splitstandin(standin)
+                    if lfdirstate[lfile] <> 'r':
+                        lfutil.updatestandin(self, standin)
+                        lfdirstate.normal(lfile)
+                    else:
+                        lfdirstate.drop(lfile)
+                lfdirstate.write()
+
+                # Cook up a new matcher that only matches regular files or
+                # standins corresponding to the big files requested by the
+                # user.  Have to modify _files to prevent commit() from
+                # complaining "not tracked" for big files.
+                lfiles = lfutil.listlfiles(repo)
+                match = copy.copy(match)
+                orig_matchfn = match.matchfn
+
+                # Check both the list of largefiles and the list of
+                # standins because if a largefile was removed, it
+                # won't be in the list of largefiles at this point
+                match._files += sorted(standins)
+
+                actualfiles = []
+                for f in match._files:
+                    fstandin = lfutil.standin(f)
+
+                    # ignore known largefiles and standins
+                    if f in lfiles or fstandin in standins:
+                        continue
+
+                    # append directory separator to avoid collisions
+                    if not fstandin.endswith(os.sep):
+                        fstandin += os.sep
+
+                    # prevalidate matching standin directories
+                    if lfutil.any_(st for st in match._files
+                                   if st.startswith(fstandin)):
+                        continue
+                    actualfiles.append(f)
+                match._files = actualfiles
+
+                def matchfn(f):
+                    if orig_matchfn(f):
+                        return f not in lfiles
+                    else:
+                        return f in standins
+
+                match.matchfn = matchfn
+                return orig(text=text, user=user, date=date, match=match,
+                                force=force, editor=editor, extra=extra)
+            finally:
+                wlock.release()
+
+        def push(self, remote, force=False, revs=None, newbranch=False):
+            o = lfutil.findoutgoing(repo, remote, force)
+            if o:
+                toupload = set()
+                o = repo.changelog.nodesbetween(o, revs)[0]
+                for n in o:
+                    parents = [p for p in repo.changelog.parents(n)
+                               if p != node.nullid]
+                    ctx = repo[n]
+                    files = set(ctx.files())
+                    if len(parents) == 2:
+                        mc = ctx.manifest()
+                        mp1 = ctx.parents()[0].manifest()
+                        mp2 = ctx.parents()[1].manifest()
+                        for f in mp1:
+                            if f not in mc:
+                                files.add(f)
+                        for f in mp2:
+                            if f not in mc:
+                                files.add(f)
+                        for f in mc:
+                            if mc[f] != mp1.get(f, None) or mc[f] != mp2.get(f,
+                                    None):
+                                files.add(f)
+
+                    toupload = toupload.union(
+                        set([ctx[f].data().strip()
+                             for f in files
+                             if lfutil.isstandin(f) and f in ctx]))
+                lfcommands.uploadlfiles(ui, self, remote, toupload)
+            return super(lfiles_repo, self).push(remote, force, revs,
+                newbranch)
+
+    repo.__class__ = lfiles_repo
+
+    def checkrequireslfiles(ui, repo, **kwargs):
+        if 'largefiles' not in repo.requirements and lfutil.any_(
+                lfutil.shortname+'/' in f[0] for f in repo.store.datafiles()):
+            # workaround bug in Mercurial 1.9 whereby requirements is
+            # a list on newly-cloned repos
+            repo.requirements = set(repo.requirements)
+
+            repo.requirements |= set(['largefiles'])
+            repo._writerequirements()
+
+    checkrequireslfiles(ui, repo)
+
+    ui.setconfig('hooks', 'changegroup.lfiles', checkrequireslfiles)
+    ui.setconfig('hooks', 'commit.lfiles', checkrequireslfiles)

hgext/largefiles/uisetup.py

@@ -0,0 +1,138
+# Copyright 2009-2010 Gregory P. Ward
+# Copyright 2009-2010 Intelerad Medical Systems Incorporated
+# Copyright 2010-2011 Fog Creek Software
+# Copyright 2010-2011 Unity Technologies
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+'''setup for largefiles extension: uisetup'''
+
+from mercurial import archival, cmdutil, commands, extensions, filemerge, hg, \
+    httprepo, localrepo, sshrepo, sshserver, util, wireproto
+from mercurial.i18n import _
+from mercurial.hgweb import hgweb_mod, protocol
+
+import overrides
+import proto
+
+def uisetup(ui):
+    # Disable auto-status for some commands which assume that all
+    # files in the result are under Mercurial's control
+
+    entry = extensions.wrapcommand(commands.table, 'add',
+                                   overrides.override_add)
+    addopt = [('', 'large', None, _('add as largefile')),
+            ('', 'lfsize', '', _('add all files above this size (in megabytes)'
+                                 'as largefiles (default: 10)'))]
+    entry[1].extend(addopt)
+
+    entry = extensions.wrapcommand(commands.table, 'addremove',
+            overrides.override_addremove)
+    entry = extensions.wrapcommand(commands.table, 'remove',
+                                   overrides.override_remove)
+    entry = extensions.wrapcommand(commands.table, 'forget',
+                                   overrides.override_forget)
+    entry = extensions.wrapcommand(commands.table, 'status',
+                                   overrides.override_status)
+    entry = extensions.wrapcommand(commands.table, 'log',
+                                   overrides.override_log)
+    entry = extensions.wrapcommand(commands.table, 'rollback',
+                                   overrides.override_rollback)
+    entry = extensions.wrapcommand(commands.table, 'verify',
+                                   overrides.override_verify)
+
+    verifyopt = [('', 'large', None, _('verify largefiles')),
+                 ('', 'lfa', None,
+                     _('verify all revisions of largefiles not just current')),
+                 ('', 'lfc', None,
+                     _('verify largefile contents not just existence'))]
+    entry[1].extend(verifyopt)
+
+    entry = extensions.wrapcommand(commands.table, 'outgoing',
+        overrides.override_outgoing)
+    outgoingopt = [('', 'large', None, _('display outgoing largefiles'))]
+    entry[1].extend(outgoingopt)
+    entry = extensions.wrapcommand(commands.table, 'summary',
+                                   overrides.override_summary)
+    summaryopt = [('', 'large', None, _('display outgoing largefiles'))]
+    entry[1].extend(summaryopt)
+
+    entry = extensions.wrapcommand(commands.table, 'update',
+                                   overrides.override_update)
+    entry = extensions.wrapcommand(commands.table, 'pull',
+                                   overrides.override_pull)
+    entry = extensions.wrapfunction(filemerge, 'filemerge',
+                                    overrides.override_filemerge)
+    entry = extensions.wrapfunction(cmdutil, 'copy',
+                                    overrides.override_copy)
+
+    # Backout calls revert so we need to override both the command and the
+    # function
+    entry = extensions.wrapcommand(commands.table, 'revert',
+                                   overrides.override_revert)
+    entry = extensions.wrapfunction(commands, 'revert',
+                                    overrides.override_revert)
+
+    # clone uses hg._update instead of hg.update even though they are the
+    # same function... so wrap both of them)
+    extensions.wrapfunction(hg, 'update', overrides.hg_update)
+    extensions.wrapfunction(hg, '_update', overrides.hg_update)
+    extensions.wrapfunction(hg, 'clean', overrides.hg_clean)
+    extensions.wrapfunction(hg, 'merge', overrides.hg_merge)
+
+    extensions.wrapfunction(archival, 'archive', overrides.override_archive)
+    if util.safehasattr(cmdutil, 'bailifchanged'):
+        extensions.wrapfunction(cmdutil, 'bailifchanged',
+            overrides.override_bailifchanged)
+    else:
+        extensions.wrapfunction(cmdutil, 'bail_if_changed',
+            overrides.override_bailifchanged)
+
+    # create the new wireproto commands ...
+    wireproto.commands['putlfile'] = (proto.putlfile, 'sha')
+    wireproto.commands['getlfile'] = (proto.getlfile, 'sha')
+    wireproto.commands['statlfile'] = (proto.statlfile, 'sha')
+
+    # ... and wrap some existing ones
+    wireproto.commands['capabilities'] = (proto.capabilities, '')
+    wireproto.commands['heads'] = (proto.heads, '')
+    wireproto.commands['lheads'] = (wireproto.heads, '')
+
+    # make putlfile behave the same as push and {get,stat}lfile behave
+    # the same as pull w.r.t. permissions checks
+    hgweb_mod.perms['putlfile'] = 'push'
+    hgweb_mod.perms['getlfile'] = 'pull'
+    hgweb_mod.perms['statlfile'] = 'pull'
+
+    # the hello wireproto command uses wireproto.capabilities, so it won't see
+    # our largefiles capability unless we replace the actual function as well.
+    proto.capabilities_orig = wireproto.capabilities
+    wireproto.capabilities = proto.capabilities
+
+    # these let us reject non-largefiles clients and make them display
+    # our error messages
+    protocol.webproto.refuseclient = proto.webproto_refuseclient
+    sshserver.sshserver.refuseclient = proto.sshproto_refuseclient
+
+    # can't do this in reposetup because it needs to have happened before
+    # wirerepo.__init__ is called
+    proto.ssh_oldcallstream = sshrepo.sshrepository._callstream
+    proto.http_oldcallstream = httprepo.httprepository._callstream
+    sshrepo.sshrepository._callstream = proto.sshrepo_callstream
+    httprepo.httprepository._callstream = proto.httprepo_callstream
+
+    # don't die on seeing a repo with the largefiles requirement
+    localrepo.localrepository.supported |= set(['largefiles'])
+
+    # override some extensions' stuff as well
+    for name, module in extensions.extensions():
+        if name == 'fetch':
+            extensions.wrapcommand(getattr(module, 'cmdtable'), 'fetch',
+                overrides.override_fetch)
+        if name == 'purge':
+            extensions.wrapcommand(getattr(module, 'cmdtable'), 'purge',
+                overrides.override_purge)
+        if name == 'rebase':
+            extensions.wrapcommand(getattr(module, 'cmdtable'), 'rebase',
+                overrides.override_rebase)

hgext/largefiles/usage.txt

@@ -0,0 +1,51
+Largefiles allows for tracking large, incompressible binary files in Mercurial
+without requiring excessive bandwidth for clones and pulls.  Files added as
+largefiles are not tracked directly by Mercurial; rather, their revisions are
+identified by a checksum, and Mercurial tracks these checksums.  This way, when
+you clone a repository or pull in changesets, the large files in older
+revisions of the repository are not needed, and only the ones needed to update
+to the current version are downloaded.  This saves both disk space and
+bandwidth.
+
+If you are starting a new repository or adding new large binary files, using
+largefiles for them is as easy as adding '--large' to your hg add command.  For
+example:
+
+$ dd if=/dev/urandom of=thisfileislarge count=2000
+$ hg add --large thisfileislarge
+$ hg commit -m 'add thisfileislarge, which is large, as a largefile'
+
+When you push a changeset that affects largefiles to a remote repository, its
+largefile revisions will be uploaded along with it.  Note that the remote
+Mercurial must also have the largefiles extension enabled for this to work.
+
+When you pull a changeset that affects largefiles from a remote repository,
+nothing different from Mercurial's normal behavior happens.  However, when you
+update to such a revision, any largefiles needed by that revision are
+downloaded and cached if they have never been downloaded before.  This means
+that network access is required to update to revision you have not yet updated
+to.
+
+If you already have large files tracked by Mercurial without the largefiles
+extension, you will need to convert your repository in order to benefit from
+largefiles.  This is done with the 'hg lfconvert' command:
+
+$ hg lfconvert --size 10 oldrepo newrepo
+
+By default, in repositories that already have largefiles in them, any new file
+over 10MB will automatically be added as largefiles.  To change this
+threshhold, set [largefiles].size in your Mercurial config file to the minimum
+size in megabytes to track as a largefile, or use the --lfsize option to the
+add command (also in megabytes):
+
+[largefiles]
+size = 2
+
+$ hg add --lfsize 2
+
+The [largefiles].patterns config option allows you to specify specific
+space-separated filename patterns (in shell glob syntax) that should always be
+tracked as largefiles:
+
+[largefiles]
+pattens = *.jpg *.{png,bmp} library.zip content/audio/*

hgext/largefiles/wirestore.py

@@ -0,0 +1,29
+# Copyright 2010-2011 Fog Creek Software
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+'''largefile store working over Mercurial's wire protocol'''
+
+import lfutil
+import remotestore
+
+class wirestore(remotestore.remotestore):
+    def __init__(self, ui, repo, remote):
+        cap = remote.capable('largefiles')
+        if not cap:
+            raise lfutil.storeprotonotcapable([])
+        storetypes = cap.split(',')
+        if not 'serve' in storetypes:
+            raise lfutil.storeprotonotcapable(storetypes)
+        self.remote = remote
+        super(wirestore, self).__init__(ui, repo, remote.url())
+
+    def _put(self, hash, fd):
+        return self.remote.putlfile(hash, fd)
+
+    def _get(self, hash):
+        return self.remote.getlfile(hash)
+
+    def _stat(self, hash):
+        return self.remote.statlfile(hash)

contrib/check-code.py

@@ -1,384 +1,386
 #!/usr/bin/env python
 #
 # check-code - a style and portability checker for Mercurial
 #
 # Copyright 2010 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 import re, glob, os, sys
 import keyword
 import optparse
 
 def repquote(m):
     t = re.sub(r"\w", "x", m.group('text'))
     t = re.sub(r"[^\sx]", "o", t)
     return m.group('quote') + t + m.group('quote')
 
 def reppython(m):
     comment = m.group('comment')
     if comment:
         return "#" * len(comment)
     return repquote(m)
 
 def repcomment(m):
     return m.group(1) + "#" * len(m.group(2))
 
 def repccomment(m):
     t = re.sub(r"((?<=\n) )|\S", "x", m.group(2))
     return m.group(1) + t + "*/"
 
 def repcallspaces(m):
     t = re.sub(r"\n\s+", "\n", m.group(2))
     return m.group(1) + t
 
 def repinclude(m):
     return m.group(1) + "<foo>"
 
 def rephere(m):
     t = re.sub(r"\S", "x", m.group(2))
     return m.group(1) + t
 
 
 testpats = [
   [
     (r'(pushd|popd)', "don't use 'pushd' or 'popd', use 'cd'"),
     (r'\W\$?\(\([^\)]*\)\)', "don't use (()) or $(()), use 'expr'"),
     (r'^function', "don't use 'function', use old style"),
     (r'grep.*-q', "don't use 'grep -q', redirect to /dev/null"),
     (r'echo.*\\n', "don't use 'echo \\n', use printf"),
     (r'echo -n', "don't use 'echo -n', use printf"),
     (r'^diff.*-\w*N', "don't use 'diff -N'"),
     (r'(^| )wc[^|]*$', "filter wc output"),
     (r'head -c', "don't use 'head -c', use 'dd'"),
     (r'ls.*-\w*R', "don't use 'ls -R', use 'find'"),
     (r'printf.*\\\d\d\d', "don't use 'printf \NNN', use Python"),
     (r'printf.*\\x', "don't use printf \\x, use Python"),
     (r'\$\(.*\)', "don't use $(expr), use `expr`"),
     (r'rm -rf \*', "don't use naked rm -rf, target a directory"),
     (r'(^|\|\s*)grep (-\w\s+)*[^|]*[(|]\w',
      "use egrep for extended grep syntax"),
     (r'/bin/', "don't use explicit paths for tools"),
     (r'\$PWD', "don't use $PWD, use `pwd`"),
     (r'[^\n]\Z', "no trailing newline"),
     (r'export.*=', "don't export and assign at once"),
     ('^([^"\']|("[^"]*")|(\'[^\']*\'))*\\^', "^ must be quoted"),
     (r'^source\b', "don't use 'source', use '.'"),
     (r'touch -d', "don't use 'touch -d', use 'touch -t' instead"),
     (r'ls\s+[^|-]+\s+-', "options to 'ls' must come before filenames"),
     (r'[^>]>\s*\$HGRCPATH', "don't overwrite $HGRCPATH, append to it"),
     (r'stop\(\)', "don't use 'stop' as a shell function name"),
   ],
   # warnings
   []
 ]
 
 testfilters = [
     (r"( *)(#([^\n]*\S)?)", repcomment),
     (r"<<(\S+)((.|\n)*?\n\1)", rephere),
 ]
 
 uprefix = r"^  \$ "
 uprefixc = r"^  > "
 utestpats = [
   [
     (r'^(\S|  $ ).*(\S\s+|^\s+)\n', "trailing whitespace on non-output"),
     (uprefix + r'.*\|\s*sed', "use regex test output patterns instead of sed"),
     (uprefix + r'(true|exit 0)', "explicit zero exit unnecessary"),
     (uprefix + r'.*\$\?', "explicit exit code checks unnecessary"),
     (uprefix + r'.*\|\| echo.*(fail|error)',
      "explicit exit code checks unnecessary"),
     (uprefix + r'set -e', "don't use set -e"),
     (uprefixc + r'( *)\t', "don't use tabs to indent"),
   ],
   # warnings
   []
 ]
 
 for i in [0, 1]:
     for p, m in testpats[i]:
         if p.startswith('^'):
             p = uprefix + p[1:]
         else:
             p = uprefix + p
         utestpats[i].append((p, m))
 
 utestfilters = [
     (r"( *)(#([^\n]*\S)?)", repcomment),
 ]
 
 pypats = [
   [
     (r'^\s*def\s*\w+\s*\(.*,\s*\(',
      "tuple parameter unpacking not available in Python 3+"),
     (r'lambda\s*\(.*,.*\)',
      "tuple parameter unpacking not available in Python 3+"),
     (r'(?<!def)\s+(cmp)\(', "cmp is not available in Python 3+"),
     (r'\breduce\s*\(.*', "reduce is not available in Python 3+"),
     (r'\.has_key\b', "dict.has_key is not available in Python 3+"),
     (r'^\s*\t', "don't use tabs"),
     (r'\S;\s*\n', "semicolon"),
     (r'\w,\w', "missing whitespace after ,"),
     (r'\w[+/*\-<>]\w', "missing whitespace in expression"),
     (r'^\s+\w+=\w+[^,)]$', "missing whitespace in assignment"),
     (r'.{85}', "line too long"),
     (r'[^\n]\Z', "no trailing newline"),
     (r'(\S\s+|^\s+)\n', "trailing whitespace"),
 #    (r'^\s+[^_ ][^_. ]+_[^_]+\s*=', "don't use underbars in identifiers"),
 #    (r'\w*[a-z][A-Z]\w*\s*=', "don't use camelcase in identifiers"),
     (r'^\s*(if|while|def|class|except|try)\s[^[]*:\s*[^\]#\s]+',
      "linebreak after :"),
     (r'class\s[^( ]+:', "old-style class, use class foo(object)"),
     (r'class\s[^( ]+\(\):',
      "class foo() not available in Python 2.4, use class foo(object)"),
     (r'\b(%s)\(' % '|'.join(keyword.kwlist),
      "Python keyword is not a function"),
     (r',]', "unneeded trailing ',' in list"),
 #    (r'class\s[A-Z][^\(]*\((?!Exception)',
 #     "don't capitalize non-exception classes"),
 #    (r'in range\(', "use xrange"),
 #    (r'^\s*print\s+', "avoid using print in core and extensions"),
     (r'[\x80-\xff]', "non-ASCII character literal"),
     (r'("\')\.format\(', "str.format() not available in Python 2.4"),
     (r'^\s*with\s+', "with not available in Python 2.4"),
     (r'\.isdisjoint\(', "set.isdisjoint not available in Python 2.4"),
     (r'^\s*except.* as .*:', "except as not available in Python 2.4"),
     (r'^\s*os\.path\.relpath', "relpath not available in Python 2.4"),
     (r'(?<!def)\s+(any|all|format)\(',
      "any/all/format not available in Python 2.4"),
     (r'(?<!def)\s+(callable)\(',
-     "callable not available in Python 3, use hasattr(f, '__call__')"),
+     "callable not available in Python 3, use getattr(f, '__call__', None)"),
     (r'if\s.*\selse', "if ... else form not available in Python 2.4"),
     (r'^\s*(%s)\s\s' % '|'.join(keyword.kwlist),
      "gratuitous whitespace after Python keyword"),
     (r'([\(\[]\s\S)|(\S\s[\)\]])', "gratuitous whitespace in () or []"),
 #    (r'\s\s=', "gratuitous whitespace before ="),
     (r'[^>< ](\+=|-=|!=|<>|<=|>=|<<=|>>=)\S',
      "missing whitespace around operator"),
     (r'[^>< ](\+=|-=|!=|<>|<=|>=|<<=|>>=)\s',
      "missing whitespace around operator"),
     (r'\s(\+=|-=|!=|<>|<=|>=|<<=|>>=)\S',
      "missing whitespace around operator"),
     (r'[^+=*/!<>&| -](\s=|=\s)[^= ]',
      "wrong whitespace around ="),
     (r'raise Exception', "don't raise generic exceptions"),
     (r' is\s+(not\s+)?["\'0-9-]', "object comparison with literal"),
     (r' [=!]=\s+(True|False|None)',
      "comparison with singleton, use 'is' or 'is not' instead"),
     (r'^\s*(while|if) [01]:',
      "use True/False for constant Boolean expression"),
+    (r'(?<!def)\s+hasattr',
+     'hasattr(foo, bar) is broken, use util.safehasattr(foo, bar) instead'),
     (r'opener\([^)]*\).read\(',
      "use opener.read() instead"),
     (r'opener\([^)]*\).write\(',
      "use opener.write() instead"),
     (r'[\s\(](open|file)\([^)]*\)\.read\(',
      "use util.readfile() instead"),
     (r'[\s\(](open|file)\([^)]*\)\.write\(',
      "use util.readfile() instead"),
     (r'^[\s\(]*(open(er)?|file)\([^)]*\)',
      "always assign an opened file to a variable, and close it afterwards"),
     (r'[\s\(](open|file)\([^)]*\)\.',
      "always assign an opened file to a variable, and close it afterwards"),
     (r'(?i)descendent', "the proper spelling is descendAnt"),
     (r'\.debug\(\_', "don't mark debug messages for translation"),
   ],
   # warnings
   [
     (r'.{81}', "warning: line over 80 characters"),
     (r'^\s*except:$', "warning: naked except clause"),
     (r'ui\.(status|progress|write|note|warn)\([\'\"]x',
      "warning: unwrapped ui message"),
   ]
 ]
 
 pyfilters = [
     (r"""(?msx)(?P<comment>\#.*?$)|
          ((?P<quote>('''|\"\"\"|(?<!')'(?!')|(?<!")"(?!")))
           (?P<text>(([^\\]|\\.)*?))
           (?P=quote))""", reppython),
 ]
 
 cpats = [
   [
     (r'//', "don't use //-style comments"),
     (r'^  ', "don't use spaces to indent"),
     (r'\S\t', "don't use tabs except for indent"),
     (r'(\S\s+|^\s+)\n', "trailing whitespace"),
     (r'.{85}', "line too long"),
     (r'(while|if|do|for)\(', "use space after while/if/do/for"),
     (r'return\(', "return is not a function"),
     (r' ;', "no space before ;"),
     (r'\w+\* \w+', "use int *foo, not int* foo"),
     (r'\([^\)]+\) \w+', "use (int)foo, not (int) foo"),
     (r'\S+ (\+\+|--)', "use foo++, not foo ++"),
     (r'\w,\w', "missing whitespace after ,"),
     (r'^[^#]\w[+/*]\w', "missing whitespace in expression"),
     (r'^#\s+\w', "use #foo, not # foo"),
     (r'[^\n]\Z', "no trailing newline"),
     (r'^\s*#import\b', "use only #include in standard C code"),
   ],
   # warnings
   []
 ]
 
 cfilters = [
     (r'(/\*)(((\*(?!/))|[^*])*)\*/', repccomment),
     (r'''(?P<quote>(?<!")")(?P<text>([^"]|\\")+)"(?!")''', repquote),
     (r'''(#\s*include\s+<)([^>]+)>''', repinclude),
     (r'(\()([^)]+\))', repcallspaces),
 ]
 
 inutilpats = [
   [
     (r'\bui\.', "don't use ui in util"),
   ],
   # warnings
   []
 ]
 
 inrevlogpats = [
   [
     (r'\brepo\.', "don't use repo in revlog"),
   ],
   # warnings
   []
 ]
 
 checks = [
     ('python', r'.*\.(py|cgi)$', pyfilters, pypats),
     ('test script', r'(.*/)?test-[^.~]*$', testfilters, testpats),
     ('c', r'.*\.c$', cfilters, cpats),
     ('unified test', r'.*\.t$', utestfilters, utestpats),
     ('layering violation repo in revlog', r'mercurial/revlog\.py', pyfilters,
      inrevlogpats),
     ('layering violation ui in util', r'mercurial/util\.py', pyfilters,
      inutilpats),
 ]
 
 class norepeatlogger(object):
     def __init__(self):
         self._lastseen = None
 
     def log(self, fname, lineno, line, msg, blame):
         """print error related a to given line of a given file.
 
         The faulty line will also be printed but only once in the case
         of multiple errors.
 
         :fname: filename
         :lineno: line number
         :line: actual content of the line
         :msg: error message
         """
         msgid = fname, lineno, line
         if msgid != self._lastseen:
             if blame:
                 print "%s:%d (%s):" % (fname, lineno, blame)
             else:
                 print "%s:%d:" % (fname, lineno)
             print " > %s" % line
             self._lastseen = msgid
         print " " + msg
 
 _defaultlogger = norepeatlogger()
 
 def getblame(f):
     lines = []
     for l in os.popen('hg annotate -un %s' % f):
         start, line = l.split(':', 1)
         user, rev = start.split()
         lines.append((line[1:-1], user, rev))
     return lines
 
 def checkfile(f, logfunc=_defaultlogger.log, maxerr=None, warnings=False,
               blame=False, debug=False):
     """checks style and portability of a given file
 
     :f: filepath
     :logfunc: function used to report error
               logfunc(filename, linenumber, linecontent, errormessage)
     :maxerr: number of error to display before arborting.
              Set to None (default) to report all errors
 
     return True if no error is found, False otherwise.
     """
     blamecache = None
     result = True
     for name, match, filters, pats in checks:
         if debug:
             print name, f
         fc = 0
         if not re.match(match, f):
             if debug:
                 print "Skipping %s for %s it doesn't match %s" % (
                        name, match, f)
             continue
         fp = open(f)
         pre = post = fp.read()
         fp.close()
         if "no-" + "check-code" in pre:
             if debug:
                 print "Skipping %s for %s it has no- and check-code" % (
                        name, f)
             break
         for p, r in filters:
             post = re.sub(p, r, post)
         if warnings:
             pats = pats[0] + pats[1]
         else:
             pats = pats[0]
         # print post # uncomment to show filtered version
         z = enumerate(zip(pre.splitlines(), post.splitlines(True)))
         if debug:
             print "Checking %s for %s" % (name, f)
         for n, l in z:
             if "check-code" + "-ignore" in l[0]:
                 if debug:
                     print "Skipping %s for %s:%s (check-code -ignore)" % (
                            name, f, n)
                 continue
             for p, msg in pats:
                 if re.search(p, l[1]):
                     bd = ""
                     if blame:
                         bd = 'working directory'
                         if not blamecache:
                             blamecache = getblame(f)
                         if n < len(blamecache):
                             bl, bu, br = blamecache[n]
                             if bl == l[0]:
                                 bd = '%s@%s' % (bu, br)
                     logfunc(f, n + 1, l[0], msg, bd)
                     fc += 1
                     result = False
             if maxerr is not None and fc >= maxerr:
                 print " (too many errors, giving up)"
                 break
     return result
 
 if __name__ == "__main__":
     parser = optparse.OptionParser("%prog [options] [files]")
     parser.add_option("-w", "--warnings", action="store_true",
                       help="include warning-level checks")
     parser.add_option("-p", "--per-file", type="int",
                       help="max warnings per file")
     parser.add_option("-b", "--blame", action="store_true",
                       help="use annotate to generate blame info")
     parser.add_option("", "--debug", action="store_true",
                       help="show debug information")
 
     parser.set_defaults(per_file=15, warnings=False, blame=False, debug=False)
     (options, args) = parser.parse_args()
 
     if len(args) == 0:
         check = glob.glob("*")
     else:
         check = args
 
     for f in check:
         ret = 0
         if not checkfile(f, maxerr=options.per_file, warnings=options.warnings,
                          blame=options.blame, debug=options.debug):
             ret = 1
     sys.exit(ret)

contrib/setup3k.py

@@ -1,373 +1,373
 #
 # This is an experimental py3k-enabled mercurial setup script.
 #
 # 'python setup.py install', or
 # 'python setup.py --help' for more options
 
 from distutils.command.build_py import build_py_2to3
 from lib2to3.refactor import get_fixers_from_package as getfixers
 
 import sys
-if not hasattr(sys, 'version_info') or sys.version_info < (2, 4, 0, 'final'):
+if getattr(sys, 'version_info', (0, 0, 0)) < (2, 4, 0, 'final'):
     raise SystemExit("Mercurial requires Python 2.4 or later.")
 
 if sys.version_info[0] >= 3:
     def b(s):
         '''A helper function to emulate 2.6+ bytes literals using string
         literals.'''
         return s.encode('latin1')
 else:
     def b(s):
         '''A helper function to emulate 2.6+ bytes literals using string
         literals.'''
         return s
 
 # Solaris Python packaging brain damage
 try:
     import hashlib
     sha = hashlib.sha1()
 except:
     try:
         import sha
     except:
         raise SystemExit(
             "Couldn't import standard hashlib (incomplete Python install).")
 
 try:
     import zlib
 except:
     raise SystemExit(
         "Couldn't import standard zlib (incomplete Python install).")
 
 try:
     import bz2
 except:
     raise SystemExit(
         "Couldn't import standard bz2 (incomplete Python install).")
 
 import os, subprocess, time
 import shutil
 import tempfile
 from distutils import log
 from distutils.core import setup, Extension
 from distutils.dist import Distribution
 from distutils.command.build import build
 from distutils.command.build_ext import build_ext
 from distutils.command.build_py import build_py
 from distutils.spawn import spawn, find_executable
 from distutils.ccompiler import new_compiler
 from distutils.errors import CCompilerError
 
 scripts = ['hg']
 if os.name == 'nt':
     scripts.append('contrib/win32/hg.bat')
 
 # simplified version of distutils.ccompiler.CCompiler.has_function
 # that actually removes its temporary files.
 def hasfunction(cc, funcname):
     tmpdir = tempfile.mkdtemp(prefix='hg-install-')
     devnull = oldstderr = None
     try:
         try:
             fname = os.path.join(tmpdir, 'funcname.c')
             f = open(fname, 'w')
             f.write('int main(void) {\n')
             f.write('    %s();\n' % funcname)
             f.write('}\n')
             f.close()
             # Redirect stderr to /dev/null to hide any error messages
             # from the compiler.
             # This will have to be changed if we ever have to check
             # for a function on Windows.
             devnull = open('/dev/null', 'w')
             oldstderr = os.dup(sys.stderr.fileno())
             os.dup2(devnull.fileno(), sys.stderr.fileno())
             objects = cc.compile([fname], output_dir=tmpdir)
             cc.link_executable(objects, os.path.join(tmpdir, "a.out"))
         except:
             return False
         return True
     finally:
         if oldstderr is not None:
             os.dup2(oldstderr, sys.stderr.fileno())
         if devnull is not None:
             devnull.close()
         shutil.rmtree(tmpdir)
 
 # py2exe needs to be installed to work
 try:
     import py2exe
     py2exeloaded = True
 
     # Help py2exe to find win32com.shell
     try:
         import modulefinder
         import win32com
         for p in win32com.__path__[1:]: # Take the path to win32comext
             modulefinder.AddPackagePath("win32com", p)
         pn = "win32com.shell"
         __import__(pn)
         m = sys.modules[pn]
         for p in m.__path__[1:]:
             modulefinder.AddPackagePath(pn, p)
     except ImportError:
         pass
 
 except ImportError:
     py2exeloaded = False
     pass
 
 def runcmd(cmd, env):
     p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                          stderr=subprocess.PIPE, env=env)
     out, err = p.communicate()
     # If root is executing setup.py, but the repository is owned by
     # another user (as in "sudo python setup.py install") we will get
     # trust warnings since the .hg/hgrc file is untrusted. That is
     # fine, we don't want to load it anyway.  Python may warn about
     # a missing __init__.py in mercurial/locale, we also ignore that.
     err = [e for e in err.splitlines()
            if not e.startswith(b('Not trusting file')) \
               and not e.startswith(b('warning: Not importing'))]
     if err:
         return ''
     return out
 
 version = ''
 
 if os.path.isdir('.hg'):
     # Execute hg out of this directory with a custom environment which
     # includes the pure Python modules in mercurial/pure. We also take
     # care to not use any hgrc files and do no localization.
     pypath = ['mercurial', os.path.join('mercurial', 'pure')]
     env = {'PYTHONPATH': os.pathsep.join(pypath),
            'HGRCPATH': '',
            'LANGUAGE': 'C'}
     if 'LD_LIBRARY_PATH' in os.environ:
         env['LD_LIBRARY_PATH'] = os.environ['LD_LIBRARY_PATH']
     if 'SystemRoot' in os.environ:
         # Copy SystemRoot into the custom environment for Python 2.6
         # under Windows. Otherwise, the subprocess will fail with
         # error 0xc0150004. See: http://bugs.python.org/issue3440
         env['SystemRoot'] = os.environ['SystemRoot']
     cmd = [sys.executable, 'hg', 'id', '-i', '-t']
     l = runcmd(cmd, env).split()
     while len(l) > 1 and l[-1][0].isalpha(): # remove non-numbered tags
         l.pop()
     if len(l) > 1: # tag found
         version = l[-1]
         if l[0].endswith('+'): # propagate the dirty status to the tag
             version += '+'
     elif len(l) == 1: # no tag found
         cmd = [sys.executable, 'hg', 'parents', '--template',
                '{latesttag}+{latesttagdistance}-']
         version = runcmd(cmd, env) + l[0]
     if version.endswith('+'):
         version += time.strftime('%Y%m%d')
 elif os.path.exists('.hg_archival.txt'):
     kw = dict([[t.strip() for t in l.split(':', 1)]
                for l in open('.hg_archival.txt')])
     if 'tag' in kw:
         version =  kw['tag']
     elif 'latesttag' in kw:
         version = '%(latesttag)s+%(latesttagdistance)s-%(node).12s' % kw
     else:
         version = kw.get('node', '')[:12]
 
 if version:
     f = open("mercurial/__version__.py", "w")
     f.write('# this file is autogenerated by setup.py\n')
     f.write('version = "%s"\n' % version)
     f.close()
 
 
 try:
     from mercurial import __version__
     version = __version__.version
 except ImportError:
     version = 'unknown'
 
 class hgbuildmo(build):
 
     description = "build translations (.mo files)"
 
     def run(self):
         if not find_executable('msgfmt'):
             self.warn("could not find msgfmt executable, no translations "
                      "will be built")
             return
 
         podir = 'i18n'
         if not os.path.isdir(podir):
             self.warn("could not find %s/ directory" % podir)
             return
 
         join = os.path.join
         for po in os.listdir(podir):
             if not po.endswith('.po'):
                 continue
             pofile = join(podir, po)
             modir = join('locale', po[:-3], 'LC_MESSAGES')
             mofile = join(modir, 'hg.mo')
             mobuildfile = join('mercurial', mofile)
             cmd = ['msgfmt', '-v', '-o', mobuildfile, pofile]
             if sys.platform != 'sunos5':
                 # msgfmt on Solaris does not know about -c
                 cmd.append('-c')
             self.mkpath(join('mercurial', modir))
             self.make_file([pofile], mobuildfile, spawn, (cmd,))
 
 # Insert hgbuildmo first so that files in mercurial/locale/ are found
 # when build_py is run next.
 build.sub_commands.insert(0, ('build_mo', None))
 # We also need build_ext before build_py. Otherwise, when 2to3 is called (in
 # build_py), it will not find osutil & friends, thinking that those modules are
 # global and, consequently, making a mess, now that all module imports are
 # global.
 build.sub_commands.insert(1, ('build_ext', None))
 
 Distribution.pure = 0
 Distribution.global_options.append(('pure', None, "use pure (slow) Python "
                                     "code instead of C extensions"))
 
 class hgbuildext(build_ext):
 
     def build_extension(self, ext):
         try:
             build_ext.build_extension(self, ext)
         except CCompilerError:
-            if not hasattr(ext, 'optional') or not ext.optional:
+            if getattr(ext, 'optional', False):
                 raise
             log.warn("Failed to build optional extension '%s' (skipping)",
                      ext.name)
 
 class hgbuildpy(build_py_2to3):
     fixer_names = sorted(set(getfixers("lib2to3.fixes") +
                              getfixers("hgfixes")))
 
     def finalize_options(self):
         build_py.finalize_options(self)
 
         if self.distribution.pure:
             if self.py_modules is None:
                 self.py_modules = []
             for ext in self.distribution.ext_modules:
                 if ext.name.startswith("mercurial."):
                     self.py_modules.append("mercurial.pure.%s" % ext.name[10:])
             self.distribution.ext_modules = []
 
     def find_modules(self):
         modules = build_py.find_modules(self)
         for module in modules:
             if module[0] == "mercurial.pure":
                 if module[1] != "__init__":
                     yield ("mercurial", module[1], module[2])
             else:
                 yield module
 
     def run(self):
         # In the build_py_2to3 class, self.updated_files = [], but I couldn't
         # see when that variable was updated to point to the updated files, as
         # its names suggests. Thus, I decided to just find_all_modules and feed
         # them to 2to3. Unfortunately, subsequent calls to setup3k.py will
         # incur in 2to3 analysis overhead.
         self.updated_files = [i[2] for i in self.find_all_modules()]
 
         # Base class code
         if self.py_modules:
             self.build_modules()
         if self.packages:
             self.build_packages()
             self.build_package_data()
 
         # 2to3
         self.run_2to3(self.updated_files)
 
         # Remaining base class code
         self.byte_compile(self.get_outputs(include_bytecode=0))
 
 cmdclass = {'build_mo': hgbuildmo,
             'build_ext': hgbuildext,
             'build_py': hgbuildpy}
 
 packages = ['mercurial', 'mercurial.hgweb', 'hgext', 'hgext.convert',
             'hgext.highlight', 'hgext.zeroconf']
 
 pymodules = []
 
 extmodules = [
     Extension('mercurial.base85', ['mercurial/base85.c']),
     Extension('mercurial.bdiff', ['mercurial/bdiff.c']),
     Extension('mercurial.diffhelpers', ['mercurial/diffhelpers.c']),
     Extension('mercurial.mpatch', ['mercurial/mpatch.c']),
     Extension('mercurial.parsers', ['mercurial/parsers.c']),
     ]
 
 # disable osutil.c under windows + python 2.4 (issue1364)
 if sys.platform == 'win32' and sys.version_info < (2, 5, 0, 'final'):
     pymodules.append('mercurial.pure.osutil')
 else:
     extmodules.append(Extension('mercurial.osutil', ['mercurial/osutil.c']))
 
 if sys.platform.startswith('linux') and os.uname()[2] > '2.6':
     # The inotify extension is only usable with Linux 2.6 kernels.
     # You also need a reasonably recent C library.
     # In any case, if it fails to build the error will be skipped ('optional').
     cc = new_compiler()
     if hasfunction(cc, 'inotify_add_watch'):
         inotify = Extension('hgext.inotify.linux._inotify',
                             ['hgext/inotify/linux/_inotify.c'],
                             ['mercurial'])
         inotify.optional = True
         extmodules.append(inotify)
         packages.extend(['hgext.inotify', 'hgext.inotify.linux'])
 
 packagedata = {'mercurial': ['locale/*/LC_MESSAGES/hg.mo',
                              'help/*.txt']}
 
 def ordinarypath(p):
     return p and p[0] != '.' and p[-1] != '~'
 
 for root in ('templates',):
     for curdir, dirs, files in os.walk(os.path.join('mercurial', root)):
         curdir = curdir.split(os.sep, 1)[1]
         dirs[:] = filter(ordinarypath, dirs)
         for f in filter(ordinarypath, files):
             f = os.path.join(curdir, f)
             packagedata['mercurial'].append(f)
 
 datafiles = []
 setupversion = version
 extra = {}
 
 if py2exeloaded:
     extra['console'] = [
         {'script':'hg',
          'copyright':'Copyright (C) 2005-2010 Matt Mackall and others',
          'product_version':version}]
 
 if os.name == 'nt':
     # Windows binary file versions for exe/dll files must have the
     # form W.X.Y.Z, where W,X,Y,Z are numbers in the range 0..65535
     setupversion = version.split('+', 1)[0]
 
 setup(name='mercurial',
       version=setupversion,
       author='Matt Mackall',
       author_email='[e-mail]',
       url='http://mercurial.selenic.com/',
       description='Scalable distributed SCM',
       license='GNU GPLv2+',
       scripts=scripts,
       packages=packages,
       py_modules=pymodules,
       ext_modules=extmodules,
       data_files=datafiles,
       package_data=packagedata,
       cmdclass=cmdclass,
       options=dict(py2exe=dict(packages=['hgext', 'email']),
                    bdist_mpkg=dict(zipdist=True,
                                    license='COPYING',
                                    readme='contrib/macosx/Readme.html',
                                    welcome='contrib/macosx/Welcome.html')),
       **extra)

contrib/win32/hgwebdir_wsgi.py

@@ -1,95 +1,95
 # An example WSGI script for IIS/isapi-wsgi to export multiple hgweb repos
 # Copyright 2010 Sune Foldager <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 #
 # Requirements:
 # - Python 2.6
 # - PyWin32 build 214 or newer
 # - Mercurial installed from source (python setup.py install)
 # - IIS 7
 #
 # Earlier versions will in general work as well, but the PyWin32 version is
 # necessary for win32traceutil to work correctly.
 #
 #
 # Installation and use:
 #
 # - Download the isapi-wsgi source and run python setup.py install:
 #   http://code.google.com/p/isapi-wsgi/
 #
 # - Run this script (i.e. python hgwebdir_wsgi.py) to get a shim dll. The
 #   shim is identical for all scripts, so you can just copy and rename one
 #   from an earlier run, if you wish.
 #
 # - Setup an IIS application where your hgwebdir is to be served from.
 #   On 64-bit systems, make sure it's assigned a 32-bit app pool.
 #
 # - In the application, setup a wildcard script handler mapping of type
 #   IpsapiModule with the shim dll as its executable. This file MUST reside
 #   in the same directory as the shim. Remove all other handlers, if you wish.
 #
 # - Make sure the ISAPI and CGI restrictions (configured globally on the
 #   web server) includes the shim dll, to allow it to run.
 #
 # - Adjust the configuration variables below to match your needs.
 #
 
 # Configuration file location
 hgweb_config = r'c:\src\iis\hg\hgweb.config'
 
 # Global settings for IIS path translation
 path_strip = 0   # Strip this many path elements off (when using url rewrite)
 path_prefix = 1  # This many path elements are prefixes (depends on the
                  # virtual path of the IIS application).
 
 import sys
 
 # Adjust python path if this is not a system-wide install
 #sys.path.insert(0, r'c:\path\to\python\lib')
 
 # Enable tracing. Run 'python -m win32traceutil' to debug
-if hasattr(sys, 'isapidllhandle'):
+if getattr(sys, 'isapidllhandle', None) is not None:
     import win32traceutil
 
 # To serve pages in local charset instead of UTF-8, remove the two lines below
 import os
 os.environ['HGENCODING'] = 'UTF-8'
 
 
 import isapi_wsgi
 from mercurial import demandimport; demandimport.enable()
 from mercurial.hgweb.hgwebdir_mod import hgwebdir
 
 # Example tweak: Replace isapi_wsgi's handler to provide better error message
 # Other stuff could also be done here, like logging errors etc.
 class WsgiHandler(isapi_wsgi.IsapiWsgiHandler):
     error_status = '500 Internal Server Error' # less silly error message
 
 isapi_wsgi.IsapiWsgiHandler = WsgiHandler
 
 # Only create the hgwebdir instance once
 application = hgwebdir(hgweb_config)
 
 def handler(environ, start_response):
 
     # Translate IIS's weird URLs
     url = environ['SCRIPT_NAME'] + environ['PATH_INFO']
     paths = url[1:].split('/')[path_strip:]
     script_name = '/' + '/'.join(paths[:path_prefix])
     path_info = '/'.join(paths[path_prefix:])
     if path_info:
         path_info = '/' + path_info
     environ['SCRIPT_NAME'] = script_name
     environ['PATH_INFO'] = path_info
 
     return application(environ, start_response)
 
 def __ExtensionFactory__():
     return isapi_wsgi.ISAPISimpleHandler(handler)
 
 if __name__=='__main__':
     from isapi.install import *
     params = ISAPIParameters()
     HandleCommandLine(params)

contrib/zsh_completion

@@ -1,1010 +1,1023
 #compdef hg
 
 # Zsh completion script for mercurial.  Rename this file to _hg and copy
 # it into your zsh function path (/usr/share/zsh/site-functions for
 # instance)
 #
 # If you do not want to install it globally, you can copy it somewhere
 # else and add that directory to $fpath. This must be done before
 # compinit is called. If the file is copied to ~/.zsh.d, your ~/.zshrc
 # file could look like this:
 #
 # fpath=("$HOME/.zsh.d" $fpath)
 # autoload -U compinit
 # compinit
 #
 # Copyright (C) 2005, 2006 Steve Borho <[e-mail]>
 # Copyright (C) 2006-10 Brendan Cully <[e-mail]>
 #
 # Permission is hereby granted, without written agreement and without
 # licence or royalty fees, to use, copy, modify, and distribute this
 # software and to distribute modified versions of this software for any
 # purpose, provided that the above copyright notice and the following
 # two paragraphs appear in all copies of this software.
 #
 # In no event shall the authors be liable to any party for direct,
 # indirect, special, incidental, or consequential damages arising out of
 # the use of this software and its documentation, even if the authors
 # have been advised of the possibility of such damage.
 #
 # The authors specifically disclaim any warranties, including, but not
 # limited to, the implied warranties of merchantability and fitness for
 # a particular purpose.  The software provided hereunder is on an "as
 # is" basis, and the authors have no obligation to provide maintenance,
 # support, updates, enhancements, or modifications.
 
 emulate -LR zsh
 setopt extendedglob
 
 local curcontext="$curcontext" state line
 typeset -A _hg_cmd_globals
 
 _hg() {
   local cmd _hg_root
   integer i=2
   _hg_cmd_globals=()
 
   while (( i < $#words ))
   do
     case "$words[$i]" in
       -R|--repository)
         eval _hg_root="$words[$i+1]"
         _hg_cmd_globals+=("$words[$i]" "$_hg_root")
         (( i += 2 ))
         continue
       ;;
       -R*)
         _hg_cmd_globals+="$words[$i]"
         eval _hg_root="${words[$i]#-R}"
        (( i++ ))
        continue
       ;;
       --cwd|--config)
         # pass along arguments to hg completer
         _hg_cmd_globals+=("$words[$i]" "$words[$i+1]")
         (( i += 2 ))
         continue
       ;;
       -*)
         # skip option
         (( i++ ))
         continue
       ;;
     esac
     if [[ -z "$cmd" ]]
     then
       cmd="$words[$i]"
       words[$i]=()
       (( CURRENT-- ))
     fi
     (( i++ ))
   done
 
   if [[ -z "$cmd" ]]
   then
     _arguments -s -w : $_hg_global_opts \
     ':mercurial command:_hg_commands'
     return
   fi
 
   # resolve abbreviations and aliases
   if ! (( $+functions[_hg_cmd_${cmd}] ))
   then
     local cmdexp
     (( $#_hg_cmd_list )) || _hg_get_commands
 
     cmdexp=$_hg_cmd_list[(r)${cmd}*]
     if [[ $cmdexp == $_hg_cmd_list[(R)${cmd}*] ]]
     then
       # might be nice to rewrite the command line with the expansion
       cmd="$cmdexp"
     fi
     if [[ -n $_hg_alias_list[$cmd] ]]
     then
       cmd=$_hg_alias_list[$cmd]
     fi
   fi
 
   curcontext="${curcontext%:*:*}:hg-${cmd}:"
 
   zstyle -s ":completion:$curcontext:" cache-policy update_policy
 
   if [[ -z "$update_policy" ]]
   then
     zstyle ":completion:$curcontext:" cache-policy _hg_cache_policy
   fi
 
   if (( $+functions[_hg_cmd_${cmd}] ))
   then
     _hg_cmd_${cmd}
   else
     # complete unknown commands normally
     _arguments -s -w : $_hg_global_opts \
       '*:files:_hg_files'
   fi
 }
 
 _hg_cache_policy() {
   typeset -a old
 
   # cache for a minute
   old=( "$1"(mm+10) )
   (( $#old )) && return 0
 
   return 1
 }
 
 _hg_get_commands() {
   typeset -ga _hg_cmd_list
   typeset -gA _hg_alias_list
   local hline cmd cmdalias
 
   _call_program hg hg debugcomplete -v | while read -A hline
   do
     cmd=$hline[1]
     _hg_cmd_list+=($cmd)
 
     for cmdalias in $hline[2,-1]
     do
       _hg_cmd_list+=($cmdalias)
       _hg_alias_list+=($cmdalias $cmd)
     done
   done
 }
 
 _hg_commands() {
   (( $#_hg_cmd_list )) || _hg_get_commands
   _describe -t commands 'mercurial command' _hg_cmd_list
 }
 
 _hg_revrange() {
   compset -P 1 '*:'
   _hg_labels "$@"
 }
 
 _hg_labels() {
   _hg_tags "$@"
   _hg_bookmarks "$@"
+  _hg_branches "$@"
 }
 
 _hg_tags() {
   typeset -a tags
   local tag rev
 
   _hg_cmd tags | while read tag
   do
     tags+=(${tag/ #    [0-9]#:*})
   done
   (( $#tags )) && _describe -t tags 'tags' tags
 }
 
 _hg_bookmarks() {
   typeset -a bookmark bookmarks
 
   _hg_cmd bookmarks | while read -A bookmark
   do
     if test -z ${bookmark[-1]:#[0-9]*}
     then
       bookmarks+=($bookmark[-2])
     fi
   done
   (( $#bookmarks )) && _describe -t bookmarks 'bookmarks' bookmarks
 }
 
+_hg_branches() {
+  typeset -a branches
+  local branch
+
+  _hg_cmd branches | while read branch
+  do
+    branches+=(${branch/ #    [0-9]#:*})
+  done
+  (( $#branches )) && _describe -t branches 'branches' branches
+}
+
 # likely merge candidates
 _hg_mergerevs() {
   typeset -a heads
   local myrev
 
   heads=(${(f)"$(_hg_cmd heads --template '{rev}\\n')"})
   # exclude own revision
   myrev=$(_hg_cmd log -r . --template '{rev}\\n')
   heads=(${heads:#$myrev})
 
   (( $#heads )) && _describe -t heads 'heads' heads
 }
 
 _hg_files() {
   if [[ -n "$_hg_root" ]]
   then
     [[ -d "$_hg_root/.hg" ]] || return
     case "$_hg_root" in
       /*)
         _files -W $_hg_root
       ;;
       *)
         _files -W $PWD/$_hg_root
       ;;
     esac
   else
     _files
   fi
 }
 
 _hg_status() {
   [[ -d $PREFIX ]] || PREFIX=$PREFIX:h
   status_files=(${(ps:\0:)"$(_hg_cmd status -0n$1 ./$PREFIX)"})
 }
 
 _hg_unknown() {
   typeset -a status_files
   _hg_status u
   _wanted files expl 'unknown files' _multi_parts / status_files
 }
 
 _hg_missing() {
   typeset -a status_files
   _hg_status d
   _wanted files expl 'missing files' _multi_parts / status_files
 }
 
 _hg_modified() {
   typeset -a status_files
   _hg_status m
   _wanted files expl 'modified files' _multi_parts / status_files
 }
 
 _hg_resolve() {
   local rstate rpath
 
   [[ -d $PREFIX ]] || PREFIX=$PREFIX:h
 
   _hg_cmd resolve -l ./$PREFIX | while read rstate rpath
   do
     [[ $rstate == 'R' ]] && resolved_files+=($rpath)
     [[ $rstate == 'U' ]] && unresolved_files+=($rpath)
   done
 }
 
 _hg_resolved() {
   typeset -a resolved_files unresolved_files
   _hg_resolve
   _wanted files expl 'resolved files' _multi_parts / resolved_files
 }
 
 _hg_unresolved() {
   typeset -a resolved_files unresolved_files
   _hg_resolve
   _wanted files expl 'unresolved files' _multi_parts / unresolved_files
 }
 
 _hg_config() {
     typeset -a items
     items=(${${(%f)"$(_call_program hg hg showconfig)"}%%\=*})
     (( $#items )) && _describe -t config 'config item' items
 }
 
 _hg_addremove() {
   _alternative 'files:unknown files:_hg_unknown' \
     'files:missing files:_hg_missing'
 }
 
 _hg_ssh_urls() {
   if [[ -prefix */ ]]
   then
     if zstyle -T ":completion:${curcontext}:files" remote-access
     then
       local host=${PREFIX%%/*}
       typeset -a remdirs
       compset -p $(( $#host + 1 ))
       local rempath=${(M)PREFIX##*/}
       local cacheid="hg:${host}-${rempath//\//_}"
       cacheid=${cacheid%[-_]}
       compset -P '*/'
       if _cache_invalid "$cacheid" || ! _retrieve_cache "$cacheid"
       then
         remdirs=(${${(M)${(f)"$(_call_program files ssh -a -x $host ls -1FL "${(q)rempath}")"}##*/}%/})
         _store_cache "$cacheid" remdirs
       fi
       _describe -t directories 'remote directory' remdirs -S/
     else
       _message 'remote directory'
     fi
   else
     if compset -P '*@'
     then
       _hosts -S/
     else
       _alternative 'hosts:remote host name:_hosts -S/' \
         'users:user:_users -S@'
     fi
   fi
 }
 
 _hg_urls() {
   if compset -P bundle://
   then
     _files
   elif compset -P ssh://
   then
     _hg_ssh_urls
   elif [[ -prefix *: ]]
   then
     _urls
   else
     local expl
     compset -S '[^:]*'
     _wanted url-schemas expl 'URL schema' compadd -S '' - \
       http:// https:// ssh:// bundle://
   fi
 }
 
 _hg_paths() {
   typeset -a paths pnames
   _hg_cmd paths | while read -A pnames
   do
     paths+=($pnames[1])
   done
   (( $#paths )) && _describe -t path-aliases 'repository alias' paths
 }
 
 _hg_remote() {
   _alternative 'path-aliases:repository alias:_hg_paths' \
     'directories:directory:_files -/' \
     'urls:URL:_hg_urls'
 }
 
 _hg_clone_dest() {
   _alternative 'directories:directory:_files -/' \
     'urls:URL:_hg_urls'
 }
 
 # Common options
 _hg_global_opts=(
     '(--repository -R)'{-R+,--repository}'[repository root directory]:repository:_files -/'
     '--cwd[change working directory]:new working directory:_files -/'
     '(--noninteractive -y)'{-y,--noninteractive}'[do not prompt, assume yes for any required answers]'
     '(--verbose -v)'{-v,--verbose}'[enable additional output]'
     '*--config[set/override config option]:defined config items:_hg_config'
     '(--quiet -q)'{-q,--quiet}'[suppress output]'
     '(--help -h)'{-h,--help}'[display help and exit]'
     '--debug[debug mode]'
     '--debugger[start debugger]'
     '--encoding[set the charset encoding]'
     '--encodingmode[set the charset encoding mode]'
     '--lsprof[print improved command execution profile]'
     '--traceback[print traceback on exception]'
     '--time[time how long the command takes]'
     '--profile[profile]'
     '--version[output version information and exit]'
 )
 
 _hg_pat_opts=(
   '*'{-I+,--include}'[include names matching the given patterns]:dir:_files -W $(_hg_cmd root) -/'
   '*'{-X+,--exclude}'[exclude names matching the given patterns]:dir:_files -W $(_hg_cmd root) -/')
 
 _hg_diff_opts=(
   '(--text -a)'{-a,--text}'[treat all files as text]'
   '(--git -g)'{-g,--git}'[use git extended diff format]'
   "--nodates[omit dates from diff headers]")
 
 _hg_dryrun_opts=(
   '(--dry-run -n)'{-n,--dry-run}'[do not perform actions, just print output]')
 
 _hg_style_opts=(
   '--style[display using template map file]:'
   '--template[display with template]:')
 
 _hg_commit_opts=(
   '(-m --message -l --logfile --edit -e)'{-e,--edit}'[edit commit message]'
   '(-e --edit -l --logfile --message -m)'{-m+,--message}'[use <text> as commit message]:message:'
   '(-e --edit -m --message --logfile -l)'{-l+,--logfile}'[read the commit message from <file>]:log file:_files')
 
 _hg_remote_opts=(
   '(--ssh -e)'{-e+,--ssh}'[specify ssh command to use]:'
   '--remotecmd[specify hg command to run on the remote side]:')
 
 _hg_cmd() {
   _call_program hg HGPLAIN=1 hg "$_hg_cmd_globals[@]" "$@" 2> /dev/null
 }
 
 _hg_cmd_add() {
   _arguments -s -w : $_hg_global_opts $_hg_pat_opts $_hg_dryrun_opts \
   '*:unknown files:_hg_unknown'
 }
 
 _hg_cmd_addremove() {
   _arguments -s -w : $_hg_global_opts $_hg_pat_opts $_hg_dryrun_opts \
   '(--similarity -s)'{-s+,--similarity}'[guess renamed files by similarity (0<=s<=100)]:' \
   '*:unknown or missing files:_hg_addremove'
 }
 
 _hg_cmd_annotate() {
   _arguments -s -w : $_hg_global_opts $_hg_pat_opts \
   '(--rev -r)'{-r+,--rev}'[annotate the specified revision]:revision:_hg_labels' \
   '(--follow -f)'{-f,--follow}'[follow file copies and renames]' \
   '(--text -a)'{-a,--text}'[treat all files as text]' \
   '(--user -u)'{-u,--user}'[list the author]' \
   '(--date -d)'{-d,--date}'[list the date]' \
   '(--number -n)'{-n,--number}'[list the revision number (default)]' \
   '(--changeset -c)'{-c,--changeset}'[list the changeset]' \
   '*:files:_hg_files'
 }
 
 _hg_cmd_archive() {
   _arguments -s -w : $_hg_global_opts $_hg_pat_opts \
   '--no-decode[do not pass files through decoders]' \
   '(--prefix -p)'{-p+,--prefix}'[directory prefix for files in archive]:' \
   '(--rev -r)'{-r+,--rev}'[revision to distribute]:revision:_hg_labels' \
   '(--type -t)'{-t+,--type}'[type of distribution to create]:archive type:(files tar tbz2 tgz uzip zip)' \
   '*:destination:_files'
 }
 
 _hg_cmd_backout() {
   _arguments -s -w : $_hg_global_opts $_hg_pat_opts \
     '--merge[merge with old dirstate parent after backout]' \
     '(--date -d)'{-d+,--date}'[record datecode as commit date]:date code:' \
     '--parent[parent to choose when backing out merge]' \
     '(--user -u)'{-u+,--user}'[record user as commiter]:user:' \
     '(--rev -r)'{-r+,--rev}'[revision]:revision:_hg_labels' \
     '(--message -m)'{-m+,--message}'[use <text> as commit message]:text:' \
     '(--logfile -l)'{-l+,--logfile}'[read commit message from <file>]:log file:_files -g \*.txt'
 }
 
 _hg_cmd_bisect() {
   _arguments -s -w : $_hg_global_opts \
   '(-)'{-r,--reset}'[reset bisect state]' \
   '(--good -g --bad -b --skip -s --reset -r)'{-g,--good}'[mark changeset good]'::revision:_hg_labels \
   '(--good -g --bad -b --skip -s --reset -r)'{-b,--bad}'[mark changeset bad]'::revision:_hg_labels \
   '(--good -g --bad -b --skip -s --reset -r)'{-s,--skip}'[skip testing changeset]' \
   '(--command -c --noupdate -U)'{-c+,--command}'[use command to check changeset state]':commands:_command_names \
   '(--command -c --noupdate -U)'{-U,--noupdate}'[do not update to target]'
 }
 
 _hg_cmd_bookmarks() {
   _arguments -s -w : $_hg_global_opts \
   '(--force -f)'{-f,--force}'[force]' \
   '(--rev -r --delete -d --rename -m)'{-r+,--rev}'[revision]:revision:_hg_labels' \
   '(--rev -r --delete -d --rename -m)'{-d,--delete}'[delete a given bookmark]' \
   '(--rev -r --delete -d --rename -m)'{-m+,--rename}'[rename a given bookmark]:bookmark:_hg_bookmarks' \
   ':bookmark:_hg_bookmarks'
 }
 
 _hg_cmd_branch() {
   _arguments -s -w : $_hg_global_opts \
   '(--force -f)'{-f,--force}'[set branch name even if it shadows an existing branch]' \
   '(--clean -C)'{-C,--clean}'[reset branch name to parent branch name]'
 }
 
 _hg_cmd_branches() {
   _arguments -s -w : $_hg_global_opts \
   '(--active -a)'{-a,--active}'[show only branches that have unmerge heads]'
 }
 
 _hg_cmd_bundle() {
   _arguments -s -w : $_hg_global_opts $_hg_remote_opts \
   '(--force -f)'{-f,--force}'[run even when remote repository is unrelated]' \
   '(2)*--base[a base changeset to specify instead of a destination]:revision:_hg_labels' \
   ':output file:_files' \
   ':destination repository:_files -/'
 }
 
 _hg_cmd_cat() {
   _arguments -s -w : $_hg_global_opts $_hg_pat_opts \
   '(--output -o)'{-o+,--output}'[print output to file with formatted name]:filespec:' \
   '(--rev -r)'{-r+,--rev}'[revision]:revision:_hg_labels' \
   '*:file:_hg_files'
 }
 
 _hg_cmd_clone() {
   _arguments -s -w : $_hg_global_opts $_hg_remote_opts \
   '(--noupdate -U)'{-U,--noupdate}'[do not update the new working directory]' \
   '(--rev -r)'{-r+,--rev}'[a changeset you would like to have after cloning]:' \
   '--uncompressed[use uncompressed transfer (fast over LAN)]' \
   ':source repository:_hg_remote' \
   ':destination:_hg_clone_dest'
 }
 
 _hg_cmd_commit() {
   _arguments -s -w : $_hg_global_opts $_hg_pat_opts \
   '(--addremove -A)'{-A,--addremove}'[mark new/missing files as added/removed before committing]' \
   '(--message -m)'{-m+,--message}'[use <text> as commit message]:text:' \
   '(--logfile -l)'{-l+,--logfile}'[read commit message from <file>]:log file:_files -g \*.txt' \
   '(--date -d)'{-d+,--date}'[record datecode as commit date]:date code:' \
   '(--user -u)'{-u+,--user}'[record user as commiter]:user:' \
   '*:file:_hg_files'
 }
 
 _hg_cmd_copy() {
   _arguments -s -w : $_hg_global_opts $_hg_pat_opts $_hg_dryrun_opts \
   '(--after -A)'{-A,--after}'[record a copy that has already occurred]' \
   '(--force -f)'{-f,--force}'[forcibly copy over an existing managed file]' \
   '*:file:_hg_files'
 }
 
 _hg_cmd_diff() {
   typeset -A opt_args
   _arguments -s -w : $_hg_global_opts $_hg_pat_opts $_hg_diff_opts \
   '*'{-r,--rev}'+[revision]:revision:_hg_revrange' \
   '(--show-function -p)'{-p,--show-function}'[show which function each change is in]' \
   '(--ignore-all-space -w)'{-w,--ignore-all-space}'[ignore white space when comparing lines]' \
   '(--ignore-space-change -b)'{-b,--ignore-space-change}'[ignore changes in the amount of white space]' \
   '(--ignore-blank-lines -B)'{-B,--ignore-blank-lines}'[ignore changes whose lines are all blank]' \
   '*:file:->diff_files'
 
   if [[ $state == 'diff_files' ]]
   then
     if [[ -n $opt_args[-r] ]]
     then
       _hg_files
     else
       _hg_modified
     fi
   fi
 }
 
 _hg_cmd_export() {
   _arguments -s -w : $_hg_global_opts $_hg_diff_opts \
   '(--outout -o)'{-o+,--output}'[print output to file with formatted name]:filespec:' \
   '--switch-parent[diff against the second parent]' \
   '*:revision:_hg_labels'
 }
 
 _hg_cmd_grep() {
   _arguments -s -w : $_hg_global_opts $_hg_pat_opts \
   '(--print0 -0)'{-0,--print0}'[end filenames with NUL]' \
   '--all[print all revisions with matches]' \
   '(--follow -f)'{-f,--follow}'[follow changeset or file history]' \
   '(--ignore-case -i)'{-i,--ignore-case}'[ignore case when matching]' \
   '(--files-with-matches -l)'{-l,--files-with-matches}'[print only filenames and revs that match]' \
   '(--line-number -n)'{-n,--line-number}'[print matching line numbers]' \
   '*'{-r+,--rev}'[search in given revision range]:revision:_hg_revrange' \
   '(--user -u)'{-u,--user}'[print user who committed change]' \
   '1:search pattern:' \
   '*:files:_hg_files'
 }
 
 _hg_cmd_heads() {
   _arguments -s -w : $_hg_global_opts $_hg_style_opts \
   '(--rev -r)'{-r+,--rev}'[show only heads which are descendants of rev]:revision:_hg_labels'
 }
 
 _hg_cmd_help() {
   _arguments -s -w : $_hg_global_opts \
   '*:mercurial command:_hg_commands'
 }
 
 _hg_cmd_identify() {
   _arguments -s -w : $_hg_global_opts \
   '(--rev -r)'{-r+,--rev}'[identify the specified rev]:revision:_hg_labels' \
   '(--num -n)'{-n+,--num}'[show local revision number]' \
   '(--id -i)'{-i+,--id}'[show global revision id]' \
   '(--branch -b)'{-b+,--branch}'[show branch]' \
   '(--tags -t)'{-t+,--tags}'[show tags]'
 }
 
 _hg_cmd_import() {
   _arguments -s -w : $_hg_global_opts \
   '(--strip -p)'{-p+,--strip}'[directory strip option for patch (default: 1)]:count:' \
   '(--message -m)'{-m+,--message}'[use <text> as commit message]:text:' \
   '(--force -f)'{-f,--force}'[skip check for outstanding uncommitted changes]' \
   '*:patch:_files'
 }
 
 _hg_cmd_incoming() {
   _arguments -s -w : $_hg_global_opts $_hg_remote_opts $_hg_style_opts \
   '(--no-merges -M)'{-M,--no-merges}'[do not show merge revisions]' \
   '(--force -f)'{-f,--force}'[run even when the remote repository is unrelated]' \
   '(--patch -p)'{-p,--patch}'[show patch]' \
   '(--rev -r)'{-r+,--rev}'[a specific revision up to which you would like to pull]:revision:_hg_tags' \
   '(--newest-first -n)'{-n,--newest-first}'[show newest record first]' \
   '--bundle[file to store the bundles into]:bundle file:_files' \
   ':source:_hg_remote'
 }
 
 _hg_cmd_init() {
   _arguments -s -w : $_hg_global_opts $_hg_remote_opts \
   ':dir:_files -/'
 }
 
 _hg_cmd_locate() {
   _arguments -s -w : $_hg_global_opts $_hg_pat_opts \
   '(--rev -r)'{-r+,--rev}'[search repository as it stood at revision]:revision:_hg_labels' \
   '(--print0 -0)'{-0,--print0}'[end filenames with NUL, for use with xargs]' \
   '(--fullpath -f)'{-f,--fullpath}'[print complete paths]' \
   '*:search pattern:_hg_files'
 }
 
 _hg_cmd_log() {
   _arguments -s -w : $_hg_global_opts $_hg_pat_opts $_hg_style_opts \
   '(--follow --follow-first -f)'{-f,--follow}'[follow changeset or history]' \
   '(-f --follow)--follow-first[only follow the first parent of merge changesets]' \
   '(--copies -C)'{-C,--copies}'[show copied files]' \
   '(--keyword -k)'{-k+,--keyword}'[search for a keyword]:' \
   '(--limit -l)'{-l+,--limit}'[limit number of changes displayed]:' \
   '*'{-r,--rev}'[show the specified revision or range]:revision:_hg_revrange' \
   '(--no-merges -M)'{-M,--no-merges}'[do not show merges]' \
   '(--only-merges -m)'{-m,--only-merges}'[show only merges]' \
   '(--patch -p)'{-p,--patch}'[show patch]' \
   '(--prune -P)'{-P+,--prune}'[do not display revision or any of its ancestors]:revision:_hg_labels' \
+  '(--branch -b)'{-b+,--branch}'[show changesets within the given named branch]:branch:_hg_branches' \
   '*:files:_hg_files'
 }
 
 _hg_cmd_manifest() {
   _arguments -s -w : $_hg_global_opts \
   ':revision:_hg_labels'
 }
 
 _hg_cmd_merge() {
   _arguments -s -w : $_hg_global_opts \
   '(--force -f)'{-f,--force}'[force a merge with outstanding changes]' \
   '(--rev -r 1)'{-r,--rev}'[revision to merge]:revision:_hg_mergerevs' \
   '(--preview -P)'{-P,--preview}'[review revisions to merge (no merge is performed)]' \
   ':revision:_hg_mergerevs'
 }
 
 _hg_cmd_outgoing() {
   _arguments -s -w : $_hg_global_opts $_hg_remote_opts $_hg_style_opts \
   '(--no-merges -M)'{-M,--no-merges}'[do not show merge revisions]' \
   '(--force -f)'{-f,--force}'[run even when the remote repository is unrelated]' \
   '(--patch -p)'{-p,--patch}'[show patch]' \
   '(--rev -r)'{-r+,--rev}'[a specific revision you would like to push]' \
   '(--newest-first -n)'{-n,--newest-first}'[show newest record first]' \
   ':destination:_hg_remote'
 }
 
 _hg_cmd_parents() {
   _arguments -s -w : $_hg_global_opts $_hg_style_opts \
   '(--rev -r)'{-r+,--rev}'[show parents of the specified rev]:revision:_hg_labels' \
   ':last modified file:_hg_files'
 }
 
 _hg_cmd_paths() {
   _arguments -s -w : $_hg_global_opts \
   ':path:_hg_paths'
 }
 
 _hg_cmd_pull() {
   _arguments -s -w : $_hg_global_opts $_hg_remote_opts \
   '(--force -f)'{-f,--force}'[run even when the remote repository is unrelated]' \
   '(--update -u)'{-u,--update}'[update to new tip if changesets were pulled]' \
   '(--rev -r)'{-r+,--rev}'[a specific revision up to which you would like to pull]:revision:' \
   ':source:_hg_remote'
 }
 
 _hg_cmd_push() {
   _arguments -s -w : $_hg_global_opts $_hg_remote_opts \
   '(--force -f)'{-f,--force}'[force push]' \
   '(--rev -r)'{-r+,--rev}'[a specific revision you would like to push]:revision:_hg_labels' \
   ':destination:_hg_remote'
 }
 
 _hg_cmd_remove() {
   _arguments -s -w : $_hg_global_opts $_hg_pat_opts \
   '(--after -A)'{-A,--after}'[record remove that has already occurred]' \
   '(--force -f)'{-f,--force}'[remove file even if modified]' \
   '*:file:_hg_files'
 }
 
 _hg_cmd_rename() {
   _arguments -s -w : $_hg_global_opts $_hg_pat_opts $_hg_dryrun_opts \
   '(--after -A)'{-A,--after}'[record a rename that has already occurred]' \
   '(--force -f)'{-f,--force}'[forcibly copy over an existing managed file]' \
   '*:file:_hg_files'
 }
 
 _hg_cmd_resolve() {
   local context state line
   typeset -A opt_args
 
   _arguments -s -w : $_hg_global_opts \
   '(--list -l --mark -m --unmark -u)'{-l,--list}'[list state of files needing merge]:*:merged files:->resolve_files' \
   '(--mark -m --list -l --unmark -u)'{-m,--mark}'[mark files as resolved]:*:unresolved files:_hg_unresolved' \
   '(--unmark -u --list -l --mark -m)'{-u,--unmark}'[unmark files as resolved]:*:resolved files:_hg_resolved' \
   '*:file:_hg_unresolved'
 
   if [[ $state == 'resolve_files' ]]
   then
     _alternative 'files:resolved files:_hg_resolved' \
       'files:unresolved files:_hg_unresolved'
   fi
 }
 
 _hg_cmd_revert() {
   local context state line
   typeset -A opt_args
 
   _arguments -s -w : $_hg_global_opts $_hg_pat_opts $_hg_dryrun_opts \
   '(--all -a :)'{-a,--all}'[revert all changes when no arguments given]' \
   '(--rev -r)'{-r+,--rev}'[revision to revert to]:revision:_hg_labels' \
   '--no-backup[do not save backup copies of files]' \
   '*:file:->diff_files'
 
   if [[ $state == 'diff_files' ]]
   then
     if [[ -n $opt_args[-r] ]]
     then
       _hg_files
     else
       typeset -a status_files
       _hg_status mard
       _wanted files expl 'modified, added, removed or deleted file' _multi_parts / status_files
     fi
   fi
 }
 
 _hg_cmd_serve() {
   _arguments -s -w : $_hg_global_opts \
   '(--accesslog -A)'{-A+,--accesslog}'[name of access log file]:log file:_files' \
   '(--errorlog -E)'{-E+,--errorlog}'[name of error log file]:log file:_files' \
   '(--daemon -d)'{-d,--daemon}'[run server in background]' \
   '(--port -p)'{-p+,--port}'[listen port]:listen port:' \
   '(--address -a)'{-a+,--address}'[interface address]:interface address:' \
   '(--name -n)'{-n+,--name}'[name to show in web pages]:repository name:' \
   '(--templates -t)'{-t,--templates}'[web template directory]:template dir:_files -/' \
   '--style[web template style]:style' \
   '--stdio[for remote clients]' \
   '(--ipv6 -6)'{-6,--ipv6}'[use IPv6 in addition to IPv4]'
 }
 
 _hg_cmd_showconfig() {
   _arguments -s -w : $_hg_global_opts \
   '(--untrusted -u)'{-u+,--untrusted}'[show untrusted configuration options]' \
   ':config item:_hg_config'
 }
 
 _hg_cmd_status() {
   _arguments -s -w : $_hg_global_opts $_hg_pat_opts \
   '(--all -A)'{-A,--all}'[show status of all files]' \
   '(--modified -m)'{-m,--modified}'[show only modified files]' \
   '(--added -a)'{-a,--added}'[show only added files]' \
   '(--removed -r)'{-r,--removed}'[show only removed files]' \
   '(--deleted -d)'{-d,--deleted}'[show only deleted (but tracked) files]' \
   '(--clean -c)'{-c,--clean}'[show only files without changes]' \
   '(--unknown -u)'{-u,--unknown}'[show only unknown files]' \
   '(--ignored -i)'{-i,--ignored}'[show ignored files]' \
   '(--no-status -n)'{-n,--no-status}'[hide status prefix]' \
   '(--copies -C)'{-C,--copies}'[show source of copied files]' \
   '(--print0 -0)'{-0,--print0}'[end filenames with NUL, for use with xargs]' \
   '--rev[show difference from revision]:revision:_hg_labels' \
   '*:files:_files'
 }
 
 _hg_cmd_summary() {
   _arguments -s -w : $_hg_global_opts \
   '--remote[check for push and pull]'
 }
 
 _hg_cmd_tag() {
   _arguments -s -w : $_hg_global_opts \
   '(--local -l)'{-l,--local}'[make the tag local]' \
   '(--message -m)'{-m+,--message}'[message for tag commit log entry]:message:' \
   '(--date -d)'{-d+,--date}'[record datecode as commit date]:date code:' \
   '(--user -u)'{-u+,--user}'[record user as commiter]:user:' \
   '(--rev -r)'{-r+,--rev}'[revision to tag]:revision:_hg_labels' \
   ':tag name:'
 }
 
 _hg_cmd_tip() {
   _arguments -s -w : $_hg_global_opts $_hg_style_opts \
   '(--patch -p)'{-p,--patch}'[show patch]'
 }
 
 _hg_cmd_unbundle() {
   _arguments -s -w : $_hg_global_opts \
   '(--update -u)'{-u,--update}'[update to new tip if changesets were unbundled]' \
   ':files:_files'
 }
 
 _hg_cmd_update() {
   _arguments -s -w : $_hg_global_opts \
   '(--clean -C)'{-C,--clean}'[overwrite locally modified files]' \
   '(--rev -r)'{-r+,--rev}'[revision]:revision:_hg_labels' \
   ':revision:_hg_labels'
 }
 
 ## extensions ##
 
 # HGK
 _hg_cmd_view() {
   _arguments -s -w : $_hg_global_opts \
   '(--limit -l)'{-l+,--limit}'[limit number of changes displayed]:' \
   ':revision range:_hg_tags'
 }
 
 # MQ
 _hg_qseries() {
   typeset -a patches
   patches=(${(f)"$(_hg_cmd qseries)"})
   (( $#patches )) && _describe -t hg-patches 'patches' patches
 }
 
 _hg_qapplied() {
   typeset -a patches
   patches=(${(f)"$(_hg_cmd qapplied)"})
   if (( $#patches ))
   then
     patches+=(qbase qtip)
     _describe -t hg-applied-patches 'applied patches' patches
   fi
 }
 
 _hg_qunapplied() {
   typeset -a patches
   patches=(${(f)"$(_hg_cmd qunapplied)"})
   (( $#patches )) && _describe -t hg-unapplied-patches 'unapplied patches' patches
 }
 
 # unapplied, including guarded patches
 _hg_qdeletable() {
   typeset -a unapplied
   unapplied=(${(f)"$(_hg_cmd qseries)"})
   for p in $(_hg_cmd qapplied)
   do
     unapplied=(${unapplied:#$p})
   done
 
   (( $#unapplied )) && _describe -t hg-allunapplied-patches 'all unapplied patches' unapplied
 }
 
 _hg_qguards() {
   typeset -a guards
   local guard
   compset -P "+|-"
   _hg_cmd qselect -s | while read guard
   do
     guards+=(${guard#(+|-)})
   done
   (( $#guards )) && _describe -t hg-guards 'guards' guards
 }
 
 _hg_qseries_opts=(
   '(--summary -s)'{-s,--summary}'[print first line of patch header]')
 
 _hg_cmd_qapplied() {
   _arguments -s -w : $_hg_global_opts $_hg_qseries_opts
 }
 
 _hg_cmd_qdelete() {
   _arguments -s -w : $_hg_global_opts \
   '(--keep -k)'{-k,--keep}'[keep patch file]' \
   '*'{-r+,--rev}'[stop managing a revision]:applied patch:_hg_revrange' \
   '*:unapplied patch:_hg_qdeletable'
 }
 
 _hg_cmd_qdiff() {
   _arguments -s -w : $_hg_global_opts $_hg_pat_opts \
   '*:pattern:_hg_files'
 }
 
 _hg_cmd_qfold() {
   _arguments -s -w : $_hg_global_opts $_h_commit_opts \
   '(--keep,-k)'{-k,--keep}'[keep folded patch files]' \
   '*:unapplied patch:_hg_qunapplied'
 }
 
 _hg_cmd_qgoto() {
   _arguments -s -w : $_hg_global_opts \
   '(--force -f)'{-f,--force}'[overwrite any local changes]' \
   ':patch:_hg_qseries'
 }
 
 _hg_cmd_qguard() {
   _arguments -s -w : $_hg_global_opts \
   '(--list -l)'{-l,--list}'[list all patches and guards]' \
   '(--none -n)'{-n,--none}'[drop all guards]' \
   ':patch:_hg_qseries' \
   '*:guards:_hg_qguards'
 }
 
 _hg_cmd_qheader() {
   _arguments -s -w : $_hg_global_opts \
   ':patch:_hg_qseries'
 }
 
 _hg_cmd_qimport() {
   _arguments -s -w : $_hg_global_opts \
   '(--existing -e)'{-e,--existing}'[import file in patch dir]' \
   '(--name -n 2)'{-n+,--name}'[patch file name]:name:' \
   '(--force -f)'{-f,--force}'[overwrite existing files]' \
   '*'{-r+,--rev}'[place existing revisions under mq control]:revision:_hg_revrange' \
   '*:patch:_files'
 }
 
 _hg_cmd_qnew() {
   _arguments -s -w : $_hg_global_opts $_hg_commit_opts \
   '(--force -f)'{-f,--force}'[import uncommitted changes into patch]' \
   ':patch:'
 }
 
 _hg_cmd_qnext() {
   _arguments -s -w : $_hg_global_opts $_hg_qseries_opts
 }
 
 _hg_cmd_qpop() {
   _arguments -s -w : $_hg_global_opts \
   '(--all -a :)'{-a,--all}'[pop all patches]' \
   '(--name -n)'{-n+,--name}'[queue name to pop]:' \
   '(--force -f)'{-f,--force}'[forget any local changes]' \
   ':patch:_hg_qapplied'
 }
 
 _hg_cmd_qprev() {
   _arguments -s -w : $_hg_global_opts $_hg_qseries_opts
 }
 
 _hg_cmd_qpush() {
   _arguments -s -w : $_hg_global_opts \
   '(--all -a :)'{-a,--all}'[apply all patches]' \
   '(--list -l)'{-l,--list}'[list patch name in commit text]' \
   '(--merge -m)'{-m+,--merge}'[merge from another queue]:' \
   '(--name -n)'{-n+,--name}'[merge queue name]:' \
   '(--force -f)'{-f,--force}'[apply if the patch has rejects]' \
   '--move[reorder patch series and apply only the patch]' \
   ':patch:_hg_qunapplied'
 }
 
 _hg_cmd_qrefresh() {
   _arguments -s -w : $_hg_global_opts $_hg_pat_opts $_hg_commit_opts \
   '(--git -g)'{-g,--git}'[use git extended diff format]' \
   '(--short -s)'{-s,--short}'[short refresh]' \
   '*:files:_hg_files'
 }
 
 _hg_cmd_qrename() {
   _arguments -s -w : $_hg_global_opts \
   ':patch:_hg_qseries' \
   ':destination:'
 }
 
 _hg_cmd_qselect() {
   _arguments -s -w : $_hg_global_opts \
   '(--none -n :)'{-n,--none}'[disable all guards]' \
   '(--series -s :)'{-s,--series}'[list all guards in series file]' \
   '--pop[pop to before first guarded applied patch]' \
   '--reapply[pop and reapply patches]' \
   '*:guards:_hg_qguards'
 }
 
 _hg_cmd_qseries() {
   _arguments -s -w : $_hg_global_opts $_hg_qseries_opts \
   '(--missing -m)'{-m,--missing}'[print patches not in series]'
 }
 
 _hg_cmd_qunapplied() {
   _arguments -s -w : $_hg_global_opts $_hg_qseries_opts
 }
 
 _hg_cmd_qtop() {
   _arguments -s -w : $_hg_global_opts $_hg_qseries_opts
 }
 
 _hg_cmd_strip() {
   _arguments -s -w : $_hg_global_opts \
   '(--force -f)'{-f,--force}'[force multi-head removal]' \
   '(--backup -b)'{-b,--backup}'[bundle unrelated changesets]' \
   '(--nobackup -n)'{-n,--nobackup}'[no backups]' \
   ':revision:_hg_labels'
 }
 
 # Patchbomb
 _hg_cmd_email() {
   _arguments -s -w : $_hg_global_opts $_hg_remote_opts \
   '(--git -g)'{-g,--git}'[use git extended diff format]' \
   '--plain[omit hg patch header]' \
   '(--outgoing -o)'{-o,--outgoing}'[send changes not found in the target repository]' \
   '(--bundle -b)'{-b,--bundle}'[send changes not in target as a binary bundle]' \
   '--bundlename[name of the bundle attachment file (default: bundle)]:' \
   '*'{-r+,--rev}'[search in given revision range]:revision:_hg_revrange' \
   '--force[run even when remote repository is unrelated (with -b/--bundle)]' \
   '*--base[a base changeset to specify instead of a destination (with -b/--bundle)]:revision:_hg_labels' \
   '--intro[send an introduction email for a single patch]' \
   '(--inline -i --attach -a)'{-a,--attach}'[send patches as attachments]' \
   '(--attach -a --inline -i)'{-i,--inline}'[send patches as inline attachments]' \
   '*--bcc[email addresses of blind carbon copy recipients]:email:' \
   '*'{-c+,--cc}'[email addresses of copy recipients]:email:' \
   '(--diffstat -d)'{-d,--diffstat}'[add diffstat output to messages]' \
   '--date[use the given date as the sending date]:date:' \
   '--desc[use the given file as the series description]:files:_files' \
   '(--from -f)'{-f,--from}'[email address of sender]:email:' \
   '(--test -n)'{-n,--test}'[print messages that would be sent]' \
   '(--mbox -m)'{-m,--mbox}'[write messages to mbox file instead of sending them]:file:' \
   '*--reply-to[email addresses replies should be sent to]:email:' \
   '(--subject -s)'{-s,--subject}'[subject of first message (intro or single patch)]:subject:' \
   '--in-reply-to[message identifier to reply to]:msgid:' \
   '*--flag[flags to add in subject prefixes]:flag:' \
   '*'{-t,--to}'[email addresses of recipients]:email:' \
   ':revision:_hg_revrange'
 }
 
 _hg "$@"

doc/gendoc.py

@@ -1,167 +1,168
 import os, sys, textwrap
 # import from the live mercurial repo
 sys.path.insert(0, "..")
 # fall back to pure modules if required C extensions are not available
 sys.path.append(os.path.join('..', 'mercurial', 'pure'))
 from mercurial import demandimport; demandimport.enable()
 from mercurial import encoding
 from mercurial.commands import table, globalopts
 from mercurial.i18n import _
 from mercurial.help import helptable
 from mercurial import extensions
+from mercurial import util
 
 def get_desc(docstr):
     if not docstr:
         return "", ""
     # sanitize
     docstr = docstr.strip("\n")
     docstr = docstr.rstrip()
     shortdesc = docstr.splitlines()[0].strip()
 
     i = docstr.find("\n")
     if i != -1:
         desc = docstr[i + 2:]
     else:
         desc = shortdesc
 
     desc = textwrap.dedent(desc)
 
     return (shortdesc, desc)
 
 def get_opts(opts):
     for opt in opts:
         if len(opt) == 5:
             shortopt, longopt, default, desc, optlabel = opt
         else:
             shortopt, longopt, default, desc = opt
         allopts = []
         if shortopt:
             allopts.append("-%s" % shortopt)
         if longopt:
             allopts.append("--%s" % longopt)
         desc += default and _(" (default: %s)") % default or ""
         yield (", ".join(allopts), desc)
 
 def get_cmd(cmd, cmdtable):
     d = {}
     attr = cmdtable[cmd]
     cmds = cmd.lstrip("^").split("|")
 
     d['cmd'] = cmds[0]
     d['aliases'] = cmd.split("|")[1:]
     d['desc'] = get_desc(attr[0].__doc__)
     d['opts'] = list(get_opts(attr[1]))
 
     s = 'hg ' + cmds[0]
     if len(attr) > 2:
         if not attr[2].startswith('hg'):
             s += ' ' + attr[2]
         else:
             s = attr[2]
     d['synopsis'] = s.strip()
 
     return d
 
 def section(ui, s):
     ui.write("%s\n%s\n\n" % (s, "-" * encoding.colwidth(s)))
 
 def subsection(ui, s):
     ui.write("%s\n%s\n\n" % (s, '"' * encoding.colwidth(s)))
 
 def subsubsection(ui, s):
     ui.write("%s\n%s\n\n" % (s, "." * encoding.colwidth(s)))
 
 def subsubsubsection(ui, s):
     ui.write("%s\n%s\n\n" % (s, "#" * encoding.colwidth(s)))
 
 
 def show_doc(ui):
     # print options
     section(ui, _("Options"))
     for optstr, desc in get_opts(globalopts):
         ui.write("%s\n    %s\n\n" % (optstr, desc))
 
     # print cmds
     section(ui, _("Commands"))
     commandprinter(ui, table, subsection)
 
     # print topics
     for names, sec, doc in helptable:
         if names[0] == "config":
             # The config help topic is included in the hgrc.5 man
             # page.
             continue
         for name in names:
             ui.write(".. _%s:\n" % name)
         ui.write("\n")
         section(ui, sec)
-        if hasattr(doc, '__call__'):
+        if util.safehasattr(doc, '__call__'):
             doc = doc()
         ui.write(doc)
         ui.write("\n")
 
     section(ui, _("Extensions"))
     ui.write(_("This section contains help for extensions that are distributed "
                "together with Mercurial. Help for other extensions is available "
                "in the help system."))
     ui.write("\n\n"
              ".. contents::\n"
              "   :class: htmlonly\n"
              "   :local:\n"
              "   :depth: 1\n\n")
 
     for extensionname in sorted(allextensionnames()):
         mod = extensions.load(None, extensionname, None)
         subsection(ui, extensionname)
         ui.write("%s\n\n" % mod.__doc__)
         cmdtable = getattr(mod, 'cmdtable', None)
         if cmdtable:
             subsubsection(ui, _('Commands'))
             commandprinter(ui, cmdtable, subsubsubsection)
 
 def commandprinter(ui, cmdtable, sectionfunc):
     h = {}
     for c, attr in cmdtable.items():
         f = c.split("|")[0]
         f = f.lstrip("^")
         h[f] = c
     cmds = h.keys()
     cmds.sort()
 
     for f in cmds:
         if f.startswith("debug"):
             continue
         d = get_cmd(h[f], cmdtable)
         sectionfunc(ui, d['cmd'])
         # synopsis
         ui.write("::\n\n")
         synopsislines = d['synopsis'].splitlines()
         for line in synopsislines:
             # some commands (such as rebase) have a multi-line
             # synopsis
             ui.write("   %s\n" % line)
         ui.write('\n')
         # description
         ui.write("%s\n\n" % d['desc'][1])
         # options
         opt_output = list(d['opts'])
         if opt_output:
             opts_len = max([len(line[0]) for line in opt_output])
             ui.write(_("Options:\n\n"))
             for optstr, desc in opt_output:
                 if desc:
                     s = "%-*s  %s" % (opts_len, optstr, desc)
                 else:
                     s = optstr
                 ui.write("%s\n" % s)
             ui.write("\n")
         # aliases
         if d['aliases']:
             ui.write(_("    aliases: %s\n\n") % " ".join(d['aliases']))
 
 
 def allextensionnames():
     return extensions.enabled().keys() + extensions.disabled().keys()
 
 if __name__ == "__main__":
     show_doc(sys.stdout)

hgext/acl.py

@@ -1,250 +1,252
 # acl.py - changeset access control for mercurial
 #
 # Copyright 2006 Vadim Gelfer <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''hooks for controlling repository access
 
 This hook makes it possible to allow or deny write access to given
 branches and paths of a repository when receiving incoming changesets
 via pretxnchangegroup and pretxncommit.
 
 The authorization is matched based on the local user name on the
 system where the hook runs, and not the committer of the original
 changeset (since the latter is merely informative).
 
 The acl hook is best used along with a restricted shell like hgsh,
 preventing authenticating users from doing anything other than pushing
 or pulling. The hook is not safe to use if users have interactive
 shell access, as they can then disable the hook. Nor is it safe if
 remote users share an account, because then there is no way to
 distinguish them.
 
 The order in which access checks are performed is:
 
 1) Deny  list for branches (section ``acl.deny.branches``)
 2) Allow list for branches (section ``acl.allow.branches``)
 3) Deny  list for paths    (section ``acl.deny``)
 4) Allow list for paths    (section ``acl.allow``)
 
 The allow and deny sections take key-value pairs.
 
 Branch-based Access Control
 ...........................
 
 Use the ``acl.deny.branches`` and ``acl.allow.branches`` sections to
 have branch-based access control. Keys in these sections can be
 either:
 
 - a branch name, or
 - an asterisk, to match any branch;
 
 The corresponding values can be either:
 
 - a comma-separated list containing users and groups, or
 - an asterisk, to match anyone;
 
 Path-based Access Control
 .........................
 
 Use the ``acl.deny`` and ``acl.allow`` sections to have path-based
 access control. Keys in these sections accept a subtree pattern (with
 a glob syntax by default). The corresponding values follow the same
 syntax as the other sections above.
 
 Groups
 ......
 
 Group names must be prefixed with an ``@`` symbol. Specifying a group
 name has the same effect as specifying all the users in that group.
 
 You can define group members in the ``acl.groups`` section.
 If a group name is not defined there, and Mercurial is running under
 a Unix-like system, the list of users will be taken from the OS.
 Otherwise, an exception will be raised.
 
 Example Configuration
 .....................
 
 ::
 
   [hooks]
 
   # Use this if you want to check access restrictions at commit time
   pretxncommit.acl = python:hgext.acl.hook
 
   # Use this if you want to check access restrictions for pull, push,
   # bundle and serve.
   pretxnchangegroup.acl = python:hgext.acl.hook
 
   [acl]
   # Allow or deny access for incoming changes only if their source is
   # listed here, let them pass otherwise. Source is "serve" for all
   # remote access (http or ssh), "push", "pull" or "bundle" when the
   # related commands are run locally.
   # Default: serve
   sources = serve
 
   [acl.deny.branches]
 
   # Everyone is denied to the frozen branch:
   frozen-branch = *
 
   # A bad user is denied on all branches:
   * = bad-user
 
   [acl.allow.branches]
 
   # A few users are allowed on branch-a:
   branch-a = user-1, user-2, user-3
 
   # Only one user is allowed on branch-b:
   branch-b = user-1
 
   # The super user is allowed on any branch:
   * = super-user
 
   # Everyone is allowed on branch-for-tests:
   branch-for-tests = *
 
   [acl.deny]
   # This list is checked first. If a match is found, acl.allow is not
   # checked. All users are granted access if acl.deny is not present.
   # Format for both lists: glob pattern = user, ..., @group, ...
 
   # To match everyone, use an asterisk for the user:
   # my/glob/pattern = *
 
   # user6 will not have write access to any file:
   ** = user6
 
   # Group "hg-denied" will not have write access to any file:
   ** = @hg-denied
 
   # Nobody will be able to change "DONT-TOUCH-THIS.txt", despite
   # everyone being able to change all other files. See below.
   src/main/resources/DONT-TOUCH-THIS.txt = *
 
   [acl.allow]
   # if acl.allow is not present, all users are allowed by default
   # empty acl.allow = no users allowed
 
   # User "doc_writer" has write access to any file under the "docs"
   # folder:
   docs/** = doc_writer
 
   # User "jack" and group "designers" have write access to any file
   # under the "images" folder:
   images/** = jack, @designers
 
   # Everyone (except for "user6" - see acl.deny above) will have write
   # access to any file under the "resources" folder (except for 1
   # file. See acl.deny):
   src/main/resources/** = *
 
   .hgtags = release_engineer
 
 '''
 
 from mercurial.i18n import _
 from mercurial import util, match
 import getpass, urllib
 
 def _getusers(ui, group):
 
     # First, try to use group definition from section [acl.groups]
     hgrcusers = ui.configlist('acl.groups', group)
     if hgrcusers:
         return hgrcusers
 
     ui.debug('acl: "%s" not defined in [acl.groups]\n' % group)
     # If no users found in group definition, get users from OS-level group
     try:
         return util.groupmembers(group)
     except KeyError:
         raise util.Abort(_("group '%s' is undefined") % group)
 
 def _usermatch(ui, user, usersorgroups):
 
     if usersorgroups == '*':
         return True
 
     for ug in usersorgroups.replace(',', ' ').split():
         if user == ug or ug.find('@') == 0 and user in _getusers(ui, ug[1:]):
             return True
 
     return False
 
 def buildmatch(ui, repo, user, key):
     '''return tuple of (match function, list enabled).'''
     if not ui.has_section(key):
         ui.debug('acl: %s not enabled\n' % key)
         return None
 
     pats = [pat for pat, users in ui.configitems(key)
             if _usermatch(ui, user, users)]
     ui.debug('acl: %s enabled, %d entries for user %s\n' %
              (key, len(pats), user))
 
     if not repo:
         if pats:
             return lambda b: '*' in pats or b in pats
         return lambda b: False
 
     if pats:
         return match.match(repo.root, '', pats)
     return match.exact(repo.root, '', [])
 
 
 def hook(ui, repo, hooktype, node=None, source=None, **kwargs):
     if hooktype not in ['pretxnchangegroup', 'pretxncommit']:
         raise util.Abort(_('config error - hook type "%s" cannot stop '
                            'incoming changesets nor commits') % hooktype)
     if (hooktype == 'pretxnchangegroup' and
         source not in ui.config('acl', 'sources', 'serve').split()):
         ui.debug('acl: changes have source "%s" - skipping\n' % source)
         return
 
     user = None
     if source == 'serve' and 'url' in kwargs:
         url = kwargs['url'].split(':')
         if url[0] == 'remote' and url[1].startswith('http'):
             user = urllib.unquote(url[3])
 
     if user is None:
         user = getpass.getuser()
 
+    ui.debug('acl: checking access for user "%s"\n' % user)
+
     cfg = ui.config('acl', 'config')
     if cfg:
         ui.readconfig(cfg, sections = ['acl.groups', 'acl.allow.branches',
         'acl.deny.branches', 'acl.allow', 'acl.deny'])
 
     allowbranches = buildmatch(ui, None, user, 'acl.allow.branches')
     denybranches = buildmatch(ui, None, user, 'acl.deny.branches')
     allow = buildmatch(ui, repo, user, 'acl.allow')
     deny = buildmatch(ui, repo, user, 'acl.deny')
 
     for rev in xrange(repo[node], len(repo)):
         ctx = repo[rev]
         branch = ctx.branch()
         if denybranches and denybranches(branch):
             raise util.Abort(_('acl: user "%s" denied on branch "%s"'
                                ' (changeset "%s")')
                                % (user, branch, ctx))
         if allowbranches and not allowbranches(branch):
             raise util.Abort(_('acl: user "%s" not allowed on branch "%s"'
                                ' (changeset "%s")')
                                % (user, branch, ctx))
         ui.debug('acl: branch access granted: "%s" on branch "%s"\n'
         % (ctx, branch))
 
         for f in ctx.files():
             if deny and deny(f):
-                ui.debug('acl: user %s denied on %s\n' % (user, f))
-                raise util.Abort(_('acl: access denied for changeset %s') % ctx)
+                raise util.Abort(_('acl: user "%s" denied on "%s"'
+                ' (changeset "%s")') % (user, f, ctx))
             if allow and not allow(f):
-                ui.debug('acl: user %s not allowed on %s\n' % (user, f))
-                raise util.Abort(_('acl: access denied for changeset %s') % ctx)
-        ui.debug('acl: allowing changeset %s\n' % ctx)
+                raise util.Abort(_('acl: user "%s" not allowed on "%s"'
+                ' (changeset "%s")') % (user, f, ctx))
+        ui.debug('acl: path access granted: "%s"\n' % ctx)

hgext/color.py

@@ -1,507 +1,512
 # color.py color output for the status and qseries commands
 #
 # Copyright (C) 2007 Kevin Christen <[e-mail]>
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
 # Free Software Foundation; either version 2 of the License, or (at your
 # option) any later version.
 #
 # This program is distributed in the hope that it will be useful, but
 # WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
 # Public License for more details.
 #
 # You should have received a copy of the GNU General Public License along
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 '''colorize output from some commands
 
 This extension modifies the status and resolve commands to add color
 to their output to reflect file status, the qseries command to add
 color to reflect patch status (applied, unapplied, missing), and to
 diff-related commands to highlight additions, removals, diff headers,
 and trailing whitespace.
 
 Other effects in addition to color, like bold and underlined text, are
 also available. By default, the terminfo database is used to find the
 terminal codes used to change color and effect.  If terminfo is not
 available, then effects are rendered with the ECMA-48 SGR control
 function (aka ANSI escape codes).
 
 Default effects may be overridden from your configuration file::
 
   [color]
   status.modified = blue bold underline red_background
   status.added = green bold
   status.removed = red bold blue_background
   status.deleted = cyan bold underline
   status.unknown = magenta bold underline
   status.ignored = black bold
 
   # 'none' turns off all effects
   status.clean = none
   status.copied = none
 
   qseries.applied = blue bold underline
   qseries.unapplied = black bold
   qseries.missing = red bold
 
   diff.diffline = bold
   diff.extended = cyan bold
   diff.file_a = red bold
   diff.file_b = green bold
   diff.hunk = magenta
   diff.deleted = red
   diff.inserted = green
   diff.changed = white
   diff.trailingwhitespace = bold red_background
 
   resolve.unresolved = red bold
   resolve.resolved = green bold
 
   bookmarks.current = green
 
   branches.active = none
   branches.closed = black bold
   branches.current = green
   branches.inactive = none
 
+  tags.normal = green
+  tags.local = black bold
+
 The available effects in terminfo mode are 'blink', 'bold', 'dim',
 'inverse', 'invisible', 'italic', 'standout', and 'underline'; in
 ECMA-48 mode, the options are 'bold', 'inverse', 'italic', and
 'underline'.  How each is rendered depends on the terminal emulator.
 Some may not be available for a given terminal type, and will be
 silently ignored.
 
 Note that on some systems, terminfo mode may cause problems when using
 color with the pager extension and less -R. less with the -R option
 will only display ECMA-48 color codes, and terminfo mode may sometimes
 emit codes that less doesn't understand. You can work around this by
 either using ansi mode (or auto mode), or by using less -r (which will
 pass through all terminal control codes, not just color control
 codes).
 
 Because there are only eight standard colors, this module allows you
 to define color names for other color slots which might be available
 for your terminal type, assuming terminfo mode.  For instance::
 
   color.brightblue = 12
   color.pink = 207
   color.orange = 202
 
 to set 'brightblue' to color slot 12 (useful for 16 color terminals
 that have brighter colors defined in the upper eight) and, 'pink' and
 'orange' to colors in 256-color xterm's default color cube.  These
 defined colors may then be used as any of the pre-defined eight,
 including appending '_background' to set the background to that color.
 
 By default, the color extension will use ANSI mode (or win32 mode on
 Windows) if it detects a terminal. To override auto mode (to enable
 terminfo mode, for example), set the following configuration option::
 
   [color]
   mode = terminfo
 
 Any value other than 'ansi', 'win32', 'terminfo', or 'auto' will
 disable color.
 '''
 
 import os
 
 from mercurial import commands, dispatch, extensions, ui as uimod, util
 from mercurial.i18n import _
 
 # start and stop parameters for effects
 _effects = {'none': 0, 'black': 30, 'red': 31, 'green': 32, 'yellow': 33,
             'blue': 34, 'magenta': 35, 'cyan': 36, 'white': 37, 'bold': 1,
             'italic': 3, 'underline': 4, 'inverse': 7,
             'black_background': 40, 'red_background': 41,
             'green_background': 42, 'yellow_background': 43,
             'blue_background': 44, 'purple_background': 45,
             'cyan_background': 46, 'white_background': 47}
 
 def _terminfosetup(ui, mode):
     '''Initialize terminfo data and the terminal if we're in terminfo mode.'''
 
     global _terminfo_params
     # If we failed to load curses, we go ahead and return.
     if not _terminfo_params:
         return
     # Otherwise, see what the config file says.
     if mode not in ('auto', 'terminfo'):
         return
 
     _terminfo_params.update((key[6:], (False, int(val)))
         for key, val in ui.configitems('color')
         if key.startswith('color.'))
 
     try:
         curses.setupterm()
     except curses.error, e:
         _terminfo_params = {}
         return
 
     for key, (b, e) in _terminfo_params.items():
         if not b:
             continue
         if not curses.tigetstr(e):
             # Most terminals don't support dim, invis, etc, so don't be
             # noisy and use ui.debug().
             ui.debug("no terminfo entry for %s\n" % e)
             del _terminfo_params[key]
     if not curses.tigetstr('setaf') or not curses.tigetstr('setab'):
         # Only warn about missing terminfo entries if we explicitly asked for
         # terminfo mode.
         if mode == "terminfo":
             ui.warn(_("no terminfo entry for setab/setaf: reverting to "
               "ECMA-48 color\n"))
         _terminfo_params = {}
 
 def _modesetup(ui, opts):
     global _terminfo_params
 
     coloropt = opts['color']
     auto = coloropt == 'auto'
     always = not auto and util.parsebool(coloropt)
     if not always and not auto:
         return None
 
     formatted = always or (os.environ.get('TERM') != 'dumb' and ui.formatted())
 
     mode = ui.config('color', 'mode', 'auto')
     realmode = mode
     if mode == 'auto':
         if os.name == 'nt' and 'TERM' not in os.environ:
             # looks line a cmd.exe console, use win32 API or nothing
             realmode = 'win32'
         else:
             realmode = 'ansi'
 
     if realmode == 'win32':
         _terminfo_params = {}
         if not w32effects:
             if mode == 'win32':
                 # only warn if color.mode is explicitly set to win32
                 ui.warn(_('warning: failed to set color mode to %s\n') % mode)
             return None
         _effects.update(w32effects)
     elif realmode == 'ansi':
         _terminfo_params = {}
     elif realmode == 'terminfo':
         _terminfosetup(ui, mode)
         if not _terminfo_params:
             if mode == 'terminfo':
                 ## FIXME Shouldn't we return None in this case too?
                 # only warn if color.mode is explicitly set to win32
                 ui.warn(_('warning: failed to set color mode to %s\n') % mode)
             realmode = 'ansi'
     else:
         return None
 
     if always or (auto and formatted):
         return realmode
     return None
 
 try:
     import curses
     # Mapping from effect name to terminfo attribute name or color number.
     # This will also force-load the curses module.
     _terminfo_params = {'none': (True, 'sgr0'),
                         'standout': (True, 'smso'),
                         'underline': (True, 'smul'),
                         'reverse': (True, 'rev'),
                         'inverse': (True, 'rev'),
                         'blink': (True, 'blink'),
                         'dim': (True, 'dim'),
                         'bold': (True, 'bold'),
                         'invisible': (True, 'invis'),
                         'italic': (True, 'sitm'),
                         'black': (False, curses.COLOR_BLACK),
                         'red': (False, curses.COLOR_RED),
                         'green': (False, curses.COLOR_GREEN),
                         'yellow': (False, curses.COLOR_YELLOW),
                         'blue': (False, curses.COLOR_BLUE),
                         'magenta': (False, curses.COLOR_MAGENTA),
                         'cyan': (False, curses.COLOR_CYAN),
                         'white': (False, curses.COLOR_WHITE)}
 except ImportError:
     _terminfo_params = False
 
 _styles = {'grep.match': 'red bold',
            'bookmarks.current': 'green',
            'branches.active': 'none',
            'branches.closed': 'black bold',
            'branches.current': 'green',
            'branches.inactive': 'none',
            'diff.changed': 'white',
            'diff.deleted': 'red',
            'diff.diffline': 'bold',
            'diff.extended': 'cyan bold',
            'diff.file_a': 'red bold',
            'diff.file_b': 'green bold',
            'diff.hunk': 'magenta',
            'diff.inserted': 'green',
            'diff.trailingwhitespace': 'bold red_background',
            'diffstat.deleted': 'red',
            'diffstat.inserted': 'green',
            'ui.prompt': 'yellow',
            'log.changeset': 'yellow',
            'resolve.resolved': 'green bold',
            'resolve.unresolved': 'red bold',
            'status.added': 'green bold',
            'status.clean': 'none',
            'status.copied': 'none',
            'status.deleted': 'cyan bold underline',
            'status.ignored': 'black bold',
            'status.modified': 'blue bold',
            'status.removed': 'red bold',
-           'status.unknown': 'magenta bold underline'}
+           'status.unknown': 'magenta bold underline',
+           'tags.normal': 'green',
+           'tags.local': 'black bold'}
 
 
 def _effect_str(effect):
     '''Helper function for render_effects().'''
 
     bg = False
     if effect.endswith('_background'):
         bg = True
         effect = effect[:-11]
     attr, val = _terminfo_params[effect]
     if attr:
         return curses.tigetstr(val)
     elif bg:
         return curses.tparm(curses.tigetstr('setab'), val)
     else:
         return curses.tparm(curses.tigetstr('setaf'), val)
 
 def render_effects(text, effects):
     'Wrap text in commands to turn on each effect.'
     if not text:
         return text
     if not _terminfo_params:
         start = [str(_effects[e]) for e in ['none'] + effects.split()]
         start = '\033[' + ';'.join(start) + 'm'
         stop = '\033[' + str(_effects['none']) + 'm'
     else:
         start = ''.join(_effect_str(effect)
                         for effect in ['none'] + effects.split())
         stop = _effect_str('none')
     return ''.join([start, text, stop])
 
 def extstyles():
     for name, ext in extensions.extensions():
         _styles.update(getattr(ext, 'colortable', {}))
 
 def configstyles(ui):
     for status, cfgeffects in ui.configitems('color'):
         if '.' not in status or status.startswith('color.'):
             continue
         cfgeffects = ui.configlist('color', status)
         if cfgeffects:
             good = []
             for e in cfgeffects:
                 if not _terminfo_params and e in _effects:
                     good.append(e)
                 elif e in _terminfo_params or e[:-11] in _terminfo_params:
                     good.append(e)
                 else:
                     ui.warn(_("ignoring unknown color/effect %r "
                               "(configured in color.%s)\n")
                             % (e, status))
             _styles[status] = ' '.join(good)
 
 class colorui(uimod.ui):
     def popbuffer(self, labeled=False):
         if labeled:
             return ''.join(self.label(a, label) for a, label
                            in self._buffers.pop())
         return ''.join(a for a, label in self._buffers.pop())
 
     _colormode = 'ansi'
     def write(self, *args, **opts):
         label = opts.get('label', '')
         if self._buffers:
             self._buffers[-1].extend([(str(a), label) for a in args])
         elif self._colormode == 'win32':
             for a in args:
                 win32print(a, super(colorui, self).write, **opts)
         else:
             return super(colorui, self).write(
                 *[self.label(str(a), label) for a in args], **opts)
 
     def write_err(self, *args, **opts):
         label = opts.get('label', '')
         if self._colormode == 'win32':
             for a in args:
                 win32print(a, super(colorui, self).write_err, **opts)
         else:
             return super(colorui, self).write_err(
                 *[self.label(str(a), label) for a in args], **opts)
 
     def label(self, msg, label):
         effects = []
         for l in label.split():
             s = _styles.get(l, '')
             if s:
                 effects.append(s)
         effects = ' '.join(effects)
         if effects:
             return '\n'.join([render_effects(s, effects)
                               for s in msg.split('\n')])
         return msg
 
 
 def uisetup(ui):
     global _terminfo_params
     if ui.plain():
         return
     def colorcmd(orig, ui_, opts, cmd, cmdfunc):
         mode = _modesetup(ui_, opts)
         if mode:
             colorui._colormode = mode
             if not issubclass(ui_.__class__, colorui):
                 colorui.__bases__ = (ui_.__class__,)
                 ui_.__class__ = colorui
             extstyles()
             configstyles(ui_)
         return orig(ui_, opts, cmd, cmdfunc)
     extensions.wrapfunction(dispatch, '_runcommand', colorcmd)
 
 def extsetup(ui):
     commands.globalopts.append(
         ('', 'color', 'auto',
          # i18n: 'always', 'auto', and 'never' are keywords and should
          # not be translated
          _("when to colorize (boolean, always, auto, or never)"),
          _('TYPE')))
 
 if os.name != 'nt':
     w32effects = None
 else:
     import re, ctypes
 
     _kernel32 = ctypes.windll.kernel32
 
     _WORD = ctypes.c_ushort
 
     _INVALID_HANDLE_VALUE = -1
 
     class _COORD(ctypes.Structure):
         _fields_ = [('X', ctypes.c_short),
                     ('Y', ctypes.c_short)]
 
     class _SMALL_RECT(ctypes.Structure):
         _fields_ = [('Left', ctypes.c_short),
                     ('Top', ctypes.c_short),
                     ('Right', ctypes.c_short),
                     ('Bottom', ctypes.c_short)]
 
     class _CONSOLE_SCREEN_BUFFER_INFO(ctypes.Structure):
         _fields_ = [('dwSize', _COORD),
                     ('dwCursorPosition', _COORD),
                     ('wAttributes', _WORD),
                     ('srWindow', _SMALL_RECT),
                     ('dwMaximumWindowSize', _COORD)]
 
     _STD_OUTPUT_HANDLE = 0xfffffff5L # (DWORD)-11
     _STD_ERROR_HANDLE = 0xfffffff4L  # (DWORD)-12
 
     _FOREGROUND_BLUE = 0x0001
     _FOREGROUND_GREEN = 0x0002
     _FOREGROUND_RED = 0x0004
     _FOREGROUND_INTENSITY = 0x0008
 
     _BACKGROUND_BLUE = 0x0010
     _BACKGROUND_GREEN = 0x0020
     _BACKGROUND_RED = 0x0040
     _BACKGROUND_INTENSITY = 0x0080
 
     _COMMON_LVB_REVERSE_VIDEO = 0x4000
     _COMMON_LVB_UNDERSCORE = 0x8000
 
     # http://msdn.microsoft.com/en-us/library/ms682088%28VS.85%29.aspx
     w32effects = {
         'none': -1,
         'black': 0,
         'red': _FOREGROUND_RED,
         'green': _FOREGROUND_GREEN,
         'yellow': _FOREGROUND_RED | _FOREGROUND_GREEN,
         'blue': _FOREGROUND_BLUE,
         'magenta': _FOREGROUND_BLUE | _FOREGROUND_RED,
         'cyan': _FOREGROUND_BLUE | _FOREGROUND_GREEN,
         'white': _FOREGROUND_RED | _FOREGROUND_GREEN | _FOREGROUND_BLUE,
         'bold': _FOREGROUND_INTENSITY,
         'black_background': 0x100,                  # unused value > 0x0f
         'red_background': _BACKGROUND_RED,
         'green_background': _BACKGROUND_GREEN,
         'yellow_background': _BACKGROUND_RED | _BACKGROUND_GREEN,
         'blue_background': _BACKGROUND_BLUE,
         'purple_background': _BACKGROUND_BLUE | _BACKGROUND_RED,
         'cyan_background': _BACKGROUND_BLUE | _BACKGROUND_GREEN,
         'white_background': (_BACKGROUND_RED | _BACKGROUND_GREEN |
                              _BACKGROUND_BLUE),
         'bold_background': _BACKGROUND_INTENSITY,
         'underline': _COMMON_LVB_UNDERSCORE,  # double-byte charsets only
         'inverse': _COMMON_LVB_REVERSE_VIDEO, # double-byte charsets only
     }
 
     passthrough = set([_FOREGROUND_INTENSITY,
                        _BACKGROUND_INTENSITY,
                        _COMMON_LVB_UNDERSCORE,
                        _COMMON_LVB_REVERSE_VIDEO])
 
     stdout = _kernel32.GetStdHandle(
                   _STD_OUTPUT_HANDLE)  # don't close the handle returned
     if stdout is None or stdout == _INVALID_HANDLE_VALUE:
         w32effects = None
     else:
         csbi = _CONSOLE_SCREEN_BUFFER_INFO()
         if not _kernel32.GetConsoleScreenBufferInfo(
                     stdout, ctypes.byref(csbi)):
             # stdout may not support GetConsoleScreenBufferInfo()
             # when called from subprocess or redirected
             w32effects = None
         else:
             origattr = csbi.wAttributes
             ansire = re.compile('\033\[([^m]*)m([^\033]*)(.*)',
                                 re.MULTILINE | re.DOTALL)
 
     def win32print(text, orig, **opts):
         label = opts.get('label', '')
         attr = origattr
 
         def mapcolor(val, attr):
             if val == -1:
                 return origattr
             elif val in passthrough:
                 return attr | val
             elif val > 0x0f:
                 return (val & 0x70) | (attr & 0x8f)
             else:
                 return (val & 0x07) | (attr & 0xf8)
 
         # determine console attributes based on labels
         for l in label.split():
             style = _styles.get(l, '')
             for effect in style.split():
                 attr = mapcolor(w32effects[effect], attr)
 
         # hack to ensure regexp finds data
         if not text.startswith('\033['):
             text = '\033[m' + text
 
         # Look for ANSI-like codes embedded in text
         m = re.match(ansire, text)
 
         try:
             while m:
                 for sattr in m.group(1).split(';'):
                     if sattr:
                         attr = mapcolor(int(sattr), attr)
                 _kernel32.SetConsoleTextAttribute(stdout, attr)
                 orig(m.group(2), **opts)
                 m = re.match(ansire, m.group(3))
         finally:
             # Explicity reset original attributes
             _kernel32.SetConsoleTextAttribute(stdout, origattr)

hgext/convert/cvsps.py

@@ -1,847 +1,854
 # Mercurial built-in replacement for cvsps.
 #
 # Copyright 2008, Frank Kingswood <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 import os
 import re
 import cPickle as pickle
 from mercurial import util
 from mercurial.i18n import _
 from mercurial import hook
+from mercurial import util
 
 class logentry(object):
     '''Class logentry has the following attributes:
         .author    - author name as CVS knows it
         .branch    - name of branch this revision is on
         .branches  - revision tuple of branches starting at this revision
         .comment   - commit message
         .date      - the commit date as a (time, tz) tuple
         .dead      - true if file revision is dead
         .file      - Name of file
         .lines     - a tuple (+lines, -lines) or None
         .parent    - Previous revision of this entry
         .rcs       - name of file as returned from CVS
         .revision  - revision number as tuple
         .tags      - list of tags on the file
         .synthetic - is this a synthetic "file ... added on ..." revision?
         .mergepoint- the branch that has been merged from
                      (if present in rlog output)
         .branchpoints- the branches that start at the current entry
     '''
     def __init__(self, **entries):
         self.synthetic = False
         self.__dict__.update(entries)
 
     def __repr__(self):
         return "<%s at 0x%x: %s %s>" % (self.__class__.__name__,
                                         id(self),
                                         self.file,
                                         ".".join(map(str, self.revision)))
 
 class logerror(Exception):
     pass
 
 def getrepopath(cvspath):
     """Return the repository path from a CVS path.
 
     >>> getrepopath('/foo/bar')
     '/foo/bar'
     >>> getrepopath('c:/foo/bar')
     'c:/foo/bar'
     >>> getrepopath(':pserver:10/foo/bar')
     '/foo/bar'
     >>> getrepopath(':pserver:10c:/foo/bar')
     '/foo/bar'
     >>> getrepopath(':pserver:/foo/bar')
     '/foo/bar'
     >>> getrepopath(':pserver:c:/foo/bar')
     'c:/foo/bar'
     >>> getrepopath(':pserver:[e-mail]:/foo/bar')
     '/foo/bar'
     >>> getrepopath(':pserver:[e-mail]:c:/foo/bar')
     'c:/foo/bar'
     """
     # According to CVS manual, CVS paths are expressed like:
     # [:method:][[user][:password]@]hostname[:[port]]/path/to/repository
     #
     # Unfortunately, Windows absolute paths start with a drive letter
     # like 'c:' making it harder to parse. Here we assume that drive
     # letters are only one character long and any CVS component before
     # the repository path is at least 2 characters long, and use this
     # to disambiguate.
     parts = cvspath.split(':')
     if len(parts) == 1:
         return parts[0]
     # Here there is an ambiguous case if we have a port number
     # immediately followed by a Windows driver letter. We assume this
     # never happens and decide it must be CVS path component,
     # therefore ignoring it.
     if len(parts[-2]) > 1:
         return parts[-1].lstrip('0123456789')
     return parts[-2] + ':' + parts[-1]
 
 def createlog(ui, directory=None, root="", rlog=True, cache=None):
     '''Collect the CVS rlog'''
 
     # Because we store many duplicate commit log messages, reusing strings
     # saves a lot of memory and pickle storage space.
     _scache = {}
     def scache(s):
         "return a shared version of a string"
         return _scache.setdefault(s, s)
 
     ui.status(_('collecting CVS rlog\n'))
 
     log = []      # list of logentry objects containing the CVS state
 
     # patterns to match in CVS (r)log output, by state of use
     re_00 = re.compile('RCS file: (.+)$')
     re_01 = re.compile('cvs \\[r?log aborted\\]: (.+)$')
     re_02 = re.compile('cvs (r?log|server): (.+)\n$')
     re_03 = re.compile("(Cannot access.+CVSROOT)|"
                        "(can't create temporary directory.+)$")
     re_10 = re.compile('Working file: (.+)$')
     re_20 = re.compile('symbolic names:')
     re_30 = re.compile('\t(.+): ([\\d.]+)$')
     re_31 = re.compile('----------------------------$')
     re_32 = re.compile('======================================='
                        '======================================$')
     re_50 = re.compile('revision ([\\d.]+)(\s+locked by:\s+.+;)?$')
     re_60 = re.compile(r'date:\s+(.+);\s+author:\s+(.+);\s+state:\s+(.+?);'
                        r'(\s+lines:\s+(\+\d+)?\s+(-\d+)?;)?'
                        r'(.*mergepoint:\s+([^;]+);)?')
     re_70 = re.compile('branches: (.+);$')
 
     file_added_re = re.compile(r'file [^/]+ was (initially )?added on branch')
 
     prefix = ''   # leading path to strip of what we get from CVS
 
     if directory is None:
         # Current working directory
 
         # Get the real directory in the repository
         try:
             prefix = open(os.path.join('CVS','Repository')).read().strip()
             directory = prefix
             if prefix == ".":
                 prefix = ""
         except IOError:
             raise logerror(_('not a CVS sandbox'))
 
         if prefix and not prefix.endswith(os.sep):
             prefix += os.sep
 
         # Use the Root file in the sandbox, if it exists
         try:
             root = open(os.path.join('CVS','Root')).read().strip()
         except IOError:
             pass
 
     if not root:
         root = os.environ.get('CVSROOT', '')
 
     # read log cache if one exists
     oldlog = []
     date = None
 
     if cache:
         cachedir = os.path.expanduser('~/.hg.cvsps')
         if not os.path.exists(cachedir):
             os.mkdir(cachedir)
 
         # The cvsps cache pickle needs a uniquified name, based on the
         # repository location. The address may have all sort of nasties
         # in it, slashes, colons and such. So here we take just the
         # alphanumerics, concatenated in a way that does not mix up the
         # various components, so that
         #    :pserver:user@server:/path
         # and
         #    /pserver/user/server/path
         # are mapped to different cache file names.
         cachefile = root.split(":") + [directory, "cache"]
         cachefile = ['-'.join(re.findall(r'\w+', s)) for s in cachefile if s]
         cachefile = os.path.join(cachedir,
                                  '.'.join([s for s in cachefile if s]))
 
     if cache == 'update':
         try:
             ui.note(_('reading cvs log cache %s\n') % cachefile)
             oldlog = pickle.load(open(cachefile))
             ui.note(_('cache has %d log entries\n') % len(oldlog))
         except Exception, e:
             ui.note(_('error reading cache: %r\n') % e)
 
         if oldlog:
             date = oldlog[-1].date    # last commit date as a (time,tz) tuple
             date = util.datestr(date, '%Y/%m/%d %H:%M:%S %1%2')
 
     # build the CVS commandline
     cmd = ['cvs', '-q']
     if root:
         cmd.append('-d%s' % root)
         p = util.normpath(getrepopath(root))
         if not p.endswith('/'):
             p += '/'
         if prefix:
             # looks like normpath replaces "" by "."
             prefix = p + util.normpath(prefix)
         else:
             prefix = p
     cmd.append(['log', 'rlog'][rlog])
     if date:
         # no space between option and date string
         cmd.append('-d>%s' % date)
     cmd.append(directory)
 
     # state machine begins here
     tags = {}     # dictionary of revisions on current file with their tags
     branchmap = {} # mapping between branch names and revision numbers
     state = 0
     store = False # set when a new record can be appended
 
     cmd = [util.shellquote(arg) for arg in cmd]
     ui.note(_("running %s\n") % (' '.join(cmd)))
     ui.debug("prefix=%r directory=%r root=%r\n" % (prefix, directory, root))
 
     pfp = util.popen(' '.join(cmd))
     peek = pfp.readline()
     while True:
         line = peek
         if line == '':
             break
         peek = pfp.readline()
         if line.endswith('\n'):
             line = line[:-1]
         #ui.debug('state=%d line=%r\n' % (state, line))
 
         if state == 0:
             # initial state, consume input until we see 'RCS file'
             match = re_00.match(line)
             if match:
                 rcs = match.group(1)
                 tags = {}
                 if rlog:
                     filename = util.normpath(rcs[:-2])
                     if filename.startswith(prefix):
                         filename = filename[len(prefix):]
                     if filename.startswith('/'):
                         filename = filename[1:]
                     if filename.startswith('Attic/'):
                         filename = filename[6:]
                     else:
                         filename = filename.replace('/Attic/', '/')
                     state = 2
                     continue
                 state = 1
                 continue
             match = re_01.match(line)
             if match:
                 raise logerror(match.group(1))
             match = re_02.match(line)
             if match:
                 raise logerror(match.group(2))
             if re_03.match(line):
                 raise logerror(line)
 
         elif state == 1:
             # expect 'Working file' (only when using log instead of rlog)
             match = re_10.match(line)
             assert match, _('RCS file must be followed by working file')
             filename = util.normpath(match.group(1))
             state = 2
 
         elif state == 2:
             # expect 'symbolic names'
             if re_20.match(line):
                 branchmap = {}
                 state = 3
 
         elif state == 3:
             # read the symbolic names and store as tags
             match = re_30.match(line)
             if match:
                 rev = [int(x) for x in match.group(2).split('.')]
 
                 # Convert magic branch number to an odd-numbered one
                 revn = len(rev)
                 if revn > 3 and (revn % 2) == 0 and rev[-2] == 0:
                     rev = rev[:-2] + rev[-1:]
                 rev = tuple(rev)
 
                 if rev not in tags:
                     tags[rev] = []
                 tags[rev].append(match.group(1))
                 branchmap[match.group(1)] = match.group(2)
 
             elif re_31.match(line):
                 state = 5
             elif re_32.match(line):
                 state = 0
 
         elif state == 4:
             # expecting '------' separator before first revision
             if re_31.match(line):
                 state = 5
             else:
                 assert not re_32.match(line), _('must have at least '
                                                 'some revisions')
 
         elif state == 5:
             # expecting revision number and possibly (ignored) lock indication
             # we create the logentry here from values stored in states 0 to 4,
             # as this state is re-entered for subsequent revisions of a file.
             match = re_50.match(line)
             assert match, _('expected revision number')
             e = logentry(rcs=scache(rcs), file=scache(filename),
                     revision=tuple([int(x) for x in match.group(1).split('.')]),
                     branches=[], parent=None)
             state = 6
 
         elif state == 6:
             # expecting date, author, state, lines changed
             match = re_60.match(line)
             assert match, _('revision must be followed by date line')
             d = match.group(1)
             if d[2] == '/':
                 # Y2K
                 d = '19' + d
 
             if len(d.split()) != 3:
                 # cvs log dates always in GMT
                 d = d + ' UTC'
             e.date = util.parsedate(d, ['%y/%m/%d %H:%M:%S',
                                         '%Y/%m/%d %H:%M:%S',
                                         '%Y-%m-%d %H:%M:%S'])
             e.author = scache(match.group(2))
             e.dead = match.group(3).lower() == 'dead'
 
             if match.group(5):
                 if match.group(6):
                     e.lines = (int(match.group(5)), int(match.group(6)))
                 else:
                     e.lines = (int(match.group(5)), 0)
             elif match.group(6):
                 e.lines = (0, int(match.group(6)))
             else:
                 e.lines = None
 
             if match.group(7): # cvsnt mergepoint
                 myrev = match.group(8).split('.')
                 if len(myrev) == 2: # head
                     e.mergepoint = 'HEAD'
                 else:
                     myrev = '.'.join(myrev[:-2] + ['0', myrev[-2]])
                     branches = [b for b in branchmap if branchmap[b] == myrev]
                     assert len(branches) == 1, 'unknown branch: %s' % e.mergepoint
                     e.mergepoint = branches[0]
             else:
                 e.mergepoint = None
             e.comment = []
             state = 7
 
         elif state == 7:
             # read the revision numbers of branches that start at this revision
             # or store the commit log message otherwise
             m = re_70.match(line)
             if m:
                 e.branches = [tuple([int(y) for y in x.strip().split('.')])
                                 for x in m.group(1).split(';')]
                 state = 8
             elif re_31.match(line) and re_50.match(peek):
                 state = 5
                 store = True
             elif re_32.match(line):
                 state = 0
                 store = True
             else:
                 e.comment.append(line)
 
         elif state == 8:
             # store commit log message
             if re_31.match(line):
-                state = 5
-                store = True
+                cpeek = peek
+                if cpeek.endswith('\n'):
+                    cpeek = cpeek[:-1]
+                if re_50.match(cpeek):
+                    state = 5
+                    store = True
+                else:
+                    e.comment.append(line)
             elif re_32.match(line):
                 state = 0
                 store = True
             else:
                 e.comment.append(line)
 
         # When a file is added on a branch B1, CVS creates a synthetic
         # dead trunk revision 1.1 so that the branch has a root.
         # Likewise, if you merge such a file to a later branch B2 (one
         # that already existed when the file was added on B1), CVS
         # creates a synthetic dead revision 1.1.x.1 on B2.  Don't drop
         # these revisions now, but mark them synthetic so
         # createchangeset() can take care of them.
         if (store and
               e.dead and
               e.revision[-1] == 1 and      # 1.1 or 1.1.x.1
               len(e.comment) == 1 and
               file_added_re.match(e.comment[0])):
             ui.debug('found synthetic revision in %s: %r\n'
                      % (e.rcs, e.comment[0]))
             e.synthetic = True
 
         if store:
             # clean up the results and save in the log.
             store = False
             e.tags = sorted([scache(x) for x in tags.get(e.revision, [])])
             e.comment = scache('\n'.join(e.comment))
 
             revn = len(e.revision)
             if revn > 3 and (revn % 2) == 0:
                 e.branch = tags.get(e.revision[:-1], [None])[0]
             else:
                 e.branch = None
 
             # find the branches starting from this revision
             branchpoints = set()
             for branch, revision in branchmap.iteritems():
                 revparts = tuple([int(i) for i in revision.split('.')])
                 if len(revparts) < 2: # bad tags
                     continue
                 if revparts[-2] == 0 and revparts[-1] % 2 == 0:
                     # normal branch
                     if revparts[:-2] == e.revision:
                         branchpoints.add(branch)
                 elif revparts == (1, 1, 1): # vendor branch
                     if revparts in e.branches:
                         branchpoints.add(branch)
             e.branchpoints = branchpoints
 
             log.append(e)
 
             if len(log) % 100 == 0:
                 ui.status(util.ellipsis('%d %s' % (len(log), e.file), 80)+'\n')
 
     log.sort(key=lambda x: (x.rcs, x.revision))
 
     # find parent revisions of individual files
     versions = {}
     for e in log:
         branch = e.revision[:-1]
         p = versions.get((e.rcs, branch), None)
         if p is None:
             p = e.revision[:-2]
         e.parent = p
         versions[(e.rcs, branch)] = e.revision
 
     # update the log cache
     if cache:
         if log:
             # join up the old and new logs
             log.sort(key=lambda x: x.date)
 
             if oldlog and oldlog[-1].date >= log[0].date:
                 raise logerror(_('log cache overlaps with new log entries,'
                                  ' re-run without cache.'))
 
             log = oldlog + log
 
             # write the new cachefile
             ui.note(_('writing cvs log cache %s\n') % cachefile)
             pickle.dump(log, open(cachefile, 'w'))
         else:
             log = oldlog
 
     ui.status(_('%d log entries\n') % len(log))
 
     hook.hook(ui, None, "cvslog", True, log=log)
 
     return log
 
 
 class changeset(object):
     '''Class changeset has the following attributes:
         .id        - integer identifying this changeset (list index)
         .author    - author name as CVS knows it
         .branch    - name of branch this changeset is on, or None
         .comment   - commit message
         .date      - the commit date as a (time,tz) tuple
         .entries   - list of logentry objects in this changeset
         .parents   - list of one or two parent changesets
         .tags      - list of tags on this changeset
         .synthetic - from synthetic revision "file ... added on branch ..."
         .mergepoint- the branch that has been merged from
                      (if present in rlog output)
         .branchpoints- the branches that start at the current entry
     '''
     def __init__(self, **entries):
         self.synthetic = False
         self.__dict__.update(entries)
 
     def __repr__(self):
         return "<%s at 0x%x: %s>" % (self.__class__.__name__,
                                      id(self),
                                      getattr(self, 'id', "(no id)"))
 
 def createchangeset(ui, log, fuzz=60, mergefrom=None, mergeto=None):
     '''Convert log into changesets.'''
 
     ui.status(_('creating changesets\n'))
 
     # Merge changesets
 
     log.sort(key=lambda x: (x.comment, x.author, x.branch, x.date))
 
     changesets = []
     files = set()
     c = None
     for i, e in enumerate(log):
 
         # Check if log entry belongs to the current changeset or not.
 
         # Since CVS is file centric, two different file revisions with
         # different branchpoints should be treated as belonging to two
         # different changesets (and the ordering is important and not
         # honoured by cvsps at this point).
         #
         # Consider the following case:
         # foo 1.1 branchpoints: [MYBRANCH]
         # bar 1.1 branchpoints: [MYBRANCH, MYBRANCH2]
         #
         # Here foo is part only of MYBRANCH, but not MYBRANCH2, e.g. a
         # later version of foo may be in MYBRANCH2, so foo should be the
         # first changeset and bar the next and MYBRANCH and MYBRANCH2
         # should both start off of the bar changeset. No provisions are
         # made to ensure that this is, in fact, what happens.
         if not (c and
                   e.comment == c.comment and
                   e.author == c.author and
                   e.branch == c.branch and
-                  (not hasattr(e, 'branchpoints') or
-                    not hasattr (c, 'branchpoints') or
+                  (not util.safehasattr(e, 'branchpoints') or
+                    not util.safehasattr (c, 'branchpoints') or
                     e.branchpoints == c.branchpoints) and
                   ((c.date[0] + c.date[1]) <=
                    (e.date[0] + e.date[1]) <=
                    (c.date[0] + c.date[1]) + fuzz) and
                   e.file not in files):
             c = changeset(comment=e.comment, author=e.author,
                           branch=e.branch, date=e.date, entries=[],
                           mergepoint=getattr(e, 'mergepoint', None),
                           branchpoints=getattr(e, 'branchpoints', set()))
             changesets.append(c)
             files = set()
             if len(changesets) % 100 == 0:
                 t = '%d %s' % (len(changesets), repr(e.comment)[1:-1])
                 ui.status(util.ellipsis(t, 80) + '\n')
 
         c.entries.append(e)
         files.add(e.file)
         c.date = e.date       # changeset date is date of latest commit in it
 
     # Mark synthetic changesets
 
     for c in changesets:
         # Synthetic revisions always get their own changeset, because
         # the log message includes the filename.  E.g. if you add file3
         # and file4 on a branch, you get four log entries and three
         # changesets:
         #   "File file3 was added on branch ..." (synthetic, 1 entry)
         #   "File file4 was added on branch ..." (synthetic, 1 entry)
         #   "Add file3 and file4 to fix ..."     (real, 2 entries)
         # Hence the check for 1 entry here.
         c.synthetic = len(c.entries) == 1 and c.entries[0].synthetic
 
     # Sort files in each changeset
 
     for c in changesets:
         def pathcompare(l, r):
             'Mimic cvsps sorting order'
             l = l.split('/')
             r = r.split('/')
             nl = len(l)
             nr = len(r)
             n = min(nl, nr)
             for i in range(n):
                 if i + 1 == nl and nl < nr:
                     return -1
                 elif i + 1 == nr and nl > nr:
                     return +1
                 elif l[i] < r[i]:
                     return -1
                 elif l[i] > r[i]:
                     return +1
             return 0
         def entitycompare(l, r):
             return pathcompare(l.file, r.file)
 
         c.entries.sort(entitycompare)
 
     # Sort changesets by date
 
     def cscmp(l, r):
         d = sum(l.date) - sum(r.date)
         if d:
             return d
 
         # detect vendor branches and initial commits on a branch
         le = {}
         for e in l.entries:
             le[e.rcs] = e.revision
         re = {}
         for e in r.entries:
             re[e.rcs] = e.revision
 
         d = 0
         for e in l.entries:
             if re.get(e.rcs, None) == e.parent:
                 assert not d
                 d = 1
                 break
 
         for e in r.entries:
             if le.get(e.rcs, None) == e.parent:
                 assert not d
                 d = -1
                 break
 
         return d
 
     changesets.sort(cscmp)
 
     # Collect tags
 
     globaltags = {}
     for c in changesets:
         for e in c.entries:
             for tag in e.tags:
                 # remember which is the latest changeset to have this tag
                 globaltags[tag] = c
 
     for c in changesets:
         tags = set()
         for e in c.entries:
             tags.update(e.tags)
         # remember tags only if this is the latest changeset to have it
         c.tags = sorted(tag for tag in tags if globaltags[tag] is c)
 
     # Find parent changesets, handle {{mergetobranch BRANCHNAME}}
     # by inserting dummy changesets with two parents, and handle
     # {{mergefrombranch BRANCHNAME}} by setting two parents.
 
     if mergeto is None:
         mergeto = r'{{mergetobranch ([-\w]+)}}'
     if mergeto:
         mergeto = re.compile(mergeto)
 
     if mergefrom is None:
         mergefrom = r'{{mergefrombranch ([-\w]+)}}'
     if mergefrom:
         mergefrom = re.compile(mergefrom)
 
     versions = {}    # changeset index where we saw any particular file version
     branches = {}    # changeset index where we saw a branch
     n = len(changesets)
     i = 0
     while i < n:
         c = changesets[i]
 
         for f in c.entries:
             versions[(f.rcs, f.revision)] = i
 
         p = None
         if c.branch in branches:
             p = branches[c.branch]
         else:
             # first changeset on a new branch
             # the parent is a changeset with the branch in its
             # branchpoints such that it is the latest possible
             # commit without any intervening, unrelated commits.
 
             for candidate in xrange(i):
                 if c.branch not in changesets[candidate].branchpoints:
                     if p is not None:
                         break
                     continue
                 p = candidate
 
         c.parents = []
         if p is not None:
             p = changesets[p]
 
             # Ensure no changeset has a synthetic changeset as a parent.
             while p.synthetic:
                 assert len(p.parents) <= 1, \
                        _('synthetic changeset cannot have multiple parents')
                 if p.parents:
                     p = p.parents[0]
                 else:
                     p = None
                     break
 
             if p is not None:
                 c.parents.append(p)
 
         if c.mergepoint:
             if c.mergepoint == 'HEAD':
                 c.mergepoint = None
             c.parents.append(changesets[branches[c.mergepoint]])
 
         if mergefrom:
             m = mergefrom.search(c.comment)
             if m:
                 m = m.group(1)
                 if m == 'HEAD':
                     m = None
                 try:
                     candidate = changesets[branches[m]]
                 except KeyError:
                     ui.warn(_("warning: CVS commit message references "
                               "non-existent branch %r:\n%s\n")
                             % (m, c.comment))
                 if m in branches and c.branch != m and not candidate.synthetic:
                     c.parents.append(candidate)
 
         if mergeto:
             m = mergeto.search(c.comment)
             if m:
                 try:
                     m = m.group(1)
                     if m == 'HEAD':
                         m = None
                 except:
                     m = None   # if no group found then merge to HEAD
                 if m in branches and c.branch != m:
                     # insert empty changeset for merge
                     cc = changeset(
                         author=c.author, branch=m, date=c.date,
                         comment='convert-repo: CVS merge from branch %s'
                         % c.branch,
                         entries=[], tags=[],
                         parents=[changesets[branches[m]], c])
                     changesets.insert(i + 1, cc)
                     branches[m] = i + 1
 
                     # adjust our loop counters now we have inserted a new entry
                     n += 1
                     i += 2
                     continue
 
         branches[c.branch] = i
         i += 1
 
     # Drop synthetic changesets (safe now that we have ensured no other
     # changesets can have them as parents).
     i = 0
     while i < len(changesets):
         if changesets[i].synthetic:
             del changesets[i]
         else:
             i += 1
 
     # Number changesets
 
     for i, c in enumerate(changesets):
         c.id = i + 1
 
     ui.status(_('%d changeset entries\n') % len(changesets))
 
     hook.hook(ui, None, "cvschangesets", True, changesets=changesets)
 
     return changesets
 
 
 def debugcvsps(ui, *args, **opts):
     '''Read CVS rlog for current directory or named path in
     repository, and convert the log to changesets based on matching
     commit log entries and dates.
     '''
     if opts["new_cache"]:
         cache = "write"
     elif opts["update_cache"]:
         cache = "update"
     else:
         cache = None
 
     revisions = opts["revisions"]
 
     try:
         if args:
             log = []
             for d in args:
                 log += createlog(ui, d, root=opts["root"], cache=cache)
         else:
             log = createlog(ui, root=opts["root"], cache=cache)
     except logerror, e:
         ui.write("%r\n"%e)
         return
 
     changesets = createchangeset(ui, log, opts["fuzz"])
     del log
 
     # Print changesets (optionally filtered)
 
     off = len(revisions)
     branches = {}    # latest version number in each branch
     ancestors = {}   # parent branch
     for cs in changesets:
 
         if opts["ancestors"]:
             if cs.branch not in branches and cs.parents and cs.parents[0].id:
                 ancestors[cs.branch] = (changesets[cs.parents[0].id - 1].branch,
                                         cs.parents[0].id)
             branches[cs.branch] = cs.id
 
         # limit by branches
         if opts["branches"] and (cs.branch or 'HEAD') not in opts["branches"]:
             continue
 
         if not off:
             # Note: trailing spaces on several lines here are needed to have
             #       bug-for-bug compatibility with cvsps.
             ui.write('---------------------\n')
             ui.write('PatchSet %d \n' % cs.id)
             ui.write('Date: %s\n' % util.datestr(cs.date,
                                                  '%Y/%m/%d %H:%M:%S %1%2'))
             ui.write('Author: %s\n' % cs.author)
             ui.write('Branch: %s\n' % (cs.branch or 'HEAD'))
             ui.write('Tag%s: %s \n' % (['', 's'][len(cs.tags) > 1],
                                   ','.join(cs.tags) or '(none)'))
             branchpoints = getattr(cs, 'branchpoints', None)
             if branchpoints:
                 ui.write('Branchpoints: %s \n' % ', '.join(branchpoints))
             if opts["parents"] and cs.parents:
                 if len(cs.parents) > 1:
                     ui.write('Parents: %s\n' %
                              (','.join([str(p.id) for p in cs.parents])))
                 else:
                     ui.write('Parent: %d\n' % cs.parents[0].id)
 
             if opts["ancestors"]:
                 b = cs.branch
                 r = []
                 while b:
                     b, c = ancestors[b]
                     r.append('%s:%d:%d' % (b or "HEAD", c, branches[b]))
                 if r:
                     ui.write('Ancestors: %s\n' % (','.join(r)))
 
             ui.write('Log:\n')
             ui.write('%s\n\n' % cs.comment)
             ui.write('Members: \n')
             for f in cs.entries:
                 fn = f.file
                 if fn.startswith(opts["prefix"]):
                     fn = fn[len(opts["prefix"]):]
                 ui.write('\t%s:%s->%s%s \n' % (
                         fn, '.'.join([str(x) for x in f.parent]) or 'INITIAL',
                         '.'.join([str(x) for x in f.revision]),
                         ['', '(DEAD)'][f.dead]))
             ui.write('\n')
 
         # have we seen the start tag?
         if revisions and off:
             if revisions[0] == str(cs.id) or \
                 revisions[0] in cs.tags:
                 off = False
 
         # see if we reached the end tag
         if len(revisions) > 1 and not off:
             if revisions[1] == str(cs.id) or \
                 revisions[1] in cs.tags:
                 break

hgext/convert/filemap.py

@@ -1,377 +1,380
 # Copyright 2007 Bryan O'Sullivan <[e-mail]>
 # Copyright 2007 Alexis S. L. Carvalho <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 import shlex
 from mercurial.i18n import _
 from mercurial import util
 from common import SKIPREV, converter_source
 
 def rpairs(name):
     e = len(name)
     while e != -1:
         yield name[:e], name[e + 1:]
         e = name.rfind('/', 0, e)
     yield '.', name
 
 class filemapper(object):
     '''Map and filter filenames when importing.
     A name can be mapped to itself, a new name, or None (omit from new
     repository).'''
 
     def __init__(self, ui, path=None):
         self.ui = ui
         self.include = {}
         self.exclude = {}
         self.rename = {}
         if path:
             if self.parse(path):
                 raise util.Abort(_('errors in filemap'))
 
     def parse(self, path):
         errs = 0
         def check(name, mapping, listname):
             if not name:
                 self.ui.warn(_('%s:%d: path to %s is missing\n') %
                              (lex.infile, lex.lineno, listname))
                 return 1
             if name in mapping:
                 self.ui.warn(_('%s:%d: %r already in %s list\n') %
                              (lex.infile, lex.lineno, name, listname))
                 return 1
             if (name.startswith('/') or
                 name.endswith('/') or
                 '//' in name):
                 self.ui.warn(_('%s:%d: superfluous / in %s %r\n') %
                              (lex.infile, lex.lineno, listname, name))
                 return 1
             return 0
         lex = shlex.shlex(open(path), path, True)
         lex.wordchars += '!@#$%^&*()-=+[]{}|;:,./<>?'
         cmd = lex.get_token()
         while cmd:
             if cmd == 'include':
                 name = lex.get_token()
                 errs += check(name, self.exclude, 'exclude')
                 self.include[name] = name
             elif cmd == 'exclude':
                 name = lex.get_token()
                 errs += check(name, self.include, 'include')
                 errs += check(name, self.rename, 'rename')
                 self.exclude[name] = name
             elif cmd == 'rename':
                 src = lex.get_token()
                 dest = lex.get_token()
                 errs += check(src, self.exclude, 'exclude')
                 self.rename[src] = dest
             elif cmd == 'source':
                 errs += self.parse(lex.get_token())
             else:
                 self.ui.warn(_('%s:%d: unknown directive %r\n') %
                              (lex.infile, lex.lineno, cmd))
                 errs += 1
             cmd = lex.get_token()
         return errs
 
     def lookup(self, name, mapping):
         for pre, suf in rpairs(name):
             try:
                 return mapping[pre], pre, suf
             except KeyError:
                 pass
         return '', name, ''
 
     def __call__(self, name):
         if self.include:
             inc = self.lookup(name, self.include)[0]
         else:
             inc = name
         if self.exclude:
             exc = self.lookup(name, self.exclude)[0]
         else:
             exc = ''
         if (not self.include and exc) or (len(inc) <= len(exc)):
             return None
         newpre, pre, suf = self.lookup(name, self.rename)
         if newpre:
             if newpre == '.':
                 return suf
             if suf:
                 return newpre + '/' + suf
             return newpre
         return name
 
     def active(self):
         return bool(self.include or self.exclude or self.rename)
 
 # This class does two additional things compared to a regular source:
 #
 # - Filter and rename files.  This is mostly wrapped by the filemapper
 #   class above. We hide the original filename in the revision that is
 #   returned by getchanges to be able to find things later in getfile.
 #
 # - Return only revisions that matter for the files we're interested in.
 #   This involves rewriting the parents of the original revision to
 #   create a graph that is restricted to those revisions.
 #
 #   This set of revisions includes not only revisions that directly
 #   touch files we're interested in, but also merges that merge two
 #   or more interesting revisions.
 
 class filemap_source(converter_source):
     def __init__(self, ui, baseconverter, filemap):
         super(filemap_source, self).__init__(ui)
         self.base = baseconverter
         self.filemapper = filemapper(ui, filemap)
         self.commits = {}
         # if a revision rev has parent p in the original revision graph, then
         # rev will have parent self.parentmap[p] in the restricted graph.
         self.parentmap = {}
         # self.wantedancestors[rev] is the set of all ancestors of rev that
         # are in the restricted graph.
         self.wantedancestors = {}
         self.convertedorder = None
         self._rebuilt = False
         self.origparents = {}
         self.children = {}
         self.seenchildren = {}
 
     def before(self):
         self.base.before()
 
     def after(self):
         self.base.after()
 
     def setrevmap(self, revmap):
         # rebuild our state to make things restartable
         #
         # To avoid calling getcommit for every revision that has already
         # been converted, we rebuild only the parentmap, delaying the
         # rebuild of wantedancestors until we need it (i.e. until a
         # merge).
         #
         # We assume the order argument lists the revisions in
         # topological order, so that we can infer which revisions were
         # wanted by previous runs.
         self._rebuilt = not revmap
         seen = {SKIPREV: SKIPREV}
         dummyset = set()
         converted = []
         for rev in revmap.order:
             mapped = revmap[rev]
             wanted = mapped not in seen
             if wanted:
                 seen[mapped] = rev
                 self.parentmap[rev] = rev
             else:
                 self.parentmap[rev] = seen[mapped]
             self.wantedancestors[rev] = dummyset
             arg = seen[mapped]
             if arg == SKIPREV:
                 arg = None
             converted.append((rev, wanted, arg))
         self.convertedorder = converted
         return self.base.setrevmap(revmap)
 
     def rebuild(self):
         if self._rebuilt:
             return True
         self._rebuilt = True
         self.parentmap.clear()
         self.wantedancestors.clear()
         self.seenchildren.clear()
         for rev, wanted, arg in self.convertedorder:
             if rev not in self.origparents:
                 self.origparents[rev] = self.getcommit(rev).parents
             if arg is not None:
                 self.children[arg] = self.children.get(arg, 0) + 1
 
         for rev, wanted, arg in self.convertedorder:
             parents = self.origparents[rev]
             if wanted:
                 self.mark_wanted(rev, parents)
             else:
                 self.mark_not_wanted(rev, arg)
             self._discard(arg, *parents)
 
         return True
 
     def getheads(self):
         return self.base.getheads()
 
     def getcommit(self, rev):
         # We want to save a reference to the commit objects to be able
         # to rewrite their parents later on.
         c = self.commits[rev] = self.base.getcommit(rev)
         for p in c.parents:
             self.children[p] = self.children.get(p, 0) + 1
         return c
 
     def _cachedcommit(self, rev):
         if rev in self.commits:
             return self.commits[rev]
         return self.base.getcommit(rev)
 
     def _discard(self, *revs):
         for r in revs:
             if r is None:
                 continue
             self.seenchildren[r] = self.seenchildren.get(r, 0) + 1
             if self.seenchildren[r] == self.children[r]:
                 del self.wantedancestors[r]
                 del self.parentmap[r]
                 del self.seenchildren[r]
                 if self._rebuilt:
                     del self.children[r]
 
     def wanted(self, rev, i):
         # Return True if we're directly interested in rev.
         #
         # i is an index selecting one of the parents of rev (if rev
         # has no parents, i is None).  getchangedfiles will give us
         # the list of files that are different in rev and in the parent
         # indicated by i.  If we're interested in any of these files,
         # we're interested in rev.
         try:
             files = self.base.getchangedfiles(rev, i)
         except NotImplementedError:
             raise util.Abort(_("source repository doesn't support --filemap"))
         for f in files:
             if self.filemapper(f):
                 return True
         return False
 
     def mark_not_wanted(self, rev, p):
         # Mark rev as not interesting and update data structures.
 
         if p is None:
             # A root revision. Use SKIPREV to indicate that it doesn't
             # map to any revision in the restricted graph.  Put SKIPREV
             # in the set of wanted ancestors to simplify code elsewhere
             self.parentmap[rev] = SKIPREV
             self.wantedancestors[rev] = set((SKIPREV,))
             return
 
         # Reuse the data from our parent.
         self.parentmap[rev] = self.parentmap[p]
         self.wantedancestors[rev] = self.wantedancestors[p]
 
     def mark_wanted(self, rev, parents):
         # Mark rev ss wanted and update data structures.
 
         # rev will be in the restricted graph, so children of rev in
         # the original graph should still have rev as a parent in the
         # restricted graph.
         self.parentmap[rev] = rev
 
         # The set of wanted ancestors of rev is the union of the sets
         # of wanted ancestors of its parents. Plus rev itself.
         wrev = set()
         for p in parents:
             wrev.update(self.wantedancestors[p])
         wrev.add(rev)
         self.wantedancestors[rev] = wrev
 
     def getchanges(self, rev):
         parents = self.commits[rev].parents
         if len(parents) > 1:
             self.rebuild()
 
         # To decide whether we're interested in rev we:
         #
         # - calculate what parents rev will have if it turns out we're
         #   interested in it.  If it's going to have more than 1 parent,
         #   we're interested in it.
         #
         # - otherwise, we'll compare it with the single parent we found.
         #   If any of the files we're interested in is different in the
         #   the two revisions, we're interested in rev.
 
         # A parent p is interesting if its mapped version (self.parentmap[p]):
         # - is not SKIPREV
         # - is still not in the list of parents (we don't want duplicates)
         # - is not an ancestor of the mapped versions of the other parents
         mparents = []
         wp = None
         for i, p1 in enumerate(parents):
             mp1 = self.parentmap[p1]
             if mp1 == SKIPREV or mp1 in mparents:
                 continue
             for p2 in parents:
                 if p1 == p2 or mp1 == self.parentmap[p2]:
                     continue
                 if mp1 in self.wantedancestors[p2]:
                     break
             else:
                 mparents.append(mp1)
                 wp = i
 
         if wp is None and parents:
             wp = 0
 
         self.origparents[rev] = parents
 
         closed = False
         if 'close' in self.commits[rev].extra:
             # A branch closing revision is only useful if one of its
             # parents belong to the branch being closed
             branch = self.commits[rev].branch
             pbranches = [self._cachedcommit(p).branch for p in mparents]
             if branch in pbranches:
                 closed = True
 
         if len(mparents) < 2 and not closed and not self.wanted(rev, wp):
             # We don't want this revision.
             # Update our state and tell the convert process to map this
             # revision to the same revision its parent as mapped to.
             p = None
             if parents:
                 p = parents[wp]
             self.mark_not_wanted(rev, p)
             self.convertedorder.append((rev, False, p))
             self._discard(*parents)
             return self.parentmap[rev]
 
         # We want this revision.
         # Rewrite the parents of the commit object
         self.commits[rev].parents = mparents
         self.mark_wanted(rev, parents)
         self.convertedorder.append((rev, True, None))
         self._discard(*parents)
 
         # Get the real changes and do the filtering/mapping. To be
         # able to get the files later on in getfile, we hide the
         # original filename in the rev part of the return value.
         changes, copies = self.base.getchanges(rev)
         newnames = {}
         files = []
         for f, r in changes:
             newf = self.filemapper(f)
             if newf:
                 files.append((newf, (f, r)))
                 newnames[f] = newf
 
         ncopies = {}
         for c in copies:
             newc = self.filemapper(c)
             if newc:
                 newsource = self.filemapper(copies[c])
                 if newsource:
                     ncopies[newc] = newsource
 
         return files, ncopies
 
     def getfile(self, name, rev):
         realname, realrev = rev
         return self.base.getfile(realname, realrev)
 
     def gettags(self):
         return self.base.gettags()
 
     def hasnativeorder(self):
         return self.base.hasnativeorder()
 
     def lookuprev(self, rev):
         return self.base.lookuprev(rev)
+
+    def getbookmarks(self):
+        return self.base.getbookmarks()

hgext/convert/git.py

@@ -1,205 +1,205
 # git.py - git support for the convert extension
 #
 #  Copyright 2005-2009 Matt Mackall <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 import os
 from mercurial import util
 from mercurial.node import hex, nullid
 from mercurial.i18n import _
 
 from common import NoRepo, commit, converter_source, checktool
 
 class convert_git(converter_source):
     # Windows does not support GIT_DIR= construct while other systems
     # cannot remove environment variable. Just assume none have
     # both issues.
-    if hasattr(os, 'unsetenv'):
+    if util.safehasattr(os, 'unsetenv'):
         def gitopen(self, s, noerr=False):
             prevgitdir = os.environ.get('GIT_DIR')
             os.environ['GIT_DIR'] = self.path
             try:
                 if noerr:
                     (stdin, stdout, stderr) = util.popen3(s)
                     return stdout
                 else:
                     return util.popen(s, 'rb')
             finally:
                 if prevgitdir is None:
                     del os.environ['GIT_DIR']
                 else:
                     os.environ['GIT_DIR'] = prevgitdir
     else:
         def gitopen(self, s, noerr=False):
             if noerr:
                 (sin, so, se) = util.popen3('GIT_DIR=%s %s' % (self.path, s))
                 return so
             else:
                 return util.popen('GIT_DIR=%s %s' % (self.path, s), 'rb')
 
     def gitread(self, s):
         fh = self.gitopen(s)
         data = fh.read()
         return data, fh.close()
 
     def __init__(self, ui, path, rev=None):
         super(convert_git, self).__init__(ui, path, rev=rev)
 
         if os.path.isdir(path + "/.git"):
             path += "/.git"
         if not os.path.exists(path + "/objects"):
             raise NoRepo(_("%s does not look like a Git repository") % path)
 
         checktool('git', 'git')
 
         self.path = path
 
     def getheads(self):
         if not self.rev:
             heads, ret = self.gitread('git rev-parse --branches --remotes')
             heads = heads.splitlines()
         else:
             heads, ret = self.gitread("git rev-parse --verify %s" % self.rev)
             heads = [heads[:-1]]
         if ret:
             raise util.Abort(_('cannot retrieve git heads'))
         return heads
 
     def catfile(self, rev, type):
         if rev == hex(nullid):
             raise IOError()
         data, ret = self.gitread("git cat-file %s %s" % (type, rev))
         if ret:
             raise util.Abort(_('cannot read %r object at %s') % (type, rev))
         return data
 
     def getfile(self, name, rev):
         data = self.catfile(rev, "blob")
         mode = self.modecache[(name, rev)]
         return data, mode
 
     def getchanges(self, version):
         self.modecache = {}
         fh = self.gitopen("git diff-tree -z --root -m -r %s" % version)
         changes = []
         seen = set()
         entry = None
         for l in fh.read().split('\x00'):
             if not entry:
                 if not l.startswith(':'):
                     continue
                 entry = l
                 continue
             f = l
             if f not in seen:
                 seen.add(f)
                 entry = entry.split()
                 h = entry[3]
                 p = (entry[1] == "100755")
                 s = (entry[1] == "120000")
                 self.modecache[(f, h)] = (p and "x") or (s and "l") or ""
                 changes.append((f, h))
             entry = None
         if fh.close():
             raise util.Abort(_('cannot read changes in %s') % version)
         return (changes, {})
 
     def getcommit(self, version):
         c = self.catfile(version, "commit") # read the commit hash
         end = c.find("\n\n")
         message = c[end + 2:]
         message = self.recode(message)
         l = c[:end].splitlines()
         parents = []
         author = committer = None
         for e in l[1:]:
             n, v = e.split(" ", 1)
             if n == "author":
                 p = v.split()
                 tm, tz = p[-2:]
                 author = " ".join(p[:-2])
                 if author[0] == "<": author = author[1:-1]
                 author = self.recode(author)
             if n == "committer":
                 p = v.split()
                 tm, tz = p[-2:]
                 committer = " ".join(p[:-2])
                 if committer[0] == "<": committer = committer[1:-1]
                 committer = self.recode(committer)
             if n == "parent":
                 parents.append(v)
 
         if committer and committer != author:
             message += "\ncommitter: %s\n" % committer
         tzs, tzh, tzm = tz[-5:-4] + "1", tz[-4:-2], tz[-2:]
         tz = -int(tzs) * (int(tzh) * 3600 + int(tzm))
         date = tm + " " + str(tz)
 
         c = commit(parents=parents, date=date, author=author, desc=message,
                    rev=version)
         return c
 
     def gettags(self):
         tags = {}
         fh = self.gitopen('git ls-remote --tags "%s"' % self.path)
         prefix = 'refs/tags/'
         for line in fh:
             line = line.strip()
             if not line.endswith("^{}"):
                 continue
             node, tag = line.split(None, 1)
             if not tag.startswith(prefix):
                 continue
             tag = tag[len(prefix):-3]
             tags[tag] = node
         if fh.close():
             raise util.Abort(_('cannot read tags from %s') % self.path)
 
         return tags
 
     def getchangedfiles(self, version, i):
         changes = []
         if i is None:
             fh = self.gitopen("git diff-tree --root -m -r %s" % version)
             for l in fh:
                 if "\t" not in l:
                     continue
                 m, f = l[:-1].split("\t")
                 changes.append(f)
         else:
             fh = self.gitopen('git diff-tree --name-only --root -r %s "%s^%s" --'
                              % (version, version, i + 1))
             changes = [f.rstrip('\n') for f in fh]
         if fh.close():
             raise util.Abort(_('cannot read changes in %s') % version)
 
         return changes
 
     def getbookmarks(self):
         bookmarks = {}
 
         # Interesting references in git are prefixed
         prefix = 'refs/heads/'
         prefixlen = len(prefix)
 
         # factor two commands
         gitcmd = { 'remote/': 'git ls-remote --heads origin',
                           '': 'git show-ref'}
 
         # Origin heads
         for reftype in gitcmd:
             try:
                 fh = self.gitopen(gitcmd[reftype], noerr=True)
                 for line in fh:
                     line = line.strip()
                     rev, name = line.split(None, 1)
                     if not name.startswith(prefix):
                         continue
                     name = '%s%s' % (reftype, name[prefixlen:])
                     bookmarks[name] = rev
             except:
                 pass
 
         return bookmarks

hgext/convert/hg.py

@@ -1,388 +1,387
 # hg.py - hg backend for convert extension
 #
 #  Copyright 2005-2009 Matt Mackall <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # Notes for hg->hg conversion:
 #
 # * Old versions of Mercurial didn't trim the whitespace from the ends
 #   of commit messages, but new versions do.  Changesets created by
 #   those older versions, then converted, may thus have different
 #   hashes for changesets that are otherwise identical.
 #
 # * Using "--config convert.hg.saverev=true" will make the source
 #   identifier to be stored in the converted revision. This will cause
 #   the converted revision to have a different identity than the
 #   source.
 
 
 import os, time, cStringIO
 from mercurial.i18n import _
 from mercurial.node import bin, hex, nullid
 from mercurial import hg, util, context, bookmarks, error
 
 from common import NoRepo, commit, converter_source, converter_sink
 
 class mercurial_sink(converter_sink):
     def __init__(self, ui, path):
         converter_sink.__init__(self, ui, path)
         self.branchnames = ui.configbool('convert', 'hg.usebranchnames', True)
         self.clonebranches = ui.configbool('convert', 'hg.clonebranches', False)
         self.tagsbranch = ui.config('convert', 'hg.tagsbranch', 'default')
         self.lastbranch = None
         if os.path.isdir(path) and len(os.listdir(path)) > 0:
             try:
                 self.repo = hg.repository(self.ui, path)
                 if not self.repo.local():
                     raise NoRepo(_('%s is not a local Mercurial repository')
                                  % path)
             except error.RepoError, err:
                 ui.traceback()
                 raise NoRepo(err.args[0])
         else:
             try:
                 ui.status(_('initializing destination %s repository\n') % path)
                 self.repo = hg.repository(self.ui, path, create=True)
                 if not self.repo.local():
                     raise NoRepo(_('%s is not a local Mercurial repository')
                                  % path)
                 self.created.append(path)
             except error.RepoError:
                 ui.traceback()
                 raise NoRepo(_("could not create hg repository %s as sink")
                              % path)
         self.lock = None
         self.wlock = None
         self.filemapmode = False
 
     def before(self):
         self.ui.debug('run hg sink pre-conversion action\n')
         self.wlock = self.repo.wlock()
         self.lock = self.repo.lock()
 
     def after(self):
         self.ui.debug('run hg sink post-conversion action\n')
         if self.lock:
             self.lock.release()
         if self.wlock:
             self.wlock.release()
 
     def revmapfile(self):
-        return os.path.join(self.path, ".hg", "shamap")
+        return self.repo.join("shamap")
 
     def authorfile(self):
-        return os.path.join(self.path, ".hg", "authormap")
+        return self.repo.join("authormap")
 
     def getheads(self):
         h = self.repo.changelog.heads()
         return [hex(x) for x in h]
 
     def setbranch(self, branch, pbranches):
         if not self.clonebranches:
             return
 
         setbranch = (branch != self.lastbranch)
         self.lastbranch = branch
         if not branch:
             branch = 'default'
         pbranches = [(b[0], b[1] and b[1] or 'default') for b in pbranches]
         pbranch = pbranches and pbranches[0][1] or 'default'
 
         branchpath = os.path.join(self.path, branch)
         if setbranch:
             self.after()
             try:
                 self.repo = hg.repository(self.ui, branchpath)
             except:
                 self.repo = hg.repository(self.ui, branchpath, create=True)
             self.before()
 
         # pbranches may bring revisions from other branches (merge parents)
         # Make sure we have them, or pull them.
         missings = {}
         for b in pbranches:
             try:
                 self.repo.lookup(b[0])
             except:
                 missings.setdefault(b[1], []).append(b[0])
 
         if missings:
             self.after()
             for pbranch, heads in missings.iteritems():
                 pbranchpath = os.path.join(self.path, pbranch)
                 prepo = hg.peer(self.ui, {}, pbranchpath)
                 self.ui.note(_('pulling from %s into %s\n') % (pbranch, branch))
                 self.repo.pull(prepo, [prepo.lookup(h) for h in heads])
             self.before()
 
     def _rewritetags(self, source, revmap, data):
         fp = cStringIO.StringIO()
         for line in data.splitlines():
             s = line.split(' ', 1)
             if len(s) != 2:
                 continue
             revid = revmap.get(source.lookuprev(s[0]))
             if not revid:
                 continue
             fp.write('%s %s\n' % (revid, s[1]))
         return fp.getvalue()
 
     def putcommit(self, files, copies, parents, commit, source, revmap):
 
         files = dict(files)
         def getfilectx(repo, memctx, f):
             v = files[f]
             data, mode = source.getfile(f, v)
             if f == '.hgtags':
                 data = self._rewritetags(source, revmap, data)
             return context.memfilectx(f, data, 'l' in mode, 'x' in mode,
                                       copies.get(f))
 
         pl = []
         for p in parents:
             if p not in pl:
                 pl.append(p)
         parents = pl
         nparents = len(parents)
         if self.filemapmode and nparents == 1:
             m1node = self.repo.changelog.read(bin(parents[0]))[0]
             parent = parents[0]
 
         if len(parents) < 2:
             parents.append(nullid)
         if len(parents) < 2:
             parents.append(nullid)
         p2 = parents.pop(0)
 
         text = commit.desc
         extra = commit.extra.copy()
         if self.branchnames and commit.branch:
             extra['branch'] = commit.branch
         if commit.rev:
             extra['convert_revision'] = commit.rev
 
         while parents:
             p1 = p2
             p2 = parents.pop(0)
             ctx = context.memctx(self.repo, (p1, p2), text, files.keys(),
                                  getfilectx, commit.author, commit.date, extra)
             self.repo.commitctx(ctx)
             text = "(octopus merge fixup)\n"
             p2 = hex(self.repo.changelog.tip())
 
         if self.filemapmode and nparents == 1:
             man = self.repo.manifest
             mnode = self.repo.changelog.read(bin(p2))[0]
             closed = 'close' in commit.extra
             if not closed and not man.cmp(m1node, man.revision(mnode)):
                 self.ui.status(_("filtering out empty revision\n"))
-                self.repo.rollback()
+                self.repo.rollback(force=True)
                 return parent
         return p2
 
     def puttags(self, tags):
         try:
             parentctx = self.repo[self.tagsbranch]
             tagparent = parentctx.node()
         except error.RepoError:
             parentctx = None
             tagparent = nullid
 
         try:
             oldlines = sorted(parentctx['.hgtags'].data().splitlines(True))
         except:
             oldlines = []
 
         newlines = sorted([("%s %s\n" % (tags[tag], tag)) for tag in tags])
         if newlines == oldlines:
             return None, None
         data = "".join(newlines)
         def getfilectx(repo, memctx, f):
             return context.memfilectx(f, data, False, False, None)
 
         self.ui.status(_("updating tags\n"))
         date = "%s 0" % int(time.mktime(time.gmtime()))
         extra = {'branch': self.tagsbranch}
         ctx = context.memctx(self.repo, (tagparent, None), "update tags",
                              [".hgtags"], getfilectx, "convert-repo", date,
                              extra)
         self.repo.commitctx(ctx)
         return hex(self.repo.changelog.tip()), hex(tagparent)
 
     def setfilemapmode(self, active):
         self.filemapmode = active
 
     def putbookmarks(self, updatedbookmark):
         if not len(updatedbookmark):
             return
 
         self.ui.status(_("updating bookmarks\n"))
         for bookmark in updatedbookmark:
             self.repo._bookmarks[bookmark] = bin(updatedbookmark[bookmark])
             bookmarks.write(self.repo)
 
 
 class mercurial_source(converter_source):
     def __init__(self, ui, path, rev=None):
         converter_source.__init__(self, ui, path, rev)
         self.ignoreerrors = ui.configbool('convert', 'hg.ignoreerrors', False)
         self.ignored = set()
         self.saverev = ui.configbool('convert', 'hg.saverev', False)
         try:
             self.repo = hg.repository(self.ui, path)
             # try to provoke an exception if this isn't really a hg
             # repo, but some other bogus compatible-looking url
             if not self.repo.local():
                 raise error.RepoError()
         except error.RepoError:
             ui.traceback()
             raise NoRepo(_("%s is not a local Mercurial repository") % path)
         self.lastrev = None
         self.lastctx = None
         self._changescache = None
         self.convertfp = None
         # Restrict converted revisions to startrev descendants
         startnode = ui.config('convert', 'hg.startrev')
         if startnode is not None:
             try:
                 startnode = self.repo.lookup(startnode)
             except error.RepoError:
                 raise util.Abort(_('%s is not a valid start revision')
                                  % startnode)
             startrev = self.repo.changelog.rev(startnode)
             children = {startnode: 1}
             for rev in self.repo.changelog.descendants(startrev):
                 children[self.repo.changelog.node(rev)] = 1
             self.keep = children.__contains__
         else:
             self.keep = util.always
 
     def changectx(self, rev):
         if self.lastrev != rev:
             self.lastctx = self.repo[rev]
             self.lastrev = rev
         return self.lastctx
 
     def parents(self, ctx):
         return [p for p in ctx.parents() if p and self.keep(p.node())]
 
     def getheads(self):
         if self.rev:
             heads = [self.repo[self.rev].node()]
         else:
             heads = self.repo.heads()
         return [hex(h) for h in heads if self.keep(h)]
 
     def getfile(self, name, rev):
         try:
             fctx = self.changectx(rev)[name]
             return fctx.data(), fctx.flags()
         except error.LookupError, err:
             raise IOError(err)
 
     def getchanges(self, rev):
         ctx = self.changectx(rev)
         parents = self.parents(ctx)
         if not parents:
             files = sorted(ctx.manifest())
             # getcopies() is not needed for roots, but it is a simple way to
             # detect missing revlogs and abort on errors or populate self.ignored
             self.getcopies(ctx, parents, files)
             return [(f, rev) for f in files if f not in self.ignored], {}
         if self._changescache and self._changescache[0] == rev:
             m, a, r = self._changescache[1]
         else:
             m, a, r = self.repo.status(parents[0].node(), ctx.node())[:3]
         # getcopies() detects missing revlogs early, run it before
         # filtering the changes.
         copies = self.getcopies(ctx, parents, m + a)
         changes = [(name, rev) for name in m + a + r
                    if name not in self.ignored]
         return sorted(changes), copies
 
     def getcopies(self, ctx, parents, files):
         copies = {}
         for name in files:
             if name in self.ignored:
                 continue
             try:
                 copysource, copynode = ctx.filectx(name).renamed()
                 if copysource in self.ignored or not self.keep(copynode):
                     continue
                 # Ignore copy sources not in parent revisions
                 found = False
                 for p in parents:
                     if copysource in p:
                         found = True
                         break
                 if not found:
                     continue
                 copies[name] = copysource
             except TypeError:
                 pass
             except error.LookupError, e:
                 if not self.ignoreerrors:
                     raise
                 self.ignored.add(name)
                 self.ui.warn(_('ignoring: %s\n') % e)
         return copies
 
     def getcommit(self, rev):
         ctx = self.changectx(rev)
         parents = [p.hex() for p in self.parents(ctx)]
         if self.saverev:
             crev = rev
         else:
             crev = None
         return commit(author=ctx.user(), date=util.datestr(ctx.date()),
                       desc=ctx.description(), rev=crev, parents=parents,
                       branch=ctx.branch(), extra=ctx.extra(),
                       sortkey=ctx.rev())
 
     def gettags(self):
         tags = [t for t in self.repo.tagslist() if t[0] != 'tip']
         return dict([(name, hex(node)) for name, node in tags
                      if self.keep(node)])
 
     def getchangedfiles(self, rev, i):
         ctx = self.changectx(rev)
         parents = self.parents(ctx)
         if not parents and i is None:
             i = 0
             changes = [], ctx.manifest().keys(), []
         else:
             i = i or 0
             changes = self.repo.status(parents[i].node(), ctx.node())[:3]
         changes = [[f for f in l if f not in self.ignored] for l in changes]
 
         if i == 0:
             self._changescache = (rev, changes)
 
         return changes[0] + changes[1] + changes[2]
 
     def converted(self, rev, destrev):
         if self.convertfp is None:
-            self.convertfp = open(os.path.join(self.path, '.hg', 'shamap'),
-                                  'a')
+            self.convertfp = open(self.repo.join('shamap'), 'a')
         self.convertfp.write('%s %s\n' % (destrev, rev))
         self.convertfp.flush()
 
     def before(self):
         self.ui.debug('run hg source pre-conversion action\n')
 
     def after(self):
         self.ui.debug('run hg source post-conversion action\n')
 
     def hasnativeorder(self):
         return True
 
     def lookuprev(self, rev):
         try:
             return hex(self.repo.lookup(rev))
         except error.RepoError:
             return None
 
     def getbookmarks(self):
         return bookmarks.listbookmarks(self.repo)

hgext/convert/subversion.py

@@ -1,1175 +1,1175
 # Subversion 1.4/1.5 Python API backend
 #
 # Copyright(C) 2007 Daniel Holth et al
 
 import os
 import re
 import sys
 import cPickle as pickle
 import tempfile
 import urllib
 import urllib2
 
 from mercurial import strutil, scmutil, util, encoding
 from mercurial.i18n import _
 
 # Subversion stuff. Works best with very recent Python SVN bindings
 # e.g. SVN 1.5 or backports. Thanks to the bzr folks for enhancing
 # these bindings.
 
 from cStringIO import StringIO
 
 from common import NoRepo, MissingTool, commit, encodeargs, decodeargs
 from common import commandline, converter_source, converter_sink, mapfile
 
 try:
     from svn.core import SubversionException, Pool
     import svn
     import svn.client
     import svn.core
     import svn.ra
     import svn.delta
     import transport
     import warnings
     warnings.filterwarnings('ignore',
             module='svn.core',
             category=DeprecationWarning)
 
 except ImportError:
     svn = None
 
 class SvnPathNotFound(Exception):
     pass
 
 def revsplit(rev):
     """Parse a revision string and return (uuid, path, revnum)."""
     url, revnum = rev.rsplit('@', 1)
     parts = url.split('/', 1)
     mod = ''
     if len(parts) > 1:
         mod = '/' + parts[1]
     return parts[0][4:], mod, int(revnum)
 
 def geturl(path):
     try:
         return svn.client.url_from_path(svn.core.svn_path_canonicalize(path))
     except SubversionException:
         pass
     if os.path.isdir(path):
         path = os.path.normpath(os.path.abspath(path))
         if os.name == 'nt':
             path = '/' + util.normpath(path)
         # Module URL is later compared with the repository URL returned
         # by svn API, which is UTF-8.
         path = encoding.tolocal(path)
         return 'file://%s' % urllib.quote(path)
     return path
 
 def optrev(number):
     optrev = svn.core.svn_opt_revision_t()
     optrev.kind = svn.core.svn_opt_revision_number
     optrev.value.number = number
     return optrev
 
 class changedpath(object):
     def __init__(self, p):
         self.copyfrom_path = p.copyfrom_path
         self.copyfrom_rev = p.copyfrom_rev
         self.action = p.action
 
 def get_log_child(fp, url, paths, start, end, limit=0, discover_changed_paths=True,
                     strict_node_history=False):
     protocol = -1
     def receiver(orig_paths, revnum, author, date, message, pool):
         if orig_paths is not None:
             for k, v in orig_paths.iteritems():
                 orig_paths[k] = changedpath(v)
         pickle.dump((orig_paths, revnum, author, date, message),
                     fp, protocol)
 
     try:
         # Use an ra of our own so that our parent can consume
         # our results without confusing the server.
         t = transport.SvnRaTransport(url=url)
         svn.ra.get_log(t.ra, paths, start, end, limit,
                        discover_changed_paths,
                        strict_node_history,
                        receiver)
     except SubversionException, (inst, num):
         pickle.dump(num, fp, protocol)
     except IOError:
         # Caller may interrupt the iteration
         pickle.dump(None, fp, protocol)
     else:
         pickle.dump(None, fp, protocol)
     fp.close()
     # With large history, cleanup process goes crazy and suddenly
     # consumes *huge* amount of memory. The output file being closed,
     # there is no need for clean termination.
     os._exit(0)
 
 def debugsvnlog(ui, **opts):
     """Fetch SVN log in a subprocess and channel them back to parent to
     avoid memory collection issues.
     """
     util.setbinary(sys.stdin)
     util.setbinary(sys.stdout)
     args = decodeargs(sys.stdin.read())
     get_log_child(sys.stdout, *args)
 
 class logstream(object):
     """Interruptible revision log iterator."""
     def __init__(self, stdout):
         self._stdout = stdout
 
     def __iter__(self):
         while True:
             try:
                 entry = pickle.load(self._stdout)
             except EOFError:
                 raise util.Abort(_('Mercurial failed to run itself, check'
                                    ' hg executable is in PATH'))
             try:
                 orig_paths, revnum, author, date, message = entry
             except:
                 if entry is None:
                     break
                 raise SubversionException("child raised exception", entry)
             yield entry
 
     def close(self):
         if self._stdout:
             self._stdout.close()
             self._stdout = None
 
 
 # Check to see if the given path is a local Subversion repo. Verify this by
 # looking for several svn-specific files and directories in the given
 # directory.
 def filecheck(ui, path, proto):
     for x in ('locks', 'hooks', 'format', 'db'):
         if not os.path.exists(os.path.join(path, x)):
             return False
     return True
 
 # Check to see if a given path is the root of an svn repo over http. We verify
 # this by requesting a version-controlled URL we know can't exist and looking
 # for the svn-specific "not found" XML.
 def httpcheck(ui, path, proto):
     try:
         opener = urllib2.build_opener()
         rsp = opener.open('%s://%s/!svn/ver/0/.svn' % (proto, path))
         data = rsp.read()
     except urllib2.HTTPError, inst:
         if inst.code != 404:
             # Except for 404 we cannot know for sure this is not an svn repo
             ui.warn(_('svn: cannot probe remote repository, assume it could '
                       'be a subversion repository. Use --source-type if you '
                       'know better.\n'))
             return True
         data = inst.fp.read()
     except:
         # Could be urllib2.URLError if the URL is invalid or anything else.
         return False
     return '<m:human-readable errcode="160013">' in data
 
 protomap = {'http': httpcheck,
             'https': httpcheck,
             'file': filecheck,
             }
 def issvnurl(ui, url):
     try:
         proto, path = url.split('://', 1)
         if proto == 'file':
             path = urllib.url2pathname(path)
     except ValueError:
         proto = 'file'
         path = os.path.abspath(url)
     if proto == 'file':
         path = path.replace(os.sep, '/')
     check = protomap.get(proto, lambda *args: False)
     while '/' in path:
         if check(ui, path, proto):
             return True
         path = path.rsplit('/', 1)[0]
     return False
 
 # SVN conversion code stolen from bzr-svn and tailor
 #
 # Subversion looks like a versioned filesystem, branches structures
 # are defined by conventions and not enforced by the tool. First,
 # we define the potential branches (modules) as "trunk" and "branches"
 # children directories. Revisions are then identified by their
 # module and revision number (and a repository identifier).
 #
 # The revision graph is really a tree (or a forest). By default, a
 # revision parent is the previous revision in the same module. If the
 # module directory is copied/moved from another module then the
 # revision is the module root and its parent the source revision in
 # the parent module. A revision has at most one parent.
 #
 class svn_source(converter_source):
     def __init__(self, ui, url, rev=None):
         super(svn_source, self).__init__(ui, url, rev=rev)
 
         if not (url.startswith('svn://') or url.startswith('svn+ssh://') or
                 (os.path.exists(url) and
                  os.path.exists(os.path.join(url, '.svn'))) or
                 issvnurl(ui, url)):
             raise NoRepo(_("%s does not look like a Subversion repository")
                          % url)
         if svn is None:
             raise MissingTool(_('Could not load Subversion python bindings'))
 
         try:
             version = svn.core.SVN_VER_MAJOR, svn.core.SVN_VER_MINOR
             if version < (1, 4):
                 raise MissingTool(_('Subversion python bindings %d.%d found, '
                                     '1.4 or later required') % version)
         except AttributeError:
             raise MissingTool(_('Subversion python bindings are too old, 1.4 '
                                 'or later required'))
 
         self.lastrevs = {}
 
         latest = None
         try:
             # Support file://path@rev syntax. Useful e.g. to convert
             # deleted branches.
             at = url.rfind('@')
             if at >= 0:
                 latest = int(url[at + 1:])
                 url = url[:at]
         except ValueError:
             pass
         self.url = geturl(url)
         self.encoding = 'UTF-8' # Subversion is always nominal UTF-8
         try:
             self.transport = transport.SvnRaTransport(url=self.url)
             self.ra = self.transport.ra
             self.ctx = self.transport.client
             self.baseurl = svn.ra.get_repos_root(self.ra)
             # Module is either empty or a repository path starting with
             # a slash and not ending with a slash.
             self.module = urllib.unquote(self.url[len(self.baseurl):])
             self.prevmodule = None
             self.rootmodule = self.module
             self.commits = {}
             self.paths = {}
             self.uuid = svn.ra.get_uuid(self.ra)
         except SubversionException:
             ui.traceback()
             raise NoRepo(_("%s does not look like a Subversion repository")
                          % self.url)
 
         if rev:
             try:
                 latest = int(rev)
             except ValueError:
                 raise util.Abort(_('svn: revision %s is not an integer') % rev)
 
         self.trunkname = self.ui.config('convert', 'svn.trunk', 'trunk').strip('/')
         self.startrev = self.ui.config('convert', 'svn.startrev', default=0)
         try:
             self.startrev = int(self.startrev)
             if self.startrev < 0:
                 self.startrev = 0
         except ValueError:
             raise util.Abort(_('svn: start revision %s is not an integer')
                              % self.startrev)
 
         try:
             self.head = self.latest(self.module, latest)
         except SvnPathNotFound:
             self.head = None
         if not self.head:
             raise util.Abort(_('no revision found in module %s')
                              % self.module)
         self.last_changed = self.revnum(self.head)
 
         self._changescache = None
 
         if os.path.exists(os.path.join(url, '.svn/entries')):
             self.wc = url
         else:
             self.wc = None
         self.convertfp = None
 
     def setrevmap(self, revmap):
         lastrevs = {}
         for revid in revmap.iterkeys():
             uuid, module, revnum = revsplit(revid)
             lastrevnum = lastrevs.setdefault(module, revnum)
             if revnum > lastrevnum:
                 lastrevs[module] = revnum
         self.lastrevs = lastrevs
 
     def exists(self, path, optrev):
         try:
             svn.client.ls(self.url.rstrip('/') + '/' + urllib.quote(path),
                                  optrev, False, self.ctx)
             return True
         except SubversionException:
             return False
 
     def getheads(self):
 
         def isdir(path, revnum):
             kind = self._checkpath(path, revnum)
             return kind == svn.core.svn_node_dir
 
         def getcfgpath(name, rev):
             cfgpath = self.ui.config('convert', 'svn.' + name)
             if cfgpath is not None and cfgpath.strip() == '':
                 return None
             path = (cfgpath or name).strip('/')
             if not self.exists(path, rev):
                 if self.module.endswith(path) and name == 'trunk':
                     # we are converting from inside this directory
                     return None
                 if cfgpath:
                     raise util.Abort(_('expected %s to be at %r, but not found')
                                  % (name, path))
                 return None
             self.ui.note(_('found %s at %r\n') % (name, path))
             return path
 
         rev = optrev(self.last_changed)
         oldmodule = ''
         trunk = getcfgpath('trunk', rev)
         self.tags = getcfgpath('tags', rev)
         branches = getcfgpath('branches', rev)
 
         # If the project has a trunk or branches, we will extract heads
         # from them. We keep the project root otherwise.
         if trunk:
             oldmodule = self.module or ''
             self.module += '/' + trunk
             self.head = self.latest(self.module, self.last_changed)
             if not self.head:
                 raise util.Abort(_('no revision found in module %s')
                                  % self.module)
 
         # First head in the list is the module's head
         self.heads = [self.head]
         if self.tags is not None:
             self.tags = '%s/%s' % (oldmodule , (self.tags or 'tags'))
 
         # Check if branches bring a few more heads to the list
         if branches:
             rpath = self.url.strip('/')
             branchnames = svn.client.ls(rpath + '/' + urllib.quote(branches),
                                         rev, False, self.ctx)
             for branch in branchnames.keys():
                 module = '%s/%s/%s' % (oldmodule, branches, branch)
                 if not isdir(module, self.last_changed):
                     continue
                 brevid = self.latest(module, self.last_changed)
                 if not brevid:
                     self.ui.note(_('ignoring empty branch %s\n') % branch)
                     continue
                 self.ui.note(_('found branch %s at %d\n') %
                              (branch, self.revnum(brevid)))
                 self.heads.append(brevid)
 
         if self.startrev and self.heads:
             if len(self.heads) > 1:
                 raise util.Abort(_('svn: start revision is not supported '
                                    'with more than one branch'))
             revnum = self.revnum(self.heads[0])
             if revnum < self.startrev:
                 raise util.Abort(
                     _('svn: no revision found after start revision %d')
                                  % self.startrev)
 
         return self.heads
 
     def getchanges(self, rev):
         if self._changescache and self._changescache[0] == rev:
             return self._changescache[1]
         self._changescache = None
         (paths, parents) = self.paths[rev]
         if parents:
             files, self.removed, copies = self.expandpaths(rev, paths, parents)
         else:
             # Perform a full checkout on roots
             uuid, module, revnum = revsplit(rev)
             entries = svn.client.ls(self.baseurl + urllib.quote(module),
                                     optrev(revnum), True, self.ctx)
             files = [n for n, e in entries.iteritems()
                      if e.kind == svn.core.svn_node_file]
             copies = {}
             self.removed = set()
 
         files.sort()
         files = zip(files, [rev] * len(files))
 
         # caller caches the result, so free it here to release memory
         del self.paths[rev]
         return (files, copies)
 
     def getchangedfiles(self, rev, i):
         changes = self.getchanges(rev)
         self._changescache = (rev, changes)
         return [f[0] for f in changes[0]]
 
     def getcommit(self, rev):
         if rev not in self.commits:
             uuid, module, revnum = revsplit(rev)
             self.module = module
             self.reparent(module)
             # We assume that:
             # - requests for revisions after "stop" come from the
             # revision graph backward traversal. Cache all of them
             # down to stop, they will be used eventually.
             # - requests for revisions before "stop" come to get
             # isolated branches parents. Just fetch what is needed.
             stop = self.lastrevs.get(module, 0)
             if revnum < stop:
                 stop = revnum + 1
             self._fetch_revisions(revnum, stop)
         commit = self.commits[rev]
         # caller caches the result, so free it here to release memory
         del self.commits[rev]
         return commit
 
     def gettags(self):
         tags = {}
         if self.tags is None:
             return tags
 
         # svn tags are just a convention, project branches left in a
         # 'tags' directory. There is no other relationship than
         # ancestry, which is expensive to discover and makes them hard
         # to update incrementally.  Worse, past revisions may be
         # referenced by tags far away in the future, requiring a deep
         # history traversal on every calculation.  Current code
         # performs a single backward traversal, tracking moves within
         # the tags directory (tag renaming) and recording a new tag
         # everytime a project is copied from outside the tags
         # directory. It also lists deleted tags, this behaviour may
         # change in the future.
         pendings = []
         tagspath = self.tags
         start = svn.ra.get_latest_revnum(self.ra)
         stream = self._getlog([self.tags], start, self.startrev)
         try:
             for entry in stream:
                 origpaths, revnum, author, date, message = entry
                 copies = [(e.copyfrom_path, e.copyfrom_rev, p) for p, e
                           in origpaths.iteritems() if e.copyfrom_path]
                 # Apply moves/copies from more specific to general
                 copies.sort(reverse=True)
 
                 srctagspath = tagspath
                 if copies and copies[-1][2] == tagspath:
                     # Track tags directory moves
                     srctagspath = copies.pop()[0]
 
                 for source, sourcerev, dest in copies:
                     if not dest.startswith(tagspath + '/'):
                         continue
                     for tag in pendings:
                         if tag[0].startswith(dest):
                             tagpath = source + tag[0][len(dest):]
                             tag[:2] = [tagpath, sourcerev]
                             break
                     else:
                         pendings.append([source, sourcerev, dest])
 
                 # Filter out tags with children coming from different
                 # parts of the repository like:
                 # /tags/tag.1 (from /trunk:10)
                 # /tags/tag.1/foo (from /branches/foo:12)
                 # Here/tags/tag.1 discarded as well as its children.
                 # It happens with tools like cvs2svn. Such tags cannot
                 # be represented in mercurial.
                 addeds = dict((p, e.copyfrom_path) for p, e
                               in origpaths.iteritems()
                               if e.action == 'A' and e.copyfrom_path)
                 badroots = set()
                 for destroot in addeds:
                     for source, sourcerev, dest in pendings:
                         if (not dest.startswith(destroot + '/')
                             or source.startswith(addeds[destroot] + '/')):
                             continue
                         badroots.add(destroot)
                         break
 
                 for badroot in badroots:
                     pendings = [p for p in pendings if p[2] != badroot
                                 and not p[2].startswith(badroot + '/')]
 
                 # Tell tag renamings from tag creations
-                remainings = []
+                renamings = []
                 for source, sourcerev, dest in pendings:
                     tagname = dest.split('/')[-1]
                     if source.startswith(srctagspath):
-                        remainings.append([source, sourcerev, tagname])
+                        renamings.append([source, sourcerev, tagname])
                         continue
                     if tagname in tags:
                         # Keep the latest tag value
                         continue
                     # From revision may be fake, get one with changes
                     try:
                         tagid = self.latest(source, sourcerev)
                         if tagid and tagname not in tags:
                             tags[tagname] = tagid
                     except SvnPathNotFound:
                         # It happens when we are following directories
                         # we assumed were copied with their parents
                         # but were really created in the tag
                         # directory.
                         pass
-                pendings = remainings
+                pendings = renamings
                 tagspath = srctagspath
         finally:
             stream.close()
         return tags
 
     def converted(self, rev, destrev):
         if not self.wc:
             return
         if self.convertfp is None:
             self.convertfp = open(os.path.join(self.wc, '.svn', 'hg-shamap'),
                                   'a')
         self.convertfp.write('%s %d\n' % (destrev, self.revnum(rev)))
         self.convertfp.flush()
 
     def revid(self, revnum, module=None):
         return 'svn:%s%s@%s' % (self.uuid, module or self.module, revnum)
 
     def revnum(self, rev):
         return int(rev.split('@')[-1])
 
     def latest(self, path, stop=0):
         """Find the latest revid affecting path, up to stop. It may return
         a revision in a different module, since a branch may be moved without
         a change being reported. Return None if computed module does not
         belong to rootmodule subtree.
         """
         if not path.startswith(self.rootmodule):
             # Requests on foreign branches may be forbidden at server level
             self.ui.debug('ignoring foreign branch %r\n' % path)
             return None
 
         if not stop:
             stop = svn.ra.get_latest_revnum(self.ra)
         try:
             prevmodule = self.reparent('')
             dirent = svn.ra.stat(self.ra, path.strip('/'), stop)
             self.reparent(prevmodule)
         except SubversionException:
             dirent = None
         if not dirent:
             raise SvnPathNotFound(_('%s not found up to revision %d')
                                   % (path, stop))
 
         # stat() gives us the previous revision on this line of
         # development, but it might be in *another module*. Fetch the
         # log and detect renames down to the latest revision.
         stream = self._getlog([path], stop, dirent.created_rev)
         try:
             for entry in stream:
                 paths, revnum, author, date, message = entry
                 if revnum <= dirent.created_rev:
                     break
 
                 for p in paths:
                     if not path.startswith(p) or not paths[p].copyfrom_path:
                         continue
                     newpath = paths[p].copyfrom_path + path[len(p):]
                     self.ui.debug("branch renamed from %s to %s at %d\n" %
                                   (path, newpath, revnum))
                     path = newpath
                     break
         finally:
             stream.close()
 
         if not path.startswith(self.rootmodule):
             self.ui.debug('ignoring foreign branch %r\n' % path)
             return None
         return self.revid(dirent.created_rev, path)
 
     def reparent(self, module):
         """Reparent the svn transport and return the previous parent."""
         if self.prevmodule == module:
             return module
         svnurl = self.baseurl + urllib.quote(module)
         prevmodule = self.prevmodule
         if prevmodule is None:
             prevmodule = ''
         self.ui.debug("reparent to %s\n" % svnurl)
         svn.ra.reparent(self.ra, svnurl)
         self.prevmodule = module
         return prevmodule
 
     def expandpaths(self, rev, paths, parents):
         changed, removed = set(), set()
         copies = {}
 
         new_module, revnum = revsplit(rev)[1:]
         if new_module != self.module:
             self.module = new_module
             self.reparent(self.module)
 
         for i, (path, ent) in enumerate(paths):
             self.ui.progress(_('scanning paths'), i, item=path,
                              total=len(paths))
             entrypath = self.getrelpath(path)
 
             kind = self._checkpath(entrypath, revnum)
             if kind == svn.core.svn_node_file:
                 changed.add(self.recode(entrypath))
                 if not ent.copyfrom_path or not parents:
                     continue
                 # Copy sources not in parent revisions cannot be
                 # represented, ignore their origin for now
                 pmodule, prevnum = revsplit(parents[0])[1:]
                 if ent.copyfrom_rev < prevnum:
                     continue
                 copyfrom_path = self.getrelpath(ent.copyfrom_path, pmodule)
                 if not copyfrom_path:
                     continue
                 self.ui.debug("copied to %s from %s@%s\n" %
                               (entrypath, copyfrom_path, ent.copyfrom_rev))
                 copies[self.recode(entrypath)] = self.recode(copyfrom_path)
             elif kind == 0: # gone, but had better be a deleted *file*
                 self.ui.debug("gone from %s\n" % ent.copyfrom_rev)
                 pmodule, prevnum = revsplit(parents[0])[1:]
                 parentpath = pmodule + "/" + entrypath
                 fromkind = self._checkpath(entrypath, prevnum, pmodule)
 
                 if fromkind == svn.core.svn_node_file:
                     removed.add(self.recode(entrypath))
                 elif fromkind == svn.core.svn_node_dir:
                     oroot = parentpath.strip('/')
                     nroot = path.strip('/')
                     children = self._iterfiles(oroot, prevnum)
                     for childpath in children:
                         childpath = childpath.replace(oroot, nroot)
                         childpath = self.getrelpath("/" + childpath, pmodule)
                         if childpath:
                             removed.add(self.recode(childpath))
                 else:
                     self.ui.debug('unknown path in revision %d: %s\n' % \
                                   (revnum, path))
             elif kind == svn.core.svn_node_dir:
                 if ent.action == 'M':
                     # If the directory just had a prop change,
                     # then we shouldn't need to look for its children.
                     continue
                 if ent.action == 'R' and parents:
                     # If a directory is replacing a file, mark the previous
                     # file as deleted
                     pmodule, prevnum = revsplit(parents[0])[1:]
                     pkind = self._checkpath(entrypath, prevnum, pmodule)
                     if pkind == svn.core.svn_node_file:
                         removed.add(self.recode(entrypath))
                     elif pkind == svn.core.svn_node_dir:
                         # We do not know what files were kept or removed,
                         # mark them all as changed.
                         for childpath in self._iterfiles(pmodule, prevnum):
                             childpath = self.getrelpath("/" + childpath)
                             if childpath:
                                 changed.add(self.recode(childpath))
 
                 for childpath in self._iterfiles(path, revnum):
                     childpath = self.getrelpath("/" + childpath)
                     if childpath:
                         changed.add(self.recode(childpath))
 
                 # Handle directory copies
                 if not ent.copyfrom_path or not parents:
                     continue
                 # Copy sources not in parent revisions cannot be
                 # represented, ignore their origin for now
                 pmodule, prevnum = revsplit(parents[0])[1:]
                 if ent.copyfrom_rev < prevnum:
                     continue
                 copyfrompath = self.getrelpath(ent.copyfrom_path, pmodule)
                 if not copyfrompath:
                     continue
                 self.ui.debug("mark %s came from %s:%d\n"
                               % (path, copyfrompath, ent.copyfrom_rev))
                 children = self._iterfiles(ent.copyfrom_path, ent.copyfrom_rev)
                 for childpath in children:
                     childpath = self.getrelpath("/" + childpath, pmodule)
                     if not childpath:
                         continue
                     copytopath = path + childpath[len(copyfrompath):]
                     copytopath = self.getrelpath(copytopath)
                     copies[self.recode(copytopath)] = self.recode(childpath)
 
         self.ui.progress(_('scanning paths'), None)
         changed.update(removed)
         return (list(changed), removed, copies)
 
     def _fetch_revisions(self, from_revnum, to_revnum):
         if from_revnum < to_revnum:
             from_revnum, to_revnum = to_revnum, from_revnum
 
         self.child_cset = None
 
         def parselogentry(orig_paths, revnum, author, date, message):
             """Return the parsed commit object or None, and True if
             the revision is a branch root.
             """
             self.ui.debug("parsing revision %d (%d changes)\n" %
                           (revnum, len(orig_paths)))
 
             branched = False
             rev = self.revid(revnum)
             # branch log might return entries for a parent we already have
 
             if rev in self.commits or revnum < to_revnum:
                 return None, branched
 
             parents = []
             # check whether this revision is the start of a branch or part
             # of a branch renaming
             orig_paths = sorted(orig_paths.iteritems())
             root_paths = [(p, e) for p, e in orig_paths
                           if self.module.startswith(p)]
             if root_paths:
                 path, ent = root_paths[-1]
                 if ent.copyfrom_path:
                     branched = True
                     newpath = ent.copyfrom_path + self.module[len(path):]
                     # ent.copyfrom_rev may not be the actual last revision
                     previd = self.latest(newpath, ent.copyfrom_rev)
                     if previd is not None:
                         prevmodule, prevnum = revsplit(previd)[1:]
                         if prevnum >= self.startrev:
                             parents = [previd]
                             self.ui.note(
                                 _('found parent of branch %s at %d: %s\n') %
                                 (self.module, prevnum, prevmodule))
                 else:
                     self.ui.debug("no copyfrom path, don't know what to do.\n")
 
             paths = []
             # filter out unrelated paths
             for path, ent in orig_paths:
                 if self.getrelpath(path) is None:
                     continue
                 paths.append((path, ent))
 
             # Example SVN datetime. Includes microseconds.
             # ISO-8601 conformant
             # '2007-01-04T17:35:00.902377Z'
             date = util.parsedate(date[:19] + " UTC", ["%Y-%m-%dT%H:%M:%S"])
 
             log = message and self.recode(message) or ''
             author = author and self.recode(author) or ''
             try:
                 branch = self.module.split("/")[-1]
                 if branch == self.trunkname:
                     branch = None
             except IndexError:
                 branch = None
 
             cset = commit(author=author,
                           date=util.datestr(date),
                           desc=log,
                           parents=parents,
                           branch=branch,
                           rev=rev)
 
             self.commits[rev] = cset
             # The parents list is *shared* among self.paths and the
             # commit object. Both will be updated below.
             self.paths[rev] = (paths, cset.parents)
             if self.child_cset and not self.child_cset.parents:
                 self.child_cset.parents[:] = [rev]
             self.child_cset = cset
             return cset, branched
 
         self.ui.note(_('fetching revision log for "%s" from %d to %d\n') %
                      (self.module, from_revnum, to_revnum))
 
         try:
             firstcset = None
             lastonbranch = False
             stream = self._getlog([self.module], from_revnum, to_revnum)
             try:
                 for entry in stream:
                     paths, revnum, author, date, message = entry
                     if revnum < self.startrev:
                         lastonbranch = True
                         break
                     if not paths:
                         self.ui.debug('revision %d has no entries\n' % revnum)
                         # If we ever leave the loop on an empty
                         # revision, do not try to get a parent branch
                         lastonbranch = lastonbranch or revnum == 0
                         continue
                     cset, lastonbranch = parselogentry(paths, revnum, author,
                                                        date, message)
                     if cset:
                         firstcset = cset
                     if lastonbranch:
                         break
             finally:
                 stream.close()
 
             if not lastonbranch and firstcset and not firstcset.parents:
                 # The first revision of the sequence (the last fetched one)
                 # has invalid parents if not a branch root. Find the parent
                 # revision now, if any.
                 try:
                     firstrevnum = self.revnum(firstcset.rev)
                     if firstrevnum > 1:
                         latest = self.latest(self.module, firstrevnum - 1)
                         if latest:
                             firstcset.parents.append(latest)
                 except SvnPathNotFound:
                     pass
         except SubversionException, (inst, num):
             if num == svn.core.SVN_ERR_FS_NO_SUCH_REVISION:
                 raise util.Abort(_('svn: branch has no revision %s') % to_revnum)
             raise
 
     def getfile(self, file, rev):
         # TODO: ra.get_file transmits the whole file instead of diffs.
         if file in self.removed:
             raise IOError()
         mode = ''
         try:
             new_module, revnum = revsplit(rev)[1:]
             if self.module != new_module:
                 self.module = new_module
                 self.reparent(self.module)
             io = StringIO()
             info = svn.ra.get_file(self.ra, file, revnum, io)
             data = io.getvalue()
             # ra.get_files() seems to keep a reference on the input buffer
             # preventing collection. Release it explicitely.
             io.close()
             if isinstance(info, list):
                 info = info[-1]
             mode = ("svn:executable" in info) and 'x' or ''
             mode = ("svn:special" in info) and 'l' or mode
         except SubversionException, e:
             notfound = (svn.core.SVN_ERR_FS_NOT_FOUND,
                 svn.core.SVN_ERR_RA_DAV_PATH_NOT_FOUND)
             if e.apr_err in notfound: # File not found
                 raise IOError()
             raise
         if mode == 'l':
             link_prefix = "link "
             if data.startswith(link_prefix):
                 data = data[len(link_prefix):]
         return data, mode
 
     def _iterfiles(self, path, revnum):
         """Enumerate all files in path at revnum, recursively."""
         path = path.strip('/')
         pool = Pool()
         rpath = '/'.join([self.baseurl, urllib.quote(path)]).strip('/')
         entries = svn.client.ls(rpath, optrev(revnum), True, self.ctx, pool)
         if path:
             path += '/'
         return ((path + p) for p, e in entries.iteritems()
                 if e.kind == svn.core.svn_node_file)
 
     def getrelpath(self, path, module=None):
         if module is None:
             module = self.module
         # Given the repository url of this wc, say
         #   "http://server/plone/CMFPlone/branches/Plone-2_0-branch"
         # extract the "entry" portion (a relative path) from what
         # svn log --xml says, ie
         #   "/CMFPlone/branches/Plone-2_0-branch/tests/PloneTestCase.py"
         # that is to say "tests/PloneTestCase.py"
         if path.startswith(module):
             relative = path.rstrip('/')[len(module):]
             if relative.startswith('/'):
                 return relative[1:]
             elif relative == '':
                 return relative
 
         # The path is outside our tracked tree...
         self.ui.debug('%r is not under %r, ignoring\n' % (path, module))
         return None
 
     def _checkpath(self, path, revnum, module=None):
         if module is not None:
             prevmodule = self.reparent('')
             path = module + '/' + path
         try:
             # ra.check_path does not like leading slashes very much, it leads
             # to PROPFIND subversion errors
             return svn.ra.check_path(self.ra, path.strip('/'), revnum)
         finally:
             if module is not None:
                 self.reparent(prevmodule)
 
     def _getlog(self, paths, start, end, limit=0, discover_changed_paths=True,
                 strict_node_history=False):
         # Normalize path names, svn >= 1.5 only wants paths relative to
         # supplied URL
         relpaths = []
         for p in paths:
             if not p.startswith('/'):
                 p = self.module + '/' + p
             relpaths.append(p.strip('/'))
         args = [self.baseurl, relpaths, start, end, limit, discover_changed_paths,
                 strict_node_history]
         arg = encodeargs(args)
         hgexe = util.hgexecutable()
         cmd = '%s debugsvnlog' % util.shellquote(hgexe)
         stdin, stdout = util.popen2(util.quotecommand(cmd))
         stdin.write(arg)
         try:
             stdin.close()
         except IOError:
             raise util.Abort(_('Mercurial failed to run itself, check'
                                ' hg executable is in PATH'))
         return logstream(stdout)
 
 pre_revprop_change = '''#!/bin/sh
 
 REPOS="$1"
 REV="$2"
 USER="$3"
 PROPNAME="$4"
 ACTION="$5"
 
 if [ "$ACTION" = "M" -a "$PROPNAME" = "svn:log" ]; then exit 0; fi
 if [ "$ACTION" = "A" -a "$PROPNAME" = "hg:convert-branch" ]; then exit 0; fi
 if [ "$ACTION" = "A" -a "$PROPNAME" = "hg:convert-rev" ]; then exit 0; fi
 
 echo "Changing prohibited revision property" >&2
 exit 1
 '''
 
 class svn_sink(converter_sink, commandline):
     commit_re = re.compile(r'Committed revision (\d+).', re.M)
     uuid_re = re.compile(r'Repository UUID:\s*(\S+)', re.M)
 
     def prerun(self):
         if self.wc:
             os.chdir(self.wc)
 
     def postrun(self):
         if self.wc:
             os.chdir(self.cwd)
 
     def join(self, name):
         return os.path.join(self.wc, '.svn', name)
 
     def revmapfile(self):
         return self.join('hg-shamap')
 
     def authorfile(self):
         return self.join('hg-authormap')
 
     def __init__(self, ui, path):
 
         converter_sink.__init__(self, ui, path)
         commandline.__init__(self, ui, 'svn')
         self.delete = []
         self.setexec = []
         self.delexec = []
         self.copies = []
         self.wc = None
         self.cwd = os.getcwd()
 
         path = os.path.realpath(path)
 
         created = False
         if os.path.isfile(os.path.join(path, '.svn', 'entries')):
             self.wc = path
             self.run0('update')
         else:
             wcpath = os.path.join(os.getcwd(), os.path.basename(path) + '-wc')
 
             if os.path.isdir(os.path.dirname(path)):
                 if not os.path.exists(os.path.join(path, 'db', 'fs-type')):
                     ui.status(_('initializing svn repository %r\n') %
                               os.path.basename(path))
                     commandline(ui, 'svnadmin').run0('create', path)
                     created = path
                 path = util.normpath(path)
                 if not path.startswith('/'):
                     path = '/' + path
                 path = 'file://' + path
 
             ui.status(_('initializing svn working copy %r\n')
                       % os.path.basename(wcpath))
             self.run0('checkout', path, wcpath)
 
             self.wc = wcpath
         self.opener = scmutil.opener(self.wc)
         self.wopener = scmutil.opener(self.wc)
         self.childmap = mapfile(ui, self.join('hg-childmap'))
         self.is_exec = util.checkexec(self.wc) and util.isexec or None
 
         if created:
             hook = os.path.join(created, 'hooks', 'pre-revprop-change')
             fp = open(hook, 'w')
             fp.write(pre_revprop_change)
             fp.close()
             util.setflags(hook, False, True)
 
         output = self.run0('info')
         self.uuid = self.uuid_re.search(output).group(1).strip()
 
     def wjoin(self, *names):
         return os.path.join(self.wc, *names)
 
     def putfile(self, filename, flags, data):
         if 'l' in flags:
             self.wopener.symlink(data, filename)
         else:
             try:
                 if os.path.islink(self.wjoin(filename)):
                     os.unlink(filename)
             except OSError:
                 pass
             self.wopener.write(filename, data)
 
             if self.is_exec:
                 was_exec = self.is_exec(self.wjoin(filename))
             else:
                 # On filesystems not supporting execute-bit, there is no way
                 # to know if it is set but asking subversion. Setting it
                 # systematically is just as expensive and much simpler.
                 was_exec = 'x' not in flags
 
             util.setflags(self.wjoin(filename), False, 'x' in flags)
             if was_exec:
                 if 'x' not in flags:
                     self.delexec.append(filename)
             else:
                 if 'x' in flags:
                     self.setexec.append(filename)
 
     def _copyfile(self, source, dest):
         # SVN's copy command pukes if the destination file exists, but
         # our copyfile method expects to record a copy that has
         # already occurred.  Cross the semantic gap.
         wdest = self.wjoin(dest)
         exists = os.path.lexists(wdest)
         if exists:
             fd, tempname = tempfile.mkstemp(
                 prefix='hg-copy-', dir=os.path.dirname(wdest))
             os.close(fd)
             os.unlink(tempname)
             os.rename(wdest, tempname)
         try:
             self.run0('copy', source, dest)
         finally:
             if exists:
                 try:
                     os.unlink(wdest)
                 except OSError:
                     pass
                 os.rename(tempname, wdest)
 
     def dirs_of(self, files):
         dirs = set()
         for f in files:
             if os.path.isdir(self.wjoin(f)):
                 dirs.add(f)
             for i in strutil.rfindall(f, '/'):
                 dirs.add(f[:i])
         return dirs
 
     def add_dirs(self, files):
         add_dirs = [d for d in sorted(self.dirs_of(files))
                     if not os.path.exists(self.wjoin(d, '.svn', 'entries'))]
         if add_dirs:
             self.xargs(add_dirs, 'add', non_recursive=True, quiet=True)
         return add_dirs
 
     def add_files(self, files):
         if files:
             self.xargs(files, 'add', quiet=True)
         return files
 
     def tidy_dirs(self, names):
         deleted = []
         for d in sorted(self.dirs_of(names), reverse=True):
             wd = self.wjoin(d)
             if os.listdir(wd) == '.svn':
                 self.run0('delete', d)
                 deleted.append(d)
         return deleted
 
     def addchild(self, parent, child):
         self.childmap[parent] = child
 
     def revid(self, rev):
         return u"svn:%s@%s" % (self.uuid, rev)
 
     def putcommit(self, files, copies, parents, commit, source, revmap):
         # Apply changes to working copy
         for f, v in files:
             try:
                 data, mode = source.getfile(f, v)
             except IOError:
                 self.delete.append(f)
             else:
                 self.putfile(f, mode, data)
                 if f in copies:
                     self.copies.append([copies[f], f])
         files = [f[0] for f in files]
 
         for parent in parents:
             try:
                 return self.revid(self.childmap[parent])
             except KeyError:
                 pass
         entries = set(self.delete)
         files = frozenset(files)
         entries.update(self.add_dirs(files.difference(entries)))
         if self.copies:
             for s, d in self.copies:
                 self._copyfile(s, d)
             self.copies = []
         if self.delete:
             self.xargs(self.delete, 'delete')
             self.delete = []
         entries.update(self.add_files(files.difference(entries)))
         entries.update(self.tidy_dirs(entries))
         if self.delexec:
             self.xargs(self.delexec, 'propdel', 'svn:executable')
             self.delexec = []
         if self.setexec:
             self.xargs(self.setexec, 'propset', 'svn:executable', '*')
             self.setexec = []
 
         fd, messagefile = tempfile.mkstemp(prefix='hg-convert-')
         fp = os.fdopen(fd, 'w')
         fp.write(commit.desc)
         fp.close()
         try:
             output = self.run0('commit',
                                username=util.shortuser(commit.author),
                                file=messagefile,
                                encoding='utf-8')
             try:
                 rev = self.commit_re.search(output).group(1)
             except AttributeError:
                 if not files:
                     return parents[0]
                 self.ui.warn(_('unexpected svn output:\n'))
                 self.ui.warn(output)
                 raise util.Abort(_('unable to cope with svn output'))
             if commit.rev:
                 self.run('propset', 'hg:convert-rev', commit.rev,
                          revprop=True, revision=rev)
             if commit.branch and commit.branch != 'default':
                 self.run('propset', 'hg:convert-branch', commit.branch,
                          revprop=True, revision=rev)
             for parent in parents:
                 self.addchild(parent, rev)
             return self.revid(rev)
         finally:
             os.unlink(messagefile)
 
     def puttags(self, tags):
         self.ui.warn(_('writing Subversion tags is not yet implemented\n'))
         return None, None

hgext/convert/transport.py

@@ -1,128 +1,128
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2007 Daniel Holth <[e-mail]>
 # This is a stripped-down version of the original bzr-svn transport.py,
 # Copyright (C) 2006 Jelmer Vernooij <[e-mail]>
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 2 of the License, or
 # (at your option) any later version.
 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 from svn.core import SubversionException, Pool
 import svn.ra
 import svn.client
 import svn.core
 
 # Some older versions of the Python bindings need to be
 # explicitly initialized. But what we want to do probably
 # won't work worth a darn against those libraries anyway!
 svn.ra.initialize()
 
 svn_config = svn.core.svn_config_get_config(None)
 
 
 def _create_auth_baton(pool):
     """Create a Subversion authentication baton. """
     import svn.client
     # Give the client context baton a suite of authentication
     # providers.h
     providers = [
         svn.client.get_simple_provider(pool),
         svn.client.get_username_provider(pool),
         svn.client.get_ssl_client_cert_file_provider(pool),
         svn.client.get_ssl_client_cert_pw_file_provider(pool),
         svn.client.get_ssl_server_trust_file_provider(pool),
         ]
     # Platform-dependant authentication methods
     getprovider = getattr(svn.core, 'svn_auth_get_platform_specific_provider',
                           None)
     if getprovider:
         # Available in svn >= 1.6
         for name in ('gnome_keyring', 'keychain', 'kwallet', 'windows'):
             for type in ('simple', 'ssl_client_cert_pw', 'ssl_server_trust'):
                 p = getprovider(name, type, pool)
                 if p:
                     providers.append(p)
     else:
-        if hasattr(svn.client, 'get_windows_simple_provider'):
+        if util.safehasattr(svn.client, 'get_windows_simple_provider'):
             providers.append(svn.client.get_windows_simple_provider(pool))
 
     return svn.core.svn_auth_open(providers, pool)
 
 class NotBranchError(SubversionException):
     pass
 
 class SvnRaTransport(object):
     """
     Open an ra connection to a Subversion repository.
     """
     def __init__(self, url="", ra=None):
         self.pool = Pool()
         self.svn_url = url
         self.username = ''
         self.password = ''
 
         # Only Subversion 1.4 has reparent()
-        if ra is None or not hasattr(svn.ra, 'reparent'):
+        if ra is None or not util.safehasattr(svn.ra, 'reparent'):
             self.client = svn.client.create_context(self.pool)
             ab = _create_auth_baton(self.pool)
             if False:
                 svn.core.svn_auth_set_parameter(
                     ab, svn.core.SVN_AUTH_PARAM_DEFAULT_USERNAME, self.username)
                 svn.core.svn_auth_set_parameter(
                     ab, svn.core.SVN_AUTH_PARAM_DEFAULT_PASSWORD, self.password)
             self.client.auth_baton = ab
             self.client.config = svn_config
             try:
                 self.ra = svn.client.open_ra_session(
                     self.svn_url.encode('utf8'),
                     self.client, self.pool)
             except SubversionException, (inst, num):
                 if num in (svn.core.SVN_ERR_RA_ILLEGAL_URL,
                            svn.core.SVN_ERR_RA_LOCAL_REPOS_OPEN_FAILED,
                            svn.core.SVN_ERR_BAD_URL):
                     raise NotBranchError(url)
                 raise
         else:
             self.ra = ra
             svn.ra.reparent(self.ra, self.svn_url.encode('utf8'))
 
     class Reporter(object):
         def __init__(self, reporter_data):
             self._reporter, self._baton = reporter_data
 
         def set_path(self, path, revnum, start_empty, lock_token, pool=None):
             svn.ra.reporter2_invoke_set_path(self._reporter, self._baton,
                         path, revnum, start_empty, lock_token, pool)
 
         def delete_path(self, path, pool=None):
             svn.ra.reporter2_invoke_delete_path(self._reporter, self._baton,
                     path, pool)
 
         def link_path(self, path, url, revision, start_empty, lock_token,
                       pool=None):
             svn.ra.reporter2_invoke_link_path(self._reporter, self._baton,
                     path, url, revision, start_empty, lock_token,
                     pool)
 
         def finish_report(self, pool=None):
             svn.ra.reporter2_invoke_finish_report(self._reporter,
                     self._baton, pool)
 
         def abort_report(self, pool=None):
             svn.ra.reporter2_invoke_abort_report(self._reporter,
                     self._baton, pool)
 
     def do_update(self, revnum, path, *args, **kwargs):
         return self.Reporter(svn.ra.do_update(self.ra, revnum, path,
                                               *args, **kwargs))

hgext/eol.py

@@ -1,335 +1,347
 """automatically manage newlines in repository files
 
 This extension allows you to manage the type of line endings (CRLF or
 LF) that are used in the repository and in the local working
 directory. That way you can get CRLF line endings on Windows and LF on
 Unix/Mac, thereby letting everybody use their OS native line endings.
 
 The extension reads its configuration from a versioned ``.hgeol``
 configuration file found in the root of the working copy. The
 ``.hgeol`` file use the same syntax as all other Mercurial
 configuration files. It uses two sections, ``[patterns]`` and
 ``[repository]``.
 
 The ``[patterns]`` section specifies how line endings should be
 converted between the working copy and the repository. The format is
 specified by a file pattern. The first match is used, so put more
 specific patterns first. The available line endings are ``LF``,
 ``CRLF``, and ``BIN``.
 
 Files with the declared format of ``CRLF`` or ``LF`` are always
 checked out and stored in the repository in that format and files
 declared to be binary (``BIN``) are left unchanged. Additionally,
 ``native`` is an alias for checking out in the platform's default line
 ending: ``LF`` on Unix (including Mac OS X) and ``CRLF`` on
 Windows. Note that ``BIN`` (do nothing to line endings) is Mercurial's
 default behaviour; it is only needed if you need to override a later,
 more general pattern.
 
 The optional ``[repository]`` section specifies the line endings to
 use for files stored in the repository. It has a single setting,
 ``native``, which determines the storage line endings for files
 declared as ``native`` in the ``[patterns]`` section. It can be set to
 ``LF`` or ``CRLF``. The default is ``LF``. For example, this means
 that on Windows, files configured as ``native`` (``CRLF`` by default)
 will be converted to ``LF`` when stored in the repository. Files
 declared as ``LF``, ``CRLF``, or ``BIN`` in the ``[patterns]`` section
 are always stored as-is in the repository.
 
 Example versioned ``.hgeol`` file::
 
   [patterns]
   **.py = native
   **.vcproj = CRLF
   **.txt = native
   Makefile = LF
   **.jpg = BIN
 
   [repository]
   native = LF
 
 .. note::
    The rules will first apply when files are touched in the working
    copy, e.g. by updating to null and back to tip to touch all files.
 
-The extension uses an optional ``[eol]`` section in your hgrc file
-(not the ``.hgeol`` file) for settings that control the overall
-behavior. There are two settings:
+The extension uses an optional ``[eol]`` section read from both the
+normal Mercurial configuration files and the ``.hgeol`` file, with the
+latter overriding the former. You can use that section to control the
+overall behavior. There are three settings:
 
 - ``eol.native`` (default ``os.linesep``) can be set to ``LF`` or
   ``CRLF`` to override the default interpretation of ``native`` for
   checkout. This can be used with :hg:`archive` on Unix, say, to
   generate an archive where files have line endings for Windows.
 
 - ``eol.only-consistent`` (default True) can be set to False to make
   the extension convert files with inconsistent EOLs. Inconsistent
   means that there is both ``CRLF`` and ``LF`` present in the file.
   Such files are normally not touched under the assumption that they
   have mixed EOLs on purpose.
 
+- ``eol.fix-trailing-newline`` (default False) can be set to True to
+  ensure that converted files end with a EOL character (either ``\\n``
+  or ``\\r\\n`` as per the configured patterns).
+
 The extension provides ``cleverencode:`` and ``cleverdecode:`` filters
 like the deprecated win32text extension does. This means that you can
 disable win32text and enable eol and your filters will still work. You
 only need to these filters until you have prepared a ``.hgeol`` file.
 
 The ``win32text.forbid*`` hooks provided by the win32text extension
 have been unified into a single hook named ``eol.checkheadshook``. The
 hook will lookup the expected line endings from the ``.hgeol`` file,
 which means you must migrate to a ``.hgeol`` file first before using
 the hook. ``eol.checkheadshook`` only checks heads, intermediate
 invalid revisions will be pushed. To forbid them completely, use the
 ``eol.checkallhook`` hook. These hooks are best used as
 ``pretxnchangegroup`` hooks.
 
 See :hg:`help patterns` for more information about the glob patterns
 used.
 """
 
 from mercurial.i18n import _
 from mercurial import util, config, extensions, match, error
 import re, os
 
 # Matches a lone LF, i.e., one that is not part of CRLF.
 singlelf = re.compile('(^|[^\r])\n')
 # Matches a single EOL which can either be a CRLF where repeated CR
 # are removed or a LF. We do not care about old Machintosh files, so a
 # stray CR is an error.
 eolre = re.compile('\r*\n')
 
 
 def inconsistenteol(data):
     return '\r\n' in data and singlelf.search(data)
 
 def tolf(s, params, ui, **kwargs):
     """Filter to convert to LF EOLs."""
     if util.binary(s):
         return s
     if ui.configbool('eol', 'only-consistent', True) and inconsistenteol(s):
         return s
+    if ui.configbool('eol', 'fix-trailing-newline', False) and s and s[-1] != '\n':
+        s = s + '\n'
     return eolre.sub('\n', s)
 
 def tocrlf(s, params, ui, **kwargs):
     """Filter to convert to CRLF EOLs."""
     if util.binary(s):
         return s
     if ui.configbool('eol', 'only-consistent', True) and inconsistenteol(s):
         return s
+    if ui.configbool('eol', 'fix-trailing-newline', False) and s and s[-1] != '\n':
+        s = s + '\n'
     return eolre.sub('\r\n', s)
 
 def isbinary(s, params):
     """Filter to do nothing with the file."""
     return s
 
 filters = {
     'to-lf': tolf,
     'to-crlf': tocrlf,
     'is-binary': isbinary,
     # The following provide backwards compatibility with win32text
     'cleverencode:': tolf,
     'cleverdecode:': tocrlf
 }
 
 class eolfile(object):
     def __init__(self, ui, root, data):
         self._decode = {'LF': 'to-lf', 'CRLF': 'to-crlf', 'BIN': 'is-binary'}
         self._encode = {'LF': 'to-lf', 'CRLF': 'to-crlf', 'BIN': 'is-binary'}
 
         self.cfg = config.config()
         # Our files should not be touched. The pattern must be
         # inserted first override a '** = native' pattern.
         self.cfg.set('patterns', '.hg*', 'BIN')
         # We can then parse the user's patterns.
         self.cfg.parse('.hgeol', data)
 
         isrepolf = self.cfg.get('repository', 'native') != 'CRLF'
         self._encode['NATIVE'] = isrepolf and 'to-lf' or 'to-crlf'
         iswdlf = ui.config('eol', 'native', os.linesep) in ('LF', '\n')
         self._decode['NATIVE'] = iswdlf and 'to-lf' or 'to-crlf'
 
         include = []
         exclude = []
         for pattern, style in self.cfg.items('patterns'):
             key = style.upper()
             if key == 'BIN':
                 exclude.append(pattern)
             else:
                 include.append(pattern)
         # This will match the files for which we need to care
         # about inconsistent newlines.
         self.match = match.match(root, '', [], include, exclude)
 
-    def setfilters(self, ui):
+    def copytoui(self, ui):
         for pattern, style in self.cfg.items('patterns'):
             key = style.upper()
             try:
                 ui.setconfig('decode', pattern, self._decode[key])
                 ui.setconfig('encode', pattern, self._encode[key])
             except KeyError:
                 ui.warn(_("ignoring unknown EOL style '%s' from %s\n")
                         % (style, self.cfg.source('patterns', pattern)))
+        # eol.only-consistent can be specified in ~/.hgrc or .hgeol
+        for k, v in self.cfg.items('eol'):
+            ui.setconfig('eol', k, v)
 
     def checkrev(self, repo, ctx, files):
         failed = []
         for f in (files or ctx.files()):
             if f not in ctx:
                 continue
             for pattern, style in self.cfg.items('patterns'):
                 if not match.match(repo.root, '', [pattern])(f):
                     continue
                 target = self._encode[style.upper()]
                 data = ctx[f].data()
                 if (target == "to-lf" and "\r\n" in data
                     or target == "to-crlf" and singlelf.search(data)):
                     failed.append((str(ctx), target, f))
                 break
         return failed
 
 def parseeol(ui, repo, nodes):
     try:
         for node in nodes:
             try:
                 if node is None:
                     # Cannot use workingctx.data() since it would load
                     # and cache the filters before we configure them.
                     data = repo.wfile('.hgeol').read()
                 else:
                     data = repo[node]['.hgeol'].data()
                 return eolfile(ui, repo.root, data)
             except (IOError, LookupError):
                 pass
     except error.ParseError, inst:
         ui.warn(_("warning: ignoring .hgeol file due to parse error "
                   "at %s: %s\n") % (inst.args[1], inst.args[0]))
     return None
 
 def _checkhook(ui, repo, node, headsonly):
     # Get revisions to check and touched files at the same time
     files = set()
     revs = set()
     for rev in xrange(repo[node].rev(), len(repo)):
         revs.add(rev)
         if headsonly:
             ctx = repo[rev]
             files.update(ctx.files())
             for pctx in ctx.parents():
                 revs.discard(pctx.rev())
     failed = []
     for rev in revs:
         ctx = repo[rev]
         eol = parseeol(ui, repo, [ctx.node()])
         if eol:
             failed.extend(eol.checkrev(repo, ctx, files))
 
     if failed:
         eols = {'to-lf': 'CRLF', 'to-crlf': 'LF'}
         msgs = []
         for node, target, f in failed:
             msgs.append(_("  %s in %s should not have %s line endings") %
                         (f, node, eols[target]))
         raise util.Abort(_("end-of-line check failed:\n") + "\n".join(msgs))
 
 def checkallhook(ui, repo, node, hooktype, **kwargs):
     """verify that files have expected EOLs"""
     _checkhook(ui, repo, node, False)
 
 def checkheadshook(ui, repo, node, hooktype, **kwargs):
     """verify that files have expected EOLs"""
     _checkhook(ui, repo, node, True)
 
 # "checkheadshook" used to be called "hook"
 hook = checkheadshook
 
 def preupdate(ui, repo, hooktype, parent1, parent2):
     #print "preupdate for %s: %s -> %s" % (repo.root, parent1, parent2)
     repo.loadeol([parent1])
     return False
 
 def uisetup(ui):
     ui.setconfig('hooks', 'preupdate.eol', preupdate)
 
 def extsetup(ui):
     try:
         extensions.find('win32text')
         ui.warn(_("the eol extension is incompatible with the "
                   "win32text extension\n"))
     except KeyError:
         pass
 
 
 def reposetup(ui, repo):
     uisetup(repo.ui)
     #print "reposetup for", repo.root
 
     if not repo.local():
         return
     for name, fn in filters.iteritems():
         repo.adddatafilter(name, fn)
 
     ui.setconfig('patch', 'eol', 'auto')
 
     class eolrepo(repo.__class__):
 
         def loadeol(self, nodes):
             eol = parseeol(self.ui, self, nodes)
             if eol is None:
                 return None
-            eol.setfilters(self.ui)
+            eol.copytoui(self.ui)
             return eol.match
 
         def _hgcleardirstate(self):
             self._eolfile = self.loadeol([None, 'tip'])
             if not self._eolfile:
                 self._eolfile = util.never
                 return
 
             try:
                 cachemtime = os.path.getmtime(self.join("eol.cache"))
             except OSError:
                 cachemtime = 0
 
             try:
                 eolmtime = os.path.getmtime(self.wjoin(".hgeol"))
             except OSError:
                 eolmtime = 0
 
             if eolmtime > cachemtime:
                 ui.debug("eol: detected change in .hgeol\n")
                 wlock = None
                 try:
                     wlock = self.wlock()
                     for f in self.dirstate:
                         if self.dirstate[f] == 'n':
                             # all normal files need to be looked at
                             # again since the new .hgeol file might no
                             # longer match a file it matched before
                             self.dirstate.normallookup(f)
                     # Touch the cache to update mtime.
                     self.opener("eol.cache", "w").close()
                     wlock.release()
                 except error.LockUnavailable:
                     # If we cannot lock the repository and clear the
                     # dirstate, then a commit might not see all files
                     # as modified. But if we cannot lock the
                     # repository, then we can also not make a commit,
                     # so ignore the error.
                     pass
 
         def commitctx(self, ctx, error=False):
             for f in sorted(ctx.added() + ctx.modified()):
                 if not self._eolfile(f):
                     continue
                 try:
                     data = ctx[f].data()
                 except IOError:
                     continue
                 if util.binary(data):
                     # We should not abort here, since the user should
                     # be able to say "** = native" to automatically
                     # have all non-binary files taken care of.
                     continue
                 if inconsistenteol(data):
                     raise util.Abort(_("inconsistent newline style "
                                        "in %s\n" % f))
             return super(eolrepo, self).commitctx(ctx, error)
     repo.__class__ = eolrepo
     repo._hgcleardirstate()

hgext/inotify/__init__.py

@@ -1,89 +1,90
 # __init__.py - inotify-based status acceleration for Linux
 #
 # Copyright 2006, 2007, 2008 Bryan O'Sullivan <[e-mail]>
 # Copyright 2007, 2008 Brendan Cully <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''accelerate status report using Linux's inotify service'''
 
 # todo: socket permissions
 
 from mercurial.i18n import _
+from mercurial import util
 import server
 from client import client, QueryFailed
 
 def serve(ui, repo, **opts):
     '''start an inotify server for this repository'''
     server.start(ui, repo.dirstate, repo.root, opts)
 
 def debuginotify(ui, repo, **opts):
     '''debugging information for inotify extension
 
     Prints the list of directories being watched by the inotify server.
     '''
     cli = client(ui, repo)
     response = cli.debugquery()
 
     ui.write(_('directories being watched:\n'))
     for path in response:
         ui.write(('  %s/\n') % path)
 
 def reposetup(ui, repo):
-    if not hasattr(repo, 'dirstate'):
+    if not util.safehasattr(repo, 'dirstate'):
         return
 
     class inotifydirstate(repo.dirstate.__class__):
 
         # We'll set this to false after an unsuccessful attempt so that
         # next calls of status() within the same instance don't try again
         # to start an inotify server if it won't start.
         _inotifyon = True
 
         def status(self, match, subrepos, ignored, clean, unknown):
             files = match.files()
             if '.' in files:
                 files = []
             if self._inotifyon and not ignored and not subrepos and not self._dirty:
                 cli = client(ui, repo)
                 try:
                     result = cli.statusquery(files, match, False,
                                             clean, unknown)
                 except QueryFailed, instr:
                     ui.debug(str(instr))
                     # don't retry within the same hg instance
                     inotifydirstate._inotifyon = False
                     pass
                 else:
                     if ui.config('inotify', 'debug'):
                         r2 = super(inotifydirstate, self).status(
                             match, [], False, clean, unknown)
                         for c, a, b in zip('LMARDUIC', result, r2):
                             for f in a:
                                 if f not in b:
                                     ui.warn('*** inotify: %s +%s\n' % (c, f))
                             for f in b:
                                 if f not in a:
                                     ui.warn('*** inotify: %s -%s\n' % (c, f))
                         result = r2
                     return result
             return super(inotifydirstate, self).status(
                 match, subrepos, ignored, clean, unknown)
 
     repo.dirstate.__class__ = inotifydirstate
 
 cmdtable = {
     'debuginotify':
         (debuginotify, [], ('hg debuginotify')),
     '^inserve':
         (serve,
          [('d', 'daemon', None, _('run server in background')),
           ('', 'daemon-pipefds', '',
            _('used internally by daemon mode'), _('NUM')),
           ('t', 'idle-timeout', '',
            _('minutes to sit idle before exiting'), _('NUM')),
           ('', 'pid-file', '',
            _('name of file to write process ID to'), _('FILE'))],
          _('hg inserve [OPTION]...')),
     }

hgext/keyword.py

@@ -1,693 +1,701
 # keyword.py - $Keyword$ expansion for Mercurial
 #
 # Copyright 2007-2010 Christian Ebert <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 #
 # $Id$
 #
 # Keyword expansion hack against the grain of a DSCM
 #
 # There are many good reasons why this is not needed in a distributed
 # SCM, still it may be useful in very small projects based on single
 # files (like LaTeX packages), that are mostly addressed to an
 # audience not running a version control system.
 #
 # For in-depth discussion refer to
 # <http://mercurial.selenic.com/wiki/KeywordPlan>.
 #
 # Keyword expansion is based on Mercurial's changeset template mappings.
 #
 # Binary files are not touched.
 #
 # Files to act upon/ignore are specified in the [keyword] section.
 # Customized keyword template mappings in the [keywordmaps] section.
 #
 # Run "hg help keyword" and "hg kwdemo" to get info on configuration.
 
 '''expand keywords in tracked files
 
 This extension expands RCS/CVS-like or self-customized $Keywords$ in
 tracked text files selected by your configuration.
 
 Keywords are only expanded in local repositories and not stored in the
 change history. The mechanism can be regarded as a convenience for the
 current user or for archive distribution.
 
 Keywords expand to the changeset data pertaining to the latest change
 relative to the working directory parent of each file.
 
 Configuration is done in the [keyword], [keywordset] and [keywordmaps]
 sections of hgrc files.
 
 Example::
 
     [keyword]
     # expand keywords in every python file except those matching "x*"
     **.py =
     x*    = ignore
 
     [keywordset]
     # prefer svn- over cvs-like default keywordmaps
     svn = True
 
 .. note::
    The more specific you are in your filename patterns the less you
    lose speed in huge repositories.
 
 For [keywordmaps] template mapping and expansion demonstration and
 control run :hg:`kwdemo`. See :hg:`help templates` for a list of
 available templates and filters.
 
 Three additional date template filters are provided:
 
 :``utcdate``:    "2006/09/18 15:13:13"
 :``svnutcdate``: "2006-09-18 15:13:13Z"
 :``svnisodate``: "2006-09-18 08:13:13 -700 (Mon, 18 Sep 2006)"
 
 The default template mappings (view with :hg:`kwdemo -d`) can be
 replaced with customized keywords and templates. Again, run
 :hg:`kwdemo` to control the results of your configuration changes.
 
 Before changing/disabling active keywords, you must run :hg:`kwshrink`
 to avoid storing expanded keywords in the change history.
 
 To force expansion after enabling it, or a configuration change, run
 :hg:`kwexpand`.
 
 Expansions spanning more than one line and incremental expansions,
 like CVS' $Log$, are not supported. A keyword template map "Log =
 {desc}" expands to the first line of the changeset description.
 '''
 
 from mercurial import commands, context, cmdutil, dispatch, filelog, extensions
 from mercurial import localrepo, match, patch, templatefilters, templater, util
 from mercurial import scmutil
 from mercurial.hgweb import webcommands
 from mercurial.i18n import _
 import os, re, shutil, tempfile
 
 commands.optionalrepo += ' kwdemo'
 
 cmdtable = {}
 command = cmdutil.command(cmdtable)
 
 # hg commands that do not act on keywords
 nokwcommands = ('add addremove annotate bundle export grep incoming init log'
                 ' outgoing push tip verify convert email glog')
 
 # hg commands that trigger expansion only when writing to working dir,
 # not when reading filelog, and unexpand when reading from working dir
 restricted = 'merge kwexpand kwshrink record qrecord resolve transplant'
 
 # names of extensions using dorecord
 recordextensions = 'record'
 
 colortable = {
     'kwfiles.enabled': 'green bold',
     'kwfiles.deleted': 'cyan bold underline',
     'kwfiles.enabledunknown': 'green',
     'kwfiles.ignored': 'bold',
     'kwfiles.ignoredunknown': 'none'
 }
 
 # date like in cvs' $Date
 def utcdate(text):
     ''':utcdate: Date. Returns a UTC-date in this format: "2009/08/18 11:00:13".
     '''
     return util.datestr((text[0], 0), '%Y/%m/%d %H:%M:%S')
 # date like in svn's $Date
 def svnisodate(text):
     ''':svnisodate: Date. Returns a date in this format: "2009-08-18 13:00:13
     +0200 (Tue, 18 Aug 2009)".
     '''
     return util.datestr(text, '%Y-%m-%d %H:%M:%S %1%2 (%a, %d %b %Y)')
 # date like in svn's $Id
 def svnutcdate(text):
     ''':svnutcdate: Date. Returns a UTC-date in this format: "2009-08-18
     11:00:13Z".
     '''
     return util.datestr((text[0], 0), '%Y-%m-%d %H:%M:%SZ')
 
 templatefilters.filters.update({'utcdate': utcdate,
                                 'svnisodate': svnisodate,
                                 'svnutcdate': svnutcdate})
 
 # make keyword tools accessible
 kwtools = {'templater': None, 'hgcmd': ''}
 
 def _defaultkwmaps(ui):
     '''Returns default keywordmaps according to keywordset configuration.'''
     templates = {
         'Revision': '{node|short}',
         'Author': '{author|user}',
     }
     kwsets = ({
         'Date': '{date|utcdate}',
         'RCSfile': '{file|basename},v',
         'RCSFile': '{file|basename},v', # kept for backwards compatibility
                                         # with hg-keyword
         'Source': '{root}/{file},v',
         'Id': '{file|basename},v {node|short} {date|utcdate} {author|user}',
         'Header': '{root}/{file},v {node|short} {date|utcdate} {author|user}',
     }, {
         'Date': '{date|svnisodate}',
         'Id': '{file|basename},v {node|short} {date|svnutcdate} {author|user}',
         'LastChangedRevision': '{node|short}',
         'LastChangedBy': '{author|user}',
         'LastChangedDate': '{date|svnisodate}',
     })
     templates.update(kwsets[ui.configbool('keywordset', 'svn')])
     return templates
 
 def _shrinktext(text, subfunc):
     '''Helper for keyword expansion removal in text.
     Depending on subfunc also returns number of substitutions.'''
     return subfunc(r'$\1$', text)
 
 def _preselect(wstatus, changed):
     '''Retrieves modfied and added files from a working directory state
     and returns the subset of each contained in given changed files
     retrieved from a change context.'''
     modified, added = wstatus[:2]
     modified = [f for f in modified if f in changed]
     added = [f for f in added if f in changed]
     return modified, added
 
 
 class kwtemplater(object):
     '''
     Sets up keyword templates, corresponding keyword regex, and
     provides keyword substitution functions.
     '''
 
     def __init__(self, ui, repo, inc, exc):
         self.ui = ui
         self.repo = repo
         self.match = match.match(repo.root, '', [], inc, exc)
         self.restrict = kwtools['hgcmd'] in restricted.split()
         self.record = False
 
         kwmaps = self.ui.configitems('keywordmaps')
         if kwmaps: # override default templates
             self.templates = dict((k, templater.parsestring(v, False))
                                   for k, v in kwmaps)
         else:
             self.templates = _defaultkwmaps(self.ui)
 
     @util.propertycache
     def escape(self):
         '''Returns bar-separated and escaped keywords.'''
         return '|'.join(map(re.escape, self.templates.keys()))
 
     @util.propertycache
     def rekw(self):
         '''Returns regex for unexpanded keywords.'''
         return re.compile(r'\$(%s)\$' % self.escape)
 
     @util.propertycache
     def rekwexp(self):
         '''Returns regex for expanded keywords.'''
         return re.compile(r'\$(%s): [^$\n\r]*? \$' % self.escape)
 
     def substitute(self, data, path, ctx, subfunc):
         '''Replaces keywords in data with expanded template.'''
         def kwsub(mobj):
             kw = mobj.group(1)
             ct = cmdutil.changeset_templater(self.ui, self.repo,
                                              False, None, '', False)
             ct.use_template(self.templates[kw])
             self.ui.pushbuffer()
             ct.show(ctx, root=self.repo.root, file=path)
             ekw = templatefilters.firstline(self.ui.popbuffer())
             return '$%s: %s $' % (kw, ekw)
         return subfunc(kwsub, data)
 
     def linkctx(self, path, fileid):
         '''Similar to filelog.linkrev, but returns a changectx.'''
         return self.repo.filectx(path, fileid=fileid).changectx()
 
     def expand(self, path, node, data):
         '''Returns data with keywords expanded.'''
         if not self.restrict and self.match(path) and not util.binary(data):
             ctx = self.linkctx(path, node)
             return self.substitute(data, path, ctx, self.rekw.sub)
         return data
 
     def iskwfile(self, cand, ctx):
         '''Returns subset of candidates which are configured for keyword
         expansion are not symbolic links.'''
         return [f for f in cand if self.match(f) and not 'l' in ctx.flags(f)]
 
     def overwrite(self, ctx, candidates, lookup, expand, rekw=False):
         '''Overwrites selected files expanding/shrinking keywords.'''
         if self.restrict or lookup or self.record: # exclude kw_copy
             candidates = self.iskwfile(candidates, ctx)
         if not candidates:
             return
         kwcmd = self.restrict and lookup # kwexpand/kwshrink
         if self.restrict or expand and lookup:
             mf = ctx.manifest()
-        lctx = ctx
-        re_kw = (self.restrict or rekw) and self.rekw or self.rekwexp
-        msg = (expand and _('overwriting %s expanding keywords\n')
-               or _('overwriting %s shrinking keywords\n'))
+        if self.restrict or rekw:
+            re_kw = self.rekw
+        else:
+            re_kw = self.rekwexp
+        if expand:
+            msg = _('overwriting %s expanding keywords\n')
+        else:
+            msg = _('overwriting %s shrinking keywords\n')
         for f in candidates:
             if self.restrict:
                 data = self.repo.file(f).read(mf[f])
             else:
                 data = self.repo.wread(f)
             if util.binary(data):
                 continue
             if expand:
                 if lookup:
-                    lctx = self.linkctx(f, mf[f])
-                data, found = self.substitute(data, f, lctx, re_kw.subn)
+                    ctx = self.linkctx(f, mf[f])
+                data, found = self.substitute(data, f, ctx, re_kw.subn)
             elif self.restrict:
                 found = re_kw.search(data)
             else:
                 data, found = _shrinktext(data, re_kw.subn)
             if found:
                 self.ui.note(msg % f)
-                fpath = self.repo.wjoin(f)
-                mode = os.lstat(fpath).st_mode
-                self.repo.wwrite(f, data, ctx.flags(f))
-                os.chmod(fpath, mode)
+                fp = self.repo.wopener(f, "wb", atomictemp=True)
+                fp.write(data)
+                fp.close()
                 if kwcmd:
                     self.repo.dirstate.normal(f)
                 elif self.record:
                     self.repo.dirstate.normallookup(f)
 
     def shrink(self, fname, text):
         '''Returns text with all keyword substitutions removed.'''
         if self.match(fname) and not util.binary(text):
             return _shrinktext(text, self.rekwexp.sub)
         return text
 
     def shrinklines(self, fname, lines):
         '''Returns lines with keyword substitutions removed.'''
         if self.match(fname):
             text = ''.join(lines)
             if not util.binary(text):
                 return _shrinktext(text, self.rekwexp.sub).splitlines(True)
         return lines
 
     def wread(self, fname, data):
         '''If in restricted mode returns data read from wdir with
         keyword substitutions removed.'''
-        return self.restrict and self.shrink(fname, data) or data
+        if self.restrict:
+            return self.shrink(fname, data)
+        return data
 
 class kwfilelog(filelog.filelog):
     '''
     Subclass of filelog to hook into its read, add, cmp methods.
     Keywords are "stored" unexpanded, and processed on reading.
     '''
     def __init__(self, opener, kwt, path):
         super(kwfilelog, self).__init__(opener, path)
         self.kwt = kwt
         self.path = path
 
     def read(self, node):
         '''Expands keywords when reading filelog.'''
         data = super(kwfilelog, self).read(node)
         if self.renamed(node):
             return data
         return self.kwt.expand(self.path, node, data)
 
     def add(self, text, meta, tr, link, p1=None, p2=None):
         '''Removes keyword substitutions when adding to filelog.'''
         text = self.kwt.shrink(self.path, text)
         return super(kwfilelog, self).add(text, meta, tr, link, p1, p2)
 
     def cmp(self, node, text):
         '''Removes keyword substitutions for comparison.'''
         text = self.kwt.shrink(self.path, text)
         return super(kwfilelog, self).cmp(node, text)
 
-def _status(ui, repo, kwt, *pats, **opts):
+def _status(ui, repo, wctx, kwt, *pats, **opts):
     '''Bails out if [keyword] configuration is not active.
     Returns status of working directory.'''
     if kwt:
-        return repo.status(match=scmutil.match(repo[None], pats, opts), clean=True,
+        return repo.status(match=scmutil.match(wctx, pats, opts), clean=True,
                            unknown=opts.get('unknown') or opts.get('all'))
     if ui.configitems('keyword'):
         raise util.Abort(_('[keyword] patterns cannot match'))
     raise util.Abort(_('no [keyword] patterns configured'))
 
 def _kwfwrite(ui, repo, expand, *pats, **opts):
     '''Selects files and passes them to kwtemplater.overwrite.'''
     wctx = repo[None]
     if len(wctx.parents()) > 1:
         raise util.Abort(_('outstanding uncommitted merge'))
     kwt = kwtools['templater']
     wlock = repo.wlock()
     try:
-        status = _status(ui, repo, kwt, *pats, **opts)
+        status = _status(ui, repo, wctx, kwt, *pats, **opts)
         modified, added, removed, deleted, unknown, ignored, clean = status
         if modified or added or removed or deleted:
             raise util.Abort(_('outstanding uncommitted changes'))
         kwt.overwrite(wctx, clean, True, expand)
     finally:
         wlock.release()
 
 @command('kwdemo',
          [('d', 'default', None, _('show default keyword template maps')),
           ('f', 'rcfile', '',
            _('read maps from rcfile'), _('FILE'))],
          _('hg kwdemo [-d] [-f RCFILE] [TEMPLATEMAP]...'))
 def demo(ui, repo, *args, **opts):
     '''print [keywordmaps] configuration and an expansion example
 
     Show current, custom, or default keyword template maps and their
     expansions.
 
     Extend the current configuration by specifying maps as arguments
     and using -f/--rcfile to source an external hgrc file.
 
     Use -d/--default to disable current configuration.
 
     See :hg:`help templates` for information on templates and filters.
     '''
     def demoitems(section, items):
         ui.write('[%s]\n' % section)
         for k, v in sorted(items):
             ui.write('%s = %s\n' % (k, v))
 
     fn = 'demo.txt'
     tmpdir = tempfile.mkdtemp('', 'kwdemo.')
     ui.note(_('creating temporary repository at %s\n') % tmpdir)
     repo = localrepo.localrepository(ui, tmpdir, True)
     ui.setconfig('keyword', fn, '')
     svn = ui.configbool('keywordset', 'svn')
     # explicitly set keywordset for demo output
     ui.setconfig('keywordset', 'svn', svn)
 
     uikwmaps = ui.configitems('keywordmaps')
     if args or opts.get('rcfile'):
         ui.status(_('\n\tconfiguration using custom keyword template maps\n'))
         if uikwmaps:
             ui.status(_('\textending current template maps\n'))
         if opts.get('default') or not uikwmaps:
             if svn:
                 ui.status(_('\toverriding default svn keywordset\n'))
             else:
                 ui.status(_('\toverriding default cvs keywordset\n'))
         if opts.get('rcfile'):
             ui.readconfig(opts.get('rcfile'))
         if args:
             # simulate hgrc parsing
             rcmaps = ['[keywordmaps]\n'] + [a + '\n' for a in args]
             fp = repo.opener('hgrc', 'w')
             fp.writelines(rcmaps)
             fp.close()
             ui.readconfig(repo.join('hgrc'))
         kwmaps = dict(ui.configitems('keywordmaps'))
     elif opts.get('default'):
         if svn:
             ui.status(_('\n\tconfiguration using default svn keywordset\n'))
         else:
             ui.status(_('\n\tconfiguration using default cvs keywordset\n'))
         kwmaps = _defaultkwmaps(ui)
         if uikwmaps:
             ui.status(_('\tdisabling current template maps\n'))
             for k, v in kwmaps.iteritems():
                 ui.setconfig('keywordmaps', k, v)
     else:
         ui.status(_('\n\tconfiguration using current keyword template maps\n'))
-        kwmaps = dict(uikwmaps) or _defaultkwmaps(ui)
+        if uikwmaps:
+            kwmaps = dict(uikwmaps)
+        else:
+            kwmaps = _defaultkwmaps(ui)
 
     uisetup(ui)
     reposetup(ui, repo)
     ui.write('[extensions]\nkeyword =\n')
     demoitems('keyword', ui.configitems('keyword'))
     demoitems('keywordset', ui.configitems('keywordset'))
     demoitems('keywordmaps', kwmaps.iteritems())
     keywords = '$' + '$\n$'.join(sorted(kwmaps.keys())) + '$\n'
     repo.wopener.write(fn, keywords)
     repo[None].add([fn])
     ui.note(_('\nkeywords written to %s:\n') % fn)
     ui.note(keywords)
     repo.dirstate.setbranch('demobranch')
     for name, cmd in ui.configitems('hooks'):
         if name.split('.', 1)[0].find('commit') > -1:
             repo.ui.setconfig('hooks', name, '')
     msg = _('hg keyword configuration and expansion example')
     ui.note("hg ci -m '%s'\n" % msg)
     repo.commit(text=msg)
     ui.status(_('\n\tkeywords expanded\n'))
     ui.write(repo.wread(fn))
     shutil.rmtree(tmpdir, ignore_errors=True)
 
 @command('kwexpand', commands.walkopts, _('hg kwexpand [OPTION]... [FILE]...'))
 def expand(ui, repo, *pats, **opts):
     '''expand keywords in the working directory
 
     Run after (re)enabling keyword expansion.
 
     kwexpand refuses to run if given files contain local changes.
     '''
     # 3rd argument sets expansion to True
     _kwfwrite(ui, repo, True, *pats, **opts)
 
 @command('kwfiles',
          [('A', 'all', None, _('show keyword status flags of all files')),
           ('i', 'ignore', None, _('show files excluded from expansion')),
           ('u', 'unknown', None, _('only show unknown (not tracked) files')),
          ] + commands.walkopts,
          _('hg kwfiles [OPTION]... [FILE]...'))
 def files(ui, repo, *pats, **opts):
     '''show files configured for keyword expansion
 
     List which files in the working directory are matched by the
     [keyword] configuration patterns.
 
     Useful to prevent inadvertent keyword expansion and to speed up
     execution by including only files that are actual candidates for
     expansion.
 
     See :hg:`help keyword` on how to construct patterns both for
     inclusion and exclusion of files.
 
     With -A/--all and -v/--verbose the codes used to show the status
     of files are::
 
       K = keyword expansion candidate
       k = keyword expansion candidate (not tracked)
       I = ignored
       i = ignored (not tracked)
     '''
     kwt = kwtools['templater']
-    status = _status(ui, repo, kwt, *pats, **opts)
+    wctx = repo[None]
+    status = _status(ui, repo, wctx, kwt, *pats, **opts)
     cwd = pats and repo.getcwd() or ''
     modified, added, removed, deleted, unknown, ignored, clean = status
     files = []
     if not opts.get('unknown') or opts.get('all'):
         files = sorted(modified + added + clean)
-    wctx = repo[None]
     kwfiles = kwt.iskwfile(files, wctx)
     kwdeleted = kwt.iskwfile(deleted, wctx)
     kwunknown = kwt.iskwfile(unknown, wctx)
     if not opts.get('ignore') or opts.get('all'):
         showfiles = kwfiles, kwdeleted, kwunknown
     else:
         showfiles = [], [], []
     if opts.get('all') or opts.get('ignore'):
         showfiles += ([f for f in files if f not in kwfiles],
                       [f for f in unknown if f not in kwunknown])
     kwlabels = 'enabled deleted enabledunknown ignored ignoredunknown'.split()
     kwstates = zip('K!kIi', showfiles, kwlabels)
     for char, filenames, kwstate in kwstates:
         fmt = (opts.get('all') or ui.verbose) and '%s %%s\n' % char or '%s\n'
         for f in filenames:
             ui.write(fmt % repo.pathto(f, cwd), label='kwfiles.' + kwstate)
 
 @command('kwshrink', commands.walkopts, _('hg kwshrink [OPTION]... [FILE]...'))
 def shrink(ui, repo, *pats, **opts):
     '''revert expanded keywords in the working directory
 
     Must be run before changing/disabling active keywords.
 
     kwshrink refuses to run if given files contain local changes.
     '''
     # 3rd argument sets expansion to False
     _kwfwrite(ui, repo, False, *pats, **opts)
 
 
 def uisetup(ui):
     ''' Monkeypatches dispatch._parse to retrieve user command.'''
 
     def kwdispatch_parse(orig, ui, args):
         '''Monkeypatch dispatch._parse to obtain running hg command.'''
         cmd, func, args, options, cmdoptions = orig(ui, args)
         kwtools['hgcmd'] = cmd
         return cmd, func, args, options, cmdoptions
 
     extensions.wrapfunction(dispatch, '_parse', kwdispatch_parse)
 
 def reposetup(ui, repo):
     '''Sets up repo as kwrepo for keyword substitution.
     Overrides file method to return kwfilelog instead of filelog
     if file matches user configuration.
     Wraps commit to overwrite configured files with updated
     keyword substitutions.
     Monkeypatches patch and webcommands.'''
 
     try:
         if (not repo.local() or kwtools['hgcmd'] in nokwcommands.split()
             or '.hg' in util.splitpath(repo.root)
             or repo._url.startswith('bundle:')):
             return
     except AttributeError:
         pass
 
     inc, exc = [], ['.hg*']
     for pat, opt in ui.configitems('keyword'):
         if opt != 'ignore':
             inc.append(pat)
         else:
             exc.append(pat)
     if not inc:
         return
 
     kwtools['templater'] = kwt = kwtemplater(ui, repo, inc, exc)
 
     class kwrepo(repo.__class__):
         def file(self, f):
             if f[0] == '/':
                 f = f[1:]
             return kwfilelog(self.sopener, kwt, f)
 
         def wread(self, filename):
             data = super(kwrepo, self).wread(filename)
             return kwt.wread(filename, data)
 
         def commit(self, *args, **opts):
             # use custom commitctx for user commands
             # other extensions can still wrap repo.commitctx directly
             self.commitctx = self.kwcommitctx
             try:
                 return super(kwrepo, self).commit(*args, **opts)
             finally:
                 del self.commitctx
 
         def kwcommitctx(self, ctx, error=False):
             n = super(kwrepo, self).commitctx(ctx, error)
             # no lock needed, only called from repo.commit() which already locks
             if not kwt.record:
                 restrict = kwt.restrict
                 kwt.restrict = True
                 kwt.overwrite(self[n], sorted(ctx.added() + ctx.modified()),
                               False, True)
                 kwt.restrict = restrict
             return n
 
-        def rollback(self, dryrun=False):
+        def rollback(self, dryrun=False, force=False):
             wlock = self.wlock()
             try:
                 if not dryrun:
                     changed = self['.'].files()
-                ret = super(kwrepo, self).rollback(dryrun)
+                ret = super(kwrepo, self).rollback(dryrun, force)
                 if not dryrun:
                     ctx = self['.']
                     modified, added = _preselect(self[None].status(), changed)
                     kwt.overwrite(ctx, modified, True, True)
                     kwt.overwrite(ctx, added, True, False)
                 return ret
             finally:
                 wlock.release()
 
     # monkeypatches
     def kwpatchfile_init(orig, self, ui, gp, backend, store, eolmode=None):
         '''Monkeypatch/wrap patch.patchfile.__init__ to avoid
         rejects or conflicts due to expanded keywords in working dir.'''
         orig(self, ui, gp, backend, store, eolmode)
         # shrink keywords read from working dir
         self.lines = kwt.shrinklines(self.fname, self.lines)
 
     def kw_diff(orig, repo, node1=None, node2=None, match=None, changes=None,
                 opts=None, prefix=''):
         '''Monkeypatch patch.diff to avoid expansion.'''
         kwt.restrict = True
         return orig(repo, node1, node2, match, changes, opts, prefix)
 
     def kwweb_skip(orig, web, req, tmpl):
         '''Wraps webcommands.x turning off keyword expansion.'''
         kwt.match = util.never
         return orig(web, req, tmpl)
 
     def kw_copy(orig, ui, repo, pats, opts, rename=False):
         '''Wraps cmdutil.copy so that copy/rename destinations do not
         contain expanded keywords.
         Note that the source of a regular file destination may also be a
         symlink:
         hg cp sym x                -> x is symlink
         cp sym x; hg cp -A sym x   -> x is file (maybe expanded keywords)
         For the latter we have to follow the symlink to find out whether its
         target is configured for expansion and we therefore must unexpand the
         keywords in the destination.'''
         orig(ui, repo, pats, opts, rename)
         if opts.get('dry_run'):
             return
         wctx = repo[None]
         cwd = repo.getcwd()
 
         def haskwsource(dest):
             '''Returns true if dest is a regular file and configured for
             expansion or a symlink which points to a file configured for
             expansion. '''
             source = repo.dirstate.copied(dest)
             if 'l' in wctx.flags(source):
                 source = scmutil.canonpath(repo.root, cwd,
                                            os.path.realpath(source))
             return kwt.match(source)
 
         candidates = [f for f in repo.dirstate.copies() if
                       not 'l' in wctx.flags(f) and haskwsource(f)]
         kwt.overwrite(wctx, candidates, False, False)
 
     def kw_dorecord(orig, ui, repo, commitfunc, *pats, **opts):
         '''Wraps record.dorecord expanding keywords after recording.'''
         wlock = repo.wlock()
         try:
             # record returns 0 even when nothing has changed
             # therefore compare nodes before and after
             kwt.record = True
             ctx = repo['.']
             wstatus = repo[None].status()
             ret = orig(ui, repo, commitfunc, *pats, **opts)
             recctx = repo['.']
             if ctx != recctx:
                 modified, added = _preselect(wstatus, recctx.files())
                 kwt.restrict = False
                 kwt.overwrite(recctx, modified, False, True)
                 kwt.overwrite(recctx, added, False, True, True)
                 kwt.restrict = True
             return ret
         finally:
             wlock.release()
 
     def kwfilectx_cmp(orig, self, fctx):
         # keyword affects data size, comparing wdir and filelog size does
         # not make sense
         if (fctx._filerev is None and
             (self._repo._encodefilterpats or
              kwt.match(fctx.path()) and not 'l' in fctx.flags()) or
             self.size() == fctx.size()):
             return self._filelog.cmp(self._filenode, fctx.data())
         return True
 
     extensions.wrapfunction(context.filectx, 'cmp', kwfilectx_cmp)
     extensions.wrapfunction(patch.patchfile, '__init__', kwpatchfile_init)
     extensions.wrapfunction(patch, 'diff', kw_diff)
     extensions.wrapfunction(cmdutil, 'copy', kw_copy)
     for c in 'annotate changeset rev filediff diff'.split():
         extensions.wrapfunction(webcommands, c, kwweb_skip)
     for name in recordextensions.split():
         try:
             record = extensions.find(name)
             extensions.wrapfunction(record, 'dorecord', kw_dorecord)
         except KeyError:
             pass
 
     repo.__class__ = kwrepo