largefiles: remove trivial portability wrappers

hgext/largefiles/lfutil.py

@@ -1,443 +1,430
 # Copyright 2009-2010 Gregory P. Ward
 # Copyright 2009-2010 Intelerad Medical Systems Incorporated
 # Copyright 2010-2011 Fog Creek Software
 # Copyright 2010-2011 Unity Technologies
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''largefiles utility code: must not import other modules in this package.'''
 
 import os
 import errno
 import platform
 import shutil
 import stat
 
 from mercurial import dirstate, httpconnection, match as match_, util, scmutil
 from mercurial.i18n import _
 
 shortname = '.hglf'
 shortnameslash = shortname + '/'
 longname = 'largefiles'
 
 
-# -- Portability wrappers ----------------------------------------------
-
-def dirstatewalk(dirstate, matcher, unknown=False, ignored=False):
-    return dirstate.walk(matcher, [], unknown, ignored)
-
-def repoadd(repo, list):
-    add = repo[None].add
-    return add(list)
-
-def repoforget(repo, list):
-    forget = repo[None].forget
-    return forget(list)
-
 # -- Private worker functions ------------------------------------------
 
 def getminsize(ui, assumelfiles, opt, default=10):
     lfsize = opt
     if not lfsize and assumelfiles:
         lfsize = ui.config(longname, 'minsize', default=default)
     if lfsize:
         try:
             lfsize = float(lfsize)
         except ValueError:
             raise util.Abort(_('largefiles: size must be number (not %s)\n')
                              % lfsize)
     if lfsize is None:
         raise util.Abort(_('minimum size for largefiles must be specified'))
     return lfsize
 
 def link(src, dest):
     try:
         util.oslink(src, dest)
     except OSError:
         # if hardlinks fail, fallback on atomic copy
         dst = util.atomictempfile(dest)
         for chunk in util.filechunkiter(open(src, 'rb')):
             dst.write(chunk)
         dst.close()
         os.chmod(dest, os.stat(src).st_mode)
 
 def usercachepath(ui, hash):
     path = ui.configpath(longname, 'usercache', None)
     if path:
         path = os.path.join(path, hash)
     else:
         if os.name == 'nt':
             appdata = os.getenv('LOCALAPPDATA', os.getenv('APPDATA'))
             if appdata:
                 path = os.path.join(appdata, longname, hash)
         elif platform.system() == 'Darwin':
             home = os.getenv('HOME')
             if home:
                 path = os.path.join(home, 'Library', 'Caches',
                                     longname, hash)
         elif os.name == 'posix':
             path = os.getenv('XDG_CACHE_HOME')
             if path:
                 path = os.path.join(path, longname, hash)
             else:
                 home = os.getenv('HOME')
                 if home:
                     path = os.path.join(home, '.cache', longname, hash)
         else:
             raise util.Abort(_('unknown operating system: %s\n') % os.name)
     return path
 
 def inusercache(ui, hash):
     path = usercachepath(ui, hash)
     return path and os.path.exists(path)
 
 def findfile(repo, hash):
     if instore(repo, hash):
         repo.ui.note(_('found %s in store\n') % hash)
         return storepath(repo, hash)
     elif inusercache(repo.ui, hash):
         repo.ui.note(_('found %s in system cache\n') % hash)
         path = storepath(repo, hash)
         util.makedirs(os.path.dirname(path))
         link(usercachepath(repo.ui, hash), path)
         return path
     return None
 
 class largefilesdirstate(dirstate.dirstate):
     def __getitem__(self, key):
         return super(largefilesdirstate, self).__getitem__(unixpath(key))
     def normal(self, f):
         return super(largefilesdirstate, self).normal(unixpath(f))
     def remove(self, f):
         return super(largefilesdirstate, self).remove(unixpath(f))
     def add(self, f):
         return super(largefilesdirstate, self).add(unixpath(f))
     def drop(self, f):
         return super(largefilesdirstate, self).drop(unixpath(f))
     def forget(self, f):
         return super(largefilesdirstate, self).forget(unixpath(f))
     def normallookup(self, f):
         return super(largefilesdirstate, self).normallookup(unixpath(f))
     def _ignore(self):
         return False
 
 def openlfdirstate(ui, repo, create=True):
     '''
     Return a dirstate object that tracks largefiles: i.e. its root is
     the repo root, but it is saved in .hg/largefiles/dirstate.
     '''
     lfstoredir = repo.join(longname)
     opener = scmutil.opener(lfstoredir)
     lfdirstate = largefilesdirstate(opener, ui, repo.root,
                                      repo.dirstate._validate)
 
     # If the largefiles dirstate does not exist, populate and create
     # it. This ensures that we create it on the first meaningful
     # largefiles operation in a new clone.
     if create and not os.path.exists(os.path.join(lfstoredir, 'dirstate')):
         util.makedirs(lfstoredir)
         matcher = getstandinmatcher(repo)
-        for standin in dirstatewalk(repo.dirstate, matcher):
+        for standin in repo.dirstate.walk(matcher, [], False, False):
             lfile = splitstandin(standin)
             hash = readstandin(repo, lfile)
             lfdirstate.normallookup(lfile)
             try:
                 if hash == hashfile(repo.wjoin(lfile)):
                     lfdirstate.normal(lfile)
             except OSError, err:
                 if err.errno != errno.ENOENT:
                     raise
     return lfdirstate
 
 def lfdirstatestatus(lfdirstate, repo, rev):
     match = match_.always(repo.root, repo.getcwd())
     s = lfdirstate.status(match, [], False, False, False)
     unsure, modified, added, removed, missing, unknown, ignored, clean = s
     for lfile in unsure:
         if repo[rev][standin(lfile)].data().strip() != \
                 hashfile(repo.wjoin(lfile)):
             modified.append(lfile)
         else:
             clean.append(lfile)
             lfdirstate.normal(lfile)
     return (modified, added, removed, missing, unknown, ignored, clean)
 
 def listlfiles(repo, rev=None, matcher=None):
     '''return a list of largefiles in the working copy or the
     specified changeset'''
 
     if matcher is None:
         matcher = getstandinmatcher(repo)
 
     # ignore unknown files in working directory
     return [splitstandin(f)
             for f in repo[rev].walk(matcher)
             if rev is not None or repo.dirstate[f] != '?']
 
 def instore(repo, hash):
     return os.path.exists(storepath(repo, hash))
 
 def storepath(repo, hash):
     return repo.join(os.path.join(longname, hash))
 
 def copyfromcache(repo, hash, filename):
     '''Copy the specified largefile from the repo or system cache to
     filename in the repository. Return true on success or false if the
     file was not found in either cache (which should not happened:
     this is meant to be called only after ensuring that the needed
     largefile exists in the cache).'''
     path = findfile(repo, hash)
     if path is None:
         return False
     util.makedirs(os.path.dirname(repo.wjoin(filename)))
     # The write may fail before the file is fully written, but we
     # don't use atomic writes in the working copy.
     shutil.copy(path, repo.wjoin(filename))
     return True
 
 def copytostore(repo, rev, file, uploaded=False):
     hash = readstandin(repo, file, rev)
     if instore(repo, hash):
         return
     copytostoreabsolute(repo, repo.wjoin(file), hash)
 
 def copyalltostore(repo, node):
     '''Copy all largefiles in a given revision to the store'''
 
     ctx = repo[node]
     for filename in ctx.files():
         if isstandin(filename) and filename in ctx.manifest():
             realfile = splitstandin(filename)
             copytostore(repo, ctx.node(), realfile)
 
 
 def copytostoreabsolute(repo, file, hash):
     util.makedirs(os.path.dirname(storepath(repo, hash)))
     if inusercache(repo.ui, hash):
         link(usercachepath(repo.ui, hash), storepath(repo, hash))
     elif not getattr(repo, "_isconverting", False):
         dst = util.atomictempfile(storepath(repo, hash),
                                   createmode=repo.store.createmode)
         for chunk in util.filechunkiter(open(file, 'rb')):
             dst.write(chunk)
         dst.close()
         linktousercache(repo, hash)
 
 def linktousercache(repo, hash):
     path = usercachepath(repo.ui, hash)
     if path:
         util.makedirs(os.path.dirname(path))
         link(storepath(repo, hash), path)
 
 def getstandinmatcher(repo, pats=[], opts={}):
     '''Return a match object that applies pats to the standin directory'''
     standindir = repo.wjoin(shortname)
     if pats:
         # patterns supplied: search standin directory relative to current dir
         cwd = repo.getcwd()
         if os.path.isabs(cwd):
             # cwd is an absolute path for hg -R <reponame>
             # work relative to the repository root in this case
             cwd = ''
         pats = [os.path.join(standindir, cwd, pat) for pat in pats]
     elif os.path.isdir(standindir):
         # no patterns: relative to repo root
         pats = [standindir]
     else:
         # no patterns and no standin dir: return matcher that matches nothing
         return match_.match(repo.root, None, [], exact=True)
 
     # no warnings about missing files or directories
     match = scmutil.match(repo[None], pats, opts)
     match.bad = lambda f, msg: None
     return match
 
 def composestandinmatcher(repo, rmatcher):
     '''Return a matcher that accepts standins corresponding to the
     files accepted by rmatcher. Pass the list of files in the matcher
     as the paths specified by the user.'''
     smatcher = getstandinmatcher(repo, rmatcher.files())
     isstandin = smatcher.matchfn
     def composedmatchfn(f):
         return isstandin(f) and rmatcher.matchfn(splitstandin(f))
     smatcher.matchfn = composedmatchfn
 
     return smatcher
 
 def standin(filename):
     '''Return the repo-relative path to the standin for the specified big
     file.'''
     # Notes:
     # 1) Some callers want an absolute path, but for instance addlargefiles
-    #    needs it repo-relative so it can be passed to repoadd().  So leave
-    #    it up to the caller to use repo.wjoin() to get an absolute path.
+    #    needs it repo-relative so it can be passed to repo[None].add().  So
+    #    leave it up to the caller to use repo.wjoin() to get an absolute path.
     # 2) Join with '/' because that's what dirstate always uses, even on
     #    Windows. Change existing separator to '/' first in case we are
     #    passed filenames from an external source (like the command line).
     return shortnameslash + util.pconvert(filename)
 
 def isstandin(filename):
     '''Return true if filename is a big file standin. filename must be
     in Mercurial's internal form (slash-separated).'''
     return filename.startswith(shortnameslash)
 
 def splitstandin(filename):
     # Split on / because that's what dirstate always uses, even on Windows.
     # Change local separator to / first just in case we are passed filenames
     # from an external source (like the command line).
     bits = util.pconvert(filename).split('/', 1)
     if len(bits) == 2 and bits[0] == shortname:
         return bits[1]
     else:
         return None
 
 def updatestandin(repo, standin):
     file = repo.wjoin(splitstandin(standin))
     if os.path.exists(file):
         hash = hashfile(file)
         executable = getexecutable(file)
         writestandin(repo, standin, hash, executable)
 
 def readstandin(repo, filename, node=None):
     '''read hex hash from standin for filename at given node, or working
     directory if no node is given'''
     return repo[node][standin(filename)].data().strip()
 
 def writestandin(repo, standin, hash, executable):
     '''write hash to <repo.root>/<standin>'''
     writehash(hash, repo.wjoin(standin), executable)
 
 def copyandhash(instream, outfile):
     '''Read bytes from instream (iterable) and write them to outfile,
     computing the SHA-1 hash of the data along the way.  Close outfile
     when done and return the binary hash.'''
     hasher = util.sha1('')
     for data in instream:
         hasher.update(data)
         outfile.write(data)
 
     # Blecch: closing a file that somebody else opened is rude and
     # wrong. But it's so darn convenient and practical! After all,
     # outfile was opened just to copy and hash.
     outfile.close()
 
     return hasher.digest()
 
 def hashrepofile(repo, file):
     return hashfile(repo.wjoin(file))
 
 def hashfile(file):
     if not os.path.exists(file):
         return ''
     hasher = util.sha1('')
     fd = open(file, 'rb')
     for data in blockstream(fd):
         hasher.update(data)
     fd.close()
     return hasher.hexdigest()
 
 class limitreader(object):
     def __init__(self, f, limit):
         self.f = f
         self.limit = limit
 
     def read(self, length):
         if self.limit == 0:
             return ''
         length = length > self.limit and self.limit or length
         self.limit -= length
         return self.f.read(length)
 
     def close(self):
         pass
 
 def blockstream(infile, blocksize=128 * 1024):
     """Generator that yields blocks of data from infile and closes infile."""
     while True:
         data = infile.read(blocksize)
         if not data:
             break
         yield data
     # same blecch as copyandhash() above
     infile.close()
 
 def writehash(hash, filename, executable):
     util.makedirs(os.path.dirname(filename))
     util.writefile(filename, hash + '\n')
     os.chmod(filename, getmode(executable))
 
 def getexecutable(filename):
     mode = os.stat(filename).st_mode
     return ((mode & stat.S_IXUSR) and
             (mode & stat.S_IXGRP) and
             (mode & stat.S_IXOTH))
 
 def getmode(executable):
     if executable:
         return 0755
     else:
         return 0644
 
 def urljoin(first, second, *arg):
     def join(left, right):
         if not left.endswith('/'):
             left += '/'
         if right.startswith('/'):
             right = right[1:]
         return left + right
 
     url = join(first, second)
     for a in arg:
         url = join(url, a)
     return url
 
 def hexsha1(data):
     """hexsha1 returns the hex-encoded sha1 sum of the data in the file-like
     object data"""
     h = util.sha1()
     for chunk in util.filechunkiter(data):
         h.update(chunk)
     return h.hexdigest()
 
 def httpsendfile(ui, filename):
     return httpconnection.httpsendfile(ui, filename, 'rb')
 
 def unixpath(path):
     '''Return a version of path normalized for use with the lfdirstate.'''
     return util.pconvert(os.path.normpath(path))
 
 def islfilesrepo(repo):
     if ('largefiles' in repo.requirements and
             util.any(shortnameslash in f[0] for f in repo.store.datafiles())):
         return True
 
     return util.any(openlfdirstate(repo.ui, repo, False))
 
 class storeprotonotcapable(Exception):
     def __init__(self, storetypes):
         self.storetypes = storetypes
 
 def getcurrentheads(repo):
     branches = repo.branchmap()
     heads = []
     for branch in branches:
         newheads = repo.branchheads(branch)
         heads = heads + newheads
     return heads
 
 def getstandinsstate(repo):
     standins = []
     matcher = getstandinmatcher(repo)
-    for standin in dirstatewalk(repo.dirstate, matcher):
+    for standin in repo.dirstate.walk(matcher, [], False, False):
         lfile = splitstandin(standin)
         standins.append((lfile, readstandin(repo, lfile)))
     return standins
 
 def getlfilestoupdate(oldstandins, newstandins):
     changedstandins = set(oldstandins).symmetric_difference(set(newstandins))
     filelist = []
     for f in changedstandins:
         if f[0] not in filelist:
             filelist.append(f[0])
     return filelist

hgext/largefiles/overrides.py

@@ -1,1141 +1,1141
 # Copyright 2009-2010 Gregory P. Ward
 # Copyright 2009-2010 Intelerad Medical Systems Incorporated
 # Copyright 2010-2011 Fog Creek Software
 # Copyright 2010-2011 Unity Technologies
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''Overridden Mercurial commands and functions for the largefiles extension'''
 
 import os
 import copy
 
 from mercurial import hg, commands, util, cmdutil, scmutil, match as match_, \
     node, archival, error, merge, discovery
 from mercurial.i18n import _
 from mercurial.node import hex
 from hgext import rebase
 
 import lfutil
 import lfcommands
 
 # -- Utility functions: commonly/repeatedly needed functionality ---------------
 
 def installnormalfilesmatchfn(manifest):
     '''overrides scmutil.match so that the matcher it returns will ignore all
     largefiles'''
     oldmatch = None # for the closure
     def overridematch(ctx, pats=[], opts={}, globbed=False,
             default='relpath'):
         match = oldmatch(ctx, pats, opts, globbed, default)
         m = copy.copy(match)
         notlfile = lambda f: not (lfutil.isstandin(f) or lfutil.standin(f) in
                 manifest)
         m._files = filter(notlfile, m._files)
         m._fmap = set(m._files)
         origmatchfn = m.matchfn
         m.matchfn = lambda f: notlfile(f) and origmatchfn(f) or None
         return m
     oldmatch = installmatchfn(overridematch)
 
 def installmatchfn(f):
     oldmatch = scmutil.match
     setattr(f, 'oldmatch', oldmatch)
     scmutil.match = f
     return oldmatch
 
 def restorematchfn():
     '''restores scmutil.match to what it was before installnormalfilesmatchfn
     was called.  no-op if scmutil.match is its original function.
 
     Note that n calls to installnormalfilesmatchfn will require n calls to
     restore matchfn to reverse'''
     scmutil.match = getattr(scmutil.match, 'oldmatch', scmutil.match)
 
 def addlargefiles(ui, repo, *pats, **opts):
     large = opts.pop('large', None)
     lfsize = lfutil.getminsize(
         ui, lfutil.islfilesrepo(repo), opts.pop('lfsize', None))
 
     lfmatcher = None
     if lfutil.islfilesrepo(repo):
         lfpats = ui.configlist(lfutil.longname, 'patterns', default=[])
         if lfpats:
             lfmatcher = match_.match(repo.root, '', list(lfpats))
 
     lfnames = []
     m = scmutil.match(repo[None], pats, opts)
     m.bad = lambda x, y: None
     wctx = repo[None]
     for f in repo.walk(m):
         exact = m.exact(f)
         lfile = lfutil.standin(f) in wctx
         nfile = f in wctx
         exists = lfile or nfile
 
         # Don't warn the user when they attempt to add a normal tracked file.
         # The normal add code will do that for us.
         if exact and exists:
             if lfile:
                 ui.warn(_('%s already a largefile\n') % f)
             continue
 
         if (exact or not exists) and not lfutil.isstandin(f):
             wfile = repo.wjoin(f)
 
             # In case the file was removed previously, but not committed
             # (issue3507)
             if not os.path.exists(wfile):
                 continue
 
             abovemin = (lfsize and
                         os.lstat(wfile).st_size >= lfsize * 1024 * 1024)
             if large or abovemin or (lfmatcher and lfmatcher(f)):
                 lfnames.append(f)
                 if ui.verbose or not exact:
                     ui.status(_('adding %s as a largefile\n') % m.rel(f))
 
     bad = []
     standins = []
 
     # Need to lock, otherwise there could be a race condition between
     # when standins are created and added to the repo.
     wlock = repo.wlock()
     try:
         if not opts.get('dry_run'):
             lfdirstate = lfutil.openlfdirstate(ui, repo)
             for f in lfnames:
                 standinname = lfutil.standin(f)
                 lfutil.writestandin(repo, standinname, hash='',
                     executable=lfutil.getexecutable(repo.wjoin(f)))
                 standins.append(standinname)
                 if lfdirstate[f] == 'r':
                     lfdirstate.normallookup(f)
                 else:
                     lfdirstate.add(f)
             lfdirstate.write()
             bad += [lfutil.splitstandin(f)
-                    for f in lfutil.repoadd(repo, standins)
+                    for f in repo[None].add(standins)
                     if f in m.files()]
     finally:
         wlock.release()
     return bad
 
 def removelargefiles(ui, repo, *pats, **opts):
     after = opts.get('after')
     if not pats and not after:
         raise util.Abort(_('no files specified'))
     m = scmutil.match(repo[None], pats, opts)
     try:
         repo.lfstatus = True
         s = repo.status(match=m, clean=True)
     finally:
         repo.lfstatus = False
     manifest = repo[None].manifest()
     modified, added, deleted, clean = [[f for f in list
                                         if lfutil.standin(f) in manifest]
                                        for list in [s[0], s[1], s[3], s[6]]]
 
     def warn(files, msg):
         for f in files:
             ui.warn(msg % m.rel(f))
         return int(len(files) > 0)
 
     result = 0
 
     if after:
         remove, forget = deleted, []
         result = warn(modified + added + clean,
                       _('not removing %s: file still exists\n'))
     else:
         remove, forget = deleted + clean, []
         result = warn(modified, _('not removing %s: file is modified (use -f'
                                   ' to force removal)\n'))
         result = warn(added, _('not removing %s: file has been marked for add'
                                ' (use forget to undo)\n')) or result
 
     for f in sorted(remove + forget):
         if ui.verbose or not m.exact(f):
             ui.status(_('removing %s\n') % m.rel(f))
 
     # Need to lock because standin files are deleted then removed from the
     # repository and we could race in-between.
     wlock = repo.wlock()
     try:
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         for f in remove:
             if not after:
                 # If this is being called by addremove, notify the user that we
                 # are removing the file.
                 if getattr(repo, "_isaddremove", False):
                     ui.status(_('removing %s\n') % f)
                 if os.path.exists(repo.wjoin(f)):
                     util.unlinkpath(repo.wjoin(f))
             lfdirstate.remove(f)
         lfdirstate.write()
         forget = [lfutil.standin(f) for f in forget]
         remove = [lfutil.standin(f) for f in remove]
-        lfutil.repoforget(repo, forget)
+        repo[None].forget(forget)
         # If this is being called by addremove, let the original addremove
         # function handle this.
         if not getattr(repo, "_isaddremove", False):
             for f in remove:
                 util.unlinkpath(repo.wjoin(f), ignoremissing=True)
         repo[None].forget(remove)
     finally:
         wlock.release()
 
     return result
 
 # For overriding mercurial.hgweb.webcommands so that largefiles will
 # appear at their right place in the manifests.
 def decodepath(orig, path):
     return lfutil.splitstandin(path) or path
 
 # -- Wrappers: modify existing commands --------------------------------
 
 # Add works by going through the files that the user wanted to add and
 # checking if they should be added as largefiles. Then it makes a new
 # matcher which matches only the normal files and runs the original
 # version of add.
 def overrideadd(orig, ui, repo, *pats, **opts):
     normal = opts.pop('normal')
     if normal:
         if opts.get('large'):
             raise util.Abort(_('--normal cannot be used with --large'))
         return orig(ui, repo, *pats, **opts)
     bad = addlargefiles(ui, repo, *pats, **opts)
     installnormalfilesmatchfn(repo[None].manifest())
     result = orig(ui, repo, *pats, **opts)
     restorematchfn()
 
     return (result == 1 or bad) and 1 or 0
 
 def overrideremove(orig, ui, repo, *pats, **opts):
     installnormalfilesmatchfn(repo[None].manifest())
     result = orig(ui, repo, *pats, **opts)
     restorematchfn()
     return removelargefiles(ui, repo, *pats, **opts) or result
 
 def overridestatusfn(orig, repo, rev2, **opts):
     try:
         repo._repo.lfstatus = True
         return orig(repo, rev2, **opts)
     finally:
         repo._repo.lfstatus = False
 
 def overridestatus(orig, ui, repo, *pats, **opts):
     try:
         repo.lfstatus = True
         return orig(ui, repo, *pats, **opts)
     finally:
         repo.lfstatus = False
 
 def overridedirty(orig, repo, ignoreupdate=False):
     try:
         repo._repo.lfstatus = True
         return orig(repo, ignoreupdate)
     finally:
         repo._repo.lfstatus = False
 
 def overridelog(orig, ui, repo, *pats, **opts):
     try:
         repo.lfstatus = True
         return orig(ui, repo, *pats, **opts)
     finally:
         repo.lfstatus = False
 
 def overrideverify(orig, ui, repo, *pats, **opts):
     large = opts.pop('large', False)
     all = opts.pop('lfa', False)
     contents = opts.pop('lfc', False)
 
     result = orig(ui, repo, *pats, **opts)
     if large:
         result = result or lfcommands.verifylfiles(ui, repo, all, contents)
     return result
 
 def overridedebugstate(orig, ui, repo, *pats, **opts):
     large = opts.pop('large', False)
     if large:
         lfcommands.debugdirstate(ui, repo)
     else:
         orig(ui, repo, *pats, **opts)
 
 # Override needs to refresh standins so that update's normal merge
 # will go through properly. Then the other update hook (overriding repo.update)
 # will get the new files. Filemerge is also overridden so that the merge
 # will merge standins correctly.
 def overrideupdate(orig, ui, repo, *pats, **opts):
     lfdirstate = lfutil.openlfdirstate(ui, repo)
     s = lfdirstate.status(match_.always(repo.root, repo.getcwd()), [], False,
         False, False)
     (unsure, modified, added, removed, missing, unknown, ignored, clean) = s
 
     # Need to lock between the standins getting updated and their
     # largefiles getting updated
     wlock = repo.wlock()
     try:
         if opts['check']:
             mod = len(modified) > 0
             for lfile in unsure:
                 standin = lfutil.standin(lfile)
                 if repo['.'][standin].data().strip() != \
                         lfutil.hashfile(repo.wjoin(lfile)):
                     mod = True
                 else:
                     lfdirstate.normal(lfile)
             lfdirstate.write()
             if mod:
                 raise util.Abort(_('uncommitted local changes'))
         # XXX handle removed differently
         if not opts['clean']:
             for lfile in unsure + modified + added:
                 lfutil.updatestandin(repo, lfutil.standin(lfile))
     finally:
         wlock.release()
     return orig(ui, repo, *pats, **opts)
 
 # Before starting the manifest merge, merge.updates will call
 # _checkunknown to check if there are any files in the merged-in
 # changeset that collide with unknown files in the working copy.
 #
 # The largefiles are seen as unknown, so this prevents us from merging
 # in a file 'foo' if we already have a largefile with the same name.
 #
 # The overridden function filters the unknown files by removing any
 # largefiles. This makes the merge proceed and we can then handle this
 # case further in the overridden manifestmerge function below.
 def overridecheckunknownfile(origfn, repo, wctx, mctx, f):
     if lfutil.standin(f) in wctx:
         return False
     return origfn(repo, wctx, mctx, f)
 
 # The manifest merge handles conflicts on the manifest level. We want
 # to handle changes in largefile-ness of files at this level too.
 #
 # The strategy is to run the original manifestmerge and then process
 # the action list it outputs. There are two cases we need to deal with:
 #
 # 1. Normal file in p1, largefile in p2. Here the largefile is
 #    detected via its standin file, which will enter the working copy
 #    with a "get" action. It is not "merge" since the standin is all
 #    Mercurial is concerned with at this level -- the link to the
 #    existing normal file is not relevant here.
 #
 # 2. Largefile in p1, normal file in p2. Here we get a "merge" action
 #    since the largefile will be present in the working copy and
 #    different from the normal file in p2. Mercurial therefore
 #    triggers a merge action.
 #
 # In both cases, we prompt the user and emit new actions to either
 # remove the standin (if the normal file was kept) or to remove the
 # normal file and get the standin (if the largefile was kept). The
 # default prompt answer is to use the largefile version since it was
 # presumably changed on purpose.
 #
 # Finally, the merge.applyupdates function will then take care of
 # writing the files into the working copy and lfcommands.updatelfiles
 # will update the largefiles.
 def overridemanifestmerge(origfn, repo, p1, p2, pa, overwrite, partial):
     actions = origfn(repo, p1, p2, pa, overwrite, partial)
     processed = []
 
     for action in actions:
         if overwrite:
             processed.append(action)
             continue
         f, m = action[:2]
 
         choices = (_('&Largefile'), _('&Normal file'))
         if m == "g" and lfutil.splitstandin(f) in p1 and f in p2:
             # Case 1: normal file in the working copy, largefile in
             # the second parent
             lfile = lfutil.splitstandin(f)
             standin = f
             msg = _('%s has been turned into a largefile\n'
                     'use (l)argefile or keep as (n)ormal file?') % lfile
             if repo.ui.promptchoice(msg, choices, 0) == 0:
                 processed.append((lfile, "r"))
                 processed.append((standin, "g", p2.flags(standin)))
             else:
                 processed.append((standin, "r"))
         elif m == "g" and lfutil.standin(f) in p1 and f in p2:
             # Case 2: largefile in the working copy, normal file in
             # the second parent
             standin = lfutil.standin(f)
             lfile = f
             msg = _('%s has been turned into a normal file\n'
                     'keep as (l)argefile or use (n)ormal file?') % lfile
             if repo.ui.promptchoice(msg, choices, 0) == 0:
                 processed.append((lfile, "r"))
             else:
                 processed.append((standin, "r"))
                 processed.append((lfile, "g", p2.flags(lfile)))
         else:
             processed.append(action)
 
     return processed
 
 # Override filemerge to prompt the user about how they wish to merge
 # largefiles. This will handle identical edits, and copy/rename +
 # edit without prompting the user.
 def overridefilemerge(origfn, repo, mynode, orig, fcd, fco, fca):
     # Use better variable names here. Because this is a wrapper we cannot
     # change the variable names in the function declaration.
     fcdest, fcother, fcancestor = fcd, fco, fca
     if not lfutil.isstandin(orig):
         return origfn(repo, mynode, orig, fcdest, fcother, fcancestor)
     else:
         if not fcother.cmp(fcdest): # files identical?
             return None
 
         # backwards, use working dir parent as ancestor
         if fcancestor == fcother:
             fcancestor = fcdest.parents()[0]
 
         if orig != fcother.path():
             repo.ui.status(_('merging %s and %s to %s\n')
                            % (lfutil.splitstandin(orig),
                               lfutil.splitstandin(fcother.path()),
                               lfutil.splitstandin(fcdest.path())))
         else:
             repo.ui.status(_('merging %s\n')
                            % lfutil.splitstandin(fcdest.path()))
 
         if fcancestor.path() != fcother.path() and fcother.data() == \
                 fcancestor.data():
             return 0
         if fcancestor.path() != fcdest.path() and fcdest.data() == \
                 fcancestor.data():
             repo.wwrite(fcdest.path(), fcother.data(), fcother.flags())
             return 0
 
         if repo.ui.promptchoice(_('largefile %s has a merge conflict\n'
                              'keep (l)ocal or take (o)ther?') %
                              lfutil.splitstandin(orig),
                              (_('&Local'), _('&Other')), 0) == 0:
             return 0
         else:
             repo.wwrite(fcdest.path(), fcother.data(), fcother.flags())
             return 0
 
 # Copy first changes the matchers to match standins instead of
 # largefiles.  Then it overrides util.copyfile in that function it
 # checks if the destination largefile already exists. It also keeps a
 # list of copied files so that the largefiles can be copied and the
 # dirstate updated.
 def overridecopy(orig, ui, repo, pats, opts, rename=False):
     # doesn't remove largefile on rename
     if len(pats) < 2:
         # this isn't legal, let the original function deal with it
         return orig(ui, repo, pats, opts, rename)
 
     def makestandin(relpath):
         path = scmutil.canonpath(repo.root, repo.getcwd(), relpath)
         return os.path.join(repo.wjoin(lfutil.standin(path)))
 
     fullpats = scmutil.expandpats(pats)
     dest = fullpats[-1]
 
     if os.path.isdir(dest):
         if not os.path.isdir(makestandin(dest)):
             os.makedirs(makestandin(dest))
     # This could copy both lfiles and normal files in one command,
     # but we don't want to do that. First replace their matcher to
     # only match normal files and run it, then replace it to just
     # match largefiles and run it again.
     nonormalfiles = False
     nolfiles = False
     try:
         try:
             installnormalfilesmatchfn(repo[None].manifest())
             result = orig(ui, repo, pats, opts, rename)
         except util.Abort, e:
             if str(e) != _('no files to copy'):
                 raise e
             else:
                 nonormalfiles = True
             result = 0
     finally:
         restorematchfn()
 
     # The first rename can cause our current working directory to be removed.
     # In that case there is nothing left to copy/rename so just quit.
     try:
         repo.getcwd()
     except OSError:
         return result
 
     try:
         try:
             # When we call orig below it creates the standins but we don't add
             # them to the dir state until later so lock during that time.
             wlock = repo.wlock()
 
             manifest = repo[None].manifest()
             oldmatch = None # for the closure
             def overridematch(ctx, pats=[], opts={}, globbed=False,
                     default='relpath'):
                 newpats = []
                 # The patterns were previously mangled to add the standin
                 # directory; we need to remove that now
                 for pat in pats:
                     if match_.patkind(pat) is None and lfutil.shortname in pat:
                         newpats.append(pat.replace(lfutil.shortname, ''))
                     else:
                         newpats.append(pat)
                 match = oldmatch(ctx, newpats, opts, globbed, default)
                 m = copy.copy(match)
                 lfile = lambda f: lfutil.standin(f) in manifest
                 m._files = [lfutil.standin(f) for f in m._files if lfile(f)]
                 m._fmap = set(m._files)
                 origmatchfn = m.matchfn
                 m.matchfn = lambda f: (lfutil.isstandin(f) and
                                     (f in manifest) and
                                     origmatchfn(lfutil.splitstandin(f)) or
                                     None)
                 return m
             oldmatch = installmatchfn(overridematch)
             listpats = []
             for pat in pats:
                 if match_.patkind(pat) is not None:
                     listpats.append(pat)
                 else:
                     listpats.append(makestandin(pat))
 
             try:
                 origcopyfile = util.copyfile
                 copiedfiles = []
                 def overridecopyfile(src, dest):
                     if (lfutil.shortname in src and
                         dest.startswith(repo.wjoin(lfutil.shortname))):
                         destlfile = dest.replace(lfutil.shortname, '')
                         if not opts['force'] and os.path.exists(destlfile):
                             raise IOError('',
                                 _('destination largefile already exists'))
                     copiedfiles.append((src, dest))
                     origcopyfile(src, dest)
 
                 util.copyfile = overridecopyfile
                 result += orig(ui, repo, listpats, opts, rename)
             finally:
                 util.copyfile = origcopyfile
 
             lfdirstate = lfutil.openlfdirstate(ui, repo)
             for (src, dest) in copiedfiles:
                 if (lfutil.shortname in src and
                     dest.startswith(repo.wjoin(lfutil.shortname))):
                     srclfile = src.replace(repo.wjoin(lfutil.standin('')), '')
                     destlfile = dest.replace(repo.wjoin(lfutil.standin('')), '')
                     destlfiledir = os.path.dirname(repo.wjoin(destlfile)) or '.'
                     if not os.path.isdir(destlfiledir):
                         os.makedirs(destlfiledir)
                     if rename:
                         os.rename(repo.wjoin(srclfile), repo.wjoin(destlfile))
                         lfdirstate.remove(srclfile)
                     else:
                         util.copyfile(repo.wjoin(srclfile),
                                       repo.wjoin(destlfile))
 
                     lfdirstate.add(destlfile)
             lfdirstate.write()
         except util.Abort, e:
             if str(e) != _('no files to copy'):
                 raise e
             else:
                 nolfiles = True
     finally:
         restorematchfn()
         wlock.release()
 
     if nolfiles and nonormalfiles:
         raise util.Abort(_('no files to copy'))
 
     return result
 
 # When the user calls revert, we have to be careful to not revert any
 # changes to other largefiles accidentally. This means we have to keep
 # track of the largefiles that are being reverted so we only pull down
 # the necessary largefiles.
 #
 # Standins are only updated (to match the hash of largefiles) before
 # commits. Update the standins then run the original revert, changing
 # the matcher to hit standins instead of largefiles. Based on the
 # resulting standins update the largefiles. Then return the standins
 # to their proper state
 def overriderevert(orig, ui, repo, *pats, **opts):
     # Because we put the standins in a bad state (by updating them)
     # and then return them to a correct state we need to lock to
     # prevent others from changing them in their incorrect state.
     wlock = repo.wlock()
     try:
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         (modified, added, removed, missing, unknown, ignored, clean) = \
             lfutil.lfdirstatestatus(lfdirstate, repo, repo['.'].rev())
         lfdirstate.write()
         for lfile in modified:
             lfutil.updatestandin(repo, lfutil.standin(lfile))
         for lfile in missing:
             if (os.path.exists(repo.wjoin(lfutil.standin(lfile)))):
                 os.unlink(repo.wjoin(lfutil.standin(lfile)))
 
         try:
             ctx = scmutil.revsingle(repo, opts.get('rev'))
             oldmatch = None # for the closure
             def overridematch(ctx, pats=[], opts={}, globbed=False,
                     default='relpath'):
                 match = oldmatch(ctx, pats, opts, globbed, default)
                 m = copy.copy(match)
                 def tostandin(f):
                     if lfutil.standin(f) in ctx:
                         return lfutil.standin(f)
                     elif lfutil.standin(f) in repo[None]:
                         return None
                     return f
                 m._files = [tostandin(f) for f in m._files]
                 m._files = [f for f in m._files if f is not None]
                 m._fmap = set(m._files)
                 origmatchfn = m.matchfn
                 def matchfn(f):
                     if lfutil.isstandin(f):
                         # We need to keep track of what largefiles are being
                         # matched so we know which ones to update later --
                         # otherwise we accidentally revert changes to other
                         # largefiles. This is repo-specific, so duckpunch the
                         # repo object to keep the list of largefiles for us
                         # later.
                         if origmatchfn(lfutil.splitstandin(f)) and \
                                 (f in repo[None] or f in ctx):
                             lfileslist = getattr(repo, '_lfilestoupdate', [])
                             lfileslist.append(lfutil.splitstandin(f))
                             repo._lfilestoupdate = lfileslist
                             return True
                         else:
                             return False
                     return origmatchfn(f)
                 m.matchfn = matchfn
                 return m
             oldmatch = installmatchfn(overridematch)
             scmutil.match
             matches = overridematch(repo[None], pats, opts)
             orig(ui, repo, *pats, **opts)
         finally:
             restorematchfn()
         lfileslist = getattr(repo, '_lfilestoupdate', [])
         lfcommands.updatelfiles(ui, repo, filelist=lfileslist,
                                 printmessage=False)
 
         # empty out the largefiles list so we start fresh next time
         repo._lfilestoupdate = []
         for lfile in modified:
             if lfile in lfileslist:
                 if os.path.exists(repo.wjoin(lfutil.standin(lfile))) and lfile\
                         in repo['.']:
                     lfutil.writestandin(repo, lfutil.standin(lfile),
                         repo['.'][lfile].data().strip(),
                         'x' in repo['.'][lfile].flags())
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         for lfile in added:
             standin = lfutil.standin(lfile)
             if standin not in ctx and (standin in matches or opts.get('all')):
                 if lfile in lfdirstate:
                     lfdirstate.drop(lfile)
                 util.unlinkpath(repo.wjoin(standin))
         lfdirstate.write()
     finally:
         wlock.release()
 
 def hgupdate(orig, repo, node):
     # Only call updatelfiles the standins that have changed to save time
     oldstandins = lfutil.getstandinsstate(repo)
     result = orig(repo, node)
     newstandins = lfutil.getstandinsstate(repo)
     filelist = lfutil.getlfilestoupdate(oldstandins, newstandins)
     lfcommands.updatelfiles(repo.ui, repo, filelist=filelist, printmessage=True)
     return result
 
 def hgclean(orig, repo, node, show_stats=True):
     result = orig(repo, node, show_stats)
     lfcommands.updatelfiles(repo.ui, repo)
     return result
 
 def hgmerge(orig, repo, node, force=None, remind=True):
     # Mark the repo as being in the middle of a merge, so that
     # updatelfiles() will know that it needs to trust the standins in
     # the working copy, not in the standins in the current node
     repo._ismerging = True
     try:
         result = orig(repo, node, force, remind)
         lfcommands.updatelfiles(repo.ui, repo)
     finally:
         repo._ismerging = False
     return result
 
 # When we rebase a repository with remotely changed largefiles, we need to
 # take some extra care so that the largefiles are correctly updated in the
 # working copy
 def overridepull(orig, ui, repo, source=None, **opts):
     revsprepull = len(repo)
     if opts.get('rebase', False):
         repo._isrebasing = True
         try:
             if opts.get('update'):
                 del opts['update']
                 ui.debug('--update and --rebase are not compatible, ignoring '
                          'the update flag\n')
             del opts['rebase']
             cmdutil.bailifchanged(repo)
             origpostincoming = commands.postincoming
             def _dummy(*args, **kwargs):
                 pass
             commands.postincoming = _dummy
             if not source:
                 source = 'default'
             repo.lfpullsource = source
             try:
                 result = commands.pull(ui, repo, source, **opts)
             finally:
                 commands.postincoming = origpostincoming
             revspostpull = len(repo)
             if revspostpull > revsprepull:
                 result = result or rebase.rebase(ui, repo)
         finally:
             repo._isrebasing = False
     else:
         if not source:
             source = 'default'
         repo.lfpullsource = source
         oldheads = lfutil.getcurrentheads(repo)
         result = orig(ui, repo, source, **opts)
         # If we do not have the new largefiles for any new heads we pulled, we
         # will run into a problem later if we try to merge or rebase with one of
         # these heads, so cache the largefiles now directly into the system
         # cache.
         ui.status(_("caching new largefiles\n"))
         numcached = 0
         heads = lfutil.getcurrentheads(repo)
         newheads = set(heads).difference(set(oldheads))
         for head in newheads:
             (cached, missing) = lfcommands.cachelfiles(ui, repo, head)
             numcached += len(cached)
         ui.status(_("%d largefiles cached\n") % numcached)
     if opts.get('all_largefiles'):
         revspostpull = len(repo)
         revs = []
         for rev in xrange(revsprepull + 1, revspostpull):
             revs.append(repo[rev].rev())
         lfcommands.downloadlfiles(ui, repo, revs)
     return result
 
 def overrideclone(orig, ui, source, dest=None, **opts):
     d = dest
     if d is None:
         d = hg.defaultdest(source)
     if opts.get('all_largefiles') and not hg.islocal(d):
             raise util.Abort(_(
             '--all-largefiles is incompatible with non-local destination %s' %
             d))
 
     return orig(ui, source, dest, **opts)
 
 def hgclone(orig, ui, opts, *args, **kwargs):
     result = orig(ui, opts, *args, **kwargs)
 
     if result is not None:
         sourcerepo, destrepo = result
         repo = destrepo.local()
 
         # The .hglf directory must exist for the standin matcher to match
         # anything (which listlfiles uses for each rev), and .hg/largefiles is
         # assumed to exist by the code that caches the downloaded file.  These
         # directories exist if clone updated to any rev.  (If the repo does not
         # have largefiles, download never gets to the point of needing
         # .hg/largefiles, and the standin matcher won't match anything anyway.)
         if 'largefiles' in repo.requirements:
             if opts.get('noupdate'):
                 util.makedirs(repo.wjoin(lfutil.shortname))
                 util.makedirs(repo.join(lfutil.longname))
 
         # Caching is implicitly limited to 'rev' option, since the dest repo was
         # truncated at that point.  The user may expect a download count with
         # this option, so attempt whether or not this is a largefile repo.
         if opts.get('all_largefiles'):
             success, missing = lfcommands.downloadlfiles(ui, repo, None)
 
             if missing != 0:
                 return None
 
     return result
 
 def overriderebase(orig, ui, repo, **opts):
     repo._isrebasing = True
     try:
         return orig(ui, repo, **opts)
     finally:
         repo._isrebasing = False
 
 def overridearchive(orig, repo, dest, node, kind, decode=True, matchfn=None,
             prefix=None, mtime=None, subrepos=None):
     # No need to lock because we are only reading history and
     # largefile caches, neither of which are modified.
     lfcommands.cachelfiles(repo.ui, repo, node)
 
     if kind not in archival.archivers:
         raise util.Abort(_("unknown archive type '%s'") % kind)
 
     ctx = repo[node]
 
     if kind == 'files':
         if prefix:
             raise util.Abort(
                 _('cannot give prefix when archiving to files'))
     else:
         prefix = archival.tidyprefix(dest, kind, prefix)
 
     def write(name, mode, islink, getdata):
         if matchfn and not matchfn(name):
             return
         data = getdata()
         if decode:
             data = repo.wwritedata(name, data)
         archiver.addfile(prefix + name, mode, islink, data)
 
     archiver = archival.archivers[kind](dest, mtime or ctx.date()[0])
 
     if repo.ui.configbool("ui", "archivemeta", True):
         def metadata():
             base = 'repo: %s\nnode: %s\nbranch: %s\n' % (
                 hex(repo.changelog.node(0)), hex(node), ctx.branch())
 
             tags = ''.join('tag: %s\n' % t for t in ctx.tags()
                            if repo.tagtype(t) == 'global')
             if not tags:
                 repo.ui.pushbuffer()
                 opts = {'template': '{latesttag}\n{latesttagdistance}',
                         'style': '', 'patch': None, 'git': None}
                 cmdutil.show_changeset(repo.ui, repo, opts).show(ctx)
                 ltags, dist = repo.ui.popbuffer().split('\n')
                 tags = ''.join('latesttag: %s\n' % t for t in ltags.split(':'))
                 tags += 'latesttagdistance: %s\n' % dist
 
             return base + tags
 
         write('.hg_archival.txt', 0644, False, metadata)
 
     for f in ctx:
         ff = ctx.flags(f)
         getdata = ctx[f].data
         if lfutil.isstandin(f):
             path = lfutil.findfile(repo, getdata().strip())
             if path is None:
                 raise util.Abort(
                     _('largefile %s not found in repo store or system cache')
                     % lfutil.splitstandin(f))
             f = lfutil.splitstandin(f)
 
             def getdatafn():
                 fd = None
                 try:
                     fd = open(path, 'rb')
                     return fd.read()
                 finally:
                     if fd:
                         fd.close()
 
             getdata = getdatafn
         write(f, 'x' in ff and 0755 or 0644, 'l' in ff, getdata)
 
     if subrepos:
         for subpath in ctx.substate:
             sub = ctx.sub(subpath)
             submatch = match_.narrowmatcher(subpath, matchfn)
             sub.archive(repo.ui, archiver, prefix, submatch)
 
     archiver.done()
 
 def hgsubrepoarchive(orig, repo, ui, archiver, prefix, match=None):
     repo._get(repo._state + ('hg',))
     rev = repo._state[1]
     ctx = repo._repo[rev]
 
     lfcommands.cachelfiles(ui, repo._repo, ctx.node())
 
     def write(name, mode, islink, getdata):
         # At this point, the standin has been replaced with the largefile name,
         # so the normal matcher works here without the lfutil variants.
         if match and not match(f):
             return
         data = getdata()
 
         archiver.addfile(prefix + repo._path + '/' + name, mode, islink, data)
 
     for f in ctx:
         ff = ctx.flags(f)
         getdata = ctx[f].data
         if lfutil.isstandin(f):
             path = lfutil.findfile(repo._repo, getdata().strip())
             if path is None:
                 raise util.Abort(
                     _('largefile %s not found in repo store or system cache')
                     % lfutil.splitstandin(f))
             f = lfutil.splitstandin(f)
 
             def getdatafn():
                 fd = None
                 try:
                     fd = open(os.path.join(prefix, path), 'rb')
                     return fd.read()
                 finally:
                     if fd:
                         fd.close()
 
             getdata = getdatafn
 
         write(f, 'x' in ff and 0755 or 0644, 'l' in ff, getdata)
 
     for subpath in ctx.substate:
         sub = ctx.sub(subpath)
         submatch = match_.narrowmatcher(subpath, match)
         sub.archive(ui, archiver, os.path.join(prefix, repo._path) + '/',
                     submatch)
 
 # If a largefile is modified, the change is not reflected in its
 # standin until a commit. cmdutil.bailifchanged() raises an exception
 # if the repo has uncommitted changes. Wrap it to also check if
 # largefiles were changed. This is used by bisect and backout.
 def overridebailifchanged(orig, repo):
     orig(repo)
     repo.lfstatus = True
     modified, added, removed, deleted = repo.status()[:4]
     repo.lfstatus = False
     if modified or added or removed or deleted:
         raise util.Abort(_('outstanding uncommitted changes'))
 
 # Fetch doesn't use cmdutil.bailifchanged so override it to add the check
 def overridefetch(orig, ui, repo, *pats, **opts):
     repo.lfstatus = True
     modified, added, removed, deleted = repo.status()[:4]
     repo.lfstatus = False
     if modified or added or removed or deleted:
         raise util.Abort(_('outstanding uncommitted changes'))
     return orig(ui, repo, *pats, **opts)
 
 def overrideforget(orig, ui, repo, *pats, **opts):
     installnormalfilesmatchfn(repo[None].manifest())
     result = orig(ui, repo, *pats, **opts)
     restorematchfn()
     m = scmutil.match(repo[None], pats, opts)
 
     try:
         repo.lfstatus = True
         s = repo.status(match=m, clean=True)
     finally:
         repo.lfstatus = False
     forget = sorted(s[0] + s[1] + s[3] + s[6])
     forget = [f for f in forget if lfutil.standin(f) in repo[None].manifest()]
 
     for f in forget:
         if lfutil.standin(f) not in repo.dirstate and not \
                 os.path.isdir(m.rel(lfutil.standin(f))):
             ui.warn(_('not removing %s: file is already untracked\n')
                     % m.rel(f))
             result = 1
 
     for f in forget:
         if ui.verbose or not m.exact(f):
             ui.status(_('removing %s\n') % m.rel(f))
 
     # Need to lock because standin files are deleted then removed from the
     # repository and we could race in-between.
     wlock = repo.wlock()
     try:
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         for f in forget:
             if lfdirstate[f] == 'a':
                 lfdirstate.drop(f)
             else:
                 lfdirstate.remove(f)
         lfdirstate.write()
         standins = [lfutil.standin(f) for f in forget]
         for f in standins:
             util.unlinkpath(repo.wjoin(f), ignoremissing=True)
         repo[None].forget(standins)
     finally:
         wlock.release()
 
     return result
 
 def getoutgoinglfiles(ui, repo, dest=None, **opts):
     dest = ui.expandpath(dest or 'default-push', dest or 'default')
     dest, branches = hg.parseurl(dest, opts.get('branch'))
     revs, checkout = hg.addbranchrevs(repo, repo, branches, opts.get('rev'))
     if revs:
         revs = [repo.lookup(rev) for rev in scmutil.revrange(repo, revs)]
 
     try:
         remote = hg.peer(repo, opts, dest)
     except error.RepoError:
         return None
     outgoing = discovery.findcommonoutgoing(repo, remote.peer(), force=False)
     if not outgoing.missing:
         return outgoing.missing
     o = repo.changelog.nodesbetween(outgoing.missing, revs)[0]
     if opts.get('newest_first'):
         o.reverse()
 
     toupload = set()
     for n in o:
         parents = [p for p in repo.changelog.parents(n) if p != node.nullid]
         ctx = repo[n]
         files = set(ctx.files())
         if len(parents) == 2:
             mc = ctx.manifest()
             mp1 = ctx.parents()[0].manifest()
             mp2 = ctx.parents()[1].manifest()
             for f in mp1:
                 if f not in mc:
                         files.add(f)
             for f in mp2:
                 if f not in mc:
                     files.add(f)
             for f in mc:
                 if mc[f] != mp1.get(f, None) or mc[f] != mp2.get(f, None):
                     files.add(f)
         toupload = toupload.union(
             set([f for f in files if lfutil.isstandin(f) and f in ctx]))
     return toupload
 
 def overrideoutgoing(orig, ui, repo, dest=None, **opts):
     result = orig(ui, repo, dest, **opts)
 
     if opts.pop('large', None):
         toupload = getoutgoinglfiles(ui, repo, dest, **opts)
         if toupload is None:
             ui.status(_('largefiles: No remote repo\n'))
         elif not toupload:
             ui.status(_('largefiles: no files to upload\n'))
         else:
             ui.status(_('largefiles to upload:\n'))
             for file in toupload:
                 ui.status(lfutil.splitstandin(file) + '\n')
             ui.status('\n')
 
     return result
 
 def overridesummary(orig, ui, repo, *pats, **opts):
     try:
         repo.lfstatus = True
         orig(ui, repo, *pats, **opts)
     finally:
         repo.lfstatus = False
 
     if opts.pop('large', None):
         toupload = getoutgoinglfiles(ui, repo, None, **opts)
         if toupload is None:
             # i18n: column positioning for "hg summary"
             ui.status(_('largefiles: (no remote repo)\n'))
         elif not toupload:
             # i18n: column positioning for "hg summary"
             ui.status(_('largefiles: (no files to upload)\n'))
         else:
             # i18n: column positioning for "hg summary"
             ui.status(_('largefiles: %d to upload\n') % len(toupload))
 
 def scmutiladdremove(orig, repo, pats=[], opts={}, dry_run=None,
                      similarity=None):
     if not lfutil.islfilesrepo(repo):
         return orig(repo, pats, opts, dry_run, similarity)
     # Get the list of missing largefiles so we can remove them
     lfdirstate = lfutil.openlfdirstate(repo.ui, repo)
     s = lfdirstate.status(match_.always(repo.root, repo.getcwd()), [], False,
         False, False)
     (unsure, modified, added, removed, missing, unknown, ignored, clean) = s
 
     # Call into the normal remove code, but the removing of the standin, we want
     # to have handled by original addremove.  Monkey patching here makes sure
     # we don't remove the standin in the largefiles code, preventing a very
     # confused state later.
     if missing:
         m = [repo.wjoin(f) for f in missing]
         repo._isaddremove = True
         removelargefiles(repo.ui, repo, *m, **opts)
         repo._isaddremove = False
     # Call into the normal add code, and any files that *should* be added as
     # largefiles will be
     addlargefiles(repo.ui, repo, *pats, **opts)
     # Now that we've handled largefiles, hand off to the original addremove
     # function to take care of the rest.  Make sure it doesn't do anything with
     # largefiles by installing a matcher that will ignore them.
     installnormalfilesmatchfn(repo[None].manifest())
     result = orig(repo, pats, opts, dry_run, similarity)
     restorematchfn()
     return result
 
 # Calling purge with --all will cause the largefiles to be deleted.
 # Override repo.status to prevent this from happening.
 def overridepurge(orig, ui, repo, *dirs, **opts):
     # XXX large file status is buggy when used on repo proxy.
     # XXX this needs to be investigate.
     repo = repo.unfiltered()
     oldstatus = repo.status
     def overridestatus(node1='.', node2=None, match=None, ignored=False,
                         clean=False, unknown=False, listsubrepos=False):
         r = oldstatus(node1, node2, match, ignored, clean, unknown,
                       listsubrepos)
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         modified, added, removed, deleted, unknown, ignored, clean = r
         unknown = [f for f in unknown if lfdirstate[f] == '?']
         ignored = [f for f in ignored if lfdirstate[f] == '?']
         return modified, added, removed, deleted, unknown, ignored, clean
     repo.status = overridestatus
     orig(ui, repo, *dirs, **opts)
     repo.status = oldstatus
 
 def overriderollback(orig, ui, repo, **opts):
     result = orig(ui, repo, **opts)
     merge.update(repo, node=None, branchmerge=False, force=True,
         partial=lfutil.isstandin)
     wlock = repo.wlock()
     try:
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         lfiles = lfutil.listlfiles(repo)
         oldlfiles = lfutil.listlfiles(repo, repo[None].parents()[0].rev())
         for file in lfiles:
             if file in oldlfiles:
                 lfdirstate.normallookup(file)
             else:
                 lfdirstate.add(file)
         lfdirstate.write()
     finally:
         wlock.release()
     return result
 
 def overridetransplant(orig, ui, repo, *revs, **opts):
     try:
         oldstandins = lfutil.getstandinsstate(repo)
         repo._istransplanting = True
         result = orig(ui, repo, *revs, **opts)
         newstandins = lfutil.getstandinsstate(repo)
         filelist = lfutil.getlfilestoupdate(oldstandins, newstandins)
         lfcommands.updatelfiles(repo.ui, repo, filelist=filelist,
                                 printmessage=True)
     finally:
         repo._istransplanting = False
     return result
 
 def overridecat(orig, ui, repo, file1, *pats, **opts):
     ctx = scmutil.revsingle(repo, opts.get('rev'))
     if not lfutil.standin(file1) in ctx:
         result = orig(ui, repo, file1, *pats, **opts)
         return result
     return lfcommands.catlfile(repo, file1, ctx.rev(), opts.get('output'))
 
 def mercurialsinkbefore(orig, sink):
     sink.repo._isconverting = True
     orig(sink)
 
 def mercurialsinkafter(orig, sink):
     sink.repo._isconverting = False
     orig(sink)

hgext/largefiles/reposetup.py

@@ -1,504 +1,504
 # Copyright 2009-2010 Gregory P. Ward
 # Copyright 2009-2010 Intelerad Medical Systems Incorporated
 # Copyright 2010-2011 Fog Creek Software
 # Copyright 2010-2011 Unity Technologies
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''setup for largefiles repositories: reposetup'''
 import copy
 import types
 import os
 
 from mercurial import context, error, manifest, match as match_, util, \
     discovery
 from mercurial import node as node_
 from mercurial.i18n import _
 from mercurial import localrepo
 
 import lfcommands
 import proto
 import lfutil
 
 def reposetup(ui, repo):
     # wire repositories should be given new wireproto functions but not the
     # other largefiles modifications
     if not repo.local():
         return proto.wirereposetup(ui, repo)
 
     for name in ('status', 'commitctx', 'commit', 'push'):
         method = getattr(repo, name)
         if (isinstance(method, types.FunctionType) and
             method.func_name == 'wrap'):
             ui.warn(_('largefiles: repo method %r appears to have already been'
                     ' wrapped by another extension: '
                     'largefiles may behave incorrectly\n')
                     % name)
 
     class lfilesrepo(repo.__class__):
         lfstatus = False
         def status_nolfiles(self, *args, **kwargs):
             return super(lfilesrepo, self).status(*args, **kwargs)
 
         # When lfstatus is set, return a context that gives the names
         # of largefiles instead of their corresponding standins and
         # identifies the largefiles as always binary, regardless of
         # their actual contents.
         def __getitem__(self, changeid):
             ctx = super(lfilesrepo, self).__getitem__(changeid)
             if self.lfstatus:
                 class lfilesmanifestdict(manifest.manifestdict):
                     def __contains__(self, filename):
                         if super(lfilesmanifestdict,
                                 self).__contains__(filename):
                             return True
                         return super(lfilesmanifestdict,
                             self).__contains__(lfutil.standin(filename))
                 class lfilesctx(ctx.__class__):
                     def files(self):
                         filenames = super(lfilesctx, self).files()
                         return [lfutil.splitstandin(f) or f for f in filenames]
                     def manifest(self):
                         man1 = super(lfilesctx, self).manifest()
                         man1.__class__ = lfilesmanifestdict
                         return man1
                     def filectx(self, path, fileid=None, filelog=None):
                         try:
                             if filelog is not None:
                                 result = super(lfilesctx, self).filectx(
                                     path, fileid, filelog)
                             else:
                                 result = super(lfilesctx, self).filectx(
                                     path, fileid)
                         except error.LookupError:
                             # Adding a null character will cause Mercurial to
                             # identify this as a binary file.
                             if filelog is not None:
                                 result = super(lfilesctx, self).filectx(
                                     lfutil.standin(path), fileid, filelog)
                             else:
                                 result = super(lfilesctx, self).filectx(
                                     lfutil.standin(path), fileid)
                             olddata = result.data
                             result.data = lambda: olddata() + '\0'
                         return result
                 ctx.__class__ = lfilesctx
             return ctx
 
         # Figure out the status of big files and insert them into the
         # appropriate list in the result. Also removes standin files
         # from the listing. Revert to the original status if
         # self.lfstatus is False.
         # XXX large file status is buggy when used on repo proxy.
         # XXX this needs to be investigated.
         @localrepo.unfilteredmethod
         def status(self, node1='.', node2=None, match=None, ignored=False,
                 clean=False, unknown=False, listsubrepos=False):
             listignored, listclean, listunknown = ignored, clean, unknown
             if not self.lfstatus:
                 return super(lfilesrepo, self).status(node1, node2, match,
                     listignored, listclean, listunknown, listsubrepos)
             else:
                 # some calls in this function rely on the old version of status
                 self.lfstatus = False
                 if isinstance(node1, context.changectx):
                     ctx1 = node1
                 else:
                     ctx1 = self[node1]
                 if isinstance(node2, context.changectx):
                     ctx2 = node2
                 else:
                     ctx2 = self[node2]
                 working = ctx2.rev() is None
                 parentworking = working and ctx1 == self['.']
 
                 def inctx(file, ctx):
                     try:
                         if ctx.rev() is None:
                             return file in ctx.manifest()
                         ctx[file]
                         return True
                     except KeyError:
                         return False
 
                 if match is None:
                     match = match_.always(self.root, self.getcwd())
 
                 # First check if there were files specified on the
                 # command line.  If there were, and none of them were
                 # largefiles, we should just bail here and let super
                 # handle it -- thus gaining a big performance boost.
                 lfdirstate = lfutil.openlfdirstate(ui, self)
                 if match.files() and not match.anypats():
                     for f in lfdirstate:
                         if match(f):
                             break
                     else:
                         return super(lfilesrepo, self).status(node1, node2,
                                 match, listignored, listclean,
                                 listunknown, listsubrepos)
 
                 # Create a copy of match that matches standins instead
                 # of largefiles.
                 def tostandins(files):
                     if not working:
                         return files
                     newfiles = []
                     dirstate = self.dirstate
                     for f in files:
                         sf = lfutil.standin(f)
                         if sf in dirstate:
                             newfiles.append(sf)
                         elif sf in dirstate.dirs():
                             # Directory entries could be regular or
                             # standin, check both
                             newfiles.extend((f, sf))
                         else:
                             newfiles.append(f)
                     return newfiles
 
                 m = copy.copy(match)
                 m._files = tostandins(m._files)
 
                 result = super(lfilesrepo, self).status(node1, node2, m,
                     ignored, clean, unknown, listsubrepos)
                 if working:
 
                     def sfindirstate(f):
                         sf = lfutil.standin(f)
                         dirstate = self.dirstate
                         return sf in dirstate or sf in dirstate.dirs()
 
                     match._files = [f for f in match._files
                                     if sfindirstate(f)]
                     # Don't waste time getting the ignored and unknown
                     # files from lfdirstate
                     s = lfdirstate.status(match, [], False,
                             listclean, False)
                     (unsure, modified, added, removed, missing, _unknown,
                             _ignored, clean) = s
                     if parentworking:
                         for lfile in unsure:
                             standin = lfutil.standin(lfile)
                             if standin not in ctx1:
                                 # from second parent
                                 modified.append(lfile)
                             elif ctx1[standin].data().strip() \
                                     != lfutil.hashfile(self.wjoin(lfile)):
                                 modified.append(lfile)
                             else:
                                 clean.append(lfile)
                                 lfdirstate.normal(lfile)
                     else:
                         tocheck = unsure + modified + added + clean
                         modified, added, clean = [], [], []
 
                         for lfile in tocheck:
                             standin = lfutil.standin(lfile)
                             if inctx(standin, ctx1):
                                 if ctx1[standin].data().strip() != \
                                         lfutil.hashfile(self.wjoin(lfile)):
                                     modified.append(lfile)
                                 else:
                                     clean.append(lfile)
                             else:
                                 added.append(lfile)
 
                     # Standins no longer found in lfdirstate has been removed
                     for standin in ctx1.manifest():
                         if not lfutil.isstandin(standin):
                             continue
                         lfile = lfutil.splitstandin(standin)
                         if not match(lfile):
                             continue
                         if lfile not in lfdirstate:
                             removed.append(lfile)
 
                     # Filter result lists
                     result = list(result)
 
                     # Largefiles are not really removed when they're
                     # still in the normal dirstate. Likewise, normal
                     # files are not really removed if they are still in
                     # lfdirstate. This happens in merges where files
                     # change type.
                     removed = [f for f in removed if f not in self.dirstate]
                     result[2] = [f for f in result[2] if f not in lfdirstate]
 
                     lfiles = set(lfdirstate._map)
                     # Unknown files
                     result[4] = set(result[4]).difference(lfiles)
                     # Ignored files
                     result[5] = set(result[5]).difference(lfiles)
                     # combine normal files and largefiles
                     normals = [[fn for fn in filelist
                                 if not lfutil.isstandin(fn)]
                                for filelist in result]
                     lfiles = (modified, added, removed, missing, [], [], clean)
                     result = [sorted(list1 + list2)
                               for (list1, list2) in zip(normals, lfiles)]
                 else:
                     def toname(f):
                         if lfutil.isstandin(f):
                             return lfutil.splitstandin(f)
                         return f
                     result = [[toname(f) for f in items] for items in result]
 
                 lfdirstate.write()
 
                 if not listunknown:
                     result[4] = []
                 if not listignored:
                     result[5] = []
                 if not listclean:
                     result[6] = []
                 self.lfstatus = True
                 return result
 
         # As part of committing, copy all of the largefiles into the
         # cache.
         def commitctx(self, *args, **kwargs):
             node = super(lfilesrepo, self).commitctx(*args, **kwargs)
             lfutil.copyalltostore(self, node)
             return node
 
         # Before commit, largefile standins have not had their
         # contents updated to reflect the hash of their largefile.
         # Do that here.
         def commit(self, text="", user=None, date=None, match=None,
                 force=False, editor=False, extra={}):
             orig = super(lfilesrepo, self).commit
 
             wlock = self.wlock()
             try:
                 # Case 0: Rebase or Transplant
                 # We have to take the time to pull down the new largefiles now.
                 # Otherwise, any largefiles that were modified in the
                 # destination changesets get overwritten, either by the rebase
                 # or in the first commit after the rebase or transplant.
                 # updatelfiles will update the dirstate to mark any pulled
                 # largefiles as modified
                 if getattr(self, "_isrebasing", False) or \
                         getattr(self, "_istransplanting", False):
                     lfcommands.updatelfiles(self.ui, self, filelist=None,
                                             printmessage=False)
                     result = orig(text=text, user=user, date=date, match=match,
                                     force=force, editor=editor, extra=extra)
                     return result
                 # Case 1: user calls commit with no specific files or
                 # include/exclude patterns: refresh and commit all files that
                 # are "dirty".
                 if ((match is None) or
                     (not match.anypats() and not match.files())):
                     # Spend a bit of time here to get a list of files we know
                     # are modified so we can compare only against those.
                     # It can cost a lot of time (several seconds)
                     # otherwise to update all standins if the largefiles are
                     # large.
                     lfdirstate = lfutil.openlfdirstate(ui, self)
                     dirtymatch = match_.always(self.root, self.getcwd())
                     s = lfdirstate.status(dirtymatch, [], False, False, False)
                     modifiedfiles = []
                     for i in s:
                         modifiedfiles.extend(i)
                     lfiles = lfutil.listlfiles(self)
                     # this only loops through largefiles that exist (not
                     # removed/renamed)
                     for lfile in lfiles:
                         if lfile in modifiedfiles:
                             if os.path.exists(
                                     self.wjoin(lfutil.standin(lfile))):
                                 # this handles the case where a rebase is being
                                 # performed and the working copy is not updated
                                 # yet.
                                 if os.path.exists(self.wjoin(lfile)):
                                     lfutil.updatestandin(self,
                                         lfutil.standin(lfile))
                                     lfdirstate.normal(lfile)
 
                     result = orig(text=text, user=user, date=date, match=match,
                                     force=force, editor=editor, extra=extra)
 
                     if result is not None:
                         for lfile in lfdirstate:
                             if lfile in modifiedfiles:
                                 if (not os.path.exists(self.wjoin(
                                    lfutil.standin(lfile)))) or \
                                    (not os.path.exists(self.wjoin(lfile))):
                                     lfdirstate.drop(lfile)
 
                     # This needs to be after commit; otherwise precommit hooks
                     # get the wrong status
                     lfdirstate.write()
                     return result
 
                 lfiles = lfutil.listlfiles(self)
                 match._files = self._subdirlfs(match.files(), lfiles)
 
                 # Case 2: user calls commit with specified patterns: refresh
                 # any matching big files.
                 smatcher = lfutil.composestandinmatcher(self, match)
-                standins = lfutil.dirstatewalk(self.dirstate, smatcher)
+                standins = self.dirstate.walk(smatcher, [], False, False)
 
                 # No matching big files: get out of the way and pass control to
                 # the usual commit() method.
                 if not standins:
                     return orig(text=text, user=user, date=date, match=match,
                                     force=force, editor=editor, extra=extra)
 
                 # Refresh all matching big files.  It's possible that the
                 # commit will end up failing, in which case the big files will
                 # stay refreshed.  No harm done: the user modified them and
                 # asked to commit them, so sooner or later we're going to
                 # refresh the standins.  Might as well leave them refreshed.
                 lfdirstate = lfutil.openlfdirstate(ui, self)
                 for standin in standins:
                     lfile = lfutil.splitstandin(standin)
                     if lfdirstate[lfile] <> 'r':
                         lfutil.updatestandin(self, standin)
                         lfdirstate.normal(lfile)
                     else:
                         lfdirstate.drop(lfile)
 
                 # Cook up a new matcher that only matches regular files or
                 # standins corresponding to the big files requested by the
                 # user.  Have to modify _files to prevent commit() from
                 # complaining "not tracked" for big files.
                 match = copy.copy(match)
                 origmatchfn = match.matchfn
 
                 # Check both the list of largefiles and the list of
                 # standins because if a largefile was removed, it
                 # won't be in the list of largefiles at this point
                 match._files += sorted(standins)
 
                 actualfiles = []
                 for f in match._files:
                     fstandin = lfutil.standin(f)
 
                     # ignore known largefiles and standins
                     if f in lfiles or fstandin in standins:
                         continue
 
                     # append directory separator to avoid collisions
                     if not fstandin.endswith(os.sep):
                         fstandin += os.sep
 
                     actualfiles.append(f)
                 match._files = actualfiles
 
                 def matchfn(f):
                     if origmatchfn(f):
                         return f not in lfiles
                     else:
                         return f in standins
 
                 match.matchfn = matchfn
                 result = orig(text=text, user=user, date=date, match=match,
                                 force=force, editor=editor, extra=extra)
                 # This needs to be after commit; otherwise precommit hooks
                 # get the wrong status
                 lfdirstate.write()
                 return result
             finally:
                 wlock.release()
 
         def push(self, remote, force=False, revs=None, newbranch=False):
             outgoing = discovery.findcommonoutgoing(repo, remote.peer(),
                                                     force=force)
             if outgoing.missing:
                 toupload = set()
                 o = self.changelog.nodesbetween(outgoing.missing, revs)[0]
                 for n in o:
                     parents = [p for p in self.changelog.parents(n)
                                if p != node_.nullid]
                     ctx = self[n]
                     files = set(ctx.files())
                     if len(parents) == 2:
                         mc = ctx.manifest()
                         mp1 = ctx.parents()[0].manifest()
                         mp2 = ctx.parents()[1].manifest()
                         for f in mp1:
                             if f not in mc:
                                 files.add(f)
                         for f in mp2:
                             if f not in mc:
                                 files.add(f)
                         for f in mc:
                             if mc[f] != mp1.get(f, None) or mc[f] != mp2.get(f,
                                     None):
                                 files.add(f)
 
                     toupload = toupload.union(
                         set([ctx[f].data().strip()
                              for f in files
                              if lfutil.isstandin(f) and f in ctx]))
                 lfcommands.uploadlfiles(ui, self, remote, toupload)
             return super(lfilesrepo, self).push(remote, force, revs,
                 newbranch)
 
         def _subdirlfs(self, files, lfiles):
             '''
             Adjust matched file list
             If we pass a directory to commit whose only commitable files
             are largefiles, the core commit code aborts before finding
             the largefiles.
             So we do the following:
             For directories that only have largefiles as matches,
             we explicitly add the largefiles to the matchlist and remove
             the directory.
             In other cases, we leave the match list unmodified.
             '''
             actualfiles = []
             dirs = []
             regulars = []
 
             for f in files:
                 if lfutil.isstandin(f + '/'):
                     raise util.Abort(
                         _('file "%s" is a largefile standin') % f,
                         hint=('commit the largefile itself instead'))
                 # Scan directories
                 if os.path.isdir(self.wjoin(f)):
                     dirs.append(f)
                 else:
                     regulars.append(f)
 
             for f in dirs:
                 matcheddir = False
                 d = self.dirstate.normalize(f) + '/'
                 # Check for matched normal files
                 for mf in regulars:
                     if self.dirstate.normalize(mf).startswith(d):
                         actualfiles.append(f)
                         matcheddir = True
                         break
                 if not matcheddir:
                     # If no normal match, manually append
                     # any matching largefiles
                     for lf in lfiles:
                         if self.dirstate.normalize(lf).startswith(d):
                             actualfiles.append(lf)
                             if not matcheddir:
                                 actualfiles.append(lfutil.standin(f))
                                 matcheddir = True
                 # Nothing in dir, so readd it
                 # and let commit reject it
                 if not matcheddir:
                     actualfiles.append(f)
 
             # Always add normal files
             actualfiles += regulars
             return actualfiles
 
     repo.__class__ = lfilesrepo
 
     def checkrequireslfiles(ui, repo, **kwargs):
         if 'largefiles' not in repo.requirements and util.any(
                 lfutil.shortname+'/' in f[0] for f in repo.store.datafiles()):
             repo.requirements.add('largefiles')
             repo._writerequirements()
 
     ui.setconfig('hooks', 'changegroup.lfiles', checkrequireslfiles)
     ui.setconfig('hooks', 'commit.lfiles', checkrequireslfiles)