largefiles: inline _updatelfile, prepare for further refactorings

hgext/largefiles/lfcommands.py

@@ -1,576 +1,566 b''
 # Copyright 2009-2010 Gregory P. Ward
 # Copyright 2009-2010 Intelerad Medical Systems Incorporated
 # Copyright 2010-2011 Fog Creek Software
 # Copyright 2010-2011 Unity Technologies
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''High-level command function for lfconvert, plus the cmdtable.'''
 
 import os, errno
 import shutil
 
 from mercurial import util, match as match_, hg, node, context, error, \
     cmdutil, scmutil, commands
 from mercurial.i18n import _
 from mercurial.lock import release
 
 import lfutil
 import basestore
 
 # -- Commands ----------------------------------------------------------
 
 def lfconvert(ui, src, dest, *pats, **opts):
     '''convert a normal repository to a largefiles repository
 
     Convert repository SOURCE to a new repository DEST, identical to
     SOURCE except that certain files will be converted as largefiles:
     specifically, any file that matches any PATTERN *or* whose size is
     above the minimum size threshold is converted as a largefile. The
     size used to determine whether or not to track a file as a
     largefile is the size of the first version of the file. The
     minimum size can be specified either with --size or in
     configuration as ``largefiles.size``.
 
     After running this command you will need to make sure that
     largefiles is enabled anywhere you intend to push the new
     repository.
 
     Use --to-normal to convert largefiles back to normal files; after
     this, the DEST repository can be used without largefiles at all.'''
 
     if opts['to_normal']:
         tolfile = False
     else:
         tolfile = True
         size = lfutil.getminsize(ui, True, opts.get('size'), default=None)
 
     if not hg.islocal(src):
         raise util.Abort(_('%s is not a local Mercurial repo') % src)
     if not hg.islocal(dest):
         raise util.Abort(_('%s is not a local Mercurial repo') % dest)
 
     rsrc = hg.repository(ui, src)
     ui.status(_('initializing destination %s\n') % dest)
     rdst = hg.repository(ui, dest, create=True)
 
     success = False
     dstwlock = dstlock = None
     try:
         # Lock destination to prevent modification while it is converted to.
         # Don't need to lock src because we are just reading from its history
         # which can't change.
         dstwlock = rdst.wlock()
         dstlock = rdst.lock()
 
         # Get a list of all changesets in the source.  The easy way to do this
         # is to simply walk the changelog, using changelog.nodesbetween().
         # Take a look at mercurial/revlog.py:639 for more details.
         # Use a generator instead of a list to decrease memory usage
         ctxs = (rsrc[ctx] for ctx in rsrc.changelog.nodesbetween(None,
             rsrc.heads())[0])
         revmap = {node.nullid: node.nullid}
         if tolfile:
             lfiles = set()
             normalfiles = set()
             if not pats:
                 pats = ui.configlist(lfutil.longname, 'patterns', default=[])
             if pats:
                 matcher = match_.match(rsrc.root, '', list(pats))
             else:
                 matcher = None
 
             lfiletohash = {}
             for ctx in ctxs:
                 ui.progress(_('converting revisions'), ctx.rev(),
                     unit=_('revision'), total=rsrc['tip'].rev())
                 _lfconvert_addchangeset(rsrc, rdst, ctx, revmap,
                     lfiles, normalfiles, matcher, size, lfiletohash)
             ui.progress(_('converting revisions'), None)
 
             if os.path.exists(rdst.wjoin(lfutil.shortname)):
                 shutil.rmtree(rdst.wjoin(lfutil.shortname))
 
             for f in lfiletohash.keys():
                 if os.path.isfile(rdst.wjoin(f)):
                     os.unlink(rdst.wjoin(f))
                 try:
                     os.removedirs(os.path.dirname(rdst.wjoin(f)))
                 except OSError:
                     pass
 
             # If there were any files converted to largefiles, add largefiles
             # to the destination repository's requirements.
             if lfiles:
                 rdst.requirements.add('largefiles')
                 rdst._writerequirements()
         else:
             for ctx in ctxs:
                 ui.progress(_('converting revisions'), ctx.rev(),
                     unit=_('revision'), total=rsrc['tip'].rev())
                 _addchangeset(ui, rsrc, rdst, ctx, revmap)
 
             ui.progress(_('converting revisions'), None)
         success = True
     finally:
         rdst.dirstate.clear()
         release(dstlock, dstwlock)
         if not success:
             # we failed, remove the new directory
             shutil.rmtree(rdst.root)
 
 def _addchangeset(ui, rsrc, rdst, ctx, revmap):
     # Convert src parents to dst parents
     parents = _convertparents(ctx, revmap)
 
     # Generate list of changed files
     files = _getchangedfiles(ctx, parents)
 
     def getfilectx(repo, memctx, f):
         if lfutil.standin(f) in files:
             # if the file isn't in the manifest then it was removed
             # or renamed, raise IOError to indicate this
             try:
                 fctx = ctx.filectx(lfutil.standin(f))
             except error.LookupError:
                 raise IOError
             renamed = fctx.renamed()
             if renamed:
                 renamed = lfutil.splitstandin(renamed[0])
 
             hash = fctx.data().strip()
             path = lfutil.findfile(rsrc, hash)
 
             # If one file is missing, likely all files from this rev are
             if path is None:
                 cachelfiles(ui, rsrc, ctx.node())
                 path = lfutil.findfile(rsrc, hash)
 
                 if path is None:
                     raise util.Abort(
                         _("missing largefile \'%s\' from revision %s")
                          % (f, node.hex(ctx.node())))
 
             data = ''
             fd = None
             try:
                 fd = open(path, 'rb')
                 data = fd.read()
             finally:
                 if fd:
                     fd.close()
             return context.memfilectx(f, data, 'l' in fctx.flags(),
                                       'x' in fctx.flags(), renamed)
         else:
             return _getnormalcontext(repo.ui, ctx, f, revmap)
 
     dstfiles = []
     for file in files:
         if lfutil.isstandin(file):
             dstfiles.append(lfutil.splitstandin(file))
         else:
             dstfiles.append(file)
     # Commit
     _commitcontext(rdst, parents, ctx, dstfiles, getfilectx, revmap)
 
 def _lfconvert_addchangeset(rsrc, rdst, ctx, revmap, lfiles, normalfiles,
         matcher, size, lfiletohash):
     # Convert src parents to dst parents
     parents = _convertparents(ctx, revmap)
 
     # Generate list of changed files
     files = _getchangedfiles(ctx, parents)
 
     dstfiles = []
     for f in files:
         if f not in lfiles and f not in normalfiles:
             islfile = _islfile(f, ctx, matcher, size)
             # If this file was renamed or copied then copy
             # the largefile-ness of its predecessor
             if f in ctx.manifest():
                 fctx = ctx.filectx(f)
                 renamed = fctx.renamed()
                 renamedlfile = renamed and renamed[0] in lfiles
                 islfile |= renamedlfile
                 if 'l' in fctx.flags():
                     if renamedlfile:
                         raise util.Abort(
                             _('renamed/copied largefile %s becomes symlink')
                             % f)
                     islfile = False
             if islfile:
                 lfiles.add(f)
             else:
                 normalfiles.add(f)
 
         if f in lfiles:
             dstfiles.append(lfutil.standin(f))
             # largefile in manifest if it has not been removed/renamed
             if f in ctx.manifest():
                 fctx = ctx.filectx(f)
                 if 'l' in fctx.flags():
                     renamed = fctx.renamed()
                     if renamed and renamed[0] in lfiles:
                         raise util.Abort(_('largefile %s becomes symlink') % f)
 
                 # largefile was modified, update standins
                 m = util.sha1('')
                 m.update(ctx[f].data())
                 hash = m.hexdigest()
                 if f not in lfiletohash or lfiletohash[f] != hash:
                     rdst.wwrite(f, ctx[f].data(), ctx[f].flags())
                     executable = 'x' in ctx[f].flags()
                     lfutil.writestandin(rdst, lfutil.standin(f), hash,
                         executable)
                     lfiletohash[f] = hash
         else:
             # normal file
             dstfiles.append(f)
 
     def getfilectx(repo, memctx, f):
         if lfutil.isstandin(f):
             # if the file isn't in the manifest then it was removed
             # or renamed, raise IOError to indicate this
             srcfname = lfutil.splitstandin(f)
             try:
                 fctx = ctx.filectx(srcfname)
             except error.LookupError:
                 raise IOError
             renamed = fctx.renamed()
             if renamed:
                 # standin is always a largefile because largefile-ness
                 # doesn't change after rename or copy
                 renamed = lfutil.standin(renamed[0])
 
             return context.memfilectx(f, lfiletohash[srcfname] + '\n', 'l' in
                 fctx.flags(), 'x' in fctx.flags(), renamed)
         else:
             return _getnormalcontext(repo.ui, ctx, f, revmap)
 
     # Commit
     _commitcontext(rdst, parents, ctx, dstfiles, getfilectx, revmap)
 
 def _commitcontext(rdst, parents, ctx, dstfiles, getfilectx, revmap):
     mctx = context.memctx(rdst, parents, ctx.description(), dstfiles,
                           getfilectx, ctx.user(), ctx.date(), ctx.extra())
     ret = rdst.commitctx(mctx)
     rdst.setparents(ret)
     revmap[ctx.node()] = rdst.changelog.tip()
 
 # Generate list of changed files
 def _getchangedfiles(ctx, parents):
     files = set(ctx.files())
     if node.nullid not in parents:
         mc = ctx.manifest()
         mp1 = ctx.parents()[0].manifest()
         mp2 = ctx.parents()[1].manifest()
         files |= (set(mp1) | set(mp2)) - set(mc)
         for f in mc:
             if mc[f] != mp1.get(f, None) or mc[f] != mp2.get(f, None):
                 files.add(f)
     return files
 
 # Convert src parents to dst parents
 def _convertparents(ctx, revmap):
     parents = []
     for p in ctx.parents():
         parents.append(revmap[p.node()])
     while len(parents) < 2:
         parents.append(node.nullid)
     return parents
 
 # Get memfilectx for a normal file
 def _getnormalcontext(ui, ctx, f, revmap):
     try:
         fctx = ctx.filectx(f)
     except error.LookupError:
         raise IOError
     renamed = fctx.renamed()
     if renamed:
         renamed = renamed[0]
 
     data = fctx.data()
     if f == '.hgtags':
         data = _converttags (ui, revmap, data)
     return context.memfilectx(f, data, 'l' in fctx.flags(),
                               'x' in fctx.flags(), renamed)
 
 # Remap tag data using a revision map
 def _converttags(ui, revmap, data):
     newdata = []
     for line in data.splitlines():
         try:
             id, name = line.split(' ', 1)
         except ValueError:
             ui.warn(_('skipping incorrectly formatted tag %s\n'
                 % line))
             continue
         try:
             newid = node.bin(id)
         except TypeError:
             ui.warn(_('skipping incorrectly formatted id %s\n'
                 % id))
             continue
         try:
             newdata.append('%s %s\n' % (node.hex(revmap[newid]),
                 name))
         except KeyError:
             ui.warn(_('no mapping for id %s\n') % id)
             continue
     return ''.join(newdata)
 
 def _islfile(file, ctx, matcher, size):
     '''Return true if file should be considered a largefile, i.e.
     matcher matches it or it is larger than size.'''
     # never store special .hg* files as largefiles
     if file == '.hgtags' or file == '.hgignore' or file == '.hgsigs':
         return False
     if matcher and matcher(file):
         return True
     try:
         return ctx.filectx(file).size() >= size * 1024 * 1024
     except error.LookupError:
         return False
 
 def uploadlfiles(ui, rsrc, rdst, files):
     '''upload largefiles to the central store'''
 
     if not files:
         return
 
     store = basestore._openstore(rsrc, rdst, put=True)
 
     at = 0
     ui.debug("sending statlfile command for %d largefiles\n" % len(files))
     retval = store.exists(files)
     files = filter(lambda h: not retval[h], files)
     ui.debug("%d largefiles need to be uploaded\n" % len(files))
 
     for hash in files:
         ui.progress(_('uploading largefiles'), at, unit='largefile',
                     total=len(files))
         source = lfutil.findfile(rsrc, hash)
         if not source:
             raise util.Abort(_('largefile %s missing from store'
                                ' (needs to be uploaded)') % hash)
         # XXX check for errors here
         store.put(source, hash)
         at += 1
     ui.progress(_('uploading largefiles'), None)
 
 def verifylfiles(ui, repo, all=False, contents=False):
     '''Verify that every largefile revision in the current changeset
     exists in the central store.  With --contents, also verify that
     the contents of each local largefile file revision are correct (SHA-1 hash
     matches the revision ID).  With --all, check every changeset in
     this repository.'''
     if all:
         # Pass a list to the function rather than an iterator because we know a
         # list will work.
         revs = range(len(repo))
     else:
         revs = ['.']
 
     store = basestore._openstore(repo)
     return store.verify(revs, contents=contents)
 
 def debugdirstate(ui, repo):
     '''Show basic information for the largefiles dirstate'''
     lfdirstate = lfutil.openlfdirstate(ui, repo)
     for file_, ent in sorted(lfdirstate._map.iteritems()):
         mode = '%3o' % (ent[1] & 0777 & ~util.umask)
         ui.write("%c %s %10d %s\n" % (ent[0], mode, ent[2], file_))
 
 def cachelfiles(ui, repo, node, filelist=None):
     '''cachelfiles ensures that all largefiles needed by the specified revision
     are present in the repository's largefile cache.
 
     returns a tuple (cached, missing).  cached is the list of files downloaded
     by this operation; missing is the list of files that were needed but could
     not be found.'''
     lfiles = lfutil.listlfiles(repo, node)
     if filelist:
         lfiles = set(lfiles) & set(filelist)
     toget = []
 
     for lfile in lfiles:
         try:
             expectedhash = repo[node][lfutil.standin(lfile)].data().strip()
         except IOError, err:
             if err.errno == errno.ENOENT:
                 continue # node must be None and standin wasn't found in wctx
             raise
         if not lfutil.findfile(repo, expectedhash):
             toget.append((lfile, expectedhash))
 
     if toget:
         store = basestore._openstore(repo)
         ret = store.get(toget)
         return ret
 
     return ([], [])
 
 def downloadlfiles(ui, repo, rev=None):
     matchfn = scmutil.match(repo[None],
                             [repo.wjoin(lfutil.shortname)], {})
     def prepare(ctx, fns):
         pass
     totalsuccess = 0
     totalmissing = 0
     if rev != []: # walkchangerevs on empty list would return all revs
         for ctx in cmdutil.walkchangerevs(repo, matchfn, {'rev' : rev},
                                           prepare):
             success, missing = cachelfiles(ui, repo, ctx.node())
             totalsuccess += len(success)
             totalmissing += len(missing)
     ui.status(_("%d additional largefiles cached\n") % totalsuccess)
     if totalmissing > 0:
         ui.status(_("%d largefiles failed to download\n") % totalmissing)
     return totalsuccess, totalmissing
 
 def updatelfiles(ui, repo, filelist=None, printmessage=True):
     wlock = repo.wlock()
     try:
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         lfiles = set(lfutil.listlfiles(repo)) | set(lfdirstate)
 
         if filelist is not None:
             lfiles = [f for f in lfiles if f in filelist]
 
         if lfiles:
             if printmessage:
                 ui.status(_('getting changed largefiles\n'))
             cachelfiles(ui, repo, None, lfiles)
 
         updated, removed = 0, 0
-        for f in lfiles:
-            i = _updatelfile(repo, lfdirstate, f)
-            if i:
-                if i > 0:
-                    updated += i
+        for lfile in lfiles:
+            # updates a single largefile and copies the state of its standin from
+            # the repository's dirstate to its state in the lfdirstate.
+            abslfile = repo.wjoin(lfile)
+            absstandin = repo.wjoin(lfutil.standin(lfile))
+            if os.path.exists(absstandin):
+                if (os.path.exists(absstandin + '.orig') and
+                    os.path.exists(abslfile)):
+                    shutil.copyfile(abslfile, abslfile + '.orig')
+                update1 = 0
+                expecthash = lfutil.readstandin(repo, lfile)
+                if (expecthash != '' and
+                    (not os.path.exists(abslfile) or
+                     expecthash != lfutil.hashfile(abslfile))):
+                    if not lfutil.copyfromcache(repo, expecthash, lfile):
+                        # use normallookup() to allocate entry in largefiles
+                        # dirstate, because lack of it misleads
+                        # lfilesrepo.status() into recognition that such cache
+                        # missing files are REMOVED.
+                        if lfile not in repo[None]: # not switched to normal
+                            util.unlinkpath(abslfile, ignoremissing=True)
+                        lfdirstate.normallookup(lfile)
+                        continue # don't try to set the mode
+                    else:
+                        # Synchronize largefile dirstate to the last modified
+                        # time of the file
+                        lfdirstate.normal(lfile)
+                    update1 = 1
+                mode = os.stat(absstandin).st_mode
+                if mode != os.stat(abslfile).st_mode:
+                    os.chmod(abslfile, mode)
+                    update1 = 1
+                updated += update1
+            else:
+                # Remove lfiles for which the standin is deleted, unless the
+                # lfile is added to the repository again. This happens when a
+                # largefile is converted back to a normal file: the standin
+                # disappears, but a new (normal) file appears as the lfile.
+                if (os.path.exists(abslfile) and
+                    repo.dirstate.normalize(lfile) not in repo[None]):
+                    util.unlinkpath(abslfile)
+                    removed += 1
+            state = repo.dirstate[lfutil.standin(lfile)]
+            if state == 'n':
+                # When rebasing, we need to synchronize the standin and the
+                # largefile, because otherwise the largefile will get reverted.
+                # But for commit's sake, we have to mark the file as unclean.
+                if getattr(repo, "_isrebasing", False):
+                    lfdirstate.normallookup(lfile)
                 else:
-                    removed -= i
+                    lfdirstate.normal(lfile)
+            elif state == 'r':
+                lfdirstate.remove(lfile)
+            elif state == 'a':
+                lfdirstate.add(lfile)
+            elif state == '?':
+                lfdirstate.drop(lfile)
 
         lfdirstate.write()
         if printmessage and lfiles:
             ui.status(_('%d largefiles updated, %d removed\n') % (updated,
                 removed))
     finally:
         wlock.release()
 
-def _updatelfile(repo, lfdirstate, lfile):
-    '''updates a single largefile and copies the state of its standin from
-    the repository's dirstate to its state in the lfdirstate.
-
-    returns 1 if the file was modified, -1 if the file was removed, 0 if the
-    file was unchanged, and None if the needed largefile was missing from the
-    cache.'''
-    ret = 0
-    abslfile = repo.wjoin(lfile)
-    absstandin = repo.wjoin(lfutil.standin(lfile))
-    if os.path.exists(absstandin):
-        if os.path.exists(absstandin + '.orig') and os.path.exists(abslfile):
-            shutil.copyfile(abslfile, abslfile + '.orig')
-        expecthash = lfutil.readstandin(repo, lfile)
-        if (expecthash != '' and
-            (not os.path.exists(abslfile) or
-             expecthash != lfutil.hashfile(abslfile))):
-            if not lfutil.copyfromcache(repo, expecthash, lfile):
-                # use normallookup() to allocate entry in largefiles dirstate,
-                # because lack of it misleads lfilesrepo.status() into
-                # recognition that such cache missing files are REMOVED.
-                if lfile not in repo[None]: # not switched to normal file
-                    util.unlinkpath(abslfile, ignoremissing=True)
-                lfdirstate.normallookup(lfile)
-                return None # don't try to set the mode
-            else:
-                # Synchronize largefile dirstate to the last modified time of
-                # the file
-                lfdirstate.normal(lfile)
-            ret = 1
-        mode = os.stat(absstandin).st_mode
-        if mode != os.stat(abslfile).st_mode:
-            os.chmod(abslfile, mode)
-            ret = 1
-    else:
-        # Remove lfiles for which the standin is deleted, unless the
-        # lfile is added to the repository again. This happens when a
-        # largefile is converted back to a normal file: the standin
-        # disappears, but a new (normal) file appears as the lfile.
-        if (os.path.exists(abslfile) and
-            repo.dirstate.normalize(lfile) not in repo[None]):
-            util.unlinkpath(abslfile)
-            ret = -1
-    state = repo.dirstate[lfutil.standin(lfile)]
-    if state == 'n':
-        # When rebasing, we need to synchronize the standin and the largefile,
-        # because otherwise the largefile will get reverted.  But for commit's
-        # sake, we have to mark the file as unclean.
-        if getattr(repo, "_isrebasing", False):
-            lfdirstate.normallookup(lfile)
-        else:
-            lfdirstate.normal(lfile)
-    elif state == 'r':
-        lfdirstate.remove(lfile)
-    elif state == 'a':
-        lfdirstate.add(lfile)
-    elif state == '?':
-        lfdirstate.drop(lfile)
-    return ret
-
 def lfpull(ui, repo, source="default", **opts):
     """pull largefiles for the specified revisions from the specified source
 
     Pull largefiles that are referenced from local changesets but missing
     locally, pulling from a remote repository to the local cache.
 
     If SOURCE is omitted, the 'default' path will be used.
     See :hg:`help urls` for more information.
 
     .. container:: verbose
 
       Some examples:
 
       - pull largefiles for all branch heads::
 
           hg lfpull -r "head() and not closed()"
 
       - pull largefiles on the default branch::
 
           hg lfpull -r "branch(default)"
     """
     repo.lfpullsource = source
 
     revs = opts.get('rev', [])
     if not revs:
         raise util.Abort(_('no revisions specified'))
     revs = scmutil.revrange(repo, revs)
 
     numcached = 0
     for rev in revs:
         ui.note(_('pulling largefiles for revision %s\n') % rev)
         (cached, missing) = cachelfiles(ui, repo, rev)
         numcached += len(cached)
     ui.status(_("%d largefiles cached\n") % numcached)
 
 # -- hg commands declarations ------------------------------------------------
 
 cmdtable = {
     'lfconvert': (lfconvert,
                   [('s', 'size', '',
                     _('minimum size (MB) for files to be converted '
                       'as largefiles'),
                     'SIZE'),
                   ('', 'to-normal', False,
                    _('convert from a largefiles repo to a normal repo')),
                   ],
                   _('hg lfconvert SOURCE DEST [FILE ...]')),
     'lfpull': (lfpull,
                [('r', 'rev', [], _('pull largefiles for these revisions'))
                 ] +  commands.remoteopts,
                _('-r REV... [-e CMD] [--remotecmd CMD] [SOURCE]')
                ),
     }
 
 commands.inferrepo += " lfconvert"