largefiles: add --normal option to hg add (issue3061)

hgext/largefiles/overrides.py

@@ -1,951 +1,956
 # Copyright 2009-2010 Gregory P. Ward
 # Copyright 2009-2010 Intelerad Medical Systems Incorporated
 # Copyright 2010-2011 Fog Creek Software
 # Copyright 2010-2011 Unity Technologies
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''Overridden Mercurial commands and functions for the largefiles extension'''
 
 import os
 import copy
 
 from mercurial import hg, commands, util, cmdutil, scmutil, match as match_, \
     node, archival, error, merge
 from mercurial.i18n import _
 from mercurial.node import hex
 from hgext import rebase
 
 import lfutil
 import lfcommands
 
 # -- Utility functions: commonly/repeatedly needed functionality ---------------
 
 def installnormalfilesmatchfn(manifest):
     '''overrides scmutil.match so that the matcher it returns will ignore all
     largefiles'''
     oldmatch = None # for the closure
     def override_match(ctx, pats=[], opts={}, globbed=False,
             default='relpath'):
         match = oldmatch(ctx, pats, opts, globbed, default)
         m = copy.copy(match)
         notlfile = lambda f: not (lfutil.isstandin(f) or lfutil.standin(f) in
                 manifest)
         m._files = filter(notlfile, m._files)
         m._fmap = set(m._files)
         orig_matchfn = m.matchfn
         m.matchfn = lambda f: notlfile(f) and orig_matchfn(f) or None
         return m
     oldmatch = installmatchfn(override_match)
 
 def installmatchfn(f):
     oldmatch = scmutil.match
     setattr(f, 'oldmatch', oldmatch)
     scmutil.match = f
     return oldmatch
 
 def restorematchfn():
     '''restores scmutil.match to what it was before installnormalfilesmatchfn
     was called.  no-op if scmutil.match is its original function.
 
     Note that n calls to installnormalfilesmatchfn will require n calls to
     restore matchfn to reverse'''
     scmutil.match = getattr(scmutil.match, 'oldmatch', scmutil.match)
 
 def add_largefiles(ui, repo, *pats, **opts):
     large = opts.pop('large', None)
     lfsize = lfutil.getminsize(
         ui, lfutil.islfilesrepo(repo), opts.pop('lfsize', None))
 
     lfmatcher = None
     if lfutil.islfilesrepo(repo):
         lfpats = ui.configlist(lfutil.longname, 'patterns', default=[])
         if lfpats:
             lfmatcher = match_.match(repo.root, '', list(lfpats))
 
     lfnames = []
     m = scmutil.match(repo[None], pats, opts)
     m.bad = lambda x, y: None
     wctx = repo[None]
     for f in repo.walk(m):
         exact = m.exact(f)
         lfile = lfutil.standin(f) in wctx
         nfile = f in wctx
         exists = lfile or nfile
 
         # Don't warn the user when they attempt to add a normal tracked file.
         # The normal add code will do that for us.
         if exact and exists:
             if lfile:
                 ui.warn(_('%s already a largefile\n') % f)
             continue
 
         if exact or not exists:
             abovemin = (lfsize and
                         os.lstat(repo.wjoin(f)).st_size >= lfsize * 1024 * 1024)
             if large or abovemin or (lfmatcher and lfmatcher(f)):
                 lfnames.append(f)
                 if ui.verbose or not exact:
                     ui.status(_('adding %s as a largefile\n') % m.rel(f))
 
     bad = []
     standins = []
 
     # Need to lock, otherwise there could be a race condition between
     # when standins are created and added to the repo.
     wlock = repo.wlock()
     try:
         if not opts.get('dry_run'):
             lfdirstate = lfutil.openlfdirstate(ui, repo)
             for f in lfnames:
                 standinname = lfutil.standin(f)
                 lfutil.writestandin(repo, standinname, hash='',
                     executable=lfutil.getexecutable(repo.wjoin(f)))
                 standins.append(standinname)
                 if lfdirstate[f] == 'r':
                     lfdirstate.normallookup(f)
                 else:
                     lfdirstate.add(f)
             lfdirstate.write()
             bad += [lfutil.splitstandin(f)
                     for f in lfutil.repo_add(repo, standins)
                     if f in m.files()]
     finally:
         wlock.release()
     return bad
 
 def remove_largefiles(ui, repo, *pats, **opts):
     after = opts.get('after')
     if not pats and not after:
         raise util.Abort(_('no files specified'))
     m = scmutil.match(repo[None], pats, opts)
     try:
         repo.lfstatus = True
         s = repo.status(match=m, clean=True)
     finally:
         repo.lfstatus = False
     manifest = repo[None].manifest()
     modified, added, deleted, clean = [[f for f in list
                                         if lfutil.standin(f) in manifest]
                                        for list in [s[0], s[1], s[3], s[6]]]
 
     def warn(files, reason):
         for f in files:
             ui.warn(_('not removing %s: %s (use forget to undo)\n')
                     % (m.rel(f), reason))
 
     if after:
         remove, forget = deleted, []
         warn(modified + added + clean, _('file still exists'))
     else:
         remove, forget = deleted + clean, []
         warn(modified, _('file is modified'))
         warn(added, _('file has been marked for add'))
 
     for f in sorted(remove + forget):
         if ui.verbose or not m.exact(f):
             ui.status(_('removing %s\n') % m.rel(f))
 
     # Need to lock because standin files are deleted then removed from the
     # repository and we could race inbetween.
     wlock = repo.wlock()
     try:
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         for f in remove:
             if not after:
                 # If this is being called by addremove, notify the user that we
                 # are removing the file.
                 if getattr(repo, "_isaddremove", False):
                     ui.status(_('removing %s\n' % f))
                 if os.path.exists(repo.wjoin(f)):
                     util.unlinkpath(repo.wjoin(f))
             lfdirstate.remove(f)
         lfdirstate.write()
         forget = [lfutil.standin(f) for f in forget]
         remove = [lfutil.standin(f) for f in remove]
         lfutil.repo_forget(repo, forget)
         # If this is being called by addremove, let the original addremove
         # function handle this.
         if not getattr(repo, "_isaddremove", False):
             lfutil.repo_remove(repo, remove, unlink=True)
     finally:
         wlock.release()
 
 # -- Wrappers: modify existing commands --------------------------------
 
 # Add works by going through the files that the user wanted to add and
 # checking if they should be added as largefiles. Then it makes a new
 # matcher which matches only the normal files and runs the original
 # version of add.
 def override_add(orig, ui, repo, *pats, **opts):
+    normal = opts.pop('normal')
+    if normal:
+        if opts.get('large'):
+            raise util.Abort(_('--normal cannot be used with --large'))
+        return orig(ui, repo, *pats, **opts)
     bad = add_largefiles(ui, repo, *pats, **opts)
     installnormalfilesmatchfn(repo[None].manifest())
     result = orig(ui, repo, *pats, **opts)
     restorematchfn()
 
     return (result == 1 or bad) and 1 or 0
 
 def override_remove(orig, ui, repo, *pats, **opts):
     installnormalfilesmatchfn(repo[None].manifest())
     orig(ui, repo, *pats, **opts)
     restorematchfn()
     remove_largefiles(ui, repo, *pats, **opts)
 
 def override_status(orig, ui, repo, *pats, **opts):
     try:
         repo.lfstatus = True
         return orig(ui, repo, *pats, **opts)
     finally:
         repo.lfstatus = False
 
 def override_log(orig, ui, repo, *pats, **opts):
     try:
         repo.lfstatus = True
         orig(ui, repo, *pats, **opts)
     finally:
         repo.lfstatus = False
 
 def override_verify(orig, ui, repo, *pats, **opts):
     large = opts.pop('large', False)
     all = opts.pop('lfa', False)
     contents = opts.pop('lfc', False)
 
     result = orig(ui, repo, *pats, **opts)
     if large:
         result = result or lfcommands.verifylfiles(ui, repo, all, contents)
     return result
 
 # Override needs to refresh standins so that update's normal merge
 # will go through properly. Then the other update hook (overriding repo.update)
 # will get the new files. Filemerge is also overriden so that the merge
 # will merge standins correctly.
 def override_update(orig, ui, repo, *pats, **opts):
     lfdirstate = lfutil.openlfdirstate(ui, repo)
     s = lfdirstate.status(match_.always(repo.root, repo.getcwd()), [], False,
         False, False)
     (unsure, modified, added, removed, missing, unknown, ignored, clean) = s
 
     # Need to lock between the standins getting updated and their
     # largefiles getting updated
     wlock = repo.wlock()
     try:
         if opts['check']:
             mod = len(modified) > 0
             for lfile in unsure:
                 standin = lfutil.standin(lfile)
                 if repo['.'][standin].data().strip() != \
                         lfutil.hashfile(repo.wjoin(lfile)):
                     mod = True
                 else:
                     lfdirstate.normal(lfile)
             lfdirstate.write()
             if mod:
                 raise util.Abort(_('uncommitted local changes'))
         # XXX handle removed differently
         if not opts['clean']:
             for lfile in unsure + modified + added:
                 lfutil.updatestandin(repo, lfutil.standin(lfile))
     finally:
         wlock.release()
     return orig(ui, repo, *pats, **opts)
 
 # Before starting the manifest merge, merge.updates will call
 # _checkunknown to check if there are any files in the merged-in
 # changeset that collide with unknown files in the working copy.
 #
 # The largefiles are seen as unknown, so this prevents us from merging
 # in a file 'foo' if we already have a largefile with the same name.
 #
 # The overridden function filters the unknown files by removing any
 # largefiles. This makes the merge proceed and we can then handle this
 # case further in the overridden manifestmerge function below.
 def override_checkunknown(origfn, wctx, mctx, folding):
     origunknown = wctx.unknown()
     wctx._unknown = filter(lambda f: lfutil.standin(f) not in wctx, origunknown)
     try:
         return origfn(wctx, mctx, folding)
     finally:
         wctx._unknown = origunknown
 
 # The manifest merge handles conflicts on the manifest level. We want
 # to handle changes in largefile-ness of files at this level too.
 #
 # The strategy is to run the original manifestmerge and then process
 # the action list it outputs. There are two cases we need to deal with:
 #
 # 1. Normal file in p1, largefile in p2. Here the largefile is
 #    detected via its standin file, which will enter the working copy
 #    with a "get" action. It is not "merge" since the standin is all
 #    Mercurial is concerned with at this level -- the link to the
 #    existing normal file is not relevant here.
 #
 # 2. Largefile in p1, normal file in p2. Here we get a "merge" action
 #    since the largefile will be present in the working copy and
 #    different from the normal file in p2. Mercurial therefore
 #    triggers a merge action.
 #
 # In both cases, we prompt the user and emit new actions to either
 # remove the standin (if the normal file was kept) or to remove the
 # normal file and get the standin (if the largefile was kept). The
 # default prompt answer is to use the largefile version since it was
 # presumably changed on purpose.
 #
 # Finally, the merge.applyupdates function will then take care of
 # writing the files into the working copy and lfcommands.updatelfiles
 # will update the largefiles.
 def override_manifestmerge(origfn, repo, p1, p2, pa, overwrite, partial):
     actions = origfn(repo, p1, p2, pa, overwrite, partial)
     processed = []
 
     for action in actions:
         if overwrite:
             processed.append(action)
             continue
         f, m = action[:2]
 
         choices = (_('&Largefile'), _('&Normal file'))
         if m == "g" and lfutil.splitstandin(f) in p1 and f in p2:
             # Case 1: normal file in the working copy, largefile in
             # the second parent
             lfile = lfutil.splitstandin(f)
             standin = f
             msg = _('%s has been turned into a largefile\n'
                     'use (l)argefile or keep as (n)ormal file?') % lfile
             if repo.ui.promptchoice(msg, choices, 0) == 0:
                 processed.append((lfile, "r"))
                 processed.append((standin, "g", p2.flags(standin)))
             else:
                 processed.append((standin, "r"))
         elif m == "m" and lfutil.standin(f) in p1 and f in p2:
             # Case 2: largefile in the working copy, normal file in
             # the second parent
             standin = lfutil.standin(f)
             lfile = f
             msg = _('%s has been turned into a normal file\n'
                     'keep as (l)argefile or use (n)ormal file?') % lfile
             if repo.ui.promptchoice(msg, choices, 0) == 0:
                 processed.append((lfile, "r"))
             else:
                 processed.append((standin, "r"))
                 processed.append((lfile, "g", p2.flags(lfile)))
         else:
             processed.append(action)
 
     return processed
 
 # Override filemerge to prompt the user about how they wish to merge
 # largefiles. This will handle identical edits, and copy/rename +
 # edit without prompting the user.
 def override_filemerge(origfn, repo, mynode, orig, fcd, fco, fca):
     # Use better variable names here. Because this is a wrapper we cannot
     # change the variable names in the function declaration.
     fcdest, fcother, fcancestor = fcd, fco, fca
     if not lfutil.isstandin(orig):
         return origfn(repo, mynode, orig, fcdest, fcother, fcancestor)
     else:
         if not fcother.cmp(fcdest): # files identical?
             return None
 
         # backwards, use working dir parent as ancestor
         if fcancestor == fcother:
             fcancestor = fcdest.parents()[0]
 
         if orig != fcother.path():
             repo.ui.status(_('merging %s and %s to %s\n')
                            % (lfutil.splitstandin(orig),
                               lfutil.splitstandin(fcother.path()),
                               lfutil.splitstandin(fcdest.path())))
         else:
             repo.ui.status(_('merging %s\n')
                            % lfutil.splitstandin(fcdest.path()))
 
         if fcancestor.path() != fcother.path() and fcother.data() == \
                 fcancestor.data():
             return 0
         if fcancestor.path() != fcdest.path() and fcdest.data() == \
                 fcancestor.data():
             repo.wwrite(fcdest.path(), fcother.data(), fcother.flags())
             return 0
 
         if repo.ui.promptchoice(_('largefile %s has a merge conflict\n'
                              'keep (l)ocal or take (o)ther?') %
                              lfutil.splitstandin(orig),
                              (_('&Local'), _('&Other')), 0) == 0:
             return 0
         else:
             repo.wwrite(fcdest.path(), fcother.data(), fcother.flags())
             return 0
 
 # Copy first changes the matchers to match standins instead of
 # largefiles.  Then it overrides util.copyfile in that function it
 # checks if the destination largefile already exists. It also keeps a
 # list of copied files so that the largefiles can be copied and the
 # dirstate updated.
 def override_copy(orig, ui, repo, pats, opts, rename=False):
     # doesn't remove largefile on rename
     if len(pats) < 2:
         # this isn't legal, let the original function deal with it
         return orig(ui, repo, pats, opts, rename)
 
     def makestandin(relpath):
         path = scmutil.canonpath(repo.root, repo.getcwd(), relpath)
         return os.path.join(repo.wjoin(lfutil.standin(path)))
 
     fullpats = scmutil.expandpats(pats)
     dest = fullpats[-1]
 
     if os.path.isdir(dest):
         if not os.path.isdir(makestandin(dest)):
             os.makedirs(makestandin(dest))
     # This could copy both lfiles and normal files in one command,
     # but we don't want to do that. First replace their matcher to
     # only match normal files and run it, then replace it to just
     # match largefiles and run it again.
     nonormalfiles = False
     nolfiles = False
     try:
         try:
             installnormalfilesmatchfn(repo[None].manifest())
             result = orig(ui, repo, pats, opts, rename)
         except util.Abort, e:
             if str(e) != 'no files to copy':
                 raise e
             else:
                 nonormalfiles = True
             result = 0
     finally:
         restorematchfn()
 
     # The first rename can cause our current working directory to be removed.
     # In that case there is nothing left to copy/rename so just quit.
     try:
         repo.getcwd()
     except OSError:
         return result
 
     try:
         try:
             # When we call orig below it creates the standins but we don't add them
             # to the dir state until later so lock during that time.
             wlock = repo.wlock()
 
             manifest = repo[None].manifest()
             oldmatch = None # for the closure
             def override_match(ctx, pats=[], opts={}, globbed=False,
                     default='relpath'):
                 newpats = []
                 # The patterns were previously mangled to add the standin
                 # directory; we need to remove that now
                 for pat in pats:
                     if match_.patkind(pat) is None and lfutil.shortname in pat:
                         newpats.append(pat.replace(lfutil.shortname, ''))
                     else:
                         newpats.append(pat)
                 match = oldmatch(ctx, newpats, opts, globbed, default)
                 m = copy.copy(match)
                 lfile = lambda f: lfutil.standin(f) in manifest
                 m._files = [lfutil.standin(f) for f in m._files if lfile(f)]
                 m._fmap = set(m._files)
                 orig_matchfn = m.matchfn
                 m.matchfn = lambda f: (lfutil.isstandin(f) and
                                     lfile(lfutil.splitstandin(f)) and
                                     orig_matchfn(lfutil.splitstandin(f)) or
                                     None)
                 return m
             oldmatch = installmatchfn(override_match)
             listpats = []
             for pat in pats:
                 if match_.patkind(pat) is not None:
                     listpats.append(pat)
                 else:
                     listpats.append(makestandin(pat))
 
             try:
                 origcopyfile = util.copyfile
                 copiedfiles = []
                 def override_copyfile(src, dest):
                     if (lfutil.shortname in src and
                         dest.startswith(repo.wjoin(lfutil.shortname))):
                         destlfile = dest.replace(lfutil.shortname, '')
                         if not opts['force'] and os.path.exists(destlfile):
                             raise IOError('',
                                 _('destination largefile already exists'))
                     copiedfiles.append((src, dest))
                     origcopyfile(src, dest)
 
                 util.copyfile = override_copyfile
                 result += orig(ui, repo, listpats, opts, rename)
             finally:
                 util.copyfile = origcopyfile
 
             lfdirstate = lfutil.openlfdirstate(ui, repo)
             for (src, dest) in copiedfiles:
                 if (lfutil.shortname in src and
                     dest.startswith(repo.wjoin(lfutil.shortname))):
                     srclfile = src.replace(repo.wjoin(lfutil.standin('')), '')
                     destlfile = dest.replace(repo.wjoin(lfutil.standin('')), '')
                     destlfiledir = os.path.dirname(destlfile) or '.'
                     if not os.path.isdir(destlfiledir):
                         os.makedirs(destlfiledir)
                     if rename:
                         os.rename(repo.wjoin(srclfile), repo.wjoin(destlfile))
                         lfdirstate.remove(srclfile)
                     else:
                         util.copyfile(srclfile, destlfile)
                     lfdirstate.add(destlfile)
             lfdirstate.write()
         except util.Abort, e:
             if str(e) != 'no files to copy':
                 raise e
             else:
                 nolfiles = True
     finally:
         restorematchfn()
         wlock.release()
 
     if nolfiles and nonormalfiles:
         raise util.Abort(_('no files to copy'))
 
     return result
 
 # When the user calls revert, we have to be careful to not revert any
 # changes to other largefiles accidentally. This means we have to keep
 # track of the largefiles that are being reverted so we only pull down
 # the necessary largefiles.
 #
 # Standins are only updated (to match the hash of largefiles) before
 # commits. Update the standins then run the original revert, changing
 # the matcher to hit standins instead of largefiles. Based on the
 # resulting standins update the largefiles. Then return the standins
 # to their proper state
 def override_revert(orig, ui, repo, *pats, **opts):
     # Because we put the standins in a bad state (by updating them)
     # and then return them to a correct state we need to lock to
     # prevent others from changing them in their incorrect state.
     wlock = repo.wlock()
     try:
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         (modified, added, removed, missing, unknown, ignored, clean) = \
             lfutil.lfdirstate_status(lfdirstate, repo, repo['.'].rev())
         for lfile in modified:
             lfutil.updatestandin(repo, lfutil.standin(lfile))
 
         try:
             ctx = repo[opts.get('rev')]
             oldmatch = None # for the closure
             def override_match(ctx, pats=[], opts={}, globbed=False,
                     default='relpath'):
                 match = oldmatch(ctx, pats, opts, globbed, default)
                 m = copy.copy(match)
                 def tostandin(f):
                     if lfutil.standin(f) in ctx or lfutil.standin(f) in ctx:
                         return lfutil.standin(f)
                     elif lfutil.standin(f) in repo[None]:
                         return None
                     return f
                 m._files = [tostandin(f) for f in m._files]
                 m._files = [f for f in m._files if f is not None]
                 m._fmap = set(m._files)
                 orig_matchfn = m.matchfn
                 def matchfn(f):
                     if lfutil.isstandin(f):
                         # We need to keep track of what largefiles are being
                         # matched so we know which ones to update later --
                         # otherwise we accidentally revert changes to other
                         # largefiles. This is repo-specific, so duckpunch the
                         # repo object to keep the list of largefiles for us
                         # later.
                         if orig_matchfn(lfutil.splitstandin(f)) and \
                                 (f in repo[None] or f in ctx):
                             lfileslist = getattr(repo, '_lfilestoupdate', [])
                             lfileslist.append(lfutil.splitstandin(f))
                             repo._lfilestoupdate = lfileslist
                             return True
                         else:
                             return False
                     return orig_matchfn(f)
                 m.matchfn = matchfn
                 return m
             oldmatch = installmatchfn(override_match)
             scmutil.match
             matches = override_match(repo[None], pats, opts)
             orig(ui, repo, *pats, **opts)
         finally:
             restorematchfn()
         lfileslist = getattr(repo, '_lfilestoupdate', [])
         lfcommands.updatelfiles(ui, repo, filelist=lfileslist,
                                 printmessage=False)
 
         # empty out the largefiles list so we start fresh next time
         repo._lfilestoupdate = []
         for lfile in modified:
             if lfile in lfileslist:
                 if os.path.exists(repo.wjoin(lfutil.standin(lfile))) and lfile\
                         in repo['.']:
                     lfutil.writestandin(repo, lfutil.standin(lfile),
                         repo['.'][lfile].data().strip(),
                         'x' in repo['.'][lfile].flags())
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         for lfile in added:
             standin = lfutil.standin(lfile)
             if standin not in ctx and (standin in matches or opts.get('all')):
                 if lfile in lfdirstate:
                     lfdirstate.drop(lfile)
                 util.unlinkpath(repo.wjoin(standin))
         lfdirstate.write()
     finally:
         wlock.release()
 
 def hg_update(orig, repo, node):
     result = orig(repo, node)
     lfcommands.updatelfiles(repo.ui, repo)
     return result
 
 def hg_clean(orig, repo, node, show_stats=True):
     result = orig(repo, node, show_stats)
     lfcommands.updatelfiles(repo.ui, repo)
     return result
 
 def hg_merge(orig, repo, node, force=None, remind=True):
     # Mark the repo as being in the middle of a merge, so that
     # updatelfiles() will know that it needs to trust the standins in
     # the working copy, not in the standins in the current node
     repo._ismerging = True
     try:
         result = orig(repo, node, force, remind)
         lfcommands.updatelfiles(repo.ui, repo)
     finally:
         repo._ismerging = False
     return result
 
 # When we rebase a repository with remotely changed largefiles, we need to
 # take some extra care so that the largefiles are correctly updated in the
 # working copy
 def override_pull(orig, ui, repo, source=None, **opts):
     if opts.get('rebase', False):
         repo._isrebasing = True
         try:
             if opts.get('update'):
                  del opts['update']
                  ui.debug('--update and --rebase are not compatible, ignoring '
                           'the update flag\n')
             del opts['rebase']
             cmdutil.bailifchanged(repo)
             revsprepull = len(repo)
             origpostincoming = commands.postincoming
             def _dummy(*args, **kwargs):
                 pass
             commands.postincoming = _dummy
             repo.lfpullsource = source
             if not source:
                 source = 'default'
             try:
                 result = commands.pull(ui, repo, source, **opts)
             finally:
                 commands.postincoming = origpostincoming
             revspostpull = len(repo)
             if revspostpull > revsprepull:
                 result = result or rebase.rebase(ui, repo)
         finally:
             repo._isrebasing = False
     else:
         repo.lfpullsource = source
         if not source:
             source = 'default'
         result = orig(ui, repo, source, **opts)
         # If we do not have the new largefiles for any new heads we pulled, we
         # will run into a problem later if we try to merge or rebase with one of
         # these heads, so cache the largefiles now direclty into the system
         # cache.
         ui.status(_("caching new largefiles\n"))
         numcached = 0
         branches = repo.branchmap()
         for branch in branches:
             heads = repo.branchheads(branch)
             for head in heads:
                 (cached, missing) = lfcommands.cachelfiles(ui, repo, head)
                 numcached += len(cached)
         ui.status(_("%d largefiles cached\n" % numcached))
     return result
 
 def override_rebase(orig, ui, repo, **opts):
     repo._isrebasing = True
     try:
         orig(ui, repo, **opts)
     finally:
         repo._isrebasing = False
 
 def override_archive(orig, repo, dest, node, kind, decode=True, matchfn=None,
             prefix=None, mtime=None, subrepos=None):
     # No need to lock because we are only reading history and
     # largefile caches, neither of which are modified.
     lfcommands.cachelfiles(repo.ui, repo, node)
 
     if kind not in archival.archivers:
         raise util.Abort(_("unknown archive type '%s'") % kind)
 
     ctx = repo[node]
 
     if kind == 'files':
         if prefix:
             raise util.Abort(
                 _('cannot give prefix when archiving to files'))
     else:
         prefix = archival.tidyprefix(dest, kind, prefix)
 
     def write(name, mode, islink, getdata):
         if matchfn and not matchfn(name):
             return
         data = getdata()
         if decode:
             data = repo.wwritedata(name, data)
         archiver.addfile(prefix + name, mode, islink, data)
 
     archiver = archival.archivers[kind](dest, mtime or ctx.date()[0])
 
     if repo.ui.configbool("ui", "archivemeta", True):
         def metadata():
             base = 'repo: %s\nnode: %s\nbranch: %s\n' % (
                 hex(repo.changelog.node(0)), hex(node), ctx.branch())
 
             tags = ''.join('tag: %s\n' % t for t in ctx.tags()
                            if repo.tagtype(t) == 'global')
             if not tags:
                 repo.ui.pushbuffer()
                 opts = {'template': '{latesttag}\n{latesttagdistance}',
                         'style': '', 'patch': None, 'git': None}
                 cmdutil.show_changeset(repo.ui, repo, opts).show(ctx)
                 ltags, dist = repo.ui.popbuffer().split('\n')
                 tags = ''.join('latesttag: %s\n' % t for t in ltags.split(':'))
                 tags += 'latesttagdistance: %s\n' % dist
 
             return base + tags
 
         write('.hg_archival.txt', 0644, False, metadata)
 
     for f in ctx:
         ff = ctx.flags(f)
         getdata = ctx[f].data
         if lfutil.isstandin(f):
             path = lfutil.findfile(repo, getdata().strip())
             if path is None:
                 raise util.Abort(
                     _('largefile %s not found in repo store or system cache')
                     % lfutil.splitstandin(f))
             f = lfutil.splitstandin(f)
 
             def getdatafn():
                 fd = None
                 try:
                     fd = open(path, 'rb')
                     return fd.read()
                 finally:
                     if fd:
                         fd.close()
 
             getdata = getdatafn
         write(f, 'x' in ff and 0755 or 0644, 'l' in ff, getdata)
 
     if subrepos:
         for subpath in ctx.substate:
             sub = ctx.sub(subpath)
             sub.archive(repo.ui, archiver, prefix)
 
     archiver.done()
 
 # If a largefile is modified, the change is not reflected in its
 # standin until a commit. cmdutil.bailifchanged() raises an exception
 # if the repo has uncommitted changes. Wrap it to also check if
 # largefiles were changed. This is used by bisect and backout.
 def override_bailifchanged(orig, repo):
     orig(repo)
     repo.lfstatus = True
     modified, added, removed, deleted = repo.status()[:4]
     repo.lfstatus = False
     if modified or added or removed or deleted:
         raise util.Abort(_('outstanding uncommitted changes'))
 
 # Fetch doesn't use cmdutil.bail_if_changed so override it to add the check
 def override_fetch(orig, ui, repo, *pats, **opts):
     repo.lfstatus = True
     modified, added, removed, deleted = repo.status()[:4]
     repo.lfstatus = False
     if modified or added or removed or deleted:
         raise util.Abort(_('outstanding uncommitted changes'))
     return orig(ui, repo, *pats, **opts)
 
 def override_forget(orig, ui, repo, *pats, **opts):
     installnormalfilesmatchfn(repo[None].manifest())
     orig(ui, repo, *pats, **opts)
     restorematchfn()
     m = scmutil.match(repo[None], pats, opts)
 
     try:
         repo.lfstatus = True
         s = repo.status(match=m, clean=True)
     finally:
         repo.lfstatus = False
     forget = sorted(s[0] + s[1] + s[3] + s[6])
     forget = [f for f in forget if lfutil.standin(f) in repo[None].manifest()]
 
     for f in forget:
         if lfutil.standin(f) not in repo.dirstate and not \
                 os.path.isdir(m.rel(lfutil.standin(f))):
             ui.warn(_('not removing %s: file is already untracked\n')
                     % m.rel(f))
 
     for f in forget:
         if ui.verbose or not m.exact(f):
             ui.status(_('removing %s\n') % m.rel(f))
 
     # Need to lock because standin files are deleted then removed from the
     # repository and we could race inbetween.
     wlock = repo.wlock()
     try:
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         for f in forget:
             if lfdirstate[f] == 'a':
                 lfdirstate.drop(f)
             else:
                 lfdirstate.remove(f)
         lfdirstate.write()
         lfutil.repo_remove(repo, [lfutil.standin(f) for f in forget],
             unlink=True)
     finally:
         wlock.release()
 
 def getoutgoinglfiles(ui, repo, dest=None, **opts):
     dest = ui.expandpath(dest or 'default-push', dest or 'default')
     dest, branches = hg.parseurl(dest, opts.get('branch'))
     revs, checkout = hg.addbranchrevs(repo, repo, branches, opts.get('rev'))
     if revs:
         revs = [repo.lookup(rev) for rev in revs]
 
     remoteui = hg.remoteui
 
     try:
         remote = hg.repository(remoteui(repo, opts), dest)
     except error.RepoError:
         return None
     o = lfutil.findoutgoing(repo, remote, False)
     if not o:
         return None
     o = repo.changelog.nodesbetween(o, revs)[0]
     if opts.get('newest_first'):
         o.reverse()
 
     toupload = set()
     for n in o:
         parents = [p for p in repo.changelog.parents(n) if p != node.nullid]
         ctx = repo[n]
         files = set(ctx.files())
         if len(parents) == 2:
             mc = ctx.manifest()
             mp1 = ctx.parents()[0].manifest()
             mp2 = ctx.parents()[1].manifest()
             for f in mp1:
                 if f not in mc:
                         files.add(f)
             for f in mp2:
                 if f not in mc:
                     files.add(f)
             for f in mc:
                 if mc[f] != mp1.get(f, None) or mc[f] != mp2.get(f, None):
                     files.add(f)
         toupload = toupload.union(
             set([f for f in files if lfutil.isstandin(f) and f in ctx]))
     return toupload
 
 def override_outgoing(orig, ui, repo, dest=None, **opts):
     orig(ui, repo, dest, **opts)
 
     if opts.pop('large', None):
         toupload = getoutgoinglfiles(ui, repo, dest, **opts)
         if toupload is None:
             ui.status(_('largefiles: No remote repo\n'))
         else:
             ui.status(_('largefiles to upload:\n'))
             for file in toupload:
                 ui.status(lfutil.splitstandin(file) + '\n')
             ui.status('\n')
 
 def override_summary(orig, ui, repo, *pats, **opts):
     try:
         repo.lfstatus = True
         orig(ui, repo, *pats, **opts)
     finally:
         repo.lfstatus = False
 
     if opts.pop('large', None):
         toupload = getoutgoinglfiles(ui, repo, None, **opts)
         if toupload is None:
             ui.status(_('largefiles: No remote repo\n'))
         else:
             ui.status(_('largefiles: %d to upload\n') % len(toupload))
 
 def override_addremove(orig, ui, repo, *pats, **opts):
     # Get the list of missing largefiles so we can remove them
     lfdirstate = lfutil.openlfdirstate(ui, repo)
     s = lfdirstate.status(match_.always(repo.root, repo.getcwd()), [], False,
         False, False)
     (unsure, modified, added, removed, missing, unknown, ignored, clean) = s
 
     # Call into the normal remove code, but the removing of the standin, we want
     # to have handled by original addremove.  Monkey patching here makes sure
     # we don't remove the standin in the largefiles code, preventing a very
     # confused state later.
     repo._isaddremove = True
     remove_largefiles(ui, repo, *missing, **opts)
     repo._isaddremove = False
     # Call into the normal add code, and any files that *should* be added as
     # largefiles will be
     add_largefiles(ui, repo, *pats, **opts)
     # Now that we've handled largefiles, hand off to the original addremove
     # function to take care of the rest.  Make sure it doesn't do anything with
     # largefiles by installing a matcher that will ignore them.
     installnormalfilesmatchfn(repo[None].manifest())
     result = orig(ui, repo, *pats, **opts)
     restorematchfn()
     return result
 
 # Calling purge with --all will cause the largefiles to be deleted.
 # Override repo.status to prevent this from happening.
 def override_purge(orig, ui, repo, *dirs, **opts):
     oldstatus = repo.status
     def override_status(node1='.', node2=None, match=None, ignored=False,
                         clean=False, unknown=False, listsubrepos=False):
         r = oldstatus(node1, node2, match, ignored, clean, unknown,
                       listsubrepos)
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         modified, added, removed, deleted, unknown, ignored, clean = r
         unknown = [f for f in unknown if lfdirstate[f] == '?']
         ignored = [f for f in ignored if lfdirstate[f] == '?']
         return modified, added, removed, deleted, unknown, ignored, clean
     repo.status = override_status
     orig(ui, repo, *dirs, **opts)
     repo.status = oldstatus
 
 def override_rollback(orig, ui, repo, **opts):
     result = orig(ui, repo, **opts)
     merge.update(repo, node=None, branchmerge=False, force=True,
         partial=lfutil.isstandin)
     wlock = repo.wlock()
     try:
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         lfiles = lfutil.listlfiles(repo)
         oldlfiles = lfutil.listlfiles(repo, repo[None].parents()[0].rev())
         for file in lfiles:
             if file in oldlfiles:
                 lfdirstate.normallookup(file)
             else:
                 lfdirstate.add(file)
         lfdirstate.write()
     finally:
         wlock.release()
     return result
 
 def override_transplant(orig, ui, repo, *revs, **opts):
     result = orig(ui, repo, *revs, **opts)
     lfcommands.updatelfiles(repo.ui, repo)
     return result

hgext/largefiles/uisetup.py

@@ -1,142 +1,143
 # Copyright 2009-2010 Gregory P. Ward
 # Copyright 2009-2010 Intelerad Medical Systems Incorporated
 # Copyright 2010-2011 Fog Creek Software
 # Copyright 2010-2011 Unity Technologies
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''setup for largefiles extension: uisetup'''
 
 from mercurial import archival, cmdutil, commands, extensions, filemerge, hg, \
     httprepo, localrepo, merge, sshrepo, sshserver, wireproto
 from mercurial.i18n import _
 from mercurial.hgweb import hgweb_mod, protocol
 
 import overrides
 import proto
 
 def uisetup(ui):
     # Disable auto-status for some commands which assume that all
     # files in the result are under Mercurial's control
 
     entry = extensions.wrapcommand(commands.table, 'add',
                                    overrides.override_add)
     addopt = [('', 'large', None, _('add as largefile')),
+              ('', 'normal', None, _('add as normal file')),
               ('', 'lfsize', '', _('add all files above this size '
                                    '(in megabytes) as largefiles '
                                    '(default: 10)'))]
     entry[1].extend(addopt)
 
     entry = extensions.wrapcommand(commands.table, 'addremove',
             overrides.override_addremove)
     entry = extensions.wrapcommand(commands.table, 'remove',
                                    overrides.override_remove)
     entry = extensions.wrapcommand(commands.table, 'forget',
                                    overrides.override_forget)
     entry = extensions.wrapcommand(commands.table, 'status',
                                    overrides.override_status)
     entry = extensions.wrapcommand(commands.table, 'log',
                                    overrides.override_log)
     entry = extensions.wrapcommand(commands.table, 'rollback',
                                    overrides.override_rollback)
     entry = extensions.wrapcommand(commands.table, 'verify',
                                    overrides.override_verify)
 
     verifyopt = [('', 'large', None, _('verify largefiles')),
                  ('', 'lfa', None,
                      _('verify all revisions of largefiles not just current')),
                  ('', 'lfc', None,
                      _('verify largefile contents not just existence'))]
     entry[1].extend(verifyopt)
 
     entry = extensions.wrapcommand(commands.table, 'outgoing',
         overrides.override_outgoing)
     outgoingopt = [('', 'large', None, _('display outgoing largefiles'))]
     entry[1].extend(outgoingopt)
     entry = extensions.wrapcommand(commands.table, 'summary',
                                    overrides.override_summary)
     summaryopt = [('', 'large', None, _('display outgoing largefiles'))]
     entry[1].extend(summaryopt)
 
     entry = extensions.wrapcommand(commands.table, 'update',
                                    overrides.override_update)
     entry = extensions.wrapcommand(commands.table, 'pull',
                                    overrides.override_pull)
     entry = extensions.wrapfunction(merge, '_checkunknown',
                                     overrides.override_checkunknown)
     entry = extensions.wrapfunction(merge, 'manifestmerge',
                                     overrides.override_manifestmerge)
     entry = extensions.wrapfunction(filemerge, 'filemerge',
                                     overrides.override_filemerge)
     entry = extensions.wrapfunction(cmdutil, 'copy',
                                     overrides.override_copy)
 
     # Backout calls revert so we need to override both the command and the
     # function
     entry = extensions.wrapcommand(commands.table, 'revert',
                                    overrides.override_revert)
     entry = extensions.wrapfunction(commands, 'revert',
                                     overrides.override_revert)
 
     # clone uses hg._update instead of hg.update even though they are the
     # same function... so wrap both of them)
     extensions.wrapfunction(hg, 'update', overrides.hg_update)
     extensions.wrapfunction(hg, '_update', overrides.hg_update)
     extensions.wrapfunction(hg, 'clean', overrides.hg_clean)
     extensions.wrapfunction(hg, 'merge', overrides.hg_merge)
 
     extensions.wrapfunction(archival, 'archive', overrides.override_archive)
     extensions.wrapfunction(cmdutil, 'bailifchanged',
                             overrides.override_bailifchanged)
 
     # create the new wireproto commands ...
     wireproto.commands['putlfile'] = (proto.putlfile, 'sha')
     wireproto.commands['getlfile'] = (proto.getlfile, 'sha')
     wireproto.commands['statlfile'] = (proto.statlfile, 'sha')
 
     # ... and wrap some existing ones
     wireproto.commands['capabilities'] = (proto.capabilities, '')
     wireproto.commands['heads'] = (proto.heads, '')
     wireproto.commands['lheads'] = (wireproto.heads, '')
 
     # make putlfile behave the same as push and {get,stat}lfile behave
     # the same as pull w.r.t. permissions checks
     hgweb_mod.perms['putlfile'] = 'push'
     hgweb_mod.perms['getlfile'] = 'pull'
     hgweb_mod.perms['statlfile'] = 'pull'
 
     # the hello wireproto command uses wireproto.capabilities, so it won't see
     # our largefiles capability unless we replace the actual function as well.
     proto.capabilities_orig = wireproto.capabilities
     wireproto.capabilities = proto.capabilities
 
     # these let us reject non-largefiles clients and make them display
     # our error messages
     protocol.webproto.refuseclient = proto.webproto_refuseclient
     sshserver.sshserver.refuseclient = proto.sshproto_refuseclient
 
     # can't do this in reposetup because it needs to have happened before
     # wirerepo.__init__ is called
     proto.ssh_oldcallstream = sshrepo.sshrepository._callstream
     proto.http_oldcallstream = httprepo.httprepository._callstream
     sshrepo.sshrepository._callstream = proto.sshrepo_callstream
     httprepo.httprepository._callstream = proto.httprepo_callstream
 
     # don't die on seeing a repo with the largefiles requirement
     localrepo.localrepository.supported |= set(['largefiles'])
 
     # override some extensions' stuff as well
     for name, module in extensions.extensions():
         if name == 'fetch':
             extensions.wrapcommand(getattr(module, 'cmdtable'), 'fetch',
                 overrides.override_fetch)
         if name == 'purge':
             extensions.wrapcommand(getattr(module, 'cmdtable'), 'purge',
                 overrides.override_purge)
         if name == 'rebase':
             extensions.wrapcommand(getattr(module, 'cmdtable'), 'rebase',
                 overrides.override_rebase)
         if name == 'transplant':
             extensions.wrapcommand(getattr(module, 'cmdtable'), 'transplant',
                 overrides.override_transplant)

tests/test-largefiles.t

@@ -1,944 +1,955
   $ "$TESTDIR/hghave" symlink unix-permissions serve || exit 80
   $ USERCACHE=`pwd`/cache; export USERCACHE
   $ mkdir -p ${USERCACHE}
   $ cat >> $HGRCPATH <<EOF
   > [extensions]
   > largefiles=
   > purge=
   > rebase=
   > transplant=
   > [phases]
   > publish=False
   > [largefiles]
   > minsize=2
   > patterns=glob:**.dat
   > usercache=${USERCACHE}
   > [hooks]
   > precommit=echo "Invoking status precommit hook"; hg status
   > EOF
 
 Create the repo with a couple of revisions of both large and normal
 files, testing that status correctly shows largefiles and that summary output
 is correct.
 
   $ hg init a
   $ cd a
   $ mkdir sub
   $ echo normal1 > normal1
   $ echo normal2 > sub/normal2
   $ echo large1 > large1
   $ echo large2 > sub/large2
   $ hg add normal1 sub/normal2
   $ hg add --large large1 sub/large2
   $ hg commit -m "add files"
   Invoking status precommit hook
   A large1
   A normal1
   A sub/large2
   A sub/normal2
   $ echo normal11 > normal1
   $ echo normal22 > sub/normal2
   $ echo large11 > large1
   $ echo large22 > sub/large2
   $ hg commit -m "edit files"
   Invoking status precommit hook
   M large1
   M normal1
   M sub/large2
   M sub/normal2
   $ hg sum --large
   parent: 1:ce8896473775 tip
    edit files
   branch: default
   commit: (clean)
   update: (current)
   largefiles: No remote repo
 
 Commit preserved largefile contents.
 
   $ cat normal1
   normal11
   $ cat large1
   large11
   $ cat sub/normal2
   normal22
   $ cat sub/large2
   large22
 
 Remove both largefiles and normal files.
  
   $ hg remove normal1 large1
   $ hg commit -m "remove files"
   Invoking status precommit hook
   R large1
   R normal1
   $ ls
   sub
   $ echo "testlargefile" > large1-test
   $ hg add --large large1-test
   $ hg st
   A large1-test
   $ hg rm large1-test
   not removing large1-test: file has been marked for add (use forget to undo)
   $ hg st
   A large1-test
   $ hg forget large1-test
   $ hg st
   ? large1-test
   $ rm large1-test
 
 Copy both largefiles and normal files (testing that status output is correct).
 
   $ hg cp sub/normal2 normal1
   $ hg cp sub/large2 large1
   $ hg commit -m "copy files"
   Invoking status precommit hook
   A large1
   A normal1
   $ cat normal1
   normal22
   $ cat large1
   large22
 
 Test moving largefiles and verify that normal files are also unaffected.
 
   $ hg mv normal1 normal3
   $ hg mv large1 large3
   $ hg mv sub/normal2 sub/normal4
   $ hg mv sub/large2 sub/large4
   $ hg commit -m "move files"
   Invoking status precommit hook
   A large3
   A normal3
   A sub/large4
   A sub/normal4
   R large1
   R normal1
   R sub/large2
   R sub/normal2
   $ cat normal3
   normal22
   $ cat large3
   large22
   $ cat sub/normal4
   normal22
   $ cat sub/large4
   large22
 
 Test archiving the various revisions.  These hit corner cases known with
 archiving.
 
   $ hg archive -r 0 ../archive0
   $ hg archive -r 1 ../archive1
   $ hg archive -r 2 ../archive2
   $ hg archive -r 3 ../archive3
   $ hg archive -r 4 ../archive4
   $ cd ../archive0
   $ cat normal1 
   normal1
   $ cat large1
   large1
   $ cat sub/normal2
   normal2
   $ cat sub/large2
   large2
   $ cd ../archive1
   $ cat normal1
   normal11
   $ cat large1
   large11
   $ cat sub/normal2
   normal22
   $ cat sub/large2
   large22
   $ cd ../archive2
   $ ls
   sub
   $ cat sub/normal2
   normal22
   $ cat sub/large2
   large22
   $ cd ../archive3
   $ cat normal1
   normal22
   $ cat large1
   large22
   $ cat sub/normal2
   normal22
   $ cat sub/large2
   large22
   $ cd ../archive4
   $ cat normal3
   normal22
   $ cat large3
   large22
   $ cat sub/normal4
   normal22
   $ cat sub/large4
   large22
 
 Commit corner case: specify files to commit.
 
   $ cd ../a
   $ echo normal3 > normal3
   $ echo large3 > large3
   $ echo normal4 > sub/normal4
   $ echo large4 > sub/large4
   $ hg commit normal3 large3 sub/normal4 sub/large4 -m "edit files again"
   Invoking status precommit hook
   M large3
   M normal3
   M sub/large4
   M sub/normal4
   $ cat normal3
   normal3
   $ cat large3
   large3
   $ cat sub/normal4
   normal4
   $ cat sub/large4
   large4
 
 One more commit corner case: commit from a subdirectory.
 
   $ cd ../a
   $ echo normal33 > normal3
   $ echo large33 > large3
   $ echo normal44 > sub/normal4
   $ echo large44 > sub/large4
   $ cd sub
   $ hg commit -m "edit files yet again"
   Invoking status precommit hook
   M large3
   M normal3
   M sub/large4
   M sub/normal4
   $ cat ../normal3
   normal33
   $ cat ../large3
   large33
   $ cat normal4
   normal44
   $ cat large4
   large44
 
 Committing standins is not allowed.
 
   $ cd ..
   $ echo large3 > large3
   $ hg commit .hglf/large3 -m "try to commit standin"
   abort: file ".hglf/large3" is a largefile standin
   (commit the largefile itself instead)
   [255]
 
 Corner cases for adding largefiles.
 
   $ echo large5 > large5
   $ hg add --large large5
   $ hg add --large large5
   large5 already a largefile
   $ mkdir sub2
   $ echo large6 > sub2/large6
   $ echo large7 > sub2/large7
   $ hg add --large sub2
   adding sub2/large6 as a largefile (glob)
   adding sub2/large7 as a largefile (glob)
   $ hg st
   M large3
   A large5
   A sub2/large6
   A sub2/large7
 
 Config settings (pattern **.dat, minsize 2 MB) are respected.
 
   $ echo testdata > test.dat
   $ dd bs=1k count=2k if=/dev/zero of=reallylarge > /dev/null 2> /dev/null
   $ hg add
   adding reallylarge as a largefile
   adding test.dat as a largefile
 
 Test that minsize and --lfsize handle float values;
 also tests that --lfsize overrides largefiles.minsize.
 (0.250 MB = 256 kB = 262144 B)
 
   $ dd if=/dev/zero of=ratherlarge bs=1024 count=256 > /dev/null 2> /dev/null
   $ dd if=/dev/zero of=medium bs=1024 count=128 > /dev/null 2> /dev/null
   $ hg --config largefiles.minsize=.25 add
   adding ratherlarge as a largefile
   adding medium
   $ hg forget medium
   $ hg --config largefiles.minsize=.25 add --lfsize=.125
   adding medium as a largefile
   $ dd if=/dev/zero of=notlarge bs=1024 count=127 > /dev/null 2> /dev/null
   $ hg --config largefiles.minsize=.25 add --lfsize=.125
   adding notlarge
   $ hg forget notlarge
 
 Test forget on largefiles.
 
   $ hg forget large3 large5 test.dat reallylarge ratherlarge medium
   $ hg commit -m "add/edit more largefiles"
   Invoking status precommit hook
   A sub2/large6
   A sub2/large7
   R large3
   ? large5
   ? medium
   ? notlarge
   ? ratherlarge
   ? reallylarge
   ? test.dat
   $ hg st
   ? large3
   ? large5
   ? medium
   ? notlarge
   ? ratherlarge
   ? reallylarge
   ? test.dat
 
 Purge with largefiles: verify that largefiles are still in the working
 dir after a purge.
 
   $ hg purge --all
   $ cat sub/large4
   large44
   $ cat sub2/large6
   large6
   $ cat sub2/large7
   large7
 
 Test addremove: verify that files that should be added as largfiles are added as
 such and that already-existing largfiles are not added as normal files by
 accident.
 
   $ rm normal3
   $ rm sub/large4
   $ echo "testing addremove with patterns" > testaddremove.dat
   $ echo "normaladdremove" > normaladdremove
   $ hg addremove
   removing sub/large4
   adding testaddremove.dat as a largefile
   removing normal3
   adding normaladdremove
 
 Clone a largefiles repo.
 
   $ hg clone . ../b
   updating to branch default
   5 files updated, 0 files merged, 0 files removed, 0 files unresolved
   getting changed largefiles
   3 largefiles updated, 0 removed
   $ cd ../b
   $ hg log --template '{rev}:{node|short}  {desc|firstline}\n'
   7:daea875e9014  add/edit more largefiles
   6:4355d653f84f  edit files yet again
   5:9d5af5072dbd  edit files again
   4:74c02385b94c  move files
   3:9e8fbc4bce62  copy files
   2:51a0ae4d5864  remove files
   1:ce8896473775  edit files
   0:30d30fe6a5be  add files
   $ cat normal3
   normal33
   $ cat sub/normal4
   normal44
   $ cat sub/large4
   large44
   $ cat sub2/large6
   large6
   $ cat sub2/large7
   large7
   $ cd ..
   $ hg clone a -r 3 c
   adding changesets
   adding manifests
   adding file changes
   added 4 changesets with 10 changes to 4 files
   updating to branch default
   4 files updated, 0 files merged, 0 files removed, 0 files unresolved
   getting changed largefiles
   2 largefiles updated, 0 removed
   $ cd c
   $ hg log --template '{rev}:{node|short}  {desc|firstline}\n'
   3:9e8fbc4bce62  copy files
   2:51a0ae4d5864  remove files
   1:ce8896473775  edit files
   0:30d30fe6a5be  add files
   $ cat normal1
   normal22
   $ cat large1
   large22
   $ cat sub/normal2
   normal22
   $ cat sub/large2
   large22
 
 Old revisions of a clone have correct largefiles content (this also
 tests update).
 
   $ hg update -r 1 
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   getting changed largefiles
   1 largefiles updated, 0 removed
   $ cat large1
   large11
   $ cat sub/large2
   large22
 
 Rebasing between two repositories does not revert largefiles to old
 revisions (this was a very bad bug that took a lot of work to fix).
 
   $ cd ..
   $ hg clone a d
   updating to branch default
   5 files updated, 0 files merged, 0 files removed, 0 files unresolved
   getting changed largefiles
   3 largefiles updated, 0 removed
   $ cd b
   $ echo large4-modified > sub/large4
   $ echo normal3-modified > normal3
   $ hg commit -m "modify normal file and largefile in repo b"
   Invoking status precommit hook
   M normal3
   M sub/large4
   $ cd ../d
   $ echo large6-modified > sub2/large6
   $ echo normal4-modified > sub/normal4
   $ hg commit -m "modify normal file largefile in repo d"
   Invoking status precommit hook
   M sub/normal4
   M sub2/large6
   $ cd ..
   $ hg clone d e
   updating to branch default
   5 files updated, 0 files merged, 0 files removed, 0 files unresolved
   getting changed largefiles
   3 largefiles updated, 0 removed
   $ cd d
   $ hg pull --rebase ../b
   pulling from ../b
   searching for changes
   adding changesets
   adding manifests
   adding file changes
   added 1 changesets with 2 changes to 2 files (+1 heads)
   getting changed largefiles
   1 largefiles updated, 0 removed
   Invoking status precommit hook
   M sub/normal4
   M sub2/large6
   saved backup bundle to $TESTTMP/d/.hg/strip-backup/f574fb32bb45-backup.hg
   nothing to rebase
   $ hg log --template '{rev}:{node|short}  {desc|firstline}\n'
   9:598410d3eb9a  modify normal file largefile in repo d
   8:a381d2c8c80e  modify normal file and largefile in repo b
   7:daea875e9014  add/edit more largefiles
   6:4355d653f84f  edit files yet again
   5:9d5af5072dbd  edit files again
   4:74c02385b94c  move files
   3:9e8fbc4bce62  copy files
   2:51a0ae4d5864  remove files
   1:ce8896473775  edit files
   0:30d30fe6a5be  add files
   $ cat normal3
   normal3-modified
   $ cat sub/normal4
   normal4-modified
   $ cat sub/large4
   large4-modified
   $ cat sub2/large6
   large6-modified
   $ cat sub2/large7
   large7
   $ cd ../e
   $ hg pull ../b
   pulling from ../b
   searching for changes
   adding changesets
   adding manifests
   adding file changes
   added 1 changesets with 2 changes to 2 files (+1 heads)
   (run 'hg heads' to see heads, 'hg merge' to merge)
   caching new largefiles
   0 largefiles cached
   $ hg rebase
   getting changed largefiles
   1 largefiles updated, 0 removed
   Invoking status precommit hook
   M sub/normal4
   M sub2/large6
   saved backup bundle to $TESTTMP/e/.hg/strip-backup/f574fb32bb45-backup.hg
   $ hg log --template '{rev}:{node|short}  {desc|firstline}\n'
   9:598410d3eb9a  modify normal file largefile in repo d
   8:a381d2c8c80e  modify normal file and largefile in repo b
   7:daea875e9014  add/edit more largefiles
   6:4355d653f84f  edit files yet again
   5:9d5af5072dbd  edit files again
   4:74c02385b94c  move files
   3:9e8fbc4bce62  copy files
   2:51a0ae4d5864  remove files
   1:ce8896473775  edit files
   0:30d30fe6a5be  add files
   $ cat normal3
   normal3-modified
   $ cat sub/normal4
   normal4-modified
   $ cat sub/large4
   large4-modified
   $ cat sub2/large6
   large6-modified
   $ cat sub2/large7
   large7
 
 Rollback on largefiles.
 
   $ echo large4-modified-again > sub/large4 
   $ hg commit -m "Modify large4 again"
   Invoking status precommit hook
   M sub/large4
   $ hg rollback
   repository tip rolled back to revision 9 (undo commit)
   working directory now based on revision 9
   $ hg st
   M sub/large4
   $ hg log --template '{rev}:{node|short}  {desc|firstline}\n'
   9:598410d3eb9a  modify normal file largefile in repo d
   8:a381d2c8c80e  modify normal file and largefile in repo b
   7:daea875e9014  add/edit more largefiles
   6:4355d653f84f  edit files yet again
   5:9d5af5072dbd  edit files again
   4:74c02385b94c  move files
   3:9e8fbc4bce62  copy files
   2:51a0ae4d5864  remove files
   1:ce8896473775  edit files
   0:30d30fe6a5be  add files
   $ cat sub/large4
   large4-modified-again
 
 "update --check" refuses to update with uncommitted changes.
   $ hg update --check 8
   abort: uncommitted local changes
   [255]
 
 "update --clean" leaves correct largefiles in working copy.
 
   $ hg update --clean 
   0 files updated, 0 files merged, 0 files removed, 0 files unresolved
   getting changed largefiles
   1 largefiles updated, 0 removed
   $ cat normal3
   normal3-modified
   $ cat sub/normal4
   normal4-modified
   $ cat sub/large4
   large4-modified
   $ cat sub2/large6
   large6-modified
   $ cat sub2/large7
   large7
 
 Now "update check" is happy.
   $ hg update --check 8
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   getting changed largefiles
   1 largefiles updated, 0 removed
   $ hg update --check
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   getting changed largefiles
   1 largefiles updated, 0 removed
 
 Test removing empty largefiles directories on update
   $ test -d sub2 && echo "sub2 exists"
   sub2 exists
   $ hg update -q null
   $ test -d sub2 && echo "error: sub2 should not exist anymore"
   [1]
   $ hg update -q
 
 Test hg remove removes empty largefiles directories
   $ test -d sub2 && echo "sub2 exists"
   sub2 exists
   $ hg remove sub2/*
   $ test -d sub2 && echo "error: sub2 should not exist anymore"
   [1]
   $ hg revert sub2/large6 sub2/large7
 
 "revert" works on largefiles (and normal files too).
   $ echo hack3 >> normal3
   $ echo hack4 >> sub/normal4
   $ echo hack4 >> sub/large4
   $ hg rm sub2/large6
   $ echo new >> sub2/large8
   $ hg add --large sub2/large8
 # XXX we don't really want to report that we're reverting the standin;
 # that's just an implementation detail. But I don't see an obvious fix. ;-(
   $ hg revert sub
   reverting .hglf/sub/large4 (glob)
   reverting sub/normal4 (glob)
   $ hg status
   M normal3
   A sub2/large8
   R sub2/large6
   ? sub/large4.orig
   ? sub/normal4.orig
   $ cat sub/normal4
   normal4-modified
   $ cat sub/large4
   large4-modified
   $ hg revert -a --no-backup
   undeleting .hglf/sub2/large6 (glob)
   forgetting .hglf/sub2/large8 (glob)
   reverting normal3
   $ hg status
   ? sub/large4.orig
   ? sub/normal4.orig
   ? sub2/large8
   $ cat normal3
   normal3-modified
   $ cat sub2/large6
   large6-modified
   $ rm sub/*.orig sub2/large8
 
 revert some files to an older revision
   $ hg revert --no-backup -r 8 sub2
   reverting .hglf/sub2/large6 (glob)
   $ cat sub2/large6
   large6
   $ hg revert --no-backup sub2
   reverting .hglf/sub2/large6 (glob)
   $ hg status
 
 "verify --large" actually verifies largefiles
 
   $ hg verify --large
   checking changesets
   checking manifests
   crosschecking files in changesets and manifests
   checking files
   10 files, 10 changesets, 28 total revisions
   searching 1 changesets for largefiles
   verified existence of 3 revisions of 3 largefiles
 
 Merging does not revert to old versions of largefiles and also check
 that merging after having pulled from a non-default remote works
 correctly.
 
   $ cd ..
   $ hg clone -r 7 e temp
   adding changesets
   adding manifests
   adding file changes
   added 8 changesets with 24 changes to 10 files
   updating to branch default
   5 files updated, 0 files merged, 0 files removed, 0 files unresolved
   getting changed largefiles
   3 largefiles updated, 0 removed
   $ hg clone temp f
   updating to branch default
   5 files updated, 0 files merged, 0 files removed, 0 files unresolved
   getting changed largefiles
   3 largefiles updated, 0 removed
 # Delete the largefiles in the largefiles system cache so that we have an
 # opportunity to test that caching after a pull works.
   $ rm ${USERCACHE}/*
   $ cd f
   $ echo "large4-merge-test" > sub/large4
   $ hg commit -m "Modify large4 to test merge"
   Invoking status precommit hook
   M sub/large4
   $ hg pull ../e
   pulling from ../e
   searching for changes
   adding changesets
   adding manifests
   adding file changes
   added 2 changesets with 4 changes to 4 files (+1 heads)
   (run 'hg heads' to see heads, 'hg merge' to merge)
   caching new largefiles
   2 largefiles cached
   $ hg merge
   merging sub/large4
   largefile sub/large4 has a merge conflict
   keep (l)ocal or take (o)ther? l
   3 files updated, 1 files merged, 0 files removed, 0 files unresolved
   (branch merge, don't forget to commit)
   getting changed largefiles
   1 largefiles updated, 0 removed
   $ hg commit -m "Merge repos e and f"
   Invoking status precommit hook
   M normal3
   M sub/normal4
   M sub2/large6
   $ cat normal3
   normal3-modified
   $ cat sub/normal4
   normal4-modified
   $ cat sub/large4
   large4-merge-test
   $ cat sub2/large6
   large6-modified
   $ cat sub2/large7
   large7
 
 Test status after merging with a branch that introduces a new largefile:
 
   $ echo large > large
   $ hg add --large large
   $ hg commit -m 'add largefile'
   Invoking status precommit hook
   A large
   $ hg update -q ".^"
   $ echo change >> normal3
   $ hg commit -m 'some change'
   Invoking status precommit hook
   M normal3
   created new head
   $ hg merge
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   (branch merge, don't forget to commit)
   getting changed largefiles
   1 largefiles updated, 0 removed
   $ hg status
   M large
 
 Test that a normal file and a largefile with the same name and path cannot
 coexist.
 
   $ rm sub2/large7
   $ echo "largeasnormal" > sub2/large7
   $ hg add sub2/large7
   sub2/large7 already a largefile
 
 Test that transplanting a largefile change works correctly.
 
   $ cd ..
   $ hg clone -r 8 d g
   adding changesets
   adding manifests
   adding file changes
   added 9 changesets with 26 changes to 10 files
   updating to branch default
   5 files updated, 0 files merged, 0 files removed, 0 files unresolved
   getting changed largefiles
   3 largefiles updated, 0 removed
   $ cd g
   $ hg transplant -s ../d 598410d3eb9a 
   searching for changes
   searching for changes
   adding changesets
   adding manifests
   adding file changes
   added 1 changesets with 2 changes to 2 files
   getting changed largefiles
   1 largefiles updated, 0 removed
   $ hg log --template '{rev}:{node|short}  {desc|firstline}\n'
   9:598410d3eb9a  modify normal file largefile in repo d
   8:a381d2c8c80e  modify normal file and largefile in repo b
   7:daea875e9014  add/edit more largefiles
   6:4355d653f84f  edit files yet again
   5:9d5af5072dbd  edit files again
   4:74c02385b94c  move files
   3:9e8fbc4bce62  copy files
   2:51a0ae4d5864  remove files
   1:ce8896473775  edit files
   0:30d30fe6a5be  add files
   $ cat normal3
   normal3-modified
   $ cat sub/normal4
   normal4-modified
   $ cat sub/large4
   large4-modified
   $ cat sub2/large6
   large6-modified
   $ cat sub2/large7
   large7
 
 Test that renaming a largefile results in correct output for status
 
   $ hg rename sub/large4 large4-renamed
   $ hg commit -m "test rename output"
   Invoking status precommit hook
   A large4-renamed
   R sub/large4
   $ cat large4-renamed
   large4-modified
   $ cd sub2
   $ hg rename large6 large6-renamed
   $ hg st
   A sub2/large6-renamed
   R sub2/large6
-  $ cd ../..
+  $ cd ..
+
+Test --normal flag
+
+  $ dd if=/dev/urandom bs=2k count=11k > new-largefile 2> /dev/null
+  $ hg add --normal --large new-largefile
+  abort: --normal cannot be used with --large
+  [255]
+  $ hg add --normal new-largefile
+  new-largefile: up to 69 MB of RAM may be required to manage this file
+  (use 'hg revert new-largefile' to cancel the pending addition)
+  $ cd ..
 
 vanilla clients not locked out from largefiles servers on vanilla repos
   $ mkdir r1
   $ cd r1
   $ hg init
   $ echo c1 > f1
   $ hg add f1
   $ hg commit -m "m1"
   Invoking status precommit hook
   A f1
   $ cd ..
   $ hg serve -R r1 -d -p $HGPORT --pid-file hg.pid
   $ cat hg.pid >> $DAEMON_PIDS
   $ hg --config extensions.largefiles=! clone http://localhost:$HGPORT r2
   requesting all changes
   adding changesets
   adding manifests
   adding file changes
   added 1 changesets with 1 changes to 1 files
   updating to branch default
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
 
 largefiles clients still work with vanilla servers
   $ hg --config extensions.largefiles=! serve -R r1 -d -p $HGPORT1 --pid-file hg.pid
   $ cat hg.pid >> $DAEMON_PIDS
   $ hg clone http://localhost:$HGPORT1 r3
   requesting all changes
   adding changesets
   adding manifests
   adding file changes
   added 1 changesets with 1 changes to 1 files
   updating to branch default
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
 
 vanilla clients locked out from largefiles http repos
   $ mkdir r4
   $ cd r4
   $ hg init
   $ echo c1 > f1
   $ hg add --large f1
   $ hg commit -m "m1"
   Invoking status precommit hook
   A f1
   $ cd ..
   $ hg serve -R r4 -d -p $HGPORT2 --pid-file hg.pid
   $ cat hg.pid >> $DAEMON_PIDS
   $ hg --config extensions.largefiles=! clone http://localhost:$HGPORT2 r5
   abort: remote error:
   
   This repository uses the largefiles extension.
   
   Please enable it in your Mercurial config file.
   [255]
 
 used all HGPORTs, kill all daemons
   $ "$TESTDIR/killdaemons.py"
 
 vanilla clients locked out from largefiles ssh repos
   $ hg --config extensions.largefiles=! clone -e "python $TESTDIR/dummyssh" ssh://user@dummy/r4 r5
   abort: remote error:
   
   This repository uses the largefiles extension.
   
   Please enable it in your Mercurial config file.
   [255]
 
 largefiles clients refuse to push largefiles repos to vanilla servers
   $ mkdir r6
   $ cd r6
   $ hg init
   $ echo c1 > f1
   $ hg add f1
   $ hg commit -m "m1"
   Invoking status precommit hook
   A f1
   $ cat >> .hg/hgrc <<!
   > [web]
   > push_ssl = false
   > allow_push = *
   > !
   $ cd ..
   $ hg clone r6 r7
   updating to branch default
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ cd r7
   $ echo c2 > f2
   $ hg add --large f2
   $ hg commit -m "m2"
   Invoking status precommit hook
   A f2
   $ hg --config extensions.largefiles=! -R ../r6 serve -d -p $HGPORT --pid-file ../hg.pid
   $ cat ../hg.pid >> $DAEMON_PIDS
   $ hg push http://localhost:$HGPORT
   pushing to http://localhost:$HGPORT/
   searching for changes
   abort: http://localhost:$HGPORT/ does not appear to be a largefile store
   [255]
   $ cd ..
 
 putlfile errors are shown (issue3123)
 Corrupt the cached largefile in r7
   $ echo corruption > $USERCACHE/4cdac4d8b084d0b599525cf732437fb337d422a8
   $ hg init empty
   $ hg serve -R empty -d -p $HGPORT1 --pid-file hg.pid \
   >   --config 'web.allow_push=*' --config web.push_ssl=False
   $ cat hg.pid >> $DAEMON_PIDS
   $ hg push -R r7 http://localhost:$HGPORT1
   pushing to http://localhost:$HGPORT1/
   searching for changes
   remote: largefiles: failed to put 4cdac4d8b084d0b599525cf732437fb337d422a8 into store: largefile contents do not match hash
   abort: remotestore: could not put $TESTTMP/r7/.hg/largefiles/4cdac4d8b084d0b599525cf732437fb337d422a8 to remote store http://localhost:$HGPORT1/
   [255]
   $ rm -rf empty
 
 Clone a local repository owned by another user
 We have to simulate that here by setting $HOME and removing write permissions
   $ ORIGHOME="$HOME"
   $ mkdir alice
   $ HOME="`pwd`/alice"
   $ cd alice
   $ hg init pubrepo
   $ cd pubrepo
   $ dd if=/dev/urandom bs=1k count=11k > a-large-file 2> /dev/null
   $ hg add --large a-large-file
   $ hg commit -m "Add a large file"
   Invoking status precommit hook
   A a-large-file
   $ cd ..
   $ chmod -R a-w pubrepo
   $ cd ..
   $ mkdir bob
   $ HOME="`pwd`/bob"
   $ cd bob
   $ hg clone --pull ../alice/pubrepo pubrepo
   requesting all changes
   adding changesets
   adding manifests
   adding file changes
   added 1 changesets with 1 changes to 1 files
   updating to branch default
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   getting changed largefiles
   1 largefiles updated, 0 removed
   $ cd ..
   $ chmod -R u+w alice/pubrepo
   $ HOME="$ORIGHOME"
 
 Symlink to a large largefile should behave the same as a symlink to a normal file
   $ hg init largesymlink
   $ cd largesymlink
   $ dd if=/dev/zero bs=1k count=10k of=largefile 2>/dev/null
   $ hg add --large largefile
   $ hg commit -m "commit a large file"
   Invoking status precommit hook
   A largefile
   $ ln -s largefile largelink
   $ hg add largelink
   $ hg commit -m "commit a large symlink"
   Invoking status precommit hook
   A largelink
   $ rm -f largelink
   $ hg up >/dev/null
   $ test -f largelink
   [1]
   $ test -L largelink
   [1]
   $ rm -f largelink # make next part of the test independent of the previous
   $ hg up -C >/dev/null
   $ test -f largelink
   $ test -L largelink
   $ cd ..