merge: fix race that could cause wrong size in dirstate

The problem is that hg merge/update/etc work the following way:
1. figure out what files to update
2. apply the update to disk
3. apply the update to in-memory dirstate
4. write dirstate

where step3 looks at the filesystem and assumes it sees the result of
step2. If a file is changed between step2 and step3, step3 will record
incorrect information in the dirstate.

I avoid this by passing the size step3 needs directly from step2, for
the common path (not implemented for change/delete conflicts for
instance).

I didn't fix the same race for the exec bit for now, because it's less
likely to be problematic and I had trouble due to the fact that the
dirstate stores the permissions differently from the manifest (st_mode
vs '' 'l' 'x'), in combination with tests that pretend that symlinks
are not supported.

However, I moved the lstat from step3 to step2, which should tighten
the race window markedly, both for the exec bit and for the mtime.

Differential Revision: https://phab.mercurial-scm.org/D6475

hgext/largefiles/overrides.py

@@ -1,1503 +1,1503
 # Copyright 2009-2010 Gregory P. Ward
 # Copyright 2009-2010 Intelerad Medical Systems Incorporated
 # Copyright 2010-2011 Fog Creek Software
 # Copyright 2010-2011 Unity Technologies
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''Overridden Mercurial commands and functions for the largefiles extension'''
 from __future__ import absolute_import
 
 import copy
 import os
 
 from mercurial.i18n import _
 
 from mercurial.hgweb import (
     webcommands,
 )
 
 from mercurial import (
     archival,
     cmdutil,
     copies as copiesmod,
     error,
     exchange,
     extensions,
     exthelper,
     filemerge,
     hg,
     logcmdutil,
     match as matchmod,
     merge,
     pathutil,
     pycompat,
     scmutil,
     smartset,
     subrepo,
     upgrade,
     url as urlmod,
     util,
 )
 
 from . import (
     lfcommands,
     lfutil,
     storefactory,
 )
 
 eh = exthelper.exthelper()
 
 # -- Utility functions: commonly/repeatedly needed functionality ---------------
 
 def composelargefilematcher(match, manifest):
     '''create a matcher that matches only the largefiles in the original
     matcher'''
     m = copy.copy(match)
     lfile = lambda f: lfutil.standin(f) in manifest
     m._files = [lf for lf in m._files if lfile(lf)]
     m._fileset = set(m._files)
     m.always = lambda: False
     origmatchfn = m.matchfn
     m.matchfn = lambda f: lfile(f) and origmatchfn(f)
     return m
 
 def composenormalfilematcher(match, manifest, exclude=None):
     excluded = set()
     if exclude is not None:
         excluded.update(exclude)
 
     m = copy.copy(match)
     notlfile = lambda f: not (lfutil.isstandin(f) or lfutil.standin(f) in
             manifest or f in excluded)
     m._files = [lf for lf in m._files if notlfile(lf)]
     m._fileset = set(m._files)
     m.always = lambda: False
     origmatchfn = m.matchfn
     m.matchfn = lambda f: notlfile(f) and origmatchfn(f)
     return m
 
 def addlargefiles(ui, repo, isaddremove, matcher, uipathfn, **opts):
     large = opts.get(r'large')
     lfsize = lfutil.getminsize(
         ui, lfutil.islfilesrepo(repo), opts.get(r'lfsize'))
 
     lfmatcher = None
     if lfutil.islfilesrepo(repo):
         lfpats = ui.configlist(lfutil.longname, 'patterns')
         if lfpats:
             lfmatcher = matchmod.match(repo.root, '', list(lfpats))
 
     lfnames = []
     m = matcher
 
     wctx = repo[None]
     for f in wctx.walk(matchmod.badmatch(m, lambda x, y: None)):
         exact = m.exact(f)
         lfile = lfutil.standin(f) in wctx
         nfile = f in wctx
         exists = lfile or nfile
 
         # Don't warn the user when they attempt to add a normal tracked file.
         # The normal add code will do that for us.
         if exact and exists:
             if lfile:
                 ui.warn(_('%s already a largefile\n') % uipathfn(f))
             continue
 
         if (exact or not exists) and not lfutil.isstandin(f):
             # In case the file was removed previously, but not committed
             # (issue3507)
             if not repo.wvfs.exists(f):
                 continue
 
             abovemin = (lfsize and
                         repo.wvfs.lstat(f).st_size >= lfsize * 1024 * 1024)
             if large or abovemin or (lfmatcher and lfmatcher(f)):
                 lfnames.append(f)
                 if ui.verbose or not exact:
                     ui.status(_('adding %s as a largefile\n') % uipathfn(f))
 
     bad = []
 
     # Need to lock, otherwise there could be a race condition between
     # when standins are created and added to the repo.
     with repo.wlock():
         if not opts.get(r'dry_run'):
             standins = []
             lfdirstate = lfutil.openlfdirstate(ui, repo)
             for f in lfnames:
                 standinname = lfutil.standin(f)
                 lfutil.writestandin(repo, standinname, hash='',
                     executable=lfutil.getexecutable(repo.wjoin(f)))
                 standins.append(standinname)
                 if lfdirstate[f] == 'r':
                     lfdirstate.normallookup(f)
                 else:
                     lfdirstate.add(f)
             lfdirstate.write()
             bad += [lfutil.splitstandin(f)
                     for f in repo[None].add(standins)
                     if f in m.files()]
 
         added = [f for f in lfnames if f not in bad]
     return added, bad
 
 def removelargefiles(ui, repo, isaddremove, matcher, uipathfn, dryrun, **opts):
     after = opts.get(r'after')
     m = composelargefilematcher(matcher, repo[None].manifest())
     try:
         repo.lfstatus = True
         s = repo.status(match=m, clean=not isaddremove)
     finally:
         repo.lfstatus = False
     manifest = repo[None].manifest()
     modified, added, deleted, clean = [[f for f in list
                                         if lfutil.standin(f) in manifest]
                                        for list in (s.modified, s.added,
                                                     s.deleted, s.clean)]
 
     def warn(files, msg):
         for f in files:
             ui.warn(msg % uipathfn(f))
         return int(len(files) > 0)
 
     if after:
         remove = deleted
         result = warn(modified + added + clean,
                       _('not removing %s: file still exists\n'))
     else:
         remove = deleted + clean
         result = warn(modified, _('not removing %s: file is modified (use -f'
                                   ' to force removal)\n'))
         result = warn(added, _('not removing %s: file has been marked for add'
                                ' (use forget to undo)\n')) or result
 
     # Need to lock because standin files are deleted then removed from the
     # repository and we could race in-between.
     with repo.wlock():
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         for f in sorted(remove):
             if ui.verbose or not m.exact(f):
                 ui.status(_('removing %s\n') % uipathfn(f))
 
             if not dryrun:
                 if not after:
                     repo.wvfs.unlinkpath(f, ignoremissing=True)
 
         if dryrun:
             return result
 
         remove = [lfutil.standin(f) for f in remove]
         # If this is being called by addremove, let the original addremove
         # function handle this.
         if not isaddremove:
             for f in remove:
                 repo.wvfs.unlinkpath(f, ignoremissing=True)
         repo[None].forget(remove)
 
         for f in remove:
             lfutil.synclfdirstate(repo, lfdirstate, lfutil.splitstandin(f),
                                   False)
 
         lfdirstate.write()
 
     return result
 
 # For overriding mercurial.hgweb.webcommands so that largefiles will
 # appear at their right place in the manifests.
 @eh.wrapfunction(webcommands, 'decodepath')
 def decodepath(orig, path):
     return lfutil.splitstandin(path) or path
 
 # -- Wrappers: modify existing commands --------------------------------
 
 @eh.wrapcommand('add',
     opts=[('', 'large', None, _('add as largefile')),
           ('', 'normal', None, _('add as normal file')),
           ('', 'lfsize', '', _('add all files above this size (in megabytes) '
                                'as largefiles (default: 10)'))])
 def overrideadd(orig, ui, repo, *pats, **opts):
     if opts.get(r'normal') and opts.get(r'large'):
         raise error.Abort(_('--normal cannot be used with --large'))
     return orig(ui, repo, *pats, **opts)
 
 @eh.wrapfunction(cmdutil, 'add')
 def cmdutiladd(orig, ui, repo, matcher, prefix, uipathfn, explicitonly, **opts):
     # The --normal flag short circuits this override
     if opts.get(r'normal'):
         return orig(ui, repo, matcher, prefix, uipathfn, explicitonly, **opts)
 
     ladded, lbad = addlargefiles(ui, repo, False, matcher, uipathfn, **opts)
     normalmatcher = composenormalfilematcher(matcher, repo[None].manifest(),
                                              ladded)
     bad = orig(ui, repo, normalmatcher, prefix, uipathfn, explicitonly, **opts)
 
     bad.extend(f for f in lbad)
     return bad
 
 @eh.wrapfunction(cmdutil, 'remove')
 def cmdutilremove(orig, ui, repo, matcher, prefix, uipathfn, after, force,
                   subrepos, dryrun):
     normalmatcher = composenormalfilematcher(matcher, repo[None].manifest())
     result = orig(ui, repo, normalmatcher, prefix, uipathfn, after, force,
                   subrepos, dryrun)
     return removelargefiles(ui, repo, False, matcher, uipathfn, dryrun,
                             after=after, force=force) or result
 
 @eh.wrapfunction(subrepo.hgsubrepo, 'status')
 def overridestatusfn(orig, repo, rev2, **opts):
     try:
         repo._repo.lfstatus = True
         return orig(repo, rev2, **opts)
     finally:
         repo._repo.lfstatus = False
 
 @eh.wrapcommand('status')
 def overridestatus(orig, ui, repo, *pats, **opts):
     try:
         repo.lfstatus = True
         return orig(ui, repo, *pats, **opts)
     finally:
         repo.lfstatus = False
 
 @eh.wrapfunction(subrepo.hgsubrepo, 'dirty')
 def overridedirty(orig, repo, ignoreupdate=False, missing=False):
     try:
         repo._repo.lfstatus = True
         return orig(repo, ignoreupdate=ignoreupdate, missing=missing)
     finally:
         repo._repo.lfstatus = False
 
 @eh.wrapcommand('log')
 def overridelog(orig, ui, repo, *pats, **opts):
     def overridematchandpats(orig, ctx, pats=(), opts=None, globbed=False,
             default='relpath', badfn=None):
         """Matcher that merges root directory with .hglf, suitable for log.
         It is still possible to match .hglf directly.
         For any listed files run log on the standin too.
         matchfn tries both the given filename and with .hglf stripped.
         """
         if opts is None:
             opts = {}
         matchandpats = orig(ctx, pats, opts, globbed, default, badfn=badfn)
         m, p = copy.copy(matchandpats)
 
         if m.always():
             # We want to match everything anyway, so there's no benefit trying
             # to add standins.
             return matchandpats
 
         pats = set(p)
 
         def fixpats(pat, tostandin=lfutil.standin):
             if pat.startswith('set:'):
                 return pat
 
             kindpat = matchmod._patsplit(pat, None)
 
             if kindpat[0] is not None:
                 return kindpat[0] + ':' + tostandin(kindpat[1])
             return tostandin(kindpat[1])
 
         cwd = repo.getcwd()
         if cwd:
             hglf = lfutil.shortname
             back = util.pconvert(repo.pathto(hglf)[:-len(hglf)])
 
             def tostandin(f):
                 # The file may already be a standin, so truncate the back
                 # prefix and test before mangling it.  This avoids turning
                 # 'glob:../.hglf/foo*' into 'glob:../.hglf/../.hglf/foo*'.
                 if f.startswith(back) and lfutil.splitstandin(f[len(back):]):
                     return f
 
                 # An absolute path is from outside the repo, so truncate the
                 # path to the root before building the standin.  Otherwise cwd
                 # is somewhere in the repo, relative to root, and needs to be
                 # prepended before building the standin.
                 if os.path.isabs(cwd):
                     f = f[len(back):]
                 else:
                     f = cwd + '/' + f
                 return back + lfutil.standin(f)
         else:
             def tostandin(f):
                 if lfutil.isstandin(f):
                     return f
                 return lfutil.standin(f)
         pats.update(fixpats(f, tostandin) for f in p)
 
         for i in range(0, len(m._files)):
             # Don't add '.hglf' to m.files, since that is already covered by '.'
             if m._files[i] == '.':
                 continue
             standin = lfutil.standin(m._files[i])
             # If the "standin" is a directory, append instead of replace to
             # support naming a directory on the command line with only
             # largefiles.  The original directory is kept to support normal
             # files.
             if standin in ctx:
                 m._files[i] = standin
             elif m._files[i] not in ctx and repo.wvfs.isdir(standin):
                 m._files.append(standin)
 
         m._fileset = set(m._files)
         m.always = lambda: False
         origmatchfn = m.matchfn
         def lfmatchfn(f):
             lf = lfutil.splitstandin(f)
             if lf is not None and origmatchfn(lf):
                 return True
             r = origmatchfn(f)
             return r
         m.matchfn = lfmatchfn
 
         ui.debug('updated patterns: %s\n' % ', '.join(sorted(pats)))
         return m, pats
 
     # For hg log --patch, the match object is used in two different senses:
     # (1) to determine what revisions should be printed out, and
     # (2) to determine what files to print out diffs for.
     # The magic matchandpats override should be used for case (1) but not for
     # case (2).
     oldmatchandpats = scmutil.matchandpats
     def overridemakefilematcher(orig, repo, pats, opts, badfn=None):
         wctx = repo[None]
         match, pats = oldmatchandpats(wctx, pats, opts, badfn=badfn)
         return lambda ctx: match
 
     wrappedmatchandpats = extensions.wrappedfunction(scmutil, 'matchandpats',
                                                      overridematchandpats)
     wrappedmakefilematcher = extensions.wrappedfunction(
         logcmdutil, '_makenofollowfilematcher', overridemakefilematcher)
     with wrappedmatchandpats, wrappedmakefilematcher:
         return orig(ui, repo, *pats, **opts)
 
 @eh.wrapcommand('verify',
     opts=[('', 'large', None,
                 _('verify that all largefiles in current revision exists')),
           ('', 'lfa', None,
                 _('verify largefiles in all revisions, not just current')),
           ('', 'lfc', None,
                 _('verify local largefile contents, not just existence'))])
 def overrideverify(orig, ui, repo, *pats, **opts):
     large = opts.pop(r'large', False)
     all = opts.pop(r'lfa', False)
     contents = opts.pop(r'lfc', False)
 
     result = orig(ui, repo, *pats, **opts)
     if large or all or contents:
         result = result or lfcommands.verifylfiles(ui, repo, all, contents)
     return result
 
 @eh.wrapcommand('debugstate',
     opts=[('', 'large', None, _('display largefiles dirstate'))])
 def overridedebugstate(orig, ui, repo, *pats, **opts):
     large = opts.pop(r'large', False)
     if large:
         class fakerepo(object):
             dirstate = lfutil.openlfdirstate(ui, repo)
         orig(ui, fakerepo, *pats, **opts)
     else:
         orig(ui, repo, *pats, **opts)
 
 # Before starting the manifest merge, merge.updates will call
 # _checkunknownfile to check if there are any files in the merged-in
 # changeset that collide with unknown files in the working copy.
 #
 # The largefiles are seen as unknown, so this prevents us from merging
 # in a file 'foo' if we already have a largefile with the same name.
 #
 # The overridden function filters the unknown files by removing any
 # largefiles. This makes the merge proceed and we can then handle this
 # case further in the overridden calculateupdates function below.
 @eh.wrapfunction(merge, '_checkunknownfile')
 def overridecheckunknownfile(origfn, repo, wctx, mctx, f, f2=None):
     if lfutil.standin(repo.dirstate.normalize(f)) in wctx:
         return False
     return origfn(repo, wctx, mctx, f, f2)
 
 # The manifest merge handles conflicts on the manifest level. We want
 # to handle changes in largefile-ness of files at this level too.
 #
 # The strategy is to run the original calculateupdates and then process
 # the action list it outputs. There are two cases we need to deal with:
 #
 # 1. Normal file in p1, largefile in p2. Here the largefile is
 #    detected via its standin file, which will enter the working copy
 #    with a "get" action. It is not "merge" since the standin is all
 #    Mercurial is concerned with at this level -- the link to the
 #    existing normal file is not relevant here.
 #
 # 2. Largefile in p1, normal file in p2. Here we get a "merge" action
 #    since the largefile will be present in the working copy and
 #    different from the normal file in p2. Mercurial therefore
 #    triggers a merge action.
 #
 # In both cases, we prompt the user and emit new actions to either
 # remove the standin (if the normal file was kept) or to remove the
 # normal file and get the standin (if the largefile was kept). The
 # default prompt answer is to use the largefile version since it was
 # presumably changed on purpose.
 #
 # Finally, the merge.applyupdates function will then take care of
 # writing the files into the working copy and lfcommands.updatelfiles
 # will update the largefiles.
 @eh.wrapfunction(merge, 'calculateupdates')
 def overridecalculateupdates(origfn, repo, p1, p2, pas, branchmerge, force,
                              acceptremote, *args, **kwargs):
     overwrite = force and not branchmerge
     actions, diverge, renamedelete = origfn(
         repo, p1, p2, pas, branchmerge, force, acceptremote, *args, **kwargs)
 
     if overwrite:
         return actions, diverge, renamedelete
 
     # Convert to dictionary with filename as key and action as value.
     lfiles = set()
     for f in actions:
         splitstandin = lfutil.splitstandin(f)
         if splitstandin in p1:
             lfiles.add(splitstandin)
         elif lfutil.standin(f) in p1:
             lfiles.add(f)
 
     for lfile in sorted(lfiles):
         standin = lfutil.standin(lfile)
         (lm, largs, lmsg) = actions.get(lfile, (None, None, None))
         (sm, sargs, smsg) = actions.get(standin, (None, None, None))
         if sm in ('g', 'dc') and lm != 'r':
             if sm == 'dc':
                 f1, f2, fa, move, anc = sargs
                 sargs = (p2[f2].flags(), False)
             # Case 1: normal file in the working copy, largefile in
             # the second parent
             usermsg = _('remote turned local normal file %s into a largefile\n'
                         'use (l)argefile or keep (n)ormal file?'
                         '$$ &Largefile $$ &Normal file') % lfile
             if repo.ui.promptchoice(usermsg, 0) == 0: # pick remote largefile
                 actions[lfile] = ('r', None, 'replaced by standin')
                 actions[standin] = ('g', sargs, 'replaces standin')
             else: # keep local normal file
                 actions[lfile] = ('k', None, 'replaces standin')
                 if branchmerge:
                     actions[standin] = ('k', None, 'replaced by non-standin')
                 else:
                     actions[standin] = ('r', None, 'replaced by non-standin')
         elif lm in ('g', 'dc') and sm != 'r':
             if lm == 'dc':
                 f1, f2, fa, move, anc = largs
                 largs = (p2[f2].flags(), False)
             # Case 2: largefile in the working copy, normal file in
             # the second parent
             usermsg = _('remote turned local largefile %s into a normal file\n'
                     'keep (l)argefile or use (n)ormal file?'
                     '$$ &Largefile $$ &Normal file') % lfile
             if repo.ui.promptchoice(usermsg, 0) == 0: # keep local largefile
                 if branchmerge:
                     # largefile can be restored from standin safely
                     actions[lfile] = ('k', None, 'replaced by standin')
                     actions[standin] = ('k', None, 'replaces standin')
                 else:
                     # "lfile" should be marked as "removed" without
                     # removal of itself
                     actions[lfile] = ('lfmr', None,
                                       'forget non-standin largefile')
 
                     # linear-merge should treat this largefile as 're-added'
                     actions[standin] = ('a', None, 'keep standin')
             else: # pick remote normal file
                 actions[lfile] = ('g', largs, 'replaces standin')
                 actions[standin] = ('r', None, 'replaced by non-standin')
 
     return actions, diverge, renamedelete
 
 @eh.wrapfunction(merge, 'recordupdates')
-def mergerecordupdates(orig, repo, actions, branchmerge):
+def mergerecordupdates(orig, repo, actions, branchmerge, getfiledata):
     if 'lfmr' in actions:
         lfdirstate = lfutil.openlfdirstate(repo.ui, repo)
         for lfile, args, msg in actions['lfmr']:
             # this should be executed before 'orig', to execute 'remove'
             # before all other actions
             repo.dirstate.remove(lfile)
             # make sure lfile doesn't get synclfdirstate'd as normal
             lfdirstate.add(lfile)
         lfdirstate.write()
 
-    return orig(repo, actions, branchmerge)
+    return orig(repo, actions, branchmerge, getfiledata)
 
 # Override filemerge to prompt the user about how they wish to merge
 # largefiles. This will handle identical edits without prompting the user.
 @eh.wrapfunction(filemerge, '_filemerge')
 def overridefilemerge(origfn, premerge, repo, wctx, mynode, orig, fcd, fco, fca,
                       labels=None):
     if not lfutil.isstandin(orig) or fcd.isabsent() or fco.isabsent():
         return origfn(premerge, repo, wctx, mynode, orig, fcd, fco, fca,
                       labels=labels)
 
     ahash = lfutil.readasstandin(fca).lower()
     dhash = lfutil.readasstandin(fcd).lower()
     ohash = lfutil.readasstandin(fco).lower()
     if (ohash != ahash and
         ohash != dhash and
         (dhash == ahash or
          repo.ui.promptchoice(
              _('largefile %s has a merge conflict\nancestor was %s\n'
                'keep (l)ocal %s or\ntake (o)ther %s?'
                '$$ &Local $$ &Other') %
                (lfutil.splitstandin(orig), ahash, dhash, ohash),
              0) == 1)):
         repo.wwrite(fcd.path(), fco.data(), fco.flags())
     return True, 0, False
 
 @eh.wrapfunction(copiesmod, 'pathcopies')
 def copiespathcopies(orig, ctx1, ctx2, match=None):
     copies = orig(ctx1, ctx2, match=match)
     updated = {}
 
     for k, v in copies.iteritems():
         updated[lfutil.splitstandin(k) or k] = lfutil.splitstandin(v) or v
 
     return updated
 
 # Copy first changes the matchers to match standins instead of
 # largefiles.  Then it overrides util.copyfile in that function it
 # checks if the destination largefile already exists. It also keeps a
 # list of copied files so that the largefiles can be copied and the
 # dirstate updated.
 @eh.wrapfunction(cmdutil, 'copy')
 def overridecopy(orig, ui, repo, pats, opts, rename=False):
     # doesn't remove largefile on rename
     if len(pats) < 2:
         # this isn't legal, let the original function deal with it
         return orig(ui, repo, pats, opts, rename)
 
     # This could copy both lfiles and normal files in one command,
     # but we don't want to do that. First replace their matcher to
     # only match normal files and run it, then replace it to just
     # match largefiles and run it again.
     nonormalfiles = False
     nolfiles = False
     manifest = repo[None].manifest()
     def normalfilesmatchfn(orig, ctx, pats=(), opts=None, globbed=False,
         default='relpath', badfn=None):
         if opts is None:
             opts = {}
         match = orig(ctx, pats, opts, globbed, default, badfn=badfn)
         return composenormalfilematcher(match, manifest)
     with extensions.wrappedfunction(scmutil, 'match', normalfilesmatchfn):
         try:
             result = orig(ui, repo, pats, opts, rename)
         except error.Abort as e:
             if pycompat.bytestr(e) != _('no files to copy'):
                 raise e
             else:
                 nonormalfiles = True
             result = 0
 
     # The first rename can cause our current working directory to be removed.
     # In that case there is nothing left to copy/rename so just quit.
     try:
         repo.getcwd()
     except OSError:
         return result
 
     def makestandin(relpath):
         path = pathutil.canonpath(repo.root, repo.getcwd(), relpath)
         return repo.wvfs.join(lfutil.standin(path))
 
     fullpats = scmutil.expandpats(pats)
     dest = fullpats[-1]
 
     if os.path.isdir(dest):
         if not os.path.isdir(makestandin(dest)):
             os.makedirs(makestandin(dest))
 
     try:
         # When we call orig below it creates the standins but we don't add
         # them to the dir state until later so lock during that time.
         wlock = repo.wlock()
 
         manifest = repo[None].manifest()
         def overridematch(orig, ctx, pats=(), opts=None, globbed=False,
                 default='relpath', badfn=None):
             if opts is None:
                 opts = {}
             newpats = []
             # The patterns were previously mangled to add the standin
             # directory; we need to remove that now
             for pat in pats:
                 if matchmod.patkind(pat) is None and lfutil.shortname in pat:
                     newpats.append(pat.replace(lfutil.shortname, ''))
                 else:
                     newpats.append(pat)
             match = orig(ctx, newpats, opts, globbed, default, badfn=badfn)
             m = copy.copy(match)
             lfile = lambda f: lfutil.standin(f) in manifest
             m._files = [lfutil.standin(f) for f in m._files if lfile(f)]
             m._fileset = set(m._files)
             origmatchfn = m.matchfn
             def matchfn(f):
                 lfile = lfutil.splitstandin(f)
                 return (lfile is not None and
                         (f in manifest) and
                         origmatchfn(lfile) or
                         None)
             m.matchfn = matchfn
             return m
         listpats = []
         for pat in pats:
             if matchmod.patkind(pat) is not None:
                 listpats.append(pat)
             else:
                 listpats.append(makestandin(pat))
 
         copiedfiles = []
         def overridecopyfile(orig, src, dest, *args, **kwargs):
             if (lfutil.shortname in src and
                 dest.startswith(repo.wjoin(lfutil.shortname))):
                 destlfile = dest.replace(lfutil.shortname, '')
                 if not opts['force'] and os.path.exists(destlfile):
                     raise IOError('',
                                   _('destination largefile already exists'))
             copiedfiles.append((src, dest))
             orig(src, dest, *args, **kwargs)
         with extensions.wrappedfunction(util, 'copyfile', overridecopyfile):
             with extensions.wrappedfunction(scmutil, 'match', overridematch):
                 result += orig(ui, repo, listpats, opts, rename)
 
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         for (src, dest) in copiedfiles:
             if (lfutil.shortname in src and
                 dest.startswith(repo.wjoin(lfutil.shortname))):
                 srclfile = src.replace(repo.wjoin(lfutil.standin('')), '')
                 destlfile = dest.replace(repo.wjoin(lfutil.standin('')), '')
                 destlfiledir = repo.wvfs.dirname(repo.wjoin(destlfile)) or '.'
                 if not os.path.isdir(destlfiledir):
                     os.makedirs(destlfiledir)
                 if rename:
                     os.rename(repo.wjoin(srclfile), repo.wjoin(destlfile))
 
                     # The file is gone, but this deletes any empty parent
                     # directories as a side-effect.
                     repo.wvfs.unlinkpath(srclfile, ignoremissing=True)
                     lfdirstate.remove(srclfile)
                 else:
                     util.copyfile(repo.wjoin(srclfile),
                                   repo.wjoin(destlfile))
 
                 lfdirstate.add(destlfile)
         lfdirstate.write()
     except error.Abort as e:
         if pycompat.bytestr(e) != _('no files to copy'):
             raise e
         else:
             nolfiles = True
     finally:
         wlock.release()
 
     if nolfiles and nonormalfiles:
         raise error.Abort(_('no files to copy'))
 
     return result
 
 # When the user calls revert, we have to be careful to not revert any
 # changes to other largefiles accidentally. This means we have to keep
 # track of the largefiles that are being reverted so we only pull down
 # the necessary largefiles.
 #
 # Standins are only updated (to match the hash of largefiles) before
 # commits. Update the standins then run the original revert, changing
 # the matcher to hit standins instead of largefiles. Based on the
 # resulting standins update the largefiles.
 @eh.wrapfunction(cmdutil, 'revert')
 def overriderevert(orig, ui, repo, ctx, parents, *pats, **opts):
     # Because we put the standins in a bad state (by updating them)
     # and then return them to a correct state we need to lock to
     # prevent others from changing them in their incorrect state.
     with repo.wlock():
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         s = lfutil.lfdirstatestatus(lfdirstate, repo)
         lfdirstate.write()
         for lfile in s.modified:
             lfutil.updatestandin(repo, lfile, lfutil.standin(lfile))
         for lfile in s.deleted:
             fstandin = lfutil.standin(lfile)
             if (repo.wvfs.exists(fstandin)):
                 repo.wvfs.unlink(fstandin)
 
         oldstandins = lfutil.getstandinsstate(repo)
 
         def overridematch(orig, mctx, pats=(), opts=None, globbed=False,
                 default='relpath', badfn=None):
             if opts is None:
                 opts = {}
             match = orig(mctx, pats, opts, globbed, default, badfn=badfn)
             m = copy.copy(match)
 
             # revert supports recursing into subrepos, and though largefiles
             # currently doesn't work correctly in that case, this match is
             # called, so the lfdirstate above may not be the correct one for
             # this invocation of match.
             lfdirstate = lfutil.openlfdirstate(mctx.repo().ui, mctx.repo(),
                                                False)
 
             wctx = repo[None]
             matchfiles = []
             for f in m._files:
                 standin = lfutil.standin(f)
                 if standin in ctx or standin in mctx:
                     matchfiles.append(standin)
                 elif standin in wctx or lfdirstate[f] == 'r':
                     continue
                 else:
                     matchfiles.append(f)
             m._files = matchfiles
             m._fileset = set(m._files)
             origmatchfn = m.matchfn
             def matchfn(f):
                 lfile = lfutil.splitstandin(f)
                 if lfile is not None:
                     return (origmatchfn(lfile) and
                             (f in ctx or f in mctx))
                 return origmatchfn(f)
             m.matchfn = matchfn
             return m
         with extensions.wrappedfunction(scmutil, 'match', overridematch):
             orig(ui, repo, ctx, parents, *pats, **opts)
 
         newstandins = lfutil.getstandinsstate(repo)
         filelist = lfutil.getlfilestoupdate(oldstandins, newstandins)
         # lfdirstate should be 'normallookup'-ed for updated files,
         # because reverting doesn't touch dirstate for 'normal' files
         # when target revision is explicitly specified: in such case,
         # 'n' and valid timestamp in dirstate doesn't ensure 'clean'
         # of target (standin) file.
         lfcommands.updatelfiles(ui, repo, filelist, printmessage=False,
                                 normallookup=True)
 
 # after pulling changesets, we need to take some extra care to get
 # largefiles updated remotely
 @eh.wrapcommand('pull',
     opts=[('', 'all-largefiles', None,
                 _('download all pulled versions of largefiles (DEPRECATED)')),
           ('', 'lfrev', [],
                 _('download largefiles for these revisions'), _('REV'))])
 def overridepull(orig, ui, repo, source=None, **opts):
     revsprepull = len(repo)
     if not source:
         source = 'default'
     repo.lfpullsource = source
     result = orig(ui, repo, source, **opts)
     revspostpull = len(repo)
     lfrevs = opts.get(r'lfrev', [])
     if opts.get(r'all_largefiles'):
         lfrevs.append('pulled()')
     if lfrevs and revspostpull > revsprepull:
         numcached = 0
         repo.firstpulled = revsprepull # for pulled() revset expression
         try:
             for rev in scmutil.revrange(repo, lfrevs):
                 ui.note(_('pulling largefiles for revision %d\n') % rev)
                 (cached, missing) = lfcommands.cachelfiles(ui, repo, rev)
                 numcached += len(cached)
         finally:
             del repo.firstpulled
         ui.status(_("%d largefiles cached\n") % numcached)
     return result
 
 @eh.wrapcommand('push',
     opts=[('', 'lfrev', [],
                _('upload largefiles for these revisions'), _('REV'))])
 def overridepush(orig, ui, repo, *args, **kwargs):
     """Override push command and store --lfrev parameters in opargs"""
     lfrevs = kwargs.pop(r'lfrev', None)
     if lfrevs:
         opargs = kwargs.setdefault(r'opargs', {})
         opargs['lfrevs'] = scmutil.revrange(repo, lfrevs)
     return orig(ui, repo, *args, **kwargs)
 
 @eh.wrapfunction(exchange, 'pushoperation')
 def exchangepushoperation(orig, *args, **kwargs):
     """Override pushoperation constructor and store lfrevs parameter"""
     lfrevs = kwargs.pop(r'lfrevs', None)
     pushop = orig(*args, **kwargs)
     pushop.lfrevs = lfrevs
     return pushop
 
 @eh.revsetpredicate('pulled()')
 def pulledrevsetsymbol(repo, subset, x):
     """Changesets that just has been pulled.
 
     Only available with largefiles from pull --lfrev expressions.
 
     .. container:: verbose
 
       Some examples:
 
       - pull largefiles for all new changesets::
 
           hg pull -lfrev "pulled()"
 
       - pull largefiles for all new branch heads::
 
           hg pull -lfrev "head(pulled()) and not closed()"
 
     """
 
     try:
         firstpulled = repo.firstpulled
     except AttributeError:
         raise error.Abort(_("pulled() only available in --lfrev"))
     return smartset.baseset([r for r in subset if r >= firstpulled])
 
 @eh.wrapcommand('clone',
     opts=[('', 'all-largefiles', None,
                _('download all versions of all largefiles'))])
 def overrideclone(orig, ui, source, dest=None, **opts):
     d = dest
     if d is None:
         d = hg.defaultdest(source)
     if opts.get(r'all_largefiles') and not hg.islocal(d):
             raise error.Abort(_(
             '--all-largefiles is incompatible with non-local destination %s') %
             d)
 
     return orig(ui, source, dest, **opts)
 
 @eh.wrapfunction(hg, 'clone')
 def hgclone(orig, ui, opts, *args, **kwargs):
     result = orig(ui, opts, *args, **kwargs)
 
     if result is not None:
         sourcerepo, destrepo = result
         repo = destrepo.local()
 
         # When cloning to a remote repo (like through SSH), no repo is available
         # from the peer.   Therefore the largefiles can't be downloaded and the
         # hgrc can't be updated.
         if not repo:
             return result
 
         # Caching is implicitly limited to 'rev' option, since the dest repo was
         # truncated at that point.  The user may expect a download count with
         # this option, so attempt whether or not this is a largefile repo.
         if opts.get('all_largefiles'):
             success, missing = lfcommands.downloadlfiles(ui, repo, None)
 
             if missing != 0:
                 return None
 
     return result
 
 @eh.wrapcommand('rebase', extension='rebase')
 def overriderebase(orig, ui, repo, **opts):
     if not util.safehasattr(repo, '_largefilesenabled'):
         return orig(ui, repo, **opts)
 
     resuming = opts.get(r'continue')
     repo._lfcommithooks.append(lfutil.automatedcommithook(resuming))
     repo._lfstatuswriters.append(lambda *msg, **opts: None)
     try:
         return orig(ui, repo, **opts)
     finally:
         repo._lfstatuswriters.pop()
         repo._lfcommithooks.pop()
 
 @eh.wrapcommand('archive')
 def overridearchivecmd(orig, ui, repo, dest, **opts):
     repo.unfiltered().lfstatus = True
 
     try:
         return orig(ui, repo.unfiltered(), dest, **opts)
     finally:
         repo.unfiltered().lfstatus = False
 
 @eh.wrapfunction(webcommands, 'archive')
 def hgwebarchive(orig, web):
     web.repo.lfstatus = True
 
     try:
         return orig(web)
     finally:
         web.repo.lfstatus = False
 
 @eh.wrapfunction(archival, 'archive')
 def overridearchive(orig, repo, dest, node, kind, decode=True, match=None,
             prefix='', mtime=None, subrepos=None):
     # For some reason setting repo.lfstatus in hgwebarchive only changes the
     # unfiltered repo's attr, so check that as well.
     if not repo.lfstatus and not repo.unfiltered().lfstatus:
         return orig(repo, dest, node, kind, decode, match, prefix, mtime,
                     subrepos)
 
     # No need to lock because we are only reading history and
     # largefile caches, neither of which are modified.
     if node is not None:
         lfcommands.cachelfiles(repo.ui, repo, node)
 
     if kind not in archival.archivers:
         raise error.Abort(_("unknown archive type '%s'") % kind)
 
     ctx = repo[node]
 
     if kind == 'files':
         if prefix:
             raise error.Abort(
                 _('cannot give prefix when archiving to files'))
     else:
         prefix = archival.tidyprefix(dest, kind, prefix)
 
     def write(name, mode, islink, getdata):
         if match and not match(name):
             return
         data = getdata()
         if decode:
             data = repo.wwritedata(name, data)
         archiver.addfile(prefix + name, mode, islink, data)
 
     archiver = archival.archivers[kind](dest, mtime or ctx.date()[0])
 
     if repo.ui.configbool("ui", "archivemeta"):
         write('.hg_archival.txt', 0o644, False,
               lambda: archival.buildmetadata(ctx))
 
     for f in ctx:
         ff = ctx.flags(f)
         getdata = ctx[f].data
         lfile = lfutil.splitstandin(f)
         if lfile is not None:
             if node is not None:
                 path = lfutil.findfile(repo, getdata().strip())
 
                 if path is None:
                     raise error.Abort(
                        _('largefile %s not found in repo store or system cache')
                        % lfile)
             else:
                 path = lfile
 
             f = lfile
 
             getdata = lambda: util.readfile(path)
         write(f, 'x' in ff and 0o755 or 0o644, 'l' in ff, getdata)
 
     if subrepos:
         for subpath in sorted(ctx.substate):
             sub = ctx.workingsub(subpath)
             submatch = matchmod.subdirmatcher(subpath, match)
             subprefix = prefix + subpath + '/'
             sub._repo.lfstatus = True
             sub.archive(archiver, subprefix, submatch)
 
     archiver.done()
 
 @eh.wrapfunction(subrepo.hgsubrepo, 'archive')
 def hgsubrepoarchive(orig, repo, archiver, prefix, match=None, decode=True):
     lfenabled = util.safehasattr(repo._repo, '_largefilesenabled')
     if not lfenabled or not repo._repo.lfstatus:
         return orig(repo, archiver, prefix, match, decode)
 
     repo._get(repo._state + ('hg',))
     rev = repo._state[1]
     ctx = repo._repo[rev]
 
     if ctx.node() is not None:
         lfcommands.cachelfiles(repo.ui, repo._repo, ctx.node())
 
     def write(name, mode, islink, getdata):
         # At this point, the standin has been replaced with the largefile name,
         # so the normal matcher works here without the lfutil variants.
         if match and not match(f):
             return
         data = getdata()
         if decode:
             data = repo._repo.wwritedata(name, data)
 
         archiver.addfile(prefix + name, mode, islink, data)
 
     for f in ctx:
         ff = ctx.flags(f)
         getdata = ctx[f].data
         lfile = lfutil.splitstandin(f)
         if lfile is not None:
             if ctx.node() is not None:
                 path = lfutil.findfile(repo._repo, getdata().strip())
 
                 if path is None:
                     raise error.Abort(
                        _('largefile %s not found in repo store or system cache')
                        % lfile)
             else:
                 path = lfile
 
             f = lfile
 
             getdata = lambda: util.readfile(os.path.join(prefix, path))
 
         write(f, 'x' in ff and 0o755 or 0o644, 'l' in ff, getdata)
 
     for subpath in sorted(ctx.substate):
         sub = ctx.workingsub(subpath)
         submatch = matchmod.subdirmatcher(subpath, match)
         subprefix = prefix + subpath + '/'
         sub._repo.lfstatus = True
         sub.archive(archiver, subprefix, submatch, decode)
 
 # If a largefile is modified, the change is not reflected in its
 # standin until a commit. cmdutil.bailifchanged() raises an exception
 # if the repo has uncommitted changes. Wrap it to also check if
 # largefiles were changed. This is used by bisect, backout and fetch.
 @eh.wrapfunction(cmdutil, 'bailifchanged')
 def overridebailifchanged(orig, repo, *args, **kwargs):
     orig(repo, *args, **kwargs)
     repo.lfstatus = True
     s = repo.status()
     repo.lfstatus = False
     if s.modified or s.added or s.removed or s.deleted:
         raise error.Abort(_('uncommitted changes'))
 
 @eh.wrapfunction(cmdutil, 'postcommitstatus')
 def postcommitstatus(orig, repo, *args, **kwargs):
     repo.lfstatus = True
     try:
         return orig(repo, *args, **kwargs)
     finally:
         repo.lfstatus = False
 
 @eh.wrapfunction(cmdutil, 'forget')
 def cmdutilforget(orig, ui, repo, match, prefix, uipathfn, explicitonly, dryrun,
                   interactive):
     normalmatcher = composenormalfilematcher(match, repo[None].manifest())
     bad, forgot = orig(ui, repo, normalmatcher, prefix, uipathfn, explicitonly,
                        dryrun, interactive)
     m = composelargefilematcher(match, repo[None].manifest())
 
     try:
         repo.lfstatus = True
         s = repo.status(match=m, clean=True)
     finally:
         repo.lfstatus = False
     manifest = repo[None].manifest()
     forget = sorted(s.modified + s.added + s.deleted + s.clean)
     forget = [f for f in forget if lfutil.standin(f) in manifest]
 
     for f in forget:
         fstandin = lfutil.standin(f)
         if fstandin not in repo.dirstate and not repo.wvfs.isdir(fstandin):
             ui.warn(_('not removing %s: file is already untracked\n')
                     % uipathfn(f))
             bad.append(f)
 
     for f in forget:
         if ui.verbose or not m.exact(f):
             ui.status(_('removing %s\n') % uipathfn(f))
 
     # Need to lock because standin files are deleted then removed from the
     # repository and we could race in-between.
     with repo.wlock():
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         for f in forget:
             if lfdirstate[f] == 'a':
                 lfdirstate.drop(f)
             else:
                 lfdirstate.remove(f)
         lfdirstate.write()
         standins = [lfutil.standin(f) for f in forget]
         for f in standins:
             repo.wvfs.unlinkpath(f, ignoremissing=True)
         rejected = repo[None].forget(standins)
 
     bad.extend(f for f in rejected if f in m.files())
     forgot.extend(f for f in forget if f not in rejected)
     return bad, forgot
 
 def _getoutgoings(repo, other, missing, addfunc):
     """get pairs of filename and largefile hash in outgoing revisions
     in 'missing'.
 
     largefiles already existing on 'other' repository are ignored.
 
     'addfunc' is invoked with each unique pairs of filename and
     largefile hash value.
     """
     knowns = set()
     lfhashes = set()
     def dedup(fn, lfhash):
         k = (fn, lfhash)
         if k not in knowns:
             knowns.add(k)
             lfhashes.add(lfhash)
     lfutil.getlfilestoupload(repo, missing, dedup)
     if lfhashes:
         lfexists = storefactory.openstore(repo, other).exists(lfhashes)
         for fn, lfhash in knowns:
             if not lfexists[lfhash]: # lfhash doesn't exist on "other"
                 addfunc(fn, lfhash)
 
 def outgoinghook(ui, repo, other, opts, missing):
     if opts.pop('large', None):
         lfhashes = set()
         if ui.debugflag:
             toupload = {}
             def addfunc(fn, lfhash):
                 if fn not in toupload:
                     toupload[fn] = []
                 toupload[fn].append(lfhash)
                 lfhashes.add(lfhash)
             def showhashes(fn):
                 for lfhash in sorted(toupload[fn]):
                     ui.debug('    %s\n' % (lfhash))
         else:
             toupload = set()
             def addfunc(fn, lfhash):
                 toupload.add(fn)
                 lfhashes.add(lfhash)
             def showhashes(fn):
                 pass
         _getoutgoings(repo, other, missing, addfunc)
 
         if not toupload:
             ui.status(_('largefiles: no files to upload\n'))
         else:
             ui.status(_('largefiles to upload (%d entities):\n')
                       % (len(lfhashes)))
             for file in sorted(toupload):
                 ui.status(lfutil.splitstandin(file) + '\n')
                 showhashes(file)
             ui.status('\n')
 
 @eh.wrapcommand('outgoing',
     opts=[('', 'large', None, _('display outgoing largefiles'))])
 def _outgoingcmd(orig, *args, **kwargs):
     # Nothing to do here other than add the extra help option- the hook above
     # processes it.
     return orig(*args, **kwargs)
 
 def summaryremotehook(ui, repo, opts, changes):
     largeopt = opts.get('large', False)
     if changes is None:
         if largeopt:
             return (False, True) # only outgoing check is needed
         else:
             return (False, False)
     elif largeopt:
         url, branch, peer, outgoing = changes[1]
         if peer is None:
             # i18n: column positioning for "hg summary"
             ui.status(_('largefiles: (no remote repo)\n'))
             return
 
         toupload = set()
         lfhashes = set()
         def addfunc(fn, lfhash):
             toupload.add(fn)
             lfhashes.add(lfhash)
         _getoutgoings(repo, peer, outgoing.missing, addfunc)
 
         if not toupload:
             # i18n: column positioning for "hg summary"
             ui.status(_('largefiles: (no files to upload)\n'))
         else:
             # i18n: column positioning for "hg summary"
             ui.status(_('largefiles: %d entities for %d files to upload\n')
                       % (len(lfhashes), len(toupload)))
 
 @eh.wrapcommand('summary',
     opts=[('', 'large', None, _('display outgoing largefiles'))])
 def overridesummary(orig, ui, repo, *pats, **opts):
     try:
         repo.lfstatus = True
         orig(ui, repo, *pats, **opts)
     finally:
         repo.lfstatus = False
 
 @eh.wrapfunction(scmutil, 'addremove')
 def scmutiladdremove(orig, repo, matcher, prefix, uipathfn, opts=None):
     if opts is None:
         opts = {}
     if not lfutil.islfilesrepo(repo):
         return orig(repo, matcher, prefix, uipathfn, opts)
     # Get the list of missing largefiles so we can remove them
     lfdirstate = lfutil.openlfdirstate(repo.ui, repo)
     unsure, s = lfdirstate.status(matchmod.always(), subrepos=[],
                                   ignored=False, clean=False, unknown=False)
 
     # Call into the normal remove code, but the removing of the standin, we want
     # to have handled by original addremove.  Monkey patching here makes sure
     # we don't remove the standin in the largefiles code, preventing a very
     # confused state later.
     if s.deleted:
         m = copy.copy(matcher)
 
         # The m._files and m._map attributes are not changed to the deleted list
         # because that affects the m.exact() test, which in turn governs whether
         # or not the file name is printed, and how.  Simply limit the original
         # matches to those in the deleted status list.
         matchfn = m.matchfn
         m.matchfn = lambda f: f in s.deleted and matchfn(f)
 
         removelargefiles(repo.ui, repo, True, m, uipathfn, opts.get('dry_run'),
                          **pycompat.strkwargs(opts))
     # Call into the normal add code, and any files that *should* be added as
     # largefiles will be
     added, bad = addlargefiles(repo.ui, repo, True, matcher, uipathfn,
                                **pycompat.strkwargs(opts))
     # Now that we've handled largefiles, hand off to the original addremove
     # function to take care of the rest.  Make sure it doesn't do anything with
     # largefiles by passing a matcher that will ignore them.
     matcher = composenormalfilematcher(matcher, repo[None].manifest(), added)
     return orig(repo, matcher, prefix, uipathfn, opts)
 
 # Calling purge with --all will cause the largefiles to be deleted.
 # Override repo.status to prevent this from happening.
 @eh.wrapcommand('purge', extension='purge')
 def overridepurge(orig, ui, repo, *dirs, **opts):
     # XXX Monkey patching a repoview will not work. The assigned attribute will
     # be set on the unfiltered repo, but we will only lookup attributes in the
     # unfiltered repo if the lookup in the repoview object itself fails. As the
     # monkey patched method exists on the repoview class the lookup will not
     # fail. As a result, the original version will shadow the monkey patched
     # one, defeating the monkey patch.
     #
     # As a work around we use an unfiltered repo here. We should do something
     # cleaner instead.
     repo = repo.unfiltered()
     oldstatus = repo.status
     def overridestatus(node1='.', node2=None, match=None, ignored=False,
                         clean=False, unknown=False, listsubrepos=False):
         r = oldstatus(node1, node2, match, ignored, clean, unknown,
                       listsubrepos)
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         unknown = [f for f in r.unknown if lfdirstate[f] == '?']
         ignored = [f for f in r.ignored if lfdirstate[f] == '?']
         return scmutil.status(r.modified, r.added, r.removed, r.deleted,
                               unknown, ignored, r.clean)
     repo.status = overridestatus
     orig(ui, repo, *dirs, **opts)
     repo.status = oldstatus
 
 @eh.wrapcommand('rollback')
 def overriderollback(orig, ui, repo, **opts):
     with repo.wlock():
         before = repo.dirstate.parents()
         orphans = set(f for f in repo.dirstate
                       if lfutil.isstandin(f) and repo.dirstate[f] != 'r')
         result = orig(ui, repo, **opts)
         after = repo.dirstate.parents()
         if before == after:
             return result # no need to restore standins
 
         pctx = repo['.']
         for f in repo.dirstate:
             if lfutil.isstandin(f):
                 orphans.discard(f)
                 if repo.dirstate[f] == 'r':
                     repo.wvfs.unlinkpath(f, ignoremissing=True)
                 elif f in pctx:
                     fctx = pctx[f]
                     repo.wwrite(f, fctx.data(), fctx.flags())
                 else:
                     # content of standin is not so important in 'a',
                     # 'm' or 'n' (coming from the 2nd parent) cases
                     lfutil.writestandin(repo, f, '', False)
         for standin in orphans:
             repo.wvfs.unlinkpath(standin, ignoremissing=True)
 
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         orphans = set(lfdirstate)
         lfiles = lfutil.listlfiles(repo)
         for file in lfiles:
             lfutil.synclfdirstate(repo, lfdirstate, file, True)
             orphans.discard(file)
         for lfile in orphans:
             lfdirstate.drop(lfile)
         lfdirstate.write()
     return result
 
 @eh.wrapcommand('transplant', extension='transplant')
 def overridetransplant(orig, ui, repo, *revs, **opts):
     resuming = opts.get(r'continue')
     repo._lfcommithooks.append(lfutil.automatedcommithook(resuming))
     repo._lfstatuswriters.append(lambda *msg, **opts: None)
     try:
         result = orig(ui, repo, *revs, **opts)
     finally:
         repo._lfstatuswriters.pop()
         repo._lfcommithooks.pop()
     return result
 
 @eh.wrapcommand('cat')
 def overridecat(orig, ui, repo, file1, *pats, **opts):
     opts = pycompat.byteskwargs(opts)
     ctx = scmutil.revsingle(repo, opts.get('rev'))
     err = 1
     notbad = set()
     m = scmutil.match(ctx, (file1,) + pats, opts)
     origmatchfn = m.matchfn
     def lfmatchfn(f):
         if origmatchfn(f):
             return True
         lf = lfutil.splitstandin(f)
         if lf is None:
             return False
         notbad.add(lf)
         return origmatchfn(lf)
     m.matchfn = lfmatchfn
     origbadfn = m.bad
     def lfbadfn(f, msg):
         if not f in notbad:
             origbadfn(f, msg)
     m.bad = lfbadfn
 
     origvisitdirfn = m.visitdir
     def lfvisitdirfn(dir):
         if dir == lfutil.shortname:
             return True
         ret = origvisitdirfn(dir)
         if ret:
             return ret
         lf = lfutil.splitstandin(dir)
         if lf is None:
             return False
         return origvisitdirfn(lf)
     m.visitdir = lfvisitdirfn
 
     for f in ctx.walk(m):
         with cmdutil.makefileobj(ctx, opts.get('output'), pathname=f) as fp:
             lf = lfutil.splitstandin(f)
             if lf is None or origmatchfn(f):
                 # duplicating unreachable code from commands.cat
                 data = ctx[f].data()
                 if opts.get('decode'):
                     data = repo.wwritedata(f, data)
                 fp.write(data)
             else:
                 hash = lfutil.readasstandin(ctx[f])
                 if not lfutil.inusercache(repo.ui, hash):
                     store = storefactory.openstore(repo)
                     success, missing = store.get([(lf, hash)])
                     if len(success) != 1:
                         raise error.Abort(
                             _('largefile %s is not in cache and could not be '
                               'downloaded')  % lf)
                 path = lfutil.usercachepath(repo.ui, hash)
                 with open(path, "rb") as fpin:
                     for chunk in util.filechunkiter(fpin):
                         fp.write(chunk)
         err = 0
     return err
 
 @eh.wrapfunction(merge, 'update')
 def mergeupdate(orig, repo, node, branchmerge, force,
                 *args, **kwargs):
     matcher = kwargs.get(r'matcher', None)
     # note if this is a partial update
     partial = matcher and not matcher.always()
     with repo.wlock():
         # branch |       |         |
         #  merge | force | partial | action
         # -------+-------+---------+--------------
         #    x   |   x   |    x    | linear-merge
         #    o   |   x   |    x    | branch-merge
         #    x   |   o   |    x    | overwrite (as clean update)
         #    o   |   o   |    x    | force-branch-merge (*1)
         #    x   |   x   |    o    |   (*)
         #    o   |   x   |    o    |   (*)
         #    x   |   o   |    o    | overwrite (as revert)
         #    o   |   o   |    o    |   (*)
         #
         # (*) don't care
         # (*1) deprecated, but used internally (e.g: "rebase --collapse")
 
         lfdirstate = lfutil.openlfdirstate(repo.ui, repo)
         unsure, s = lfdirstate.status(matchmod.always(), subrepos=[],
                                       ignored=False, clean=True, unknown=False)
         oldclean = set(s.clean)
         pctx = repo['.']
         dctx = repo[node]
         for lfile in unsure + s.modified:
             lfileabs = repo.wvfs.join(lfile)
             if not repo.wvfs.exists(lfileabs):
                 continue
             lfhash = lfutil.hashfile(lfileabs)
             standin = lfutil.standin(lfile)
             lfutil.writestandin(repo, standin, lfhash,
                                 lfutil.getexecutable(lfileabs))
             if (standin in pctx and
                 lfhash == lfutil.readasstandin(pctx[standin])):
                 oldclean.add(lfile)
         for lfile in s.added:
             fstandin = lfutil.standin(lfile)
             if fstandin not in dctx:
                 # in this case, content of standin file is meaningless
                 # (in dctx, lfile is unknown, or normal file)
                 continue
             lfutil.updatestandin(repo, lfile, fstandin)
         # mark all clean largefiles as dirty, just in case the update gets
         # interrupted before largefiles and lfdirstate are synchronized
         for lfile in oldclean:
             lfdirstate.normallookup(lfile)
         lfdirstate.write()
 
         oldstandins = lfutil.getstandinsstate(repo)
         # Make sure the merge runs on disk, not in-memory. largefiles is not a
         # good candidate for in-memory merge (large files, custom dirstate,
         # matcher usage).
         kwargs[r'wc'] = repo[None]
         result = orig(repo, node, branchmerge, force, *args, **kwargs)
 
         newstandins = lfutil.getstandinsstate(repo)
         filelist = lfutil.getlfilestoupdate(oldstandins, newstandins)
 
         # to avoid leaving all largefiles as dirty and thus rehash them, mark
         # all the ones that didn't change as clean
         for lfile in oldclean.difference(filelist):
             lfdirstate.normal(lfile)
         lfdirstate.write()
 
         if branchmerge or force or partial:
             filelist.extend(s.deleted + s.removed)
 
         lfcommands.updatelfiles(repo.ui, repo, filelist=filelist,
                                 normallookup=partial)
 
         return result
 
 @eh.wrapfunction(scmutil, 'marktouched')
 def scmutilmarktouched(orig, repo, files, *args, **kwargs):
     result = orig(repo, files, *args, **kwargs)
 
     filelist = []
     for f in files:
         lf = lfutil.splitstandin(f)
         if lf is not None:
             filelist.append(lf)
     if filelist:
         lfcommands.updatelfiles(repo.ui, repo, filelist=filelist,
                                 printmessage=False, normallookup=True)
 
     return result
 
 @eh.wrapfunction(upgrade, 'preservedrequirements')
 @eh.wrapfunction(upgrade, 'supporteddestrequirements')
 def upgraderequirements(orig, repo):
     reqs = orig(repo)
     if 'largefiles' in repo.requirements:
         reqs.add('largefiles')
     return reqs
 
 _lfscheme = 'largefile://'
 
 @eh.wrapfunction(urlmod, 'open')
 def openlargefile(orig, ui, url_, data=None):
     if url_.startswith(_lfscheme):
         if data:
             msg = "cannot use data on a 'largefile://' url"
             raise error.ProgrammingError(msg)
         lfid = url_[len(_lfscheme):]
         return storefactory.getlfile(ui, lfid)
     else:
         return orig(ui, url_, data=data)

hgext/narrow/narrowdirstate.py

@@ -1,63 +1,63
 # narrowdirstate.py - extensions to mercurial dirstate to support narrow clones
 #
 # Copyright 2017 Google, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 from mercurial.i18n import _
 from mercurial import (
     error,
 )
 
 def wrapdirstate(repo, dirstate):
     """Add narrow spec dirstate ignore, block changes outside narrow spec."""
 
     def _editfunc(fn):
-        def _wrapper(self, *args):
+        def _wrapper(self, *args, **kwargs):
             narrowmatch = repo.narrowmatch()
             for f in args:
                 if f is not None and not narrowmatch(f) and f not in self:
                     raise error.Abort(_("cannot track '%s' - it is outside " +
                         "the narrow clone") % f)
-            return fn(self, *args)
+            return fn(self, *args, **kwargs)
         return _wrapper
 
     class narrowdirstate(dirstate.__class__):
         # Prevent adding/editing/copying/deleting files that are outside the
         # sparse checkout
         @_editfunc
-        def normal(self, *args):
-            return super(narrowdirstate, self).normal(*args)
+        def normal(self, *args, **kwargs):
+            return super(narrowdirstate, self).normal(*args, **kwargs)
 
         @_editfunc
         def add(self, *args):
             return super(narrowdirstate, self).add(*args)
 
         @_editfunc
         def normallookup(self, *args):
             return super(narrowdirstate, self).normallookup(*args)
 
         @_editfunc
         def copy(self, *args):
             return super(narrowdirstate, self).copy(*args)
 
         @_editfunc
         def remove(self, *args):
             return super(narrowdirstate, self).remove(*args)
 
         @_editfunc
         def merge(self, *args):
             return super(narrowdirstate, self).merge(*args)
 
         def rebuild(self, parent, allfiles, changedfiles=None):
             if changedfiles is None:
                 # Rebuilding entire dirstate, let's filter allfiles to match the
                 # narrowspec.
                 allfiles = [f for f in allfiles if repo.narrowmatch()(f)]
             super(narrowdirstate, self).rebuild(parent, allfiles, changedfiles)
 
     dirstate.__class__ = narrowdirstate
     return dirstate

hgext/remotefilelog/__init__.py

@@ -1,1111 +1,1113
 # __init__.py - remotefilelog extension
 #
 # Copyright 2013 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 """remotefilelog causes Mercurial to lazilly fetch file contents (EXPERIMENTAL)
 
 This extension is HIGHLY EXPERIMENTAL. There are NO BACKWARDS COMPATIBILITY
 GUARANTEES. This means that repositories created with this extension may
 only be usable with the exact version of this extension/Mercurial that was
 used. The extension attempts to enforce this in order to prevent repository
 corruption.
 
 remotefilelog works by fetching file contents lazily and storing them
 in a cache on the client rather than in revlogs. This allows enormous
 histories to be transferred only partially, making them easier to
 operate on.
 
 Configs:
 
     ``packs.maxchainlen`` specifies the maximum delta chain length in pack files
 
     ``packs.maxpacksize`` specifies the maximum pack file size
 
     ``packs.maxpackfilecount`` specifies the maximum number of packs in the
       shared cache (trees only for now)
 
     ``remotefilelog.backgroundprefetch`` runs prefetch in background when True
 
     ``remotefilelog.bgprefetchrevs`` specifies revisions to fetch on commit and
       update, and on other commands that use them. Different from pullprefetch.
 
     ``remotefilelog.gcrepack`` does garbage collection during repack when True
 
     ``remotefilelog.nodettl`` specifies maximum TTL of a node in seconds before
       it is garbage collected
 
     ``remotefilelog.repackonhggc`` runs repack on hg gc when True
 
     ``remotefilelog.prefetchdays`` specifies the maximum age of a commit in
       days after which it is no longer prefetched.
 
     ``remotefilelog.prefetchdelay`` specifies delay between background
       prefetches in seconds after operations that change the working copy parent
 
     ``remotefilelog.data.gencountlimit`` constraints the minimum number of data
       pack files required to be considered part of a generation. In particular,
       minimum number of packs files > gencountlimit.
 
     ``remotefilelog.data.generations`` list for specifying the lower bound of
       each generation of the data pack files. For example, list ['100MB','1MB']
       or ['1MB', '100MB'] will lead to three generations: [0, 1MB), [
       1MB, 100MB) and [100MB, infinity).
 
     ``remotefilelog.data.maxrepackpacks`` the maximum number of pack files to
       include in an incremental data repack.
 
     ``remotefilelog.data.repackmaxpacksize`` the maximum size of a pack file for
       it to be considered for an incremental data repack.
 
     ``remotefilelog.data.repacksizelimit`` the maximum total size of pack files
       to include in an incremental data repack.
 
     ``remotefilelog.history.gencountlimit`` constraints the minimum number of
       history pack files required to be considered part of a generation. In
       particular, minimum number of packs files > gencountlimit.
 
     ``remotefilelog.history.generations`` list for specifying the lower bound of
       each generation of the history pack files. For example, list [
       '100MB', '1MB'] or ['1MB', '100MB'] will lead to three generations: [
       0, 1MB), [1MB, 100MB) and [100MB, infinity).
 
     ``remotefilelog.history.maxrepackpacks`` the maximum number of pack files to
       include in an incremental history repack.
 
     ``remotefilelog.history.repackmaxpacksize`` the maximum size of a pack file
       for it to be considered for an incremental history repack.
 
     ``remotefilelog.history.repacksizelimit`` the maximum total size of pack
       files to include in an incremental history repack.
 
     ``remotefilelog.backgroundrepack`` automatically consolidate packs in the
       background
 
     ``remotefilelog.cachepath`` path to cache
 
     ``remotefilelog.cachegroup`` if set, make cache directory sgid to this
       group
 
     ``remotefilelog.cacheprocess`` binary to invoke for fetching file data
 
     ``remotefilelog.debug`` turn on remotefilelog-specific debug output
 
     ``remotefilelog.excludepattern`` pattern of files to exclude from pulls
 
     ``remotefilelog.includepattern`` pattern of files to include in pulls
 
     ``remotefilelog.fetchwarning``: message to print when too many
       single-file fetches occur
 
     ``remotefilelog.getfilesstep`` number of files to request in a single RPC
 
     ``remotefilelog.getfilestype`` if set to 'threaded' use threads to fetch
       files, otherwise use optimistic fetching
 
     ``remotefilelog.pullprefetch`` revset for selecting files that should be
       eagerly downloaded rather than lazily
 
     ``remotefilelog.reponame`` name of the repo. If set, used to partition
       data from other repos in a shared store.
 
     ``remotefilelog.server`` if true, enable server-side functionality
 
     ``remotefilelog.servercachepath`` path for caching blobs on the server
 
     ``remotefilelog.serverexpiration`` number of days to keep cached server
       blobs
 
     ``remotefilelog.validatecache`` if set, check cache entries for corruption
       before returning blobs
 
     ``remotefilelog.validatecachelog`` if set, check cache entries for
       corruption before returning metadata
 
 """
 from __future__ import absolute_import
 
 import os
 import time
 import traceback
 
 from mercurial.node import hex
 from mercurial.i18n import _
 from mercurial import (
     changegroup,
     changelog,
     cmdutil,
     commands,
     configitems,
     context,
     copies,
     debugcommands as hgdebugcommands,
     dispatch,
     error,
     exchange,
     extensions,
     hg,
     localrepo,
     match,
     merge,
     node as nodemod,
     patch,
     pycompat,
     registrar,
     repair,
     repoview,
     revset,
     scmutil,
     smartset,
     streamclone,
     util,
 )
 from . import (
     constants,
     debugcommands,
     fileserverclient,
     remotefilectx,
     remotefilelog,
     remotefilelogserver,
     repack as repackmod,
     shallowbundle,
     shallowrepo,
     shallowstore,
     shallowutil,
     shallowverifier,
 )
 
 # ensures debug commands are registered
 hgdebugcommands.command
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 configitem('remotefilelog', 'debug', default=False)
 
 configitem('remotefilelog', 'reponame', default='')
 configitem('remotefilelog', 'cachepath', default=None)
 configitem('remotefilelog', 'cachegroup', default=None)
 configitem('remotefilelog', 'cacheprocess', default=None)
 configitem('remotefilelog', 'cacheprocess.includepath', default=None)
 configitem("remotefilelog", "cachelimit", default="1000 GB")
 
 configitem('remotefilelog', 'fallbackpath', default=configitems.dynamicdefault,
            alias=[('remotefilelog', 'fallbackrepo')])
 
 configitem('remotefilelog', 'validatecachelog', default=None)
 configitem('remotefilelog', 'validatecache', default='on')
 configitem('remotefilelog', 'server', default=None)
 configitem('remotefilelog', 'servercachepath', default=None)
 configitem("remotefilelog", "serverexpiration", default=30)
 configitem('remotefilelog', 'backgroundrepack', default=False)
 configitem('remotefilelog', 'bgprefetchrevs', default=None)
 configitem('remotefilelog', 'pullprefetch', default=None)
 configitem('remotefilelog', 'backgroundprefetch', default=False)
 configitem('remotefilelog', 'prefetchdelay', default=120)
 configitem('remotefilelog', 'prefetchdays', default=14)
 
 configitem('remotefilelog', 'getfilesstep', default=10000)
 configitem('remotefilelog', 'getfilestype', default='optimistic')
 configitem('remotefilelog', 'batchsize', configitems.dynamicdefault)
 configitem('remotefilelog', 'fetchwarning', default='')
 
 configitem('remotefilelog', 'includepattern', default=None)
 configitem('remotefilelog', 'excludepattern', default=None)
 
 configitem('remotefilelog', 'gcrepack', default=False)
 configitem('remotefilelog', 'repackonhggc', default=False)
 configitem('repack', 'chainorphansbysize', default=True)
 
 configitem('packs', 'maxpacksize', default=0)
 configitem('packs', 'maxchainlen', default=1000)
 
 #  default TTL limit is 30 days
 _defaultlimit = 60 * 60 * 24 * 30
 configitem('remotefilelog', 'nodettl', default=_defaultlimit)
 
 configitem('remotefilelog', 'data.gencountlimit', default=2),
 configitem('remotefilelog', 'data.generations',
            default=['1GB', '100MB', '1MB'])
 configitem('remotefilelog', 'data.maxrepackpacks', default=50)
 configitem('remotefilelog', 'data.repackmaxpacksize', default='4GB')
 configitem('remotefilelog', 'data.repacksizelimit', default='100MB')
 
 configitem('remotefilelog', 'history.gencountlimit', default=2),
 configitem('remotefilelog', 'history.generations', default=['100MB'])
 configitem('remotefilelog', 'history.maxrepackpacks', default=50)
 configitem('remotefilelog', 'history.repackmaxpacksize', default='400MB')
 configitem('remotefilelog', 'history.repacksizelimit', default='100MB')
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
 repoclass = localrepo.localrepository
 repoclass._basesupported.add(constants.SHALLOWREPO_REQUIREMENT)
 
 isenabled = shallowutil.isenabled
 
 def uisetup(ui):
     """Wraps user facing Mercurial commands to swap them out with shallow
     versions.
     """
     hg.wirepeersetupfuncs.append(fileserverclient.peersetup)
 
     entry = extensions.wrapcommand(commands.table, 'clone', cloneshallow)
     entry[1].append(('', 'shallow', None,
                      _("create a shallow clone which uses remote file "
                        "history")))
 
     extensions.wrapcommand(commands.table, 'debugindex',
         debugcommands.debugindex)
     extensions.wrapcommand(commands.table, 'debugindexdot',
         debugcommands.debugindexdot)
     extensions.wrapcommand(commands.table, 'log', log)
     extensions.wrapcommand(commands.table, 'pull', pull)
 
     # Prevent 'hg manifest --all'
     def _manifest(orig, ui, repo, *args, **opts):
         if (isenabled(repo) and opts.get(r'all')):
             raise error.Abort(_("--all is not supported in a shallow repo"))
 
         return orig(ui, repo, *args, **opts)
     extensions.wrapcommand(commands.table, "manifest", _manifest)
 
     # Wrap remotefilelog with lfs code
     def _lfsloaded(loaded=False):
         lfsmod = None
         try:
             lfsmod = extensions.find('lfs')
         except KeyError:
             pass
         if lfsmod:
             lfsmod.wrapfilelog(remotefilelog.remotefilelog)
             fileserverclient._lfsmod = lfsmod
     extensions.afterloaded('lfs', _lfsloaded)
 
     # debugdata needs remotefilelog.len to work
     extensions.wrapcommand(commands.table, 'debugdata', debugdatashallow)
 
     changegroup.cgpacker = shallowbundle.shallowcg1packer
 
     extensions.wrapfunction(changegroup, '_addchangegroupfiles',
                             shallowbundle.addchangegroupfiles)
     extensions.wrapfunction(
         changegroup, 'makechangegroup', shallowbundle.makechangegroup)
     extensions.wrapfunction(localrepo, 'makestore', storewrapper)
     extensions.wrapfunction(exchange, 'pull', exchangepull)
     extensions.wrapfunction(merge, 'applyupdates', applyupdates)
     extensions.wrapfunction(merge, '_checkunknownfiles', checkunknownfiles)
     extensions.wrapfunction(context.workingctx, '_checklookup', checklookup)
     extensions.wrapfunction(scmutil, '_findrenames', findrenames)
     extensions.wrapfunction(copies, '_computeforwardmissing',
                             computeforwardmissing)
     extensions.wrapfunction(dispatch, 'runcommand', runcommand)
     extensions.wrapfunction(repair, '_collectbrokencsets', _collectbrokencsets)
     extensions.wrapfunction(context.changectx, 'filectx', filectx)
     extensions.wrapfunction(context.workingctx, 'filectx', workingfilectx)
     extensions.wrapfunction(patch, 'trydiff', trydiff)
     extensions.wrapfunction(hg, 'verify', _verify)
     scmutil.fileprefetchhooks.add('remotefilelog', _fileprefetchhook)
 
     # disappointing hacks below
     scmutil.getrenamedfn = getrenamedfn
     extensions.wrapfunction(revset, 'filelog', filelogrevset)
     revset.symbols['filelog'] = revset.filelog
     extensions.wrapfunction(cmdutil, 'walkfilerevs', walkfilerevs)
 
 
 def cloneshallow(orig, ui, repo, *args, **opts):
     if opts.get(r'shallow'):
         repos = []
         def pull_shallow(orig, self, *args, **kwargs):
             if not isenabled(self):
                 repos.append(self.unfiltered())
                 # set up the client hooks so the post-clone update works
                 setupclient(self.ui, self.unfiltered())
 
                 # setupclient fixed the class on the repo itself
                 # but we also need to fix it on the repoview
                 if isinstance(self, repoview.repoview):
                     self.__class__.__bases__ = (self.__class__.__bases__[0],
                                                 self.unfiltered().__class__)
                 self.requirements.add(constants.SHALLOWREPO_REQUIREMENT)
                 self._writerequirements()
 
                 # Since setupclient hadn't been called, exchange.pull was not
                 # wrapped. So we need to manually invoke our version of it.
                 return exchangepull(orig, self, *args, **kwargs)
             else:
                 return orig(self, *args, **kwargs)
         extensions.wrapfunction(exchange, 'pull', pull_shallow)
 
         # Wrap the stream logic to add requirements and to pass include/exclude
         # patterns around.
         def setup_streamout(repo, remote):
             # Replace remote.stream_out with a version that sends file
             # patterns.
             def stream_out_shallow(orig):
                 caps = remote.capabilities()
                 if constants.NETWORK_CAP_LEGACY_SSH_GETFILES in caps:
                     opts = {}
                     if repo.includepattern:
                         opts[r'includepattern'] = '\0'.join(repo.includepattern)
                     if repo.excludepattern:
                         opts[r'excludepattern'] = '\0'.join(repo.excludepattern)
                     return remote._callstream('stream_out_shallow', **opts)
                 else:
                     return orig()
             extensions.wrapfunction(remote, 'stream_out', stream_out_shallow)
         def stream_wrap(orig, op):
             setup_streamout(op.repo, op.remote)
             return orig(op)
         extensions.wrapfunction(
             streamclone, 'maybeperformlegacystreamclone', stream_wrap)
 
         def canperformstreamclone(orig, pullop, bundle2=False):
             # remotefilelog is currently incompatible with the
             # bundle2 flavor of streamclones, so force us to use
             # v1 instead.
             if 'v2' in pullop.remotebundle2caps.get('stream', []):
                 pullop.remotebundle2caps['stream'] = [
                     c for c in pullop.remotebundle2caps['stream']
                     if c != 'v2']
             if bundle2:
                 return False, None
             supported, requirements = orig(pullop, bundle2=bundle2)
             if requirements is not None:
                 requirements.add(constants.SHALLOWREPO_REQUIREMENT)
             return supported, requirements
         extensions.wrapfunction(
             streamclone, 'canperformstreamclone', canperformstreamclone)
 
     try:
         orig(ui, repo, *args, **opts)
     finally:
         if opts.get(r'shallow'):
             for r in repos:
                 if util.safehasattr(r, 'fileservice'):
                     r.fileservice.close()
 
 def debugdatashallow(orig, *args, **kwds):
     oldlen = remotefilelog.remotefilelog.__len__
     try:
         remotefilelog.remotefilelog.__len__ = lambda x: 1
         return orig(*args, **kwds)
     finally:
         remotefilelog.remotefilelog.__len__ = oldlen
 
 def reposetup(ui, repo):
     if not repo.local():
         return
 
     # put here intentionally bc doesnt work in uisetup
     ui.setconfig('hooks', 'update.prefetch', wcpprefetch)
     ui.setconfig('hooks', 'commit.prefetch', wcpprefetch)
 
     isserverenabled = ui.configbool('remotefilelog', 'server')
     isshallowclient = isenabled(repo)
 
     if isserverenabled and isshallowclient:
         raise RuntimeError("Cannot be both a server and shallow client.")
 
     if isshallowclient:
         setupclient(ui, repo)
 
     if isserverenabled:
         remotefilelogserver.setupserver(ui, repo)
 
 def setupclient(ui, repo):
     if not isinstance(repo, localrepo.localrepository):
         return
 
     # Even clients get the server setup since they need to have the
     # wireprotocol endpoints registered.
     remotefilelogserver.onetimesetup(ui)
     onetimeclientsetup(ui)
 
     shallowrepo.wraprepo(repo)
     repo.store = shallowstore.wrapstore(repo.store)
 
 def storewrapper(orig, requirements, path, vfstype):
     s = orig(requirements, path, vfstype)
     if constants.SHALLOWREPO_REQUIREMENT in requirements:
         s = shallowstore.wrapstore(s)
 
     return s
 
 # prefetch files before update
-def applyupdates(orig, repo, actions, wctx, mctx, overwrite, labels=None):
+def applyupdates(orig, repo, actions, wctx, mctx, overwrite, wantfiledata,
+                 labels=None):
     if isenabled(repo):
         manifest = mctx.manifest()
         files = []
         for f, args, msg in actions['g']:
             files.append((f, hex(manifest[f])))
         # batch fetch the needed files from the server
         repo.fileservice.prefetch(files)
-    return orig(repo, actions, wctx, mctx, overwrite, labels=labels)
+    return orig(repo, actions, wctx, mctx, overwrite, wantfiledata,
+                labels=labels)
 
 # Prefetch merge checkunknownfiles
 def checkunknownfiles(orig, repo, wctx, mctx, force, actions,
     *args, **kwargs):
     if isenabled(repo):
         files = []
         sparsematch = repo.maybesparsematch(mctx.rev())
         for f, (m, actionargs, msg) in actions.iteritems():
             if sparsematch and not sparsematch(f):
                 continue
             if m in ('c', 'dc', 'cm'):
                 files.append((f, hex(mctx.filenode(f))))
             elif m == 'dg':
                 f2 = actionargs[0]
                 files.append((f2, hex(mctx.filenode(f2))))
         # batch fetch the needed files from the server
         repo.fileservice.prefetch(files)
     return orig(repo, wctx, mctx, force, actions, *args, **kwargs)
 
 # Prefetch files before status attempts to look at their size and contents
 def checklookup(orig, self, files):
     repo = self._repo
     if isenabled(repo):
         prefetchfiles = []
         for parent in self._parents:
             for f in files:
                 if f in parent:
                     prefetchfiles.append((f, hex(parent.filenode(f))))
         # batch fetch the needed files from the server
         repo.fileservice.prefetch(prefetchfiles)
     return orig(self, files)
 
 # Prefetch the logic that compares added and removed files for renames
 def findrenames(orig, repo, matcher, added, removed, *args, **kwargs):
     if isenabled(repo):
         files = []
         pmf = repo['.'].manifest()
         for f in removed:
             if f in pmf:
                 files.append((f, hex(pmf[f])))
         # batch fetch the needed files from the server
         repo.fileservice.prefetch(files)
     return orig(repo, matcher, added, removed, *args, **kwargs)
 
 # prefetch files before pathcopies check
 def computeforwardmissing(orig, a, b, match=None):
     missing = orig(a, b, match=match)
     repo = a._repo
     if isenabled(repo):
         mb = b.manifest()
 
         files = []
         sparsematch = repo.maybesparsematch(b.rev())
         if sparsematch:
             sparsemissing = set()
             for f in missing:
                 if sparsematch(f):
                     files.append((f, hex(mb[f])))
                     sparsemissing.add(f)
             missing = sparsemissing
 
         # batch fetch the needed files from the server
         repo.fileservice.prefetch(files)
     return missing
 
 # close cache miss server connection after the command has finished
 def runcommand(orig, lui, repo, *args, **kwargs):
     fileservice = None
     # repo can be None when running in chg:
     # - at startup, reposetup was called because serve is not norepo
     # - a norepo command like "help" is called
     if repo and isenabled(repo):
         fileservice = repo.fileservice
     try:
         return orig(lui, repo, *args, **kwargs)
     finally:
         if fileservice:
             fileservice.close()
 
 # prevent strip from stripping remotefilelogs
 def _collectbrokencsets(orig, repo, files, striprev):
     if isenabled(repo):
         files = list([f for f in files if not repo.shallowmatch(f)])
     return orig(repo, files, striprev)
 
 # changectx wrappers
 def filectx(orig, self, path, fileid=None, filelog=None):
     if fileid is None:
         fileid = self.filenode(path)
     if (isenabled(self._repo) and self._repo.shallowmatch(path)):
         return remotefilectx.remotefilectx(self._repo, path, fileid=fileid,
                                            changectx=self, filelog=filelog)
     return orig(self, path, fileid=fileid, filelog=filelog)
 
 def workingfilectx(orig, self, path, filelog=None):
     if (isenabled(self._repo) and self._repo.shallowmatch(path)):
         return remotefilectx.remoteworkingfilectx(self._repo, path,
                                                   workingctx=self,
                                                   filelog=filelog)
     return orig(self, path, filelog=filelog)
 
 # prefetch required revisions before a diff
 def trydiff(orig, repo, revs, ctx1, ctx2, modified, added, removed,
     copy, getfilectx, *args, **kwargs):
     if isenabled(repo):
         prefetch = []
         mf1 = ctx1.manifest()
         for fname in modified + added + removed:
             if fname in mf1:
                 fnode = getfilectx(fname, ctx1).filenode()
                 # fnode can be None if it's a edited working ctx file
                 if fnode:
                     prefetch.append((fname, hex(fnode)))
             if fname not in removed:
                 fnode = getfilectx(fname, ctx2).filenode()
                 if fnode:
                     prefetch.append((fname, hex(fnode)))
 
         repo.fileservice.prefetch(prefetch)
 
     return orig(repo, revs, ctx1, ctx2, modified, added, removed, copy,
                 getfilectx, *args, **kwargs)
 
 # Prevent verify from processing files
 # a stub for mercurial.hg.verify()
 def _verify(orig, repo, level=None):
     lock = repo.lock()
     try:
         return shallowverifier.shallowverifier(repo).verify()
     finally:
         lock.release()
 
 
 clientonetime = False
 def onetimeclientsetup(ui):
     global clientonetime
     if clientonetime:
         return
     clientonetime = True
 
     # Don't commit filelogs until we know the commit hash, since the hash
     # is present in the filelog blob.
     # This violates Mercurial's filelog->manifest->changelog write order,
     # but is generally fine for client repos.
     pendingfilecommits = []
     def addrawrevision(orig, self, rawtext, transaction, link, p1, p2, node,
                        flags, cachedelta=None, _metatuple=None):
         if isinstance(link, int):
             pendingfilecommits.append(
                 (self, rawtext, transaction, link, p1, p2, node, flags,
                  cachedelta, _metatuple))
             return node
         else:
             return orig(self, rawtext, transaction, link, p1, p2, node, flags,
                         cachedelta, _metatuple=_metatuple)
     extensions.wrapfunction(
         remotefilelog.remotefilelog, 'addrawrevision', addrawrevision)
 
     def changelogadd(orig, self, *args):
         oldlen = len(self)
         node = orig(self, *args)
         newlen = len(self)
         if oldlen != newlen:
             for oldargs in pendingfilecommits:
                 log, rt, tr, link, p1, p2, n, fl, c, m = oldargs
                 linknode = self.node(link)
                 if linknode == node:
                     log.addrawrevision(rt, tr, linknode, p1, p2, n, fl, c, m)
                 else:
                     raise error.ProgrammingError(
                         'pending multiple integer revisions are not supported')
         else:
             # "link" is actually wrong here (it is set to len(changelog))
             # if changelog remains unchanged, skip writing file revisions
             # but still do a sanity check about pending multiple revisions
             if len(set(x[3] for x in pendingfilecommits)) > 1:
                 raise error.ProgrammingError(
                     'pending multiple integer revisions are not supported')
         del pendingfilecommits[:]
         return node
     extensions.wrapfunction(changelog.changelog, 'add', changelogadd)
 
 def getrenamedfn(repo, endrev=None):
     rcache = {}
 
     def getrenamed(fn, rev):
         '''looks up all renames for a file (up to endrev) the first
         time the file is given. It indexes on the changerev and only
         parses the manifest if linkrev != changerev.
         Returns rename info for fn at changerev rev.'''
         if rev in rcache.setdefault(fn, {}):
             return rcache[fn][rev]
 
         try:
             fctx = repo[rev].filectx(fn)
             for ancestor in fctx.ancestors():
                 if ancestor.path() == fn:
                     renamed = ancestor.renamed()
                     rcache[fn][ancestor.rev()] = renamed and renamed[0]
 
             renamed = fctx.renamed()
             return renamed and renamed[0]
         except error.LookupError:
             return None
 
     return getrenamed
 
 def walkfilerevs(orig, repo, match, follow, revs, fncache):
     if not isenabled(repo):
         return orig(repo, match, follow, revs, fncache)
 
     # remotefilelog's can't be walked in rev order, so throw.
     # The caller will see the exception and walk the commit tree instead.
     if not follow:
         raise cmdutil.FileWalkError("Cannot walk via filelog")
 
     wanted = set()
     minrev, maxrev = min(revs), max(revs)
 
     pctx = repo['.']
     for filename in match.files():
         if filename not in pctx:
             raise error.Abort(_('cannot follow file not in parent '
                                'revision: "%s"') % filename)
         fctx = pctx[filename]
 
         linkrev = fctx.linkrev()
         if linkrev >= minrev and linkrev <= maxrev:
             fncache.setdefault(linkrev, []).append(filename)
             wanted.add(linkrev)
 
         for ancestor in fctx.ancestors():
             linkrev = ancestor.linkrev()
             if linkrev >= minrev and linkrev <= maxrev:
                 fncache.setdefault(linkrev, []).append(ancestor.path())
                 wanted.add(linkrev)
 
     return wanted
 
 def filelogrevset(orig, repo, subset, x):
     """``filelog(pattern)``
     Changesets connected to the specified filelog.
 
     For performance reasons, ``filelog()`` does not show every changeset
     that affects the requested file(s). See :hg:`help log` for details. For
     a slower, more accurate result, use ``file()``.
     """
 
     if not isenabled(repo):
         return orig(repo, subset, x)
 
     # i18n: "filelog" is a keyword
     pat = revset.getstring(x, _("filelog requires a pattern"))
     m = match.match(repo.root, repo.getcwd(), [pat], default='relpath',
                        ctx=repo[None])
     s = set()
 
     if not match.patkind(pat):
         # slow
         for r in subset:
             ctx = repo[r]
             cfiles = ctx.files()
             for f in m.files():
                 if f in cfiles:
                     s.add(ctx.rev())
                     break
     else:
         # partial
         files = (f for f in repo[None] if m(f))
         for f in files:
             fctx = repo[None].filectx(f)
             s.add(fctx.linkrev())
             for actx in fctx.ancestors():
                 s.add(actx.linkrev())
 
     return smartset.baseset([r for r in subset if r in s])
 
 @command('gc', [], _('hg gc [REPO...]'), norepo=True)
 def gc(ui, *args, **opts):
     '''garbage collect the client and server filelog caches
     '''
     cachepaths = set()
 
     # get the system client cache
     systemcache = shallowutil.getcachepath(ui, allowempty=True)
     if systemcache:
         cachepaths.add(systemcache)
 
     # get repo client and server cache
     repopaths = []
     pwd = ui.environ.get('PWD')
     if pwd:
         repopaths.append(pwd)
 
     repopaths.extend(args)
     repos = []
     for repopath in repopaths:
         try:
             repo = hg.peer(ui, {}, repopath)
             repos.append(repo)
 
             repocache = shallowutil.getcachepath(repo.ui, allowempty=True)
             if repocache:
                 cachepaths.add(repocache)
         except error.RepoError:
             pass
 
     # gc client cache
     for cachepath in cachepaths:
         gcclient(ui, cachepath)
 
     # gc server cache
     for repo in repos:
         remotefilelogserver.gcserver(ui, repo._repo)
 
 def gcclient(ui, cachepath):
     # get list of repos that use this cache
     repospath = os.path.join(cachepath, 'repos')
     if not os.path.exists(repospath):
         ui.warn(_("no known cache at %s\n") % cachepath)
         return
 
     reposfile = open(repospath, 'rb')
     repos = {r[:-1] for r in reposfile.readlines()}
     reposfile.close()
 
     # build list of useful files
     validrepos = []
     keepkeys = set()
 
     sharedcache = None
     filesrepacked = False
 
     count = 0
     progress = ui.makeprogress(_("analyzing repositories"), unit="repos",
                                total=len(repos))
     for path in repos:
         progress.update(count)
         count += 1
         try:
             path = ui.expandpath(os.path.normpath(path))
         except TypeError as e:
             ui.warn(_("warning: malformed path: %r:%s\n") % (path, e))
             traceback.print_exc()
             continue
         try:
             peer = hg.peer(ui, {}, path)
             repo = peer._repo
         except error.RepoError:
             continue
 
         validrepos.append(path)
 
         # Protect against any repo or config changes that have happened since
         # this repo was added to the repos file. We'd rather this loop succeed
         # and too much be deleted, than the loop fail and nothing gets deleted.
         if not isenabled(repo):
             continue
 
         if not util.safehasattr(repo, 'name'):
             ui.warn(_("repo %s is a misconfigured remotefilelog repo\n") % path)
             continue
 
         # If garbage collection on repack and repack on hg gc are enabled
         # then loose files are repacked and garbage collected.
         # Otherwise regular garbage collection is performed.
         repackonhggc = repo.ui.configbool('remotefilelog', 'repackonhggc')
         gcrepack = repo.ui.configbool('remotefilelog', 'gcrepack')
         if repackonhggc and gcrepack:
             try:
                 repackmod.incrementalrepack(repo)
                 filesrepacked = True
                 continue
             except (IOError, repackmod.RepackAlreadyRunning):
                 # If repack cannot be performed due to not enough disk space
                 # continue doing garbage collection of loose files w/o repack
                 pass
 
         reponame = repo.name
         if not sharedcache:
             sharedcache = repo.sharedstore
 
         # Compute a keepset which is not garbage collected
         def keyfn(fname, fnode):
             return fileserverclient.getcachekey(reponame, fname, hex(fnode))
         keepkeys = repackmod.keepset(repo, keyfn=keyfn, lastkeepkeys=keepkeys)
 
     progress.complete()
 
     # write list of valid repos back
     oldumask = os.umask(0o002)
     try:
         reposfile = open(repospath, 'wb')
         reposfile.writelines([("%s\n" % r) for r in validrepos])
         reposfile.close()
     finally:
         os.umask(oldumask)
 
     # prune cache
     if sharedcache is not None:
         sharedcache.gc(keepkeys)
     elif not filesrepacked:
         ui.warn(_("warning: no valid repos in repofile\n"))
 
 def log(orig, ui, repo, *pats, **opts):
     if not isenabled(repo):
         return orig(ui, repo, *pats, **opts)
 
     follow = opts.get(r'follow')
     revs = opts.get(r'rev')
     if pats:
         # Force slowpath for non-follow patterns and follows that start from
         # non-working-copy-parent revs.
         if not follow or revs:
             # This forces the slowpath
             opts[r'removed'] = True
 
         # If this is a non-follow log without any revs specified, recommend that
         # the user add -f to speed it up.
         if not follow and not revs:
             match = scmutil.match(repo['.'], pats, pycompat.byteskwargs(opts))
             isfile = not match.anypats()
             if isfile:
                 for file in match.files():
                     if not os.path.isfile(repo.wjoin(file)):
                         isfile = False
                         break
 
             if isfile:
                 ui.warn(_("warning: file log can be slow on large repos - " +
                           "use -f to speed it up\n"))
 
     return orig(ui, repo, *pats, **opts)
 
 def revdatelimit(ui, revset):
     """Update revset so that only changesets no older than 'prefetchdays' days
     are included. The default value is set to 14 days. If 'prefetchdays' is set
     to zero or negative value then date restriction is not applied.
     """
     days = ui.configint('remotefilelog', 'prefetchdays')
     if days > 0:
         revset = '(%s) & date(-%s)' % (revset, days)
     return revset
 
 def readytofetch(repo):
     """Check that enough time has passed since the last background prefetch.
     This only relates to prefetches after operations that change the working
     copy parent. Default delay between background prefetches is 2 minutes.
     """
     timeout = repo.ui.configint('remotefilelog', 'prefetchdelay')
     fname = repo.vfs.join('lastprefetch')
 
     ready = False
     with open(fname, 'a'):
         # the with construct above is used to avoid race conditions
         modtime = os.path.getmtime(fname)
         if (time.time() - modtime) > timeout:
             os.utime(fname, None)
             ready = True
 
     return ready
 
 def wcpprefetch(ui, repo, **kwargs):
     """Prefetches in background revisions specified by bgprefetchrevs revset.
     Does background repack if backgroundrepack flag is set in config.
     """
     shallow = isenabled(repo)
     bgprefetchrevs = ui.config('remotefilelog', 'bgprefetchrevs')
     isready = readytofetch(repo)
 
     if not (shallow and bgprefetchrevs and isready):
         return
 
     bgrepack = repo.ui.configbool('remotefilelog', 'backgroundrepack')
     # update a revset with a date limit
     bgprefetchrevs = revdatelimit(ui, bgprefetchrevs)
 
     def anon():
         if util.safehasattr(repo, 'ranprefetch') and repo.ranprefetch:
             return
         repo.ranprefetch = True
         repo.backgroundprefetch(bgprefetchrevs, repack=bgrepack)
 
     repo._afterlock(anon)
 
 def pull(orig, ui, repo, *pats, **opts):
     result = orig(ui, repo, *pats, **opts)
 
     if isenabled(repo):
         # prefetch if it's configured
         prefetchrevset = ui.config('remotefilelog', 'pullprefetch')
         bgrepack = repo.ui.configbool('remotefilelog', 'backgroundrepack')
         bgprefetch = repo.ui.configbool('remotefilelog', 'backgroundprefetch')
 
         if prefetchrevset:
             ui.status(_("prefetching file contents\n"))
             revs = scmutil.revrange(repo, [prefetchrevset])
             base = repo['.'].rev()
             if bgprefetch:
                 repo.backgroundprefetch(prefetchrevset, repack=bgrepack)
             else:
                 repo.prefetch(revs, base=base)
                 if bgrepack:
                     repackmod.backgroundrepack(repo, incremental=True)
         elif bgrepack:
             repackmod.backgroundrepack(repo, incremental=True)
 
     return result
 
 def exchangepull(orig, repo, remote, *args, **kwargs):
     # Hook into the callstream/getbundle to insert bundle capabilities
     # during a pull.
     def localgetbundle(orig, source, heads=None, common=None, bundlecaps=None,
                        **kwargs):
         if not bundlecaps:
             bundlecaps = set()
         bundlecaps.add(constants.BUNDLE2_CAPABLITY)
         return orig(source, heads=heads, common=common, bundlecaps=bundlecaps,
                     **kwargs)
 
     if util.safehasattr(remote, '_callstream'):
         remote._localrepo = repo
     elif util.safehasattr(remote, 'getbundle'):
         extensions.wrapfunction(remote, 'getbundle', localgetbundle)
 
     return orig(repo, remote, *args, **kwargs)
 
 def _fileprefetchhook(repo, revs, match):
     if isenabled(repo):
         allfiles = []
         for rev in revs:
             if rev == nodemod.wdirrev or rev is None:
                 continue
             ctx = repo[rev]
             mf = ctx.manifest()
             sparsematch = repo.maybesparsematch(ctx.rev())
             for path in ctx.walk(match):
                 if path.endswith('/'):
                     # Tree manifest that's being excluded as part of narrow
                     continue
                 if (not sparsematch or sparsematch(path)) and path in mf:
                     allfiles.append((path, hex(mf[path])))
         repo.fileservice.prefetch(allfiles)
 
 @command('debugremotefilelog', [
     ('d', 'decompress', None, _('decompress the filelog first')),
     ], _('hg debugremotefilelog <path>'), norepo=True)
 def debugremotefilelog(ui, path, **opts):
     return debugcommands.debugremotefilelog(ui, path, **opts)
 
 @command('verifyremotefilelog', [
     ('d', 'decompress', None, _('decompress the filelogs first')),
     ], _('hg verifyremotefilelogs <directory>'), norepo=True)
 def verifyremotefilelog(ui, path, **opts):
     return debugcommands.verifyremotefilelog(ui, path, **opts)
 
 @command('debugdatapack', [
     ('', 'long', None, _('print the long hashes')),
     ('', 'node', '', _('dump the contents of node'), 'NODE'),
     ], _('hg debugdatapack <paths>'), norepo=True)
 def debugdatapack(ui, *paths, **opts):
     return debugcommands.debugdatapack(ui, *paths, **opts)
 
 @command('debughistorypack', [
     ], _('hg debughistorypack <path>'), norepo=True)
 def debughistorypack(ui, path, **opts):
     return debugcommands.debughistorypack(ui, path)
 
 @command('debugkeepset', [
     ], _('hg debugkeepset'))
 def debugkeepset(ui, repo, **opts):
     # The command is used to measure keepset computation time
     def keyfn(fname, fnode):
         return fileserverclient.getcachekey(repo.name, fname, hex(fnode))
     repackmod.keepset(repo, keyfn)
     return
 
 @command('debugwaitonrepack', [
     ], _('hg debugwaitonrepack'))
 def debugwaitonrepack(ui, repo, **opts):
     return debugcommands.debugwaitonrepack(repo)
 
 @command('debugwaitonprefetch', [
     ], _('hg debugwaitonprefetch'))
 def debugwaitonprefetch(ui, repo, **opts):
     return debugcommands.debugwaitonprefetch(repo)
 
 def resolveprefetchopts(ui, opts):
     if not opts.get('rev'):
         revset = ['.', 'draft()']
 
         prefetchrevset = ui.config('remotefilelog', 'pullprefetch', None)
         if prefetchrevset:
             revset.append('(%s)' % prefetchrevset)
         bgprefetchrevs = ui.config('remotefilelog', 'bgprefetchrevs', None)
         if bgprefetchrevs:
             revset.append('(%s)' % bgprefetchrevs)
         revset = '+'.join(revset)
 
         # update a revset with a date limit
         revset = revdatelimit(ui, revset)
 
         opts['rev'] = [revset]
 
     if not opts.get('base'):
         opts['base'] = None
 
     return opts
 
 @command('prefetch', [
     ('r', 'rev', [], _('prefetch the specified revisions'), _('REV')),
     ('', 'repack', False, _('run repack after prefetch')),
     ('b', 'base', '', _("rev that is assumed to already be local")),
     ] + commands.walkopts, _('hg prefetch [OPTIONS] [FILE...]'))
 def prefetch(ui, repo, *pats, **opts):
     """prefetch file revisions from the server
 
     Prefetchs file revisions for the specified revs and stores them in the
     local remotefilelog cache.  If no rev is specified, the default rev is
     used which is the union of dot, draft, pullprefetch and bgprefetchrev.
     File names or patterns can be used to limit which files are downloaded.
 
     Return 0 on success.
     """
     opts = pycompat.byteskwargs(opts)
     if not isenabled(repo):
         raise error.Abort(_("repo is not shallow"))
 
     opts = resolveprefetchopts(ui, opts)
     revs = scmutil.revrange(repo, opts.get('rev'))
     repo.prefetch(revs, opts.get('base'), pats, opts)
 
     # Run repack in background
     if opts.get('repack'):
         repackmod.backgroundrepack(repo, incremental=True)
 
 @command('repack', [
      ('', 'background', None, _('run in a background process'), None),
      ('', 'incremental', None, _('do an incremental repack'), None),
      ('', 'packsonly', None, _('only repack packs (skip loose objects)'), None),
     ], _('hg repack [OPTIONS]'))
 def repack_(ui, repo, *pats, **opts):
     if opts.get(r'background'):
         repackmod.backgroundrepack(repo, incremental=opts.get(r'incremental'),
                                    packsonly=opts.get(r'packsonly', False))
         return
 
     options = {'packsonly': opts.get(r'packsonly')}
 
     try:
         if opts.get(r'incremental'):
             repackmod.incrementalrepack(repo, options=options)
         else:
             repackmod.fullrepack(repo, options=options)
     except repackmod.RepackAlreadyRunning as ex:
         # Don't propogate the exception if the repack is already in
         # progress, since we want the command to exit 0.
         repo.ui.warn('%s\n' % ex)

hgext/sparse.py

@@ -1,348 +1,348
 # sparse.py - allow sparse checkouts of the working directory
 #
 # Copyright 2014 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """allow sparse checkouts of the working directory (EXPERIMENTAL)
 
 (This extension is not yet protected by backwards compatibility
 guarantees. Any aspect may break in future releases until this
 notice is removed.)
 
 This extension allows the working directory to only consist of a
 subset of files for the revision. This allows specific files or
 directories to be explicitly included or excluded. Many repository
 operations have performance proportional to the number of files in
 the working directory. So only realizing a subset of files in the
 working directory can improve performance.
 
 Sparse Config Files
 -------------------
 
 The set of files that are part of a sparse checkout are defined by
 a sparse config file. The file defines 3 things: includes (files to
 include in the sparse checkout), excludes (files to exclude from the
 sparse checkout), and profiles (links to other config files).
 
 The file format is newline delimited. Empty lines and lines beginning
 with ``#`` are ignored.
 
 Lines beginning with ``%include `` denote another sparse config file
 to include. e.g. ``%include tests.sparse``. The filename is relative
 to the repository root.
 
 The special lines ``[include]`` and ``[exclude]`` denote the section
 for includes and excludes that follow, respectively. It is illegal to
 have ``[include]`` after ``[exclude]``.
 
 Non-special lines resemble file patterns to be added to either includes
 or excludes. The syntax of these lines is documented by :hg:`help patterns`.
 Patterns are interpreted as ``glob:`` by default and match against the
 root of the repository.
 
 Exclusion patterns take precedence over inclusion patterns. So even
 if a file is explicitly included, an ``[exclude]`` entry can remove it.
 
 For example, say you have a repository with 3 directories, ``frontend/``,
 ``backend/``, and ``tools/``. ``frontend/`` and ``backend/`` correspond
 to different projects and it is uncommon for someone working on one
 to need the files for the other. But ``tools/`` contains files shared
 between both projects. Your sparse config files may resemble::
 
   # frontend.sparse
   frontend/**
   tools/**
 
   # backend.sparse
   backend/**
   tools/**
 
 Say the backend grows in size. Or there's a directory with thousands
 of files you wish to exclude. You can modify the profile to exclude
 certain files::
 
   [include]
   backend/**
   tools/**
 
   [exclude]
   tools/tests/**
 """
 
 from __future__ import absolute_import
 
 from mercurial.i18n import _
 from mercurial import (
     commands,
     dirstate,
     error,
     extensions,
     hg,
     logcmdutil,
     match as matchmod,
     pycompat,
     registrar,
     sparse,
     util,
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 def extsetup(ui):
     sparse.enabled = True
 
     _setupclone(ui)
     _setuplog(ui)
     _setupadd(ui)
     _setupdirstate(ui)
 
 def replacefilecache(cls, propname, replacement):
     """Replace a filecache property with a new class. This allows changing the
     cache invalidation condition."""
     origcls = cls
     assert callable(replacement)
     while cls is not object:
         if propname in cls.__dict__:
             orig = cls.__dict__[propname]
             setattr(cls, propname, replacement(orig))
             break
         cls = cls.__bases__[0]
 
     if cls is object:
         raise AttributeError(_("type '%s' has no property '%s'") % (origcls,
                              propname))
 
 def _setuplog(ui):
     entry = commands.table['log|history']
     entry[1].append(('', 'sparse', None,
         "limit to changesets affecting the sparse checkout"))
 
     def _initialrevs(orig, repo, opts):
         revs = orig(repo, opts)
         if opts.get('sparse'):
             sparsematch = sparse.matcher(repo)
             def ctxmatch(rev):
                 ctx = repo[rev]
                 return any(f for f in ctx.files() if sparsematch(f))
             revs = revs.filter(ctxmatch)
         return revs
     extensions.wrapfunction(logcmdutil, '_initialrevs', _initialrevs)
 
 def _clonesparsecmd(orig, ui, repo, *args, **opts):
     include_pat = opts.get(r'include')
     exclude_pat = opts.get(r'exclude')
     enableprofile_pat = opts.get(r'enable_profile')
     narrow_pat = opts.get(r'narrow')
     include = exclude = enableprofile = False
     if include_pat:
         pat = include_pat
         include = True
     if exclude_pat:
         pat = exclude_pat
         exclude = True
     if enableprofile_pat:
         pat = enableprofile_pat
         enableprofile = True
     if sum([include, exclude, enableprofile]) > 1:
         raise error.Abort(_("too many flags specified."))
     # if --narrow is passed, it means they are includes and excludes for narrow
     # clone
     if not narrow_pat and (include or exclude or enableprofile):
         def clonesparse(orig, self, node, overwrite, *args, **kwargs):
             sparse.updateconfig(self.unfiltered(), pat, {}, include=include,
                                 exclude=exclude, enableprofile=enableprofile,
                                 usereporootpaths=True)
             return orig(self, node, overwrite, *args, **kwargs)
         extensions.wrapfunction(hg, 'updaterepo', clonesparse)
     return orig(ui, repo, *args, **opts)
 
 def _setupclone(ui):
     entry = commands.table['clone']
     entry[1].append(('', 'enable-profile', [],
                     'enable a sparse profile'))
     entry[1].append(('', 'include', [],
                     'include sparse pattern'))
     entry[1].append(('', 'exclude', [],
                     'exclude sparse pattern'))
     extensions.wrapcommand(commands.table, 'clone', _clonesparsecmd)
 
 def _setupadd(ui):
     entry = commands.table['add']
     entry[1].append(('s', 'sparse', None,
                     'also include directories of added files in sparse config'))
 
     def _add(orig, ui, repo, *pats, **opts):
         if opts.get(r'sparse'):
             dirs = set()
             for pat in pats:
                 dirname, basename = util.split(pat)
                 dirs.add(dirname)
             sparse.updateconfig(repo, list(dirs), opts, include=True)
         return orig(ui, repo, *pats, **opts)
 
     extensions.wrapcommand(commands.table, 'add', _add)
 
 def _setupdirstate(ui):
     """Modify the dirstate to prevent stat'ing excluded files,
     and to prevent modifications to files outside the checkout.
     """
 
     def walk(orig, self, match, subrepos, unknown, ignored, full=True):
         # hack to not exclude explicitly-specified paths so that they can
         # be warned later on e.g. dirstate.add()
         em = matchmod.exact(match.files())
         sm = matchmod.unionmatcher([self._sparsematcher, em])
         match = matchmod.intersectmatchers(match, sm)
         return orig(self, match, subrepos, unknown, ignored, full)
 
     extensions.wrapfunction(dirstate.dirstate, 'walk', walk)
 
     # dirstate.rebuild should not add non-matching files
     def _rebuild(orig, self, parent, allfiles, changedfiles=None):
         matcher = self._sparsematcher
         if not matcher.always():
             allfiles = [f for f in allfiles if matcher(f)]
             if changedfiles:
                 changedfiles = [f for f in changedfiles if matcher(f)]
 
             if changedfiles is not None:
                 # In _rebuild, these files will be deleted from the dirstate
                 # when they are not found to be in allfiles
                 dirstatefilestoremove = set(f for f in self if not matcher(f))
                 changedfiles = dirstatefilestoremove.union(changedfiles)
 
         return orig(self, parent, allfiles, changedfiles)
     extensions.wrapfunction(dirstate.dirstate, 'rebuild', _rebuild)
 
     # Prevent adding files that are outside the sparse checkout
     editfuncs = ['normal', 'add', 'normallookup', 'copy', 'remove', 'merge']
     hint = _('include file with `hg debugsparse --include <pattern>` or use ' +
              '`hg add -s <file>` to include file directory while adding')
     for func in editfuncs:
-        def _wrapper(orig, self, *args):
+        def _wrapper(orig, self, *args, **kwargs):
             sparsematch = self._sparsematcher
             if not sparsematch.always():
                 for f in args:
                     if (f is not None and not sparsematch(f) and
                         f not in self):
                         raise error.Abort(_("cannot add '%s' - it is outside "
                                             "the sparse checkout") % f,
                                           hint=hint)
-            return orig(self, *args)
+            return orig(self, *args, **kwargs)
         extensions.wrapfunction(dirstate.dirstate, func, _wrapper)
 
 @command('debugsparse', [
     ('I', 'include', False, _('include files in the sparse checkout')),
     ('X', 'exclude', False, _('exclude files in the sparse checkout')),
     ('d', 'delete', False, _('delete an include/exclude rule')),
     ('f', 'force', False, _('allow changing rules even with pending changes')),
     ('', 'enable-profile', False, _('enables the specified profile')),
     ('', 'disable-profile', False, _('disables the specified profile')),
     ('', 'import-rules', False, _('imports rules from a file')),
     ('', 'clear-rules', False, _('clears local include/exclude rules')),
     ('', 'refresh', False, _('updates the working after sparseness changes')),
     ('', 'reset', False, _('makes the repo full again')),
     ] + commands.templateopts,
     _('[--OPTION] PATTERN...'),
     helpbasic=True)
 def debugsparse(ui, repo, *pats, **opts):
     """make the current checkout sparse, or edit the existing checkout
 
     The sparse command is used to make the current checkout sparse.
     This means files that don't meet the sparse condition will not be
     written to disk, or show up in any working copy operations. It does
     not affect files in history in any way.
 
     Passing no arguments prints the currently applied sparse rules.
 
     --include and --exclude are used to add and remove files from the sparse
     checkout. The effects of adding an include or exclude rule are applied
     immediately. If applying the new rule would cause a file with pending
     changes to be added or removed, the command will fail. Pass --force to
     force a rule change even with pending changes (the changes on disk will
     be preserved).
 
     --delete removes an existing include/exclude rule. The effects are
     immediate.
 
     --refresh refreshes the files on disk based on the sparse rules. This is
     only necessary if .hg/sparse was changed by hand.
 
     --enable-profile and --disable-profile accept a path to a .hgsparse file.
     This allows defining sparse checkouts and tracking them inside the
     repository. This is useful for defining commonly used sparse checkouts for
     many people to use. As the profile definition changes over time, the sparse
     checkout will automatically be updated appropriately, depending on which
     changeset is checked out. Changes to .hgsparse are not applied until they
     have been committed.
 
     --import-rules accepts a path to a file containing rules in the .hgsparse
     format, allowing you to add --include, --exclude and --enable-profile rules
     in bulk. Like the --include, --exclude and --enable-profile switches, the
     changes are applied immediately.
 
     --clear-rules removes all local include and exclude rules, while leaving
     any enabled profiles in place.
 
     Returns 0 if editing the sparse checkout succeeds.
     """
     opts = pycompat.byteskwargs(opts)
     include = opts.get('include')
     exclude = opts.get('exclude')
     force = opts.get('force')
     enableprofile = opts.get('enable_profile')
     disableprofile = opts.get('disable_profile')
     importrules = opts.get('import_rules')
     clearrules = opts.get('clear_rules')
     delete = opts.get('delete')
     refresh = opts.get('refresh')
     reset = opts.get('reset')
     count = sum([include, exclude, enableprofile, disableprofile, delete,
                  importrules, refresh, clearrules, reset])
     if count > 1:
         raise error.Abort(_("too many flags specified"))
 
     if count == 0:
         if repo.vfs.exists('sparse'):
             ui.status(repo.vfs.read("sparse") + "\n")
             temporaryincludes = sparse.readtemporaryincludes(repo)
             if temporaryincludes:
                 ui.status(_("Temporarily Included Files (for merge/rebase):\n"))
                 ui.status(("\n".join(temporaryincludes) + "\n"))
             return
         else:
             raise error.Abort(_('the debugsparse command is only supported on'
                                 ' sparse repositories'))
 
     if include or exclude or delete or reset or enableprofile or disableprofile:
         sparse.updateconfig(repo, pats, opts, include=include, exclude=exclude,
                             reset=reset, delete=delete,
                             enableprofile=enableprofile,
                             disableprofile=disableprofile, force=force)
 
     if importrules:
         sparse.importfromfiles(repo, opts, pats, force=force)
 
     if clearrules:
         sparse.clearrules(repo, force=force)
 
     if refresh:
         try:
             wlock = repo.wlock()
             fcounts = map(
                 len,
                 sparse.refreshwdir(repo, repo.status(), sparse.matcher(repo),
                                    force=force))
             sparse.printchanges(ui, opts, added=fcounts[0], dropped=fcounts[1],
                                 conflicting=fcounts[2])
         finally:
             wlock.release()

mercurial/context.py

@@ -1,2604 +1,2606
 # context.py - changeset and file context objects for mercurial
 #
 # Copyright 2006, 2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import errno
 import filecmp
 import os
 import stat
 
 from .i18n import _
 from .node import (
     addednodeid,
     hex,
     modifiednodeid,
     nullid,
     nullrev,
     short,
     wdirfilenodeids,
     wdirhex,
 )
 from . import (
     dagop,
     encoding,
     error,
     fileset,
     match as matchmod,
     obsolete as obsmod,
     patch,
     pathutil,
     phases,
     pycompat,
     repoview,
     scmutil,
     sparse,
     subrepo,
     subrepoutil,
     util,
 )
 from .utils import (
     dateutil,
     stringutil,
 )
 
 propertycache = util.propertycache
 
 class basectx(object):
     """A basectx object represents the common logic for its children:
     changectx: read-only context that is already present in the repo,
     workingctx: a context that represents the working directory and can
                 be committed,
     memctx: a context that represents changes in-memory and can also
             be committed."""
 
     def __init__(self, repo):
         self._repo = repo
 
     def __bytes__(self):
         return short(self.node())
 
     __str__ = encoding.strmethod(__bytes__)
 
     def __repr__(self):
         return r"<%s %s>" % (type(self).__name__, str(self))
 
     def __eq__(self, other):
         try:
             return type(self) == type(other) and self._rev == other._rev
         except AttributeError:
             return False
 
     def __ne__(self, other):
         return not (self == other)
 
     def __contains__(self, key):
         return key in self._manifest
 
     def __getitem__(self, key):
         return self.filectx(key)
 
     def __iter__(self):
         return iter(self._manifest)
 
     def _buildstatusmanifest(self, status):
         """Builds a manifest that includes the given status results, if this is
         a working copy context. For non-working copy contexts, it just returns
         the normal manifest."""
         return self.manifest()
 
     def _matchstatus(self, other, match):
         """This internal method provides a way for child objects to override the
         match operator.
         """
         return match
 
     def _buildstatus(self, other, s, match, listignored, listclean,
                      listunknown):
         """build a status with respect to another context"""
         # Load earliest manifest first for caching reasons. More specifically,
         # if you have revisions 1000 and 1001, 1001 is probably stored as a
         # delta against 1000. Thus, if you read 1000 first, we'll reconstruct
         # 1000 and cache it so that when you read 1001, we just need to apply a
         # delta to what's in the cache. So that's one full reconstruction + one
         # delta application.
         mf2 = None
         if self.rev() is not None and self.rev() < other.rev():
             mf2 = self._buildstatusmanifest(s)
         mf1 = other._buildstatusmanifest(s)
         if mf2 is None:
             mf2 = self._buildstatusmanifest(s)
 
         modified, added = [], []
         removed = []
         clean = []
         deleted, unknown, ignored = s.deleted, s.unknown, s.ignored
         deletedset = set(deleted)
         d = mf1.diff(mf2, match=match, clean=listclean)
         for fn, value in d.iteritems():
             if fn in deletedset:
                 continue
             if value is None:
                 clean.append(fn)
                 continue
             (node1, flag1), (node2, flag2) = value
             if node1 is None:
                 added.append(fn)
             elif node2 is None:
                 removed.append(fn)
             elif flag1 != flag2:
                 modified.append(fn)
             elif node2 not in wdirfilenodeids:
                 # When comparing files between two commits, we save time by
                 # not comparing the file contents when the nodeids differ.
                 # Note that this means we incorrectly report a reverted change
                 # to a file as a modification.
                 modified.append(fn)
             elif self[fn].cmp(other[fn]):
                 modified.append(fn)
             else:
                 clean.append(fn)
 
         if removed:
             # need to filter files if they are already reported as removed
             unknown = [fn for fn in unknown if fn not in mf1 and
                                                (not match or match(fn))]
             ignored = [fn for fn in ignored if fn not in mf1 and
                                                (not match or match(fn))]
             # if they're deleted, don't report them as removed
             removed = [fn for fn in removed if fn not in deletedset]
 
         return scmutil.status(modified, added, removed, deleted, unknown,
                               ignored, clean)
 
     @propertycache
     def substate(self):
         return subrepoutil.state(self, self._repo.ui)
 
     def subrev(self, subpath):
         return self.substate[subpath][1]
 
     def rev(self):
         return self._rev
     def node(self):
         return self._node
     def hex(self):
         return hex(self.node())
     def manifest(self):
         return self._manifest
     def manifestctx(self):
         return self._manifestctx
     def repo(self):
         return self._repo
     def phasestr(self):
         return phases.phasenames[self.phase()]
     def mutable(self):
         return self.phase() > phases.public
 
     def matchfileset(self, expr, badfn=None):
         return fileset.match(self, expr, badfn=badfn)
 
     def obsolete(self):
         """True if the changeset is obsolete"""
         return self.rev() in obsmod.getrevs(self._repo, 'obsolete')
 
     def extinct(self):
         """True if the changeset is extinct"""
         return self.rev() in obsmod.getrevs(self._repo, 'extinct')
 
     def orphan(self):
         """True if the changeset is not obsolete, but its ancestor is"""
         return self.rev() in obsmod.getrevs(self._repo, 'orphan')
 
     def phasedivergent(self):
         """True if the changeset tries to be a successor of a public changeset
 
         Only non-public and non-obsolete changesets may be phase-divergent.
         """
         return self.rev() in obsmod.getrevs(self._repo, 'phasedivergent')
 
     def contentdivergent(self):
         """Is a successor of a changeset with multiple possible successor sets
 
         Only non-public and non-obsolete changesets may be content-divergent.
         """
         return self.rev() in obsmod.getrevs(self._repo, 'contentdivergent')
 
     def isunstable(self):
         """True if the changeset is either orphan, phase-divergent or
         content-divergent"""
         return self.orphan() or self.phasedivergent() or self.contentdivergent()
 
     def instabilities(self):
         """return the list of instabilities affecting this changeset.
 
         Instabilities are returned as strings. possible values are:
         - orphan,
         - phase-divergent,
         - content-divergent.
         """
         instabilities = []
         if self.orphan():
             instabilities.append('orphan')
         if self.phasedivergent():
             instabilities.append('phase-divergent')
         if self.contentdivergent():
             instabilities.append('content-divergent')
         return instabilities
 
     def parents(self):
         """return contexts for each parent changeset"""
         return self._parents
 
     def p1(self):
         return self._parents[0]
 
     def p2(self):
         parents = self._parents
         if len(parents) == 2:
             return parents[1]
         return self._repo[nullrev]
 
     def _fileinfo(self, path):
         if r'_manifest' in self.__dict__:
             try:
                 return self._manifest[path], self._manifest.flags(path)
             except KeyError:
                 raise error.ManifestLookupError(self._node, path,
                                                 _('not found in manifest'))
         if r'_manifestdelta' in self.__dict__ or path in self.files():
             if path in self._manifestdelta:
                 return (self._manifestdelta[path],
                         self._manifestdelta.flags(path))
         mfl = self._repo.manifestlog
         try:
             node, flag = mfl[self._changeset.manifest].find(path)
         except KeyError:
             raise error.ManifestLookupError(self._node, path,
                                             _('not found in manifest'))
 
         return node, flag
 
     def filenode(self, path):
         return self._fileinfo(path)[0]
 
     def flags(self, path):
         try:
             return self._fileinfo(path)[1]
         except error.LookupError:
             return ''
 
     @propertycache
     def _copies(self):
         p1copies = {}
         p2copies = {}
         p1 = self.p1()
         p2 = self.p2()
         narrowmatch = self._repo.narrowmatch()
         for dst in self.files():
             if not narrowmatch(dst) or dst not in self:
                 continue
             copied = self[dst].renamed()
             if not copied:
                 continue
             src, srcnode = copied
             if src in p1 and p1[src].filenode() == srcnode:
                 p1copies[dst] = src
             elif src in p2 and p2[src].filenode() == srcnode:
                 p2copies[dst] = src
         return p1copies, p2copies
     def p1copies(self):
         return self._copies[0]
     def p2copies(self):
         return self._copies[1]
 
     def sub(self, path, allowcreate=True):
         '''return a subrepo for the stored revision of path, never wdir()'''
         return subrepo.subrepo(self, path, allowcreate=allowcreate)
 
     def nullsub(self, path, pctx):
         return subrepo.nullsubrepo(self, path, pctx)
 
     def workingsub(self, path):
         '''return a subrepo for the stored revision, or wdir if this is a wdir
         context.
         '''
         return subrepo.subrepo(self, path, allowwdir=True)
 
     def match(self, pats=None, include=None, exclude=None, default='glob',
               listsubrepos=False, badfn=None):
         r = self._repo
         return matchmod.match(r.root, r.getcwd(), pats,
                               include, exclude, default,
                               auditor=r.nofsauditor, ctx=self,
                               listsubrepos=listsubrepos, badfn=badfn)
 
     def diff(self, ctx2=None, match=None, changes=None, opts=None,
              losedatafn=None, pathfn=None, copy=None,
              copysourcematch=None, hunksfilterfn=None):
         """Returns a diff generator for the given contexts and matcher"""
         if ctx2 is None:
             ctx2 = self.p1()
         if ctx2 is not None:
             ctx2 = self._repo[ctx2]
         return patch.diff(self._repo, ctx2, self, match=match, changes=changes,
                           opts=opts, losedatafn=losedatafn, pathfn=pathfn,
                           copy=copy, copysourcematch=copysourcematch,
                           hunksfilterfn=hunksfilterfn)
 
     def dirs(self):
         return self._manifest.dirs()
 
     def hasdir(self, dir):
         return self._manifest.hasdir(dir)
 
     def status(self, other=None, match=None, listignored=False,
                listclean=False, listunknown=False, listsubrepos=False):
         """return status of files between two nodes or node and working
         directory.
 
         If other is None, compare this node with working directory.
 
         returns (modified, added, removed, deleted, unknown, ignored, clean)
         """
 
         ctx1 = self
         ctx2 = self._repo[other]
 
         # This next code block is, admittedly, fragile logic that tests for
         # reversing the contexts and wouldn't need to exist if it weren't for
         # the fast (and common) code path of comparing the working directory
         # with its first parent.
         #
         # What we're aiming for here is the ability to call:
         #
         # workingctx.status(parentctx)
         #
         # If we always built the manifest for each context and compared those,
         # then we'd be done. But the special case of the above call means we
         # just copy the manifest of the parent.
         reversed = False
         if (not isinstance(ctx1, changectx)
             and isinstance(ctx2, changectx)):
             reversed = True
             ctx1, ctx2 = ctx2, ctx1
 
         match = self._repo.narrowmatch(match)
         match = ctx2._matchstatus(ctx1, match)
         r = scmutil.status([], [], [], [], [], [], [])
         r = ctx2._buildstatus(ctx1, r, match, listignored, listclean,
                               listunknown)
 
         if reversed:
             # Reverse added and removed. Clear deleted, unknown and ignored as
             # these make no sense to reverse.
             r = scmutil.status(r.modified, r.removed, r.added, [], [], [],
                                r.clean)
 
         if listsubrepos:
             for subpath, sub in scmutil.itersubrepos(ctx1, ctx2):
                 try:
                     rev2 = ctx2.subrev(subpath)
                 except KeyError:
                     # A subrepo that existed in node1 was deleted between
                     # node1 and node2 (inclusive). Thus, ctx2's substate
                     # won't contain that subpath. The best we can do ignore it.
                     rev2 = None
                 submatch = matchmod.subdirmatcher(subpath, match)
                 s = sub.status(rev2, match=submatch, ignored=listignored,
                                clean=listclean, unknown=listunknown,
                                listsubrepos=True)
                 for rfiles, sfiles in zip(r, s):
                     rfiles.extend("%s/%s" % (subpath, f) for f in sfiles)
 
         for l in r:
             l.sort()
 
         return r
 
 class changectx(basectx):
     """A changecontext object makes access to data related to a particular
     changeset convenient. It represents a read-only context already present in
     the repo."""
     def __init__(self, repo, rev, node):
         super(changectx, self).__init__(repo)
         self._rev = rev
         self._node = node
 
     def __hash__(self):
         try:
             return hash(self._rev)
         except AttributeError:
             return id(self)
 
     def __nonzero__(self):
         return self._rev != nullrev
 
     __bool__ = __nonzero__
 
     @propertycache
     def _changeset(self):
         return self._repo.changelog.changelogrevision(self.rev())
 
     @propertycache
     def _manifest(self):
         return self._manifestctx.read()
 
     @property
     def _manifestctx(self):
         return self._repo.manifestlog[self._changeset.manifest]
 
     @propertycache
     def _manifestdelta(self):
         return self._manifestctx.readdelta()
 
     @propertycache
     def _parents(self):
         repo = self._repo
         p1, p2 = repo.changelog.parentrevs(self._rev)
         if p2 == nullrev:
             return [repo[p1]]
         return [repo[p1], repo[p2]]
 
     def changeset(self):
         c = self._changeset
         return (
             c.manifest,
             c.user,
             c.date,
             c.files,
             c.description,
             c.extra,
         )
     def manifestnode(self):
         return self._changeset.manifest
 
     def user(self):
         return self._changeset.user
     def date(self):
         return self._changeset.date
     def files(self):
         return self._changeset.files
     def filesmodified(self):
         modified = set(self.files())
         modified.difference_update(self.filesadded())
         modified.difference_update(self.filesremoved())
         return sorted(modified)
     def filesadded(self):
         source = self._repo.ui.config('experimental', 'copies.read-from')
         if (source == 'changeset-only' or
             (source == 'compatibility' and
              self._changeset.filesadded is not None)):
             return self._changeset.filesadded or []
 
         added = []
         for f in self.files():
             if not any(f in p for p in self.parents()):
                 added.append(f)
         return added
     def filesremoved(self):
         source = self._repo.ui.config('experimental', 'copies.read-from')
         if (source == 'changeset-only' or
             (source == 'compatibility' and
              self._changeset.filesremoved is not None)):
             return self._changeset.filesremoved or []
 
         removed = []
         for f in self.files():
             if f not in self:
                 removed.append(f)
         return removed
 
     @propertycache
     def _copies(self):
         source = self._repo.ui.config('experimental', 'copies.read-from')
         p1copies = self._changeset.p1copies
         p2copies = self._changeset.p2copies
         # If config says to get copy metadata only from changeset, then return
         # that, defaulting to {} if there was no copy metadata.
         # In compatibility mode, we return copy data from the changeset if
         # it was recorded there, and otherwise we fall back to getting it from
         # the filelogs (below).
         if (source == 'changeset-only' or
             (source == 'compatibility' and p1copies is not None)):
             return p1copies or {}, p2copies or {}
 
         # Otherwise (config said to read only from filelog, or we are in
         # compatiblity mode and there is not data in the changeset), we get
         # the copy metadata from the filelogs.
         return super(changectx, self)._copies
     def description(self):
         return self._changeset.description
     def branch(self):
         return encoding.tolocal(self._changeset.extra.get("branch"))
     def closesbranch(self):
         return 'close' in self._changeset.extra
     def extra(self):
         """Return a dict of extra information."""
         return self._changeset.extra
     def tags(self):
         """Return a list of byte tag names"""
         return self._repo.nodetags(self._node)
     def bookmarks(self):
         """Return a list of byte bookmark names."""
         return self._repo.nodebookmarks(self._node)
     def phase(self):
         return self._repo._phasecache.phase(self._repo, self._rev)
     def hidden(self):
         return self._rev in repoview.filterrevs(self._repo, 'visible')
 
     def isinmemory(self):
         return False
 
     def children(self):
         """return list of changectx contexts for each child changeset.
 
         This returns only the immediate child changesets. Use descendants() to
         recursively walk children.
         """
         c = self._repo.changelog.children(self._node)
         return [self._repo[x] for x in c]
 
     def ancestors(self):
         for a in self._repo.changelog.ancestors([self._rev]):
             yield self._repo[a]
 
     def descendants(self):
         """Recursively yield all children of the changeset.
 
         For just the immediate children, use children()
         """
         for d in self._repo.changelog.descendants([self._rev]):
             yield self._repo[d]
 
     def filectx(self, path, fileid=None, filelog=None):
         """get a file context from this changeset"""
         if fileid is None:
             fileid = self.filenode(path)
         return filectx(self._repo, path, fileid=fileid,
                        changectx=self, filelog=filelog)
 
     def ancestor(self, c2, warn=False):
         """return the "best" ancestor context of self and c2
 
         If there are multiple candidates, it will show a message and check
         merge.preferancestor configuration before falling back to the
         revlog ancestor."""
         # deal with workingctxs
         n2 = c2._node
         if n2 is None:
             n2 = c2._parents[0]._node
         cahs = self._repo.changelog.commonancestorsheads(self._node, n2)
         if not cahs:
             anc = nullid
         elif len(cahs) == 1:
             anc = cahs[0]
         else:
             # experimental config: merge.preferancestor
             for r in self._repo.ui.configlist('merge', 'preferancestor'):
                 try:
                     ctx = scmutil.revsymbol(self._repo, r)
                 except error.RepoLookupError:
                     continue
                 anc = ctx.node()
                 if anc in cahs:
                     break
             else:
                 anc = self._repo.changelog.ancestor(self._node, n2)
             if warn:
                 self._repo.ui.status(
                     (_("note: using %s as ancestor of %s and %s\n") %
                      (short(anc), short(self._node), short(n2))) +
                     ''.join(_("      alternatively, use --config "
                               "merge.preferancestor=%s\n") %
                             short(n) for n in sorted(cahs) if n != anc))
         return self._repo[anc]
 
     def isancestorof(self, other):
         """True if this changeset is an ancestor of other"""
         return self._repo.changelog.isancestorrev(self._rev, other._rev)
 
     def walk(self, match):
         '''Generates matching file names.'''
 
         # Wrap match.bad method to have message with nodeid
         def bad(fn, msg):
             # The manifest doesn't know about subrepos, so don't complain about
             # paths into valid subrepos.
             if any(fn == s or fn.startswith(s + '/')
                    for s in self.substate):
                 return
             match.bad(fn, _('no such file in rev %s') % self)
 
         m = matchmod.badmatch(self._repo.narrowmatch(match), bad)
         return self._manifest.walk(m)
 
     def matches(self, match):
         return self.walk(match)
 
 class basefilectx(object):
     """A filecontext object represents the common logic for its children:
     filectx: read-only access to a filerevision that is already present
              in the repo,
     workingfilectx: a filecontext that represents files from the working
                     directory,
     memfilectx: a filecontext that represents files in-memory,
     """
     @propertycache
     def _filelog(self):
         return self._repo.file(self._path)
 
     @propertycache
     def _changeid(self):
         if r'_changectx' in self.__dict__:
             return self._changectx.rev()
         elif r'_descendantrev' in self.__dict__:
             # this file context was created from a revision with a known
             # descendant, we can (lazily) correct for linkrev aliases
             return self._adjustlinkrev(self._descendantrev)
         else:
             return self._filelog.linkrev(self._filerev)
 
     @propertycache
     def _filenode(self):
         if r'_fileid' in self.__dict__:
             return self._filelog.lookup(self._fileid)
         else:
             return self._changectx.filenode(self._path)
 
     @propertycache
     def _filerev(self):
         return self._filelog.rev(self._filenode)
 
     @propertycache
     def _repopath(self):
         return self._path
 
     def __nonzero__(self):
         try:
             self._filenode
             return True
         except error.LookupError:
             # file is missing
             return False
 
     __bool__ = __nonzero__
 
     def __bytes__(self):
         try:
             return "%s@%s" % (self.path(), self._changectx)
         except error.LookupError:
             return "%s@???" % self.path()
 
     __str__ = encoding.strmethod(__bytes__)
 
     def __repr__(self):
         return r"<%s %s>" % (type(self).__name__, str(self))
 
     def __hash__(self):
         try:
             return hash((self._path, self._filenode))
         except AttributeError:
             return id(self)
 
     def __eq__(self, other):
         try:
             return (type(self) == type(other) and self._path == other._path
                     and self._filenode == other._filenode)
         except AttributeError:
             return False
 
     def __ne__(self, other):
         return not (self == other)
 
     def filerev(self):
         return self._filerev
     def filenode(self):
         return self._filenode
     @propertycache
     def _flags(self):
         return self._changectx.flags(self._path)
     def flags(self):
         return self._flags
     def filelog(self):
         return self._filelog
     def rev(self):
         return self._changeid
     def linkrev(self):
         return self._filelog.linkrev(self._filerev)
     def node(self):
         return self._changectx.node()
     def hex(self):
         return self._changectx.hex()
     def user(self):
         return self._changectx.user()
     def date(self):
         return self._changectx.date()
     def files(self):
         return self._changectx.files()
     def description(self):
         return self._changectx.description()
     def branch(self):
         return self._changectx.branch()
     def extra(self):
         return self._changectx.extra()
     def phase(self):
         return self._changectx.phase()
     def phasestr(self):
         return self._changectx.phasestr()
     def obsolete(self):
         return self._changectx.obsolete()
     def instabilities(self):
         return self._changectx.instabilities()
     def manifest(self):
         return self._changectx.manifest()
     def changectx(self):
         return self._changectx
     def renamed(self):
         return self._copied
     def copysource(self):
         return self._copied and self._copied[0]
     def repo(self):
         return self._repo
     def size(self):
         return len(self.data())
 
     def path(self):
         return self._path
 
     def isbinary(self):
         try:
             return stringutil.binary(self.data())
         except IOError:
             return False
     def isexec(self):
         return 'x' in self.flags()
     def islink(self):
         return 'l' in self.flags()
 
     def isabsent(self):
         """whether this filectx represents a file not in self._changectx
 
         This is mainly for merge code to detect change/delete conflicts. This is
         expected to be True for all subclasses of basectx."""
         return False
 
     _customcmp = False
     def cmp(self, fctx):
         """compare with other file context
 
         returns True if different than fctx.
         """
         if fctx._customcmp:
             return fctx.cmp(self)
 
         if self._filenode is None:
             raise error.ProgrammingError(
                 'filectx.cmp() must be reimplemented if not backed by revlog')
 
         if fctx._filenode is None:
             if self._repo._encodefilterpats:
                 # can't rely on size() because wdir content may be decoded
                 return self._filelog.cmp(self._filenode, fctx.data())
             if self.size() - 4 == fctx.size():
                 # size() can match:
                 # if file data starts with '\1\n', empty metadata block is
                 # prepended, which adds 4 bytes to filelog.size().
                 return self._filelog.cmp(self._filenode, fctx.data())
         if self.size() == fctx.size():
             # size() matches: need to compare content
             return self._filelog.cmp(self._filenode, fctx.data())
 
         # size() differs
         return True
 
     def _adjustlinkrev(self, srcrev, inclusive=False, stoprev=None):
         """return the first ancestor of <srcrev> introducing <fnode>
 
         If the linkrev of the file revision does not point to an ancestor of
         srcrev, we'll walk down the ancestors until we find one introducing
         this file revision.
 
         :srcrev: the changeset revision we search ancestors from
         :inclusive: if true, the src revision will also be checked
         :stoprev: an optional revision to stop the walk at. If no introduction
                   of this file content could be found before this floor
                   revision, the function will returns "None" and stops its
                   iteration.
         """
         repo = self._repo
         cl = repo.unfiltered().changelog
         mfl = repo.manifestlog
         # fetch the linkrev
         lkr = self.linkrev()
         if srcrev == lkr:
             return lkr
         # hack to reuse ancestor computation when searching for renames
         memberanc = getattr(self, '_ancestrycontext', None)
         iteranc = None
         if srcrev is None:
             # wctx case, used by workingfilectx during mergecopy
             revs = [p.rev() for p in self._repo[None].parents()]
             inclusive = True # we skipped the real (revless) source
         else:
             revs = [srcrev]
         if memberanc is None:
             memberanc = iteranc = cl.ancestors(revs, lkr,
                                                inclusive=inclusive)
         # check if this linkrev is an ancestor of srcrev
         if lkr not in memberanc:
             if iteranc is None:
                 iteranc = cl.ancestors(revs, lkr, inclusive=inclusive)
             fnode = self._filenode
             path = self._path
             for a in iteranc:
                 if stoprev is not None and a < stoprev:
                     return None
                 ac = cl.read(a) # get changeset data (we avoid object creation)
                 if path in ac[3]: # checking the 'files' field.
                     # The file has been touched, check if the content is
                     # similar to the one we search for.
                     if fnode == mfl[ac[0]].readfast().get(path):
                         return a
             # In theory, we should never get out of that loop without a result.
             # But if manifest uses a buggy file revision (not children of the
             # one it replaces) we could. Such a buggy situation will likely
             # result is crash somewhere else at to some point.
         return lkr
 
     def isintroducedafter(self, changelogrev):
         """True if a filectx has been introduced after a given floor revision
         """
         if self.linkrev() >= changelogrev:
             return True
         introrev = self._introrev(stoprev=changelogrev)
         if introrev is None:
             return False
         return introrev >= changelogrev
 
     def introrev(self):
         """return the rev of the changeset which introduced this file revision
 
         This method is different from linkrev because it take into account the
         changeset the filectx was created from. It ensures the returned
         revision is one of its ancestors. This prevents bugs from
         'linkrev-shadowing' when a file revision is used by multiple
         changesets.
         """
         return self._introrev()
 
     def _introrev(self, stoprev=None):
         """
         Same as `introrev` but, with an extra argument to limit changelog
         iteration range in some internal usecase.
 
         If `stoprev` is set, the `introrev` will not be searched past that
         `stoprev` revision and "None" might be returned. This is useful to
         limit the iteration range.
         """
         toprev = None
         attrs = vars(self)
         if r'_changeid' in attrs:
             # We have a cached value already
             toprev = self._changeid
         elif r'_changectx' in attrs:
             # We know which changelog entry we are coming from
             toprev = self._changectx.rev()
 
         if toprev is not None:
             return self._adjustlinkrev(toprev, inclusive=True, stoprev=stoprev)
         elif r'_descendantrev' in attrs:
             introrev = self._adjustlinkrev(self._descendantrev, stoprev=stoprev)
             # be nice and cache the result of the computation
             if introrev is not None:
                 self._changeid = introrev
             return introrev
         else:
             return self.linkrev()
 
     def introfilectx(self):
         """Return filectx having identical contents, but pointing to the
         changeset revision where this filectx was introduced"""
         introrev = self.introrev()
         if self.rev() == introrev:
             return self
         return self.filectx(self.filenode(), changeid=introrev)
 
     def _parentfilectx(self, path, fileid, filelog):
         """create parent filectx keeping ancestry info for _adjustlinkrev()"""
         fctx = filectx(self._repo, path, fileid=fileid, filelog=filelog)
         if r'_changeid' in vars(self) or r'_changectx' in vars(self):
             # If self is associated with a changeset (probably explicitly
             # fed), ensure the created filectx is associated with a
             # changeset that is an ancestor of self.changectx.
             # This lets us later use _adjustlinkrev to get a correct link.
             fctx._descendantrev = self.rev()
             fctx._ancestrycontext = getattr(self, '_ancestrycontext', None)
         elif r'_descendantrev' in vars(self):
             # Otherwise propagate _descendantrev if we have one associated.
             fctx._descendantrev = self._descendantrev
             fctx._ancestrycontext = getattr(self, '_ancestrycontext', None)
         return fctx
 
     def parents(self):
         _path = self._path
         fl = self._filelog
         parents = self._filelog.parents(self._filenode)
         pl = [(_path, node, fl) for node in parents if node != nullid]
 
         r = fl.renamed(self._filenode)
         if r:
             # - In the simple rename case, both parent are nullid, pl is empty.
             # - In case of merge, only one of the parent is null id and should
             # be replaced with the rename information. This parent is -always-
             # the first one.
             #
             # As null id have always been filtered out in the previous list
             # comprehension, inserting to 0 will always result in "replacing
             # first nullid parent with rename information.
             pl.insert(0, (r[0], r[1], self._repo.file(r[0])))
 
         return [self._parentfilectx(path, fnode, l) for path, fnode, l in pl]
 
     def p1(self):
         return self.parents()[0]
 
     def p2(self):
         p = self.parents()
         if len(p) == 2:
             return p[1]
         return filectx(self._repo, self._path, fileid=-1, filelog=self._filelog)
 
     def annotate(self, follow=False, skiprevs=None, diffopts=None):
         """Returns a list of annotateline objects for each line in the file
 
         - line.fctx is the filectx of the node where that line was last changed
         - line.lineno is the line number at the first appearance in the managed
           file
         - line.text is the data on that line (including newline character)
         """
         getlog = util.lrucachefunc(lambda x: self._repo.file(x))
 
         def parents(f):
             # Cut _descendantrev here to mitigate the penalty of lazy linkrev
             # adjustment. Otherwise, p._adjustlinkrev() would walk changelog
             # from the topmost introrev (= srcrev) down to p.linkrev() if it
             # isn't an ancestor of the srcrev.
             f._changeid
             pl = f.parents()
 
             # Don't return renamed parents if we aren't following.
             if not follow:
                 pl = [p for p in pl if p.path() == f.path()]
 
             # renamed filectx won't have a filelog yet, so set it
             # from the cache to save time
             for p in pl:
                 if not r'_filelog' in p.__dict__:
                     p._filelog = getlog(p.path())
 
             return pl
 
         # use linkrev to find the first changeset where self appeared
         base = self.introfilectx()
         if getattr(base, '_ancestrycontext', None) is None:
             cl = self._repo.changelog
             if base.rev() is None:
                 # wctx is not inclusive, but works because _ancestrycontext
                 # is used to test filelog revisions
                 ac = cl.ancestors([p.rev() for p in base.parents()],
                                   inclusive=True)
             else:
                 ac = cl.ancestors([base.rev()], inclusive=True)
             base._ancestrycontext = ac
 
         return dagop.annotate(base, parents, skiprevs=skiprevs,
                               diffopts=diffopts)
 
     def ancestors(self, followfirst=False):
         visit = {}
         c = self
         if followfirst:
             cut = 1
         else:
             cut = None
 
         while True:
             for parent in c.parents()[:cut]:
                 visit[(parent.linkrev(), parent.filenode())] = parent
             if not visit:
                 break
             c = visit.pop(max(visit))
             yield c
 
     def decodeddata(self):
         """Returns `data()` after running repository decoding filters.
 
         This is often equivalent to how the data would be expressed on disk.
         """
         return self._repo.wwritedata(self.path(), self.data())
 
 class filectx(basefilectx):
     """A filecontext object makes access to data related to a particular
        filerevision convenient."""
     def __init__(self, repo, path, changeid=None, fileid=None,
                  filelog=None, changectx=None):
         """changeid must be a revision number, if specified.
            fileid can be a file revision or node."""
         self._repo = repo
         self._path = path
 
         assert (changeid is not None
                 or fileid is not None
                 or changectx is not None), (
                     "bad args: changeid=%r, fileid=%r, changectx=%r"
                     % (changeid, fileid, changectx))
 
         if filelog is not None:
             self._filelog = filelog
 
         if changeid is not None:
             self._changeid = changeid
         if changectx is not None:
             self._changectx = changectx
         if fileid is not None:
             self._fileid = fileid
 
     @propertycache
     def _changectx(self):
         try:
             return self._repo[self._changeid]
         except error.FilteredRepoLookupError:
             # Linkrev may point to any revision in the repository.  When the
             # repository is filtered this may lead to `filectx` trying to build
             # `changectx` for filtered revision. In such case we fallback to
             # creating `changectx` on the unfiltered version of the reposition.
             # This fallback should not be an issue because `changectx` from
             # `filectx` are not used in complex operations that care about
             # filtering.
             #
             # This fallback is a cheap and dirty fix that prevent several
             # crashes. It does not ensure the behavior is correct. However the
             # behavior was not correct before filtering either and "incorrect
             # behavior" is seen as better as "crash"
             #
             # Linkrevs have several serious troubles with filtering that are
             # complicated to solve. Proper handling of the issue here should be
             # considered when solving linkrev issue are on the table.
             return self._repo.unfiltered()[self._changeid]
 
     def filectx(self, fileid, changeid=None):
         '''opens an arbitrary revision of the file without
         opening a new filelog'''
         return filectx(self._repo, self._path, fileid=fileid,
                        filelog=self._filelog, changeid=changeid)
 
     def rawdata(self):
         return self._filelog.revision(self._filenode, raw=True)
 
     def rawflags(self):
         """low-level revlog flags"""
         return self._filelog.flags(self._filerev)
 
     def data(self):
         try:
             return self._filelog.read(self._filenode)
         except error.CensoredNodeError:
             if self._repo.ui.config("censor", "policy") == "ignore":
                 return ""
             raise error.Abort(_("censored node: %s") % short(self._filenode),
                              hint=_("set censor.policy to ignore errors"))
 
     def size(self):
         return self._filelog.size(self._filerev)
 
     @propertycache
     def _copied(self):
         """check if file was actually renamed in this changeset revision
 
         If rename logged in file revision, we report copy for changeset only
         if file revisions linkrev points back to the changeset in question
         or both changeset parents contain different file revisions.
         """
 
         renamed = self._filelog.renamed(self._filenode)
         if not renamed:
             return None
 
         if self.rev() == self.linkrev():
             return renamed
 
         name = self.path()
         fnode = self._filenode
         for p in self._changectx.parents():
             try:
                 if fnode == p.filenode(name):
                     return None
             except error.LookupError:
                 pass
         return renamed
 
     def children(self):
         # hard for renames
         c = self._filelog.children(self._filenode)
         return [filectx(self._repo, self._path, fileid=x,
                         filelog=self._filelog) for x in c]
 
 class committablectx(basectx):
     """A committablectx object provides common functionality for a context that
     wants the ability to commit, e.g. workingctx or memctx."""
     def __init__(self, repo, text="", user=None, date=None, extra=None,
                  changes=None, branch=None):
         super(committablectx, self).__init__(repo)
         self._rev = None
         self._node = None
         self._text = text
         if date:
             self._date = dateutil.parsedate(date)
         if user:
             self._user = user
         if changes:
             self._status = changes
 
         self._extra = {}
         if extra:
             self._extra = extra.copy()
         if branch is not None:
             self._extra['branch'] = encoding.fromlocal(branch)
         if not self._extra.get('branch'):
             self._extra['branch'] = 'default'
 
     def __bytes__(self):
         return bytes(self._parents[0]) + "+"
 
     __str__ = encoding.strmethod(__bytes__)
 
     def __nonzero__(self):
         return True
 
     __bool__ = __nonzero__
 
     @propertycache
     def _status(self):
         return self._repo.status()
 
     @propertycache
     def _user(self):
         return self._repo.ui.username()
 
     @propertycache
     def _date(self):
         ui = self._repo.ui
         date = ui.configdate('devel', 'default-date')
         if date is None:
             date = dateutil.makedate()
         return date
 
     def subrev(self, subpath):
         return None
 
     def manifestnode(self):
         return None
     def user(self):
         return self._user or self._repo.ui.username()
     def date(self):
         return self._date
     def description(self):
         return self._text
     def files(self):
         return sorted(self._status.modified + self._status.added +
                       self._status.removed)
     def modified(self):
         return self._status.modified
     def added(self):
         return self._status.added
     def removed(self):
         return self._status.removed
     def deleted(self):
         return self._status.deleted
     filesmodified = modified
     filesadded = added
     filesremoved = removed
 
     def branch(self):
         return encoding.tolocal(self._extra['branch'])
     def closesbranch(self):
         return 'close' in self._extra
     def extra(self):
         return self._extra
 
     def isinmemory(self):
         return False
 
     def tags(self):
         return []
 
     def bookmarks(self):
         b = []
         for p in self.parents():
             b.extend(p.bookmarks())
         return b
 
     def phase(self):
         phase = phases.draft # default phase to draft
         for p in self.parents():
             phase = max(phase, p.phase())
         return phase
 
     def hidden(self):
         return False
 
     def children(self):
         return []
 
     def ancestor(self, c2):
         """return the "best" ancestor context of self and c2"""
         return self._parents[0].ancestor(c2) # punt on two parents for now
 
     def ancestors(self):
         for p in self._parents:
             yield p
         for a in self._repo.changelog.ancestors(
             [p.rev() for p in self._parents]):
             yield self._repo[a]
 
     def markcommitted(self, node):
         """Perform post-commit cleanup necessary after committing this ctx
 
         Specifically, this updates backing stores this working context
         wraps to reflect the fact that the changes reflected by this
         workingctx have been committed.  For example, it marks
         modified and added files as normal in the dirstate.
 
         """
 
     def dirty(self, missing=False, merge=True, branch=True):
         return False
 
 class workingctx(committablectx):
     """A workingctx object makes access to data related to
     the current working directory convenient.
     date - any valid date string or (unixtime, offset), or None.
     user - username string, or None.
     extra - a dictionary of extra values, or None.
     changes - a list of file lists as returned by localrepo.status()
                or None to use the repository status.
     """
     def __init__(self, repo, text="", user=None, date=None, extra=None,
                  changes=None):
         branch = None
         if not extra or 'branch' not in extra:
             try:
                 branch = repo.dirstate.branch()
             except UnicodeDecodeError:
                 raise error.Abort(_('branch name not in UTF-8!'))
         super(workingctx, self).__init__(repo, text, user, date, extra, changes,
                                          branch=branch)
 
     def __iter__(self):
         d = self._repo.dirstate
         for f in d:
             if d[f] != 'r':
                 yield f
 
     def __contains__(self, key):
         return self._repo.dirstate[key] not in "?r"
 
     def hex(self):
         return wdirhex
 
     @propertycache
     def _parents(self):
         p = self._repo.dirstate.parents()
         if p[1] == nullid:
             p = p[:-1]
         # use unfiltered repo to delay/avoid loading obsmarkers
         unfi = self._repo.unfiltered()
         return [changectx(self._repo, unfi.changelog.rev(n), n) for n in p]
 
     def _fileinfo(self, path):
         # populate __dict__['_manifest'] as workingctx has no _manifestdelta
         self._manifest
         return super(workingctx, self)._fileinfo(path)
 
     def _buildflagfunc(self):
         # Create a fallback function for getting file flags when the
         # filesystem doesn't support them
 
         copiesget = self._repo.dirstate.copies().get
         parents = self.parents()
         if len(parents) < 2:
             # when we have one parent, it's easy: copy from parent
             man = parents[0].manifest()
             def func(f):
                 f = copiesget(f, f)
                 return man.flags(f)
         else:
             # merges are tricky: we try to reconstruct the unstored
             # result from the merge (issue1802)
             p1, p2 = parents
             pa = p1.ancestor(p2)
             m1, m2, ma = p1.manifest(), p2.manifest(), pa.manifest()
 
             def func(f):
                 f = copiesget(f, f) # may be wrong for merges with copies
                 fl1, fl2, fla = m1.flags(f), m2.flags(f), ma.flags(f)
                 if fl1 == fl2:
                     return fl1
                 if fl1 == fla:
                     return fl2
                 if fl2 == fla:
                     return fl1
                 return '' # punt for conflicts
 
         return func
 
     @propertycache
     def _flagfunc(self):
         return self._repo.dirstate.flagfunc(self._buildflagfunc)
 
     def flags(self, path):
         if r'_manifest' in self.__dict__:
             try:
                 return self._manifest.flags(path)
             except KeyError:
                 return ''
 
         try:
             return self._flagfunc(path)
         except OSError:
             return ''
 
     def filectx(self, path, filelog=None):
         """get a file context from the working directory"""
         return workingfilectx(self._repo, path, workingctx=self,
                               filelog=filelog)
 
     def dirty(self, missing=False, merge=True, branch=True):
         "check whether a working directory is modified"
         # check subrepos first
         for s in sorted(self.substate):
             if self.sub(s).dirty(missing=missing):
                 return True
         # check current working dir
         return ((merge and self.p2()) or
                 (branch and self.branch() != self.p1().branch()) or
                 self.modified() or self.added() or self.removed() or
                 (missing and self.deleted()))
 
     def add(self, list, prefix=""):
         with self._repo.wlock():
             ui, ds = self._repo.ui, self._repo.dirstate
             uipath = lambda f: ds.pathto(pathutil.join(prefix, f))
             rejected = []
             lstat = self._repo.wvfs.lstat
             for f in list:
                 # ds.pathto() returns an absolute file when this is invoked from
                 # the keyword extension.  That gets flagged as non-portable on
                 # Windows, since it contains the drive letter and colon.
                 scmutil.checkportable(ui, os.path.join(prefix, f))
                 try:
                     st = lstat(f)
                 except OSError:
                     ui.warn(_("%s does not exist!\n") % uipath(f))
                     rejected.append(f)
                     continue
                 limit = ui.configbytes('ui', 'large-file-limit')
                 if limit != 0 and st.st_size > limit:
                     ui.warn(_("%s: up to %d MB of RAM may be required "
                               "to manage this file\n"
                               "(use 'hg revert %s' to cancel the "
                               "pending addition)\n")
                             % (f, 3 * st.st_size // 1000000, uipath(f)))
                 if not (stat.S_ISREG(st.st_mode) or stat.S_ISLNK(st.st_mode)):
                     ui.warn(_("%s not added: only files and symlinks "
                               "supported currently\n") % uipath(f))
                     rejected.append(f)
                 elif ds[f] in 'amn':
                     ui.warn(_("%s already tracked!\n") % uipath(f))
                 elif ds[f] == 'r':
                     ds.normallookup(f)
                 else:
                     ds.add(f)
             return rejected
 
     def forget(self, files, prefix=""):
         with self._repo.wlock():
             ds = self._repo.dirstate
             uipath = lambda f: ds.pathto(pathutil.join(prefix, f))
             rejected = []
             for f in files:
                 if f not in ds:
                     self._repo.ui.warn(_("%s not tracked!\n") % uipath(f))
                     rejected.append(f)
                 elif ds[f] != 'a':
                     ds.remove(f)
                 else:
                     ds.drop(f)
             return rejected
 
     def copy(self, source, dest):
         try:
             st = self._repo.wvfs.lstat(dest)
         except OSError as err:
             if err.errno != errno.ENOENT:
                 raise
             self._repo.ui.warn(_("%s does not exist!\n")
                                % self._repo.dirstate.pathto(dest))
             return
         if not (stat.S_ISREG(st.st_mode) or stat.S_ISLNK(st.st_mode)):
             self._repo.ui.warn(_("copy failed: %s is not a file or a "
                                  "symbolic link\n")
                                % self._repo.dirstate.pathto(dest))
         else:
             with self._repo.wlock():
                 ds = self._repo.dirstate
                 if ds[dest] in '?':
                     ds.add(dest)
                 elif ds[dest] in 'r':
                     ds.normallookup(dest)
                 ds.copy(source, dest)
 
     def match(self, pats=None, include=None, exclude=None, default='glob',
               listsubrepos=False, badfn=None):
         r = self._repo
 
         # Only a case insensitive filesystem needs magic to translate user input
         # to actual case in the filesystem.
         icasefs = not util.fscasesensitive(r.root)
         return matchmod.match(r.root, r.getcwd(), pats, include, exclude,
                               default, auditor=r.auditor, ctx=self,
                               listsubrepos=listsubrepos, badfn=badfn,
                               icasefs=icasefs)
 
     def _filtersuspectsymlink(self, files):
         if not files or self._repo.dirstate._checklink:
             return files
 
         # Symlink placeholders may get non-symlink-like contents
         # via user error or dereferencing by NFS or Samba servers,
         # so we filter out any placeholders that don't look like a
         # symlink
         sane = []
         for f in files:
             if self.flags(f) == 'l':
                 d = self[f].data()
                 if (d == '' or len(d) >= 1024 or '\n' in d
                     or stringutil.binary(d)):
                     self._repo.ui.debug('ignoring suspect symlink placeholder'
                                         ' "%s"\n' % f)
                     continue
             sane.append(f)
         return sane
 
     def _checklookup(self, files):
         # check for any possibly clean files
         if not files:
             return [], [], []
 
         modified = []
         deleted = []
         fixup = []
         pctx = self._parents[0]
         # do a full compare of any files that might have changed
         for f in sorted(files):
             try:
                 # This will return True for a file that got replaced by a
                 # directory in the interim, but fixing that is pretty hard.
                 if (f not in pctx or self.flags(f) != pctx.flags(f)
                     or pctx[f].cmp(self[f])):
                     modified.append(f)
                 else:
                     fixup.append(f)
             except (IOError, OSError):
                 # A file become inaccessible in between? Mark it as deleted,
                 # matching dirstate behavior (issue5584).
                 # The dirstate has more complex behavior around whether a
                 # missing file matches a directory, etc, but we don't need to
                 # bother with that: if f has made it to this point, we're sure
                 # it's in the dirstate.
                 deleted.append(f)
 
         return modified, deleted, fixup
 
     def _poststatusfixup(self, status, fixup):
         """update dirstate for files that are actually clean"""
         poststatus = self._repo.postdsstatus()
         if fixup or poststatus:
             try:
                 oldid = self._repo.dirstate.identity()
 
                 # updating the dirstate is optional
                 # so we don't wait on the lock
                 # wlock can invalidate the dirstate, so cache normal _after_
                 # taking the lock
                 with self._repo.wlock(False):
                     if self._repo.dirstate.identity() == oldid:
                         if fixup:
                             normal = self._repo.dirstate.normal
                             for f in fixup:
                                 normal(f)
                             # write changes out explicitly, because nesting
                             # wlock at runtime may prevent 'wlock.release()'
                             # after this block from doing so for subsequent
                             # changing files
                             tr = self._repo.currenttransaction()
                             self._repo.dirstate.write(tr)
 
                         if poststatus:
                             for ps in poststatus:
                                 ps(self, status)
                     else:
                         # in this case, writing changes out breaks
                         # consistency, because .hg/dirstate was
                         # already changed simultaneously after last
                         # caching (see also issue5584 for detail)
                         self._repo.ui.debug('skip updating dirstate: '
                                             'identity mismatch\n')
             except error.LockError:
                 pass
             finally:
                 # Even if the wlock couldn't be grabbed, clear out the list.
                 self._repo.clearpostdsstatus()
 
     def _dirstatestatus(self, match, ignored=False, clean=False, unknown=False):
         '''Gets the status from the dirstate -- internal use only.'''
         subrepos = []
         if '.hgsub' in self:
             subrepos = sorted(self.substate)
         cmp, s = self._repo.dirstate.status(match, subrepos, ignored=ignored,
                                             clean=clean, unknown=unknown)
 
         # check for any possibly clean files
         fixup = []
         if cmp:
             modified2, deleted2, fixup = self._checklookup(cmp)
             s.modified.extend(modified2)
             s.deleted.extend(deleted2)
 
             if fixup and clean:
                 s.clean.extend(fixup)
 
         self._poststatusfixup(s, fixup)
 
         if match.always():
             # cache for performance
             if s.unknown or s.ignored or s.clean:
                 # "_status" is cached with list*=False in the normal route
                 self._status = scmutil.status(s.modified, s.added, s.removed,
                                               s.deleted, [], [], [])
             else:
                 self._status = s
 
         return s
 
     @propertycache
     def _copies(self):
         p1copies = {}
         p2copies = {}
         parents = self._repo.dirstate.parents()
         p1manifest = self._repo[parents[0]].manifest()
         p2manifest = self._repo[parents[1]].manifest()
         narrowmatch = self._repo.narrowmatch()
         for dst, src in self._repo.dirstate.copies().items():
             if not narrowmatch(dst):
                 continue
             if src in p1manifest:
                 p1copies[dst] = src
             elif src in p2manifest:
                 p2copies[dst] = src
         return p1copies, p2copies
     def p1copies(self):
         return self._copies[0]
     def p2copies(self):
         return self._copies[1]
 
     @propertycache
     def _manifest(self):
         """generate a manifest corresponding to the values in self._status
 
         This reuse the file nodeid from parent, but we use special node
         identifiers for added and modified files. This is used by manifests
         merge to see that files are different and by update logic to avoid
         deleting newly added files.
         """
         return self._buildstatusmanifest(self._status)
 
     def _buildstatusmanifest(self, status):
         """Builds a manifest that includes the given status results."""
         parents = self.parents()
 
         man = parents[0].manifest().copy()
 
         ff = self._flagfunc
         for i, l in ((addednodeid, status.added),
                      (modifiednodeid, status.modified)):
             for f in l:
                 man[f] = i
                 try:
                     man.setflag(f, ff(f))
                 except OSError:
                     pass
 
         for f in status.deleted + status.removed:
             if f in man:
                 del man[f]
 
         return man
 
     def _buildstatus(self, other, s, match, listignored, listclean,
                      listunknown):
         """build a status with respect to another context
 
         This includes logic for maintaining the fast path of status when
         comparing the working directory against its parent, which is to skip
         building a new manifest if self (working directory) is not comparing
         against its parent (repo['.']).
         """
         s = self._dirstatestatus(match, listignored, listclean, listunknown)
         # Filter out symlinks that, in the case of FAT32 and NTFS filesystems,
         # might have accidentally ended up with the entire contents of the file
         # they are supposed to be linking to.
         s.modified[:] = self._filtersuspectsymlink(s.modified)
         if other != self._repo['.']:
             s = super(workingctx, self)._buildstatus(other, s, match,
                                                      listignored, listclean,
                                                      listunknown)
         return s
 
     def _matchstatus(self, other, match):
         """override the match method with a filter for directory patterns
 
         We use inheritance to customize the match.bad method only in cases of
         workingctx since it belongs only to the working directory when
         comparing against the parent changeset.
 
         If we aren't comparing against the working directory's parent, then we
         just use the default match object sent to us.
         """
         if other != self._repo['.']:
             def bad(f, msg):
                 # 'f' may be a directory pattern from 'match.files()',
                 # so 'f not in ctx1' is not enough
                 if f not in other and not other.hasdir(f):
                     self._repo.ui.warn('%s: %s\n' %
                                        (self._repo.dirstate.pathto(f), msg))
             match.bad = bad
         return match
 
     def walk(self, match):
         '''Generates matching file names.'''
         return sorted(self._repo.dirstate.walk(self._repo.narrowmatch(match),
                                                subrepos=sorted(self.substate),
                                                unknown=True, ignored=False))
 
     def matches(self, match):
         match = self._repo.narrowmatch(match)
         ds = self._repo.dirstate
         return sorted(f for f in ds.matches(match) if ds[f] != 'r')
 
     def markcommitted(self, node):
         with self._repo.dirstate.parentchange():
             for f in self.modified() + self.added():
                 self._repo.dirstate.normal(f)
             for f in self.removed():
                 self._repo.dirstate.drop(f)
             self._repo.dirstate.setparents(node)
 
         # write changes out explicitly, because nesting wlock at
         # runtime may prevent 'wlock.release()' in 'repo.commit()'
         # from immediately doing so for subsequent changing files
         self._repo.dirstate.write(self._repo.currenttransaction())
 
         sparse.aftercommit(self._repo, node)
 
 class committablefilectx(basefilectx):
     """A committablefilectx provides common functionality for a file context
     that wants the ability to commit, e.g. workingfilectx or memfilectx."""
     def __init__(self, repo, path, filelog=None, ctx=None):
         self._repo = repo
         self._path = path
         self._changeid = None
         self._filerev = self._filenode = None
 
         if filelog is not None:
             self._filelog = filelog
         if ctx:
             self._changectx = ctx
 
     def __nonzero__(self):
         return True
 
     __bool__ = __nonzero__
 
     def linkrev(self):
         # linked to self._changectx no matter if file is modified or not
         return self.rev()
 
     def renamed(self):
         path = self.copysource()
         if not path:
             return None
         return path, self._changectx._parents[0]._manifest.get(path, nullid)
 
     def parents(self):
         '''return parent filectxs, following copies if necessary'''
         def filenode(ctx, path):
             return ctx._manifest.get(path, nullid)
 
         path = self._path
         fl = self._filelog
         pcl = self._changectx._parents
         renamed = self.renamed()
 
         if renamed:
             pl = [renamed + (None,)]
         else:
             pl = [(path, filenode(pcl[0], path), fl)]
 
         for pc in pcl[1:]:
             pl.append((path, filenode(pc, path), fl))
 
         return [self._parentfilectx(p, fileid=n, filelog=l)
                 for p, n, l in pl if n != nullid]
 
     def children(self):
         return []
 
 class workingfilectx(committablefilectx):
     """A workingfilectx object makes access to data related to a particular
        file in the working directory convenient."""
     def __init__(self, repo, path, filelog=None, workingctx=None):
         super(workingfilectx, self).__init__(repo, path, filelog, workingctx)
 
     @propertycache
     def _changectx(self):
         return workingctx(self._repo)
 
     def data(self):
         return self._repo.wread(self._path)
     def copysource(self):
         return self._repo.dirstate.copied(self._path)
 
     def size(self):
         return self._repo.wvfs.lstat(self._path).st_size
+    def lstat(self):
+        return self._repo.wvfs.lstat(self._path)
     def date(self):
         t, tz = self._changectx.date()
         try:
             return (self._repo.wvfs.lstat(self._path)[stat.ST_MTIME], tz)
         except OSError as err:
             if err.errno != errno.ENOENT:
                 raise
             return (t, tz)
 
     def exists(self):
         return self._repo.wvfs.exists(self._path)
 
     def lexists(self):
         return self._repo.wvfs.lexists(self._path)
 
     def audit(self):
         return self._repo.wvfs.audit(self._path)
 
     def cmp(self, fctx):
         """compare with other file context
 
         returns True if different than fctx.
         """
         # fctx should be a filectx (not a workingfilectx)
         # invert comparison to reuse the same code path
         return fctx.cmp(self)
 
     def remove(self, ignoremissing=False):
         """wraps unlink for a repo's working directory"""
         rmdir = self._repo.ui.configbool('experimental', 'removeemptydirs')
         self._repo.wvfs.unlinkpath(self._path, ignoremissing=ignoremissing,
                                    rmdir=rmdir)
 
     def write(self, data, flags, backgroundclose=False, **kwargs):
         """wraps repo.wwrite"""
-        self._repo.wwrite(self._path, data, flags,
+        return self._repo.wwrite(self._path, data, flags,
                           backgroundclose=backgroundclose,
                           **kwargs)
 
     def markcopied(self, src):
         """marks this file a copy of `src`"""
         self._repo.dirstate.copy(src, self._path)
 
     def clearunknown(self):
         """Removes conflicting items in the working directory so that
         ``write()`` can be called successfully.
         """
         wvfs = self._repo.wvfs
         f = self._path
         wvfs.audit(f)
         if self._repo.ui.configbool('experimental', 'merge.checkpathconflicts'):
             # remove files under the directory as they should already be
             # warned and backed up
             if wvfs.isdir(f) and not wvfs.islink(f):
                 wvfs.rmtree(f, forcibly=True)
             for p in reversed(list(util.finddirs(f))):
                 if wvfs.isfileorlink(p):
                     wvfs.unlink(p)
                     break
         else:
             # don't remove files if path conflicts are not processed
             if wvfs.isdir(f) and not wvfs.islink(f):
                 wvfs.removedirs(f)
 
     def setflags(self, l, x):
         self._repo.wvfs.setflags(self._path, l, x)
 
 class overlayworkingctx(committablectx):
     """Wraps another mutable context with a write-back cache that can be
     converted into a commit context.
 
     self._cache[path] maps to a dict with keys: {
         'exists': bool?
         'date': date?
         'data': str?
         'flags': str?
         'copied': str? (path or None)
     }
     If `exists` is True, `flags` must be non-None and 'date' is non-None. If it
     is `False`, the file was deleted.
     """
 
     def __init__(self, repo):
         super(overlayworkingctx, self).__init__(repo)
         self.clean()
 
     def setbase(self, wrappedctx):
         self._wrappedctx = wrappedctx
         self._parents = [wrappedctx]
         # Drop old manifest cache as it is now out of date.
         # This is necessary when, e.g., rebasing several nodes with one
         # ``overlayworkingctx`` (e.g. with --collapse).
         util.clearcachedproperty(self, '_manifest')
 
     def data(self, path):
         if self.isdirty(path):
             if self._cache[path]['exists']:
                 if self._cache[path]['data'] is not None:
                     return self._cache[path]['data']
                 else:
                     # Must fallback here, too, because we only set flags.
                     return self._wrappedctx[path].data()
             else:
                 raise error.ProgrammingError("No such file or directory: %s" %
                                              path)
         else:
             return self._wrappedctx[path].data()
 
     @propertycache
     def _manifest(self):
         parents = self.parents()
         man = parents[0].manifest().copy()
 
         flag = self._flagfunc
         for path in self.added():
             man[path] = addednodeid
             man.setflag(path, flag(path))
         for path in self.modified():
             man[path] = modifiednodeid
             man.setflag(path, flag(path))
         for path in self.removed():
             del man[path]
         return man
 
     @propertycache
     def _flagfunc(self):
         def f(path):
             return self._cache[path]['flags']
         return f
 
     def files(self):
         return sorted(self.added() + self.modified() + self.removed())
 
     def modified(self):
         return [f for f in self._cache.keys() if self._cache[f]['exists'] and
                 self._existsinparent(f)]
 
     def added(self):
         return [f for f in self._cache.keys() if self._cache[f]['exists'] and
                 not self._existsinparent(f)]
 
     def removed(self):
         return [f for f in self._cache.keys() if
                 not self._cache[f]['exists'] and self._existsinparent(f)]
 
     def p1copies(self):
         copies = self._repo._wrappedctx.p1copies().copy()
         narrowmatch = self._repo.narrowmatch()
         for f in self._cache.keys():
             if not narrowmatch(f):
                 continue
             copies.pop(f, None) # delete if it exists
             source = self._cache[f]['copied']
             if source:
                 copies[f] = source
         return copies
 
     def p2copies(self):
         copies = self._repo._wrappedctx.p2copies().copy()
         narrowmatch = self._repo.narrowmatch()
         for f in self._cache.keys():
             if not narrowmatch(f):
                 continue
             copies.pop(f, None) # delete if it exists
             source = self._cache[f]['copied']
             if source:
                 copies[f] = source
         return copies
 
     def isinmemory(self):
         return True
 
     def filedate(self, path):
         if self.isdirty(path):
             return self._cache[path]['date']
         else:
             return self._wrappedctx[path].date()
 
     def markcopied(self, path, origin):
         self._markdirty(path, exists=True, date=self.filedate(path),
                         flags=self.flags(path), copied=origin)
 
     def copydata(self, path):
         if self.isdirty(path):
             return self._cache[path]['copied']
         else:
             return None
 
     def flags(self, path):
         if self.isdirty(path):
             if self._cache[path]['exists']:
                 return self._cache[path]['flags']
             else:
                 raise error.ProgrammingError("No such file or directory: %s" %
                                              self._path)
         else:
             return self._wrappedctx[path].flags()
 
     def __contains__(self, key):
         if key in self._cache:
             return self._cache[key]['exists']
         return key in self.p1()
 
     def _existsinparent(self, path):
         try:
             # ``commitctx` raises a ``ManifestLookupError`` if a path does not
             # exist, unlike ``workingctx``, which returns a ``workingfilectx``
             # with an ``exists()`` function.
             self._wrappedctx[path]
             return True
         except error.ManifestLookupError:
             return False
 
     def _auditconflicts(self, path):
         """Replicates conflict checks done by wvfs.write().
 
         Since we never write to the filesystem and never call `applyupdates` in
         IMM, we'll never check that a path is actually writable -- e.g., because
         it adds `a/foo`, but `a` is actually a file in the other commit.
         """
         def fail(path, component):
             # p1() is the base and we're receiving "writes" for p2()'s
             # files.
             if 'l' in self.p1()[component].flags():
                 raise error.Abort("error: %s conflicts with symlink %s "
                                   "in %d." % (path, component,
                                               self.p1().rev()))
             else:
                 raise error.Abort("error: '%s' conflicts with file '%s' in "
                                   "%d." % (path, component,
                                            self.p1().rev()))
 
         # Test that each new directory to be created to write this path from p2
         # is not a file in p1.
         components = path.split('/')
         for i in pycompat.xrange(len(components)):
             component = "/".join(components[0:i])
             if component in self:
                 fail(path, component)
 
         # Test the other direction -- that this path from p2 isn't a directory
         # in p1 (test that p1 doesn't have any paths matching `path/*`).
         match = self.match([path], default=b'path')
         matches = self.p1().manifest().matches(match)
         mfiles = matches.keys()
         if len(mfiles) > 0:
             if len(mfiles) == 1 and mfiles[0] == path:
                 return
             # omit the files which are deleted in current IMM wctx
             mfiles = [m for m in mfiles if m in self]
             if not mfiles:
                 return
             raise error.Abort("error: file '%s' cannot be written because "
                               " '%s/' is a directory in %s (containing %d "
                               "entries: %s)"
                               % (path, path, self.p1(), len(mfiles),
                                  ', '.join(mfiles)))
 
     def write(self, path, data, flags='', **kwargs):
         if data is None:
             raise error.ProgrammingError("data must be non-None")
         self._auditconflicts(path)
         self._markdirty(path, exists=True, data=data, date=dateutil.makedate(),
                         flags=flags)
 
     def setflags(self, path, l, x):
         flag = ''
         if l:
             flag = 'l'
         elif x:
             flag = 'x'
         self._markdirty(path, exists=True, date=dateutil.makedate(),
                         flags=flag)
 
     def remove(self, path):
         self._markdirty(path, exists=False)
 
     def exists(self, path):
         """exists behaves like `lexists`, but needs to follow symlinks and
         return False if they are broken.
         """
         if self.isdirty(path):
             # If this path exists and is a symlink, "follow" it by calling
             # exists on the destination path.
             if (self._cache[path]['exists'] and
                         'l' in self._cache[path]['flags']):
                 return self.exists(self._cache[path]['data'].strip())
             else:
                 return self._cache[path]['exists']
 
         return self._existsinparent(path)
 
     def lexists(self, path):
         """lexists returns True if the path exists"""
         if self.isdirty(path):
             return self._cache[path]['exists']
 
         return self._existsinparent(path)
 
     def size(self, path):
         if self.isdirty(path):
             if self._cache[path]['exists']:
                 return len(self._cache[path]['data'])
             else:
                 raise error.ProgrammingError("No such file or directory: %s" %
                                              self._path)
         return self._wrappedctx[path].size()
 
     def tomemctx(self, text, branch=None, extra=None, date=None, parents=None,
                  user=None, editor=None):
         """Converts this ``overlayworkingctx`` into a ``memctx`` ready to be
         committed.
 
         ``text`` is the commit message.
         ``parents`` (optional) are rev numbers.
         """
         # Default parents to the wrapped contexts' if not passed.
         if parents is None:
             parents = self._wrappedctx.parents()
             if len(parents) == 1:
                 parents = (parents[0], None)
 
         # ``parents`` is passed as rev numbers; convert to ``commitctxs``.
         if parents[1] is None:
             parents = (self._repo[parents[0]], None)
         else:
             parents = (self._repo[parents[0]], self._repo[parents[1]])
 
         files = self.files()
         def getfile(repo, memctx, path):
             if self._cache[path]['exists']:
                 return memfilectx(repo, memctx, path,
                                   self._cache[path]['data'],
                                   'l' in self._cache[path]['flags'],
                                   'x' in self._cache[path]['flags'],
                                   self._cache[path]['copied'])
             else:
                 # Returning None, but including the path in `files`, is
                 # necessary for memctx to register a deletion.
                 return None
         return memctx(self._repo, parents, text, files, getfile, date=date,
                       extra=extra, user=user, branch=branch, editor=editor)
 
     def isdirty(self, path):
         return path in self._cache
 
     def isempty(self):
         # We need to discard any keys that are actually clean before the empty
         # commit check.
         self._compact()
         return len(self._cache) == 0
 
     def clean(self):
         self._cache = {}
 
     def _compact(self):
         """Removes keys from the cache that are actually clean, by comparing
         them with the underlying context.
 
         This can occur during the merge process, e.g. by passing --tool :local
         to resolve a conflict.
         """
         keys = []
         # This won't be perfect, but can help performance significantly when
         # using things like remotefilelog.
         scmutil.prefetchfiles(
             self.repo(), [self.p1().rev()],
             scmutil.matchfiles(self.repo(), self._cache.keys()))
 
         for path in self._cache.keys():
             cache = self._cache[path]
             try:
                 underlying = self._wrappedctx[path]
                 if (underlying.data() == cache['data'] and
                             underlying.flags() == cache['flags']):
                     keys.append(path)
             except error.ManifestLookupError:
                 # Path not in the underlying manifest (created).
                 continue
 
         for path in keys:
             del self._cache[path]
         return keys
 
     def _markdirty(self, path, exists, data=None, date=None, flags='',
         copied=None):
         # data not provided, let's see if we already have some; if not, let's
         # grab it from our underlying context, so that we always have data if
         # the file is marked as existing.
         if exists and data is None:
             oldentry = self._cache.get(path) or {}
             data = oldentry.get('data') or self._wrappedctx[path].data()
 
         self._cache[path] = {
             'exists': exists,
             'data': data,
             'date': date,
             'flags': flags,
             'copied': copied,
         }
 
     def filectx(self, path, filelog=None):
         return overlayworkingfilectx(self._repo, path, parent=self,
                                      filelog=filelog)
 
 class overlayworkingfilectx(committablefilectx):
     """Wrap a ``workingfilectx`` but intercepts all writes into an in-memory
     cache, which can be flushed through later by calling ``flush()``."""
 
     def __init__(self, repo, path, filelog=None, parent=None):
         super(overlayworkingfilectx, self).__init__(repo, path, filelog,
                                                     parent)
         self._repo = repo
         self._parent = parent
         self._path = path
 
     def cmp(self, fctx):
         return self.data() != fctx.data()
 
     def changectx(self):
         return self._parent
 
     def data(self):
         return self._parent.data(self._path)
 
     def date(self):
         return self._parent.filedate(self._path)
 
     def exists(self):
         return self.lexists()
 
     def lexists(self):
         return self._parent.exists(self._path)
 
     def copysource(self):
         return self._parent.copydata(self._path)
 
     def size(self):
         return self._parent.size(self._path)
 
     def markcopied(self, origin):
         self._parent.markcopied(self._path, origin)
 
     def audit(self):
         pass
 
     def flags(self):
         return self._parent.flags(self._path)
 
     def setflags(self, islink, isexec):
         return self._parent.setflags(self._path, islink, isexec)
 
     def write(self, data, flags, backgroundclose=False, **kwargs):
         return self._parent.write(self._path, data, flags, **kwargs)
 
     def remove(self, ignoremissing=False):
         return self._parent.remove(self._path)
 
     def clearunknown(self):
         pass
 
 class workingcommitctx(workingctx):
     """A workingcommitctx object makes access to data related to
     the revision being committed convenient.
 
     This hides changes in the working directory, if they aren't
     committed in this context.
     """
     def __init__(self, repo, changes,
                  text="", user=None, date=None, extra=None):
         super(workingcommitctx, self).__init__(repo, text, user, date, extra,
                                                changes)
 
     def _dirstatestatus(self, match, ignored=False, clean=False, unknown=False):
         """Return matched files only in ``self._status``
 
         Uncommitted files appear "clean" via this context, even if
         they aren't actually so in the working directory.
         """
         if clean:
             clean = [f for f in self._manifest if f not in self._changedset]
         else:
             clean = []
         return scmutil.status([f for f in self._status.modified if match(f)],
                               [f for f in self._status.added if match(f)],
                               [f for f in self._status.removed if match(f)],
                               [], [], [], clean)
 
     @propertycache
     def _changedset(self):
         """Return the set of files changed in this context
         """
         changed = set(self._status.modified)
         changed.update(self._status.added)
         changed.update(self._status.removed)
         return changed
 
 def makecachingfilectxfn(func):
     """Create a filectxfn that caches based on the path.
 
     We can't use util.cachefunc because it uses all arguments as the cache
     key and this creates a cycle since the arguments include the repo and
     memctx.
     """
     cache = {}
 
     def getfilectx(repo, memctx, path):
         if path not in cache:
             cache[path] = func(repo, memctx, path)
         return cache[path]
 
     return getfilectx
 
 def memfilefromctx(ctx):
     """Given a context return a memfilectx for ctx[path]
 
     This is a convenience method for building a memctx based on another
     context.
     """
     def getfilectx(repo, memctx, path):
         fctx = ctx[path]
         copysource = fctx.copysource()
         return memfilectx(repo, memctx, path, fctx.data(),
                           islink=fctx.islink(), isexec=fctx.isexec(),
                           copysource=copysource)
 
     return getfilectx
 
 def memfilefrompatch(patchstore):
     """Given a patch (e.g. patchstore object) return a memfilectx
 
     This is a convenience method for building a memctx based on a patchstore.
     """
     def getfilectx(repo, memctx, path):
         data, mode, copysource = patchstore.getfile(path)
         if data is None:
             return None
         islink, isexec = mode
         return memfilectx(repo, memctx, path, data, islink=islink,
                           isexec=isexec, copysource=copysource)
 
     return getfilectx
 
 class memctx(committablectx):
     """Use memctx to perform in-memory commits via localrepo.commitctx().
 
     Revision information is supplied at initialization time while
     related files data and is made available through a callback
     mechanism.  'repo' is the current localrepo, 'parents' is a
     sequence of two parent revisions identifiers (pass None for every
     missing parent), 'text' is the commit message and 'files' lists
     names of files touched by the revision (normalized and relative to
     repository root).
 
     filectxfn(repo, memctx, path) is a callable receiving the
     repository, the current memctx object and the normalized path of
     requested file, relative to repository root. It is fired by the
     commit function for every file in 'files', but calls order is
     undefined. If the file is available in the revision being
     committed (updated or added), filectxfn returns a memfilectx
     object. If the file was removed, filectxfn return None for recent
     Mercurial. Moved files are represented by marking the source file
     removed and the new file added with copy information (see
     memfilectx).
 
     user receives the committer name and defaults to current
     repository username, date is the commit date in any format
     supported by dateutil.parsedate() and defaults to current date, extra
     is a dictionary of metadata or is left empty.
     """
 
     # Mercurial <= 3.1 expects the filectxfn to raise IOError for missing files.
     # Extensions that need to retain compatibility across Mercurial 3.1 can use
     # this field to determine what to do in filectxfn.
     _returnnoneformissingfiles = True
 
     def __init__(self, repo, parents, text, files, filectxfn, user=None,
                  date=None, extra=None, branch=None, editor=False):
         super(memctx, self).__init__(repo, text, user, date, extra,
                                      branch=branch)
         self._rev = None
         self._node = None
         parents = [(p or nullid) for p in parents]
         p1, p2 = parents
         self._parents = [self._repo[p] for p in (p1, p2)]
         files = sorted(set(files))
         self._files = files
         self.substate = {}
 
         if isinstance(filectxfn, patch.filestore):
             filectxfn = memfilefrompatch(filectxfn)
         elif not callable(filectxfn):
             # if store is not callable, wrap it in a function
             filectxfn = memfilefromctx(filectxfn)
 
         # memoizing increases performance for e.g. vcs convert scenarios.
         self._filectxfn = makecachingfilectxfn(filectxfn)
 
         if editor:
             self._text = editor(self._repo, self, [])
             self._repo.savecommitmessage(self._text)
 
     def filectx(self, path, filelog=None):
         """get a file context from the working directory
 
         Returns None if file doesn't exist and should be removed."""
         return self._filectxfn(self._repo, self, path)
 
     def commit(self):
         """commit context to the repo"""
         return self._repo.commitctx(self)
 
     @propertycache
     def _manifest(self):
         """generate a manifest based on the return values of filectxfn"""
 
         # keep this simple for now; just worry about p1
         pctx = self._parents[0]
         man = pctx.manifest().copy()
 
         for f in self._status.modified:
             man[f] = modifiednodeid
 
         for f in self._status.added:
             man[f] = addednodeid
 
         for f in self._status.removed:
             if f in man:
                 del man[f]
 
         return man
 
     @propertycache
     def _status(self):
         """Calculate exact status from ``files`` specified at construction
         """
         man1 = self.p1().manifest()
         p2 = self._parents[1]
         # "1 < len(self._parents)" can't be used for checking
         # existence of the 2nd parent, because "memctx._parents" is
         # explicitly initialized by the list, of which length is 2.
         if p2.node() != nullid:
             man2 = p2.manifest()
             managing = lambda f: f in man1 or f in man2
         else:
             managing = lambda f: f in man1
 
         modified, added, removed = [], [], []
         for f in self._files:
             if not managing(f):
                 added.append(f)
             elif self[f]:
                 modified.append(f)
             else:
                 removed.append(f)
 
         return scmutil.status(modified, added, removed, [], [], [], [])
 
 class memfilectx(committablefilectx):
     """memfilectx represents an in-memory file to commit.
 
     See memctx and committablefilectx for more details.
     """
     def __init__(self, repo, changectx, path, data, islink=False,
                  isexec=False, copysource=None):
         """
         path is the normalized file path relative to repository root.
         data is the file content as a string.
         islink is True if the file is a symbolic link.
         isexec is True if the file is executable.
         copied is the source file path if current file was copied in the
         revision being committed, or None."""
         super(memfilectx, self).__init__(repo, path, None, changectx)
         self._data = data
         if islink:
             self._flags = 'l'
         elif isexec:
             self._flags = 'x'
         else:
             self._flags = ''
         self._copysource = copysource
 
     def copysource(self):
         return self._copysource
 
     def cmp(self, fctx):
         return self.data() != fctx.data()
 
     def data(self):
         return self._data
 
     def remove(self, ignoremissing=False):
         """wraps unlink for a repo's working directory"""
         # need to figure out what to do here
         del self._changectx[self._path]
 
     def write(self, data, flags, **kwargs):
         """wraps repo.wwrite"""
         self._data = data
 
 
 class metadataonlyctx(committablectx):
     """Like memctx but it's reusing the manifest of different commit.
     Intended to be used by lightweight operations that are creating
     metadata-only changes.
 
     Revision information is supplied at initialization time.  'repo' is the
     current localrepo, 'ctx' is original revision which manifest we're reuisng
     'parents' is a sequence of two parent revisions identifiers (pass None for
     every missing parent), 'text' is the commit.
 
     user receives the committer name and defaults to current repository
     username, date is the commit date in any format supported by
     dateutil.parsedate() and defaults to current date, extra is a dictionary of
     metadata or is left empty.
     """
     def __init__(self, repo, originalctx, parents=None, text=None, user=None,
                  date=None, extra=None, editor=False):
         if text is None:
             text = originalctx.description()
         super(metadataonlyctx, self).__init__(repo, text, user, date, extra)
         self._rev = None
         self._node = None
         self._originalctx = originalctx
         self._manifestnode = originalctx.manifestnode()
         if parents is None:
             parents = originalctx.parents()
         else:
             parents = [repo[p] for p in parents if p is not None]
         parents = parents[:]
         while len(parents) < 2:
             parents.append(repo[nullid])
         p1, p2 = self._parents = parents
 
         # sanity check to ensure that the reused manifest parents are
         # manifests of our commit parents
         mp1, mp2 = self.manifestctx().parents
         if p1 != nullid and p1.manifestnode() != mp1:
             raise RuntimeError(r"can't reuse the manifest: its p1 "
                                r"doesn't match the new ctx p1")
         if p2 != nullid and p2.manifestnode() != mp2:
             raise RuntimeError(r"can't reuse the manifest: "
                                r"its p2 doesn't match the new ctx p2")
 
         self._files = originalctx.files()
         self.substate = {}
 
         if editor:
             self._text = editor(self._repo, self, [])
             self._repo.savecommitmessage(self._text)
 
     def manifestnode(self):
         return self._manifestnode
 
     @property
     def _manifestctx(self):
         return self._repo.manifestlog[self._manifestnode]
 
     def filectx(self, path, filelog=None):
         return self._originalctx.filectx(path, filelog=filelog)
 
     def commit(self):
         """commit context to the repo"""
         return self._repo.commitctx(self)
 
     @property
     def _manifest(self):
         return self._originalctx.manifest()
 
     @propertycache
     def _status(self):
         """Calculate exact status from ``files`` specified in the ``origctx``
         and parents manifests.
         """
         man1 = self.p1().manifest()
         p2 = self._parents[1]
         # "1 < len(self._parents)" can't be used for checking
         # existence of the 2nd parent, because "metadataonlyctx._parents" is
         # explicitly initialized by the list, of which length is 2.
         if p2.node() != nullid:
             man2 = p2.manifest()
             managing = lambda f: f in man1 or f in man2
         else:
             managing = lambda f: f in man1
 
         modified, added, removed = [], [], []
         for f in self._files:
             if not managing(f):
                 added.append(f)
             elif f in self:
                 modified.append(f)
             else:
                 removed.append(f)
 
         return scmutil.status(modified, added, removed, [], [], [], [])
 
 class arbitraryfilectx(object):
     """Allows you to use filectx-like functions on a file in an arbitrary
     location on disk, possibly not in the working directory.
     """
     def __init__(self, path, repo=None):
         # Repo is optional because contrib/simplemerge uses this class.
         self._repo = repo
         self._path = path
 
     def cmp(self, fctx):
         # filecmp follows symlinks whereas `cmp` should not, so skip the fast
         # path if either side is a symlink.
         symlinks = ('l' in self.flags() or 'l' in fctx.flags())
         if not symlinks and isinstance(fctx, workingfilectx) and self._repo:
             # Add a fast-path for merge if both sides are disk-backed.
             # Note that filecmp uses the opposite return values (True if same)
             # from our cmp functions (True if different).
             return not filecmp.cmp(self.path(), self._repo.wjoin(fctx.path()))
         return self.data() != fctx.data()
 
     def path(self):
         return self._path
 
     def flags(self):
         return ''
 
     def data(self):
         return util.readfile(self._path)
 
     def decodeddata(self):
         with open(self._path, "rb") as f:
             return f.read()
 
     def remove(self):
         util.unlink(self._path)
 
     def write(self, data, flags, **kwargs):
         assert not flags
         with open(self._path, "wb") as f:
             f.write(data)

mercurial/dirstate.py

@@ -1,1506 +1,1518
 # dirstate.py - working directory tracking for mercurial
 #
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import collections
 import contextlib
 import errno
 import os
 import stat
 
 from .i18n import _
 from .node import nullid
 from . import (
     encoding,
     error,
     match as matchmod,
     pathutil,
     policy,
     pycompat,
     scmutil,
     txnutil,
     util,
 )
 
 parsers = policy.importmod(r'parsers')
 dirstatemod = policy.importrust(r'dirstate', default=parsers)
 
 propertycache = util.propertycache
 filecache = scmutil.filecache
 _rangemask = 0x7fffffff
 
 dirstatetuple = parsers.dirstatetuple
 
 class repocache(filecache):
     """filecache for files in .hg/"""
     def join(self, obj, fname):
         return obj._opener.join(fname)
 
 class rootcache(filecache):
     """filecache for files in the repository root"""
     def join(self, obj, fname):
         return obj._join(fname)
 
 def _getfsnow(vfs):
     '''Get "now" timestamp on filesystem'''
     tmpfd, tmpname = vfs.mkstemp()
     try:
         return os.fstat(tmpfd)[stat.ST_MTIME]
     finally:
         os.close(tmpfd)
         vfs.unlink(tmpname)
 
 class dirstate(object):
 
     def __init__(self, opener, ui, root, validate, sparsematchfn):
         '''Create a new dirstate object.
 
         opener is an open()-like callable that can be used to open the
         dirstate file; root is the root of the directory tracked by
         the dirstate.
         '''
         self._opener = opener
         self._validate = validate
         self._root = root
         self._sparsematchfn = sparsematchfn
         # ntpath.join(root, '') of Python 2.7.9 does not add sep if root is
         # UNC path pointing to root share (issue4557)
         self._rootdir = pathutil.normasprefix(root)
         self._dirty = False
         self._lastnormaltime = 0
         self._ui = ui
         self._filecache = {}
         self._parentwriters = 0
         self._filename = 'dirstate'
         self._pendingfilename = '%s.pending' % self._filename
         self._plchangecallbacks = {}
         self._origpl = None
         self._updatedfiles = set()
         self._mapcls = dirstatemap
         # Access and cache cwd early, so we don't access it for the first time
         # after a working-copy update caused it to not exist (accessing it then
         # raises an exception).
         self._cwd
 
     @contextlib.contextmanager
     def parentchange(self):
         '''Context manager for handling dirstate parents.
 
         If an exception occurs in the scope of the context manager,
         the incoherent dirstate won't be written when wlock is
         released.
         '''
         self._parentwriters += 1
         yield
         # Typically we want the "undo" step of a context manager in a
         # finally block so it happens even when an exception
         # occurs. In this case, however, we only want to decrement
         # parentwriters if the code in the with statement exits
         # normally, so we don't have a try/finally here on purpose.
         self._parentwriters -= 1
 
     def pendingparentchange(self):
         '''Returns true if the dirstate is in the middle of a set of changes
         that modify the dirstate parent.
         '''
         return self._parentwriters > 0
 
     @propertycache
     def _map(self):
         """Return the dirstate contents (see documentation for dirstatemap)."""
         self._map = self._mapcls(self._ui, self._opener, self._root)
         return self._map
 
     @property
     def _sparsematcher(self):
         """The matcher for the sparse checkout.
 
         The working directory may not include every file from a manifest. The
         matcher obtained by this property will match a path if it is to be
         included in the working directory.
         """
         # TODO there is potential to cache this property. For now, the matcher
         # is resolved on every access. (But the called function does use a
         # cache to keep the lookup fast.)
         return self._sparsematchfn()
 
     @repocache('branch')
     def _branch(self):
         try:
             return self._opener.read("branch").strip() or "default"
         except IOError as inst:
             if inst.errno != errno.ENOENT:
                 raise
             return "default"
 
     @property
     def _pl(self):
         return self._map.parents()
 
     def hasdir(self, d):
         return self._map.hastrackeddir(d)
 
     @rootcache('.hgignore')
     def _ignore(self):
         files = self._ignorefiles()
         if not files:
             return matchmod.never()
 
         pats = ['include:%s' % f for f in files]
         return matchmod.match(self._root, '', [], pats, warn=self._ui.warn)
 
     @propertycache
     def _slash(self):
         return self._ui.configbool('ui', 'slash') and pycompat.ossep != '/'
 
     @propertycache
     def _checklink(self):
         return util.checklink(self._root)
 
     @propertycache
     def _checkexec(self):
         return util.checkexec(self._root)
 
     @propertycache
     def _checkcase(self):
         return not util.fscasesensitive(self._join('.hg'))
 
     def _join(self, f):
         # much faster than os.path.join()
         # it's safe because f is always a relative path
         return self._rootdir + f
 
     def flagfunc(self, buildfallback):
         if self._checklink and self._checkexec:
             def f(x):
                 try:
                     st = os.lstat(self._join(x))
                     if util.statislink(st):
                         return 'l'
                     if util.statisexec(st):
                         return 'x'
                 except OSError:
                     pass
                 return ''
             return f
 
         fallback = buildfallback()
         if self._checklink:
             def f(x):
                 if os.path.islink(self._join(x)):
                     return 'l'
                 if 'x' in fallback(x):
                     return 'x'
                 return ''
             return f
         if self._checkexec:
             def f(x):
                 if 'l' in fallback(x):
                     return 'l'
                 if util.isexec(self._join(x)):
                     return 'x'
                 return ''
             return f
         else:
             return fallback
 
     @propertycache
     def _cwd(self):
         # internal config: ui.forcecwd
         forcecwd = self._ui.config('ui', 'forcecwd')
         if forcecwd:
             return forcecwd
         return encoding.getcwd()
 
     def getcwd(self):
         '''Return the path from which a canonical path is calculated.
 
         This path should be used to resolve file patterns or to convert
         canonical paths back to file paths for display. It shouldn't be
         used to get real file paths. Use vfs functions instead.
         '''
         cwd = self._cwd
         if cwd == self._root:
             return ''
         # self._root ends with a path separator if self._root is '/' or 'C:\'
         rootsep = self._root
         if not util.endswithsep(rootsep):
             rootsep += pycompat.ossep
         if cwd.startswith(rootsep):
             return cwd[len(rootsep):]
         else:
             # we're outside the repo. return an absolute path.
             return cwd
 
     def pathto(self, f, cwd=None):
         if cwd is None:
             cwd = self.getcwd()
         path = util.pathto(self._root, cwd, f)
         if self._slash:
             return util.pconvert(path)
         return path
 
     def __getitem__(self, key):
         '''Return the current state of key (a filename) in the dirstate.
 
         States are:
           n  normal
           m  needs merging
           r  marked for removal
           a  marked for addition
           ?  not tracked
         '''
         return self._map.get(key, ("?",))[0]
 
     def __contains__(self, key):
         return key in self._map
 
     def __iter__(self):
         return iter(sorted(self._map))
 
     def items(self):
         return self._map.iteritems()
 
     iteritems = items
 
     def parents(self):
         return [self._validate(p) for p in self._pl]
 
     def p1(self):
         return self._validate(self._pl[0])
 
     def p2(self):
         return self._validate(self._pl[1])
 
     def branch(self):
         return encoding.tolocal(self._branch)
 
     def setparents(self, p1, p2=nullid):
         """Set dirstate parents to p1 and p2.
 
         When moving from two parents to one, 'm' merged entries a
         adjusted to normal and previous copy records discarded and
         returned by the call.
 
         See localrepo.setparents()
         """
         if self._parentwriters == 0:
             raise ValueError("cannot set dirstate parent outside of "
                              "dirstate.parentchange context manager")
 
         self._dirty = True
         oldp2 = self._pl[1]
         if self._origpl is None:
             self._origpl = self._pl
         self._map.setparents(p1, p2)
         copies = {}
         if oldp2 != nullid and p2 == nullid:
             candidatefiles = self._map.nonnormalset.union(
                                 self._map.otherparentset)
             for f in candidatefiles:
                 s = self._map.get(f)
                 if s is None:
                     continue
 
                 # Discard 'm' markers when moving away from a merge state
                 if s[0] == 'm':
                     source = self._map.copymap.get(f)
                     if source:
                         copies[f] = source
                     self.normallookup(f)
                 # Also fix up otherparent markers
                 elif s[0] == 'n' and s[2] == -2:
                     source = self._map.copymap.get(f)
                     if source:
                         copies[f] = source
                     self.add(f)
         return copies
 
     def setbranch(self, branch):
         self.__class__._branch.set(self, encoding.fromlocal(branch))
         f = self._opener('branch', 'w', atomictemp=True, checkambig=True)
         try:
             f.write(self._branch + '\n')
             f.close()
 
             # make sure filecache has the correct stat info for _branch after
             # replacing the underlying file
             ce = self._filecache['_branch']
             if ce:
                 ce.refresh()
         except: # re-raises
             f.discard()
             raise
 
     def invalidate(self):
         '''Causes the next access to reread the dirstate.
 
         This is different from localrepo.invalidatedirstate() because it always
         rereads the dirstate. Use localrepo.invalidatedirstate() if you want to
         check whether the dirstate has changed before rereading it.'''
 
         for a in (r"_map", r"_branch", r"_ignore"):
             if a in self.__dict__:
                 delattr(self, a)
         self._lastnormaltime = 0
         self._dirty = False
         self._updatedfiles.clear()
         self._parentwriters = 0
         self._origpl = None
 
     def copy(self, source, dest):
         """Mark dest as a copy of source. Unmark dest if source is None."""
         if source == dest:
             return
         self._dirty = True
         if source is not None:
             self._map.copymap[dest] = source
             self._updatedfiles.add(source)
             self._updatedfiles.add(dest)
         elif self._map.copymap.pop(dest, None):
             self._updatedfiles.add(dest)
 
     def copied(self, file):
         return self._map.copymap.get(file, None)
 
     def copies(self):
         return self._map.copymap
 
     def _addpath(self, f, state, mode, size, mtime):
         oldstate = self[f]
         if state == 'a' or oldstate == 'r':
             scmutil.checkfilename(f)
             if self._map.hastrackeddir(f):
                 raise error.Abort(_('directory %r already in dirstate') %
                                   pycompat.bytestr(f))
             # shadows
             for d in util.finddirs(f):
                 if self._map.hastrackeddir(d):
                     break
                 entry = self._map.get(d)
                 if entry is not None and entry[0] != 'r':
                     raise error.Abort(
                         _('file %r in dirstate clashes with %r') %
                         (pycompat.bytestr(d), pycompat.bytestr(f)))
         self._dirty = True
         self._updatedfiles.add(f)
         self._map.addfile(f, oldstate, state, mode, size, mtime)
 
-    def normal(self, f):
-        '''Mark a file normal and clean.'''
+    def normal(self, f, parentfiledata=None):
+        '''Mark a file normal and clean.
+
+        parentfiledata: (mode, size, mtime) of the clean file
+
+        parentfiledata should be computed from memory (for mode,
+        size), as or close as possible from the point where we
+        determined the file was clean, to limit the risk of the
+        file having been changed by an external process between the
+        moment where the file was determined to be clean and now.'''
+        if parentfiledata:
+            (mode, size, mtime) = parentfiledata
+        else:
         s = os.lstat(self._join(f))
+            mode = s.st_mode
+            size = s.st_size
         mtime = s[stat.ST_MTIME]
-        self._addpath(f, 'n', s.st_mode,
-                      s.st_size & _rangemask, mtime & _rangemask)
+        self._addpath(f, 'n', mode, size & _rangemask, mtime & _rangemask)
         self._map.copymap.pop(f, None)
         if f in self._map.nonnormalset:
             self._map.nonnormalset.remove(f)
         if mtime > self._lastnormaltime:
             # Remember the most recent modification timeslot for status(),
             # to make sure we won't miss future size-preserving file content
             # modifications that happen within the same timeslot.
             self._lastnormaltime = mtime
 
     def normallookup(self, f):
         '''Mark a file normal, but possibly dirty.'''
         if self._pl[1] != nullid:
             # if there is a merge going on and the file was either
             # in state 'm' (-1) or coming from other parent (-2) before
             # being removed, restore that state.
             entry = self._map.get(f)
             if entry is not None:
                 if entry[0] == 'r' and entry[2] in (-1, -2):
                     source = self._map.copymap.get(f)
                     if entry[2] == -1:
                         self.merge(f)
                     elif entry[2] == -2:
                         self.otherparent(f)
                     if source:
                         self.copy(source, f)
                     return
                 if entry[0] == 'm' or entry[0] == 'n' and entry[2] == -2:
                     return
         self._addpath(f, 'n', 0, -1, -1)
         self._map.copymap.pop(f, None)
 
     def otherparent(self, f):
         '''Mark as coming from the other parent, always dirty.'''
         if self._pl[1] == nullid:
             raise error.Abort(_("setting %r to other parent "
                                "only allowed in merges") % f)
         if f in self and self[f] == 'n':
             # merge-like
             self._addpath(f, 'm', 0, -2, -1)
         else:
             # add-like
             self._addpath(f, 'n', 0, -2, -1)
         self._map.copymap.pop(f, None)
 
     def add(self, f):
         '''Mark a file added.'''
         self._addpath(f, 'a', 0, -1, -1)
         self._map.copymap.pop(f, None)
 
     def remove(self, f):
         '''Mark a file removed.'''
         self._dirty = True
         oldstate = self[f]
         size = 0
         if self._pl[1] != nullid:
             entry = self._map.get(f)
             if entry is not None:
                 # backup the previous state
                 if entry[0] == 'm': # merge
                     size = -1
                 elif entry[0] == 'n' and entry[2] == -2: # other parent
                     size = -2
                     self._map.otherparentset.add(f)
         self._updatedfiles.add(f)
         self._map.removefile(f, oldstate, size)
         if size == 0:
             self._map.copymap.pop(f, None)
 
     def merge(self, f):
         '''Mark a file merged.'''
         if self._pl[1] == nullid:
             return self.normallookup(f)
         return self.otherparent(f)
 
     def drop(self, f):
         '''Drop a file from the dirstate'''
         oldstate = self[f]
         if self._map.dropfile(f, oldstate):
             self._dirty = True
             self._updatedfiles.add(f)
             self._map.copymap.pop(f, None)
 
     def _discoverpath(self, path, normed, ignoremissing, exists, storemap):
         if exists is None:
             exists = os.path.lexists(os.path.join(self._root, path))
         if not exists:
             # Maybe a path component exists
             if not ignoremissing and '/' in path:
                 d, f = path.rsplit('/', 1)
                 d = self._normalize(d, False, ignoremissing, None)
                 folded = d + "/" + f
             else:
                 # No path components, preserve original case
                 folded = path
         else:
             # recursively normalize leading directory components
             # against dirstate
             if '/' in normed:
                 d, f = normed.rsplit('/', 1)
                 d = self._normalize(d, False, ignoremissing, True)
                 r = self._root + "/" + d
                 folded = d + "/" + util.fspath(f, r)
             else:
                 folded = util.fspath(normed, self._root)
             storemap[normed] = folded
 
         return folded
 
     def _normalizefile(self, path, isknown, ignoremissing=False, exists=None):
         normed = util.normcase(path)
         folded = self._map.filefoldmap.get(normed, None)
         if folded is None:
             if isknown:
                 folded = path
             else:
                 folded = self._discoverpath(path, normed, ignoremissing, exists,
                                             self._map.filefoldmap)
         return folded
 
     def _normalize(self, path, isknown, ignoremissing=False, exists=None):
         normed = util.normcase(path)
         folded = self._map.filefoldmap.get(normed, None)
         if folded is None:
             folded = self._map.dirfoldmap.get(normed, None)
         if folded is None:
             if isknown:
                 folded = path
             else:
                 # store discovered result in dirfoldmap so that future
                 # normalizefile calls don't start matching directories
                 folded = self._discoverpath(path, normed, ignoremissing, exists,
                                             self._map.dirfoldmap)
         return folded
 
     def normalize(self, path, isknown=False, ignoremissing=False):
         '''
         normalize the case of a pathname when on a casefolding filesystem
 
         isknown specifies whether the filename came from walking the
         disk, to avoid extra filesystem access.
 
         If ignoremissing is True, missing path are returned
         unchanged. Otherwise, we try harder to normalize possibly
         existing path components.
 
         The normalized case is determined based on the following precedence:
 
         - version of name already stored in the dirstate
         - version of name stored on disk
         - version provided via command arguments
         '''
 
         if self._checkcase:
             return self._normalize(path, isknown, ignoremissing)
         return path
 
     def clear(self):
         self._map.clear()
         self._lastnormaltime = 0
         self._updatedfiles.clear()
         self._dirty = True
 
     def rebuild(self, parent, allfiles, changedfiles=None):
         if changedfiles is None:
             # Rebuild entire dirstate
             changedfiles = allfiles
             lastnormaltime = self._lastnormaltime
             self.clear()
             self._lastnormaltime = lastnormaltime
 
         if self._origpl is None:
             self._origpl = self._pl
         self._map.setparents(parent, nullid)
         for f in changedfiles:
             if f in allfiles:
                 self.normallookup(f)
             else:
                 self.drop(f)
 
         self._dirty = True
 
     def identity(self):
         '''Return identity of dirstate itself to detect changing in storage
 
         If identity of previous dirstate is equal to this, writing
         changes based on the former dirstate out can keep consistency.
         '''
         return self._map.identity
 
     def write(self, tr):
         if not self._dirty:
             return
 
         filename = self._filename
         if tr:
             # 'dirstate.write()' is not only for writing in-memory
             # changes out, but also for dropping ambiguous timestamp.
             # delayed writing re-raise "ambiguous timestamp issue".
             # See also the wiki page below for detail:
             # https://www.mercurial-scm.org/wiki/DirstateTransactionPlan
 
             # emulate dropping timestamp in 'parsers.pack_dirstate'
             now = _getfsnow(self._opener)
             self._map.clearambiguoustimes(self._updatedfiles, now)
 
             # emulate that all 'dirstate.normal' results are written out
             self._lastnormaltime = 0
             self._updatedfiles.clear()
 
             # delay writing in-memory changes out
             tr.addfilegenerator('dirstate', (self._filename,),
                                 self._writedirstate, location='plain')
             return
 
         st = self._opener(filename, "w", atomictemp=True, checkambig=True)
         self._writedirstate(st)
 
     def addparentchangecallback(self, category, callback):
         """add a callback to be called when the wd parents are changed
 
         Callback will be called with the following arguments:
             dirstate, (oldp1, oldp2), (newp1, newp2)
 
         Category is a unique identifier to allow overwriting an old callback
         with a newer callback.
         """
         self._plchangecallbacks[category] = callback
 
     def _writedirstate(self, st):
         # notify callbacks about parents change
         if self._origpl is not None and self._origpl != self._pl:
             for c, callback in sorted(self._plchangecallbacks.iteritems()):
                 callback(self, self._origpl, self._pl)
             self._origpl = None
         # use the modification time of the newly created temporary file as the
         # filesystem's notion of 'now'
         now = util.fstat(st)[stat.ST_MTIME] & _rangemask
 
         # enough 'delaywrite' prevents 'pack_dirstate' from dropping
         # timestamp of each entries in dirstate, because of 'now > mtime'
         delaywrite = self._ui.configint('debug', 'dirstate.delaywrite')
         if delaywrite > 0:
             # do we have any files to delay for?
             for f, e in self._map.iteritems():
                 if e[0] == 'n' and e[3] == now:
                     import time # to avoid useless import
                     # rather than sleep n seconds, sleep until the next
                     # multiple of n seconds
                     clock = time.time()
                     start = int(clock) - (int(clock) % delaywrite)
                     end = start + delaywrite
                     time.sleep(end - clock)
                     now = end # trust our estimate that the end is near now
                     break
 
         self._map.write(st, now)
         self._lastnormaltime = 0
         self._dirty = False
 
     def _dirignore(self, f):
         if self._ignore(f):
             return True
         for p in util.finddirs(f):
             if self._ignore(p):
                 return True
         return False
 
     def _ignorefiles(self):
         files = []
         if os.path.exists(self._join('.hgignore')):
             files.append(self._join('.hgignore'))
         for name, path in self._ui.configitems("ui"):
             if name == 'ignore' or name.startswith('ignore.'):
                 # we need to use os.path.join here rather than self._join
                 # because path is arbitrary and user-specified
                 files.append(os.path.join(self._rootdir, util.expandpath(path)))
         return files
 
     def _ignorefileandline(self, f):
         files = collections.deque(self._ignorefiles())
         visited = set()
         while files:
             i = files.popleft()
             patterns = matchmod.readpatternfile(i, self._ui.warn,
                                                 sourceinfo=True)
             for pattern, lineno, line in patterns:
                 kind, p = matchmod._patsplit(pattern, 'glob')
                 if kind == "subinclude":
                     if p not in visited:
                         files.append(p)
                     continue
                 m = matchmod.match(self._root, '', [], [pattern],
                                    warn=self._ui.warn)
                 if m(f):
                     return (i, lineno, line)
             visited.add(i)
         return (None, -1, "")
 
     def _walkexplicit(self, match, subrepos):
         '''Get stat data about the files explicitly specified by match.
 
         Return a triple (results, dirsfound, dirsnotfound).
         - results is a mapping from filename to stat result. It also contains
           listings mapping subrepos and .hg to None.
         - dirsfound is a list of files found to be directories.
         - dirsnotfound is a list of files that the dirstate thinks are
           directories and that were not found.'''
 
         def badtype(mode):
             kind = _('unknown')
             if stat.S_ISCHR(mode):
                 kind = _('character device')
             elif stat.S_ISBLK(mode):
                 kind = _('block device')
             elif stat.S_ISFIFO(mode):
                 kind = _('fifo')
             elif stat.S_ISSOCK(mode):
                 kind = _('socket')
             elif stat.S_ISDIR(mode):
                 kind = _('directory')
             return _('unsupported file type (type is %s)') % kind
 
         matchedir = match.explicitdir
         badfn = match.bad
         dmap = self._map
         lstat = os.lstat
         getkind = stat.S_IFMT
         dirkind = stat.S_IFDIR
         regkind = stat.S_IFREG
         lnkkind = stat.S_IFLNK
         join = self._join
         dirsfound = []
         foundadd = dirsfound.append
         dirsnotfound = []
         notfoundadd = dirsnotfound.append
 
         if not match.isexact() and self._checkcase:
             normalize = self._normalize
         else:
             normalize = None
 
         files = sorted(match.files())
         subrepos.sort()
         i, j = 0, 0
         while i < len(files) and j < len(subrepos):
             subpath = subrepos[j] + "/"
             if files[i] < subpath:
                 i += 1
                 continue
             while i < len(files) and files[i].startswith(subpath):
                 del files[i]
             j += 1
 
         if not files or '' in files:
             files = ['']
             # constructing the foldmap is expensive, so don't do it for the
             # common case where files is ['']
             normalize = None
         results = dict.fromkeys(subrepos)
         results['.hg'] = None
 
         for ff in files:
             if normalize:
                 nf = normalize(ff, False, True)
             else:
                 nf = ff
             if nf in results:
                 continue
 
             try:
                 st = lstat(join(nf))
                 kind = getkind(st.st_mode)
                 if kind == dirkind:
                     if nf in dmap:
                         # file replaced by dir on disk but still in dirstate
                         results[nf] = None
                     if matchedir:
                         matchedir(nf)
                     foundadd((nf, ff))
                 elif kind == regkind or kind == lnkkind:
                     results[nf] = st
                 else:
                     badfn(ff, badtype(kind))
                     if nf in dmap:
                         results[nf] = None
             except OSError as inst: # nf not found on disk - it is dirstate only
                 if nf in dmap: # does it exactly match a missing file?
                     results[nf] = None
                 else: # does it match a missing directory?
                     if self._map.hasdir(nf):
                         if matchedir:
                             matchedir(nf)
                         notfoundadd(nf)
                     else:
                         badfn(ff, encoding.strtolocal(inst.strerror))
 
         # match.files() may contain explicitly-specified paths that shouldn't
         # be taken; drop them from the list of files found. dirsfound/notfound
         # aren't filtered here because they will be tested later.
         if match.anypats():
             for f in list(results):
                 if f == '.hg' or f in subrepos:
                     # keep sentinel to disable further out-of-repo walks
                     continue
                 if not match(f):
                     del results[f]
 
         # Case insensitive filesystems cannot rely on lstat() failing to detect
         # a case-only rename.  Prune the stat object for any file that does not
         # match the case in the filesystem, if there are multiple files that
         # normalize to the same path.
         if match.isexact() and self._checkcase:
             normed = {}
 
             for f, st in results.iteritems():
                 if st is None:
                     continue
 
                 nc = util.normcase(f)
                 paths = normed.get(nc)
 
                 if paths is None:
                     paths = set()
                     normed[nc] = paths
 
                 paths.add(f)
 
             for norm, paths in normed.iteritems():
                 if len(paths) > 1:
                     for path in paths:
                         folded = self._discoverpath(path, norm, True, None,
                                                     self._map.dirfoldmap)
                         if path != folded:
                             results[path] = None
 
         return results, dirsfound, dirsnotfound
 
     def walk(self, match, subrepos, unknown, ignored, full=True):
         '''
         Walk recursively through the directory tree, finding all files
         matched by match.
 
         If full is False, maybe skip some known-clean files.
 
         Return a dict mapping filename to stat-like object (either
         mercurial.osutil.stat instance or return value of os.stat()).
 
         '''
         # full is a flag that extensions that hook into walk can use -- this
         # implementation doesn't use it at all. This satisfies the contract
         # because we only guarantee a "maybe".
 
         if ignored:
             ignore = util.never
             dirignore = util.never
         elif unknown:
             ignore = self._ignore
             dirignore = self._dirignore
         else:
             # if not unknown and not ignored, drop dir recursion and step 2
             ignore = util.always
             dirignore = util.always
 
         matchfn = match.matchfn
         matchalways = match.always()
         matchtdir = match.traversedir
         dmap = self._map
         listdir = util.listdir
         lstat = os.lstat
         dirkind = stat.S_IFDIR
         regkind = stat.S_IFREG
         lnkkind = stat.S_IFLNK
         join = self._join
 
         exact = skipstep3 = False
         if match.isexact(): # match.exact
             exact = True
             dirignore = util.always # skip step 2
         elif match.prefix(): # match.match, no patterns
             skipstep3 = True
 
         if not exact and self._checkcase:
             normalize = self._normalize
             normalizefile = self._normalizefile
             skipstep3 = False
         else:
             normalize = self._normalize
             normalizefile = None
 
         # step 1: find all explicit files
         results, work, dirsnotfound = self._walkexplicit(match, subrepos)
 
         skipstep3 = skipstep3 and not (work or dirsnotfound)
         work = [d for d in work if not dirignore(d[0])]
 
         # step 2: visit subdirectories
         def traverse(work, alreadynormed):
             wadd = work.append
             while work:
                 nd = work.pop()
                 visitentries = match.visitchildrenset(nd)
                 if not visitentries:
                     continue
                 if visitentries == 'this' or visitentries == 'all':
                     visitentries = None
                 skip = None
                 if nd != '':
                     skip = '.hg'
                 try:
                     entries = listdir(join(nd), stat=True, skip=skip)
                 except OSError as inst:
                     if inst.errno in (errno.EACCES, errno.ENOENT):
                         match.bad(self.pathto(nd),
                                   encoding.strtolocal(inst.strerror))
                         continue
                     raise
                 for f, kind, st in entries:
                     # Some matchers may return files in the visitentries set,
                     # instead of 'this', if the matcher explicitly mentions them
                     # and is not an exactmatcher. This is acceptable; we do not
                     # make any hard assumptions about file-or-directory below
                     # based on the presence of `f` in visitentries. If
                     # visitchildrenset returned a set, we can always skip the
                     # entries *not* in the set it provided regardless of whether
                     # they're actually a file or a directory.
                     if visitentries and f not in visitentries:
                         continue
                     if normalizefile:
                         # even though f might be a directory, we're only
                         # interested in comparing it to files currently in the
                         # dmap -- therefore normalizefile is enough
                         nf = normalizefile(nd and (nd + "/" + f) or f, True,
                                            True)
                     else:
                         nf = nd and (nd + "/" + f) or f
                     if nf not in results:
                         if kind == dirkind:
                             if not ignore(nf):
                                 if matchtdir:
                                     matchtdir(nf)
                                 wadd(nf)
                             if nf in dmap and (matchalways or matchfn(nf)):
                                 results[nf] = None
                         elif kind == regkind or kind == lnkkind:
                             if nf in dmap:
                                 if matchalways or matchfn(nf):
                                     results[nf] = st
                             elif ((matchalways or matchfn(nf))
                                   and not ignore(nf)):
                                 # unknown file -- normalize if necessary
                                 if not alreadynormed:
                                     nf = normalize(nf, False, True)
                                 results[nf] = st
                         elif nf in dmap and (matchalways or matchfn(nf)):
                             results[nf] = None
 
         for nd, d in work:
             # alreadynormed means that processwork doesn't have to do any
             # expensive directory normalization
             alreadynormed = not normalize or nd == d
             traverse([d], alreadynormed)
 
         for s in subrepos:
             del results[s]
         del results['.hg']
 
         # step 3: visit remaining files from dmap
         if not skipstep3 and not exact:
             # If a dmap file is not in results yet, it was either
             # a) not matching matchfn b) ignored, c) missing, or d) under a
             # symlink directory.
             if not results and matchalways:
                 visit = [f for f in dmap]
             else:
                 visit = [f for f in dmap if f not in results and matchfn(f)]
             visit.sort()
 
             if unknown:
                 # unknown == True means we walked all dirs under the roots
                 # that wasn't ignored, and everything that matched was stat'ed
                 # and is already in results.
                 # The rest must thus be ignored or under a symlink.
                 audit_path = pathutil.pathauditor(self._root, cached=True)
 
                 for nf in iter(visit):
                     # If a stat for the same file was already added with a
                     # different case, don't add one for this, since that would
                     # make it appear as if the file exists under both names
                     # on disk.
                     if (normalizefile and
                         normalizefile(nf, True, True) in results):
                         results[nf] = None
                     # Report ignored items in the dmap as long as they are not
                     # under a symlink directory.
                     elif audit_path.check(nf):
                         try:
                             results[nf] = lstat(join(nf))
                             # file was just ignored, no links, and exists
                         except OSError:
                             # file doesn't exist
                             results[nf] = None
                     else:
                         # It's either missing or under a symlink directory
                         # which we in this case report as missing
                         results[nf] = None
             else:
                 # We may not have walked the full directory tree above,
                 # so stat and check everything we missed.
                 iv = iter(visit)
                 for st in util.statfiles([join(i) for i in visit]):
                     results[next(iv)] = st
         return results
 
     def status(self, match, subrepos, ignored, clean, unknown):
         '''Determine the status of the working copy relative to the
         dirstate and return a pair of (unsure, status), where status is of type
         scmutil.status and:
 
           unsure:
             files that might have been modified since the dirstate was
             written, but need to be read to be sure (size is the same
             but mtime differs)
           status.modified:
             files that have definitely been modified since the dirstate
             was written (different size or mode)
           status.clean:
             files that have definitely not been modified since the
             dirstate was written
         '''
         listignored, listclean, listunknown = ignored, clean, unknown
         lookup, modified, added, unknown, ignored = [], [], [], [], []
         removed, deleted, clean = [], [], []
 
         dmap = self._map
         dmap.preload()
         dcontains = dmap.__contains__
         dget = dmap.__getitem__
         ladd = lookup.append            # aka "unsure"
         madd = modified.append
         aadd = added.append
         uadd = unknown.append
         iadd = ignored.append
         radd = removed.append
         dadd = deleted.append
         cadd = clean.append
         mexact = match.exact
         dirignore = self._dirignore
         checkexec = self._checkexec
         copymap = self._map.copymap
         lastnormaltime = self._lastnormaltime
 
         # We need to do full walks when either
         # - we're listing all clean files, or
         # - match.traversedir does something, because match.traversedir should
         #   be called for every dir in the working dir
         full = listclean or match.traversedir is not None
         for fn, st in self.walk(match, subrepos, listunknown, listignored,
                                 full=full).iteritems():
             if not dcontains(fn):
                 if (listignored or mexact(fn)) and dirignore(fn):
                     if listignored:
                         iadd(fn)
                 else:
                     uadd(fn)
                 continue
 
             # This is equivalent to 'state, mode, size, time = dmap[fn]' but not
             # written like that for performance reasons. dmap[fn] is not a
             # Python tuple in compiled builds. The CPython UNPACK_SEQUENCE
             # opcode has fast paths when the value to be unpacked is a tuple or
             # a list, but falls back to creating a full-fledged iterator in
             # general. That is much slower than simply accessing and storing the
             # tuple members one by one.
             t = dget(fn)
             state = t[0]
             mode = t[1]
             size = t[2]
             time = t[3]
 
             if not st and state in "nma":
                 dadd(fn)
             elif state == 'n':
                 if (size >= 0 and
                     ((size != st.st_size and size != st.st_size & _rangemask)
                      or ((mode ^ st.st_mode) & 0o100 and checkexec))
                     or size == -2 # other parent
                     or fn in copymap):
                     madd(fn)
                 elif (time != st[stat.ST_MTIME]
                       and time != st[stat.ST_MTIME] & _rangemask):
                     ladd(fn)
                 elif st[stat.ST_MTIME] == lastnormaltime:
                     # fn may have just been marked as normal and it may have
                     # changed in the same second without changing its size.
                     # This can happen if we quickly do multiple commits.
                     # Force lookup, so we don't miss such a racy file change.
                     ladd(fn)
                 elif listclean:
                     cadd(fn)
             elif state == 'm':
                 madd(fn)
             elif state == 'a':
                 aadd(fn)
             elif state == 'r':
                 radd(fn)
 
         return (lookup, scmutil.status(modified, added, removed, deleted,
                                        unknown, ignored, clean))
 
     def matches(self, match):
         '''
         return files in the dirstate (in whatever state) filtered by match
         '''
         dmap = self._map
         if match.always():
             return dmap.keys()
         files = match.files()
         if match.isexact():
             # fast path -- filter the other way around, since typically files is
             # much smaller than dmap
             return [f for f in files if f in dmap]
         if match.prefix() and all(fn in dmap for fn in files):
             # fast path -- all the values are known to be files, so just return
             # that
             return list(files)
         return [f for f in dmap if match(f)]
 
     def _actualfilename(self, tr):
         if tr:
             return self._pendingfilename
         else:
             return self._filename
 
     def savebackup(self, tr, backupname):
         '''Save current dirstate into backup file'''
         filename = self._actualfilename(tr)
         assert backupname != filename
 
         # use '_writedirstate' instead of 'write' to write changes certainly,
         # because the latter omits writing out if transaction is running.
         # output file will be used to create backup of dirstate at this point.
         if self._dirty or not self._opener.exists(filename):
             self._writedirstate(self._opener(filename, "w", atomictemp=True,
                                              checkambig=True))
 
         if tr:
             # ensure that subsequent tr.writepending returns True for
             # changes written out above, even if dirstate is never
             # changed after this
             tr.addfilegenerator('dirstate', (self._filename,),
                                 self._writedirstate, location='plain')
 
             # ensure that pending file written above is unlinked at
             # failure, even if tr.writepending isn't invoked until the
             # end of this transaction
             tr.registertmp(filename, location='plain')
 
         self._opener.tryunlink(backupname)
         # hardlink backup is okay because _writedirstate is always called
         # with an "atomictemp=True" file.
         util.copyfile(self._opener.join(filename),
                       self._opener.join(backupname), hardlink=True)
 
     def restorebackup(self, tr, backupname):
         '''Restore dirstate by backup file'''
         # this "invalidate()" prevents "wlock.release()" from writing
         # changes of dirstate out after restoring from backup file
         self.invalidate()
         filename = self._actualfilename(tr)
         o = self._opener
         if util.samefile(o.join(backupname), o.join(filename)):
             o.unlink(backupname)
         else:
             o.rename(backupname, filename, checkambig=True)
 
     def clearbackup(self, tr, backupname):
         '''Clear backup file'''
         self._opener.unlink(backupname)
 
 class dirstatemap(object):
     """Map encapsulating the dirstate's contents.
 
     The dirstate contains the following state:
 
     - `identity` is the identity of the dirstate file, which can be used to
       detect when changes have occurred to the dirstate file.
 
     - `parents` is a pair containing the parents of the working copy. The
       parents are updated by calling `setparents`.
 
     - the state map maps filenames to tuples of (state, mode, size, mtime),
       where state is a single character representing 'normal', 'added',
       'removed', or 'merged'. It is read by treating the dirstate as a
       dict.  File state is updated by calling the `addfile`, `removefile` and
       `dropfile` methods.
 
     - `copymap` maps destination filenames to their source filename.
 
     The dirstate also provides the following views onto the state:
 
     - `nonnormalset` is a set of the filenames that have state other
       than 'normal', or are normal but have an mtime of -1 ('normallookup').
 
     - `otherparentset` is a set of the filenames that are marked as coming
       from the second parent when the dirstate is currently being merged.
 
     - `filefoldmap` is a dict mapping normalized filenames to the denormalized
       form that they appear as in the dirstate.
 
     - `dirfoldmap` is a dict mapping normalized directory names to the
       denormalized form that they appear as in the dirstate.
     """
 
     def __init__(self, ui, opener, root):
         self._ui = ui
         self._opener = opener
         self._root = root
         self._filename = 'dirstate'
 
         self._parents = None
         self._dirtyparents = False
 
         # for consistent view between _pl() and _read() invocations
         self._pendingmode = None
 
     @propertycache
     def _map(self):
         self._map = {}
         self.read()
         return self._map
 
     @propertycache
     def copymap(self):
         self.copymap = {}
         self._map
         return self.copymap
 
     def clear(self):
         self._map.clear()
         self.copymap.clear()
         self.setparents(nullid, nullid)
         util.clearcachedproperty(self, "_dirs")
         util.clearcachedproperty(self, "_alldirs")
         util.clearcachedproperty(self, "filefoldmap")
         util.clearcachedproperty(self, "dirfoldmap")
         util.clearcachedproperty(self, "nonnormalset")
         util.clearcachedproperty(self, "otherparentset")
 
     def items(self):
         return self._map.iteritems()
 
     # forward for python2,3 compat
     iteritems = items
 
     def __len__(self):
         return len(self._map)
 
     def __iter__(self):
         return iter(self._map)
 
     def get(self, key, default=None):
         return self._map.get(key, default)
 
     def __contains__(self, key):
         return key in self._map
 
     def __getitem__(self, key):
         return self._map[key]
 
     def keys(self):
         return self._map.keys()
 
     def preload(self):
         """Loads the underlying data, if it's not already loaded"""
         self._map
 
     def addfile(self, f, oldstate, state, mode, size, mtime):
         """Add a tracked file to the dirstate."""
         if oldstate in "?r" and r"_dirs" in self.__dict__:
             self._dirs.addpath(f)
         if oldstate == "?" and r"_alldirs" in self.__dict__:
             self._alldirs.addpath(f)
         self._map[f] = dirstatetuple(state, mode, size, mtime)
         if state != 'n' or mtime == -1:
             self.nonnormalset.add(f)
         if size == -2:
             self.otherparentset.add(f)
 
     def removefile(self, f, oldstate, size):
         """
         Mark a file as removed in the dirstate.
 
         The `size` parameter is used to store sentinel values that indicate
         the file's previous state.  In the future, we should refactor this
         to be more explicit about what that state is.
         """
         if oldstate not in "?r" and r"_dirs" in self.__dict__:
             self._dirs.delpath(f)
         if oldstate == "?" and r"_alldirs" in self.__dict__:
             self._alldirs.addpath(f)
         if r"filefoldmap" in self.__dict__:
             normed = util.normcase(f)
             self.filefoldmap.pop(normed, None)
         self._map[f] = dirstatetuple('r', 0, size, 0)
         self.nonnormalset.add(f)
 
     def dropfile(self, f, oldstate):
         """
         Remove a file from the dirstate.  Returns True if the file was
         previously recorded.
         """
         exists = self._map.pop(f, None) is not None
         if exists:
             if oldstate != "r" and r"_dirs" in self.__dict__:
                 self._dirs.delpath(f)
             if r"_alldirs" in self.__dict__:
                 self._alldirs.delpath(f)
         if r"filefoldmap" in self.__dict__:
             normed = util.normcase(f)
             self.filefoldmap.pop(normed, None)
         self.nonnormalset.discard(f)
         return exists
 
     def clearambiguoustimes(self, files, now):
         for f in files:
             e = self.get(f)
             if e is not None and e[0] == 'n' and e[3] == now:
                 self._map[f] = dirstatetuple(e[0], e[1], e[2], -1)
                 self.nonnormalset.add(f)
 
     def nonnormalentries(self):
         '''Compute the nonnormal dirstate entries from the dmap'''
         try:
             return parsers.nonnormalotherparententries(self._map)
         except AttributeError:
             nonnorm = set()
             otherparent = set()
             for fname, e in self._map.iteritems():
                 if e[0] != 'n' or e[3] == -1:
                     nonnorm.add(fname)
                 if e[0] == 'n' and e[2] == -2:
                     otherparent.add(fname)
             return nonnorm, otherparent
 
     @propertycache
     def filefoldmap(self):
         """Returns a dictionary mapping normalized case paths to their
         non-normalized versions.
         """
         try:
             makefilefoldmap = parsers.make_file_foldmap
         except AttributeError:
             pass
         else:
             return makefilefoldmap(self._map, util.normcasespec,
                                    util.normcasefallback)
 
         f = {}
         normcase = util.normcase
         for name, s in self._map.iteritems():
             if s[0] != 'r':
                 f[normcase(name)] = name
         f['.'] = '.' # prevents useless util.fspath() invocation
         return f
 
     def hastrackeddir(self, d):
         """
         Returns True if the dirstate contains a tracked (not removed) file
         in this directory.
         """
         return d in self._dirs
 
     def hasdir(self, d):
         """
         Returns True if the dirstate contains a file (tracked or removed)
         in this directory.
         """
         return d in self._alldirs
 
     @propertycache
     def _dirs(self):
         return util.dirs(self._map, 'r')
 
     @propertycache
     def _alldirs(self):
         return util.dirs(self._map)
 
     def _opendirstatefile(self):
         fp, mode = txnutil.trypending(self._root, self._opener, self._filename)
         if self._pendingmode is not None and self._pendingmode != mode:
             fp.close()
             raise error.Abort(_('working directory state may be '
                                 'changed parallelly'))
         self._pendingmode = mode
         return fp
 
     def parents(self):
         if not self._parents:
             try:
                 fp = self._opendirstatefile()
                 st = fp.read(40)
                 fp.close()
             except IOError as err:
                 if err.errno != errno.ENOENT:
                     raise
                 # File doesn't exist, so the current state is empty
                 st = ''
 
             l = len(st)
             if l == 40:
                 self._parents = (st[:20], st[20:40])
             elif l == 0:
                 self._parents = (nullid, nullid)
             else:
                 raise error.Abort(_('working directory state appears '
                                     'damaged!'))
 
         return self._parents
 
     def setparents(self, p1, p2):
         self._parents = (p1, p2)
         self._dirtyparents = True
 
     def read(self):
         # ignore HG_PENDING because identity is used only for writing
         self.identity = util.filestat.frompath(
             self._opener.join(self._filename))
 
         try:
             fp = self._opendirstatefile()
             try:
                 st = fp.read()
             finally:
                 fp.close()
         except IOError as err:
             if err.errno != errno.ENOENT:
                 raise
             return
         if not st:
             return
 
         if util.safehasattr(parsers, 'dict_new_presized'):
             # Make an estimate of the number of files in the dirstate based on
             # its size. From a linear regression on a set of real-world repos,
             # all over 10,000 files, the size of a dirstate entry is 85
             # bytes. The cost of resizing is significantly higher than the cost
             # of filling in a larger presized dict, so subtract 20% from the
             # size.
             #
             # This heuristic is imperfect in many ways, so in a future dirstate
             # format update it makes sense to just record the number of entries
             # on write.
             self._map = parsers.dict_new_presized(len(st) // 71)
 
         # Python's garbage collector triggers a GC each time a certain number
         # of container objects (the number being defined by
         # gc.get_threshold()) are allocated. parse_dirstate creates a tuple
         # for each file in the dirstate. The C version then immediately marks
         # them as not to be tracked by the collector. However, this has no
         # effect on when GCs are triggered, only on what objects the GC looks
         # into. This means that O(number of files) GCs are unavoidable.
         # Depending on when in the process's lifetime the dirstate is parsed,
         # this can get very expensive. As a workaround, disable GC while
         # parsing the dirstate.
         #
         # (we cannot decorate the function directly since it is in a C module)
         parse_dirstate = util.nogc(dirstatemod.parse_dirstate)
         p = parse_dirstate(self._map, self.copymap, st)
         if not self._dirtyparents:
             self.setparents(*p)
 
         # Avoid excess attribute lookups by fast pathing certain checks
         self.__contains__ = self._map.__contains__
         self.__getitem__ = self._map.__getitem__
         self.get = self._map.get
 
     def write(self, st, now):
         st.write(dirstatemod.pack_dirstate(self._map, self.copymap,
                                            self.parents(), now))
         st.close()
         self._dirtyparents = False
         self.nonnormalset, self.otherparentset = self.nonnormalentries()
 
     @propertycache
     def nonnormalset(self):
         nonnorm, otherparents = self.nonnormalentries()
         self.otherparentset = otherparents
         return nonnorm
 
     @propertycache
     def otherparentset(self):
         nonnorm, otherparents = self.nonnormalentries()
         self.nonnormalset = nonnorm
         return otherparents
 
     @propertycache
     def identity(self):
         self._map
         return self.identity
 
     @propertycache
     def dirfoldmap(self):
         f = {}
         normcase = util.normcase
         for name in self._dirs:
             f[normcase(name)] = name
         return f

mercurial/merge.py

@@ -1,2302 +1,2333
 # merge.py - directory-level update/merge handling for Mercurial
 #
 # Copyright 2006, 2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import errno
 import hashlib
 import shutil
+import stat
 import struct
 
 from .i18n import _
 from .node import (
     addednodeid,
     bin,
     hex,
     modifiednodeid,
     nullhex,
     nullid,
     nullrev,
 )
 from .thirdparty import (
     attr,
 )
 from . import (
     copies,
     encoding,
     error,
     filemerge,
     match as matchmod,
     obsutil,
     pycompat,
     scmutil,
     subrepoutil,
     util,
     worker,
 )
 
 _pack = struct.pack
 _unpack = struct.unpack
 
 def _droponode(data):
     # used for compatibility for v1
     bits = data.split('\0')
     bits = bits[:-2] + bits[-1:]
     return '\0'.join(bits)
 
 # Merge state record types. See ``mergestate`` docs for more.
 RECORD_LOCAL = b'L'
 RECORD_OTHER = b'O'
 RECORD_MERGED = b'F'
 RECORD_CHANGEDELETE_CONFLICT = b'C'
 RECORD_MERGE_DRIVER_MERGE = b'D'
 RECORD_PATH_CONFLICT = b'P'
 RECORD_MERGE_DRIVER_STATE = b'm'
 RECORD_FILE_VALUES = b'f'
 RECORD_LABELS = b'l'
 RECORD_OVERRIDE = b't'
 RECORD_UNSUPPORTED_MANDATORY = b'X'
 RECORD_UNSUPPORTED_ADVISORY = b'x'
 
 MERGE_DRIVER_STATE_UNMARKED = b'u'
 MERGE_DRIVER_STATE_MARKED = b'm'
 MERGE_DRIVER_STATE_SUCCESS = b's'
 
 MERGE_RECORD_UNRESOLVED = b'u'
 MERGE_RECORD_RESOLVED = b'r'
 MERGE_RECORD_UNRESOLVED_PATH = b'pu'
 MERGE_RECORD_RESOLVED_PATH = b'pr'
 MERGE_RECORD_DRIVER_RESOLVED = b'd'
 
 ACTION_FORGET = b'f'
 ACTION_REMOVE = b'r'
 ACTION_ADD = b'a'
 ACTION_GET = b'g'
 ACTION_PATH_CONFLICT = b'p'
 ACTION_PATH_CONFLICT_RESOLVE = b'pr'
 ACTION_ADD_MODIFIED = b'am'
 ACTION_CREATED = b'c'
 ACTION_DELETED_CHANGED = b'dc'
 ACTION_CHANGED_DELETED = b'cd'
 ACTION_MERGE = b'm'
 ACTION_LOCAL_DIR_RENAME_GET = b'dg'
 ACTION_DIR_RENAME_MOVE_LOCAL = b'dm'
 ACTION_KEEP = b'k'
 ACTION_EXEC = b'e'
 ACTION_CREATED_MERGE = b'cm'
 
 class mergestate(object):
     '''track 3-way merge state of individual files
 
     The merge state is stored on disk when needed. Two files are used: one with
     an old format (version 1), and one with a new format (version 2). Version 2
     stores a superset of the data in version 1, including new kinds of records
     in the future. For more about the new format, see the documentation for
     `_readrecordsv2`.
 
     Each record can contain arbitrary content, and has an associated type. This
     `type` should be a letter. If `type` is uppercase, the record is mandatory:
     versions of Mercurial that don't support it should abort. If `type` is
     lowercase, the record can be safely ignored.
 
     Currently known records:
 
     L: the node of the "local" part of the merge (hexified version)
     O: the node of the "other" part of the merge (hexified version)
     F: a file to be merged entry
     C: a change/delete or delete/change conflict
     D: a file that the external merge driver will merge internally
        (experimental)
     P: a path conflict (file vs directory)
     m: the external merge driver defined for this merge plus its run state
        (experimental)
     f: a (filename, dictionary) tuple of optional values for a given file
     X: unsupported mandatory record type (used in tests)
     x: unsupported advisory record type (used in tests)
     l: the labels for the parts of the merge.
 
     Merge driver run states (experimental):
     u: driver-resolved files unmarked -- needs to be run next time we're about
        to resolve or commit
     m: driver-resolved files marked -- only needs to be run before commit
     s: success/skipped -- does not need to be run any more
 
     Merge record states (stored in self._state, indexed by filename):
     u: unresolved conflict
     r: resolved conflict
     pu: unresolved path conflict (file conflicts with directory)
     pr: resolved path conflict
     d: driver-resolved conflict
 
     The resolve command transitions between 'u' and 'r' for conflicts and
     'pu' and 'pr' for path conflicts.
     '''
     statepathv1 = 'merge/state'
     statepathv2 = 'merge/state2'
 
     @staticmethod
     def clean(repo, node=None, other=None, labels=None):
         """Initialize a brand new merge state, removing any existing state on
         disk."""
         ms = mergestate(repo)
         ms.reset(node, other, labels)
         return ms
 
     @staticmethod
     def read(repo):
         """Initialize the merge state, reading it from disk."""
         ms = mergestate(repo)
         ms._read()
         return ms
 
     def __init__(self, repo):
         """Initialize the merge state.
 
         Do not use this directly! Instead call read() or clean()."""
         self._repo = repo
         self._dirty = False
         self._labels = None
 
     def reset(self, node=None, other=None, labels=None):
         self._state = {}
         self._stateextras = {}
         self._local = None
         self._other = None
         self._labels = labels
         for var in ('localctx', 'otherctx'):
             if var in vars(self):
                 delattr(self, var)
         if node:
             self._local = node
             self._other = other
         self._readmergedriver = None
         if self.mergedriver:
             self._mdstate = MERGE_DRIVER_STATE_SUCCESS
         else:
             self._mdstate = MERGE_DRIVER_STATE_UNMARKED
         shutil.rmtree(self._repo.vfs.join('merge'), True)
         self._results = {}
         self._dirty = False
 
     def _read(self):
         """Analyse each record content to restore a serialized state from disk
 
         This function process "record" entry produced by the de-serialization
         of on disk file.
         """
         self._state = {}
         self._stateextras = {}
         self._local = None
         self._other = None
         for var in ('localctx', 'otherctx'):
             if var in vars(self):
                 delattr(self, var)
         self._readmergedriver = None
         self._mdstate = MERGE_DRIVER_STATE_SUCCESS
         unsupported = set()
         records = self._readrecords()
         for rtype, record in records:
             if rtype == RECORD_LOCAL:
                 self._local = bin(record)
             elif rtype == RECORD_OTHER:
                 self._other = bin(record)
             elif rtype == RECORD_MERGE_DRIVER_STATE:
                 bits = record.split('\0', 1)
                 mdstate = bits[1]
                 if len(mdstate) != 1 or mdstate not in (
                     MERGE_DRIVER_STATE_UNMARKED, MERGE_DRIVER_STATE_MARKED,
                     MERGE_DRIVER_STATE_SUCCESS):
                     # the merge driver should be idempotent, so just rerun it
                     mdstate = MERGE_DRIVER_STATE_UNMARKED
 
                 self._readmergedriver = bits[0]
                 self._mdstate = mdstate
             elif rtype in (RECORD_MERGED, RECORD_CHANGEDELETE_CONFLICT,
                            RECORD_PATH_CONFLICT, RECORD_MERGE_DRIVER_MERGE):
                 bits = record.split('\0')
                 self._state[bits[0]] = bits[1:]
             elif rtype == RECORD_FILE_VALUES:
                 filename, rawextras = record.split('\0', 1)
                 extraparts = rawextras.split('\0')
                 extras = {}
                 i = 0
                 while i < len(extraparts):
                     extras[extraparts[i]] = extraparts[i + 1]
                     i += 2
 
                 self._stateextras[filename] = extras
             elif rtype == RECORD_LABELS:
                 labels = record.split('\0', 2)
                 self._labels = [l for l in labels if len(l) > 0]
             elif not rtype.islower():
                 unsupported.add(rtype)
         self._results = {}
         self._dirty = False
 
         if unsupported:
             raise error.UnsupportedMergeRecords(unsupported)
 
     def _readrecords(self):
         """Read merge state from disk and return a list of record (TYPE, data)
 
         We read data from both v1 and v2 files and decide which one to use.
 
         V1 has been used by version prior to 2.9.1 and contains less data than
         v2. We read both versions and check if no data in v2 contradicts
         v1. If there is not contradiction we can safely assume that both v1
         and v2 were written at the same time and use the extract data in v2. If
         there is contradiction we ignore v2 content as we assume an old version
         of Mercurial has overwritten the mergestate file and left an old v2
         file around.
 
         returns list of record [(TYPE, data), ...]"""
         v1records = self._readrecordsv1()
         v2records = self._readrecordsv2()
         if self._v1v2match(v1records, v2records):
             return v2records
         else:
             # v1 file is newer than v2 file, use it
             # we have to infer the "other" changeset of the merge
             # we cannot do better than that with v1 of the format
             mctx = self._repo[None].parents()[-1]
             v1records.append((RECORD_OTHER, mctx.hex()))
             # add place holder "other" file node information
             # nobody is using it yet so we do no need to fetch the data
             # if mctx was wrong `mctx[bits[-2]]` may fails.
             for idx, r in enumerate(v1records):
                 if r[0] == RECORD_MERGED:
                     bits = r[1].split('\0')
                     bits.insert(-2, '')
                     v1records[idx] = (r[0], '\0'.join(bits))
             return v1records
 
     def _v1v2match(self, v1records, v2records):
         oldv2 = set() # old format version of v2 record
         for rec in v2records:
             if rec[0] == RECORD_LOCAL:
                 oldv2.add(rec)
             elif rec[0] == RECORD_MERGED:
                 # drop the onode data (not contained in v1)
                 oldv2.add((RECORD_MERGED, _droponode(rec[1])))
         for rec in v1records:
             if rec not in oldv2:
                 return False
         else:
             return True
 
     def _readrecordsv1(self):
         """read on disk merge state for version 1 file
 
         returns list of record [(TYPE, data), ...]
 
         Note: the "F" data from this file are one entry short
               (no "other file node" entry)
         """
         records = []
         try:
             f = self._repo.vfs(self.statepathv1)
             for i, l in enumerate(f):
                 if i == 0:
                     records.append((RECORD_LOCAL, l[:-1]))
                 else:
                     records.append((RECORD_MERGED, l[:-1]))
             f.close()
         except IOError as err:
             if err.errno != errno.ENOENT:
                 raise
         return records
 
     def _readrecordsv2(self):
         """read on disk merge state for version 2 file
 
         This format is a list of arbitrary records of the form:
 
           [type][length][content]
 
         `type` is a single character, `length` is a 4 byte integer, and
         `content` is an arbitrary byte sequence of length `length`.
 
         Mercurial versions prior to 3.7 have a bug where if there are
         unsupported mandatory merge records, attempting to clear out the merge
         state with hg update --clean or similar aborts. The 't' record type
         works around that by writing out what those versions treat as an
         advisory record, but later versions interpret as special: the first
         character is the 'real' record type and everything onwards is the data.
 
         Returns list of records [(TYPE, data), ...]."""
         records = []
         try:
             f = self._repo.vfs(self.statepathv2)
             data = f.read()
             off = 0
             end = len(data)
             while off < end:
                 rtype = data[off:off + 1]
                 off += 1
                 length = _unpack('>I', data[off:(off + 4)])[0]
                 off += 4
                 record = data[off:(off + length)]
                 off += length
                 if rtype == RECORD_OVERRIDE:
                     rtype, record = record[0:1], record[1:]
                 records.append((rtype, record))
             f.close()
         except IOError as err:
             if err.errno != errno.ENOENT:
                 raise
         return records
 
     @util.propertycache
     def mergedriver(self):
         # protect against the following:
         # - A configures a malicious merge driver in their hgrc, then
         #   pauses the merge
         # - A edits their hgrc to remove references to the merge driver
         # - A gives a copy of their entire repo, including .hg, to B
         # - B inspects .hgrc and finds it to be clean
         # - B then continues the merge and the malicious merge driver
         #  gets invoked
         configmergedriver = self._repo.ui.config('experimental', 'mergedriver')
         if (self._readmergedriver is not None
             and self._readmergedriver != configmergedriver):
             raise error.ConfigError(
                 _("merge driver changed since merge started"),
                 hint=_("revert merge driver change or abort merge"))
 
         return configmergedriver
 
     @util.propertycache
     def localctx(self):
         if self._local is None:
             msg = "localctx accessed but self._local isn't set"
             raise error.ProgrammingError(msg)
         return self._repo[self._local]
 
     @util.propertycache
     def otherctx(self):
         if self._other is None:
             msg = "otherctx accessed but self._other isn't set"
             raise error.ProgrammingError(msg)
         return self._repo[self._other]
 
     def active(self):
         """Whether mergestate is active.
 
         Returns True if there appears to be mergestate. This is a rough proxy
         for "is a merge in progress."
         """
         # Check local variables before looking at filesystem for performance
         # reasons.
         return (bool(self._local) or bool(self._state) or
                 self._repo.vfs.exists(self.statepathv1) or
                 self._repo.vfs.exists(self.statepathv2))
 
     def commit(self):
         """Write current state on disk (if necessary)"""
         if self._dirty:
             records = self._makerecords()
             self._writerecords(records)
             self._dirty = False
 
     def _makerecords(self):
         records = []
         records.append((RECORD_LOCAL, hex(self._local)))
         records.append((RECORD_OTHER, hex(self._other)))
         if self.mergedriver:
             records.append((RECORD_MERGE_DRIVER_STATE, '\0'.join([
                 self.mergedriver, self._mdstate])))
         # Write out state items. In all cases, the value of the state map entry
         # is written as the contents of the record. The record type depends on
         # the type of state that is stored, and capital-letter records are used
         # to prevent older versions of Mercurial that do not support the feature
         # from loading them.
         for filename, v in self._state.iteritems():
             if v[0] == MERGE_RECORD_DRIVER_RESOLVED:
                 # Driver-resolved merge. These are stored in 'D' records.
                 records.append((RECORD_MERGE_DRIVER_MERGE,
                                 '\0'.join([filename] + v)))
             elif v[0] in (MERGE_RECORD_UNRESOLVED_PATH,
                           MERGE_RECORD_RESOLVED_PATH):
                 # Path conflicts. These are stored in 'P' records.  The current
                 # resolution state ('pu' or 'pr') is stored within the record.
                 records.append((RECORD_PATH_CONFLICT,
                                 '\0'.join([filename] + v)))
             elif v[1] == nullhex or v[6] == nullhex:
                 # Change/Delete or Delete/Change conflicts. These are stored in
                 # 'C' records. v[1] is the local file, and is nullhex when the
                 # file is deleted locally ('dc'). v[6] is the remote file, and
                 # is nullhex when the file is deleted remotely ('cd').
                 records.append((RECORD_CHANGEDELETE_CONFLICT,
                                 '\0'.join([filename] + v)))
             else:
                 # Normal files.  These are stored in 'F' records.
                 records.append((RECORD_MERGED,
                                 '\0'.join([filename] + v)))
         for filename, extras in sorted(self._stateextras.iteritems()):
             rawextras = '\0'.join('%s\0%s' % (k, v) for k, v in
                                   extras.iteritems())
             records.append((RECORD_FILE_VALUES,
                             '%s\0%s' % (filename, rawextras)))
         if self._labels is not None:
             labels = '\0'.join(self._labels)
             records.append((RECORD_LABELS, labels))
         return records
 
     def _writerecords(self, records):
         """Write current state on disk (both v1 and v2)"""
         self._writerecordsv1(records)
         self._writerecordsv2(records)
 
     def _writerecordsv1(self, records):
         """Write current state on disk in a version 1 file"""
         f = self._repo.vfs(self.statepathv1, 'wb')
         irecords = iter(records)
         lrecords = next(irecords)
         assert lrecords[0] == RECORD_LOCAL
         f.write(hex(self._local) + '\n')
         for rtype, data in irecords:
             if rtype == RECORD_MERGED:
                 f.write('%s\n' % _droponode(data))
         f.close()
 
     def _writerecordsv2(self, records):
         """Write current state on disk in a version 2 file
 
         See the docstring for _readrecordsv2 for why we use 't'."""
         # these are the records that all version 2 clients can read
         allowlist = (RECORD_LOCAL, RECORD_OTHER, RECORD_MERGED)
         f = self._repo.vfs(self.statepathv2, 'wb')
         for key, data in records:
             assert len(key) == 1
             if key not in allowlist:
                 key, data = RECORD_OVERRIDE, '%s%s' % (key, data)
             format = '>sI%is' % len(data)
             f.write(_pack(format, key, len(data), data))
         f.close()
 
     @staticmethod
     def getlocalkey(path):
         """hash the path of a local file context for storage in the .hg/merge
         directory."""
 
         return hex(hashlib.sha1(path).digest())
 
     def add(self, fcl, fco, fca, fd):
         """add a new (potentially?) conflicting file the merge state
         fcl: file context for local,
         fco: file context for remote,
         fca: file context for ancestors,
         fd:  file path of the resulting merge.
 
         note: also write the local version to the `.hg/merge` directory.
         """
         if fcl.isabsent():
             localkey = nullhex
         else:
             localkey = mergestate.getlocalkey(fcl.path())
             self._repo.vfs.write('merge/' + localkey, fcl.data())
         self._state[fd] = [MERGE_RECORD_UNRESOLVED, localkey, fcl.path(),
                            fca.path(), hex(fca.filenode()),
                            fco.path(), hex(fco.filenode()),
                            fcl.flags()]
         self._stateextras[fd] = {'ancestorlinknode': hex(fca.node())}
         self._dirty = True
 
     def addpath(self, path, frename, forigin):
         """add a new conflicting path to the merge state
         path:    the path that conflicts
         frename: the filename the conflicting file was renamed to
         forigin: origin of the file ('l' or 'r' for local/remote)
         """
         self._state[path] = [MERGE_RECORD_UNRESOLVED_PATH, frename, forigin]
         self._dirty = True
 
     def __contains__(self, dfile):
         return dfile in self._state
 
     def __getitem__(self, dfile):
         return self._state[dfile][0]
 
     def __iter__(self):
         return iter(sorted(self._state))
 
     def files(self):
         return self._state.keys()
 
     def mark(self, dfile, state):
         self._state[dfile][0] = state
         self._dirty = True
 
     def mdstate(self):
         return self._mdstate
 
     def unresolved(self):
         """Obtain the paths of unresolved files."""
 
         for f, entry in self._state.iteritems():
             if entry[0] in (MERGE_RECORD_UNRESOLVED,
                             MERGE_RECORD_UNRESOLVED_PATH):
                 yield f
 
     def driverresolved(self):
         """Obtain the paths of driver-resolved files."""
 
         for f, entry in self._state.items():
             if entry[0] == MERGE_RECORD_DRIVER_RESOLVED:
                 yield f
 
     def extras(self, filename):
         return self._stateextras.setdefault(filename, {})
 
     def _resolve(self, preresolve, dfile, wctx):
         """rerun merge process for file path `dfile`"""
         if self[dfile] in (MERGE_RECORD_RESOLVED,
                            MERGE_RECORD_DRIVER_RESOLVED):
             return True, 0
         stateentry = self._state[dfile]
         state, localkey, lfile, afile, anode, ofile, onode, flags = stateentry
         octx = self._repo[self._other]
         extras = self.extras(dfile)
         anccommitnode = extras.get('ancestorlinknode')
         if anccommitnode:
             actx = self._repo[anccommitnode]
         else:
             actx = None
         fcd = self._filectxorabsent(localkey, wctx, dfile)
         fco = self._filectxorabsent(onode, octx, ofile)
         # TODO: move this to filectxorabsent
         fca = self._repo.filectx(afile, fileid=anode, changectx=actx)
         # "premerge" x flags
         flo = fco.flags()
         fla = fca.flags()
         if 'x' in flags + flo + fla and 'l' not in flags + flo + fla:
             if fca.node() == nullid and flags != flo:
                 if preresolve:
                     self._repo.ui.warn(
                         _('warning: cannot merge flags for %s '
                           'without common ancestor - keeping local flags\n')
                         % afile)
             elif flags == fla:
                 flags = flo
         if preresolve:
             # restore local
             if localkey != nullhex:
                 f = self._repo.vfs('merge/' + localkey)
                 wctx[dfile].write(f.read(), flags)
                 f.close()
             else:
                 wctx[dfile].remove(ignoremissing=True)
             complete, r, deleted = filemerge.premerge(self._repo, wctx,
                                                       self._local, lfile, fcd,
                                                       fco, fca,
                                                       labels=self._labels)
         else:
             complete, r, deleted = filemerge.filemerge(self._repo, wctx,
                                                        self._local, lfile, fcd,
                                                        fco, fca,
                                                        labels=self._labels)
         if r is None:
             # no real conflict
             del self._state[dfile]
             self._stateextras.pop(dfile, None)
             self._dirty = True
         elif not r:
             self.mark(dfile, MERGE_RECORD_RESOLVED)
 
         if complete:
             action = None
             if deleted:
                 if fcd.isabsent():
                     # dc: local picked. Need to drop if present, which may
                     # happen on re-resolves.
                     action = ACTION_FORGET
                 else:
                     # cd: remote picked (or otherwise deleted)
                     action = ACTION_REMOVE
             else:
                 if fcd.isabsent(): # dc: remote picked
                     action = ACTION_GET
                 elif fco.isabsent(): # cd: local picked
                     if dfile in self.localctx:
                         action = ACTION_ADD_MODIFIED
                     else:
                         action = ACTION_ADD
                 # else: regular merges (no action necessary)
             self._results[dfile] = r, action
 
         return complete, r
 
     def _filectxorabsent(self, hexnode, ctx, f):
         if hexnode == nullhex:
             return filemerge.absentfilectx(ctx, f)
         else:
             return ctx[f]
 
     def preresolve(self, dfile, wctx):
         """run premerge process for dfile
 
         Returns whether the merge is complete, and the exit code."""
         return self._resolve(True, dfile, wctx)
 
     def resolve(self, dfile, wctx):
         """run merge process (assuming premerge was run) for dfile
 
         Returns the exit code of the merge."""
         return self._resolve(False, dfile, wctx)[1]
 
     def counts(self):
         """return counts for updated, merged and removed files in this
         session"""
         updated, merged, removed = 0, 0, 0
         for r, action in self._results.itervalues():
             if r is None:
                 updated += 1
             elif r == 0:
                 if action == ACTION_REMOVE:
                     removed += 1
                 else:
                     merged += 1
         return updated, merged, removed
 
     def unresolvedcount(self):
         """get unresolved count for this merge (persistent)"""
         return len(list(self.unresolved()))
 
     def actions(self):
         """return lists of actions to perform on the dirstate"""
         actions = {
             ACTION_REMOVE: [],
             ACTION_FORGET: [],
             ACTION_ADD: [],
             ACTION_ADD_MODIFIED: [],
             ACTION_GET: [],
         }
         for f, (r, action) in self._results.iteritems():
             if action is not None:
                 actions[action].append((f, None, "merge result"))
         return actions
 
     def recordactions(self):
         """record remove/add/get actions in the dirstate"""
         branchmerge = self._repo.dirstate.p2() != nullid
-        recordupdates(self._repo, self.actions(), branchmerge)
+        recordupdates(self._repo, self.actions(), branchmerge, None)
 
     def queueremove(self, f):
         """queues a file to be removed from the dirstate
 
         Meant for use by custom merge drivers."""
         self._results[f] = 0, ACTION_REMOVE
 
     def queueadd(self, f):
         """queues a file to be added to the dirstate
 
         Meant for use by custom merge drivers."""
         self._results[f] = 0, ACTION_ADD
 
     def queueget(self, f):
         """queues a file to be marked modified in the dirstate
 
         Meant for use by custom merge drivers."""
         self._results[f] = 0, ACTION_GET
 
 def _getcheckunknownconfig(repo, section, name):
     config = repo.ui.config(section, name)
     valid = ['abort', 'ignore', 'warn']
     if config not in valid:
         validstr = ', '.join(["'" + v + "'" for v in valid])
         raise error.ConfigError(_("%s.%s not valid "
                                   "('%s' is none of %s)")
                                 % (section, name, config, validstr))
     return config
 
 def _checkunknownfile(repo, wctx, mctx, f, f2=None):
     if wctx.isinmemory():
         # Nothing to do in IMM because nothing in the "working copy" can be an
         # unknown file.
         #
         # Note that we should bail out here, not in ``_checkunknownfiles()``,
         # because that function does other useful work.
         return False
 
     if f2 is None:
         f2 = f
     return (repo.wvfs.audit.check(f)
         and repo.wvfs.isfileorlink(f)
         and repo.dirstate.normalize(f) not in repo.dirstate
         and mctx[f2].cmp(wctx[f]))
 
 class _unknowndirschecker(object):
     """
     Look for any unknown files or directories that may have a path conflict
     with a file.  If any path prefix of the file exists as a file or link,
     then it conflicts.  If the file itself is a directory that contains any
     file that is not tracked, then it conflicts.
 
     Returns the shortest path at which a conflict occurs, or None if there is
     no conflict.
     """
     def __init__(self):
         # A set of paths known to be good.  This prevents repeated checking of
         # dirs.  It will be updated with any new dirs that are checked and found
         # to be safe.
         self._unknowndircache = set()
 
         # A set of paths that are known to be absent.  This prevents repeated
         # checking of subdirectories that are known not to exist. It will be
         # updated with any new dirs that are checked and found to be absent.
         self._missingdircache = set()
 
     def __call__(self, repo, wctx, f):
         if wctx.isinmemory():
             # Nothing to do in IMM for the same reason as ``_checkunknownfile``.
             return False
 
         # Check for path prefixes that exist as unknown files.
         for p in reversed(list(util.finddirs(f))):
             if p in self._missingdircache:
                 return
             if p in self._unknowndircache:
                 continue
             if repo.wvfs.audit.check(p):
                 if (repo.wvfs.isfileorlink(p)
                         and repo.dirstate.normalize(p) not in repo.dirstate):
                     return p
                 if not repo.wvfs.lexists(p):
                     self._missingdircache.add(p)
                     return
                 self._unknowndircache.add(p)
 
         # Check if the file conflicts with a directory containing unknown files.
         if repo.wvfs.audit.check(f) and repo.wvfs.isdir(f):
             # Does the directory contain any files that are not in the dirstate?
             for p, dirs, files in repo.wvfs.walk(f):
                 for fn in files:
                     relf = util.pconvert(repo.wvfs.reljoin(p, fn))
                     relf = repo.dirstate.normalize(relf, isknown=True)
                     if relf not in repo.dirstate:
                         return f
         return None
 
 def _checkunknownfiles(repo, wctx, mctx, force, actions, mergeforce):
     """
     Considers any actions that care about the presence of conflicting unknown
     files. For some actions, the result is to abort; for others, it is to
     choose a different action.
     """
     fileconflicts = set()
     pathconflicts = set()
     warnconflicts = set()
     abortconflicts = set()
     unknownconfig = _getcheckunknownconfig(repo, 'merge', 'checkunknown')
     ignoredconfig = _getcheckunknownconfig(repo, 'merge', 'checkignored')
     pathconfig = repo.ui.configbool('experimental', 'merge.checkpathconflicts')
     if not force:
         def collectconflicts(conflicts, config):
             if config == 'abort':
                 abortconflicts.update(conflicts)
             elif config == 'warn':
                 warnconflicts.update(conflicts)
 
         checkunknowndirs = _unknowndirschecker()
         for f, (m, args, msg) in actions.iteritems():
             if m in (ACTION_CREATED, ACTION_DELETED_CHANGED):
                 if _checkunknownfile(repo, wctx, mctx, f):
                     fileconflicts.add(f)
                 elif pathconfig and f not in wctx:
                     path = checkunknowndirs(repo, wctx, f)
                     if path is not None:
                         pathconflicts.add(path)
             elif m == ACTION_LOCAL_DIR_RENAME_GET:
                 if _checkunknownfile(repo, wctx, mctx, f, args[0]):
                     fileconflicts.add(f)
 
         allconflicts = fileconflicts | pathconflicts
         ignoredconflicts = {c for c in allconflicts
                             if repo.dirstate._ignore(c)}
         unknownconflicts = allconflicts - ignoredconflicts
         collectconflicts(ignoredconflicts, ignoredconfig)
         collectconflicts(unknownconflicts, unknownconfig)
     else:
         for f, (m, args, msg) in actions.iteritems():
             if m == ACTION_CREATED_MERGE:
                 fl2, anc = args
                 different = _checkunknownfile(repo, wctx, mctx, f)
                 if repo.dirstate._ignore(f):
                     config = ignoredconfig
                 else:
                     config = unknownconfig
 
                 # The behavior when force is True is described by this table:
                 #  config  different  mergeforce  |    action    backup
                 #    *         n          *       |      get        n
                 #    *         y          y       |     merge       -
                 #   abort      y          n       |     merge       -   (1)
                 #   warn       y          n       |  warn + get     y
                 #  ignore      y          n       |      get        y
                 #
                 # (1) this is probably the wrong behavior here -- we should
                 #     probably abort, but some actions like rebases currently
                 #     don't like an abort happening in the middle of
                 #     merge.update.
                 if not different:
                     actions[f] = (ACTION_GET, (fl2, False), 'remote created')
                 elif mergeforce or config == 'abort':
                     actions[f] = (ACTION_MERGE, (f, f, None, False, anc),
                                   'remote differs from untracked local')
                 elif config == 'abort':
                     abortconflicts.add(f)
                 else:
                     if config == 'warn':
                         warnconflicts.add(f)
                     actions[f] = (ACTION_GET, (fl2, True), 'remote created')
 
     for f in sorted(abortconflicts):
         warn = repo.ui.warn
         if f in pathconflicts:
             if repo.wvfs.isfileorlink(f):
                 warn(_("%s: untracked file conflicts with directory\n") % f)
             else:
                 warn(_("%s: untracked directory conflicts with file\n") % f)
         else:
             warn(_("%s: untracked file differs\n") % f)
     if abortconflicts:
         raise error.Abort(_("untracked files in working directory "
                             "differ from files in requested revision"))
 
     for f in sorted(warnconflicts):
         if repo.wvfs.isfileorlink(f):
             repo.ui.warn(_("%s: replacing untracked file\n") % f)
         else:
             repo.ui.warn(_("%s: replacing untracked files in directory\n") % f)
 
     for f, (m, args, msg) in actions.iteritems():
         if m == ACTION_CREATED:
             backup = (f in fileconflicts or f in pathconflicts or
                       any(p in pathconflicts for p in util.finddirs(f)))
             flags, = args
             actions[f] = (ACTION_GET, (flags, backup), msg)
 
 def _forgetremoved(wctx, mctx, branchmerge):
     """
     Forget removed files
 
     If we're jumping between revisions (as opposed to merging), and if
     neither the working directory nor the target rev has the file,
     then we need to remove it from the dirstate, to prevent the
     dirstate from listing the file when it is no longer in the
     manifest.
 
     If we're merging, and the other revision has removed a file
     that is not present in the working directory, we need to mark it
     as removed.
     """
 
     actions = {}
     m = ACTION_FORGET
     if branchmerge:
         m = ACTION_REMOVE
     for f in wctx.deleted():
         if f not in mctx:
             actions[f] = m, None, "forget deleted"
 
     if not branchmerge:
         for f in wctx.removed():
             if f not in mctx:
                 actions[f] = ACTION_FORGET, None, "forget removed"
 
     return actions
 
 def _checkcollision(repo, wmf, actions):
     """
     Check for case-folding collisions.
     """
 
     # If the repo is narrowed, filter out files outside the narrowspec.
     narrowmatch = repo.narrowmatch()
     if not narrowmatch.always():
         wmf = wmf.matches(narrowmatch)
         if actions:
             narrowactions = {}
             for m, actionsfortype in actions.iteritems():
                 narrowactions[m] = []
                 for (f, args, msg) in actionsfortype:
                     if narrowmatch(f):
                         narrowactions[m].append((f, args, msg))
             actions = narrowactions
 
     # build provisional merged manifest up
     pmmf = set(wmf)
 
     if actions:
         # KEEP and EXEC are no-op
         for m in (ACTION_ADD, ACTION_ADD_MODIFIED, ACTION_FORGET, ACTION_GET,
                   ACTION_CHANGED_DELETED, ACTION_DELETED_CHANGED):
             for f, args, msg in actions[m]:
                 pmmf.add(f)
         for f, args, msg in actions[ACTION_REMOVE]:
             pmmf.discard(f)
         for f, args, msg in actions[ACTION_DIR_RENAME_MOVE_LOCAL]:
             f2, flags = args
             pmmf.discard(f2)
             pmmf.add(f)
         for f, args, msg in actions[ACTION_LOCAL_DIR_RENAME_GET]:
             pmmf.add(f)
         for f, args, msg in actions[ACTION_MERGE]:
             f1, f2, fa, move, anc = args
             if move:
                 pmmf.discard(f1)
             pmmf.add(f)
 
     # check case-folding collision in provisional merged manifest
     foldmap = {}
     for f in pmmf:
         fold = util.normcase(f)
         if fold in foldmap:
             raise error.Abort(_("case-folding collision between %s and %s")
                              % (f, foldmap[fold]))
         foldmap[fold] = f
 
     # check case-folding of directories
     foldprefix = unfoldprefix = lastfull = ''
     for fold, f in sorted(foldmap.items()):
         if fold.startswith(foldprefix) and not f.startswith(unfoldprefix):
             # the folded prefix matches but actual casing is different
             raise error.Abort(_("case-folding collision between "
                                 "%s and directory of %s") % (lastfull, f))
         foldprefix = fold + '/'
         unfoldprefix = f + '/'
         lastfull = f
 
 def driverpreprocess(repo, ms, wctx, labels=None):
     """run the preprocess step of the merge driver, if any
 
     This is currently not implemented -- it's an extension point."""
     return True
 
 def driverconclude(repo, ms, wctx, labels=None):
     """run the conclude step of the merge driver, if any
 
     This is currently not implemented -- it's an extension point."""
     return True
 
 def _filesindirs(repo, manifest, dirs):
     """
     Generator that yields pairs of all the files in the manifest that are found
     inside the directories listed in dirs, and which directory they are found
     in.
     """
     for f in manifest:
         for p in util.finddirs(f):
             if p in dirs:
                 yield f, p
                 break
 
 def checkpathconflicts(repo, wctx, mctx, actions):
     """
     Check if any actions introduce path conflicts in the repository, updating
     actions to record or handle the path conflict accordingly.
     """
     mf = wctx.manifest()
 
     # The set of local files that conflict with a remote directory.
     localconflicts = set()
 
     # The set of directories that conflict with a remote file, and so may cause
     # conflicts if they still contain any files after the merge.
     remoteconflicts = set()
 
     # The set of directories that appear as both a file and a directory in the
     # remote manifest.  These indicate an invalid remote manifest, which
     # can't be updated to cleanly.
     invalidconflicts = set()
 
     # The set of directories that contain files that are being created.
     createdfiledirs = set()
 
     # The set of files deleted by all the actions.
     deletedfiles = set()
 
     for f, (m, args, msg) in actions.items():
         if m in (ACTION_CREATED, ACTION_DELETED_CHANGED, ACTION_MERGE,
                  ACTION_CREATED_MERGE):
             # This action may create a new local file.
             createdfiledirs.update(util.finddirs(f))
             if mf.hasdir(f):
                 # The file aliases a local directory.  This might be ok if all
                 # the files in the local directory are being deleted.  This
                 # will be checked once we know what all the deleted files are.
                 remoteconflicts.add(f)
         # Track the names of all deleted files.
         if m == ACTION_REMOVE:
             deletedfiles.add(f)
         if m == ACTION_MERGE:
             f1, f2, fa, move, anc = args
             if move:
                 deletedfiles.add(f1)
         if m == ACTION_DIR_RENAME_MOVE_LOCAL:
             f2, flags = args
             deletedfiles.add(f2)
 
     # Check all directories that contain created files for path conflicts.
     for p in createdfiledirs:
         if p in mf:
             if p in mctx:
                 # A file is in a directory which aliases both a local
                 # and a remote file.  This is an internal inconsistency
                 # within the remote manifest.
                 invalidconflicts.add(p)
             else:
                 # A file is in a directory which aliases a local file.
                 # We will need to rename the local file.
                 localconflicts.add(p)
         if p in actions and actions[p][0] in (ACTION_CREATED,
                                               ACTION_DELETED_CHANGED,
                                               ACTION_MERGE,
                                               ACTION_CREATED_MERGE):
             # The file is in a directory which aliases a remote file.
             # This is an internal inconsistency within the remote
             # manifest.
             invalidconflicts.add(p)
 
     # Rename all local conflicting files that have not been deleted.
     for p in localconflicts:
         if p not in deletedfiles:
             ctxname = bytes(wctx).rstrip('+')
             pnew = util.safename(p, ctxname, wctx, set(actions.keys()))
             actions[pnew] = (ACTION_PATH_CONFLICT_RESOLVE, (p,),
                              'local path conflict')
             actions[p] = (ACTION_PATH_CONFLICT, (pnew, 'l'),
                           'path conflict')
 
     if remoteconflicts:
         # Check if all files in the conflicting directories have been removed.
         ctxname = bytes(mctx).rstrip('+')
         for f, p in _filesindirs(repo, mf, remoteconflicts):
             if f not in deletedfiles:
                 m, args, msg = actions[p]
                 pnew = util.safename(p, ctxname, wctx, set(actions.keys()))
                 if m in (ACTION_DELETED_CHANGED, ACTION_MERGE):
                     # Action was merge, just update target.
                     actions[pnew] = (m, args, msg)
                 else:
                     # Action was create, change to renamed get action.
                     fl = args[0]
                     actions[pnew] = (ACTION_LOCAL_DIR_RENAME_GET, (p, fl),
                                      'remote path conflict')
                 actions[p] = (ACTION_PATH_CONFLICT, (pnew, ACTION_REMOVE),
                               'path conflict')
                 remoteconflicts.remove(p)
                 break
 
     if invalidconflicts:
         for p in invalidconflicts:
             repo.ui.warn(_("%s: is both a file and a directory\n") % p)
         raise error.Abort(_("destination manifest contains path conflicts"))
 
 def _filternarrowactions(narrowmatch, branchmerge, actions):
     """
     Filters out actions that can ignored because the repo is narrowed.
 
     Raise an exception if the merge cannot be completed because the repo is
     narrowed.
     """
     nooptypes = {'k'} # TODO: handle with nonconflicttypes
     nonconflicttypes = set('a am c cm f g r e'.split())
     # We mutate the items in the dict during iteration, so iterate
     # over a copy.
     for f, action in list(actions.items()):
         if narrowmatch(f):
             pass
         elif not branchmerge:
             del actions[f] # just updating, ignore changes outside clone
         elif action[0] in nooptypes:
             del actions[f] # merge does not affect file
         elif action[0] in nonconflicttypes:
             raise error.Abort(_('merge affects file \'%s\' outside narrow, '
                                 'which is not yet supported') % f,
                               hint=_('merging in the other direction '
                                      'may work'))
         else:
             raise error.Abort(_('conflict in file \'%s\' is outside '
                                 'narrow clone') % f)
 
 def manifestmerge(repo, wctx, p2, pa, branchmerge, force, matcher,
                   acceptremote, followcopies, forcefulldiff=False):
     """
     Merge wctx and p2 with ancestor pa and generate merge action list
 
     branchmerge and force are as passed in to update
     matcher = matcher to filter file lists
     acceptremote = accept the incoming changes without prompting
     """
     if matcher is not None and matcher.always():
         matcher = None
 
     copy, movewithdir, diverge, renamedelete, dirmove = {}, {}, {}, {}, {}
 
     # manifests fetched in order are going to be faster, so prime the caches
     [x.manifest() for x in
      sorted(wctx.parents() + [p2, pa], key=scmutil.intrev)]
 
     if followcopies:
         ret = copies.mergecopies(repo, wctx, p2, pa)
         copy, movewithdir, diverge, renamedelete, dirmove = ret
 
     boolbm = pycompat.bytestr(bool(branchmerge))
     boolf = pycompat.bytestr(bool(force))
     boolm = pycompat.bytestr(bool(matcher))
     repo.ui.note(_("resolving manifests\n"))
     repo.ui.debug(" branchmerge: %s, force: %s, partial: %s\n"
                   % (boolbm, boolf, boolm))
     repo.ui.debug(" ancestor: %s, local: %s, remote: %s\n" % (pa, wctx, p2))
 
     m1, m2, ma = wctx.manifest(), p2.manifest(), pa.manifest()
     copied = set(copy.values())
     copied.update(movewithdir.values())
 
     if '.hgsubstate' in m1 and wctx.rev() is None:
         # Check whether sub state is modified, and overwrite the manifest
         # to flag the change. If wctx is a committed revision, we shouldn't
         # care for the dirty state of the working directory.
         if any(wctx.sub(s).dirty() for s in wctx.substate):
             m1['.hgsubstate'] = modifiednodeid
 
     # Don't use m2-vs-ma optimization if:
     # - ma is the same as m1 or m2, which we're just going to diff again later
     # - The caller specifically asks for a full diff, which is useful during bid
     #   merge.
     if (pa not in ([wctx, p2] + wctx.parents()) and not forcefulldiff):
         # Identify which files are relevant to the merge, so we can limit the
         # total m1-vs-m2 diff to just those files. This has significant
         # performance benefits in large repositories.
         relevantfiles = set(ma.diff(m2).keys())
 
         # For copied and moved files, we need to add the source file too.
         for copykey, copyvalue in copy.iteritems():
             if copyvalue in relevantfiles:
                 relevantfiles.add(copykey)
         for movedirkey in movewithdir:
             relevantfiles.add(movedirkey)
         filesmatcher = scmutil.matchfiles(repo, relevantfiles)
         matcher = matchmod.intersectmatchers(matcher, filesmatcher)
 
     diff = m1.diff(m2, match=matcher)
 
     actions = {}
     for f, ((n1, fl1), (n2, fl2)) in diff.iteritems():
         if n1 and n2: # file exists on both local and remote side
             if f not in ma:
                 fa = copy.get(f, None)
                 if fa is not None:
                     actions[f] = (ACTION_MERGE, (f, f, fa, False, pa.node()),
                                   'both renamed from %s' % fa)
                 else:
                     actions[f] = (ACTION_MERGE, (f, f, None, False, pa.node()),
                                   'both created')
             else:
                 a = ma[f]
                 fla = ma.flags(f)
                 nol = 'l' not in fl1 + fl2 + fla
                 if n2 == a and fl2 == fla:
                     actions[f] = (ACTION_KEEP, (), 'remote unchanged')
                 elif n1 == a and fl1 == fla: # local unchanged - use remote
                     if n1 == n2: # optimization: keep local content
                         actions[f] = (ACTION_EXEC, (fl2,), 'update permissions')
                     else:
                         actions[f] = (ACTION_GET, (fl2, False),
                                       'remote is newer')
                 elif nol and n2 == a: # remote only changed 'x'
                     actions[f] = (ACTION_EXEC, (fl2,), 'update permissions')
                 elif nol and n1 == a: # local only changed 'x'
                     actions[f] = (ACTION_GET, (fl1, False), 'remote is newer')
                 else: # both changed something
                     actions[f] = (ACTION_MERGE, (f, f, f, False, pa.node()),
                                   'versions differ')
         elif n1: # file exists only on local side
             if f in copied:
                 pass # we'll deal with it on m2 side
             elif f in movewithdir: # directory rename, move local
                 f2 = movewithdir[f]
                 if f2 in m2:
                     actions[f2] = (ACTION_MERGE, (f, f2, None, True, pa.node()),
                                    'remote directory rename, both created')
                 else:
                     actions[f2] = (ACTION_DIR_RENAME_MOVE_LOCAL, (f, fl1),
                                    'remote directory rename - move from %s' % f)
             elif f in copy:
                 f2 = copy[f]
                 actions[f] = (ACTION_MERGE, (f, f2, f2, False, pa.node()),
                               'local copied/moved from %s' % f2)
             elif f in ma: # clean, a different, no remote
                 if n1 != ma[f]:
                     if acceptremote:
                         actions[f] = (ACTION_REMOVE, None, 'remote delete')
                     else:
                         actions[f] = (ACTION_CHANGED_DELETED,
                                       (f, None, f, False, pa.node()),
                                       'prompt changed/deleted')
                 elif n1 == addednodeid:
                     # This extra 'a' is added by working copy manifest to mark
                     # the file as locally added. We should forget it instead of
                     # deleting it.
                     actions[f] = (ACTION_FORGET, None, 'remote deleted')
                 else:
                     actions[f] = (ACTION_REMOVE, None, 'other deleted')
         elif n2: # file exists only on remote side
             if f in copied:
                 pass # we'll deal with it on m1 side
             elif f in movewithdir:
                 f2 = movewithdir[f]
                 if f2 in m1:
                     actions[f2] = (ACTION_MERGE,
                                    (f2, f, None, False, pa.node()),
                                    'local directory rename, both created')
                 else:
                     actions[f2] = (ACTION_LOCAL_DIR_RENAME_GET, (f, fl2),
                                    'local directory rename - get from %s' % f)
             elif f in copy:
                 f2 = copy[f]
                 if f2 in m2:
                     actions[f] = (ACTION_MERGE, (f2, f, f2, False, pa.node()),
                                   'remote copied from %s' % f2)
                 else:
                     actions[f] = (ACTION_MERGE, (f2, f, f2, True, pa.node()),
                                   'remote moved from %s' % f2)
             elif f not in ma:
                 # local unknown, remote created: the logic is described by the
                 # following table:
                 #
                 # force  branchmerge  different  |  action
                 #   n         *           *      |   create
                 #   y         n           *      |   create
                 #   y         y           n      |   create
                 #   y         y           y      |   merge
                 #
                 # Checking whether the files are different is expensive, so we
                 # don't do that when we can avoid it.
                 if not force:
                     actions[f] = (ACTION_CREATED, (fl2,), 'remote created')
                 elif not branchmerge:
                     actions[f] = (ACTION_CREATED, (fl2,), 'remote created')
                 else:
                     actions[f] = (ACTION_CREATED_MERGE, (fl2, pa.node()),
                                   'remote created, get or merge')
             elif n2 != ma[f]:
                 df = None
                 for d in dirmove:
                     if f.startswith(d):
                         # new file added in a directory that was moved
                         df = dirmove[d] + f[len(d):]
                         break
                 if df is not None and df in m1:
                     actions[df] = (ACTION_MERGE, (df, f, f, False, pa.node()),
                                    'local directory rename - respect move '
                                    'from %s' % f)
                 elif acceptremote:
                     actions[f] = (ACTION_CREATED, (fl2,), 'remote recreating')
                 else:
                     actions[f] = (ACTION_DELETED_CHANGED,
                                   (None, f, f, False, pa.node()),
                                   'prompt deleted/changed')
 
     if repo.ui.configbool('experimental', 'merge.checkpathconflicts'):
         # If we are merging, look for path conflicts.
         checkpathconflicts(repo, wctx, p2, actions)
 
     narrowmatch = repo.narrowmatch()
     if not narrowmatch.always():
         # Updates "actions" in place
         _filternarrowactions(narrowmatch, branchmerge, actions)
 
     return actions, diverge, renamedelete
 
 def _resolvetrivial(repo, wctx, mctx, ancestor, actions):
     """Resolves false conflicts where the nodeid changed but the content
        remained the same."""
     # We force a copy of actions.items() because we're going to mutate
     # actions as we resolve trivial conflicts.
     for f, (m, args, msg) in list(actions.items()):
         if (m == ACTION_CHANGED_DELETED and f in ancestor
             and not wctx[f].cmp(ancestor[f])):
             # local did change but ended up with same content
             actions[f] = ACTION_REMOVE, None, 'prompt same'
         elif (m == ACTION_DELETED_CHANGED and f in ancestor
               and not mctx[f].cmp(ancestor[f])):
             # remote did change but ended up with same content
             del actions[f] # don't get = keep local deleted
 
 def calculateupdates(repo, wctx, mctx, ancestors, branchmerge, force,
                      acceptremote, followcopies, matcher=None,
                      mergeforce=False):
     """Calculate the actions needed to merge mctx into wctx using ancestors"""
     # Avoid cycle.
     from . import sparse
 
     if len(ancestors) == 1: # default
         actions, diverge, renamedelete = manifestmerge(
             repo, wctx, mctx, ancestors[0], branchmerge, force, matcher,
             acceptremote, followcopies)
         _checkunknownfiles(repo, wctx, mctx, force, actions, mergeforce)
 
     else: # only when merge.preferancestor=* - the default
         repo.ui.note(
             _("note: merging %s and %s using bids from ancestors %s\n") %
             (wctx, mctx, _(' and ').join(pycompat.bytestr(anc)
                                             for anc in ancestors)))
 
         # Call for bids
         fbids = {} # mapping filename to bids (action method to list af actions)
         diverge, renamedelete = None, None
         for ancestor in ancestors:
             repo.ui.note(_('\ncalculating bids for ancestor %s\n') % ancestor)
             actions, diverge1, renamedelete1 = manifestmerge(
                 repo, wctx, mctx, ancestor, branchmerge, force, matcher,
                 acceptremote, followcopies, forcefulldiff=True)
             _checkunknownfiles(repo, wctx, mctx, force, actions, mergeforce)
 
             # Track the shortest set of warning on the theory that bid
             # merge will correctly incorporate more information
             if diverge is None or len(diverge1) < len(diverge):
                 diverge = diverge1
             if renamedelete is None or len(renamedelete) < len(renamedelete1):
                 renamedelete = renamedelete1
 
             for f, a in sorted(actions.iteritems()):
                 m, args, msg = a
                 repo.ui.debug(' %s: %s -> %s\n' % (f, msg, m))
                 if f in fbids:
                     d = fbids[f]
                     if m in d:
                         d[m].append(a)
                     else:
                         d[m] = [a]
                 else:
                     fbids[f] = {m: [a]}
 
         # Pick the best bid for each file
         repo.ui.note(_('\nauction for merging merge bids\n'))
         actions = {}
         for f, bids in sorted(fbids.items()):
             # bids is a mapping from action method to list af actions
             # Consensus?
             if len(bids) == 1: # all bids are the same kind of method
                 m, l = list(bids.items())[0]
                 if all(a == l[0] for a in l[1:]): # len(bids) is > 1
                     repo.ui.note(_(" %s: consensus for %s\n") % (f, m))
                     actions[f] = l[0]
                     continue
             # If keep is an option, just do it.
             if ACTION_KEEP in bids:
                 repo.ui.note(_(" %s: picking 'keep' action\n") % f)
                 actions[f] = bids[ACTION_KEEP][0]
                 continue
             # If there are gets and they all agree [how could they not?], do it.
             if ACTION_GET in bids:
                 ga0 = bids[ACTION_GET][0]
                 if all(a == ga0 for a in bids[ACTION_GET][1:]):
                     repo.ui.note(_(" %s: picking 'get' action\n") % f)
                     actions[f] = ga0
                     continue
             # TODO: Consider other simple actions such as mode changes
             # Handle inefficient democrazy.
             repo.ui.note(_(' %s: multiple bids for merge action:\n') % f)
             for m, l in sorted(bids.items()):
                 for _f, args, msg in l:
                     repo.ui.note('  %s -> %s\n' % (msg, m))
             # Pick random action. TODO: Instead, prompt user when resolving
             m, l = list(bids.items())[0]
             repo.ui.warn(_(' %s: ambiguous merge - picked %s action\n') %
                          (f, m))
             actions[f] = l[0]
             continue
         repo.ui.note(_('end of auction\n\n'))
 
     if wctx.rev() is None:
         fractions = _forgetremoved(wctx, mctx, branchmerge)
         actions.update(fractions)
 
     prunedactions = sparse.filterupdatesactions(repo, wctx, mctx, branchmerge,
                                                 actions)
     _resolvetrivial(repo, wctx, mctx, ancestors[0], actions)
 
     return prunedactions, diverge, renamedelete
 
 def _getcwd():
     try:
         return encoding.getcwd()
     except OSError as err:
         if err.errno == errno.ENOENT:
             return None
         raise
 
 def batchremove(repo, wctx, actions):
     """apply removes to the working directory
 
     yields tuples for progress updates
     """
     verbose = repo.ui.verbose
     cwd = _getcwd()
     i = 0
     for f, args, msg in actions:
         repo.ui.debug(" %s: %s -> r\n" % (f, msg))
         if verbose:
             repo.ui.note(_("removing %s\n") % f)
         wctx[f].audit()
         try:
             wctx[f].remove(ignoremissing=True)
         except OSError as inst:
             repo.ui.warn(_("update failed to remove %s: %s!\n") %
                          (f, inst.strerror))
         if i == 100:
             yield i, f
             i = 0
         i += 1
     if i > 0:
         yield i, f
 
     if cwd and not _getcwd():
         # cwd was removed in the course of removing files; print a helpful
         # warning.
         repo.ui.warn(_("current directory was removed\n"
                        "(consider changing to repo root: %s)\n") % repo.root)
 
-def batchget(repo, mctx, wctx, actions):
+def batchget(repo, mctx, wctx, wantfiledata, actions):
     """apply gets to the working directory
 
     mctx is the context to get from
 
-    yields tuples for progress updates
+    Yields arbitrarily many (False, tuple) for progress updates, followed by
+    exactly one (True, filedata). When wantfiledata is false, filedata is an
+    empty list. When wantfiledata is true, filedata[i] is a triple (mode, size,
+    mtime) of the file written for action[i].
     """
+    filedata = []
     verbose = repo.ui.verbose
     fctx = mctx.filectx
     ui = repo.ui
     i = 0
     with repo.wvfs.backgroundclosing(ui, expectedcount=len(actions)):
         for f, (flags, backup), msg in actions:
             repo.ui.debug(" %s: %s -> g\n" % (f, msg))
             if verbose:
                 repo.ui.note(_("getting %s\n") % f)
 
             if backup:
                 # If a file or directory exists with the same name, back that
                 # up.  Otherwise, look to see if there is a file that conflicts
                 # with a directory this file is in, and if so, back that up.
                 conflicting = f
                 if not repo.wvfs.lexists(f):
                     for p in util.finddirs(f):
                         if repo.wvfs.isfileorlink(p):
                             conflicting = p
                             break
                 if repo.wvfs.lexists(conflicting):
                     orig = scmutil.backuppath(ui, repo, conflicting)
                     util.rename(repo.wjoin(conflicting), orig)
-            wctx[f].clearunknown()
+            wfctx = wctx[f]
+            wfctx.clearunknown()
             atomictemp = ui.configbool("experimental", "update.atomic-file")
-            wctx[f].write(fctx(f).data(), flags, backgroundclose=True,
+            size = wfctx.write(fctx(f).data(), flags,
+                               backgroundclose=True,
                           atomictemp=atomictemp)
+            if wantfiledata:
+                s = wfctx.lstat()
+                mode = s.st_mode
+                mtime = s[stat.ST_MTIME]
+                filedata.append((mode, size, mtime)) # for dirstate.normal
             if i == 100:
-                yield i, f
+                yield False, (i, f)
                 i = 0
             i += 1
     if i > 0:
-        yield i, f
+        yield False, (i, f)
+    yield True, filedata
 
 def _prefetchfiles(repo, ctx, actions):
     """Invoke ``scmutil.prefetchfiles()`` for the files relevant to the dict
     of merge actions.  ``ctx`` is the context being merged in."""
 
     # Skipping 'a', 'am', 'f', 'r', 'dm', 'e', 'k', 'p' and 'pr', because they
     # don't touch the context to be merged in.  'cd' is skipped, because
     # changed/deleted never resolves to something from the remote side.
     oplist = [actions[a] for a in (ACTION_GET, ACTION_DELETED_CHANGED,
                                    ACTION_LOCAL_DIR_RENAME_GET, ACTION_MERGE)]
     prefetch = scmutil.prefetchfiles
     matchfiles = scmutil.matchfiles
     prefetch(repo, [ctx.rev()],
              matchfiles(repo,
                         [f for sublist in oplist for f, args, msg in sublist]))
 
 @attr.s(frozen=True)
 class updateresult(object):
     updatedcount = attr.ib()
     mergedcount = attr.ib()
     removedcount = attr.ib()
     unresolvedcount = attr.ib()
 
     def isempty(self):
         return not (self.updatedcount or self.mergedcount
                     or self.removedcount or self.unresolvedcount)
 
 def emptyactions():
     """create an actions dict, to be populated and passed to applyupdates()"""
     return dict((m, [])
                 for m in (
                     ACTION_ADD,
                     ACTION_ADD_MODIFIED,
                     ACTION_FORGET,
                     ACTION_GET,
                     ACTION_CHANGED_DELETED,
                     ACTION_DELETED_CHANGED,
                     ACTION_REMOVE,
                     ACTION_DIR_RENAME_MOVE_LOCAL,
                     ACTION_LOCAL_DIR_RENAME_GET,
                     ACTION_MERGE,
                     ACTION_EXEC,
                     ACTION_KEEP,
                     ACTION_PATH_CONFLICT,
                     ACTION_PATH_CONFLICT_RESOLVE))
 
-def applyupdates(repo, actions, wctx, mctx, overwrite, labels=None):
+def applyupdates(repo, actions, wctx, mctx, overwrite, wantfiledata,
+                 labels=None):
     """apply the merge action list to the working directory
 
     wctx is the working copy context
     mctx is the context to be merged into the working copy
 
-    Return a tuple of counts (updated, merged, removed, unresolved) that
-    describes how many files were affected by the update.
+    Return a tuple of (counts, filedata), where counts is a tuple
+    (updated, merged, removed, unresolved) that describes how many
+    files were affected by the update, and filedata is as described in
+    batchget.
     """
 
     _prefetchfiles(repo, mctx, actions)
 
     updated, merged, removed = 0, 0, 0
     ms = mergestate.clean(repo, wctx.p1().node(), mctx.node(), labels)
     moves = []
     for m, l in actions.items():
         l.sort()
 
     # 'cd' and 'dc' actions are treated like other merge conflicts
     mergeactions = sorted(actions[ACTION_CHANGED_DELETED])
     mergeactions.extend(sorted(actions[ACTION_DELETED_CHANGED]))
     mergeactions.extend(actions[ACTION_MERGE])
     for f, args, msg in mergeactions:
         f1, f2, fa, move, anc = args
         if f == '.hgsubstate': # merged internally
             continue
         if f1 is None:
             fcl = filemerge.absentfilectx(wctx, fa)
         else:
             repo.ui.debug(" preserving %s for resolve of %s\n" % (f1, f))
             fcl = wctx[f1]
         if f2 is None:
             fco = filemerge.absentfilectx(mctx, fa)
         else:
             fco = mctx[f2]
         actx = repo[anc]
         if fa in actx:
             fca = actx[fa]
         else:
             # TODO: move to absentfilectx
             fca = repo.filectx(f1, fileid=nullrev)
         ms.add(fcl, fco, fca, f)
         if f1 != f and move:
             moves.append(f1)
 
     # remove renamed files after safely stored
     for f in moves:
         if wctx[f].lexists():
             repo.ui.debug("removing %s\n" % f)
             wctx[f].audit()
             wctx[f].remove()
 
     numupdates = sum(len(l) for m, l in actions.items()
                      if m != ACTION_KEEP)
     progress = repo.ui.makeprogress(_('updating'), unit=_('files'),
                                     total=numupdates)
 
     if [a for a in actions[ACTION_REMOVE] if a[0] == '.hgsubstate']:
         subrepoutil.submerge(repo, wctx, mctx, wctx, overwrite, labels)
 
     # record path conflicts
     for f, args, msg in actions[ACTION_PATH_CONFLICT]:
         f1, fo = args
         s = repo.ui.status
         s(_("%s: path conflict - a file or link has the same name as a "
             "directory\n") % f)
         if fo == 'l':
             s(_("the local file has been renamed to %s\n") % f1)
         else:
             s(_("the remote file has been renamed to %s\n") % f1)
         s(_("resolve manually then use 'hg resolve --mark %s'\n") % f)
         ms.addpath(f, f1, fo)
         progress.increment(item=f)
 
     # When merging in-memory, we can't support worker processes, so set the
     # per-item cost at 0 in that case.
     cost = 0 if wctx.isinmemory() else 0.001
 
     # remove in parallel (must come before resolving path conflicts and getting)
     prog = worker.worker(repo.ui, cost, batchremove, (repo, wctx),
                          actions[ACTION_REMOVE])
     for i, item in prog:
         progress.increment(step=i, item=item)
     removed = len(actions[ACTION_REMOVE])
 
     # resolve path conflicts (must come before getting)
     for f, args, msg in actions[ACTION_PATH_CONFLICT_RESOLVE]:
         repo.ui.debug(" %s: %s -> pr\n" % (f, msg))
         f0, = args
         if wctx[f0].lexists():
             repo.ui.note(_("moving %s to %s\n") % (f0, f))
             wctx[f].audit()
             wctx[f].write(wctx.filectx(f0).data(), wctx.filectx(f0).flags())
             wctx[f0].remove()
         progress.increment(item=f)
 
     # get in parallel.
     threadsafe = repo.ui.configbool('experimental',
                                     'worker.wdir-get-thread-safe')
-    prog = worker.worker(repo.ui, cost, batchget, (repo, mctx, wctx),
+    prog = worker.worker(repo.ui, cost, batchget,
+                         (repo, mctx, wctx, wantfiledata),
                          actions[ACTION_GET],
-                         threadsafe=threadsafe)
-    for i, item in prog:
+                         threadsafe=threadsafe,
+                         hasretval=True)
+    getfiledata = []
+    for final, res in prog:
+        if final:
+            getfiledata = res
+        else:
+            i, item = res
         progress.increment(step=i, item=item)
     updated = len(actions[ACTION_GET])
 
     if [a for a in actions[ACTION_GET] if a[0] == '.hgsubstate']:
         subrepoutil.submerge(repo, wctx, mctx, wctx, overwrite, labels)
 
     # forget (manifest only, just log it) (must come first)
     for f, args, msg in actions[ACTION_FORGET]:
         repo.ui.debug(" %s: %s -> f\n" % (f, msg))
         progress.increment(item=f)
 
     # re-add (manifest only, just log it)
     for f, args, msg in actions[ACTION_ADD]:
         repo.ui.debug(" %s: %s -> a\n" % (f, msg))
         progress.increment(item=f)
 
     # re-add/mark as modified (manifest only, just log it)
     for f, args, msg in actions[ACTION_ADD_MODIFIED]:
         repo.ui.debug(" %s: %s -> am\n" % (f, msg))
         progress.increment(item=f)
 
     # keep (noop, just log it)
     for f, args, msg in actions[ACTION_KEEP]:
         repo.ui.debug(" %s: %s -> k\n" % (f, msg))
         # no progress
 
     # directory rename, move local
     for f, args, msg in actions[ACTION_DIR_RENAME_MOVE_LOCAL]:
         repo.ui.debug(" %s: %s -> dm\n" % (f, msg))
         progress.increment(item=f)
         f0, flags = args
         repo.ui.note(_("moving %s to %s\n") % (f0, f))
         wctx[f].audit()
         wctx[f].write(wctx.filectx(f0).data(), flags)
         wctx[f0].remove()
         updated += 1
 
     # local directory rename, get
     for f, args, msg in actions[ACTION_LOCAL_DIR_RENAME_GET]:
         repo.ui.debug(" %s: %s -> dg\n" % (f, msg))
         progress.increment(item=f)
         f0, flags = args
         repo.ui.note(_("getting %s to %s\n") % (f0, f))
         wctx[f].write(mctx.filectx(f0).data(), flags)
         updated += 1
 
     # exec
     for f, args, msg in actions[ACTION_EXEC]:
         repo.ui.debug(" %s: %s -> e\n" % (f, msg))
         progress.increment(item=f)
         flags, = args
         wctx[f].audit()
         wctx[f].setflags('l' in flags, 'x' in flags)
         updated += 1
 
     # the ordering is important here -- ms.mergedriver will raise if the merge
     # driver has changed, and we want to be able to bypass it when overwrite is
     # True
     usemergedriver = not overwrite and mergeactions and ms.mergedriver
 
     if usemergedriver:
         if wctx.isinmemory():
             raise error.InMemoryMergeConflictsError("in-memory merge does not "
                                                     "support mergedriver")
         ms.commit()
         proceed = driverpreprocess(repo, ms, wctx, labels=labels)
         # the driver might leave some files unresolved
         unresolvedf = set(ms.unresolved())
         if not proceed:
             # XXX setting unresolved to at least 1 is a hack to make sure we
             # error out
             return updateresult(updated, merged, removed,
                                 max(len(unresolvedf), 1))
         newactions = []
         for f, args, msg in mergeactions:
             if f in unresolvedf:
                 newactions.append((f, args, msg))
         mergeactions = newactions
 
     try:
         # premerge
         tocomplete = []
         for f, args, msg in mergeactions:
             repo.ui.debug(" %s: %s -> m (premerge)\n" % (f, msg))
             progress.increment(item=f)
             if f == '.hgsubstate': # subrepo states need updating
                 subrepoutil.submerge(repo, wctx, mctx, wctx.ancestor(mctx),
                                      overwrite, labels)
                 continue
             wctx[f].audit()
             complete, r = ms.preresolve(f, wctx)
             if not complete:
                 numupdates += 1
                 tocomplete.append((f, args, msg))
 
         # merge
         for f, args, msg in tocomplete:
             repo.ui.debug(" %s: %s -> m (merge)\n" % (f, msg))
             progress.increment(item=f, total=numupdates)
             ms.resolve(f, wctx)
 
     finally:
         ms.commit()
 
     unresolved = ms.unresolvedcount()
 
     if (usemergedriver and not unresolved
         and ms.mdstate() != MERGE_DRIVER_STATE_SUCCESS):
         if not driverconclude(repo, ms, wctx, labels=labels):
             # XXX setting unresolved to at least 1 is a hack to make sure we
             # error out
             unresolved = max(unresolved, 1)
 
         ms.commit()
 
     msupdated, msmerged, msremoved = ms.counts()
     updated += msupdated
     merged += msmerged
     removed += msremoved
 
     extraactions = ms.actions()
     if extraactions:
         mfiles = set(a[0] for a in actions[ACTION_MERGE])
         for k, acts in extraactions.iteritems():
             actions[k].extend(acts)
+            if k == ACTION_GET and wantfiledata:
+                # no filedata until mergestate is updated to provide it
+                getfiledata.extend([None] * len(acts))
             # Remove these files from actions[ACTION_MERGE] as well. This is
             # important because in recordupdates, files in actions[ACTION_MERGE]
             # are processed after files in other actions, and the merge driver
             # might add files to those actions via extraactions above. This can
             # lead to a file being recorded twice, with poor results. This is
             # especially problematic for actions[ACTION_REMOVE] (currently only
             # possible with the merge driver in the initial merge process;
             # interrupted merges don't go through this flow).
             #
             # The real fix here is to have indexes by both file and action so
             # that when the action for a file is changed it is automatically
             # reflected in the other action lists. But that involves a more
             # complex data structure, so this will do for now.
             #
             # We don't need to do the same operation for 'dc' and 'cd' because
             # those lists aren't consulted again.
             mfiles.difference_update(a[0] for a in acts)
 
         actions[ACTION_MERGE] = [a for a in actions[ACTION_MERGE]
                                  if a[0] in mfiles]
 
     progress.complete()
-    return updateresult(updated, merged, removed, unresolved)
+    assert len(getfiledata) == (len(actions[ACTION_GET]) if wantfiledata else 0)
+    return updateresult(updated, merged, removed, unresolved), getfiledata
 
-def recordupdates(repo, actions, branchmerge):
+def recordupdates(repo, actions, branchmerge, getfiledata):
     "record merge actions to the dirstate"
     # remove (must come first)
     for f, args, msg in actions.get(ACTION_REMOVE, []):
         if branchmerge:
             repo.dirstate.remove(f)
         else:
             repo.dirstate.drop(f)
 
     # forget (must come first)
     for f, args, msg in actions.get(ACTION_FORGET, []):
         repo.dirstate.drop(f)
 
     # resolve path conflicts
     for f, args, msg in actions.get(ACTION_PATH_CONFLICT_RESOLVE, []):
         f0, = args
         origf0 = repo.dirstate.copied(f0) or f0
         repo.dirstate.add(f)
         repo.dirstate.copy(origf0, f)
         if f0 == origf0:
             repo.dirstate.remove(f0)
         else:
             repo.dirstate.drop(f0)
 
     # re-add
     for f, args, msg in actions.get(ACTION_ADD, []):
         repo.dirstate.add(f)
 
     # re-add/mark as modified
     for f, args, msg in actions.get(ACTION_ADD_MODIFIED, []):
         if branchmerge:
             repo.dirstate.normallookup(f)
         else:
             repo.dirstate.add(f)
 
     # exec change
     for f, args, msg in actions.get(ACTION_EXEC, []):
         repo.dirstate.normallookup(f)
 
     # keep
     for f, args, msg in actions.get(ACTION_KEEP, []):
         pass
 
     # get
-    for f, args, msg in actions.get(ACTION_GET, []):
+    for i, (f, args, msg) in enumerate(actions.get(ACTION_GET, [])):
         if branchmerge:
             repo.dirstate.otherparent(f)
         else:
-            repo.dirstate.normal(f)
+            parentfiledata = getfiledata[i] if getfiledata else None
+            repo.dirstate.normal(f, parentfiledata=parentfiledata)
 
     # merge
     for f, args, msg in actions.get(ACTION_MERGE, []):
         f1, f2, fa, move, anc = args
         if branchmerge:
             # We've done a branch merge, mark this file as merged
             # so that we properly record the merger later
             repo.dirstate.merge(f)
             if f1 != f2: # copy/rename
                 if move:
                     repo.dirstate.remove(f1)
                 if f1 != f:
                     repo.dirstate.copy(f1, f)
                 else:
                     repo.dirstate.copy(f2, f)
         else:
             # We've update-merged a locally modified file, so
             # we set the dirstate to emulate a normal checkout
             # of that file some time in the past. Thus our
             # merge will appear as a normal local file
             # modification.
             if f2 == f: # file not locally copied/moved
                 repo.dirstate.normallookup(f)
             if move:
                 repo.dirstate.drop(f1)
 
     # directory rename, move local
     for f, args, msg in actions.get(ACTION_DIR_RENAME_MOVE_LOCAL, []):
         f0, flag = args
         if branchmerge:
             repo.dirstate.add(f)
             repo.dirstate.remove(f0)
             repo.dirstate.copy(f0, f)
         else:
             repo.dirstate.normal(f)
             repo.dirstate.drop(f0)
 
     # directory rename, get
     for f, args, msg in actions.get(ACTION_LOCAL_DIR_RENAME_GET, []):
         f0, flag = args
         if branchmerge:
             repo.dirstate.add(f)
             repo.dirstate.copy(f0, f)
         else:
             repo.dirstate.normal(f)
 
 def update(repo, node, branchmerge, force, ancestor=None,
            mergeancestor=False, labels=None, matcher=None, mergeforce=False,
            updatecheck=None, wc=None):
     """
     Perform a merge between the working directory and the given node
 
     node = the node to update to
     branchmerge = whether to merge between branches
     force = whether to force branch merging or file overwriting
     matcher = a matcher to filter file lists (dirstate not updated)
     mergeancestor = whether it is merging with an ancestor. If true,
       we should accept the incoming changes for any prompts that occur.
       If false, merging with an ancestor (fast-forward) is only allowed
       between different named branches. This flag is used by rebase extension
       as a temporary fix and should be avoided in general.
     labels = labels to use for base, local and other
     mergeforce = whether the merge was run with 'merge --force' (deprecated): if
       this is True, then 'force' should be True as well.
 
     The table below shows all the behaviors of the update command given the
     -c/--check and -C/--clean or no options, whether the working directory is
     dirty, whether a revision is specified, and the relationship of the parent
     rev to the target rev (linear or not). Match from top first. The -n
     option doesn't exist on the command line, but represents the
     experimental.updatecheck=noconflict option.
 
     This logic is tested by test-update-branches.t.
 
     -c  -C  -n  -m  dirty  rev  linear  |  result
      y   y   *   *    *     *     *     |    (1)
      y   *   y   *    *     *     *     |    (1)
      y   *   *   y    *     *     *     |    (1)
      *   y   y   *    *     *     *     |    (1)
      *   y   *   y    *     *     *     |    (1)
      *   *   y   y    *     *     *     |    (1)
      *   *   *   *    *     n     n     |     x
      *   *   *   *    n     *     *     |    ok
      n   n   n   n    y     *     y     |   merge
      n   n   n   n    y     y     n     |    (2)
      n   n   n   y    y     *     *     |   merge
      n   n   y   n    y     *     *     |  merge if no conflict
      n   y   n   n    y     *     *     |  discard
      y   n   n   n    y     *     *     |    (3)
 
     x = can't happen
     * = don't-care
     1 = incompatible options (checked in commands.py)
     2 = abort: uncommitted changes (commit or update --clean to discard changes)
     3 = abort: uncommitted changes (checked in commands.py)
 
     The merge is performed inside ``wc``, a workingctx-like objects. It defaults
     to repo[None] if None is passed.
 
     Return the same tuple as applyupdates().
     """
     # Avoid cycle.
     from . import sparse
 
     # This function used to find the default destination if node was None, but
     # that's now in destutil.py.
     assert node is not None
     if not branchmerge and not force:
         # TODO: remove the default once all callers that pass branchmerge=False
         # and force=False pass a value for updatecheck. We may want to allow
         # updatecheck='abort' to better suppport some of these callers.
         if updatecheck is None:
             updatecheck = 'linear'
         assert updatecheck in ('none', 'linear', 'noconflict')
     # If we're doing a partial update, we need to skip updating
     # the dirstate, so make a note of any partial-ness to the
     # update here.
     if matcher is None or matcher.always():
         partial = False
     else:
         partial = True
     with repo.wlock():
         if wc is None:
             wc = repo[None]
         pl = wc.parents()
         p1 = pl[0]
         p2 = repo[node]
         if ancestor is not None:
             pas = [repo[ancestor]]
         else:
             if repo.ui.configlist('merge', 'preferancestor') == ['*']:
                 cahs = repo.changelog.commonancestorsheads(p1.node(), p2.node())
                 pas = [repo[anc] for anc in (sorted(cahs) or [nullid])]
             else:
                 pas = [p1.ancestor(p2, warn=branchmerge)]
 
         fp1, fp2, xp1, xp2 = p1.node(), p2.node(), bytes(p1), bytes(p2)
 
         overwrite = force and not branchmerge
         ### check phase
         if not overwrite:
             if len(pl) > 1:
                 raise error.Abort(_("outstanding uncommitted merge"))
             ms = mergestate.read(repo)
             if list(ms.unresolved()):
                 raise error.Abort(_("outstanding merge conflicts"))
         if branchmerge:
             if pas == [p2]:
                 raise error.Abort(_("merging with a working directory ancestor"
                                    " has no effect"))
             elif pas == [p1]:
                 if not mergeancestor and wc.branch() == p2.branch():
                     raise error.Abort(_("nothing to merge"),
                                      hint=_("use 'hg update' "
                                             "or check 'hg heads'"))
             if not force and (wc.files() or wc.deleted()):
                 raise error.Abort(_("uncommitted changes"),
                                  hint=_("use 'hg status' to list changes"))
             if not wc.isinmemory():
                 for s in sorted(wc.substate):
                     wc.sub(s).bailifchanged()
 
         elif not overwrite:
             if p1 == p2: # no-op update
                 # call the hooks and exit early
                 repo.hook('preupdate', throw=True, parent1=xp2, parent2='')
                 repo.hook('update', parent1=xp2, parent2='', error=0)
                 return updateresult(0, 0, 0, 0)
 
             if (updatecheck == 'linear' and
                     pas not in ([p1], [p2])):  # nonlinear
                 dirty = wc.dirty(missing=True)
                 if dirty:
                     # Branching is a bit strange to ensure we do the minimal
                     # amount of call to obsutil.foreground.
                     foreground = obsutil.foreground(repo, [p1.node()])
                     # note: the <node> variable contains a random identifier
                     if repo[node].node() in foreground:
                         pass # allow updating to successors
                     else:
                         msg = _("uncommitted changes")
                         hint = _("commit or update --clean to discard changes")
                         raise error.UpdateAbort(msg, hint=hint)
                 else:
                     # Allow jumping branches if clean and specific rev given
                     pass
 
         if overwrite:
             pas = [wc]
         elif not branchmerge:
             pas = [p1]
 
         # deprecated config: merge.followcopies
         followcopies = repo.ui.configbool('merge', 'followcopies')
         if overwrite:
             followcopies = False
         elif not pas[0]:
             followcopies = False
         if not branchmerge and not wc.dirty(missing=True):
             followcopies = False
 
         ### calculate phase
         actionbyfile, diverge, renamedelete = calculateupdates(
             repo, wc, p2, pas, branchmerge, force, mergeancestor,
             followcopies, matcher=matcher, mergeforce=mergeforce)
 
         if updatecheck == 'noconflict':
             for f, (m, args, msg) in actionbyfile.iteritems():
                 if m not in (ACTION_GET, ACTION_KEEP, ACTION_EXEC,
                              ACTION_REMOVE, ACTION_PATH_CONFLICT_RESOLVE):
                     msg = _("conflicting changes")
                     hint = _("commit or update --clean to discard changes")
                     raise error.Abort(msg, hint=hint)
 
         # Prompt and create actions. Most of this is in the resolve phase
         # already, but we can't handle .hgsubstate in filemerge or
         # subrepoutil.submerge yet so we have to keep prompting for it.
         if '.hgsubstate' in actionbyfile:
             f = '.hgsubstate'
             m, args, msg = actionbyfile[f]
             prompts = filemerge.partextras(labels)
             prompts['f'] = f
             if m == ACTION_CHANGED_DELETED:
                 if repo.ui.promptchoice(
                     _("local%(l)s changed %(f)s which other%(o)s deleted\n"
                       "use (c)hanged version or (d)elete?"
                       "$$ &Changed $$ &Delete") % prompts, 0):
                     actionbyfile[f] = (ACTION_REMOVE, None, 'prompt delete')
                 elif f in p1:
                     actionbyfile[f] = (ACTION_ADD_MODIFIED, None, 'prompt keep')
                 else:
                     actionbyfile[f] = (ACTION_ADD, None, 'prompt keep')
             elif m == ACTION_DELETED_CHANGED:
                 f1, f2, fa, move, anc = args
                 flags = p2[f2].flags()
                 if repo.ui.promptchoice(
                     _("other%(o)s changed %(f)s which local%(l)s deleted\n"
                       "use (c)hanged version or leave (d)eleted?"
                       "$$ &Changed $$ &Deleted") % prompts, 0) == 0:
                     actionbyfile[f] = (ACTION_GET, (flags, False),
                                        'prompt recreating')
                 else:
                     del actionbyfile[f]
 
         # Convert to dictionary-of-lists format
         actions = emptyactions()
         for f, (m, args, msg) in actionbyfile.iteritems():
             if m not in actions:
                 actions[m] = []
             actions[m].append((f, args, msg))
 
         if not util.fscasesensitive(repo.path):
             # check collision between files only in p2 for clean update
             if (not branchmerge and
                 (force or not wc.dirty(missing=True, branch=False))):
                 _checkcollision(repo, p2.manifest(), None)
             else:
                 _checkcollision(repo, wc.manifest(), actions)
 
         # divergent renames
         for f, fl in sorted(diverge.iteritems()):
             repo.ui.warn(_("note: possible conflict - %s was renamed "
                            "multiple times to:\n") % f)
             for nf in sorted(fl):
                 repo.ui.warn(" %s\n" % nf)
 
         # rename and delete
         for f, fl in sorted(renamedelete.iteritems()):
             repo.ui.warn(_("note: possible conflict - %s was deleted "
                            "and renamed to:\n") % f)
             for nf in sorted(fl):
                 repo.ui.warn(" %s\n" % nf)
 
         ### apply phase
         if not branchmerge: # just jump to the new rev
             fp1, fp2, xp1, xp2 = fp2, nullid, xp2, ''
         if not partial and not wc.isinmemory():
             repo.hook('preupdate', throw=True, parent1=xp1, parent2=xp2)
             # note that we're in the middle of an update
             repo.vfs.write('updatestate', p2.hex())
 
         # Advertise fsmonitor when its presence could be useful.
         #
         # We only advertise when performing an update from an empty working
         # directory. This typically only occurs during initial clone.
         #
         # We give users a mechanism to disable the warning in case it is
         # annoying.
         #
         # We only allow on Linux and MacOS because that's where fsmonitor is
         # considered stable.
         fsmonitorwarning = repo.ui.configbool('fsmonitor', 'warn_when_unused')
         fsmonitorthreshold = repo.ui.configint('fsmonitor',
                                                'warn_update_file_count')
         try:
             # avoid cycle: extensions -> cmdutil -> merge
             from . import extensions
             extensions.find('fsmonitor')
             fsmonitorenabled = repo.ui.config('fsmonitor', 'mode') != 'off'
             # We intentionally don't look at whether fsmonitor has disabled
             # itself because a) fsmonitor may have already printed a warning
             # b) we only care about the config state here.
         except KeyError:
             fsmonitorenabled = False
 
         if (fsmonitorwarning
                 and not fsmonitorenabled
                 and p1.node() == nullid
                 and len(actions[ACTION_GET]) >= fsmonitorthreshold
                 and pycompat.sysplatform.startswith(('linux', 'darwin'))):
             repo.ui.warn(
                 _('(warning: large working directory being used without '
                   'fsmonitor enabled; enable fsmonitor to improve performance; '
                   'see "hg help -e fsmonitor")\n'))
 
-        stats = applyupdates(repo, actions, wc, p2, overwrite, labels=labels)
+        updatedirstate = not partial and not wc.isinmemory()
+        wantfiledata = updatedirstate and not branchmerge
+        stats, getfiledata = applyupdates(repo, actions, wc, p2, overwrite,
+                                          wantfiledata, labels=labels)
 
-        if not partial and not wc.isinmemory():
+        if updatedirstate:
             with repo.dirstate.parentchange():
                 repo.setparents(fp1, fp2)
-                recordupdates(repo, actions, branchmerge)
+                recordupdates(repo, actions, branchmerge, getfiledata)
                 # update completed, clear state
                 util.unlink(repo.vfs.join('updatestate'))
 
                 if not branchmerge:
                     repo.dirstate.setbranch(p2.branch())
 
     # If we're updating to a location, clean up any stale temporary includes
     # (ex: this happens during hg rebase --abort).
     if not branchmerge:
         sparse.prunetemporaryincludes(repo)
 
     if not partial:
         repo.hook('update', parent1=xp1, parent2=xp2,
                   error=stats.unresolvedcount)
     return stats
 
 def graft(repo, ctx, pctx, labels=None, keepparent=False,
           keepconflictparent=False):
     """Do a graft-like merge.
 
     This is a merge where the merge ancestor is chosen such that one
     or more changesets are grafted onto the current changeset. In
     addition to the merge, this fixes up the dirstate to include only
     a single parent (if keepparent is False) and tries to duplicate any
     renames/copies appropriately.
 
     ctx - changeset to rebase
     pctx - merge base, usually ctx.p1()
     labels - merge labels eg ['local', 'graft']
     keepparent - keep second parent if any
     keepconflictparent - if unresolved, keep parent used for the merge
 
     """
     # If we're grafting a descendant onto an ancestor, be sure to pass
     # mergeancestor=True to update. This does two things: 1) allows the merge if
     # the destination is the same as the parent of the ctx (so we can use graft
     # to copy commits), and 2) informs update that the incoming changes are
     # newer than the destination so it doesn't prompt about "remote changed foo
     # which local deleted".
     mergeancestor = repo.changelog.isancestor(repo['.'].node(), ctx.node())
 
     stats = update(repo, ctx.node(), True, True, pctx.node(),
                    mergeancestor=mergeancestor, labels=labels)
 
 
     if keepconflictparent and stats.unresolvedcount:
         pother = ctx.node()
     else:
         pother = nullid
         parents = ctx.parents()
         if keepparent and len(parents) == 2 and pctx in parents:
             parents.remove(pctx)
             pother = parents[0].node()
 
     with repo.dirstate.parentchange():
         repo.setparents(repo['.'].node(), pother)
         repo.dirstate.write(repo.currenttransaction())
         # fix up dirstate for copies and renames
         copies.duplicatecopies(repo, repo[None], ctx.rev(), pctx.rev())
     return stats
 
 def purge(repo, matcher, ignored=False, removeemptydirs=True,
           removefiles=True, abortonerror=False, noop=False):
     """Purge the working directory of untracked files.
 
     ``matcher`` is a matcher configured to scan the working directory -
     potentially a subset.
 
     ``ignored`` controls whether ignored files should also be purged.
 
     ``removeemptydirs`` controls whether empty directories should be removed.
 
     ``removefiles`` controls whether files are removed.
 
     ``abortonerror`` causes an exception to be raised if an error occurs
     deleting a file or directory.
 
     ``noop`` controls whether to actually remove files. If not defined, actions
     will be taken.
 
     Returns an iterable of relative paths in the working directory that were
     or would be removed.
     """
 
     def remove(removefn, path):
         try:
             removefn(path)
         except OSError:
             m = _('%s cannot be removed') % path
             if abortonerror:
                 raise error.Abort(m)
             else:
                 repo.ui.warn(_('warning: %s\n') % m)
 
     # There's no API to copy a matcher. So mutate the passed matcher and
     # restore it when we're done.
     oldexplicitdir = matcher.explicitdir
     oldtraversedir = matcher.traversedir
 
     res = []
 
     try:
         if removeemptydirs:
             directories = []
             matcher.explicitdir = matcher.traversedir = directories.append
 
         status = repo.status(match=matcher, ignored=ignored, unknown=True)
 
         if removefiles:
             for f in sorted(status.unknown + status.ignored):
                 if not noop:
                     repo.ui.note(_('removing file %s\n') % f)
                     remove(repo.wvfs.unlink, f)
                 res.append(f)
 
         if removeemptydirs:
             for f in sorted(directories, reverse=True):
                 if matcher(f) and not repo.wvfs.listdir(f):
                     if not noop:
                         repo.ui.note(_('removing directory %s\n') % f)
                         remove(repo.wvfs.rmdir, f)
                     res.append(f)
 
         return res
 
     finally:
         matcher.explicitdir = oldexplicitdir
         matcher.traversedir = oldtraversedir

mercurial/narrowspec.py

@@ -1,317 +1,317
 # narrowspec.py - methods for working with a narrow view of a repository
 #
 # Copyright 2017 Google, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 from .i18n import _
 from . import (
     error,
     match as matchmod,
     merge,
     repository,
     scmutil,
     sparse,
     util,
 )
 
 # The file in .hg/store/ that indicates which paths exit in the store
 FILENAME = 'narrowspec'
 # The file in .hg/ that indicates which paths exit in the dirstate
 DIRSTATE_FILENAME = 'narrowspec.dirstate'
 
 # Pattern prefixes that are allowed in narrow patterns. This list MUST
 # only contain patterns that are fast and safe to evaluate. Keep in mind
 # that patterns are supplied by clients and executed on remote servers
 # as part of wire protocol commands. That means that changes to this
 # data structure influence the wire protocol and should not be taken
 # lightly - especially removals.
 VALID_PREFIXES = (
     b'path:',
     b'rootfilesin:',
 )
 
 def normalizesplitpattern(kind, pat):
     """Returns the normalized version of a pattern and kind.
 
     Returns a tuple with the normalized kind and normalized pattern.
     """
     pat = pat.rstrip('/')
     _validatepattern(pat)
     return kind, pat
 
 def _numlines(s):
     """Returns the number of lines in s, including ending empty lines."""
     # We use splitlines because it is Unicode-friendly and thus Python 3
     # compatible. However, it does not count empty lines at the end, so trick
     # it by adding a character at the end.
     return len((s + 'x').splitlines())
 
 def _validatepattern(pat):
     """Validates the pattern and aborts if it is invalid.
 
     Patterns are stored in the narrowspec as newline-separated
     POSIX-style bytestring paths. There's no escaping.
     """
 
     # We use newlines as separators in the narrowspec file, so don't allow them
     # in patterns.
     if _numlines(pat) > 1:
         raise error.Abort(_('newlines are not allowed in narrowspec paths'))
 
     components = pat.split('/')
     if '.' in components or '..' in components:
         raise error.Abort(_('"." and ".." are not allowed in narrowspec paths'))
 
 def normalizepattern(pattern, defaultkind='path'):
     """Returns the normalized version of a text-format pattern.
 
     If the pattern has no kind, the default will be added.
     """
     kind, pat = matchmod._patsplit(pattern, defaultkind)
     return '%s:%s' % normalizesplitpattern(kind, pat)
 
 def parsepatterns(pats):
     """Parses an iterable of patterns into a typed pattern set.
 
     Patterns are assumed to be ``path:`` if no prefix is present.
     For safety and performance reasons, only some prefixes are allowed.
     See ``validatepatterns()``.
 
     This function should be used on patterns that come from the user to
     normalize and validate them to the internal data structure used for
     representing patterns.
     """
     res = {normalizepattern(orig) for orig in pats}
     validatepatterns(res)
     return res
 
 def validatepatterns(pats):
     """Validate that patterns are in the expected data structure and format.
 
     And that is a set of normalized patterns beginning with ``path:`` or
     ``rootfilesin:``.
 
     This function should be used to validate internal data structures
     and patterns that are loaded from sources that use the internal,
     prefixed pattern representation (but can't necessarily be fully trusted).
     """
     if not isinstance(pats, set):
         raise error.ProgrammingError('narrow patterns should be a set; '
                                      'got %r' % pats)
 
     for pat in pats:
         if not pat.startswith(VALID_PREFIXES):
             # Use a Mercurial exception because this can happen due to user
             # bugs (e.g. manually updating spec file).
             raise error.Abort(_('invalid prefix on narrow pattern: %s') % pat,
                               hint=_('narrow patterns must begin with one of '
                                      'the following: %s') %
                                    ', '.join(VALID_PREFIXES))
 
 def format(includes, excludes):
     output = '[include]\n'
     for i in sorted(includes - excludes):
         output += i + '\n'
     output += '[exclude]\n'
     for e in sorted(excludes):
         output += e + '\n'
     return output
 
 def match(root, include=None, exclude=None):
     if not include:
         # Passing empty include and empty exclude to matchmod.match()
         # gives a matcher that matches everything, so explicitly use
         # the nevermatcher.
         return matchmod.never()
     return matchmod.match(root, '', [], include=include or [],
                           exclude=exclude or [])
 
 def parseconfig(ui, spec):
     # maybe we should care about the profiles returned too
     includepats, excludepats, profiles = sparse.parseconfig(ui, spec, 'narrow')
     if profiles:
         raise error.Abort(_("including other spec files using '%include' is not"
                             " supported in narrowspec"))
 
     validatepatterns(includepats)
     validatepatterns(excludepats)
 
     return includepats, excludepats
 
 def load(repo):
     # Treat "narrowspec does not exist" the same as "narrowspec file exists
     # and is empty".
     spec = repo.svfs.tryread(FILENAME)
     return parseconfig(repo.ui, spec)
 
 def save(repo, includepats, excludepats):
     validatepatterns(includepats)
     validatepatterns(excludepats)
     spec = format(includepats, excludepats)
     repo.svfs.write(FILENAME, spec)
 
 def copytoworkingcopy(repo):
     spec = repo.svfs.read(FILENAME)
     repo.vfs.write(DIRSTATE_FILENAME, spec)
 
 def savebackup(repo, backupname):
     if repository.NARROW_REQUIREMENT not in repo.requirements:
         return
     svfs = repo.svfs
     svfs.tryunlink(backupname)
     util.copyfile(svfs.join(FILENAME), svfs.join(backupname), hardlink=True)
 
 def restorebackup(repo, backupname):
     if repository.NARROW_REQUIREMENT not in repo.requirements:
         return
     util.rename(repo.svfs.join(backupname), repo.svfs.join(FILENAME))
 
 def savewcbackup(repo, backupname):
     if repository.NARROW_REQUIREMENT not in repo.requirements:
         return
     vfs = repo.vfs
     vfs.tryunlink(backupname)
     # It may not exist in old repos
     if vfs.exists(DIRSTATE_FILENAME):
         util.copyfile(vfs.join(DIRSTATE_FILENAME), vfs.join(backupname),
                       hardlink=True)
 
 def restorewcbackup(repo, backupname):
     if repository.NARROW_REQUIREMENT not in repo.requirements:
         return
     # It may not exist in old repos
     if repo.vfs.exists(backupname):
         util.rename(repo.vfs.join(backupname), repo.vfs.join(DIRSTATE_FILENAME))
 
 def clearwcbackup(repo, backupname):
     if repository.NARROW_REQUIREMENT not in repo.requirements:
         return
     repo.vfs.tryunlink(backupname)
 
 def restrictpatterns(req_includes, req_excludes, repo_includes, repo_excludes):
     r""" Restricts the patterns according to repo settings,
     results in a logical AND operation
 
     :param req_includes: requested includes
     :param req_excludes: requested excludes
     :param repo_includes: repo includes
     :param repo_excludes: repo excludes
     :return: include patterns, exclude patterns, and invalid include patterns.
 
     >>> restrictpatterns({'f1','f2'}, {}, ['f1'], [])
     (set(['f1']), {}, [])
     >>> restrictpatterns({'f1'}, {}, ['f1','f2'], [])
     (set(['f1']), {}, [])
     >>> restrictpatterns({'f1/fc1', 'f3/fc3'}, {}, ['f1','f2'], [])
     (set(['f1/fc1']), {}, [])
     >>> restrictpatterns({'f1_fc1'}, {}, ['f1','f2'], [])
     ([], set(['path:.']), [])
     >>> restrictpatterns({'f1/../f2/fc2'}, {}, ['f1','f2'], [])
     (set(['f2/fc2']), {}, [])
     >>> restrictpatterns({'f1/../f3/fc3'}, {}, ['f1','f2'], [])
     ([], set(['path:.']), [])
     >>> restrictpatterns({'f1/$non_exitent_var'}, {}, ['f1','f2'], [])
     (set(['f1/$non_exitent_var']), {}, [])
     """
     res_excludes = set(req_excludes)
     res_excludes.update(repo_excludes)
     invalid_includes = []
     if not req_includes:
         res_includes = set(repo_includes)
     elif 'path:.' not in repo_includes:
         res_includes = []
         for req_include in req_includes:
             req_include = util.expandpath(util.normpath(req_include))
             if req_include in repo_includes:
                 res_includes.append(req_include)
                 continue
             valid = False
             for repo_include in repo_includes:
                 if req_include.startswith(repo_include + '/'):
                     valid = True
                     res_includes.append(req_include)
                     break
             if not valid:
                 invalid_includes.append(req_include)
         if len(res_includes) == 0:
             res_excludes = {'path:.'}
         else:
             res_includes = set(res_includes)
     else:
         res_includes = set(req_includes)
     return res_includes, res_excludes, invalid_includes
 
 # These two are extracted for extensions (specifically for Google's CitC file
 # system)
 def _deletecleanfiles(repo, files):
     for f in files:
         repo.wvfs.unlinkpath(f)
 
 def _writeaddedfiles(repo, pctx, files):
     actions = merge.emptyactions()
     addgaction = actions[merge.ACTION_GET].append
     mf = repo['.'].manifest()
     for f in files:
         if not repo.wvfs.exists(f):
             addgaction((f, (mf.flags(f), False), "narrowspec updated"))
     merge.applyupdates(repo, actions, wctx=repo[None],
-                       mctx=repo['.'], overwrite=False)
+                       mctx=repo['.'], overwrite=False, wantfiledata=False)
 
 def checkworkingcopynarrowspec(repo):
     # Avoid infinite recursion when updating the working copy
     if getattr(repo, '_updatingnarrowspec', False):
         return
     storespec = repo.svfs.tryread(FILENAME)
     wcspec = repo.vfs.tryread(DIRSTATE_FILENAME)
     if wcspec != storespec:
         raise error.Abort(_("working copy's narrowspec is stale"),
                           hint=_("run 'hg tracked --update-working-copy'"))
 
 def updateworkingcopy(repo, assumeclean=False):
     """updates the working copy and dirstate from the store narrowspec
 
     When assumeclean=True, files that are not known to be clean will also
     be deleted. It is then up to the caller to make sure they are clean.
     """
     oldspec = repo.vfs.tryread(DIRSTATE_FILENAME)
     newspec = repo.svfs.tryread(FILENAME)
     repo._updatingnarrowspec = True
 
     oldincludes, oldexcludes = parseconfig(repo.ui, oldspec)
     newincludes, newexcludes = parseconfig(repo.ui, newspec)
     oldmatch = match(repo.root, include=oldincludes, exclude=oldexcludes)
     newmatch = match(repo.root, include=newincludes, exclude=newexcludes)
     addedmatch = matchmod.differencematcher(newmatch, oldmatch)
     removedmatch = matchmod.differencematcher(oldmatch, newmatch)
 
     ds = repo.dirstate
     lookup, status = ds.status(removedmatch, subrepos=[], ignored=True,
                                clean=True, unknown=True)
     trackeddirty = status.modified + status.added
     clean = status.clean
     if assumeclean:
         assert not trackeddirty
         clean.extend(lookup)
     else:
         trackeddirty.extend(lookup)
     _deletecleanfiles(repo, clean)
     uipathfn = scmutil.getuipathfn(repo)
     for f in sorted(trackeddirty):
         repo.ui.status(_('not deleting possibly dirty file %s\n') % uipathfn(f))
     for f in sorted(status.unknown):
         repo.ui.status(_('not deleting unknown file %s\n') % uipathfn(f))
     for f in sorted(status.ignored):
         repo.ui.status(_('not deleting ignored file %s\n') % uipathfn(f))
     for f in clean + trackeddirty:
         ds.drop(f)
 
     pctx = repo['.']
     newfiles = [f for f in pctx.manifest().walk(addedmatch) if f not in ds]
     for f in newfiles:
         ds.normallookup(f)
     _writeaddedfiles(repo, pctx, newfiles)
     repo._updatingnarrowspec = False

mercurial/sparse.py

@@ -1,700 +1,702
 # sparse.py - functionality for sparse checkouts
 #
 # Copyright 2014 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import hashlib
 import os
 
 from .i18n import _
 from .node import (
     hex,
     nullid,
 )
 from . import (
     error,
     match as matchmod,
     merge as mergemod,
     pathutil,
     pycompat,
     scmutil,
     util,
 )
 
 # Whether sparse features are enabled. This variable is intended to be
 # temporary to facilitate porting sparse to core. It should eventually be
 # a per-repo option, possibly a repo requirement.
 enabled = False
 
 def parseconfig(ui, raw, action):
     """Parse sparse config file content.
 
     action is the command which is trigerring this read, can be narrow, sparse
 
     Returns a tuple of includes, excludes, and profiles.
     """
     includes = set()
     excludes = set()
     profiles = set()
     current = None
     havesection = False
 
     for line in raw.split('\n'):
         line = line.strip()
         if not line or line.startswith('#'):
             # empty or comment line, skip
             continue
         elif line.startswith('%include '):
             line = line[9:].strip()
             if line:
                 profiles.add(line)
         elif line == '[include]':
             if havesection and current != includes:
                 # TODO pass filename into this API so we can report it.
                 raise error.Abort(_('%(action)s config cannot have includes '
                                     'after excludes') % {'action': action})
             havesection = True
             current = includes
             continue
         elif line == '[exclude]':
             havesection = True
             current = excludes
         elif line:
             if current is None:
                 raise error.Abort(_('%(action)s config entry outside of '
                                     'section: %(line)s')
                                   % {'action': action, 'line': line},
                                   hint=_('add an [include] or [exclude] line '
                                          'to declare the entry type'))
 
             if line.strip().startswith('/'):
                 ui.warn(_('warning: %(action)s profile cannot use'
                           ' paths starting with /, ignoring %(line)s\n')
                         % {'action': action, 'line': line})
                 continue
             current.add(line)
 
     return includes, excludes, profiles
 
 # Exists as separate function to facilitate monkeypatching.
 def readprofile(repo, profile, changeid):
     """Resolve the raw content of a sparse profile file."""
     # TODO add some kind of cache here because this incurs a manifest
     # resolve and can be slow.
     return repo.filectx(profile, changeid=changeid).data()
 
 def patternsforrev(repo, rev):
     """Obtain sparse checkout patterns for the given rev.
 
     Returns a tuple of iterables representing includes, excludes, and
     patterns.
     """
     # Feature isn't enabled. No-op.
     if not enabled:
         return set(), set(), set()
 
     raw = repo.vfs.tryread('sparse')
     if not raw:
         return set(), set(), set()
 
     if rev is None:
         raise error.Abort(_('cannot parse sparse patterns from working '
                             'directory'))
 
     includes, excludes, profiles = parseconfig(repo.ui, raw, 'sparse')
     ctx = repo[rev]
 
     if profiles:
         visited = set()
         while profiles:
             profile = profiles.pop()
             if profile in visited:
                 continue
 
             visited.add(profile)
 
             try:
                 raw = readprofile(repo, profile, rev)
             except error.ManifestLookupError:
                 msg = (
                     "warning: sparse profile '%s' not found "
                     "in rev %s - ignoring it\n" % (profile, ctx))
                 # experimental config: sparse.missingwarning
                 if repo.ui.configbool(
                         'sparse', 'missingwarning'):
                     repo.ui.warn(msg)
                 else:
                     repo.ui.debug(msg)
                 continue
 
             pincludes, pexcludes, subprofs = parseconfig(repo.ui, raw, 'sparse')
             includes.update(pincludes)
             excludes.update(pexcludes)
             profiles.update(subprofs)
 
         profiles = visited
 
     if includes:
         includes.add('.hg*')
 
     return includes, excludes, profiles
 
 def activeconfig(repo):
     """Determine the active sparse config rules.
 
     Rules are constructed by reading the current sparse config and bringing in
     referenced profiles from parents of the working directory.
     """
     revs = [repo.changelog.rev(node) for node in
             repo.dirstate.parents() if node != nullid]
 
     allincludes = set()
     allexcludes = set()
     allprofiles = set()
 
     for rev in revs:
         includes, excludes, profiles = patternsforrev(repo, rev)
         allincludes |= includes
         allexcludes |= excludes
         allprofiles |= profiles
 
     return allincludes, allexcludes, allprofiles
 
 def configsignature(repo, includetemp=True):
     """Obtain the signature string for the current sparse configuration.
 
     This is used to construct a cache key for matchers.
     """
     cache = repo._sparsesignaturecache
 
     signature = cache.get('signature')
 
     if includetemp:
         tempsignature = cache.get('tempsignature')
     else:
         tempsignature = '0'
 
     if signature is None or (includetemp and tempsignature is None):
         signature = hex(hashlib.sha1(repo.vfs.tryread('sparse')).digest())
         cache['signature'] = signature
 
         if includetemp:
             raw = repo.vfs.tryread('tempsparse')
             tempsignature = hex(hashlib.sha1(raw).digest())
             cache['tempsignature'] = tempsignature
 
     return '%s %s' % (signature, tempsignature)
 
 def writeconfig(repo, includes, excludes, profiles):
     """Write the sparse config file given a sparse configuration."""
     with repo.vfs('sparse', 'wb') as fh:
         for p in sorted(profiles):
             fh.write('%%include %s\n' % p)
 
         if includes:
             fh.write('[include]\n')
             for i in sorted(includes):
                 fh.write(i)
                 fh.write('\n')
 
         if excludes:
             fh.write('[exclude]\n')
             for e in sorted(excludes):
                 fh.write(e)
                 fh.write('\n')
 
     repo._sparsesignaturecache.clear()
 
 def readtemporaryincludes(repo):
     raw = repo.vfs.tryread('tempsparse')
     if not raw:
         return set()
 
     return set(raw.split('\n'))
 
 def writetemporaryincludes(repo, includes):
     repo.vfs.write('tempsparse', '\n'.join(sorted(includes)))
     repo._sparsesignaturecache.clear()
 
 def addtemporaryincludes(repo, additional):
     includes = readtemporaryincludes(repo)
     for i in additional:
         includes.add(i)
     writetemporaryincludes(repo, includes)
 
 def prunetemporaryincludes(repo):
     if not enabled or not repo.vfs.exists('tempsparse'):
         return
 
     s = repo.status()
     if s.modified or s.added or s.removed or s.deleted:
         # Still have pending changes. Don't bother trying to prune.
         return
 
     sparsematch = matcher(repo, includetemp=False)
     dirstate = repo.dirstate
     actions = []
     dropped = []
     tempincludes = readtemporaryincludes(repo)
     for file in tempincludes:
         if file in dirstate and not sparsematch(file):
             message = _('dropping temporarily included sparse files')
             actions.append((file, None, message))
             dropped.append(file)
 
     typeactions = mergemod.emptyactions()
     typeactions['r'] = actions
-    mergemod.applyupdates(repo, typeactions, repo[None], repo['.'], False)
+    mergemod.applyupdates(repo, typeactions, repo[None], repo['.'], False,
+                          wantfiledata=False)
 
     # Fix dirstate
     for file in dropped:
         dirstate.drop(file)
 
     repo.vfs.unlink('tempsparse')
     repo._sparsesignaturecache.clear()
     msg = _('cleaned up %d temporarily added file(s) from the '
             'sparse checkout\n')
     repo.ui.status(msg % len(tempincludes))
 
 def forceincludematcher(matcher, includes):
     """Returns a matcher that returns true for any of the forced includes
     before testing against the actual matcher."""
     kindpats = [('path', include, '') for include in includes]
     includematcher = matchmod.includematcher('', kindpats)
     return matchmod.unionmatcher([includematcher, matcher])
 
 def matcher(repo, revs=None, includetemp=True):
     """Obtain a matcher for sparse working directories for the given revs.
 
     If multiple revisions are specified, the matcher is the union of all
     revs.
 
     ``includetemp`` indicates whether to use the temporary sparse profile.
     """
     # If sparse isn't enabled, sparse matcher matches everything.
     if not enabled:
         return matchmod.always()
 
     if not revs or revs == [None]:
         revs = [repo.changelog.rev(node)
                 for node in repo.dirstate.parents() if node != nullid]
 
     signature = configsignature(repo, includetemp=includetemp)
 
     key = '%s %s' % (signature, ' '.join(map(pycompat.bytestr, revs)))
 
     result = repo._sparsematchercache.get(key)
     if result:
         return result
 
     matchers = []
     for rev in revs:
         try:
             includes, excludes, profiles = patternsforrev(repo, rev)
 
             if includes or excludes:
                 matcher = matchmod.match(repo.root, '', [],
                                          include=includes, exclude=excludes,
                                          default='relpath')
                 matchers.append(matcher)
         except IOError:
             pass
 
     if not matchers:
         result = matchmod.always()
     elif len(matchers) == 1:
         result = matchers[0]
     else:
         result = matchmod.unionmatcher(matchers)
 
     if includetemp:
         tempincludes = readtemporaryincludes(repo)
         result = forceincludematcher(result, tempincludes)
 
     repo._sparsematchercache[key] = result
 
     return result
 
 def filterupdatesactions(repo, wctx, mctx, branchmerge, actions):
     """Filter updates to only lay out files that match the sparse rules."""
     if not enabled:
         return actions
 
     oldrevs = [pctx.rev() for pctx in wctx.parents()]
     oldsparsematch = matcher(repo, oldrevs)
 
     if oldsparsematch.always():
         return actions
 
     files = set()
     prunedactions = {}
 
     if branchmerge:
         # If we're merging, use the wctx filter, since we're merging into
         # the wctx.
         sparsematch = matcher(repo, [wctx.p1().rev()])
     else:
         # If we're updating, use the target context's filter, since we're
         # moving to the target context.
         sparsematch = matcher(repo, [mctx.rev()])
 
     temporaryfiles = []
     for file, action in actions.iteritems():
         type, args, msg = action
         files.add(file)
         if sparsematch(file):
             prunedactions[file] = action
         elif type == 'm':
             temporaryfiles.append(file)
             prunedactions[file] = action
         elif branchmerge:
             if type != 'k':
                 temporaryfiles.append(file)
                 prunedactions[file] = action
         elif type == 'f':
             prunedactions[file] = action
         elif file in wctx:
             prunedactions[file] = ('r', args, msg)
 
         if branchmerge and type == mergemod.ACTION_MERGE:
             f1, f2, fa, move, anc = args
             if not sparsematch(f1):
                 temporaryfiles.append(f1)
 
     if len(temporaryfiles) > 0:
         repo.ui.status(_('temporarily included %d file(s) in the sparse '
                          'checkout for merging\n') % len(temporaryfiles))
         addtemporaryincludes(repo, temporaryfiles)
 
         # Add the new files to the working copy so they can be merged, etc
         actions = []
         message = 'temporarily adding to sparse checkout'
         wctxmanifest = repo[None].manifest()
         for file in temporaryfiles:
             if file in wctxmanifest:
                 fctx = repo[None][file]
                 actions.append((file, (fctx.flags(), False), message))
 
         typeactions = mergemod.emptyactions()
         typeactions['g'] = actions
         mergemod.applyupdates(repo, typeactions, repo[None], repo['.'],
-                              False)
+                              False, wantfiledata=False)
 
         dirstate = repo.dirstate
         for file, flags, msg in actions:
             dirstate.normal(file)
 
     profiles = activeconfig(repo)[2]
     changedprofiles = profiles & files
     # If an active profile changed during the update, refresh the checkout.
     # Don't do this during a branch merge, since all incoming changes should
     # have been handled by the temporary includes above.
     if changedprofiles and not branchmerge:
         mf = mctx.manifest()
         for file in mf:
             old = oldsparsematch(file)
             new = sparsematch(file)
             if not old and new:
                 flags = mf.flags(file)
                 prunedactions[file] = ('g', (flags, False), '')
             elif old and not new:
                 prunedactions[file] = ('r', [], '')
 
     return prunedactions
 
 def refreshwdir(repo, origstatus, origsparsematch, force=False):
     """Refreshes working directory by taking sparse config into account.
 
     The old status and sparse matcher is compared against the current sparse
     matcher.
 
     Will abort if a file with pending changes is being excluded or included
     unless ``force`` is True.
     """
     # Verify there are no pending changes
     pending = set()
     pending.update(origstatus.modified)
     pending.update(origstatus.added)
     pending.update(origstatus.removed)
     sparsematch = matcher(repo)
     abort = False
 
     for f in pending:
         if not sparsematch(f):
             repo.ui.warn(_("pending changes to '%s'\n") % f)
             abort = not force
 
     if abort:
         raise error.Abort(_('could not update sparseness due to pending '
                             'changes'))
 
     # Calculate actions
     dirstate = repo.dirstate
     ctx = repo['.']
     added = []
     lookup = []
     dropped = []
     mf = ctx.manifest()
     files = set(mf)
 
     actions = {}
 
     for file in files:
         old = origsparsematch(file)
         new = sparsematch(file)
         # Add files that are newly included, or that don't exist in
         # the dirstate yet.
         if (new and not old) or (old and new and not file in dirstate):
             fl = mf.flags(file)
             if repo.wvfs.exists(file):
                 actions[file] = ('e', (fl,), '')
                 lookup.append(file)
             else:
                 actions[file] = ('g', (fl, False), '')
                 added.append(file)
         # Drop files that are newly excluded, or that still exist in
         # the dirstate.
         elif (old and not new) or (not old and not new and file in dirstate):
             dropped.append(file)
             if file not in pending:
                 actions[file] = ('r', [], '')
 
     # Verify there are no pending changes in newly included files
     abort = False
     for file in lookup:
         repo.ui.warn(_("pending changes to '%s'\n") % file)
         abort = not force
     if abort:
         raise error.Abort(_('cannot change sparseness due to pending '
                             'changes (delete the files or use '
                             '--force to bring them back dirty)'))
 
     # Check for files that were only in the dirstate.
     for file, state in dirstate.iteritems():
         if not file in files:
             old = origsparsematch(file)
             new = sparsematch(file)
             if old and not new:
                 dropped.append(file)
 
     # Apply changes to disk
     typeactions = mergemod.emptyactions()
     for f, (m, args, msg) in actions.iteritems():
         typeactions[m].append((f, args, msg))
 
-    mergemod.applyupdates(repo, typeactions, repo[None], repo['.'], False)
+    mergemod.applyupdates(repo, typeactions, repo[None], repo['.'], False,
+                          wantfiledata=False)
 
     # Fix dirstate
     for file in added:
         dirstate.normal(file)
 
     for file in dropped:
         dirstate.drop(file)
 
     for file in lookup:
         # File exists on disk, and we're bringing it back in an unknown state.
         dirstate.normallookup(file)
 
     return added, dropped, lookup
 
 def aftercommit(repo, node):
     """Perform actions after a working directory commit."""
     # This function is called unconditionally, even if sparse isn't
     # enabled.
     ctx = repo[node]
 
     profiles = patternsforrev(repo, ctx.rev())[2]
 
     # profiles will only have data if sparse is enabled.
     if profiles & set(ctx.files()):
         origstatus = repo.status()
         origsparsematch = matcher(repo)
         refreshwdir(repo, origstatus, origsparsematch, force=True)
 
     prunetemporaryincludes(repo)
 
 def _updateconfigandrefreshwdir(repo, includes, excludes, profiles,
                                 force=False, removing=False):
     """Update the sparse config and working directory state."""
     raw = repo.vfs.tryread('sparse')
     oldincludes, oldexcludes, oldprofiles = parseconfig(repo.ui, raw, 'sparse')
 
     oldstatus = repo.status()
     oldmatch = matcher(repo)
     oldrequires = set(repo.requirements)
 
     # TODO remove this try..except once the matcher integrates better
     # with dirstate. We currently have to write the updated config
     # because that will invalidate the matcher cache and force a
     # re-read. We ideally want to update the cached matcher on the
     # repo instance then flush the new config to disk once wdir is
     # updated. But this requires massive rework to matcher() and its
     # consumers.
 
     if 'exp-sparse' in oldrequires and removing:
         repo.requirements.discard('exp-sparse')
         scmutil.writerequires(repo.vfs, repo.requirements)
     elif 'exp-sparse' not in oldrequires:
         repo.requirements.add('exp-sparse')
         scmutil.writerequires(repo.vfs, repo.requirements)
 
     try:
         writeconfig(repo, includes, excludes, profiles)
         return refreshwdir(repo, oldstatus, oldmatch, force=force)
     except Exception:
         if repo.requirements != oldrequires:
             repo.requirements.clear()
             repo.requirements |= oldrequires
             scmutil.writerequires(repo.vfs, repo.requirements)
         writeconfig(repo, oldincludes, oldexcludes, oldprofiles)
         raise
 
 def clearrules(repo, force=False):
     """Clears include/exclude rules from the sparse config.
 
     The remaining sparse config only has profiles, if defined. The working
     directory is refreshed, as needed.
     """
     with repo.wlock():
         raw = repo.vfs.tryread('sparse')
         includes, excludes, profiles = parseconfig(repo.ui, raw, 'sparse')
 
         if not includes and not excludes:
             return
 
         _updateconfigandrefreshwdir(repo, set(), set(), profiles, force=force)
 
 def importfromfiles(repo, opts, paths, force=False):
     """Import sparse config rules from files.
 
     The updated sparse config is written out and the working directory
     is refreshed, as needed.
     """
     with repo.wlock():
         # read current configuration
         raw = repo.vfs.tryread('sparse')
         includes, excludes, profiles = parseconfig(repo.ui, raw, 'sparse')
         aincludes, aexcludes, aprofiles = activeconfig(repo)
 
         # Import rules on top; only take in rules that are not yet
         # part of the active rules.
         changed = False
         for p in paths:
             with util.posixfile(util.expandpath(p), mode='rb') as fh:
                 raw = fh.read()
 
             iincludes, iexcludes, iprofiles = parseconfig(repo.ui, raw,
                                                           'sparse')
             oldsize = len(includes) + len(excludes) + len(profiles)
             includes.update(iincludes - aincludes)
             excludes.update(iexcludes - aexcludes)
             profiles.update(iprofiles - aprofiles)
             if len(includes) + len(excludes) + len(profiles) > oldsize:
                 changed = True
 
         profilecount = includecount = excludecount = 0
         fcounts = (0, 0, 0)
 
         if changed:
             profilecount = len(profiles - aprofiles)
             includecount = len(includes - aincludes)
             excludecount = len(excludes - aexcludes)
 
             fcounts = map(len, _updateconfigandrefreshwdir(
                 repo, includes, excludes, profiles, force=force))
 
         printchanges(repo.ui, opts, profilecount, includecount, excludecount,
                      *fcounts)
 
 def updateconfig(repo, pats, opts, include=False, exclude=False, reset=False,
                  delete=False, enableprofile=False, disableprofile=False,
                  force=False, usereporootpaths=False):
     """Perform a sparse config update.
 
     Only one of the actions may be performed.
 
     The new config is written out and a working directory refresh is performed.
     """
     with repo.wlock():
         raw = repo.vfs.tryread('sparse')
         oldinclude, oldexclude, oldprofiles = parseconfig(repo.ui, raw,
                                                           'sparse')
 
         if reset:
             newinclude = set()
             newexclude = set()
             newprofiles = set()
         else:
             newinclude = set(oldinclude)
             newexclude = set(oldexclude)
             newprofiles = set(oldprofiles)
 
         if any(os.path.isabs(pat) for pat in pats):
             raise error.Abort(_('paths cannot be absolute'))
 
         if not usereporootpaths:
             # let's treat paths as relative to cwd
             root, cwd = repo.root, repo.getcwd()
             abspats = []
             for kindpat in pats:
                 kind, pat = matchmod._patsplit(kindpat, None)
                 if kind in matchmod.cwdrelativepatternkinds or kind is None:
                     ap = ((kind + ':' if kind else '') +
                           pathutil.canonpath(root, cwd, pat))
                     abspats.append(ap)
                 else:
                     abspats.append(kindpat)
             pats = abspats
 
         if include:
             newinclude.update(pats)
         elif exclude:
             newexclude.update(pats)
         elif enableprofile:
             newprofiles.update(pats)
         elif disableprofile:
             newprofiles.difference_update(pats)
         elif delete:
             newinclude.difference_update(pats)
             newexclude.difference_update(pats)
 
         profilecount = (len(newprofiles - oldprofiles) -
                         len(oldprofiles - newprofiles))
         includecount = (len(newinclude - oldinclude) -
                         len(oldinclude - newinclude))
         excludecount = (len(newexclude - oldexclude) -
                         len(oldexclude - newexclude))
 
         fcounts = map(len, _updateconfigandrefreshwdir(
             repo, newinclude, newexclude, newprofiles, force=force,
             removing=reset))
 
         printchanges(repo.ui, opts, profilecount, includecount,
                      excludecount, *fcounts)
 
 def printchanges(ui, opts, profilecount=0, includecount=0, excludecount=0,
                  added=0, dropped=0, conflicting=0):
     """Print output summarizing sparse config changes."""
     with ui.formatter('sparse', opts) as fm:
         fm.startitem()
         fm.condwrite(ui.verbose, 'profiles_added', _('Profiles changed: %d\n'),
                      profilecount)
         fm.condwrite(ui.verbose, 'include_rules_added',
                      _('Include rules changed: %d\n'), includecount)
         fm.condwrite(ui.verbose, 'exclude_rules_added',
                      _('Exclude rules changed: %d\n'), excludecount)
 
         # In 'plain' verbose mode, mergemod.applyupdates already outputs what
         # files are added or removed outside of the templating formatter
         # framework. No point in repeating ourselves in that case.
         if not fm.isplain():
             fm.condwrite(ui.verbose, 'files_added', _('Files added: %d\n'),
                          added)
             fm.condwrite(ui.verbose, 'files_dropped', _('Files dropped: %d\n'),
                          dropped)
             fm.condwrite(ui.verbose, 'files_conflicting',
                          _('Files conflicting: %d\n'), conflicting)

tests/test-dirstate-race2.t

@@ -1,39 +1,37
 Checking the size/permissions/file-type of files stored in the
 dirstate after an update where the files are changed concurrently
 outside of hg's control.
 
   $ hg init repo
   $ cd repo
   $ echo a > a
   $ hg commit -qAm _
   $ echo aa > a
   $ hg commit -m _
 
   $ hg debugdirstate --no-dates
   n 644          3 (set  |unset)               a (re)
 
   $ cat >> $TESTTMP/dirstaterace.py << EOF
   > from mercurial import (
   >     extensions,
   >     merge,
   > )
   > def extsetup(ui):
   >     extensions.wrapfunction(merge, 'applyupdates', wrap)
   > def wrap(orig, *args, **kwargs):
   >     res = orig(*args, **kwargs)
   >     with open("a", "w"):
   >         pass # just truncate the file
   >     return res
   > EOF
 
 Do an update where file 'a' is changed between hg writing it to disk
-and hg writing the dirstate. It results in a corrupted dirstate, which
-stores the wrong size, and thus hg status shows spuriously modified
-files.
+and hg writing the dirstate. The dirstate is correct nonetheless, and
+so hg status correctly shows a as clean.
 
   $ hg up -r 0 --config extensions.race=$TESTTMP/dirstaterace.py
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ hg debugdirstate --no-dates
-  n 644          0 (set  |unset)               a (re)
+  n 644          2 (set  |unset)               a (re)
   $ echo a > a; hg status; hg diff
-  M a

tests/test-dirstate.t

@@ -1,93 +1,93
 ------ Test dirstate._dirs refcounting
 
   $ hg init t
   $ cd t
   $ mkdir -p a/b/c/d
   $ touch a/b/c/d/x
   $ touch a/b/c/d/y
   $ touch a/b/c/d/z
   $ hg ci -Am m
   adding a/b/c/d/x
   adding a/b/c/d/y
   adding a/b/c/d/z
   $ hg mv a z
   moving a/b/c/d/x to z/b/c/d/x
   moving a/b/c/d/y to z/b/c/d/y
   moving a/b/c/d/z to z/b/c/d/z
 
 Test name collisions
 
   $ rm z/b/c/d/x
   $ mkdir z/b/c/d/x
   $ touch z/b/c/d/x/y
   $ hg add z/b/c/d/x/y
   abort: file 'z/b/c/d/x' in dirstate clashes with 'z/b/c/d/x/y'
   [255]
   $ rm -rf z/b/c/d
   $ touch z/b/c/d
   $ hg add z/b/c/d
   abort: directory 'z/b/c/d' already in dirstate
   [255]
 
   $ cd ..
 
 Issue1790: dirstate entry locked into unset if file mtime is set into
 the future
 
 Prepare test repo:
 
   $ hg init u
   $ cd u
   $ echo a > a
   $ hg add
   adding a
   $ hg ci -m1
 
 Set mtime of a into the future:
 
   $ touch -t 202101011200 a
 
 Status must not set a's entry to unset (issue1790):
 
   $ hg status
   $ hg debugstate
   n 644          2 2021-01-01 12:00:00 a
 
 Test modulo storage/comparison of absurd dates:
 
 #if no-aix
   $ touch -t 195001011200 a
   $ hg st
   $ hg debugstate
   n 644          2 2018-01-19 15:14:08 a
 #endif
 
 Verify that exceptions during a dirstate change leave the dirstate
 coherent (issue4353)
 
   $ cat > ../dirstateexception.py <<EOF
   > from __future__ import absolute_import
   > from mercurial import (
   >   error,
   >   extensions,
   >   merge,
   > )
   > 
-  > def wraprecordupdates(orig, repo, actions, branchmerge):
+  > def wraprecordupdates(*args):
   >     raise error.Abort("simulated error while recording dirstateupdates")
   > 
   > def reposetup(ui, repo):
   >     extensions.wrapfunction(merge, 'recordupdates', wraprecordupdates)
   > EOF
 
   $ hg rm a
   $ hg commit -m 'rm a'
   $ echo "[extensions]" >> .hg/hgrc
   $ echo "dirstateex=../dirstateexception.py" >> .hg/hgrc
   $ hg up 0
   abort: simulated error while recording dirstateupdates
   [255]
   $ hg log -r . -T '{rev}\n'
   1
   $ hg status
   ? a