convert: fix convert dropping p2 contents during filemap merge

When converting a merge commit using a filemap convert (i.e. when moving
contents from the root of the repo into subdir1/), convert would silently drop
the entire contents of the target repo's p2. This was because when it built the
target commit, it did so by taking the target p1 and adding only the files that
changed in the source repo's merge commit.

This breaks in the case where the target repo has files that are unrelated to
the source repo (like in the case where you use convert to import a repo as a
subdirectory of another).

The fix is to use Mercurial's merge logic to detect which files in p2 we should
carry over to the merge. It follows three rules:

1) if the file belongs to the source, don't try to merge it. Rely on the list of
files provided to putcommit to be correct.

2) if the file requires merging or user input (change vs deleted), throw an
exception. We don't have enough info to do this.

3) if p2 has the newest, non-merge-requiring version of the file, take it

I've also added a test to cover this issue.

hgext/convert/hg.py

@@ -1,568 +1,616 b''
 # hg.py - hg backend for convert extension
 #
 #  Copyright 2005-2009 Matt Mackall <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # Notes for hg->hg conversion:
 #
 # * Old versions of Mercurial didn't trim the whitespace from the ends
 #   of commit messages, but new versions do.  Changesets created by
 #   those older versions, then converted, may thus have different
 #   hashes for changesets that are otherwise identical.
 #
 # * Using "--config convert.hg.saverev=true" will make the source
 #   identifier to be stored in the converted revision. This will cause
 #   the converted revision to have a different identity than the
 #   source.
 
 
 import os, time, cStringIO
 from mercurial.i18n import _
 from mercurial.node import bin, hex, nullid
 from mercurial import hg, util, context, bookmarks, error, scmutil, exchange
 from mercurial import phases
+from mercurial import merge as mergemod
 
 from common import NoRepo, commit, converter_source, converter_sink, mapfile
 
 import re
 sha1re = re.compile(r'\b[0-9a-f]{12,40}\b')
 
 class mercurial_sink(converter_sink):
     def __init__(self, ui, path):
         converter_sink.__init__(self, ui, path)
         self.branchnames = ui.configbool('convert', 'hg.usebranchnames', True)
         self.clonebranches = ui.configbool('convert', 'hg.clonebranches', False)
         self.tagsbranch = ui.config('convert', 'hg.tagsbranch', 'default')
         self.lastbranch = None
         if os.path.isdir(path) and len(os.listdir(path)) > 0:
             try:
                 self.repo = hg.repository(self.ui, path)
                 if not self.repo.local():
                     raise NoRepo(_('%s is not a local Mercurial repository')
                                  % path)
             except error.RepoError as err:
                 ui.traceback()
                 raise NoRepo(err.args[0])
         else:
             try:
                 ui.status(_('initializing destination %s repository\n') % path)
                 self.repo = hg.repository(self.ui, path, create=True)
                 if not self.repo.local():
                     raise NoRepo(_('%s is not a local Mercurial repository')
                                  % path)
                 self.created.append(path)
             except error.RepoError:
                 ui.traceback()
                 raise NoRepo(_("could not create hg repository %s as sink")
                              % path)
         self.lock = None
         self.wlock = None
         self.filemapmode = False
         self.subrevmaps = {}
 
     def before(self):
         self.ui.debug('run hg sink pre-conversion action\n')
         self.wlock = self.repo.wlock()
         self.lock = self.repo.lock()
 
     def after(self):
         self.ui.debug('run hg sink post-conversion action\n')
         if self.lock:
             self.lock.release()
         if self.wlock:
             self.wlock.release()
 
     def revmapfile(self):
         return self.repo.join("shamap")
 
     def authorfile(self):
         return self.repo.join("authormap")
 
     def setbranch(self, branch, pbranches):
         if not self.clonebranches:
             return
 
         setbranch = (branch != self.lastbranch)
         self.lastbranch = branch
         if not branch:
             branch = 'default'
         pbranches = [(b[0], b[1] and b[1] or 'default') for b in pbranches]
         if pbranches:
             pbranch = pbranches[0][1]
         else:
             pbranch = 'default'
 
         branchpath = os.path.join(self.path, branch)
         if setbranch:
             self.after()
             try:
                 self.repo = hg.repository(self.ui, branchpath)
             except Exception:
                 self.repo = hg.repository(self.ui, branchpath, create=True)
             self.before()
 
         # pbranches may bring revisions from other branches (merge parents)
         # Make sure we have them, or pull them.
         missings = {}
         for b in pbranches:
             try:
                 self.repo.lookup(b[0])
             except Exception:
                 missings.setdefault(b[1], []).append(b[0])
 
         if missings:
             self.after()
             for pbranch, heads in sorted(missings.iteritems()):
                 pbranchpath = os.path.join(self.path, pbranch)
                 prepo = hg.peer(self.ui, {}, pbranchpath)
                 self.ui.note(_('pulling from %s into %s\n') % (pbranch, branch))
                 exchange.pull(self.repo, prepo,
                               [prepo.lookup(h) for h in heads])
             self.before()
 
     def _rewritetags(self, source, revmap, data):
         fp = cStringIO.StringIO()
         for line in data.splitlines():
             s = line.split(' ', 1)
             if len(s) != 2:
                 continue
             revid = revmap.get(source.lookuprev(s[0]))
             if not revid:
                 if s[0] == hex(nullid):
                     revid = s[0]
                 else:
                     continue
             fp.write('%s %s\n' % (revid, s[1]))
         return fp.getvalue()
 
     def _rewritesubstate(self, source, data):
         fp = cStringIO.StringIO()
         for line in data.splitlines():
             s = line.split(' ', 1)
             if len(s) != 2:
                 continue
 
             revid = s[0]
             subpath = s[1]
             if revid != hex(nullid):
                 revmap = self.subrevmaps.get(subpath)
                 if revmap is None:
                     revmap = mapfile(self.ui,
                                      self.repo.wjoin(subpath, '.hg/shamap'))
                     self.subrevmaps[subpath] = revmap
 
                     # It is reasonable that one or more of the subrepos don't
                     # need to be converted, in which case they can be cloned
                     # into place instead of converted.  Therefore, only warn
                     # once.
                     msg = _('no ".hgsubstate" updates will be made for "%s"\n')
                     if len(revmap) == 0:
                         sub = self.repo.wvfs.reljoin(subpath, '.hg')
 
                         if self.repo.wvfs.exists(sub):
                             self.ui.warn(msg % subpath)
 
                 newid = revmap.get(revid)
                 if not newid:
                     if len(revmap) > 0:
                         self.ui.warn(_("%s is missing from %s/.hg/shamap\n") %
                                      (revid, subpath))
                 else:
                     revid = newid
 
             fp.write('%s %s\n' % (revid, subpath))
 
         return fp.getvalue()
 
+    def _calculatemergedfiles(self, source, p1ctx, p2ctx):
+        """Calculates the files from p2 that we need to pull in when merging p1
+        and p2, given that the merge is coming from the given source.
+
+        This prevents us from losing files that only exist in the target p2 and
+        that don't come from the source repo (like if you're merging multiple
+        repositories together).
+        """
+        anc = [p1ctx.ancestor(p2ctx)]
+        # Calculate what files are coming from p2
+        actions, diverge, rename = mergemod.calculateupdates(
+            self.repo, p1ctx, p2ctx, anc,
+            True,  # branchmerge
+            True,  # force
+            False, # partial
+            False, # acceptremote
+            False, # followcopies
+        )
+
+        for file, (action, info, msg) in actions.iteritems():
+            if source.targetfilebelongstosource(file):
+                # If the file belongs to the source repo, ignore the p2
+                # since it will be covered by the existing fileset.
+                continue
+
+            # If the file requires actual merging, abort. We don't have enough
+            # context to resolve merges correctly.
+            if action in ['m', 'dm', 'cd', 'dc']:
+                raise util.Abort(_("unable to convert merge commit "
+                    "since target parents do not merge cleanly (file "
+                    "%s, parents %s and %s)") % (file, p1ctx,
+                                                 p2ctx))
+            elif action == 'k':
+                # 'keep' means nothing changed from p1
+                continue
+            else:
+                # Any other change means we want to take the p2 version
+                yield file
+
     def putcommit(self, files, copies, parents, commit, source, revmap, full,
                   cleanp2):
         files = dict(files)
 
         def getfilectx(repo, memctx, f):
-            if p2ctx and f in cleanp2 and f not in copies:
+            if p2ctx and f in p2files and f not in copies:
                 self.ui.debug('reusing %s from p2\n' % f)
                 return p2ctx[f]
             try:
                 v = files[f]
             except KeyError:
                 return None
             data, mode = source.getfile(f, v)
             if data is None:
                 return None
             if f == '.hgtags':
                 data = self._rewritetags(source, revmap, data)
             if f == '.hgsubstate':
                 data = self._rewritesubstate(source, data)
             return context.memfilectx(self.repo, f, data, 'l' in mode,
                                       'x' in mode, copies.get(f))
 
         pl = []
         for p in parents:
             if p not in pl:
                 pl.append(p)
         parents = pl
         nparents = len(parents)
         if self.filemapmode and nparents == 1:
             m1node = self.repo.changelog.read(bin(parents[0]))[0]
             parent = parents[0]
 
         if len(parents) < 2:
             parents.append(nullid)
         if len(parents) < 2:
             parents.append(nullid)
         p2 = parents.pop(0)
 
         text = commit.desc
 
         sha1s = re.findall(sha1re, text)
         for sha1 in sha1s:
             oldrev = source.lookuprev(sha1)
             newrev = revmap.get(oldrev)
             if newrev is not None:
                 text = text.replace(sha1, newrev[:len(sha1)])
 
         extra = commit.extra.copy()
 
         sourcename = self.repo.ui.config('convert', 'hg.sourcename')
         if sourcename:
             extra['convert_source'] = sourcename
 
         for label in ('source', 'transplant_source', 'rebase_source',
                       'intermediate-source'):
             node = extra.get(label)
 
             if node is None:
                 continue
 
             # Only transplant stores its reference in binary
             if label == 'transplant_source':
                 node = hex(node)
 
             newrev = revmap.get(node)
             if newrev is not None:
                 if label == 'transplant_source':
                     newrev = bin(newrev)
 
                 extra[label] = newrev
 
         if self.branchnames and commit.branch:
             extra['branch'] = commit.branch
         if commit.rev and commit.saverev:
             extra['convert_revision'] = commit.rev
 
         while parents:
             p1 = p2
             p2 = parents.pop(0)
+            p1ctx = self.repo[p1]
             p2ctx = None
             if p2 != nullid:
                 p2ctx = self.repo[p2]
             fileset = set(files)
             if full:
                 fileset.update(self.repo[p1])
                 fileset.update(self.repo[p2])
+
+            if p2ctx:
+                p2files = set(cleanp2)
+                for file in self._calculatemergedfiles(source, p1ctx, p2ctx):
+                    p2files.add(file)
+                    fileset.add(file)
+
             ctx = context.memctx(self.repo, (p1, p2), text, fileset,
                                  getfilectx, commit.author, commit.date, extra)
 
             # We won't know if the conversion changes the node until after the
             # commit, so copy the source's phase for now.
             self.repo.ui.setconfig('phases', 'new-commit',
                                    phases.phasenames[commit.phase], 'convert')
 
             tr = self.repo.transaction("convert")
 
             try:
                 node = hex(self.repo.commitctx(ctx))
 
                 # If the node value has changed, but the phase is lower than
                 # draft, set it back to draft since it hasn't been exposed
                 # anywhere.
                 if commit.rev != node:
                     ctx = self.repo[node]
                     if ctx.phase() < phases.draft:
                         phases.retractboundary(self.repo, tr, phases.draft,
                                                [ctx.node()])
                 tr.close()
             finally:
                 tr.release()
 
             text = "(octopus merge fixup)\n"
             p2 = node
 
         if self.filemapmode and nparents == 1:
             man = self.repo.manifest
             mnode = self.repo.changelog.read(bin(p2))[0]
             closed = 'close' in commit.extra
             if not closed and not man.cmp(m1node, man.revision(mnode)):
                 self.ui.status(_("filtering out empty revision\n"))
                 self.repo.rollback(force=True)
                 return parent
         return p2
 
     def puttags(self, tags):
         try:
             parentctx = self.repo[self.tagsbranch]
             tagparent = parentctx.node()
         except error.RepoError:
             parentctx = None
             tagparent = nullid
 
         oldlines = set()
         for branch, heads in self.repo.branchmap().iteritems():
             for h in heads:
                 if '.hgtags' in self.repo[h]:
                     oldlines.update(
                         set(self.repo[h]['.hgtags'].data().splitlines(True)))
         oldlines = sorted(list(oldlines))
 
         newlines = sorted([("%s %s\n" % (tags[tag], tag)) for tag in tags])
         if newlines == oldlines:
             return None, None
 
         # if the old and new tags match, then there is nothing to update
         oldtags = set()
         newtags = set()
         for line in oldlines:
             s = line.strip().split(' ', 1)
             if len(s) != 2:
                 continue
             oldtags.add(s[1])
         for line in newlines:
             s = line.strip().split(' ', 1)
             if len(s) != 2:
                 continue
             if s[1] not in oldtags:
                 newtags.add(s[1].strip())
 
         if not newtags:
             return None, None
 
         data = "".join(newlines)
         def getfilectx(repo, memctx, f):
             return context.memfilectx(repo, f, data, False, False, None)
 
         self.ui.status(_("updating tags\n"))
         date = "%s 0" % int(time.mktime(time.gmtime()))
         extra = {'branch': self.tagsbranch}
         ctx = context.memctx(self.repo, (tagparent, None), "update tags",
                              [".hgtags"], getfilectx, "convert-repo", date,
                              extra)
         node = self.repo.commitctx(ctx)
         return hex(node), hex(tagparent)
 
     def setfilemapmode(self, active):
         self.filemapmode = active
 
     def putbookmarks(self, updatedbookmark):
         if not len(updatedbookmark):
             return
 
         self.ui.status(_("updating bookmarks\n"))
         destmarks = self.repo._bookmarks
         for bookmark in updatedbookmark:
             destmarks[bookmark] = bin(updatedbookmark[bookmark])
         destmarks.write()
 
     def hascommitfrommap(self, rev):
         # the exact semantics of clonebranches is unclear so we can't say no
         return rev in self.repo or self.clonebranches
 
     def hascommitforsplicemap(self, rev):
         if rev not in self.repo and self.clonebranches:
             raise util.Abort(_('revision %s not found in destination '
                                'repository (lookups with clonebranches=true '
                                'are not implemented)') % rev)
         return rev in self.repo
 
 class mercurial_source(converter_source):
     def __init__(self, ui, path, revs=None):
         converter_source.__init__(self, ui, path, revs)
         if revs and len(revs) > 1:
             raise util.Abort(_("mercurial source does not support specifying "
                                "multiple revisions"))
         self.ignoreerrors = ui.configbool('convert', 'hg.ignoreerrors', False)
         self.ignored = set()
         self.saverev = ui.configbool('convert', 'hg.saverev', False)
         try:
             self.repo = hg.repository(self.ui, path)
             # try to provoke an exception if this isn't really a hg
             # repo, but some other bogus compatible-looking url
             if not self.repo.local():
                 raise error.RepoError
         except error.RepoError:
             ui.traceback()
             raise NoRepo(_("%s is not a local Mercurial repository") % path)
         self.lastrev = None
         self.lastctx = None
         self._changescache = None, None
         self.convertfp = None
         # Restrict converted revisions to startrev descendants
         startnode = ui.config('convert', 'hg.startrev')
         hgrevs = ui.config('convert', 'hg.revs')
         if hgrevs is None:
             if startnode is not None:
                 try:
                     startnode = self.repo.lookup(startnode)
                 except error.RepoError:
                     raise util.Abort(_('%s is not a valid start revision')
                                      % startnode)
                 startrev = self.repo.changelog.rev(startnode)
                 children = {startnode: 1}
                 for r in self.repo.changelog.descendants([startrev]):
                     children[self.repo.changelog.node(r)] = 1
                 self.keep = children.__contains__
             else:
                 self.keep = util.always
             if revs:
                 self._heads = [self.repo[revs[0]].node()]
             else:
                 self._heads = self.repo.heads()
         else:
             if revs or startnode is not None:
                 raise util.Abort(_('hg.revs cannot be combined with '
                                    'hg.startrev or --rev'))
             nodes = set()
             parents = set()
             for r in scmutil.revrange(self.repo, [hgrevs]):
                 ctx = self.repo[r]
                 nodes.add(ctx.node())
                 parents.update(p.node() for p in ctx.parents())
             self.keep = nodes.__contains__
             self._heads = nodes - parents
 
     def changectx(self, rev):
         if self.lastrev != rev:
             self.lastctx = self.repo[rev]
             self.lastrev = rev
         return self.lastctx
 
     def parents(self, ctx):
         return [p for p in ctx.parents() if p and self.keep(p.node())]
 
     def getheads(self):
         return [hex(h) for h in self._heads if self.keep(h)]
 
     def getfile(self, name, rev):
         try:
             fctx = self.changectx(rev)[name]
             return fctx.data(), fctx.flags()
         except error.LookupError:
             return None, None
 
     def getchanges(self, rev, full):
         ctx = self.changectx(rev)
         parents = self.parents(ctx)
         if full or not parents:
             files = copyfiles = ctx.manifest()
         if parents:
             if self._changescache[0] == rev:
                 m, a, r = self._changescache[1]
             else:
                 m, a, r = self.repo.status(parents[0].node(), ctx.node())[:3]
             if not full:
                 files = m + a + r
             copyfiles = m + a
         # getcopies() is also run for roots and before filtering so missing
         # revlogs are detected early
         copies = self.getcopies(ctx, parents, copyfiles)
         cleanp2 = set()
         if len(parents) == 2:
             cleanp2.update(self.repo.status(parents[1].node(), ctx.node(),
                                             clean=True).clean)
         changes = [(f, rev) for f in files if f not in self.ignored]
         changes.sort()
         return changes, copies, cleanp2
 
     def getcopies(self, ctx, parents, files):
         copies = {}
         for name in files:
             if name in self.ignored:
                 continue
             try:
                 copysource, _copynode = ctx.filectx(name).renamed()
                 if copysource in self.ignored:
                     continue
                 # Ignore copy sources not in parent revisions
                 found = False
                 for p in parents:
                     if copysource in p:
                         found = True
                         break
                 if not found:
                     continue
                 copies[name] = copysource
             except TypeError:
                 pass
             except error.LookupError as e:
                 if not self.ignoreerrors:
                     raise
                 self.ignored.add(name)
                 self.ui.warn(_('ignoring: %s\n') % e)
         return copies
 
     def getcommit(self, rev):
         ctx = self.changectx(rev)
         parents = [p.hex() for p in self.parents(ctx)]
         crev = rev
 
         return commit(author=ctx.user(),
                       date=util.datestr(ctx.date(), '%Y-%m-%d %H:%M:%S %1%2'),
                       desc=ctx.description(), rev=crev, parents=parents,
                       branch=ctx.branch(), extra=ctx.extra(),
                       sortkey=ctx.rev(), saverev=self.saverev,
                       phase=ctx.phase())
 
     def gettags(self):
         # This will get written to .hgtags, filter non global tags out.
         tags = [t for t in self.repo.tagslist()
                 if self.repo.tagtype(t[0]) == 'global']
         return dict([(name, hex(node)) for name, node in tags
                      if self.keep(node)])
 
     def getchangedfiles(self, rev, i):
         ctx = self.changectx(rev)
         parents = self.parents(ctx)
         if not parents and i is None:
             i = 0
             changes = [], ctx.manifest().keys(), []
         else:
             i = i or 0
             changes = self.repo.status(parents[i].node(), ctx.node())[:3]
         changes = [[f for f in l if f not in self.ignored] for l in changes]
 
         if i == 0:
             self._changescache = (rev, changes)
 
         return changes[0] + changes[1] + changes[2]
 
     def converted(self, rev, destrev):
         if self.convertfp is None:
             self.convertfp = open(self.repo.join('shamap'), 'a')
         self.convertfp.write('%s %s\n' % (destrev, rev))
         self.convertfp.flush()
 
     def before(self):
         self.ui.debug('run hg source pre-conversion action\n')
 
     def after(self):
         self.ui.debug('run hg source post-conversion action\n')
 
     def hasnativeorder(self):
         return True
 
     def hasnativeclose(self):
         return True
 
     def lookuprev(self, rev):
         try:
             return hex(self.repo.lookup(rev))
         except (error.RepoError, error.LookupError):
             return None
 
     def getbookmarks(self):
         return bookmarks.listbookmarks(self.repo)
 
     def checkrevformat(self, revstr, mapname='splicemap'):
         """ Mercurial, revision string is a 40 byte hex """
         self.checkhexformat(revstr, mapname)

tests/test-convert-filemap.t

@@ -1,673 +1,731 b''
 
   $ HGMERGE=true; export HGMERGE
   $ echo '[extensions]' >> $HGRCPATH
   $ echo 'convert =' >> $HGRCPATH
   $ glog()
   > {
   >     hg log -G --template '{rev} "{desc}" files: {files}\n' "$@"
   > }
   $ hg init source
   $ cd source
   $ echo foo > foo
   $ echo baz > baz
   $ mkdir -p dir/subdir
   $ echo dir/file >> dir/file
   $ echo dir/file2 >> dir/file2
   $ echo dir/file3 >> dir/file3 # to be corrupted in rev 0
   $ echo dir/subdir/file3 >> dir/subdir/file3
   $ echo dir/subdir/file4 >> dir/subdir/file4
   $ hg ci -d '0 0' -qAm '0: add foo baz dir/'
   $ echo bar > bar
   $ echo quux > quux
   $ echo dir/file4 >> dir/file4 # to be corrupted in rev 1
   $ hg copy foo copied
   $ hg ci -d '1 0' -qAm '1: add bar quux; copy foo to copied'
   $ echo >> foo
   $ hg ci -d '2 0' -m '2: change foo'
   $ hg up -qC 1
   $ echo >> bar
   $ echo >> quux
   $ hg ci -d '3 0' -m '3: change bar quux'
   created new head
   $ hg up -qC 2
   $ hg merge -qr 3
   $ echo >> bar
   $ echo >> baz
   $ hg ci -d '4 0' -m '4: first merge; change bar baz'
   $ echo >> bar
   $ echo 1 >> baz
   $ echo >> quux
   $ hg ci -d '5 0' -m '5: change bar baz quux'
   $ hg up -qC 4
   $ echo >> foo
   $ echo 2 >> baz
   $ hg ci -d '6 0' -m '6: change foo baz'
   created new head
   $ hg up -qC 5
   $ hg merge -qr 6
   $ echo >> bar
   $ hg ci -d '7 0' -m '7: second merge; change bar'
   $ echo >> foo
   $ hg ci -m '8: change foo'
   $ glog
   @  8 "8: change foo" files: foo
   |
   o    7 "7: second merge; change bar" files: bar baz
   |\
   | o  6 "6: change foo baz" files: baz foo
   | |
   o |  5 "5: change bar baz quux" files: bar baz quux
   |/
   o    4 "4: first merge; change bar baz" files: bar baz
   |\
   | o  3 "3: change bar quux" files: bar quux
   | |
   o |  2 "2: change foo" files: foo
   |/
   o  1 "1: add bar quux; copy foo to copied" files: bar copied dir/file4 quux
   |
   o  0 "0: add foo baz dir/" files: baz dir/file dir/file2 dir/file3 dir/subdir/file3 dir/subdir/file4 foo
   
 
 final file versions in this repo:
 
   $ hg manifest --debug
   9463f52fe115e377cf2878d4fc548117211063f2 644   bar
   94c1be4dfde2ee8d78db8bbfcf81210813307c3d 644   baz
   7711d36246cc83e61fb29cd6d4ef394c63f1ceaf 644   copied
   3e20847584beff41d7cd16136b7331ab3d754be0 644   dir/file
   75e6d3f8328f5f6ace6bf10b98df793416a09dca 644   dir/file2
   e96dce0bc6a217656a3a410e5e6bec2c4f42bf7c 644   dir/file3
   6edd55f559cdce67132b12ca09e09cee08b60442 644   dir/file4
   5fe139720576e18e34bcc9f79174db8897c8afe9 644   dir/subdir/file3
   57a1c1511590f3de52874adfa04effe8a77d64af 644   dir/subdir/file4
   9a7b52012991e4873687192c3e17e61ba3e837a3 644   foo
   bc3eca3f47023a3e70ca0d8cc95a22a6827db19d 644   quux
   $ hg debugrename copied
   copied renamed from foo:2ed2a3912a0b24502043eae84ee4b279c18b90dd
 
   $ cd ..
 
 
 Test interaction with startrev and verify that changing it is handled properly:
 
   $ > empty
   $ hg convert --filemap empty source movingstart --config convert.hg.startrev=3 -r4
   initializing destination movingstart repository
   scanning source...
   sorting...
   converting...
   1 3: change bar quux
   0 4: first merge; change bar baz
   $ hg convert --filemap empty source movingstart
   scanning source...
   sorting...
   converting...
   3 5: change bar baz quux
   2 6: change foo baz
   1 7: second merge; change bar
   warning: af455ce4166b3c9c88e6309c2b9332171dcea595 parent 61e22ca76c3b3e93df20338c4e02ce286898e825 is missing
   warning: cf908b3eeedc301c9272ebae931da966d5b326c7 parent 59e1ab45c888289513b7354484dac8a88217beab is missing
   0 8: change foo
 
 
 splitrepo tests
 
   $ splitrepo()
   > {
   >     msg="$1"
   >     files="$2"
   >     opts=$3
   >     echo "% $files: $msg"
   >     prefix=`echo "$files" | sed -e 's/ /-/g'`
   >     fmap="$prefix.fmap"
   >     repo="$prefix.repo"
   >     for i in $files; do
   >         echo "include $i" >> "$fmap"
   >     done
   >     hg -q convert $opts --filemap "$fmap" --datesort source "$repo"
   >     hg up -q -R "$repo"
   >     glog -R "$repo"
   >     hg -R "$repo" manifest --debug
   > }
   $ splitrepo 'skip unwanted merges; use 1st parent in 1st merge, 2nd in 2nd' foo
   % foo: skip unwanted merges; use 1st parent in 1st merge, 2nd in 2nd
   @  3 "8: change foo" files: foo
   |
   o  2 "6: change foo baz" files: foo
   |
   o  1 "2: change foo" files: foo
   |
   o  0 "0: add foo baz dir/" files: foo
   
   9a7b52012991e4873687192c3e17e61ba3e837a3 644   foo
   $ splitrepo 'merges are not merges anymore' bar
   % bar: merges are not merges anymore
   @  4 "7: second merge; change bar" files: bar
   |
   o  3 "5: change bar baz quux" files: bar
   |
   o  2 "4: first merge; change bar baz" files: bar
   |
   o  1 "3: change bar quux" files: bar
   |
   o  0 "1: add bar quux; copy foo to copied" files: bar
   
   9463f52fe115e377cf2878d4fc548117211063f2 644   bar
   $ splitrepo '1st merge is not a merge anymore; 2nd still is' baz
   % baz: 1st merge is not a merge anymore; 2nd still is
   @    4 "7: second merge; change bar" files: baz
   |\
   | o  3 "6: change foo baz" files: baz
   | |
   o |  2 "5: change bar baz quux" files: baz
   |/
   o  1 "4: first merge; change bar baz" files: baz
   |
   o  0 "0: add foo baz dir/" files: baz
   
   94c1be4dfde2ee8d78db8bbfcf81210813307c3d 644   baz
   $ splitrepo 'we add additional merges when they are interesting' 'foo quux'
   % foo quux: we add additional merges when they are interesting
   @  8 "8: change foo" files: foo
   |
   o    7 "7: second merge; change bar" files:
   |\
   | o  6 "6: change foo baz" files: foo
   | |
   o |  5 "5: change bar baz quux" files: quux
   |/
   o    4 "4: first merge; change bar baz" files:
   |\
   | o  3 "3: change bar quux" files: quux
   | |
   o |  2 "2: change foo" files: foo
   |/
   o  1 "1: add bar quux; copy foo to copied" files: quux
   |
   o  0 "0: add foo baz dir/" files: foo
   
   9a7b52012991e4873687192c3e17e61ba3e837a3 644   foo
   bc3eca3f47023a3e70ca0d8cc95a22a6827db19d 644   quux
   $ splitrepo 'partial conversion' 'bar quux' '-r 3'
   % bar quux: partial conversion
   @  1 "3: change bar quux" files: bar quux
   |
   o  0 "1: add bar quux; copy foo to copied" files: bar quux
   
   b79105bedc55102f394e90a789c9c380117c1b4a 644   bar
   db0421cc6b685a458c8d86c7d5c004f94429ea23 644   quux
   $ splitrepo 'complete the partial conversion' 'bar quux'
   % bar quux: complete the partial conversion
   @  4 "7: second merge; change bar" files: bar
   |
   o  3 "5: change bar baz quux" files: bar quux
   |
   o  2 "4: first merge; change bar baz" files: bar
   |
   o  1 "3: change bar quux" files: bar quux
   |
   o  0 "1: add bar quux; copy foo to copied" files: bar quux
   
   9463f52fe115e377cf2878d4fc548117211063f2 644   bar
   bc3eca3f47023a3e70ca0d8cc95a22a6827db19d 644   quux
   $ rm -r foo.repo
   $ splitrepo 'partial conversion' 'foo' '-r 3'
   % foo: partial conversion
   @  0 "0: add foo baz dir/" files: foo
   
   2ed2a3912a0b24502043eae84ee4b279c18b90dd 644   foo
   $ splitrepo 'complete the partial conversion' 'foo'
   % foo: complete the partial conversion
   @  3 "8: change foo" files: foo
   |
   o  2 "6: change foo baz" files: foo
   |
   o  1 "2: change foo" files: foo
   |
   o  0 "0: add foo baz dir/" files: foo
   
   9a7b52012991e4873687192c3e17e61ba3e837a3 644   foo
   $ splitrepo 'copied file; source not included in new repo' copied
   % copied: copied file; source not included in new repo
   @  0 "1: add bar quux; copy foo to copied" files: copied
   
   2ed2a3912a0b24502043eae84ee4b279c18b90dd 644   copied
   $ hg --cwd copied.repo debugrename copied
   copied not renamed
   $ splitrepo 'copied file; source included in new repo' 'foo copied'
   % foo copied: copied file; source included in new repo
   @  4 "8: change foo" files: foo
   |
   o  3 "6: change foo baz" files: foo
   |
   o  2 "2: change foo" files: foo
   |
   o  1 "1: add bar quux; copy foo to copied" files: copied
   |
   o  0 "0: add foo baz dir/" files: foo
   
   7711d36246cc83e61fb29cd6d4ef394c63f1ceaf 644   copied
   9a7b52012991e4873687192c3e17e61ba3e837a3 644   foo
   $ hg --cwd foo-copied.repo debugrename copied
   copied renamed from foo:2ed2a3912a0b24502043eae84ee4b279c18b90dd
 
 verify the top level 'include .' if there is no other includes:
 
   $ echo "exclude something" > default.fmap
   $ hg convert -q --filemap default.fmap -r1 source dummydest2
   $ hg -R dummydest2 log --template '{rev} {node|short} {desc|firstline}\n'
   1 61e22ca76c3b 1: add bar quux; copy foo to copied
   0 c085cf2ee7fe 0: add foo baz dir/
 
   $ echo "include somethingelse" >> default.fmap
   $ hg convert -q --filemap default.fmap -r1 source dummydest3
   $ hg -R dummydest3 log --template '{rev} {node|short} {desc|firstline}\n'
 
   $ echo "include ." >> default.fmap
   $ hg convert -q --filemap default.fmap -r1 source dummydest4
   $ hg -R dummydest4 log --template '{rev} {node|short} {desc|firstline}\n'
   1 61e22ca76c3b 1: add bar quux; copy foo to copied
   0 c085cf2ee7fe 0: add foo baz dir/
 
 ensure that the filemap contains duplicated slashes (issue3612)
 
   $ cat > renames.fmap <<EOF
   > include dir
   > exclude dir/file2
   > rename dir dir2//dir3
   > include foo
   > include copied
   > rename foo foo2/
   > rename copied ./copied2
   > exclude dir/subdir
   > include dir/subdir/file3
   > EOF
   $ rm source/.hg/store/data/dir/file3.i
   $ rm source/.hg/store/data/dir/file4.i
   $ hg -q convert --filemap renames.fmap --datesort source dummydest
   abort: data/dir/file3.i@e96dce0bc6a2: no match found!
   [255]
   $ hg -q convert --filemap renames.fmap --datesort --config convert.hg.ignoreerrors=1 source renames.repo
   ignoring: data/dir/file3.i@e96dce0bc6a2: no match found
   ignoring: data/dir/file4.i@6edd55f559cd: no match found
   $ hg up -q -R renames.repo
   $ glog -R renames.repo
   @  4 "8: change foo" files: foo2
   |
   o  3 "6: change foo baz" files: foo2
   |
   o  2 "2: change foo" files: foo2
   |
   o  1 "1: add bar quux; copy foo to copied" files: copied2
   |
   o  0 "0: add foo baz dir/" files: dir2/dir3/file dir2/dir3/subdir/file3 foo2
   
   $ hg -R renames.repo verify
   checking changesets
   checking manifests
   crosschecking files in changesets and manifests
   checking files
   4 files, 5 changesets, 7 total revisions
 
   $ hg -R renames.repo manifest --debug
   d43feacba7a4f1f2080dde4a4b985bd8a0236d46 644   copied2
   3e20847584beff41d7cd16136b7331ab3d754be0 644   dir2/dir3/file
   5fe139720576e18e34bcc9f79174db8897c8afe9 644   dir2/dir3/subdir/file3
   9a7b52012991e4873687192c3e17e61ba3e837a3 644   foo2
   $ hg --cwd renames.repo debugrename copied2
   copied2 renamed from foo2:2ed2a3912a0b24502043eae84ee4b279c18b90dd
 
 copied:
 
   $ hg --cwd source cat copied
   foo
 
 copied2:
 
   $ hg --cwd renames.repo cat copied2
   foo
 
 filemap errors
 
   $ cat > errors.fmap <<EOF
   > include dir/ # beware that comments changes error line numbers!
   > exclude /dir
   > rename dir//dir /dir//dir/ "out of sync"
   > include
   > EOF
   $ hg -q convert --filemap errors.fmap source errors.repo
   errors.fmap:3: superfluous / in include '/dir'
   errors.fmap:3: superfluous / in rename '/dir'
   errors.fmap:4: unknown directive 'out of sync'
   errors.fmap:5: path to exclude is missing
   abort: errors in filemap
   [255]
 
 test branch closing revision pruning if branch is pruned
 
   $ hg init branchpruning
   $ cd branchpruning
   $ hg branch foo
   marked working directory as branch foo
   (branches are permanent and global, did you want a bookmark?)
   $ echo a > a
   $ hg ci -Am adda
   adding a
   $ hg ci --close-branch -m closefoo
   $ hg up 0
   0 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ hg branch empty
   marked working directory as branch empty
   (branches are permanent and global, did you want a bookmark?)
   $ hg ci -m emptybranch
   $ hg ci --close-branch -m closeempty
   $ hg up 0
   0 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ hg branch default
   marked working directory as branch default
   (branches are permanent and global, did you want a bookmark?)
   $ echo b > b
   $ hg ci -Am addb
   adding b
   $ hg ci --close-branch -m closedefault
   $ cat > filemap <<EOF
   > include b
   > EOF
   $ cd ..
   $ hg convert branchpruning branchpruning-hg1
   initializing destination branchpruning-hg1 repository
   scanning source...
   sorting...
   converting...
   5 adda
   4 closefoo
   3 emptybranch
   2 closeempty
   1 addb
   0 closedefault
   $ glog -R branchpruning-hg1
   _  5 "closedefault" files:
   |
   o  4 "addb" files: b
   |
   | _  3 "closeempty" files:
   | |
   | o  2 "emptybranch" files:
   |/
   | _  1 "closefoo" files:
   |/
   o  0 "adda" files: a
   
 
 exercise incremental conversion at the same time
 
   $ hg convert -r0 --filemap branchpruning/filemap branchpruning branchpruning-hg2
   initializing destination branchpruning-hg2 repository
   scanning source...
   sorting...
   converting...
   0 adda
   $ hg convert -r4 --filemap branchpruning/filemap branchpruning branchpruning-hg2
   scanning source...
   sorting...
   converting...
   0 addb
   $ hg convert --filemap branchpruning/filemap branchpruning branchpruning-hg2
   scanning source...
   sorting...
   converting...
   3 closefoo
   2 emptybranch
   1 closeempty
   0 closedefault
   $ glog -R branchpruning-hg2
   _  1 "closedefault" files:
   |
   o  0 "addb" files: b
   
 
 Test rebuilding of map with unknown revisions in shamap - it used to crash
 
   $ cd branchpruning
   $ hg up -r 2
   0 files updated, 0 files merged, 1 files removed, 0 files unresolved
   $ hg merge 4
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   (branch merge, don't forget to commit)
   $ hg ci -m 'merging something'
   $ cd ..
   $ echo "53792d18237d2b64971fa571936869156655338d 6d955580116e82c4b029bd30f321323bae71a7f0" >> branchpruning-hg2/.hg/shamap
   $ hg convert --filemap branchpruning/filemap branchpruning branchpruning-hg2 --debug --config progress.debug=true
   run hg source pre-conversion action
   run hg sink pre-conversion action
   scanning source...
   scanning: 1 revisions
   sorting...
   converting...
   0 merging something
   source: 2503605b178fe50e8fbbb0e77b97939540aa8c87
   converting: 0/1 revisions (0.00%)
   unknown revmap source: 53792d18237d2b64971fa571936869156655338d
   run hg sink post-conversion action
   run hg source post-conversion action
 
 
 filemap rename undoing revision rename
 
   $ hg init renameundo
   $ cd renameundo
   $ echo 1 > a
   $ echo 1 > c
   $ hg ci -qAm add
   $ hg mv -q a b/a
   $ hg mv -q c b/c
   $ hg ci -qm rename
   $ echo 2 > b/a
   $ echo 2 > b/c
   $ hg ci -qm modify
   $ cd ..
 
   $ echo "rename b ." > renameundo.fmap
   $ hg convert --filemap renameundo.fmap renameundo renameundo2
   initializing destination renameundo2 repository
   scanning source...
   sorting...
   converting...
   2 add
   1 rename
   filtering out empty revision
   repository tip rolled back to revision 0 (undo convert)
   0 modify
   $ glog -R renameundo2
   o  1 "modify" files: a c
   |
   o  0 "add" files: a c
   
 
 
 test merge parents/empty merges pruning
 
   $ glog()
   > {
   >     hg log -G --template '{rev}:{node|short}@{branch} "{desc}" files: {files}\n' "$@"
   > }
 
 test anonymous branch pruning
 
   $ hg init anonymousbranch
   $ cd anonymousbranch
   $ echo a > a
   $ echo b > b
   $ hg ci -Am add
   adding a
   adding b
   $ echo a >> a
   $ hg ci -m changea
   $ hg up 0
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ echo b >> b
   $ hg ci -m changeb
   created new head
   $ hg up 1
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ hg merge
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   (branch merge, don't forget to commit)
   $ hg ci -m merge
   $ cd ..
 
   $ cat > filemap <<EOF
   > include a
   > EOF
   $ hg convert --filemap filemap anonymousbranch anonymousbranch-hg
   initializing destination anonymousbranch-hg repository
   scanning source...
   sorting...
   converting...
   3 add
   2 changea
   1 changeb
   0 merge
   $ glog -R anonymousbranch
   @    3:c71d5201a498@default "merge" files:
   |\
   | o  2:607eb44b17f9@default "changeb" files: b
   | |
   o |  1:1f60ea617824@default "changea" files: a
   |/
   o  0:0146e6129113@default "add" files: a b
   
   $ glog -R anonymousbranch-hg
   o  1:cda818e7219b@default "changea" files: a
   |
   o  0:c334dc3be0da@default "add" files: a
   
   $ cat anonymousbranch-hg/.hg/shamap
   0146e6129113dba9ac90207cfdf2d7ed35257ae5 c334dc3be0daa2a4e9ce4d2e2bdcba40c09d4916
   1f60ea61782421edf8d051ff4fcb61b330f26a4a cda818e7219b5f7f3fb9f49780054ed6a1905ec3
   607eb44b17f9348cd5cbd26e16af87ba77b0b037 c334dc3be0daa2a4e9ce4d2e2bdcba40c09d4916
   c71d5201a498b2658d105a6bf69d7a0df2649aea cda818e7219b5f7f3fb9f49780054ed6a1905ec3
 
   $ cat > filemap <<EOF
   > include b
   > EOF
   $ hg convert --filemap filemap anonymousbranch anonymousbranch-hg2
   initializing destination anonymousbranch-hg2 repository
   scanning source...
   sorting...
   converting...
   3 add
   2 changea
   1 changeb
   0 merge
   $ glog -R anonymousbranch
   @    3:c71d5201a498@default "merge" files:
   |\
   | o  2:607eb44b17f9@default "changeb" files: b
   | |
   o |  1:1f60ea617824@default "changea" files: a
   |/
   o  0:0146e6129113@default "add" files: a b
   
   $ glog -R anonymousbranch-hg2
   o  1:62dd350b0df6@default "changeb" files: b
   |
   o  0:4b9ced861657@default "add" files: b
   
   $ cat anonymousbranch-hg2/.hg/shamap
   0146e6129113dba9ac90207cfdf2d7ed35257ae5 4b9ced86165703791653059a1db6ed864630a523
   1f60ea61782421edf8d051ff4fcb61b330f26a4a 4b9ced86165703791653059a1db6ed864630a523
   607eb44b17f9348cd5cbd26e16af87ba77b0b037 62dd350b0df695f7d2c82a02e0499b16fd790f22
   c71d5201a498b2658d105a6bf69d7a0df2649aea 62dd350b0df695f7d2c82a02e0499b16fd790f22
 
 test named branch pruning
 
   $ hg init namedbranch
   $ cd namedbranch
   $ echo a > a
   $ echo b > b
   $ hg ci -Am add
   adding a
   adding b
   $ echo a >> a
   $ hg ci -m changea
   $ hg up 0
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ hg branch foo
   marked working directory as branch foo
   (branches are permanent and global, did you want a bookmark?)
   $ echo b >> b
   $ hg ci -m changeb
   $ hg up default
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ hg merge foo
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   (branch merge, don't forget to commit)
   $ hg ci -m merge
   $ cd ..
 
   $ cat > filemap <<EOF
   > include a
   > EOF
   $ hg convert --filemap filemap namedbranch namedbranch-hg
   initializing destination namedbranch-hg repository
   scanning source...
   sorting...
   converting...
   3 add
   2 changea
   1 changeb
   0 merge
   $ glog -R namedbranch
   @    3:73899bcbe45c@default "merge" files:
   |\
   | o  2:8097982d19fc@foo "changeb" files: b
   | |
   o |  1:1f60ea617824@default "changea" files: a
   |/
   o  0:0146e6129113@default "add" files: a b
   
   $ glog -R namedbranch-hg
   o  1:cda818e7219b@default "changea" files: a
   |
   o  0:c334dc3be0da@default "add" files: a
   
 
   $ cd namedbranch
   $ hg --config extensions.mq= strip tip
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   saved backup bundle to $TESTTMP/namedbranch/.hg/strip-backup/73899bcbe45c-92adf160-backup.hg (glob)
   $ hg up foo
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ hg merge default
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   (branch merge, don't forget to commit)
   $ hg ci -m merge
   $ cd ..
 
   $ hg convert --filemap filemap namedbranch namedbranch-hg2
   initializing destination namedbranch-hg2 repository
   scanning source...
   sorting...
   converting...
   3 add
   2 changea
   1 changeb
   0 merge
   $ glog -R namedbranch
   @    3:e1959de76e1b@foo "merge" files:
   |\
   | o  2:8097982d19fc@foo "changeb" files: b
   | |
   o |  1:1f60ea617824@default "changea" files: a
   |/
   o  0:0146e6129113@default "add" files: a b
   
   $ glog -R namedbranch-hg2
   o    2:dcf314454667@foo "merge" files:
   |\
   | o  1:cda818e7219b@default "changea" files: a
   |/
   o  0:c334dc3be0da@default "add" files: a
   
+  $ cd ..
+
+test converting merges into a repo that contains other files
+
+  $ hg init merge-test1
+  $ cd merge-test1
+  $ touch a && hg commit -Aqm a
+  $ hg up -q null
+  $ touch b && hg commit -Aqm b
+  $ hg merge -q 0 && hg commit -qm merge
+  $ cd ..
+  $ hg init merge-test2
+  $ cd merge-test2
+  $ mkdir converted
+  $ touch converted/a && hg commit -Aqm 'a'
+  $ touch x && hg commit -Aqm 'x'
+  $ cd ..
+  $ hg log -G -T '{node}' -R merge-test1
+  @    ea7c1a7ae9588677a715ce4f204cd89c28d5471f
+  |\
+  | o  d7486e00c6f1b633dcadc0582f78006d805c7a0f
+  |
+  o  3903775176ed42b1458a6281db4a0ccf4d9f287a
+  
+  $ hg log -G -T '{node}' -R merge-test2
+  @  34f1aa7da42559bae87920880b522d47b3ddbc0d
+  |
+  o  e01a12b07b4fdfd61ff90a2a1b4560a7a776f323
+  
+- Build a shamap where the target converted/a is in on top of an unrelated
+- change to 'x'. This simulates using convert to merge several repositories
+- together.
+  $ cat >> merge-test2/.hg/shamap <<EOF
+  > 3903775176ed42b1458a6281db4a0ccf4d9f287a 34f1aa7da42559bae87920880b522d47b3ddbc0d
+  > EOF
+  $ cat >> merge-test-filemap <<EOF
+  > rename . converted/
+  > EOF
+  $ hg convert --filemap merge-test-filemap merge-test1 merge-test2 --traceback
+  scanning source...
+  sorting...
+  converting...
+  1 b
+  0 merge
+  $ hg -R merge-test2 manifest -r tip
+  converted/a
+  converted/b
+  x
+  $ hg -R merge-test2 log -G -T '{node}\n{files % "{file}\n"}'
+  o    4b5e2f0218d3442a0c14892b18685bf9c8059c4a
+  |\
+  | o  214325dd2e4cff981dcf00cb120cd39e1ea36dcc
+  |    converted/b
+  @  34f1aa7da42559bae87920880b522d47b3ddbc0d
+  |  x
+  o  e01a12b07b4fdfd61ff90a2a1b4560a7a776f323
+     converted/a
+