convert: warn on superfluous / in paths

shlex is really a bad parser for this line-based format ...

hgext/convert/filemap.py

@@ -1,355 +1,365 b''
 # Copyright 2007 Bryan O'Sullivan <[e-mail]>
 # Copyright 2007 Alexis S. L. Carvalho <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 import shlex
 from mercurial.i18n import _
 from mercurial import util
 from common import SKIPREV, converter_source
 
 def rpairs(name):
     e = len(name)
     while e != -1:
         yield name[:e], name[e + 1:]
         e = name.rfind('/', 0, e)
     yield '.', name
 
 class filemapper(object):
     '''Map and filter filenames when importing.
     A name can be mapped to itself, a new name, or None (omit from new
     repository).'''
 
     def __init__(self, ui, path=None):
         self.ui = ui
         self.include = {}
         self.exclude = {}
         self.rename = {}
         if path:
             if self.parse(path):
                 raise util.Abort(_('errors in filemap'))
 
     def parse(self, path):
         errs = 0
         def check(name, mapping, listname):
+            if not name:
+                self.ui.warn(_('%s:%d: path to %s is missing\n') %
+                             (lex.infile, lex.lineno, listname))
+                return 1
             if name in mapping:
                 self.ui.warn(_('%s:%d: %r already in %s list\n') %
                              (lex.infile, lex.lineno, name, listname))
                 return 1
+            if (name.startswith('/') or
+                name.endswith('/') or
+                '//' in name):
+                self.ui.warn(_('%s:%d: superfluous / in %s %r\n') %
+                             (lex.infile, lex.lineno, listname, name))
+                return 1
             return 0
         lex = shlex.shlex(open(path), path, True)
         lex.wordchars += '!@#$%^&*()-=+[]{}|;:,./<>?'
         cmd = lex.get_token()
         while cmd:
             if cmd == 'include':
                 name = lex.get_token()
                 errs += check(name, self.exclude, 'exclude')
                 self.include[name] = name
             elif cmd == 'exclude':
                 name = lex.get_token()
                 errs += check(name, self.include, 'include')
                 errs += check(name, self.rename, 'rename')
                 self.exclude[name] = name
             elif cmd == 'rename':
                 src = lex.get_token()
                 dest = lex.get_token()
                 errs += check(src, self.exclude, 'exclude')
                 self.rename[src] = dest
             elif cmd == 'source':
                 errs += self.parse(lex.get_token())
             else:
                 self.ui.warn(_('%s:%d: unknown directive %r\n') %
                              (lex.infile, lex.lineno, cmd))
                 errs += 1
             cmd = lex.get_token()
         return errs
 
     def lookup(self, name, mapping):
         for pre, suf in rpairs(name):
             try:
                 return mapping[pre], pre, suf
             except KeyError:
                 pass
         return '', name, ''
 
     def __call__(self, name):
         if self.include:
             inc = self.lookup(name, self.include)[0]
         else:
             inc = name
         if self.exclude:
             exc = self.lookup(name, self.exclude)[0]
         else:
             exc = ''
         if (not self.include and exc) or (len(inc) <= len(exc)):
             return None
         newpre, pre, suf = self.lookup(name, self.rename)
         if newpre:
             if newpre == '.':
                 return suf
             if suf:
                 return newpre + '/' + suf
             return newpre
         return name
 
     def active(self):
         return bool(self.include or self.exclude or self.rename)
 
 # This class does two additional things compared to a regular source:
 #
 # - Filter and rename files.  This is mostly wrapped by the filemapper
 #   class above. We hide the original filename in the revision that is
 #   returned by getchanges to be able to find things later in getfile.
 #
 # - Return only revisions that matter for the files we're interested in.
 #   This involves rewriting the parents of the original revision to
 #   create a graph that is restricted to those revisions.
 #
 #   This set of revisions includes not only revisions that directly
 #   touch files we're interested in, but also merges that merge two
 #   or more interesting revisions.
 
 class filemap_source(converter_source):
     def __init__(self, ui, baseconverter, filemap):
         super(filemap_source, self).__init__(ui)
         self.base = baseconverter
         self.filemapper = filemapper(ui, filemap)
         self.commits = {}
         # if a revision rev has parent p in the original revision graph, then
         # rev will have parent self.parentmap[p] in the restricted graph.
         self.parentmap = {}
         # self.wantedancestors[rev] is the set of all ancestors of rev that
         # are in the restricted graph.
         self.wantedancestors = {}
         self.convertedorder = None
         self._rebuilt = False
         self.origparents = {}
         self.children = {}
         self.seenchildren = {}
 
     def before(self):
         self.base.before()
 
     def after(self):
         self.base.after()
 
     def setrevmap(self, revmap):
         # rebuild our state to make things restartable
         #
         # To avoid calling getcommit for every revision that has already
         # been converted, we rebuild only the parentmap, delaying the
         # rebuild of wantedancestors until we need it (i.e. until a
         # merge).
         #
         # We assume the order argument lists the revisions in
         # topological order, so that we can infer which revisions were
         # wanted by previous runs.
         self._rebuilt = not revmap
         seen = {SKIPREV: SKIPREV}
         dummyset = set()
         converted = []
         for rev in revmap.order:
             mapped = revmap[rev]
             wanted = mapped not in seen
             if wanted:
                 seen[mapped] = rev
                 self.parentmap[rev] = rev
             else:
                 self.parentmap[rev] = seen[mapped]
             self.wantedancestors[rev] = dummyset
             arg = seen[mapped]
             if arg == SKIPREV:
                 arg = None
             converted.append((rev, wanted, arg))
         self.convertedorder = converted
         return self.base.setrevmap(revmap)
 
     def rebuild(self):
         if self._rebuilt:
             return True
         self._rebuilt = True
         self.parentmap.clear()
         self.wantedancestors.clear()
         self.seenchildren.clear()
         for rev, wanted, arg in self.convertedorder:
             if rev not in self.origparents:
                 self.origparents[rev] = self.getcommit(rev).parents
             if arg is not None:
                 self.children[arg] = self.children.get(arg, 0) + 1
 
         for rev, wanted, arg in self.convertedorder:
             parents = self.origparents[rev]
             if wanted:
                 self.mark_wanted(rev, parents)
             else:
                 self.mark_not_wanted(rev, arg)
             self._discard(arg, *parents)
 
         return True
 
     def getheads(self):
         return self.base.getheads()
 
     def getcommit(self, rev):
         # We want to save a reference to the commit objects to be able
         # to rewrite their parents later on.
         c = self.commits[rev] = self.base.getcommit(rev)
         for p in c.parents:
             self.children[p] = self.children.get(p, 0) + 1
         return c
 
     def _discard(self, *revs):
         for r in revs:
             if r is None:
                 continue
             self.seenchildren[r] = self.seenchildren.get(r, 0) + 1
             if self.seenchildren[r] == self.children[r]:
                 del self.wantedancestors[r]
                 del self.parentmap[r]
                 del self.seenchildren[r]
                 if self._rebuilt:
                     del self.children[r]
 
     def wanted(self, rev, i):
         # Return True if we're directly interested in rev.
         #
         # i is an index selecting one of the parents of rev (if rev
         # has no parents, i is None).  getchangedfiles will give us
         # the list of files that are different in rev and in the parent
         # indicated by i.  If we're interested in any of these files,
         # we're interested in rev.
         try:
             files = self.base.getchangedfiles(rev, i)
         except NotImplementedError:
             raise util.Abort(_("source repository doesn't support --filemap"))
         for f in files:
             if self.filemapper(f):
                 return True
         return False
 
     def mark_not_wanted(self, rev, p):
         # Mark rev as not interesting and update data structures.
 
         if p is None:
             # A root revision. Use SKIPREV to indicate that it doesn't
             # map to any revision in the restricted graph.  Put SKIPREV
             # in the set of wanted ancestors to simplify code elsewhere
             self.parentmap[rev] = SKIPREV
             self.wantedancestors[rev] = set((SKIPREV,))
             return
 
         # Reuse the data from our parent.
         self.parentmap[rev] = self.parentmap[p]
         self.wantedancestors[rev] = self.wantedancestors[p]
 
     def mark_wanted(self, rev, parents):
         # Mark rev ss wanted and update data structures.
 
         # rev will be in the restricted graph, so children of rev in
         # the original graph should still have rev as a parent in the
         # restricted graph.
         self.parentmap[rev] = rev
 
         # The set of wanted ancestors of rev is the union of the sets
         # of wanted ancestors of its parents. Plus rev itself.
         wrev = set()
         for p in parents:
             wrev.update(self.wantedancestors[p])
         wrev.add(rev)
         self.wantedancestors[rev] = wrev
 
     def getchanges(self, rev):
         parents = self.commits[rev].parents
         if len(parents) > 1:
             self.rebuild()
 
         # To decide whether we're interested in rev we:
         #
         # - calculate what parents rev will have if it turns out we're
         #   interested in it.  If it's going to have more than 1 parent,
         #   we're interested in it.
         #
         # - otherwise, we'll compare it with the single parent we found.
         #   If any of the files we're interested in is different in the
         #   the two revisions, we're interested in rev.
 
         # A parent p is interesting if its mapped version (self.parentmap[p]):
         # - is not SKIPREV
         # - is still not in the list of parents (we don't want duplicates)
         # - is not an ancestor of the mapped versions of the other parents
         mparents = []
         wp = None
         for i, p1 in enumerate(parents):
             mp1 = self.parentmap[p1]
             if mp1 == SKIPREV or mp1 in mparents:
                 continue
             for p2 in parents:
                 if p1 == p2 or mp1 == self.parentmap[p2]:
                     continue
                 if mp1 in self.wantedancestors[p2]:
                     break
             else:
                 mparents.append(mp1)
                 wp = i
 
         if wp is None and parents:
             wp = 0
 
         self.origparents[rev] = parents
 
         closed = 'close' in self.commits[rev].extra
 
         if len(mparents) < 2 and not closed and not self.wanted(rev, wp):
             # We don't want this revision.
             # Update our state and tell the convert process to map this
             # revision to the same revision its parent as mapped to.
             p = None
             if parents:
                 p = parents[wp]
             self.mark_not_wanted(rev, p)
             self.convertedorder.append((rev, False, p))
             self._discard(*parents)
             return self.parentmap[rev]
 
         # We want this revision.
         # Rewrite the parents of the commit object
         self.commits[rev].parents = mparents
         self.mark_wanted(rev, parents)
         self.convertedorder.append((rev, True, None))
         self._discard(*parents)
 
         # Get the real changes and do the filtering/mapping. To be
         # able to get the files later on in getfile, we hide the
         # original filename in the rev part of the return value.
         changes, copies = self.base.getchanges(rev)
         newnames = {}
         files = []
         for f, r in changes:
             newf = self.filemapper(f)
             if newf:
                 files.append((newf, (f, r)))
                 newnames[f] = newf
 
         ncopies = {}
         for c in copies:
             newc = self.filemapper(c)
             if newc:
                 newsource = self.filemapper(copies[c])
                 if newsource:
                     ncopies[newc] = newsource
 
         return files, ncopies
 
     def getfile(self, name, rev):
         realname, realrev = rev
         return self.base.getfile(realname, realrev)
 
     def gettags(self):
         return self.base.gettags()
 
     def hasnativeorder(self):
         return self.base.hasnativeorder()
 
     def lookuprev(self, rev):
         return self.base.lookuprev(rev)

tests/test-convert-filemap

@@ -1,130 +1,141 b''
 #!/bin/sh
 
 HGMERGE=true; export HGMERGE
 
 echo '[extensions]' >> $HGRCPATH
 echo 'graphlog =' >> $HGRCPATH
 echo 'convert =' >> $HGRCPATH
 
 glog()
 {
     hg glog --template '{rev} "{desc}" files: {files}\n' "$@"
 }
 
 hg init source
 cd source
 
 echo foo > foo
 echo baz > baz
 mkdir -p dir/subdir
 echo dir/file >> dir/file
 echo dir/file2 >> dir/file2
 echo dir/subdir/file3 >> dir/subdir/file3
 echo dir/subdir/file4 >> dir/subdir/file4
 hg ci -d '0 0' -qAm '0: add foo baz dir/'
 
 echo bar > bar
 echo quux > quux
 hg copy foo copied
 hg ci -d '1 0' -qAm '1: add bar quux; copy foo to copied'
 
 echo >> foo
 hg ci -d '2 0' -m '2: change foo'
 
 hg up -qC 1
 echo >> bar
 echo >> quux
 hg ci -d '3 0' -m '3: change bar quux'
 
 hg up -qC 2
 hg merge -qr 3
 echo >> bar
 echo >> baz
 hg ci -d '4 0' -m '4: first merge; change bar baz'
 
 echo >> bar
 echo 1 >> baz
 echo >> quux
 hg ci -d '5 0' -m '5: change bar baz quux'
 
 hg up -qC 4
 echo >> foo
 echo 2 >> baz
 hg ci -d '6 0' -m '6: change foo baz'
 
 hg up -qC 5
 hg merge -qr 6
 echo >> bar
 hg ci -d '7 0' -m '7: second merge; change bar'
 
 echo >> foo
 hg ci -m '8: change foo'
 
 glog
 
 echo '% final file versions in this repo:'
 hg manifest --debug
 hg debugrename copied
 echo
 
 cd ..
 
 splitrepo()
 {
     msg="$1"
     files="$2"
     opts=$3
     echo "% $files: $msg"
     prefix=`echo "$files" | sed -e 's/ /-/g'`
     fmap="$prefix.fmap"
     repo="$prefix.repo"
     for i in $files; do
 	echo "include $i" >> "$fmap"
     done
     hg -q convert $opts --filemap "$fmap" --datesort source "$repo"
     hg up -q -R "$repo"
     glog -R "$repo"
     hg -R "$repo" manifest --debug
 }
 
 splitrepo 'skip unwanted merges; use 1st parent in 1st merge, 2nd in 2nd' foo
 
 splitrepo 'merges are not merges anymore' bar
 
 splitrepo '1st merge is not a merge anymore; 2nd still is' baz
 
 splitrepo 'we add additional merges when they are interesting' 'foo quux'
 
 splitrepo 'partial conversion' 'bar quux' '-r 3'
 splitrepo 'complete the partial conversion' 'bar quux'
 
 rm -r foo.repo
 splitrepo 'partial conversion' 'foo' '-r 3'
 splitrepo 'complete the partial conversion' 'foo'
 
 splitrepo 'copied file; source not included in new repo' copied
 hg --cwd copied.repo debugrename copied
 
 splitrepo 'copied file; source included in new repo' 'foo copied'
 hg --cwd foo-copied.repo debugrename copied
 
 cat > renames.fmap <<EOF
 include dir
 exclude dir/file2
 rename dir dir2
 include foo
 include copied
 rename foo foo2
 rename copied copied2
 exclude dir/subdir
 include dir/subdir/file3
 EOF
 hg -q convert --filemap renames.fmap --datesort source renames.repo
 hg up -q -R renames.repo
 glog -R renames.repo
 hg -R renames.repo manifest --debug
 hg --cwd renames.repo debugrename copied2
 echo 'copied:'
 hg --cwd source cat copied
 echo 'copied2:'
 hg --cwd renames.repo cat copied2
+
+echo % filemap errors
+cat > errors.fmap <<EOF
+include dir/ # beware that comments changes error line numbers!
+exclude /dir
+rename dir//dir /dir//dir/ "out of sync"
+include
+EOF
+hg -q convert --filemap errors.fmap source errors.repo
+
+true # happy ending

tests/test-convert-filemap.out

@@ -1,159 +1,167 b''
 created new head
 created new head
 @  8 "8: change foo" files: foo
 |
 o    7 "7: second merge; change bar" files: bar baz
 |\
 | o  6 "6: change foo baz" files: baz foo
 | |
 o |  5 "5: change bar baz quux" files: bar baz quux
 |/
 o    4 "4: first merge; change bar baz" files: bar baz
 |\
 | o  3 "3: change bar quux" files: bar quux
 | |
 o |  2 "2: change foo" files: foo
 |/
 o  1 "1: add bar quux; copy foo to copied" files: bar copied quux
 |
 o  0 "0: add foo baz dir/" files: baz dir/file dir/file2 dir/subdir/file3 dir/subdir/file4 foo
 
 % final file versions in this repo:
 9463f52fe115e377cf2878d4fc548117211063f2 644   bar
 94c1be4dfde2ee8d78db8bbfcf81210813307c3d 644   baz
 7711d36246cc83e61fb29cd6d4ef394c63f1ceaf 644   copied
 3e20847584beff41d7cd16136b7331ab3d754be0 644   dir/file
 75e6d3f8328f5f6ace6bf10b98df793416a09dca 644   dir/file2
 5fe139720576e18e34bcc9f79174db8897c8afe9 644   dir/subdir/file3
 57a1c1511590f3de52874adfa04effe8a77d64af 644   dir/subdir/file4
 9a7b52012991e4873687192c3e17e61ba3e837a3 644   foo
 bc3eca3f47023a3e70ca0d8cc95a22a6827db19d 644   quux
 copied renamed from foo:2ed2a3912a0b24502043eae84ee4b279c18b90dd
 
 % foo: skip unwanted merges; use 1st parent in 1st merge, 2nd in 2nd
 @  3 "8: change foo" files: foo
 |
 o  2 "6: change foo baz" files: foo
 |
 o  1 "2: change foo" files: foo
 |
 o  0 "0: add foo baz dir/" files: foo
 
 9a7b52012991e4873687192c3e17e61ba3e837a3 644   foo
 % bar: merges are not merges anymore
 @  4 "7: second merge; change bar" files: bar
 |
 o  3 "5: change bar baz quux" files: bar
 |
 o  2 "4: first merge; change bar baz" files: bar
 |
 o  1 "3: change bar quux" files: bar
 |
 o  0 "1: add bar quux; copy foo to copied" files: bar
 
 9463f52fe115e377cf2878d4fc548117211063f2 644   bar
 % baz: 1st merge is not a merge anymore; 2nd still is
 @    4 "7: second merge; change bar" files: baz
 |\
 | o  3 "6: change foo baz" files: baz
 | |
 o |  2 "5: change bar baz quux" files: baz
 |/
 o  1 "4: first merge; change bar baz" files: baz
 |
 o  0 "0: add foo baz dir/" files: baz
 
 94c1be4dfde2ee8d78db8bbfcf81210813307c3d 644   baz
 % foo quux: we add additional merges when they are interesting
 @  8 "8: change foo" files: foo
 |
 o    7 "7: second merge; change bar" files:
 |\
 | o  6 "6: change foo baz" files: foo
 | |
 o |  5 "5: change bar baz quux" files: quux
 |/
 o    4 "4: first merge; change bar baz" files:
 |\
 | o  3 "3: change bar quux" files: quux
 | |
 o |  2 "2: change foo" files: foo
 |/
 o  1 "1: add bar quux; copy foo to copied" files: quux
 |
 o  0 "0: add foo baz dir/" files: foo
 
 9a7b52012991e4873687192c3e17e61ba3e837a3 644   foo
 bc3eca3f47023a3e70ca0d8cc95a22a6827db19d 644   quux
 % bar quux: partial conversion
 @  1 "3: change bar quux" files: bar quux
 |
 o  0 "1: add bar quux; copy foo to copied" files: bar quux
 
 b79105bedc55102f394e90a789c9c380117c1b4a 644   bar
 db0421cc6b685a458c8d86c7d5c004f94429ea23 644   quux
 % bar quux: complete the partial conversion
 @  4 "7: second merge; change bar" files: bar
 |
 o  3 "5: change bar baz quux" files: bar quux
 |
 o  2 "4: first merge; change bar baz" files: bar
 |
 o  1 "3: change bar quux" files: bar quux
 |
 o  0 "1: add bar quux; copy foo to copied" files: bar quux
 
 9463f52fe115e377cf2878d4fc548117211063f2 644   bar
 bc3eca3f47023a3e70ca0d8cc95a22a6827db19d 644   quux
 % foo: partial conversion
 @  0 "0: add foo baz dir/" files: foo
 
 2ed2a3912a0b24502043eae84ee4b279c18b90dd 644   foo
 % foo: complete the partial conversion
 @  3 "8: change foo" files: foo
 |
 o  2 "6: change foo baz" files: foo
 |
 o  1 "2: change foo" files: foo
 |
 o  0 "0: add foo baz dir/" files: foo
 
 9a7b52012991e4873687192c3e17e61ba3e837a3 644   foo
 % copied: copied file; source not included in new repo
 @  0 "1: add bar quux; copy foo to copied" files: copied
 
 2ed2a3912a0b24502043eae84ee4b279c18b90dd 644   copied
 copied not renamed
 % foo copied: copied file; source included in new repo
 @  4 "8: change foo" files: foo
 |
 o  3 "6: change foo baz" files: foo
 |
 o  2 "2: change foo" files: foo
 |
 o  1 "1: add bar quux; copy foo to copied" files: copied
 |
 o  0 "0: add foo baz dir/" files: foo
 
 7711d36246cc83e61fb29cd6d4ef394c63f1ceaf 644   copied
 9a7b52012991e4873687192c3e17e61ba3e837a3 644   foo
 copied renamed from foo:2ed2a3912a0b24502043eae84ee4b279c18b90dd
 @  4 "8: change foo" files: foo2
 |
 o  3 "6: change foo baz" files: foo2
 |
 o  2 "2: change foo" files: foo2
 |
 o  1 "1: add bar quux; copy foo to copied" files: copied2
 |
 o  0 "0: add foo baz dir/" files: dir2/file dir2/subdir/file3 foo2
 
 d43feacba7a4f1f2080dde4a4b985bd8a0236d46 644   copied2
 3e20847584beff41d7cd16136b7331ab3d754be0 644   dir2/file
 5fe139720576e18e34bcc9f79174db8897c8afe9 644   dir2/subdir/file3
 9a7b52012991e4873687192c3e17e61ba3e837a3 644   foo2
 copied2 renamed from foo2:2ed2a3912a0b24502043eae84ee4b279c18b90dd
 copied:
 foo
 copied2:
 foo
+% filemap errors
+errors.fmap:1: superfluous / in exclude 'dir/'
+errors.fmap:3: superfluous / in include '/dir'
+errors.fmap:3: superfluous / in rename '/dir'
+errors.fmap:3: superfluous / in exclude 'dir//dir'
+errors.fmap:4: unknown directive 'out of sync'
+errors.fmap:5: path to exclude is missing
+abort: errors in filemap