convert: add bookmarks reading support to git backend

hgext/convert/git.py

@@ -1,170 +1,205 b''
 # git.py - git support for the convert extension
 #
 #  Copyright 2005-2009 Matt Mackall <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 import os
 from mercurial import util
 from mercurial.node import hex, nullid
 from mercurial.i18n import _
 
 from common import NoRepo, commit, converter_source, checktool
 
 class convert_git(converter_source):
     # Windows does not support GIT_DIR= construct while other systems
     # cannot remove environment variable. Just assume none have
     # both issues.
     if hasattr(os, 'unsetenv'):
-        def gitopen(self, s):
+        def gitopen(self, s, noerr=False):
             prevgitdir = os.environ.get('GIT_DIR')
             os.environ['GIT_DIR'] = self.path
             try:
-                return util.popen(s, 'rb')
+                if noerr:
+                    (stdin, stdout, stderr) = util.popen3(s)
+                    return stdout
+                else:
+                    return util.popen(s, 'rb')
             finally:
                 if prevgitdir is None:
                     del os.environ['GIT_DIR']
                 else:
                     os.environ['GIT_DIR'] = prevgitdir
     else:
-        def gitopen(self, s):
-            return util.popen('GIT_DIR=%s %s' % (self.path, s), 'rb')
+        def gitopen(self, s, noerr=False):
+            if noerr:
+                (sin, so, se) = util.popen3('GIT_DIR=%s %s' % (self.path, s))
+                return stdout
+            else:
+                util.popen('GIT_DIR=%s %s' % (self.path, s), 'rb')
 
     def gitread(self, s):
         fh = self.gitopen(s)
         data = fh.read()
         return data, fh.close()
 
     def __init__(self, ui, path, rev=None):
         super(convert_git, self).__init__(ui, path, rev=rev)
 
         if os.path.isdir(path + "/.git"):
             path += "/.git"
         if not os.path.exists(path + "/objects"):
             raise NoRepo(_("%s does not look like a Git repository") % path)
 
         checktool('git', 'git')
 
         self.path = path
 
     def getheads(self):
         if not self.rev:
             heads, ret = self.gitread('git rev-parse --branches --remotes')
             heads = heads.splitlines()
         else:
             heads, ret = self.gitread("git rev-parse --verify %s" % self.rev)
             heads = [heads[:-1]]
         if ret:
             raise util.Abort(_('cannot retrieve git heads'))
         return heads
 
     def catfile(self, rev, type):
         if rev == hex(nullid):
             raise IOError()
         data, ret = self.gitread("git cat-file %s %s" % (type, rev))
         if ret:
             raise util.Abort(_('cannot read %r object at %s') % (type, rev))
         return data
 
     def getfile(self, name, rev):
         data = self.catfile(rev, "blob")
         mode = self.modecache[(name, rev)]
         return data, mode
 
     def getchanges(self, version):
         self.modecache = {}
         fh = self.gitopen("git diff-tree -z --root -m -r %s" % version)
         changes = []
         seen = set()
         entry = None
         for l in fh.read().split('\x00'):
             if not entry:
                 if not l.startswith(':'):
                     continue
                 entry = l
                 continue
             f = l
             if f not in seen:
                 seen.add(f)
                 entry = entry.split()
                 h = entry[3]
                 p = (entry[1] == "100755")
                 s = (entry[1] == "120000")
                 self.modecache[(f, h)] = (p and "x") or (s and "l") or ""
                 changes.append((f, h))
             entry = None
         if fh.close():
             raise util.Abort(_('cannot read changes in %s') % version)
         return (changes, {})
 
     def getcommit(self, version):
         c = self.catfile(version, "commit") # read the commit hash
         end = c.find("\n\n")
         message = c[end + 2:]
         message = self.recode(message)
         l = c[:end].splitlines()
         parents = []
         author = committer = None
         for e in l[1:]:
             n, v = e.split(" ", 1)
             if n == "author":
                 p = v.split()
                 tm, tz = p[-2:]
                 author = " ".join(p[:-2])
                 if author[0] == "<": author = author[1:-1]
                 author = self.recode(author)
             if n == "committer":
                 p = v.split()
                 tm, tz = p[-2:]
                 committer = " ".join(p[:-2])
                 if committer[0] == "<": committer = committer[1:-1]
                 committer = self.recode(committer)
             if n == "parent":
                 parents.append(v)
 
         if committer and committer != author:
             message += "\ncommitter: %s\n" % committer
         tzs, tzh, tzm = tz[-5:-4] + "1", tz[-4:-2], tz[-2:]
         tz = -int(tzs) * (int(tzh) * 3600 + int(tzm))
         date = tm + " " + str(tz)
 
         c = commit(parents=parents, date=date, author=author, desc=message,
                    rev=version)
         return c
 
     def gettags(self):
         tags = {}
         fh = self.gitopen('git ls-remote --tags "%s"' % self.path)
         prefix = 'refs/tags/'
         for line in fh:
             line = line.strip()
             if not line.endswith("^{}"):
                 continue
             node, tag = line.split(None, 1)
             if not tag.startswith(prefix):
                 continue
             tag = tag[len(prefix):-3]
             tags[tag] = node
         if fh.close():
             raise util.Abort(_('cannot read tags from %s') % self.path)
 
         return tags
 
     def getchangedfiles(self, version, i):
         changes = []
         if i is None:
             fh = self.gitopen("git diff-tree --root -m -r %s" % version)
             for l in fh:
                 if "\t" not in l:
                     continue
                 m, f = l[:-1].split("\t")
                 changes.append(f)
         else:
             fh = self.gitopen('git diff-tree --name-only --root -r %s "%s^%s" --'
                              % (version, version, i + 1))
             changes = [f.rstrip('\n') for f in fh]
         if fh.close():
             raise util.Abort(_('cannot read changes in %s') % version)
 
         return changes
+
+    def getbookmarks(self):
+        bookmarks = {}
+
+        # Interesting references in git are prefixed
+        prefix = 'refs/heads/'
+        prefixlen = len(prefix)
+
+        # factor two commands
+        gitcmd = { 'remote/': 'git ls-remote --heads origin',
+                          '': 'git show-ref'}
+
+        # Origin heads
+        for reftype in gitcmd:
+            try:
+                fh = self.gitopen(gitcmd[reftype], noerr=True)
+                for line in fh:
+                    line = line.strip()
+                    rev, name = line.split(None, 1)
+                    if not name.startswith(prefix):
+                        continue
+                    name = '%s%s' % (reftype, name[prefixlen:])
+                    bookmarks[name] = rev
+            except:
+                pass
+
+        return bookmarks

tests/test-convert-git.t

@@ -1,292 +1,297 b''
 
   $ "$TESTDIR/hghave" git || exit 80
   $ echo "[extensions]" >> $HGRCPATH
   $ echo "convert=" >> $HGRCPATH
   $ echo 'hgext.graphlog =' >> $HGRCPATH
   $ GIT_AUTHOR_NAME='test'; export GIT_AUTHOR_NAME
   $ GIT_AUTHOR_EMAIL='[e-mail]'; export GIT_AUTHOR_EMAIL
   $ GIT_AUTHOR_DATE="2007-01-01 00:00:00 +0000"; export GIT_AUTHOR_DATE
   $ GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"; export GIT_COMMITTER_NAME
   $ GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"; export GIT_COMMITTER_EMAIL
   $ GIT_COMMITTER_DATE="$GIT_AUTHOR_DATE"; export GIT_COMMITTER_DATE
   $ count=10
   $ commit()
   > {
   >     GIT_AUTHOR_DATE="2007-01-01 00:00:$count +0000"
   >     GIT_COMMITTER_DATE="$GIT_AUTHOR_DATE"
   >     git commit "$@" >/dev/null 2>/dev/null || echo "git commit error"
   >     count=`expr $count + 1`
   > }
   $ mkdir git-repo
   $ cd git-repo
   $ git init-db >/dev/null 2>/dev/null
   $ echo a > a
   $ mkdir d
   $ echo b > d/b
   $ git add a d
   $ commit -a -m t1
 
 Remove the directory, then try to replace it with a file
 (issue 754)
 
   $ git rm -f d/b
   rm 'd/b'
   $ commit -m t2
   $ echo d > d
   $ git add d
   $ commit -m t3
   $ echo b >> a
   $ commit -a -m t4.1
   $ git checkout -b other HEAD~ >/dev/null 2>/dev/null
   $ echo c > a
   $ echo a >> a
   $ commit -a -m t4.2
   $ git checkout master >/dev/null 2>/dev/null
   $ git pull --no-commit . other > /dev/null 2>/dev/null
   $ commit -m 'Merge branch other'
   $ cd ..
   $ hg convert --datesort git-repo
   assuming destination git-repo-hg
   initializing destination git-repo-hg repository
   scanning source...
   sorting...
   converting...
   5 t1
   4 t2
   3 t3
   2 t4.1
   1 t4.2
   0 Merge branch other
+  updating bookmarks
   $ hg up -q -R git-repo-hg
   $ hg -R git-repo-hg tip -v
   changeset:   5:c78094926be2
+  bookmark:    master
   tag:         tip
   parent:      3:f5f5cb45432b
   parent:      4:4e174f80c67c
   user:        test <[e-mail]>
   date:        Mon Jan 01 00:00:15 2007 +0000
   files:       a
   description:
   Merge branch other
   
   
   $ count=10
   $ mkdir git-repo2
   $ cd git-repo2
   $ git init-db >/dev/null 2>/dev/null
   $ echo foo > foo
   $ git add foo
   $ commit -a -m 'add foo'
   $ echo >> foo
   $ commit -a -m 'change foo'
   $ git checkout -b Bar HEAD~ >/dev/null 2>/dev/null
   $ echo quux >> quux
   $ git add quux
   $ commit -a -m 'add quux'
   $ echo bar > bar
   $ git add bar
   $ commit -a -m 'add bar'
   $ git checkout -b Baz HEAD~ >/dev/null 2>/dev/null
   $ echo baz > baz
   $ git add baz
   $ commit -a -m 'add baz'
   $ git checkout master >/dev/null 2>/dev/null
   $ git pull --no-commit . Bar Baz > /dev/null 2>/dev/null
   $ commit -m 'Octopus merge'
   $ echo bar >> bar
   $ commit -a -m 'change bar'
   $ git checkout -b Foo HEAD~ >/dev/null 2>/dev/null
   $ echo >> foo
   $ commit -a -m 'change foo'
   $ git checkout master >/dev/null 2>/dev/null
   $ git pull --no-commit -s ours . Foo > /dev/null 2>/dev/null
   $ commit -m 'Discard change to foo'
   $ cd ..
   $ glog()
   > {
   >     hg glog --template '{rev} "{desc|firstline}" files: {files}\n' "$@"
   > }
   $ splitrepo()
   > {
   >     msg="$1"
   >     files="$2"
   >     opts=$3
   >     echo "% $files: $msg"
   >     prefix=`echo "$files" | sed -e 's/ /-/g'`
   >     fmap="$prefix.fmap"
   >     repo="$prefix.repo"
   >     for i in $files; do
   >         echo "include $i" >> "$fmap"
   >     done
   >     hg -q convert $opts --filemap "$fmap" --datesort git-repo2 "$repo"
   >     hg up -q -R "$repo"
   >     glog -R "$repo"
   >     hg -R "$repo" manifest --debug
   > }
 
 full conversion
 
   $ hg -q convert --datesort git-repo2 fullrepo
   $ hg up -q -R fullrepo
   $ glog -R fullrepo
   @    9 "Discard change to foo" files: foo
   |\
   | o  8 "change foo" files: foo
   | |
   o |  7 "change bar" files: bar
   |/
   o    6 "(octopus merge fixup)" files:
   |\
   | o    5 "Octopus merge" files: baz
   | |\
   o | |  4 "add baz" files: baz
   | | |
   +---o  3 "add bar" files: bar
   | |
   o |  2 "add quux" files: quux
   | |
   | o  1 "change foo" files: foo
   |/
   o  0 "add foo" files: foo
   
   $ hg -R fullrepo manifest --debug
   245a3b8bc653999c2b22cdabd517ccb47aecafdf 644   bar
   354ae8da6e890359ef49ade27b68bbc361f3ca88 644   baz
   9277c9cc8dd4576fc01a17939b4351e5ada93466 644   foo
   88dfeab657e8cf2cef3dec67b914f49791ae76b1 644   quux
   $ splitrepo 'octopus merge' 'foo bar baz'
   % foo bar baz: octopus merge
   @    8 "Discard change to foo" files: foo
   |\
   | o  7 "change foo" files: foo
   | |
   o |  6 "change bar" files: bar
   |/
   o    5 "(octopus merge fixup)" files:
   |\
   | o    4 "Octopus merge" files: baz
   | |\
   o | |  3 "add baz" files: baz
   | | |
   +---o  2 "add bar" files: bar
   | |
   | o  1 "change foo" files: foo
   |/
   o  0 "add foo" files: foo
   
   245a3b8bc653999c2b22cdabd517ccb47aecafdf 644   bar
   354ae8da6e890359ef49ade27b68bbc361f3ca88 644   baz
   9277c9cc8dd4576fc01a17939b4351e5ada93466 644   foo
   $ splitrepo 'only some parents of an octopus merge; "discard" a head' 'foo baz quux'
   % foo baz quux: only some parents of an octopus merge; "discard" a head
   @  6 "Discard change to foo" files: foo
   |
   o  5 "change foo" files: foo
   |
   o    4 "Octopus merge" files:
   |\
   | o  3 "add baz" files: baz
   | |
   | o  2 "add quux" files: quux
   | |
   o |  1 "change foo" files: foo
   |/
   o  0 "add foo" files: foo
   
   354ae8da6e890359ef49ade27b68bbc361f3ca88 644   baz
   9277c9cc8dd4576fc01a17939b4351e5ada93466 644   foo
   88dfeab657e8cf2cef3dec67b914f49791ae76b1 644   quux
   $ echo
   
 
 test binary conversion (issue 1359)
 
   $ mkdir git-repo3
   $ cd git-repo3
   $ git init-db >/dev/null 2>/dev/null
   $ python -c 'file("b", "wb").write("".join([chr(i) for i in range(256)])*16)'
   $ git add b
   $ commit -a -m addbinary
   $ cd ..
 
 convert binary file
 
   $ hg convert git-repo3 git-repo3-hg
   initializing destination git-repo3-hg repository
   scanning source...
   sorting...
   converting...
   0 addbinary
+  updating bookmarks
   $ cd git-repo3-hg
   $ hg up -C
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ python -c 'print len(file("b", "rb").read())'
   4096
   $ cd ..
   $ echo
   
 
 test author vs committer
 
   $ mkdir git-repo4
   $ cd git-repo4
   $ git init-db >/dev/null 2>/dev/null
   $ echo >> foo
   $ git add foo
   $ commit -a -m addfoo
   $ echo >> foo
   $ GIT_AUTHOR_NAME="nottest"
   $ commit -a -m addfoo2
   $ cd ..
 
 convert author committer
 
   $ hg convert git-repo4 git-repo4-hg
   initializing destination git-repo4-hg repository
   scanning source...
   sorting...
   converting...
   1 addfoo
   0 addfoo2
+  updating bookmarks
   $ hg -R git-repo4-hg log -v
   changeset:   1:d63e967f93da
+  bookmark:    master
   tag:         tip
   user:        nottest <[e-mail]>
   date:        Mon Jan 01 00:00:21 2007 +0000
   files:       foo
   description:
   addfoo2
   
   committer: test <[e-mail]>
   
   
   changeset:   0:0735477b0224
   user:        test <[e-mail]>
   date:        Mon Jan 01 00:00:20 2007 +0000
   files:       foo
   description:
   addfoo
   
   
 
 --sourceorder should fail
 
   $ hg convert --sourcesort git-repo4 git-repo4-sourcesort-hg
   initializing destination git-repo4-sourcesort-hg repository
   abort: --sourcesort is not supported by this data source
   [255]
 
 damage git repository and convert again
 
   $ cat > damage.py <<EOF
   > import os
   > for root, dirs, files in os.walk('git-repo4/.git/objects'):
   >     if files:
   >         path = os.path.join(root, files[0])
   >         os.remove(path)
   >         break
   > EOF
   $ python damage.py
   $ hg convert git-repo4 git-repo4-broken-hg 2>&1 | \
   >     grep 'abort:' | sed 's/abort:.*/abort:/g'
   abort:

tests/test-convert-tagsbranch-topology.t

@@ -1,82 +1,84 b''
 
   $ "$TESTDIR/hghave" git || exit 80
   $ echo "[extensions]" >> $HGRCPATH
   $ echo "convert=" >> $HGRCPATH
   $ echo 'hgext.graphlog =' >> $HGRCPATH
   $ echo '[convert]' >> $HGRCPATH
   $ echo 'hg.usebranchnames = True' >> $HGRCPATH
   $ echo 'hg.tagsbranch = tags-update' >> $HGRCPATH
   $ GIT_AUTHOR_NAME='test'; export GIT_AUTHOR_NAME
   $ GIT_AUTHOR_EMAIL='[e-mail]'; export GIT_AUTHOR_EMAIL
   $ GIT_AUTHOR_DATE="2007-01-01 00:00:00 +0000"; export GIT_AUTHOR_DATE
   $ GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"; export GIT_COMMITTER_NAME
   $ GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"; export GIT_COMMITTER_EMAIL
   $ GIT_COMMITTER_DATE="$GIT_AUTHOR_DATE"; export GIT_COMMITTER_DATE
   $ count=10
   $ action()
   > {
   >     GIT_AUTHOR_DATE="2007-01-01 00:00:$count +0000"
   >     GIT_COMMITTER_DATE="$GIT_AUTHOR_DATE"
   >     git "$@" >/dev/null 2>/dev/null || echo "git command error"
   >     count=`expr $count + 1`
   > }
   $ glog()
   > {
   >     hg glog --template '{rev} "{desc|firstline}" files: {files}\n' "$@"
   > }
   $ convertrepo()
   > {
   >     hg convert --datesort git-repo hg-repo
   > }
 
 Build a GIT repo with at least 1 tag
 
   $ mkdir git-repo
   $ cd git-repo
   $ git init >/dev/null 2>&1
   $ echo a > a
   $ git add a
   $ action commit -m "rev1"
   $ action tag -m "tag1" tag1
   $ cd ..
 
 Do a first conversion
 
   $ convertrepo
   initializing destination hg-repo repository
   scanning source...
   sorting...
   converting...
   0 rev1
   updating tags
+  updating bookmarks
 
 Simulate upstream  updates after first conversion
 
   $ cd git-repo
   $ echo b > a
   $ git add a
   $ action commit -m "rev2"
   $ action tag -m "tag2" tag2
   $ cd ..
 
 Perform an incremental conversion
 
   $ convertrepo
   scanning source...
   sorting...
   converting...
   0 rev2
   updating tags
+  updating bookmarks
 
 Print the log
 
   $ cd hg-repo
   $ glog
   o  3 "update tags" files: .hgtags
   |
   | o  2 "rev2" files: a
   | |
   o |  1 "update tags" files: .hgtags
    /
   o  0 "rev1" files: a