convert: add support for converting git submodule (issue3528)

Previously, convert aborted upon encountering a git submodule. This patch
changes it so that it now succeeds. It modifies convert_git to manually generate
'.hgsub' and '.hgsubstate' files for each git revision, so as to convert git sub
modules to non-mercurial subrepositories.

hgext/convert/git.py

@@ -6,12 +6,24
 # GNU General Public License version 2 or any later version.
 
 import os
-from mercurial import util
+from mercurial import util, config
 from mercurial.node import hex, nullid
 from mercurial.i18n import _
 
 from common import NoRepo, commit, converter_source, checktool
 
+class submodule(object):
+    def __init__(self, path, node, url):
+        self.path = path
+        self.node = node
+        self.url = url
+
+    def hgsub(self):
+        return "%s = [git]%s" % (self.path, self.url)
+
+    def hgsubstate(self):
+        return "%s %s" % (self.node, self.path)
+
 class convert_git(converter_source):
     # Windows does not support GIT_DIR= construct while other systems
     # cannot remove environment variable. Just assume none have
@@ -55,6 +67,7 class convert_git(converter_source):
         checktool('git', 'git')
 
         self.path = path
+        self.submodules = []
 
     def getheads(self):
         if not self.rev:
@@ -76,16 +89,56 class convert_git(converter_source):
         return data
 
     def getfile(self, name, rev):
+        if name == '.hgsub':
+            data = '\n'.join([m.hgsub() for m in self.submoditer()])
+            mode = ''
+        elif name == '.hgsubstate':
+            data = '\n'.join([m.hgsubstate() for m in self.submoditer()])
+            mode = ''
+        else:
         data = self.catfile(rev, "blob")
         mode = self.modecache[(name, rev)]
         return data, mode
 
+    def submoditer(self):
+        null = hex(nullid)
+        for m in sorted(self.submodules, key=lambda p: p.path):
+            if m.node != null:
+                yield m
+
+    def parsegitmodules(self, content):
+        """Parse the formatted .gitmodules file, example file format:
+        [submodule "sub"]\n
+        \tpath = sub\n
+        \turl = git://giturl\n
+        """
+        self.submodules = []
+        c = config.config()
+        # Each item in .gitmodules starts with \t that cant be parsed
+        c.parse('.gitmodules', content.replace('\t',''))
+        for sec in c.sections():
+            s = c[sec]
+            if 'url' in s and 'path' in s:
+                self.submodules.append(submodule(s['path'], '', s['url']))
+
+    def retrievegitmodules(self, version):
+        modules, ret = self.gitread("git show %s:%s" % (version, '.gitmodules'))
+        if ret:
+            raise util.Abort(_('cannot read submodules config file in %s') % version)
+        self.parsegitmodules(modules)
+        for m in self.submodules:
+            node, ret = self.gitread("git rev-parse %s:%s" % (version, m.path))
+            if ret:
+                continue
+            m.node = node.strip()
+
     def getchanges(self, version):
         self.modecache = {}
         fh = self.gitopen("git diff-tree -z --root -m -r %s" % version)
         changes = []
         seen = set()
         entry = None
+        subexists = False
         for l in fh.read().split('\x00'):
             if not entry:
                 if not l.startswith(':'):
@@ -97,15 +150,24 class convert_git(converter_source):
                 seen.add(f)
                 entry = entry.split()
                 h = entry[3]
-                if entry[1] == '160000':
-                    raise util.Abort('git submodules are not supported!')
                 p = (entry[1] == "100755")
                 s = (entry[1] == "120000")
+
+                if f == '.gitmodules':
+                    subexists = True
+                    changes.append(('.hgsub', ''))
+                elif entry[1] == '160000' or entry[0] == ':160000':
+                    subexists = True
+                else:
                 self.modecache[(f, h)] = (p and "x") or (s and "l") or ""
                 changes.append((f, h))
             entry = None
         if fh.close():
             raise util.Abort(_('cannot read changes in %s') % version)
+
+        if subexists:
+            self.retrievegitmodules(version)
+            changes.append(('.hgsubstate', ''))
         return (changes, {})
 
     def getcommit(self, version):

tests/test-convert-git.t

@@ -298,3 +298,50 damage git repository and convert again
   $ hg convert git-repo4 git-repo4-broken-hg 2>&1 | \
   >     grep 'abort:' | sed 's/abort:.*/abort:/g'
   abort:
+
+test sub modules
+
+  $ mkdir git-repo5
+  $ cd git-repo5
+  $ git init-db >/dev/null 2>/dev/null
+  $ echo 'sub' >> foo
+  $ git add foo
+  $ commit -a -m 'addfoo'
+  $ BASE=${PWD}
+  $ cd ..
+  $ mkdir git-repo6
+  $ cd git-repo6
+  $ git init-db >/dev/null 2>/dev/null
+  $ git submodule add ${BASE} >/dev/null 2>/dev/null
+  $ commit -a -m 'addsubmodule' >/dev/null 2>/dev/null
+  $ cd ..
+
+convert sub modules
+  $ hg convert git-repo6 git-repo6-hg
+  initializing destination git-repo6-hg repository
+  scanning source...
+  sorting...
+  converting...
+  0 addsubmodule
+  updating bookmarks
+  $ hg -R git-repo6-hg log -v
+  changeset:   0:* (glob)
+  bookmark:    master
+  tag:         tip
+  user:        nottest <[e-mail]>
+  date:        Mon Jan 01 00:00:23 2007 +0000
+  files:       .hgsub .hgsubstate
+  description:
+  addsubmodule
+  
+  committer: test <[e-mail]>
+  
+  
+
+  $ cd git-repo6-hg
+  $ hg up >/dev/null 2>/dev/null
+  $ cat .hgsubstate
+  * git-repo5 (glob)
+  $ cd git-repo5
+  $ cat foo
+  sub