convert: rewrite calls to Git to use the new shelling mechanism (SEC)

CVE-2016-3069 (2/5)

One test output changed because we were ignoring git return code in numcommits
before.

hgext/convert/git.py

@@ -1,432 +1,438 b''
 # git.py - git support for the convert extension
 #
 #  Copyright 2005-2009 Matt Mackall <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 import os
 import subprocess
 from mercurial import util, config, error
 from mercurial.node import hex, nullid
 from mercurial.i18n import _
 
 from common import NoRepo, commit, converter_source, checktool, commandline
 
 class submodule(object):
     def __init__(self, path, node, url):
         self.path = path
         self.node = node
         self.url = url
 
     def hgsub(self):
         return "%s = [git]%s" % (self.path, self.url)
 
     def hgsubstate(self):
         return "%s %s" % (self.node, self.path)
 
 class convert_git(converter_source, commandline):
     # Windows does not support GIT_DIR= construct while other systems
     # cannot remove environment variable. Just assume none have
     # both issues.
     if util.safehasattr(os, 'unsetenv'):
         def gitopen(self, s, err=None):
             prevgitdir = os.environ.get('GIT_DIR')
             os.environ['GIT_DIR'] = self.path
             try:
                 if err == subprocess.PIPE:
                     (stdin, stdout, stderr) = util.popen3(s)
                     return stdout
                 elif err == subprocess.STDOUT:
                     return self.popen_with_stderr(s)
                 else:
                     return util.popen(s, 'rb')
             finally:
                 if prevgitdir is None:
                     del os.environ['GIT_DIR']
                 else:
                     os.environ['GIT_DIR'] = prevgitdir
 
         def gitpipe(self, s):
             prevgitdir = os.environ.get('GIT_DIR')
             os.environ['GIT_DIR'] = self.path
             try:
                 return util.popen3(s)
             finally:
                 if prevgitdir is None:
                     del os.environ['GIT_DIR']
                 else:
                     os.environ['GIT_DIR'] = prevgitdir
 
     else:
         def gitopen(self, s, err=None):
             if err == subprocess.PIPE:
                 (sin, so, se) = util.popen3('GIT_DIR=%s %s' % (self.path, s))
                 return so
             elif err == subprocess.STDOUT:
                     return self.popen_with_stderr(s)
             else:
                 return util.popen('GIT_DIR=%s %s' % (self.path, s), 'rb')
 
         def gitpipe(self, s):
             return util.popen3('GIT_DIR=%s %s' % (self.path, s))
 
     def _gitcmd(self, cmd, *args, **kwargs):
         return cmd('--git-dir=%s' % self.path, *args, **kwargs)
 
     def gitrun0(self, *args, **kwargs):
         return self._gitcmd(self.run0, *args, **kwargs)
 
     def gitrun(self, *args, **kwargs):
         return self._gitcmd(self.run, *args, **kwargs)
 
     def gitrunlines0(self, *args, **kwargs):
         return self._gitcmd(self.runlines0, *args, **kwargs)
 
     def gitrunlines(self, *args, **kwargs):
         return self._gitcmd(self.runlines, *args, **kwargs)
 
     def popen_with_stderr(self, s):
         p = subprocess.Popen(s, shell=True, bufsize=-1,
                              close_fds=util.closefds,
                              stdin=subprocess.PIPE,
                              stdout=subprocess.PIPE,
                              stderr=subprocess.STDOUT,
                              universal_newlines=False,
                              env=None)
         return p.stdout
 
     def gitread(self, s):
         fh = self.gitopen(s)
         data = fh.read()
         return data, fh.close()
 
     def __init__(self, ui, path, revs=None):
         super(convert_git, self).__init__(ui, path, revs=revs)
         commandline.__init__(self, ui, 'git')
 
         if os.path.isdir(path + "/.git"):
             path += "/.git"
         if not os.path.exists(path + "/objects"):
             raise NoRepo(_("%s does not look like a Git repository") % path)
 
         # The default value (50) is based on the default for 'git diff'.
         similarity = ui.configint('convert', 'git.similarity', default=50)
         if similarity < 0 or similarity > 100:
             raise error.Abort(_('similarity must be between 0 and 100'))
         if similarity > 0:
-            self.simopt = '-C%d%%' % similarity
+            self.simopt = ['-C%d%%' % similarity]
             findcopiesharder = ui.configbool('convert', 'git.findcopiesharder',
                                              False)
             if findcopiesharder:
-                self.simopt += ' --find-copies-harder'
+                self.simopt.append('--find-copies-harder')
         else:
-            self.simopt = ''
+            self.simopt = []
 
         checktool('git', 'git')
 
         self.path = path
         self.submodules = []
 
         self.catfilepipe = self.gitpipe('git cat-file --batch')
 
     def after(self):
         for f in self.catfilepipe:
             f.close()
 
     def getheads(self):
         if not self.revs:
-            heads, ret = self.gitread('git rev-parse --branches --remotes')
-            heads = heads.splitlines()
-            if ret:
+            output, status = self.gitrun('rev-parse', '--branches', '--remotes')
+            heads = output.splitlines()
+            if status:
                 raise error.Abort(_('cannot retrieve git heads'))
         else:
             heads = []
             for rev in self.revs:
-                rawhead, ret = self.gitread("git rev-parse --verify %s" % rev)
+                rawhead, ret = self.gitrun('rev-parse', '--verify', rev)
                 heads.append(rawhead[:-1])
                 if ret:
                     raise error.Abort(_('cannot retrieve git head "%s"') % rev)
         return heads
 
     def catfile(self, rev, type):
         if rev == hex(nullid):
             raise IOError
         self.catfilepipe[0].write(rev+'\n')
         self.catfilepipe[0].flush()
         info = self.catfilepipe[1].readline().split()
         if info[1] != type:
             raise error.Abort(_('cannot read %r object at %s') % (type, rev))
         size = int(info[2])
         data = self.catfilepipe[1].read(size)
         if len(data) < size:
             raise error.Abort(_('cannot read %r object at %s: unexpected size')
                              % (type, rev))
         # read the trailing newline
         self.catfilepipe[1].read(1)
         return data
 
     def getfile(self, name, rev):
         if rev == hex(nullid):
             return None, None
         if name == '.hgsub':
             data = '\n'.join([m.hgsub() for m in self.submoditer()])
             mode = ''
         elif name == '.hgsubstate':
             data = '\n'.join([m.hgsubstate() for m in self.submoditer()])
             mode = ''
         else:
             data = self.catfile(rev, "blob")
             mode = self.modecache[(name, rev)]
         return data, mode
 
     def submoditer(self):
         null = hex(nullid)
         for m in sorted(self.submodules, key=lambda p: p.path):
             if m.node != null:
                 yield m
 
     def parsegitmodules(self, content):
         """Parse the formatted .gitmodules file, example file format:
         [submodule "sub"]\n
         \tpath = sub\n
         \turl = git://giturl\n
         """
         self.submodules = []
         c = config.config()
         # Each item in .gitmodules starts with whitespace that cant be parsed
         c.parse('.gitmodules', '\n'.join(line.strip() for line in
                                content.split('\n')))
         for sec in c.sections():
             s = c[sec]
             if 'url' in s and 'path' in s:
                 self.submodules.append(submodule(s['path'], '', s['url']))
 
     def retrievegitmodules(self, version):
-        modules, ret = self.gitread("git show %s:%s" % (version, '.gitmodules'))
+        modules, ret = self.gitrun('show', '%s:%s' % (version, '.gitmodules'))
         if ret:
             # This can happen if a file is in the repo that has permissions
             # 160000, but there is no .gitmodules file.
             self.ui.warn(_("warning: cannot read submodules config file in "
                            "%s\n") % version)
             return
 
         try:
             self.parsegitmodules(modules)
         except error.ParseError:
             self.ui.warn(_("warning: unable to parse .gitmodules in %s\n")
                          % version)
             return
 
         for m in self.submodules:
-            node, ret = self.gitread("git rev-parse %s:%s" % (version, m.path))
+            node, ret = self.gitrun('rev-parse', '%s:%s' % (version, m.path))
             if ret:
                 continue
             m.node = node.strip()
 
     def getchanges(self, version, full):
         if full:
             raise error.Abort(_("convert from git does not support --full"))
         self.modecache = {}
-        fh = self.gitopen("git diff-tree -z --root -m -r %s %s" % (
-            self.simopt, version))
+        cmd = ['diff-tree','-z', '--root', '-m', '-r'] + self.simopt + [version]
+        output, status = self.gitrun(*cmd)
+        if status:
+            raise error.Abort(_('cannot read changes in %s') % version)
         changes = []
         copies = {}
         seen = set()
         entry = None
         subexists = [False]
         subdeleted = [False]
-        difftree = fh.read().split('\x00')
+        difftree = output.split('\x00')
         lcount = len(difftree)
         i = 0
 
         skipsubmodules = self.ui.configbool('convert', 'git.skipsubmodules',
                                             False)
         def add(entry, f, isdest):
             seen.add(f)
             h = entry[3]
             p = (entry[1] == "100755")
             s = (entry[1] == "120000")
             renamesource = (not isdest and entry[4][0] == 'R')
 
             if f == '.gitmodules':
                 if skipsubmodules:
                     return
 
                 subexists[0] = True
                 if entry[4] == 'D' or renamesource:
                     subdeleted[0] = True
                     changes.append(('.hgsub', hex(nullid)))
                 else:
                     changes.append(('.hgsub', ''))
             elif entry[1] == '160000' or entry[0] == ':160000':
                 if not skipsubmodules:
                     subexists[0] = True
             else:
                 if renamesource:
                     h = hex(nullid)
                 self.modecache[(f, h)] = (p and "x") or (s and "l") or ""
                 changes.append((f, h))
 
         while i < lcount:
             l = difftree[i]
             i += 1
             if not entry:
                 if not l.startswith(':'):
                     continue
                 entry = l.split()
                 continue
             f = l
             if entry[4][0] == 'C':
                 copysrc = f
                 copydest = difftree[i]
                 i += 1
                 f = copydest
                 copies[copydest] = copysrc
             if f not in seen:
                 add(entry, f, False)
             # A file can be copied multiple times, or modified and copied
             # simultaneously. So f can be repeated even if fdest isn't.
             if entry[4][0] == 'R':
                 # rename: next line is the destination
                 fdest = difftree[i]
                 i += 1
                 if fdest not in seen:
                     add(entry, fdest, True)
                     # .gitmodules isn't imported at all, so it being copied to
                     # and fro doesn't really make sense
                     if f != '.gitmodules' and fdest != '.gitmodules':
                         copies[fdest] = f
             entry = None
-        if fh.close():
-            raise error.Abort(_('cannot read changes in %s') % version)
 
         if subexists[0]:
             if subdeleted[0]:
                 changes.append(('.hgsubstate', hex(nullid)))
             else:
                 self.retrievegitmodules(version)
                 changes.append(('.hgsubstate', ''))
         return (changes, copies, set())
 
     def getcommit(self, version):
         c = self.catfile(version, "commit") # read the commit hash
         end = c.find("\n\n")
         message = c[end + 2:]
         message = self.recode(message)
         l = c[:end].splitlines()
         parents = []
         author = committer = None
         for e in l[1:]:
             n, v = e.split(" ", 1)
             if n == "author":
                 p = v.split()
                 tm, tz = p[-2:]
                 author = " ".join(p[:-2])
                 if author[0] == "<": author = author[1:-1]
                 author = self.recode(author)
             if n == "committer":
                 p = v.split()
                 tm, tz = p[-2:]
                 committer = " ".join(p[:-2])
                 if committer[0] == "<": committer = committer[1:-1]
                 committer = self.recode(committer)
             if n == "parent":
                 parents.append(v)
 
         if committer and committer != author:
             message += "\ncommitter: %s\n" % committer
         tzs, tzh, tzm = tz[-5:-4] + "1", tz[-4:-2], tz[-2:]
         tz = -int(tzs) * (int(tzh) * 3600 + int(tzm))
         date = tm + " " + str(tz)
 
         c = commit(parents=parents, date=date, author=author, desc=message,
                    rev=version)
         return c
 
     def numcommits(self):
-        return len([None for _ in self.gitopen('git rev-list --all')])
+        output, ret = self.gitrunlines('rev-list', '--all')
+        if ret:
+            raise error.Abort(_('cannot retrieve number of commits in %s') \
+                              % self.path)
+        return len(output)
 
     def gettags(self):
         tags = {}
         alltags = {}
-        fh = self.gitopen('git ls-remote --tags "%s"' % self.path,
-                          err=subprocess.STDOUT)
+        output, status = self.gitrunlines('ls-remote', '--tags', self.path)
+
+        if status:
+            raise error.Abort(_('cannot read tags from %s') % self.path)
         prefix = 'refs/tags/'
 
         # Build complete list of tags, both annotated and bare ones
-        for line in fh:
+        for line in output:
             line = line.strip()
             if line.startswith("error:") or line.startswith("fatal:"):
                 raise error.Abort(_('cannot read tags from %s') % self.path)
             node, tag = line.split(None, 1)
             if not tag.startswith(prefix):
                 continue
             alltags[tag[len(prefix):]] = node
-        if fh.close():
-            raise error.Abort(_('cannot read tags from %s') % self.path)
 
         # Filter out tag objects for annotated tag refs
         for tag in alltags:
             if tag.endswith('^{}'):
                 tags[tag[:-3]] = alltags[tag]
             else:
                 if tag + '^{}' in alltags:
                     continue
                 else:
                     tags[tag] = alltags[tag]
 
         return tags
 
     def getchangedfiles(self, version, i):
         changes = []
         if i is None:
-            fh = self.gitopen("git diff-tree --root -m -r %s" % version)
-            for l in fh:
+            output, status = self.gitrunlines('diff-tree', '--root', '-m',
+                                              '-r', version)
+            if status:
+                raise error.Abort(_('cannot read changes in %s') % version)
+            for l in output:
                 if "\t" not in l:
                     continue
                 m, f = l[:-1].split("\t")
                 changes.append(f)
         else:
-            fh = self.gitopen('git diff-tree --name-only --root -r %s '
-                              '"%s^%s" --' % (version, version, i + 1))
-            changes = [f.rstrip('\n') for f in fh]
-        if fh.close():
-            raise error.Abort(_('cannot read changes in %s') % version)
+            output, status = self.gitrunlines('diff-tree', '--name-only',
+                                              '--root', '-r', version,
+                                              '%s^%s' % (version, i + 1), '--')
+            changes = [f.rstrip('\n') for f in output]
 
         return changes
 
     def getbookmarks(self):
         bookmarks = {}
 
         # Handle local and remote branches
         remoteprefix = self.ui.config('convert', 'git.remoteprefix', 'remote')
         reftypes = [
             # (git prefix, hg prefix)
             ('refs/remotes/origin/', remoteprefix + '/'),
             ('refs/heads/', '')
         ]
 
         exclude = set([
             'refs/remotes/origin/HEAD',
         ])
 
         try:
-            fh = self.gitopen('git show-ref', err=subprocess.PIPE)
-            for line in fh:
+            output, status = self.gitrunlines('show-ref')
+            for line in output:
                 line = line.strip()
                 rev, name = line.split(None, 1)
                 # Process each type of branch
                 for gitprefix, hgprefix in reftypes:
                     if not name.startswith(gitprefix) or name in exclude:
                         continue
                     name = '%s%s' % (hgprefix, name[len(gitprefix):])
                     bookmarks[name] = rev
         except Exception:
             pass
 
         return bookmarks
 
     def checkrevformat(self, revstr, mapname='splicemap'):
         """ git revision string is a 40 byte hex """
         self.checkhexformat(revstr, mapname)

tests/test-convert-git.t

@@ -1,731 +1,731 b''
 #require git
 
   $ echo "[core]" >> $HOME/.gitconfig
   $ echo "autocrlf = false" >> $HOME/.gitconfig
   $ echo "[core]" >> $HOME/.gitconfig
   $ echo "autocrlf = false" >> $HOME/.gitconfig
   $ echo "[extensions]" >> $HGRCPATH
   $ echo "convert=" >> $HGRCPATH
   $ GIT_AUTHOR_NAME='test'; export GIT_AUTHOR_NAME
   $ GIT_AUTHOR_EMAIL='[e-mail]'; export GIT_AUTHOR_EMAIL
   $ GIT_AUTHOR_DATE="2007-01-01 00:00:00 +0000"; export GIT_AUTHOR_DATE
   $ GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"; export GIT_COMMITTER_NAME
   $ GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"; export GIT_COMMITTER_EMAIL
   $ GIT_COMMITTER_DATE="$GIT_AUTHOR_DATE"; export GIT_COMMITTER_DATE
   $ INVALIDID1=afd12345af
   $ INVALIDID2=28173x36ddd1e67bf7098d541130558ef5534a86
   $ VALIDID1=39b3d83f9a69a9ba4ebb111461071a0af0027357
   $ VALIDID2=8dd6476bd09d9c7776355dc454dafe38efaec5da
   $ count=10
   $ commit()
   > {
   >     GIT_AUTHOR_DATE="2007-01-01 00:00:$count +0000"
   >     GIT_COMMITTER_DATE="$GIT_AUTHOR_DATE"
   >     git commit "$@" >/dev/null 2>/dev/null || echo "git commit error"
   >     count=`expr $count + 1`
   > }
   $ mkdir git-repo
   $ cd git-repo
   $ git init-db >/dev/null 2>/dev/null
   $ echo a > a
   $ mkdir d
   $ echo b > d/b
   $ git add a d
   $ commit -a -m t1
 
 Remove the directory, then try to replace it with a file (issue754)
 
   $ git rm -f d/b
   rm 'd/b'
   $ commit -m t2
   $ echo d > d
   $ git add d
   $ commit -m t3
   $ echo b >> a
   $ commit -a -m t4.1
   $ git checkout -b other HEAD~ >/dev/null 2>/dev/null
   $ echo c > a
   $ echo a >> a
   $ commit -a -m t4.2
   $ git checkout master >/dev/null 2>/dev/null
   $ git pull --no-commit . other > /dev/null 2>/dev/null
   $ commit -m 'Merge branch other'
   $ cd ..
   $ hg convert --config extensions.progress= --config progress.assume-tty=1 \
   >   --config progress.delay=0 --config progress.changedelay=0 \
   >   --config progress.refresh=0 --config progress.width=60 \
   >   --config progress.format='topic, bar, number' --datesort git-repo
   \r (no-eol) (esc)
   scanning [======>                                     ] 1/6\r (no-eol) (esc)
   scanning [=============>                              ] 2/6\r (no-eol) (esc)
   scanning [=====================>                      ] 3/6\r (no-eol) (esc)
   scanning [============================>               ] 4/6\r (no-eol) (esc)
   scanning [===================================>        ] 5/6\r (no-eol) (esc)
   scanning [===========================================>] 6/6\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   converting [                                          ] 0/6\r (no-eol) (esc)
   getting files [==================>                    ] 1/2\r (no-eol) (esc)
   getting files [======================================>] 2/2\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   converting [======>                                   ] 1/6\r (no-eol) (esc)
   getting files [======================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   converting [=============>                            ] 2/6\r (no-eol) (esc)
   getting files [======================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   converting [====================>                     ] 3/6\r (no-eol) (esc)
   getting files [======================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   converting [===========================>              ] 4/6\r (no-eol) (esc)
   getting files [======================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   converting [==================================>       ] 5/6\r (no-eol) (esc)
   getting files [======================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   assuming destination git-repo-hg
   initializing destination git-repo-hg repository
   scanning source...
   sorting...
   converting...
   5 t1
   4 t2
   3 t3
   2 t4.1
   1 t4.2
   0 Merge branch other
   updating bookmarks
   $ hg up -q -R git-repo-hg
   $ hg -R git-repo-hg tip -v
   changeset:   5:c78094926be2
   bookmark:    master
   tag:         tip
   parent:      3:f5f5cb45432b
   parent:      4:4e174f80c67c
   user:        test <[e-mail]>
   date:        Mon Jan 01 00:00:15 2007 +0000
   files:       a
   description:
   Merge branch other
   
   
   $ count=10
   $ mkdir git-repo2
   $ cd git-repo2
   $ git init-db >/dev/null 2>/dev/null
   $ echo foo > foo
   $ git add foo
   $ commit -a -m 'add foo'
   $ echo >> foo
   $ commit -a -m 'change foo'
   $ git checkout -b Bar HEAD~ >/dev/null 2>/dev/null
   $ echo quux >> quux
   $ git add quux
   $ commit -a -m 'add quux'
   $ echo bar > bar
   $ git add bar
   $ commit -a -m 'add bar'
   $ git checkout -b Baz HEAD~ >/dev/null 2>/dev/null
   $ echo baz > baz
   $ git add baz
   $ commit -a -m 'add baz'
   $ git checkout master >/dev/null 2>/dev/null
   $ git pull --no-commit . Bar Baz > /dev/null 2>/dev/null
   $ commit -m 'Octopus merge'
   $ echo bar >> bar
   $ commit -a -m 'change bar'
   $ git checkout -b Foo HEAD~ >/dev/null 2>/dev/null
   $ echo >> foo
   $ commit -a -m 'change foo'
   $ git checkout master >/dev/null 2>/dev/null
   $ git pull --no-commit -s ours . Foo > /dev/null 2>/dev/null
   $ commit -m 'Discard change to foo'
   $ cd ..
   $ glog()
   > {
   >     hg log -G --template '{rev} "{desc|firstline}" files: {files}\n' "$@"
   > }
   $ splitrepo()
   > {
   >     msg="$1"
   >     files="$2"
   >     opts=$3
   >     echo "% $files: $msg"
   >     prefix=`echo "$files" | sed -e 's/ /-/g'`
   >     fmap="$prefix.fmap"
   >     repo="$prefix.repo"
   >     for i in $files; do
   >         echo "include $i" >> "$fmap"
   >     done
   >     hg -q convert $opts --filemap "$fmap" --datesort git-repo2 "$repo"
   >     hg up -q -R "$repo"
   >     glog -R "$repo"
   >     hg -R "$repo" manifest --debug
   > }
 
 full conversion
 
   $ hg convert --datesort git-repo2 fullrepo \
   > --config extensions.progress= --config progress.assume-tty=1 \
   > --config progress.delay=0 --config progress.changedelay=0 \
   > --config progress.refresh=0 --config progress.width=60 \
   > --config progress.format='topic, bar, number'
   \r (no-eol) (esc)
   scanning [===>                                        ] 1/9\r (no-eol) (esc)
   scanning [========>                                   ] 2/9\r (no-eol) (esc)
   scanning [=============>                              ] 3/9\r (no-eol) (esc)
   scanning [==================>                         ] 4/9\r (no-eol) (esc)
   scanning [=======================>                    ] 5/9\r (no-eol) (esc)
   scanning [============================>               ] 6/9\r (no-eol) (esc)
   scanning [=================================>          ] 7/9\r (no-eol) (esc)
   scanning [======================================>     ] 8/9\r (no-eol) (esc)
   scanning [===========================================>] 9/9\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   converting [                                          ] 0/9\r (no-eol) (esc)
   getting files [======================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   converting [===>                                      ] 1/9\r (no-eol) (esc)
   getting files [======================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   converting [========>                                 ] 2/9\r (no-eol) (esc)
   getting files [======================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   converting [=============>                            ] 3/9\r (no-eol) (esc)
   getting files [======================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   converting [=================>                        ] 4/9\r (no-eol) (esc)
   getting files [======================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   converting [======================>                   ] 5/9\r (no-eol) (esc)
   getting files [===>                                   ] 1/8\r (no-eol) (esc)
   getting files [========>                              ] 2/8\r (no-eol) (esc)
   getting files [=============>                         ] 3/8\r (no-eol) (esc)
   getting files [==================>                    ] 4/8\r (no-eol) (esc)
   getting files [=======================>               ] 5/8\r (no-eol) (esc)
   getting files [============================>          ] 6/8\r (no-eol) (esc)
   getting files [=================================>     ] 7/8\r (no-eol) (esc)
   getting files [======================================>] 8/8\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   converting [===========================>              ] 6/9\r (no-eol) (esc)
   getting files [======================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   converting [===============================>          ] 7/9\r (no-eol) (esc)
   getting files [======================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   converting [====================================>     ] 8/9\r (no-eol) (esc)
   getting files [==================>                    ] 1/2\r (no-eol) (esc)
   getting files [======================================>] 2/2\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   initializing destination fullrepo repository
   scanning source...
   sorting...
   converting...
   8 add foo
   7 change foo
   6 add quux
   5 add bar
   4 add baz
   3 Octopus merge
   2 change bar
   1 change foo
   0 Discard change to foo
   updating bookmarks
   $ hg up -q -R fullrepo
   $ glog -R fullrepo
   @    9 "Discard change to foo" files: foo
   |\
   | o  8 "change foo" files: foo
   | |
   o |  7 "change bar" files: bar
   |/
   o    6 "(octopus merge fixup)" files:
   |\
   | o    5 "Octopus merge" files: baz
   | |\
   o | |  4 "add baz" files: baz
   | | |
   +---o  3 "add bar" files: bar
   | |
   o |  2 "add quux" files: quux
   | |
   | o  1 "change foo" files: foo
   |/
   o  0 "add foo" files: foo
   
   $ hg -R fullrepo manifest --debug
   245a3b8bc653999c2b22cdabd517ccb47aecafdf 644   bar
   354ae8da6e890359ef49ade27b68bbc361f3ca88 644   baz
   9277c9cc8dd4576fc01a17939b4351e5ada93466 644   foo
   88dfeab657e8cf2cef3dec67b914f49791ae76b1 644   quux
   $ splitrepo 'octopus merge' 'foo bar baz'
   % foo bar baz: octopus merge
   @    8 "Discard change to foo" files: foo
   |\
   | o  7 "change foo" files: foo
   | |
   o |  6 "change bar" files: bar
   |/
   o    5 "(octopus merge fixup)" files:
   |\
   | o    4 "Octopus merge" files: baz
   | |\
   o | |  3 "add baz" files: baz
   | | |
   +---o  2 "add bar" files: bar
   | |
   | o  1 "change foo" files: foo
   |/
   o  0 "add foo" files: foo
   
   245a3b8bc653999c2b22cdabd517ccb47aecafdf 644   bar
   354ae8da6e890359ef49ade27b68bbc361f3ca88 644   baz
   9277c9cc8dd4576fc01a17939b4351e5ada93466 644   foo
   $ splitrepo 'only some parents of an octopus merge; "discard" a head' 'foo baz quux'
   % foo baz quux: only some parents of an octopus merge; "discard" a head
   @  6 "Discard change to foo" files: foo
   |
   o  5 "change foo" files: foo
   |
   o    4 "Octopus merge" files:
   |\
   | o  3 "add baz" files: baz
   | |
   | o  2 "add quux" files: quux
   | |
   o |  1 "change foo" files: foo
   |/
   o  0 "add foo" files: foo
   
   354ae8da6e890359ef49ade27b68bbc361f3ca88 644   baz
   9277c9cc8dd4576fc01a17939b4351e5ada93466 644   foo
   88dfeab657e8cf2cef3dec67b914f49791ae76b1 644   quux
 
 test importing git renames and copies
 
   $ cd git-repo2
   $ git mv foo foo-renamed
 since bar is not touched in this commit, this copy will not be detected
   $ cp bar bar-copied
   $ cp baz baz-copied
   $ cp baz baz-copied2
   $ cp baz ba-copy
   $ echo baz2 >> baz
   $ git add bar-copied baz-copied baz-copied2 ba-copy
   $ commit -a -m 'rename and copy'
   $ cd ..
 
 input validation
   $ hg convert --config convert.git.similarity=foo --datesort git-repo2 fullrepo
   abort: convert.git.similarity is not an integer ('foo')
   [255]
   $ hg convert --config convert.git.similarity=-1 --datesort git-repo2 fullrepo
   abort: similarity must be between 0 and 100
   [255]
   $ hg convert --config convert.git.similarity=101 --datesort git-repo2 fullrepo
   abort: similarity must be between 0 and 100
   [255]
 
   $ hg -q convert --config convert.git.similarity=100 --datesort git-repo2 fullrepo
   $ hg -R fullrepo status -C --change master
   M baz
   A ba-copy
     baz
   A bar-copied
   A baz-copied
     baz
   A baz-copied2
     baz
   A foo-renamed
     foo
   R foo
 
 Ensure that the modification to the copy source was preserved
 (there was a bug where if the copy dest was alphabetically prior to the copy
 source, the copy source took the contents of the copy dest)
   $ hg cat -r tip fullrepo/baz
   baz
   baz2
 
   $ cd git-repo2
   $ echo bar2 >> bar
   $ commit -a -m 'change bar'
   $ cp bar bar-copied2
   $ git add bar-copied2
   $ commit -a -m 'copy with no changes'
   $ cd ..
 
   $ hg -q convert --config convert.git.similarity=100 \
   > --config convert.git.findcopiesharder=1 --datesort git-repo2 fullrepo
   $ hg -R fullrepo status -C --change master
   A bar-copied2
     bar
 
 test binary conversion (issue1359)
 
   $ count=19
   $ mkdir git-repo3
   $ cd git-repo3
   $ git init-db >/dev/null 2>/dev/null
   $ $PYTHON -c 'file("b", "wb").write("".join([chr(i) for i in range(256)])*16)'
   $ git add b
   $ commit -a -m addbinary
   $ cd ..
 
 convert binary file
 
   $ hg convert git-repo3 git-repo3-hg
   initializing destination git-repo3-hg repository
   scanning source...
   sorting...
   converting...
   0 addbinary
   updating bookmarks
   $ cd git-repo3-hg
   $ hg up -C
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ $PYTHON -c 'print len(file("b", "rb").read())'
   4096
   $ cd ..
 
 test author vs committer
 
   $ mkdir git-repo4
   $ cd git-repo4
   $ git init-db >/dev/null 2>/dev/null
   $ echo >> foo
   $ git add foo
   $ commit -a -m addfoo
   $ echo >> foo
   $ GIT_AUTHOR_NAME="nottest"
   $ commit -a -m addfoo2
   $ cd ..
 
 convert author committer
 
   $ hg convert git-repo4 git-repo4-hg
   initializing destination git-repo4-hg repository
   scanning source...
   sorting...
   converting...
   1 addfoo
   0 addfoo2
   updating bookmarks
   $ hg -R git-repo4-hg log -v
   changeset:   1:d63e967f93da
   bookmark:    master
   tag:         tip
   user:        nottest <[e-mail]>
   date:        Mon Jan 01 00:00:21 2007 +0000
   files:       foo
   description:
   addfoo2
   
   committer: test <[e-mail]>
   
   
   changeset:   0:0735477b0224
   user:        test <[e-mail]>
   date:        Mon Jan 01 00:00:20 2007 +0000
   files:       foo
   description:
   addfoo
   
   
 
 --sourceorder should fail
 
   $ hg convert --sourcesort git-repo4 git-repo4-sourcesort-hg
   initializing destination git-repo4-sourcesort-hg repository
   abort: --sourcesort is not supported by this data source
   [255]
 
 test converting certain branches
 
   $ mkdir git-testrevs
   $ cd git-testrevs
   $ git init
   Initialized empty Git repository in $TESTTMP/git-testrevs/.git/
   $ echo a >> a ; git add a > /dev/null; git commit -m 'first' > /dev/null
   $ echo a >> a ; git add a > /dev/null; git commit -m 'master commit' > /dev/null
   $ git checkout -b goodbranch 'HEAD^'
   Switched to a new branch 'goodbranch'
   $ echo a >> b ; git add b > /dev/null; git commit -m 'good branch commit' > /dev/null
   $ git checkout -b badbranch 'HEAD^'
   Switched to a new branch 'badbranch'
   $ echo a >> c ; git add c > /dev/null; git commit -m 'bad branch commit' > /dev/null
   $ cd ..
   $ hg convert git-testrevs hg-testrevs --rev master --rev goodbranch
   initializing destination hg-testrevs repository
   scanning source...
   sorting...
   converting...
   2 first
   1 good branch commit
   0 master commit
   updating bookmarks
   $ cd hg-testrevs
   $ hg log -G -T '{rev} {bookmarks}'
   o  2 master
   |
   | o  1 goodbranch
   |/
   o  0
   
   $ cd ..
 
 test sub modules
 
   $ mkdir git-repo5
   $ cd git-repo5
   $ git init-db >/dev/null 2>/dev/null
   $ echo 'sub' >> foo
   $ git add foo
   $ commit -a -m 'addfoo'
   $ BASE=`pwd`
   $ cd ..
   $ mkdir git-repo6
   $ cd git-repo6
   $ git init-db >/dev/null 2>/dev/null
   $ git submodule add ${BASE} >/dev/null 2>/dev/null
   $ commit -a -m 'addsubmodule' >/dev/null 2>/dev/null
 
 test non-tab whitespace .gitmodules
 
   $ cat >> .gitmodules <<EOF
   > [submodule "git-repo5"]
   >   path = git-repo5
   >   url = git-repo5
   > EOF
   $ git commit -q -a -m "weird white space submodule"
   $ cd ..
   $ hg convert git-repo6 hg-repo6
   initializing destination hg-repo6 repository
   scanning source...
   sorting...
   converting...
   1 addsubmodule
   0 weird white space submodule
   updating bookmarks
 
   $ rm -rf hg-repo6
   $ cd git-repo6
   $ git reset --hard 'HEAD^' > /dev/null
 
 test missing .gitmodules
 
   $ git submodule add ../git-repo4 >/dev/null 2>/dev/null
   $ git checkout HEAD .gitmodules
   $ git rm .gitmodules
   rm '.gitmodules'
   $ git commit -q -m "remove .gitmodules" .gitmodules
   $ git commit -q -m "missing .gitmodules"
   $ cd ..
   $ hg convert git-repo6 hg-repo6 --traceback 2>&1 | grep -v "fatal: Path '.gitmodules' does not exist"
   initializing destination hg-repo6 repository
   scanning source...
   sorting...
   converting...
   2 addsubmodule
   1 remove .gitmodules
   0 missing .gitmodules
   warning: cannot read submodules config file in * (glob)
   updating bookmarks
   $ rm -rf hg-repo6
   $ cd git-repo6
   $ rm -rf git-repo4
   $ git reset --hard 'HEAD^^' > /dev/null
   $ cd ..
 
 test invalid splicemap1
 
   $ cat > splicemap <<EOF
   > $VALIDID1
   > EOF
   $ hg convert --splicemap splicemap git-repo2 git-repo2-splicemap1-hg
   initializing destination git-repo2-splicemap1-hg repository
   abort: syntax error in splicemap(1): child parent1[,parent2] expected
   [255]
 
 test invalid splicemap2
 
   $ cat > splicemap <<EOF
   > $VALIDID1 $VALIDID2, $VALIDID2, $VALIDID2
   > EOF
   $ hg convert --splicemap splicemap git-repo2 git-repo2-splicemap2-hg
   initializing destination git-repo2-splicemap2-hg repository
   abort: syntax error in splicemap(1): child parent1[,parent2] expected
   [255]
 
 test invalid splicemap3
 
   $ cat > splicemap <<EOF
   > $INVALIDID1 $INVALIDID2
   > EOF
   $ hg convert --splicemap splicemap git-repo2 git-repo2-splicemap3-hg
   initializing destination git-repo2-splicemap3-hg repository
   abort: splicemap entry afd12345af is not a valid revision identifier
   [255]
 
 convert sub modules
   $ hg convert git-repo6 git-repo6-hg
   initializing destination git-repo6-hg repository
   scanning source...
   sorting...
   converting...
   0 addsubmodule
   updating bookmarks
   $ hg -R git-repo6-hg log -v
   changeset:   0:* (glob)
   bookmark:    master
   tag:         tip
   user:        nottest <[e-mail]>
   date:        Mon Jan 01 00:00:23 2007 +0000
   files:       .hgsub .hgsubstate
   description:
   addsubmodule
   
   committer: test <[e-mail]>
   
   
 
   $ cd git-repo6-hg
   $ hg up >/dev/null 2>/dev/null
   $ cat .hgsubstate
   * git-repo5 (glob)
   $ cd git-repo5
   $ cat foo
   sub
 
   $ cd ../..
 
 make sure rename detection doesn't break removing and adding gitmodules
 
   $ cd git-repo6
   $ git mv .gitmodules .gitmodules-renamed
   $ commit -a -m 'rename .gitmodules'
   $ git mv .gitmodules-renamed .gitmodules
   $ commit -a -m 'rename .gitmodules back'
   $ cd ..
 
   $ hg --config convert.git.similarity=100 convert -q git-repo6 git-repo6-hg
   $ hg -R git-repo6-hg log -r 'tip^' -T "{desc|firstline}\n"
   rename .gitmodules
   $ hg -R git-repo6-hg status -C --change 'tip^'
   A .gitmodules-renamed
   R .hgsub
   R .hgsubstate
   $ hg -R git-repo6-hg log -r tip -T "{desc|firstline}\n"
   rename .gitmodules back
   $ hg -R git-repo6-hg status -C --change tip
   A .hgsub
   A .hgsubstate
   R .gitmodules-renamed
 
 convert the revision removing '.gitmodules' itself (and related
 submodules)
 
   $ cd git-repo6
   $ git rm .gitmodules
   rm '.gitmodules'
   $ git rm --cached git-repo5
   rm 'git-repo5'
   $ commit -a -m 'remove .gitmodules and submodule git-repo5'
   $ cd ..
 
   $ hg convert -q git-repo6 git-repo6-hg
   $ hg -R git-repo6-hg tip -T "{desc|firstline}\n"
   remove .gitmodules and submodule git-repo5
   $ hg -R git-repo6-hg tip -T "{file_dels}\n"
   .hgsub .hgsubstate
 
 skip submodules in the conversion
 
   $ hg convert -q git-repo6 no-submodules --config convert.git.skipsubmodules=True
   $ hg -R no-submodules manifest --all
   .gitmodules-renamed
 
 convert using a different remote prefix
   $ git init git-repo7
   Initialized empty Git repository in $TESTTMP/git-repo7/.git/
   $ cd git-repo7
 TODO: it'd be nice to use (?) lines instead of grep -v to handle the
 git output variance, but that doesn't currently work in the middle of
 a block, so do this for now.
   $ touch a && git add a && git commit -am "commit a" | grep -v changed
   [master (root-commit) 8ae5f69] commit a
    Author: nottest <[e-mail]>
    create mode 100644 a
   $ cd ..
   $ git clone git-repo7 git-repo7-client
   Cloning into 'git-repo7-client'...
   done.
   $ hg convert --config convert.git.remoteprefix=origin git-repo7-client hg-repo7
   initializing destination hg-repo7 repository
   scanning source...
   sorting...
   converting...
   0 commit a
   updating bookmarks
   $ hg -R hg-repo7 bookmarks
      master                    0:03bf38caa4c6
      origin/master             0:03bf38caa4c6
 
 Run convert when the remote branches have changed
 (there was an old bug where the local convert read branches from the server)
 
   $ cd git-repo7
   $ echo a >> a
   $ git commit -q -am "move master forward"
   $ cd ..
   $ rm -rf hg-repo7
   $ hg convert --config convert.git.remoteprefix=origin git-repo7-client hg-repo7
   initializing destination hg-repo7 repository
   scanning source...
   sorting...
   converting...
   0 commit a
   updating bookmarks
   $ hg -R hg-repo7 bookmarks
      master                    0:03bf38caa4c6
      origin/master             0:03bf38caa4c6
 
 damaged git repository tests:
 In case the hard-coded hashes change, the following commands can be used to
 list the hashes and their corresponding types in the repository:
 cd git-repo4/.git/objects
 find . -type f | cut -c 3- | sed 's_/__' | xargs -n 1 -t git cat-file -t
 cd ../../..
 
 damage git repository by renaming a commit object
   $ COMMIT_OBJ=1c/0ce3c5886f83a1d78a7b517cdff5cf9ca17bdd
   $ mv git-repo4/.git/objects/$COMMIT_OBJ git-repo4/.git/objects/$COMMIT_OBJ.tmp
   $ hg convert git-repo4 git-repo4-broken-hg 2>&1 | grep 'abort:'
-  abort: cannot read tags from git-repo4/.git
+  abort: cannot retrieve number of commits in git-repo4/.git
   $ mv git-repo4/.git/objects/$COMMIT_OBJ.tmp git-repo4/.git/objects/$COMMIT_OBJ
 damage git repository by renaming a blob object
 
   $ BLOB_OBJ=8b/137891791fe96927ad78e64b0aad7bded08bdc
   $ mv git-repo4/.git/objects/$BLOB_OBJ git-repo4/.git/objects/$BLOB_OBJ.tmp
   $ hg convert git-repo4 git-repo4-broken-hg 2>&1 | grep 'abort:'
   abort: cannot read 'blob' object at 8b137891791fe96927ad78e64b0aad7bded08bdc
   $ mv git-repo4/.git/objects/$BLOB_OBJ.tmp git-repo4/.git/objects/$BLOB_OBJ
 damage git repository by renaming a tree object
 
   $ TREE_OBJ=72/49f083d2a63a41cc737764a86981eb5f3e4635
   $ mv git-repo4/.git/objects/$TREE_OBJ git-repo4/.git/objects/$TREE_OBJ.tmp
   $ hg convert git-repo4 git-repo4-broken-hg 2>&1 | grep 'abort:'
   abort: cannot read changes in 1c0ce3c5886f83a1d78a7b517cdff5cf9ca17bdd