convert: read git output in binary mode under Windows (issue 1359)

hgext/convert/git.py

@@ -1,142 +1,142 b''
 # git support for the convert extension
 
 import os
 from mercurial import util
 
 from common import NoRepo, commit, converter_source, checktool
 
 class convert_git(converter_source):
     # Windows does not support GIT_DIR= construct while other systems
     # cannot remove environment variable. Just assume none have
     # both issues.
     if hasattr(os, 'unsetenv'):
         def gitcmd(self, s):
             prevgitdir = os.environ.get('GIT_DIR')
             os.environ['GIT_DIR'] = self.path
             try:
-                return util.popen(s)
+                return util.popen(s, 'rb')
             finally:
                 if prevgitdir is None:
                     del os.environ['GIT_DIR']
                 else:
                     os.environ['GIT_DIR'] = prevgitdir
     else:
         def gitcmd(self, s):
-            return util.popen('GIT_DIR=%s %s' % (self.path, s))
+            return util.popen('GIT_DIR=%s %s' % (self.path, s), 'rb')
 
     def __init__(self, ui, path, rev=None):
         super(convert_git, self).__init__(ui, path, rev=rev)
 
         if os.path.isdir(path + "/.git"):
             path += "/.git"
         if not os.path.exists(path + "/objects"):
             raise NoRepo("%s does not look like a Git repo" % path)
 
         checktool('git', 'git')
 
         self.path = path
 
     def getheads(self):
         if not self.rev:
             return self.gitcmd('git rev-parse --branches').read().splitlines()
         else:
             fh = self.gitcmd("git rev-parse --verify %s" % self.rev)
             return [fh.read()[:-1]]
 
     def catfile(self, rev, type):
         if rev == "0" * 40: raise IOError()
         fh = self.gitcmd("git cat-file %s %s" % (type, rev))
         return fh.read()
 
     def getfile(self, name, rev):
         return self.catfile(rev, "blob")
 
     def getmode(self, name, rev):
         return self.modecache[(name, rev)]
 
     def getchanges(self, version):
         self.modecache = {}
         fh = self.gitcmd("git diff-tree --root -m -r %s" % version)
         changes = []
         seen = {}
         for l in fh:
             if "\t" not in l:
                 continue
             m, f = l[:-1].split("\t")
             if f in seen:
                 continue
             seen[f] = 1
             m = m.split()
             h = m[3]
             p = (m[1] == "100755")
             s = (m[1] == "120000")
             self.modecache[(f, h)] = (p and "x") or (s and "l") or ""
             changes.append((f, h))
         return (changes, {})
 
     def getcommit(self, version):
         c = self.catfile(version, "commit") # read the commit hash
         end = c.find("\n\n")
         message = c[end+2:]
         message = self.recode(message)
         l = c[:end].splitlines()
         manifest = l[0].split()[1]
         parents = []
         for e in l[1:]:
             n, v = e.split(" ", 1)
             if n == "author":
                 p = v.split()
                 tm, tz = p[-2:]
                 author = " ".join(p[:-2])
                 if author[0] == "<": author = author[1:-1]
                 author = self.recode(author)
             if n == "committer":
                 p = v.split()
                 tm, tz = p[-2:]
                 committer = " ".join(p[:-2])
                 if committer[0] == "<": committer = committer[1:-1]
                 committer = self.recode(committer)
                 message += "\ncommitter: %s\n" % committer
             if n == "parent": parents.append(v)
 
         tzs, tzh, tzm = tz[-5:-4] + "1", tz[-4:-2], tz[-2:]
         tz = -int(tzs) * (int(tzh) * 3600 + int(tzm))
         date = tm + " " + str(tz)
 
         c = commit(parents=parents, date=date, author=author, desc=message,
                    rev=version)
         return c
 
     def gettags(self):
         tags = {}
         fh = self.gitcmd('git ls-remote --tags "%s"' % self.path)
         prefix = 'refs/tags/'
         for line in fh:
             line = line.strip()
             if not line.endswith("^{}"):
                 continue
             node, tag = line.split(None, 1)
             if not tag.startswith(prefix):
                 continue
             tag = tag[len(prefix):-3]
             tags[tag] = node
 
         return tags
 
     def getchangedfiles(self, version, i):
         changes = []
         if i is None:
             fh = self.gitcmd("git diff-tree --root -m -r %s" % version)
             for l in fh:
                 if "\t" not in l:
                     continue
                 m, f = l[:-1].split("\t")
                 changes.append(f)
             fh.close()
         else:
             fh = self.gitcmd('git diff-tree --name-only --root -r %s "%s^%s" --'
                              % (version, version, i+1))
             changes = [f.rstrip('\n') for f in fh]
             fh.close()
 
         return changes

tests/test-convert-git

@@ -1,132 +1,149 b''
 #!/bin/sh
 
 "$TESTDIR/hghave" git || exit 80
 
 echo "[extensions]" >> $HGRCPATH
 echo "convert=" >> $HGRCPATH
 echo 'hgext.graphlog =' >> $HGRCPATH
 
 GIT_AUTHOR_NAME='test'; export GIT_AUTHOR_NAME
 GIT_AUTHOR_EMAIL='[e-mail]'; export GIT_AUTHOR_EMAIL
 GIT_AUTHOR_DATE="2007-01-01 00:00:00 +0000"; export GIT_AUTHOR_DATE
 GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"; export GIT_COMMITTER_NAME
 GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"; export GIT_COMMITTER_EMAIL
 GIT_COMMITTER_DATE="$GIT_AUTHOR_DATE"; export GIT_COMMITTER_DATE
 
 count=10
 commit()
 {
     GIT_AUTHOR_DATE="2007-01-01 00:00:$count +0000"
     GIT_COMMITTER_DATE="$GIT_AUTHOR_DATE"
     git commit "$@" >/dev/null 2>/dev/null || echo "git commit error"
     count=`expr $count + 1`
 }
 
 mkdir git-repo
 cd git-repo
 git init-db >/dev/null 2>/dev/null
 echo a > a
 mkdir d
 echo b > d/b
 git add a d
 commit -a -m t1
 
 # Remove the directory, then try to replace it with a file
 # (issue 754)
 git rm -f d/b
 commit -m t2
 echo d > d
 git add d
 commit -m t3
 
 echo b >> a
 commit -a -m t4.1
 
 git checkout -b other HEAD^ >/dev/null 2>/dev/null
 echo c > a
 echo a >> a
 commit -a -m t4.2
 
 git checkout master >/dev/null 2>/dev/null
 git pull --no-commit . other > /dev/null 2>/dev/null
 commit -m 'Merge branch other'
 cd ..
 
 hg convert --datesort git-repo
 
 hg -R git-repo-hg tip -v
 
 count=10
 mkdir git-repo2
 cd git-repo2
 git init-db >/dev/null 2>/dev/null
 
 echo foo > foo
 git add foo
 commit -a -m 'add foo'
 
 echo >> foo
 commit -a -m 'change foo'
 
 git checkout -b Bar HEAD^ >/dev/null 2>/dev/null
 echo quux >> quux
 git add quux
 commit -a -m 'add quux'
 
 echo bar > bar
 git add bar
 commit -a -m 'add bar'
 
 git checkout -b Baz HEAD^ >/dev/null 2>/dev/null
 echo baz > baz
 git add baz
 commit -a -m 'add baz'
 
 git checkout master >/dev/null 2>/dev/null
 git pull --no-commit . Bar Baz > /dev/null 2>/dev/null
 commit -m 'Octopus merge'
 
 echo bar >> bar
 commit -a -m 'change bar'
 
 git checkout -b Foo HEAD^ >/dev/null 2>/dev/null
 echo >> foo
 commit -a -m 'change foo'
 
 git checkout master >/dev/null 2>/dev/null
 git pull --no-commit -s ours . Foo > /dev/null 2>/dev/null
 commit -m 'Discard change to foo'
 
 cd ..
 
 glog()
 {
     hg glog --template '#rev# "#desc|firstline#" files: #files#\n' "$@"
 }
 
 splitrepo()
 {
     msg="$1"
     files="$2"
     opts=$3
     echo "% $files: $msg"
     prefix=`echo "$files" | sed -e 's/ /-/g'`
     fmap="$prefix.fmap"
     repo="$prefix.repo"
     for i in $files; do
 	echo "include $i" >> "$fmap"
     done
     hg -q convert $opts --filemap "$fmap" --datesort git-repo2 "$repo"
     glog -R "$repo"
     hg -R "$repo" manifest --debug
 }
 
 echo '% full conversion'
 hg -q convert --datesort git-repo2 fullrepo
 glog -R fullrepo
 hg -R fullrepo manifest --debug
 
 splitrepo 'octopus merge' 'foo bar baz'
 
 splitrepo 'only some parents of an octopus merge; "discard" a head' 'foo baz quux'
 
+echo
+echo '% test binary conversion (issue 1359)'
+mkdir git-repo3
+cd git-repo3
+git init-db >/dev/null 2>/dev/null
+python -c 'file("b", "wb").write("".join([chr(i) for i in range(256)])*16)'
+git add b
+commit -a -m addbinary
+cd ..
+
+echo '% convert binary file'
+hg convert git-repo3 git-repo3-hg
+
+cd git-repo3-hg
+hg up -C
+python -c 'print len(file("b", "rb").read())'
+

tests/test-convert-git.out

@@ -1,90 +1,100 b''
 rm 'd/b'
 assuming destination git-repo-hg
 initializing destination git-repo-hg repository
 scanning source...
 sorting...
 converting...
 5 t1
 4 t2
 3 t3
 2 t4.1
 1 t4.2
 0 Merge branch other
 changeset:   5:4ab1af49a271
 tag:         tip
 parent:      3:0222ab0998d7
 parent:      4:5333c870e3c2
 user:        test <[e-mail]>
 date:        Mon Jan 01 00:00:15 2007 +0000
 files:       a
 description:
 Merge branch other
 
 committer: test <[e-mail]>
 
 
 % full conversion
 o    9 "Discard change to foo" files: foo
 |\
 | o  8 "change foo" files: foo
 | |
 o |  7 "change bar" files: bar
 |/
 o    6 "(octopus merge fixup)" files:
 |\
 | o    5 "Octopus merge" files: baz
 | |\
 o | |  4 "add baz" files: baz
 | | |
 +---o  3 "add bar" files: bar
 | |
 o |  2 "add quux" files: quux
 | |
 | o  1 "change foo" files: foo
 |/
 o  0 "add foo" files: foo
 
 245a3b8bc653999c2b22cdabd517ccb47aecafdf 644   bar
 354ae8da6e890359ef49ade27b68bbc361f3ca88 644   baz
 9277c9cc8dd4576fc01a17939b4351e5ada93466 644   foo
 88dfeab657e8cf2cef3dec67b914f49791ae76b1 644   quux
 % foo bar baz: octopus merge
 o    8 "Discard change to foo" files: foo
 |\
 | o  7 "change foo" files: foo
 | |
 o |  6 "change bar" files: bar
 |/
 o    5 "(octopus merge fixup)" files:
 |\
 | o    4 "Octopus merge" files: baz
 | |\
 o | |  3 "add baz" files: baz
 | | |
 +---o  2 "add bar" files: bar
 | |
 | o  1 "change foo" files: foo
 |/
 o  0 "add foo" files: foo
 
 245a3b8bc653999c2b22cdabd517ccb47aecafdf 644   bar
 354ae8da6e890359ef49ade27b68bbc361f3ca88 644   baz
 9277c9cc8dd4576fc01a17939b4351e5ada93466 644   foo
 % foo baz quux: only some parents of an octopus merge; "discard" a head
 o  6 "Discard change to foo" files: foo
 |
 o  5 "change foo" files: foo
 |
 o    4 "Octopus merge" files:
 |\
 | o  3 "add baz" files: baz
 | |
 | o  2 "add quux" files: quux
 | |
 o |  1 "change foo" files: foo
 |/
 o  0 "add foo" files: foo
 
 354ae8da6e890359ef49ade27b68bbc361f3ca88 644   baz
 9277c9cc8dd4576fc01a17939b4351e5ada93466 644   foo
 88dfeab657e8cf2cef3dec67b914f49791ae76b1 644   quux
+
+% test binary conversion (issue 1359)
+% convert binary file
+initializing destination git-repo3-hg repository
+scanning source...
+sorting...
+converting...
+0 addbinary
+1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+4096