convert: make mapfile handle LF and CRLF shamap (issue1846)

hgext/convert/common.py

@@ -1,391 +1,391 b''
 # common.py - common code for the convert extension
 #
 #  Copyright 2005-2009 Matt Mackall <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2, incorporated herein by reference.
 
 import base64, errno
 import os
 import cPickle as pickle
 from mercurial import util
 from mercurial.i18n import _
 
 def encodeargs(args):
     def encodearg(s):
         lines = base64.encodestring(s)
         lines = [l.splitlines()[0] for l in lines]
         return ''.join(lines)
 
     s = pickle.dumps(args)
     return encodearg(s)
 
 def decodeargs(s):
     s = base64.decodestring(s)
     return pickle.loads(s)
 
 class MissingTool(Exception): pass
 
 def checktool(exe, name=None, abort=True):
     name = name or exe
     if not util.find_exe(exe):
         exc = abort and util.Abort or MissingTool
         raise exc(_('cannot find required "%s" tool') % name)
 
 class NoRepo(Exception): pass
 
 SKIPREV = 'SKIP'
 
 class commit(object):
     def __init__(self, author, date, desc, parents, branch=None, rev=None,
                  extra={}, sortkey=None):
         self.author = author or 'unknown'
         self.date = date or '0 0'
         self.desc = desc
         self.parents = parents
         self.branch = branch
         self.rev = rev
         self.extra = extra
         self.sortkey = sortkey
 
 class converter_source(object):
     """Conversion source interface"""
 
     def __init__(self, ui, path=None, rev=None):
         """Initialize conversion source (or raise NoRepo("message")
         exception if path is not a valid repository)"""
         self.ui = ui
         self.path = path
         self.rev = rev
 
         self.encoding = 'utf-8'
 
     def before(self):
         pass
 
     def after(self):
         pass
 
     def setrevmap(self, revmap):
         """set the map of already-converted revisions"""
         pass
 
     def getheads(self):
         """Return a list of this repository's heads"""
         raise NotImplementedError()
 
     def getfile(self, name, rev):
         """Return file contents as a string. rev is the identifier returned
         by a previous call to getchanges(). Raise IOError to indicate that
         name was deleted in rev.
         """
         raise NotImplementedError()
 
     def getmode(self, name, rev):
         """Return file mode, eg. '', 'x', or 'l'. rev is the identifier
         returned by a previous call to getchanges().
         """
         raise NotImplementedError()
 
     def getchanges(self, version):
         """Returns a tuple of (files, copies).
 
         files is a sorted list of (filename, id) tuples for all files
         changed between version and its first parent returned by
         getcommit(). id is the source revision id of the file.
 
         copies is a dictionary of dest: source
         """
         raise NotImplementedError()
 
     def getcommit(self, version):
         """Return the commit object for version"""
         raise NotImplementedError()
 
     def gettags(self):
         """Return the tags as a dictionary of name: revision
 
         Tag names must be UTF-8 strings.
         """
         raise NotImplementedError()
 
     def recode(self, s, encoding=None):
         if not encoding:
             encoding = self.encoding or 'utf-8'
 
         if isinstance(s, unicode):
             return s.encode("utf-8")
         try:
             return s.decode(encoding).encode("utf-8")
         except:
             try:
                 return s.decode("latin-1").encode("utf-8")
             except:
                 return s.decode(encoding, "replace").encode("utf-8")
 
     def getchangedfiles(self, rev, i):
         """Return the files changed by rev compared to parent[i].
 
         i is an index selecting one of the parents of rev.  The return
         value should be the list of files that are different in rev and
         this parent.
 
         If rev has no parents, i is None.
 
         This function is only needed to support --filemap
         """
         raise NotImplementedError()
 
     def converted(self, rev, sinkrev):
         '''Notify the source that a revision has been converted.'''
         pass
 
     def hasnativeorder(self):
         """Return true if this source has a meaningful, native revision
         order. For instance, Mercurial revisions are store sequentially
         while there is no such global ordering with Darcs.
         """
         return False
 
     def lookuprev(self, rev):
         """If rev is a meaningful revision reference in source, return
         the referenced identifier in the same format used by getcommit().
         return None otherwise.
         """
         return None
 
 class converter_sink(object):
     """Conversion sink (target) interface"""
 
     def __init__(self, ui, path):
         """Initialize conversion sink (or raise NoRepo("message")
         exception if path is not a valid repository)
 
         created is a list of paths to remove if a fatal error occurs
         later"""
         self.ui = ui
         self.path = path
         self.created = []
 
     def getheads(self):
         """Return a list of this repository's heads"""
         raise NotImplementedError()
 
     def revmapfile(self):
         """Path to a file that will contain lines
         source_rev_id sink_rev_id
         mapping equivalent revision identifiers for each system."""
         raise NotImplementedError()
 
     def authorfile(self):
         """Path to a file that will contain lines
         srcauthor=dstauthor
         mapping equivalent authors identifiers for each system."""
         return None
 
     def putcommit(self, files, copies, parents, commit, source, revmap):
         """Create a revision with all changed files listed in 'files'
         and having listed parents. 'commit' is a commit object
         containing at a minimum the author, date, and message for this
         changeset.  'files' is a list of (path, version) tuples,
         'copies' is a dictionary mapping destinations to sources,
         'source' is the source repository, and 'revmap' is a mapfile
         of source revisions to converted revisions. Only getfile(),
         getmode(), and lookuprev() should be called on 'source'.
 
         Note that the sink repository is not told to update itself to
         a particular revision (or even what that revision would be)
         before it receives the file data.
         """
         raise NotImplementedError()
 
     def puttags(self, tags):
         """Put tags into sink.
 
         tags: {tagname: sink_rev_id, ...} where tagname is an UTF-8 string.
         Return a pair (tag_revision, tag_parent_revision), or (None, None)
         if nothing was changed.
         """
         raise NotImplementedError()
 
     def setbranch(self, branch, pbranches):
         """Set the current branch name. Called before the first putcommit
         on the branch.
         branch: branch name for subsequent commits
         pbranches: (converted parent revision, parent branch) tuples"""
         pass
 
     def setfilemapmode(self, active):
         """Tell the destination that we're using a filemap
 
         Some converter_sources (svn in particular) can claim that a file
         was changed in a revision, even if there was no change.  This method
         tells the destination that we're using a filemap and that it should
         filter empty revisions.
         """
         pass
 
     def before(self):
         pass
 
     def after(self):
         pass
 
 
 class commandline(object):
     def __init__(self, ui, command):
         self.ui = ui
         self.command = command
 
     def prerun(self):
         pass
 
     def postrun(self):
         pass
 
     def _cmdline(self, cmd, *args, **kwargs):
         cmdline = [self.command, cmd] + list(args)
         for k, v in kwargs.iteritems():
             if len(k) == 1:
                 cmdline.append('-' + k)
             else:
                 cmdline.append('--' + k.replace('_', '-'))
             try:
                 if len(k) == 1:
                     cmdline.append('' + v)
                 else:
                     cmdline[-1] += '=' + v
             except TypeError:
                 pass
         cmdline = [util.shellquote(arg) for arg in cmdline]
         if not self.ui.debugflag:
             cmdline += ['2>', util.nulldev]
         cmdline += ['<', util.nulldev]
         cmdline = ' '.join(cmdline)
         return cmdline
 
     def _run(self, cmd, *args, **kwargs):
         cmdline = self._cmdline(cmd, *args, **kwargs)
         self.ui.debug(_('running: %s\n') % (cmdline,))
         self.prerun()
         try:
             return util.popen(cmdline)
         finally:
             self.postrun()
 
     def run(self, cmd, *args, **kwargs):
         fp = self._run(cmd, *args, **kwargs)
         output = fp.read()
         self.ui.debug(output)
         return output, fp.close()
 
     def runlines(self, cmd, *args, **kwargs):
         fp = self._run(cmd, *args, **kwargs)
         output = fp.readlines()
         self.ui.debug(''.join(output))
         return output, fp.close()
 
     def checkexit(self, status, output=''):
         if status:
             if output:
                 self.ui.warn(_('%s error:\n') % self.command)
                 self.ui.warn(output)
             msg = util.explain_exit(status)[0]
             raise util.Abort('%s %s' % (self.command, msg))
 
     def run0(self, cmd, *args, **kwargs):
         output, status = self.run(cmd, *args, **kwargs)
         self.checkexit(status, output)
         return output
 
     def runlines0(self, cmd, *args, **kwargs):
         output, status = self.runlines(cmd, *args, **kwargs)
         self.checkexit(status, ''.join(output))
         return output
 
     def getargmax(self):
         if '_argmax' in self.__dict__:
             return self._argmax
 
         # POSIX requires at least 4096 bytes for ARG_MAX
         self._argmax = 4096
         try:
             self._argmax = os.sysconf("SC_ARG_MAX")
         except:
             pass
 
         # Windows shells impose their own limits on command line length,
         # down to 2047 bytes for cmd.exe under Windows NT/2k and 2500 bytes
         # for older 4nt.exe. See http://support.microsoft.com/kb/830473 for
         # details about cmd.exe limitations.
 
         # Since ARG_MAX is for command line _and_ environment, lower our limit
         # (and make happy Windows shells while doing this).
 
         self._argmax = self._argmax/2 - 1
         return self._argmax
 
     def limit_arglist(self, arglist, cmd, *args, **kwargs):
         limit = self.getargmax() - len(self._cmdline(cmd, *args, **kwargs))
         bytes = 0
         fl = []
         for fn in arglist:
             b = len(fn) + 3
             if bytes + b < limit or len(fl) == 0:
                 fl.append(fn)
                 bytes += b
             else:
                 yield fl
                 fl = [fn]
                 bytes = b
         if fl:
             yield fl
 
     def xargs(self, arglist, cmd, *args, **kwargs):
         for l in self.limit_arglist(arglist, cmd, *args, **kwargs):
             self.run0(cmd, *(list(args) + l), **kwargs)
 
 class mapfile(dict):
     def __init__(self, ui, path):
         super(mapfile, self).__init__()
         self.ui = ui
         self.path = path
         self.fp = None
         self.order = []
         self._read()
 
     def _read(self):
         if not self.path:
             return
         try:
             fp = open(self.path, 'r')
         except IOError, err:
             if err.errno != errno.ENOENT:
                 raise
             return
         for i, line in enumerate(fp):
             try:
-                key, value = line[:-1].rsplit(' ', 1)
+                key, value = line.splitlines()[0].rsplit(' ', 1)
             except ValueError:
                 raise util.Abort(_('syntax error in %s(%d): key/value pair expected')
                                  % (self.path, i+1))
             if key not in self:
                 self.order.append(key)
             super(mapfile, self).__setitem__(key, value)
         fp.close()
 
     def __setitem__(self, key, value):
         if self.fp is None:
             try:
                 self.fp = open(self.path, 'a')
             except IOError, err:
                 raise util.Abort(_('could not open map file %r: %s') %
                                  (self.path, err.strerror))
         self.fp.write('%s %s\n' % (key, value))
         self.fp.flush()
         super(mapfile, self).__setitem__(key, value)
 
     def close(self):
         if self.fp:
             self.fp.close()
             self.fp = None

tests/test-convert-hg-source

@@ -1,69 +1,88 b''
 #!/bin/sh
 
 cat >> $HGRCPATH <<EOF
 [extensions]
 convert=
 [convert]
 hg.saverev=False
 EOF
 
 hg init orig
 cd orig
 
 echo foo > foo
 echo bar > bar
 hg ci -qAm 'add foo bar' -d '0 0'
 
 echo >> foo
 hg ci -m 'change foo' -d '1 0'
 
 hg up -qC 0
 hg copy --after --force foo bar
 hg copy foo baz
 hg ci -m 'make bar and baz copies of foo' -d '2 0'
 
 hg merge
 hg ci -m 'merge local copy' -d '3 0'
 
 hg up -C 1
 hg merge 2
 hg ci -m 'merge remote copy' -d '4 0'
 
 chmod +x baz
 hg ci -m 'mark baz executable' -d '5 0'
 
 cd ..
 hg convert --datesort orig new 2>&1 | grep -v 'subversion python bindings could not be loaded'
 cd new
 hg out ../orig
 cd ..
 
+echo '% check shamap LF and CRLF handling'
+cat > rewrite.py <<EOF
+import sys
+# Interlace LF and CRLF
+lines = [(l.rstrip() + ((i % 2) and '\n' or '\r\n'))
+         for i, l in enumerate(file(sys.argv[1]))]
+file(sys.argv[1], 'wb').write(''.join(lines))
+EOF
+python rewrite.py new/.hg/shamap
+cd orig
+hg up -qC 1
+echo foo >> foo
+hg ci -qm 'change foo again'
+hg up -qC 2
+echo foo >> foo
+hg ci -qm 'change foo again again'
+cd ..
+hg convert --datesort orig new 2>&1 | grep -v 'subversion python bindings could not be loaded'
+
 echo % init broken repository
 hg init broken
 cd broken
 echo a >> a
 echo b >> b
 hg ci -qAm init
 echo a >> a
 echo b >> b
 hg copy b c
 hg ci -qAm changeall
 hg up -qC 0
 echo bc >> b
 hg ci -m changebagain
 HGMERGE=internal:local hg -q merge
 hg ci -m merge
 hg mv b d
 hg ci -m moveb
 echo % break it
 rm .hg/store/data/b.*
 cd ..
 
 hg --config convert.hg.ignoreerrors=True convert broken fixed
 hg -R fixed verify
 echo '% manifest -r 0'
 hg -R fixed manifest -r 0
 echo '% manifest -r tip'
 hg -R fixed manifest -r tip
 
 true

tests/test-convert-hg-source.out

@@ -1,46 +1,52 b''
 created new head
 merging baz and foo to baz
 1 files updated, 1 files merged, 0 files removed, 0 files unresolved
 (branch merge, don't forget to commit)
 1 files updated, 0 files merged, 1 files removed, 0 files unresolved
 merging foo and baz to baz
 1 files updated, 1 files merged, 0 files removed, 0 files unresolved
 (branch merge, don't forget to commit)
 created new head
 initializing destination new repository
 scanning source...
 sorting...
 converting...
 5 add foo bar
 4 change foo
 3 make bar and baz copies of foo
 2 merge local copy
 1 merge remote copy
 0 mark baz executable
 comparing with ../orig
 searching for changes
 no changes found
+% check shamap LF and CRLF handling
+scanning source...
+sorting...
+converting...
+1 change foo again again
+0 change foo again
 % init broken repository
 created new head
 % break it
 initializing destination fixed repository
 scanning source...
 sorting...
 converting...
 4 init
 ignoring: data/b.i@1e88685f5dde: no match found
 3 changeall
 2 changebagain
 1 merge
 0 moveb
 checking changesets
 checking manifests
 crosschecking files in changesets and manifests
 checking files
 3 files, 5 changesets, 5 total revisions
 % manifest -r 0
 a
 % manifest -r tip
 a
 c
 d