convert/bzr: handle files replaced by directories (issue1623)

tests/test-convert-bzr-114

@@ -0,0 +1,30 b''
+#!/bin/sh
+
+"$TESTDIR/hghave" bzr114 || exit 80
+
+. "$TESTDIR/bzr-definitions"
+
+# The file/directory replacement can only be reproduced on
+# bzr >= 1.4. Merge it back in test-convert-bzr-directories once
+# this version becomes mainstream.
+echo % replace file with dir
+mkdir test-replace-file-with-dir
+cd test-replace-file-with-dir
+bzr init -q source
+cd source
+echo d > d
+bzr add -q d
+bzr commit -q -m 'add d file'
+rm d
+mkdir d
+bzr add -q d
+bzr commit -q -m 'replace with d dir'
+echo a > d/a
+bzr add -q d/a
+bzr commit -q -m 'add d/a'
+cd ..
+hg convert source source-hg
+manifest source-hg tip
+cd source-hg
+hg update
+cd ../..

tests/test-convert-bzr-114.out

@@ -0,0 +1,11 b''
+% replace file with dir
+initializing destination source-hg repository
+scanning source...
+sorting...
+converting...
+2 add d file
+1 replace with d dir
+0 add d/a
+% manifest of tip
+644   d/a
+1 files updated, 0 files merged, 0 files removed, 0 files unresolved

hgext/convert/bzr.py

@@ -1,219 +1,224 b''
 # bzr support for the convert extension
 # This module is for handling 'bzr', that was formerly known as Bazaar-NG;
 # it cannot access 'bar' repositories, but they were never used very much
 
 import os
 from mercurial import demandimport
 # these do not work with demandimport, blacklist
 demandimport.ignore.extend([
         'bzrlib.transactions',
         'bzrlib.urlutils',
     ])
 
 from mercurial.i18n import _
 from mercurial import util
 from common import NoRepo, commit, converter_source
 
 try:
     # bazaar imports
     from bzrlib import branch, revision, errors
     from bzrlib.revisionspec import RevisionSpec
 except ImportError:
     pass
 
 supportedkinds = ('file', 'symlink')
 
 class bzr_source(converter_source):
     """Reads Bazaar repositories by using the Bazaar Python libraries"""
 
     def __init__(self, ui, path, rev=None):
         super(bzr_source, self).__init__(ui, path, rev=rev)
 
         try:
             # access bzrlib stuff
             branch
         except NameError:
             raise NoRepo('Bazaar modules could not be loaded')
 
         if not os.path.exists(os.path.join(path, '.bzr')):
             raise NoRepo('%s does not look like a Bazaar repo' % path)
 
         path = os.path.abspath(path)
         self.branch = branch.Branch.open(path)
         self.sourcerepo = self.branch.repository
         self._parentids = {}
 
     def before(self):
         """Before the conversion begins, acquire a read lock
         for all the operations that might need it. Fortunately
         read locks don't block other reads or writes to the
         repository, so this shouldn't have any impact on the usage of
         the source repository.
 
         The alternative would be locking on every operation that
         needs locks (there are currently two: getting the file and
         getting the parent map) and releasing immediately after,
         but this approach can take even 40% longer."""
         self.sourcerepo.lock_read()
 
     def after(self):
         self.sourcerepo.unlock()
 
     def getheads(self):
         if not self.rev:
             return [self.branch.last_revision()]
         try:
             r = RevisionSpec.from_string(self.rev)
             info = r.in_history(self.branch)
         except errors.BzrError:
             raise util.Abort(_('%s is not a valid revision in current branch')
                              % self.rev)
         return [info.rev_id]
 
     def getfile(self, name, rev):
         revtree = self.sourcerepo.revision_tree(rev)
         fileid = revtree.path2id(name)
         if fileid is None or revtree.kind(fileid) not in supportedkinds:
             # the file is not available anymore - was deleted
             raise IOError(_('%s is not available in %s anymore') %
                     (name, rev))
         sio = revtree.get_file(fileid)
         return sio.read()
 
     def getmode(self, name, rev):
         return self._modecache[(name, rev)]
 
     def getchanges(self, version):
         # set up caches: modecache and revtree
         self._modecache = {}
         self._revtree = self.sourcerepo.revision_tree(version)
         # get the parentids from the cache
         parentids = self._parentids.pop(version)
         # only diff against first parent id
         prevtree = self.sourcerepo.revision_tree(parentids[0])
         return self._gettreechanges(self._revtree, prevtree)
 
     def getcommit(self, version):
         rev = self.sourcerepo.get_revision(version)
         # populate parent id cache
         if not rev.parent_ids:
             parents = []
             self._parentids[version] = (revision.NULL_REVISION,)
         else:
             parents = self._filterghosts(rev.parent_ids)
             self._parentids[version] = parents
 
         return commit(parents=parents,
                 # bzr uses 1 second timezone precision
                 date='%d %d' % (rev.timestamp, rev.timezone / 3600),
                 author=self.recode(rev.committer),
                 # bzr returns bytestrings or unicode, depending on the content
                 desc=self.recode(rev.message),
                 rev=version)
 
     def gettags(self):
         if not self.branch.supports_tags():
             return {}
         tagdict = self.branch.tags.get_tag_dict()
         bytetags = {}
         for name, rev in tagdict.iteritems():
             bytetags[self.recode(name)] = rev
         return bytetags
 
     def getchangedfiles(self, rev, i):
         self._modecache = {}
         curtree = self.sourcerepo.revision_tree(rev)
         parentids = self._parentids.pop(rev)
         if i is not None:
             parentid = parentids[i]
         else:
             # no parent id, get the empty revision
             parentid = revision.NULL_REVISION
 
         prevtree = self.sourcerepo.revision_tree(parentid)
         changes = [e[0] for e in self._gettreechanges(curtree, prevtree)[0]]
         return changes
 
     def _gettreechanges(self, current, origin):
         revid = current._revision_id;
         changes = []
         renames = {}
         for (fileid, paths, changed_content, versioned, parent, name,
             kind, executable) in current.iter_changes(origin):
 
             if paths[0] == u'' or paths[1] == u'':
                 # ignore changes to tree root
                 continue
 
             # bazaar tracks directories, mercurial does not, so
             # we have to rename the directory contents
             if kind[1] == 'directory':
+                if kind[0] not in (None, 'directory'):
+                    # Replacing 'something' with a directory, record it
+                    # so it can be removed.
+                    changes.append((self.recode(paths[0]), revid))
+
                 if None not in paths and paths[0] != paths[1]:
                     # neither an add nor an delete - a move
                     # rename all directory contents manually
                     subdir = origin.inventory.path2id(paths[0])
                     # get all child-entries of the directory
                     for name, entry in origin.inventory.iter_entries(subdir):
                         # hg does not track directory renames
                         if entry.kind == 'directory':
                             continue
                         frompath = self.recode(paths[0] + '/' + name)
                         topath = self.recode(paths[1] + '/' + name)
                         # register the files as changed
                         changes.append((frompath, revid))
                         changes.append((topath, revid))
                         # add to mode cache
                         mode = ((entry.executable and 'x') or (entry.kind == 'symlink' and 's')
                                 or '')
                         self._modecache[(topath, revid)] = mode
                         # register the change as move
                         renames[topath] = frompath
 
                 # no futher changes, go to the next change
                 continue
 
             # we got unicode paths, need to convert them
             path, topath = [self.recode(part) for part in paths]
 
             if topath is None:
                 # file deleted
                 changes.append((path, revid))
                 continue
 
             # renamed
             if path and path != topath:
                 renames[topath] = path
                 changes.append((path, revid))
 
             # populate the mode cache
             kind, executable = [e[1] for e in (kind, executable)]
             mode = ((executable and 'x') or (kind == 'symlink' and 's')
                     or '')
             self._modecache[(topath, revid)] = mode
             changes.append((topath, revid))
 
         return changes, renames
 
     def _filterghosts(self, ids):
         """Filters out ghost revisions which hg does not support, see
         <http://bazaar-vcs.org/GhostRevision>
         """
         parentmap = self.sourcerepo.get_parent_map(ids)
         parents = tuple([parent for parent in ids if parent in parentmap])
         return parents
 
     def recode(self, s, encoding=None):
         """This version of recode tries to encode unicode to bytecode,
         and preferably using the UTF-8 codec.
         Other types than Unicode are silently returned, this is by
         intention, e.g. the None-type is not going to be encoded but instead
         just passed through
         """
         if not encoding:
             encoding = self.encoding or 'utf-8'
 
         if isinstance(s, unicode):
             return s.encode(encoding)
         else:
             # leave it alone
             return s

tests/hghave

@@ -1,233 +1,242 b''
 #!/usr/bin/env python
 """Test the running system for features availability. Exit with zero
 if all features are there, non-zero otherwise. If a feature name is
 prefixed with "no-", the absence of feature is tested.
 """
 import optparse
 import os
 import re
 import sys
 import tempfile
 
 tempprefix = 'hg-hghave-'
 
 def matchoutput(cmd, regexp, ignorestatus=False):
     """Return True if cmd executes successfully and its output
     is matched by the supplied regular expression.
     """
     r = re.compile(regexp)
     fh = os.popen(cmd)
     s = fh.read()
     ret = fh.close()
     return (ignorestatus or ret is None) and r.search(s)
 
 def has_baz():
     return matchoutput('baz --version 2>&1', r'baz Bazaar version')
 
 def has_bzr():
     try:
         import bzrlib
         return bzrlib.__doc__ != None
     except ImportError:
         return False
 
+def has_bzr114():
+    try:
+        import bzrlib
+        return (bzrlib.__doc__ != None
+                and bzrlib.version_info[:2] == (1, 14))
+    except ImportError:
+        return False
+
 def has_cvs():
     re = r'Concurrent Versions System.*?server'
     return matchoutput('cvs --version 2>&1', re)
 
 def has_cvsps():
     return matchoutput('cvsps -h -q 2>&1', r'cvsps version', True)
 
 def has_darcs():
     return matchoutput('darcs', r'darcs version', True)
 
 def has_mtn():
     return matchoutput('mtn --version', r'monotone', True) and not matchoutput(
         'mtn --version', r'monotone 0\.(\d|[12]\d|3[01])[^\d]', True)
 
 def has_eol_in_paths():
     try:
         fd, path = tempfile.mkstemp(prefix=tempprefix, suffix='\n\r')
         os.close(fd)
         os.remove(path)
         return True
     except:
         return False
 
 def has_executablebit():
     fd, path = tempfile.mkstemp(prefix=tempprefix)
     os.close(fd)
     try:
         s = os.lstat(path).st_mode
         os.chmod(path, s | 0100)
         return (os.lstat(path).st_mode & 0100 != 0)
     finally:
         os.remove(path)
 
 def has_icasefs():
     # Stolen from mercurial.util
     fd, path = tempfile.mkstemp(prefix=tempprefix)
     os.close(fd)
     try:
         s1 = os.stat(path)
         d, b = os.path.split(path)
         p2 = os.path.join(d, b.upper())
         if path == p2:
             p2 = os.path.join(d, b.lower())
         try:
             s2 = os.stat(p2)
             return s2 == s1
         except:
             return False
     finally:
         os.remove(path)
 
 def has_inotify():
     try:
         import hgext.inotify.linux.watcher
         return True
     except ImportError:
         return False
 
 def has_fifo():
     return hasattr(os, "mkfifo")
 
 def has_hotshot():
     try:
         # hotshot.stats tests hotshot and many problematic dependencies
         # like profile.
         import hotshot.stats
         return True
     except ImportError:
         return False
 
 def has_lsprof():
     try:
         import _lsprof
         return True
     except ImportError:
         return False
 
 def has_git():
     return matchoutput('git --version 2>&1', r'^git version')
 
 def has_svn():
     return matchoutput('svn --version 2>&1', r'^svn, version') and \
         matchoutput('svnadmin --version 2>&1', r'^svnadmin, version')
 
 def has_svn_bindings():
     try:
         import svn.core
         version = svn.core.SVN_VER_MAJOR, svn.core.SVN_VER_MINOR
         if version < (1, 4):
             return False
         return True
     except ImportError:
         return False
 
 def has_p4():
     return matchoutput('p4 -V', r'Rev\. P4/') and matchoutput('p4d -V', r'Rev\. P4D/')
 
 def has_symlink():
     return hasattr(os, "symlink")
 
 def has_tla():
     return matchoutput('tla --version 2>&1', r'The GNU Arch Revision')
 
 def has_unix_permissions():
     d = tempfile.mkdtemp(prefix=tempprefix, dir=".")
     try:
         fname = os.path.join(d, 'foo')
         for umask in (077, 007, 022):
             os.umask(umask)
             f = open(fname, 'w')
             f.close()
             mode = os.stat(fname).st_mode
             os.unlink(fname)
             if mode & 0777 != ~umask & 0666:
                 return False
         return True
     finally:
         os.rmdir(d)
 
 def has_pygments():
     try:
         import pygments
         return True
     except ImportError:
         return False
 
 def has_outer_repo():
     return matchoutput('hg root 2>&1', r'')
 
 checks = {
     "baz": (has_baz, "GNU Arch baz client"),
     "bzr": (has_bzr, "Canonical's Bazaar client"),
+    "bzr114": (has_bzr114, "Canonical's Bazaar client >= 1.14"),
     "cvs": (has_cvs, "cvs client/server"),
     "cvsps": (has_cvsps, "cvsps utility"),
     "darcs": (has_darcs, "darcs client"),
     "eol-in-paths": (has_eol_in_paths, "end-of-lines in paths"),
     "execbit": (has_executablebit, "executable bit"),
     "fifo": (has_fifo, "named pipes"),
     "git": (has_git, "git command line client"),
     "hotshot": (has_hotshot, "python hotshot module"),
     "icasefs": (has_icasefs, "case insensitive file system"),
     "inotify": (has_inotify, "inotify extension support"),
     "lsprof": (has_lsprof, "python lsprof module"),
     "mtn": (has_mtn, "monotone client (> 0.31)"),
     "outer-repo": (has_outer_repo, "outer repo"),
     "p4": (has_p4, "Perforce server and client"),
     "pygments": (has_pygments, "Pygments source highlighting library"),
     "svn": (has_svn, "subversion client and admin tools"),
     "svn-bindings": (has_svn_bindings, "subversion python bindings"),
     "symlink": (has_symlink, "symbolic links"),
     "tla": (has_tla, "GNU Arch tla client"),
     "unix-permissions": (has_unix_permissions, "unix-style permissions"),
 }
 
 def list_features():
     for name, feature in checks.iteritems():
         desc = feature[1]
         print name + ':', desc
 
 parser = optparse.OptionParser("%prog [options] [features]")
 parser.add_option("--list-features", action="store_true",
                   help="list available features")
 parser.add_option("-q", "--quiet", action="store_true",
                   help="check features silently")
 
 if __name__ == '__main__':
     options, args = parser.parse_args()
     if options.list_features:
         list_features()
         sys.exit(0)
 
     quiet = options.quiet
 
     failures = 0
 
     def error(msg):
         global failures
         if not quiet:
             sys.stderr.write(msg + '\n')
         failures += 1
 
     for feature in args:
         negate = feature.startswith('no-')
         if negate:
             feature = feature[3:]
 
         if feature not in checks:
             error('skipped: unknown feature: ' + feature)
             continue
 
         check, desc = checks[feature]
         if not negate and not check():
             error('skipped: missing feature: ' + desc)
         elif negate and check():
             error('skipped: system supports %s' % desc)
 
     if failures != 0:
         sys.exit(1)