initial version of monotone source for convert extension

hgext/convert/monotone.py

@@ -0,0 +1,214
+# monotone support for the convert extension
+
+import os
+import re
+import time
+from mercurial import util
+
+from common import NoRepo, commit, converter_source, checktool
+
+class monotone_source(converter_source):
+    def __init__(self, ui, path=None, rev=None):
+        converter_source.__init__(self, ui, path, rev)
+        
+        self.ui = ui
+        self.path = path
+
+         
+        # regular expressions for parsing monotone output
+        
+        space    = r'\s*'
+        name     = r'\s+"((?:[^"]|\\")*)"\s*'
+        value    = name
+        revision = r'\s+\[(\w+)\]\s*'
+        lines    = r'(?:.|\n)+'
+        
+        self.dir_re      = re.compile(space + "dir"      + name)
+        self.file_re     = re.compile(space + "file"     + name + "content" + revision)
+        self.add_file_re = re.compile(space + "add_file" + name + "content" + revision)
+        self.patch_re    = re.compile(space + "patch"    + name + "from" + revision + "to" + revision)
+        self.rename_re   = re.compile(space + "rename"   + name + "to" + name)
+        self.tag_re      = re.compile(space + "tag"      + name + "revision" + revision)
+        self.cert_re     = re.compile(lines + space + "name" + name + "value" + value)
+
+        attr = space + "file" + lines + space + "attr" + space
+        self.attr_execute_re = re.compile(attr  + '"mtn:execute"' + space + '"true"')
+
+        # cached data
+        
+        self.manifest_rev = None
+        self.manifest = None
+        self.files = None   
+        self.dirs  = None     
+        
+        norepo = NoRepo("%s does not look like a monotone repo" % path)
+        if not os.path.exists(path):
+            raise norepo
+        
+        checktool('mtn')
+        
+        # test if there are are any revisions
+        self.rev = None
+        try :
+            self.getheads()
+        except :
+            raise norepo        
+
+        self.rev = rev
+
+    
+    def mtncmd(self, arg):
+        cmdline = "mtn -d %s automate %s" % (util.shellquote(self.path), arg)
+        self.ui.debug(cmdline, '\n')
+        p = util.popen(cmdline)
+        result = p.read()
+        if p.close():
+            raise IOError()
+        return result
+    
+    def mtnloadmanifest(self, rev):
+        if self.manifest_rev == rev:
+            return
+        self.manifest_rev = rev
+        self.manifest = self.mtncmd("get_manifest_of %s" % rev).split("\n\n")
+        
+        manifest = self.manifest
+        files = {}
+        dirs = {}
+
+        for e in manifest:
+            m = self.file_re.match(e)
+            if m:                
+                attr = ""
+                name = m.group(1)
+                node = m.group(2)
+                if self.attr_execute_re.match(e):
+                    attr += "x"
+                files[name] = (node, attr)
+            m = self.dir_re.match(e)
+            if m:
+                dirs[m.group(1)] = True
+        
+        self.files = files
+        self.dirs = dirs
+
+    def mtnisfile(self, name, rev):
+        # a non-file could be a directory or a deleted or renamed file
+        self.mtnloadmanifest(rev)
+        try :
+            self.files[name]
+            return True
+        except KeyError:
+            return False
+            
+    def mtnisdir(self, name, rev):
+        self.mtnloadmanifest(rev)
+        try :
+            self.dirs[name]
+            return True
+        except KeyError:
+            return False
+    
+    def mtngetcerts(self, rev):
+        certs = {"author":"<missing>", "date":"<missing>",
+            "changelog":"<missing>", "branch":"<missing>"}
+        cert_list = self.mtncmd("certs %s" % rev).split("\n\n")
+        for e in cert_list:
+            m = self.cert_re.match(e)
+            if m:
+                certs[m.group(1)] = m.group(2)
+        return certs
+        
+    def mtngetparents(self, rev):
+        parents = self.mtncmd("parents %s" % rev).strip("\n").split("\n")
+        p = []
+        for x in parents:
+            if len(x) >= 40: # blank revs have been seen otherwise
+                p.append(x)
+        return p
+
+    def mtnrenamefiles(self, files, fromdir, todir):
+        renamed = {}
+        for tofile in files:
+            suffix = tofile.lstrip(todir)
+            if todir + suffix == tofile:
+                renamed[tofile] = (fromdir + suffix).lstrip("/")
+        return renamed
+
+    
+    # implement the converter_source interface:
+    
+    def getheads(self):
+        if not self.rev or self.rev == "":
+            return self.mtncmd("leaves").splitlines()
+        else:
+            return [self.rev]
+
+    def getchanges(self, rev):
+        revision = self.mtncmd("get_revision %s" % rev).split("\n\n")
+        files = {}
+        copies = {}
+        for e in revision:
+            m = self.add_file_re.match(e)
+            if m:
+                files[m.group(1)] = rev
+            m = self.patch_re.match(e)
+            if m:
+                files[m.group(1)] = rev
+
+            # Delete/rename is handled later when the convert engine
+            # discovers an IOError exception from getfile,
+            # but only if we add the "from" file to the list of changes.
+            m = self.rename_re.match(e)
+            if m:
+                toname = m.group(2)
+                fromname = m.group(1)
+                if self.mtnisfile(toname, rev):
+                    copies[toname] = fromname
+                    files[toname] = rev
+                    files[fromname] = rev
+                if self.mtnisdir(toname, rev):
+                    renamed = self.mtnrenamefiles(self.files, fromname, toname)
+                    for tofile, fromfile in renamed.items():
+                        self.ui.debug (("copying file in renamed dir from '%s' to '%s'" % (fromfile, tofile)), "\n")
+                        files[tofile] = rev
+                    for fromfile in renamed.values():
+                        files[fromfile] = rev
+
+        return (files.items(), copies)
+        
+    def getmode(self, name, rev):
+        self.mtnloadmanifest(rev)
+        try :
+            node, attr = self.files[name]
+            return attr
+        except KeyError:
+            return ""
+        
+    def getfile(self, name, rev):
+        if not self.mtnisfile(name, rev):
+            raise IOError() # file was deleted or renamed
+        return self.mtncmd("get_file_of %s -r %s" % (util.shellquote(name), rev))
+    
+    def getcommit(self, rev):        
+        certs   = self.mtngetcerts(rev)
+        return commit(
+            author=certs["author"],
+            date=util.datestr(util.strdate(certs["date"], "%Y-%m-%dT%H:%M:%S")),
+            desc=certs["changelog"],
+            rev=rev,
+            parents=self.mtngetparents(rev),
+            branch=certs["branch"])
+
+    def gettags(self):
+        tags = {}
+        for e in self.mtncmd("tags").split("\n\n"):
+            m = self.tag_re.match(e)
+            if m:
+                tags[m.group(1)] = m.group(2)
+        return tags
+
+    def getchangedfiles(self, rev, i):
+        # This function is only needed to support --filemap
+        # ... and we don't support that
+        raise NotImplementedError()

hgext/convert/__init__.py

@@ -1,148 +1,149
 # convert.py Foreign SCM converter
 #
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms
 # of the GNU General Public License, incorporated herein by reference.
 
 import convcmd
 from mercurial import commands
 
 # Commands definition was moved elsewhere to ease demandload job.
 
 def convert(ui, src, dest=None, revmapfile=None, **opts):
     """Convert a foreign SCM repository to a Mercurial one.
 
     Accepted source formats:
     - Mercurial
     - CVS
     - Darcs
     - git
     - Subversion
+    - Monotone
     - GNU Arch
 
     Accepted destination formats:
     - Mercurial
     - Subversion (history on branches is not preserved)
 
     If no revision is given, all revisions will be converted. Otherwise,
     convert will only import up to the named revision (given in a format
     understood by the source).
 
     If no destination directory name is specified, it defaults to the
     basename of the source with '-hg' appended.  If the destination
     repository doesn't exist, it will be created.
 
     If <REVMAP> isn't given, it will be put in a default location
     (<dest>/.hg/shamap by default).  The <REVMAP> is a simple text
     file that maps each source commit ID to the destination ID for
     that revision, like so:
     <source ID> <destination ID>
 
     If the file doesn't exist, it's automatically created.  It's updated
     on each commit copied, so convert-repo can be interrupted and can
     be run repeatedly to copy new commits.
 
     The [username mapping] file is a simple text file that maps each source
     commit author to a destination commit author. It is handy for source SCMs
     that use unix logins to identify authors (eg: CVS). One line per author
     mapping and the line format is:
     srcauthor=whatever string you want
 
     The filemap is a file that allows filtering and remapping of files
     and directories.  Comment lines start with '#'.  Each line can
     contain one of the following directives:
 
       include path/to/file
 
       exclude path/to/file
 
       rename from/file to/file
 
     The 'include' directive causes a file, or all files under a
     directory, to be included in the destination repository, and the
     exclusion of all other files and dirs not explicitely included.
     The 'exclude' directive causes files or directories to be omitted.
     The 'rename' directive renames a file or directory.  To rename from a
     subdirectory into the root of the repository, use '.' as the path to
     rename to.
 
     The splicemap is a file that allows insertion of synthetic
     history, letting you specify the parents of a revision.  This is
     useful if you want to e.g. give a Subversion merge two parents, or
     graft two disconnected series of history together.  Each entry
     contains a key, followed by a space, followed by one or two
     values, separated by spaces.  The key is the revision ID in the
     source revision control system whose parents should be modified
     (same format as a key in .hg/shamap).  The values are the revision
     IDs (in either the source or destination revision control system)
     that should be used as the new parents for that node.
 
     Mercurial Source
     -----------------
 
     --config convert.hg.saverev=True          (boolean)
         allow target to preserve source revision ID
 
     Subversion Source
     -----------------
 
     Subversion source detects classical trunk/branches/tags layouts.
     By default, the supplied "svn://repo/path/" source URL is
     converted as a single branch. If "svn://repo/path/trunk" exists
     it replaces the default branch. If "svn://repo/path/branches"
     exists, its subdirectories are listed as possible branches. If
     "svn://repo/path/tags" exists, it is looked for tags referencing
     converted branches. Default "trunk", "branches" and "tags" values
     can be overriden with following options. Set them to paths
     relative to the source URL, or leave them blank to disable
     autodetection.
 
     --config convert.svn.branches=branches    (directory name)
         specify the directory containing branches
     --config convert.svn.tags=tags            (directory name)
         specify the directory containing tags
     --config convert.svn.trunk=trunk          (directory name)
         specify the name of the trunk branch
 
     Source history can be retrieved starting at a specific revision,
     instead of being integrally converted. Only single branch
     conversions are supported.
 
     --config convert.svn.startrev=0           (svn revision number)
         specify start Subversion revision.
 
     Mercurial Destination
     ---------------------
 
     --config convert.hg.clonebranches=False   (boolean)
         dispatch source branches in separate clones.
     --config convert.hg.tagsbranch=default    (branch name)
         tag revisions branch name
     --config convert.hg.usebranchnames=True   (boolean)
         preserve branch names
 
     """
     return convcmd.convert(ui, src, dest, revmapfile, **opts)
 
 def debugsvnlog(ui, **opts):
     return convcmd.debugsvnlog(ui, **opts)
 
 commands.norepo += " convert debugsvnlog"
 
 cmdtable = {
     "convert":
         (convert,
          [('A', 'authors', '', 'username mapping filename'),
           ('d', 'dest-type', '', 'destination repository type'),
           ('', 'filemap', '', 'remap file names using contents of file'),
           ('r', 'rev', '', 'import up to target revision REV'),
           ('s', 'source-type', '', 'source repository type'),
           ('', 'splicemap', '', 'splice synthesized history into place'),
           ('', 'datesort', None, 'try to sort changesets by date')],
          'hg convert [OPTION]... SOURCE [DEST [REVMAP]]'),
     "debugsvnlog":
         (debugsvnlog,
          [],
          'hg debugsvnlog'),
 }

hgext/convert/convcmd.py

@@ -1,352 +1,354
 # convcmd - convert extension commands definition
 #
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms
 # of the GNU General Public License, incorporated herein by reference.
 
 from common import NoRepo, SKIPREV, mapfile
 from cvs import convert_cvs
 from darcs import darcs_source
 from git import convert_git
 from hg import mercurial_source, mercurial_sink
 from subversion import debugsvnlog, svn_source, svn_sink
+from monotone import monotone_source
 from gnuarch import gnuarch_source
 import filemap
 
 import os, shutil
 from mercurial import hg, util
 from mercurial.i18n import _
 
 orig_encoding = 'ascii'
 
 def recode(s):
     if isinstance(s, unicode):
         return s.encode(orig_encoding, 'replace')
     else:
         return s.decode('utf-8').encode(orig_encoding, 'replace')
 
 source_converters = [
     ('cvs', convert_cvs),
     ('git', convert_git),
     ('svn', svn_source),
     ('hg', mercurial_source),
     ('darcs', darcs_source),
+    ('mtn', monotone_source),
     ('gnuarch', gnuarch_source),
     ]
 
 sink_converters = [
     ('hg', mercurial_sink),
     ('svn', svn_sink),
     ]
 
 def convertsource(ui, path, type, rev):
     exceptions = []
     for name, source in source_converters:
         try:
             if not type or name == type:
                 return source(ui, path, rev)
         except NoRepo, inst:
             exceptions.append(inst)
     if not ui.quiet:
         for inst in exceptions:
             ui.write(_("%s\n") % inst)
     raise util.Abort('%s: unknown repository type' % path)
 
 def convertsink(ui, path, type):
     for name, sink in sink_converters:
         try:
             if not type or name == type:
                 return sink(ui, path)
         except NoRepo, inst:
             ui.note(_("convert: %s\n") % inst)
     raise util.Abort('%s: unknown repository type' % path)
 
 class converter(object):
     def __init__(self, ui, source, dest, revmapfile, opts):
 
         self.source = source
         self.dest = dest
         self.ui = ui
         self.opts = opts
         self.commitcache = {}
         self.authors = {}
         self.authorfile = None
 
         self.map = mapfile(ui, revmapfile)
 
         # Read first the dst author map if any
         authorfile = self.dest.authorfile()
         if authorfile and os.path.exists(authorfile):
             self.readauthormap(authorfile)
         # Extend/Override with new author map if necessary
         if opts.get('authors'):
             self.readauthormap(opts.get('authors'))
             self.authorfile = self.dest.authorfile()
 
         self.splicemap = mapfile(ui, opts.get('splicemap'))
 
     def walktree(self, heads):
         '''Return a mapping that identifies the uncommitted parents of every
         uncommitted changeset.'''
         visit = heads
         known = {}
         parents = {}
         while visit:
             n = visit.pop(0)
             if n in known or n in self.map: continue
             known[n] = 1
             commit = self.cachecommit(n)
             parents[n] = []
             for p in commit.parents:
                 parents[n].append(p)
                 visit.append(p)
 
         return parents
 
     def toposort(self, parents):
         '''Return an ordering such that every uncommitted changeset is
         preceeded by all its uncommitted ancestors.'''
         visit = parents.keys()
         seen = {}
         children = {}
         actives = []
 
         while visit:
             n = visit.pop(0)
             if n in seen: continue
             seen[n] = 1
             # Ensure that nodes without parents are present in the 'children'
             # mapping.
             children.setdefault(n, [])
             hasparent = False
             for p in parents[n]:
                 if not p in self.map:
                     visit.append(p)
                     hasparent = True
                 children.setdefault(p, []).append(n)
             if not hasparent:
                 actives.append(n)
 
         del seen
         del visit
 
         if self.opts.get('datesort'):
             dates = {}
             def getdate(n):
                 if n not in dates:
                     dates[n] = util.parsedate(self.commitcache[n].date)
                 return dates[n]
 
             def picknext(nodes):
                 return min([(getdate(n), n) for n in nodes])[1]
         else:
             prev = [None]
             def picknext(nodes):
                 # Return the first eligible child of the previously converted
                 # revision, or any of them.
                 next = nodes[0]
                 for n in nodes:
                     if prev[0] in parents[n]:
                         next = n
                         break
                 prev[0] = next
                 return next
 
         s = []
         pendings = {}
         while actives:
             n = picknext(actives)
             actives.remove(n)
             s.append(n)
 
             # Update dependents list
             for c in children.get(n, []):
                 if c not in pendings:
                     pendings[c] = [p for p in parents[c] if p not in self.map]
                 try:
                     pendings[c].remove(n)
                 except ValueError:
                     raise util.Abort(_('cycle detected between %s and %s')
                                        % (recode(c), recode(n)))
                 if not pendings[c]:
                     # Parents are converted, node is eligible
                     actives.insert(0, c)
                     pendings[c] = None
 
         if len(s) != len(parents):
             raise util.Abort(_("not all revisions were sorted"))
 
         return s
 
     def writeauthormap(self):
         authorfile = self.authorfile
         if authorfile:
            self.ui.status('Writing author map file %s\n' % authorfile)
            ofile = open(authorfile, 'w+')
            for author in self.authors:
                ofile.write("%s=%s\n" % (author, self.authors[author]))
            ofile.close()
 
     def readauthormap(self, authorfile):
         afile = open(authorfile, 'r')
         for line in afile:
             if line.strip() == '':
                 continue
             try:
                 srcauthor, dstauthor = line.split('=', 1)
                 srcauthor = srcauthor.strip()
                 dstauthor = dstauthor.strip()
                 if srcauthor in self.authors and dstauthor != self.authors[srcauthor]:
                     self.ui.status(
                         'Overriding mapping for author %s, was %s, will be %s\n'
                         % (srcauthor, self.authors[srcauthor], dstauthor))
                 else:
                     self.ui.debug('Mapping author %s to %s\n'
                                   % (srcauthor, dstauthor))
                     self.authors[srcauthor] = dstauthor
             except IndexError:
                 self.ui.warn(
                     'Ignoring bad line in author map file %s: %s\n'
                     % (authorfile, line.rstrip()))
         afile.close()
 
     def cachecommit(self, rev):
         commit = self.source.getcommit(rev)
         commit.author = self.authors.get(commit.author, commit.author)
         self.commitcache[rev] = commit
         return commit
 
     def copy(self, rev):
         commit = self.commitcache[rev]
         do_copies = hasattr(self.dest, 'copyfile')
         filenames = []
 
         changes = self.source.getchanges(rev)
         if isinstance(changes, basestring):
             if changes == SKIPREV:
                 dest = SKIPREV
             else:
                 dest = self.map[changes]
             self.map[rev] = dest
             return
         files, copies = changes
         pbranches = []
         if commit.parents:
             for prev in commit.parents:
                 if prev not in self.commitcache:
                     self.cachecommit(prev)
                 pbranches.append((self.map[prev],
                                   self.commitcache[prev].branch))
         self.dest.setbranch(commit.branch, pbranches)
         for f, v in files:
             filenames.append(f)
             try:
                 data = self.source.getfile(f, v)
             except IOError, inst:
                 self.dest.delfile(f)
             else:
                 e = self.source.getmode(f, v)
                 self.dest.putfile(f, e, data)
                 if do_copies:
                     if f in copies:
                         copyf = copies[f]
                         # Merely marks that a copy happened.
                         self.dest.copyfile(copyf, f)
 
         try:
             parents = self.splicemap[rev].replace(',', ' ').split()
             self.ui.status('spliced in %s as parents of %s\n' %
                            (parents, rev))
             parents = [self.map.get(p, p) for p in parents]
         except KeyError:
             parents = [b[0] for b in pbranches]
         newnode = self.dest.putcommit(filenames, parents, commit)
         self.source.converted(rev, newnode)
         self.map[rev] = newnode
 
     def convert(self):
 
         try:
             self.source.before()
             self.dest.before()
             self.source.setrevmap(self.map)
             self.ui.status("scanning source...\n")
             heads = self.source.getheads()
             parents = self.walktree(heads)
             self.ui.status("sorting...\n")
             t = self.toposort(parents)
             num = len(t)
             c = None
 
             self.ui.status("converting...\n")
             for c in t:
                 num -= 1
                 desc = self.commitcache[c].desc
                 if "\n" in desc:
                     desc = desc.splitlines()[0]
                 # convert log message to local encoding without using
                 # tolocal() because util._encoding conver() use it as
                 # 'utf-8'
                 self.ui.status("%d %s\n" % (num, recode(desc)))
                 self.ui.note(_("source: %s\n" % recode(c)))
                 self.copy(c)
 
             tags = self.source.gettags()
             ctags = {}
             for k in tags:
                 v = tags[k]
                 if self.map.get(v, SKIPREV) != SKIPREV:
                     ctags[k] = self.map[v]
 
             if c and ctags:
                 nrev = self.dest.puttags(ctags)
                 # write another hash correspondence to override the previous
                 # one so we don't end up with extra tag heads
                 if nrev:
                     self.map[c] = nrev
 
             self.writeauthormap()
         finally:
             self.cleanup()
 
     def cleanup(self):
         try:
             self.dest.after()
         finally:
             self.source.after()
         self.map.close()
 
 def convert(ui, src, dest=None, revmapfile=None, **opts):
     global orig_encoding
     orig_encoding = util._encoding
     util._encoding = 'UTF-8'
 
     if not dest:
         dest = hg.defaultdest(src) + "-hg"
         ui.status("assuming destination %s\n" % dest)
 
     destc = convertsink(ui, dest, opts.get('dest_type'))
 
     try:
         srcc = convertsource(ui, src, opts.get('source_type'),
                              opts.get('rev'))
     except Exception:
         for path in destc.created:
             shutil.rmtree(path, True)
         raise
 
     fmap = opts.get('filemap')
     if fmap:
         srcc = filemap.filemap_source(ui, srcc, fmap)
         destc.setfilemapmode(True)
 
     if not revmapfile:
         try:
             revmapfile = destc.revmapfile()
         except:
             revmapfile = os.path.join(destc, "map")
 
     c = converter(ui, srcc, destc, revmapfile, opts)
     c.convert()