document conversion interface

contrib/convert-repo

@@ -1,650 +1,731 b''
 #!/usr/bin/env python
 #
 # This is a generalized framework for converting between SCM
 # repository formats.
 #
 # To use, run:
 #
 # convert-repo <source> [<dest> [<mapfile>]]
 #
 # Currently accepted source formats: git, cvs
 # Currently accepted destination formats: hg
 #
 # If destination isn't given, a new Mercurial repo named <src>-hg will
 # be created. If <mapfile> isn't given, it will be put in a default
 # location (<dest>/.hg/shamap by default)
 #
 # The <mapfile> is a simple text file that maps each source commit ID to
 # the destination ID for that revision, like so:
 #
 # <source ID> <destination ID>
 #
 # If the file doesn't exist, it's automatically created.  It's updated
 # on each commit copied, so convert-repo can be interrupted and can
 # be run repeatedly to copy new commits.
 
 import sys, os, zlib, sha, time, re, locale, socket
 os.environ["HGENCODING"] = "utf-8"
 from mercurial import hg, ui, util, fancyopts
 
 class Abort(Exception): pass
 class NoRepo(Exception): pass
 
 class commit:
     def __init__(self, **parts):
         for x in "author date desc parents".split():
             if not x in parts:
                 abort("commit missing field %s\n" % x)
         self.__dict__.update(parts)
 
 quiet = 0
 def status(msg):
     if not quiet: sys.stdout.write(str(msg))
 
 def warn(msg):
     sys.stderr.write(str(msg))
 
 def abort(msg):
     raise Abort(msg)
 
 def recode(s):
     try:
         return s.decode("utf-8").encode("utf-8")
     except:
         try:
             return s.decode("latin-1").encode("utf-8")
         except:
             return s.decode("utf-8", "replace").encode("utf-8")
 
+class converter_source:
+    """Conversion source interface"""
+
+    def __init__(self, path):
+        """Initialize conversion source (or raise NoRepo("message")
+        exception if path is not a valid repository)"""
+        raise NotImplementedError()
+
+    def getheads(self):
+        """Return a list of this repository's heads"""
+        raise NotImplementedError()
+
+    def getfile(self, name, rev):
+        """Return file contents as a string"""
+        raise NotImplementedError()
+
+    def getmode(self, name, rev):
+        """Return file mode, eg. '', 'x', or 'l'"""
+        raise NotImplementedError()
+
+    def getchanges(self, version):
+        """Return sorted list of (filename, id) tuples for all files changed in rev.
+        
+        id just tells us which revision to return in getfile(), e.g. in
+        git it's an object hash."""
+        raise NotImplementedError()
+
+    def getcommit(self, version):
+        """Return the commit object for version"""
+        raise NotImplementedError()
+
+    def gettags(self):
+        """Return the tags as a dictionary of name: revision"""
+        raise NotImplementedError()
+
+class converter_sink:
+    """Conversion sink (target) interface"""
+
+    def __init__(self, path):
+        """Initialize conversion sink (or raise NoRepo("message")
+        exception if path is not a valid repository)"""
+        raise NotImplementedError()
+
+    def getheads(self):
+        """Return a list of this repository's heads"""
+        raise NotImplementedError()
+
+    def mapfile(self):
+        """Path to a file that will contain lines
+        source_rev_id sink_rev_id
+        mapping equivalent revision identifiers for each system."""
+        raise NotImplementedError()
+
+    def putfile(self, f, e, data):
+        """Put file for next putcommit().
+        f: path to file
+        e: '', 'x', or 'l' (regular file, executable, or symlink)
+        data: file contents"""
+        raise NotImplementedError()
+
+    def delfile(self, f):
+        """Delete file for next putcommit().
+        f: path to file"""
+        raise NotImplementedError()
+
+    def putcommit(self, files, parents, commit):
+        """Create a revision with all changed files listed in 'files'
+        and having listed parents. 'commit' is a commit object containing
+        at a minimum the author, date, and message for this changeset.
+        Called after putfile() and delfile() calls. Note that the sink
+        repository is not told to update itself to a particular revision
+        (or even what that revision would be) before it receives the
+        file data."""
+        raise NotImplementedError()
+
+    def puttags(self, tags):
+        """Put tags into sink.
+        tags: {tagname: sink_rev_id, ...}"""
+        raise NotImplementedError()
+
+
 # CVS conversion code inspired by hg-cvs-import and git-cvsimport
 class convert_cvs:
     def __init__(self, path):
         self.path = path
         cvs = os.path.join(path, "CVS")
         if not os.path.exists(cvs):
             raise NoRepo("couldn't open CVS repo %s" % path)
 
         self.changeset = {}
         self.files = {}
         self.tags = {}
         self.lastbranch = {}
         self.parent = {}
         self.socket = None
         self.cvsroot = file(os.path.join(cvs, "Root")).read()[:-1]
         self.cvsrepo = file(os.path.join(cvs, "Repository")).read()[:-1]
         self.encoding = locale.getpreferredencoding()
         self._parse()
         self._connect()
 
     def _parse(self):
         if self.changeset:
             return
 
         d = os.getcwd()
         try:
             os.chdir(self.path)
             id = None
             state = 0
             for l in os.popen("cvsps -A -u --cvs-direct -q"):
                 if state == 0: # header
                     if l.startswith("PatchSet"):
                         id = l[9:-2]
                     elif l.startswith("Date"):
                         date = util.parsedate(l[6:-1], ["%Y/%m/%d %H:%M:%S"])
                         date = util.datestr(date)
                     elif l.startswith("Branch"):
                         branch = l[8:-1]
                         self.parent[id] = self.lastbranch.get(branch,'bad')
                         self.lastbranch[branch] = id
                     elif l.startswith("Ancestor branch"):
                         ancestor = l[17:-1]
                         self.parent[id] = self.lastbranch[ancestor]
                     elif l.startswith("Author"):
                         author = self.recode(l[8:-1])
                     elif l.startswith("Tag: "):
                         t = l[5:-1].rstrip()
                         if t != "(none)":
                             self.tags[t] = id
                     elif l.startswith("Log:"):
                         state = 1
                         log = ""
                 elif state == 1: # log
                     if l == "Members: \n":
                         files = {}
                         log = self.recode(log[:-1])
                         if log.isspace():
                             log = "*** empty log message ***\n"
                         state = 2
                     else:
                         log += l
                 elif state == 2:
                     if l == "\n": #
                         state = 0
                         p = [self.parent[id]]
                         if id == "1":
                             p = []
                         c = commit(author=author, date=date, parents=p,
                                    desc=log, branch=branch)
                         self.changeset[id] = c
                         self.files[id] = files
                     else:
                         file,rev = l[1:-2].rsplit(':',1)
                         rev = rev.split("->")[1]
                         files[file] = rev
 
             self.heads = self.lastbranch.values()
         finally:
             os.chdir(d)
 
     def _connect(self):
         root = self.cvsroot
         conntype = None
         user, host = None, None
         cmd = ['cvs', 'server']
 
         status("connecting to %s\n" % root)
 
         if root.startswith(":pserver:"):
             root = root[9:]
             m = re.match(r'(?:(.*?)(?::(.*?))?@)?([^:\/]*)(?::(\d*))?(.*)', root)
             if m:
                 conntype = "pserver"
                 user, passw, serv, port, root = m.groups()
                 if not user:
                     user = "anonymous"
                 rr = ":pserver:" + user + "@" + serv + ":" +  root
                 if port:
                     rr2, port = "-", int(port)
                 else:
                     rr2, port = rr, 2401
                 rr += str(port)
 
                 if not passw:
                     passw = "A"
                     pf = open(os.path.join(os.environ["HOME"], ".cvspass"))
                     for l in pf:
                         # :pserver:[e-mail]:/cvsroot/zmailer Ah<Z
                         m = re.match(r'(/\d+\s+/)?(.*)', l)
                         l = m.group(2)
                         w, p = l.split(' ', 1)
                         if w in [rr, rr2]:
                             passw = p
                             break
                     pf.close()
 
                 sck = socket.socket()
                 sck.connect((serv, port))
                 sck.send("\n".join(["BEGIN AUTH REQUEST", root, user, passw, "END AUTH REQUEST", ""]))
                 if sck.recv(128) != "I LOVE YOU\n":
                     raise NoRepo("CVS pserver authentication failed")
 
                 self.writep = self.readp = sck.makefile('r+')
 
         if not conntype and root.startswith(":local:"):
             conntype = "local"
             root = root[7:]
 
         if not conntype:
             # :ext:user@host/home/user/path/to/cvsroot
             if root.startswith(":ext:"):
                 root = root[5:]
             m = re.match(r'(?:([^@:/]+)@)?([^:/]+):?(.*)', root)
             if not m:
                 conntype = "local"
             else:
                 conntype = "rsh"
                 user, host, root = m.group(1), m.group(2), m.group(3)
 
         if conntype != "pserver":
             if conntype == "rsh": 
                 rsh = os.environ.get("CVS_RSH" or "rsh")
                 if user:
                     cmd = [rsh, '-l', user, host] + cmd
                 else:
                     cmd = [rsh, host] + cmd
 
             self.writep, self.readp = os.popen2(cmd)
 
         self.realroot = root
 
         self.writep.write("Root %s\n" % root)
         self.writep.write("Valid-responses ok error Valid-requests Mode"
                           " M Mbinary E Checked-in Created Updated"
                           " Merged Removed\n")
         self.writep.write("valid-requests\n")
         self.writep.flush()
         r = self.readp.readline()
         if not r.startswith("Valid-requests"):
             abort("server sucks\n")
         if "UseUnchanged" in r:
             self.writep.write("UseUnchanged\n")
             self.writep.flush()
             r = self.readp.readline()
 
     def getheads(self):
         return self.heads
 
     def _getfile(self, name, rev):
         if rev.endswith("(DEAD)"):
             raise IOError
 
         args = ("-N -P -kk -r %s --" % rev).split()
         args.append(os.path.join(self.cvsrepo, name))
         for x in args:
             self.writep.write("Argument %s\n" % x)
         self.writep.write("Directory .\n%s\nco\n" % self.realroot)
         self.writep.flush()
 
         data = ""
         while 1:
             line = self.readp.readline()
             if line.startswith("Created ") or line.startswith("Updated "):
                 self.readp.readline() # path
                 self.readp.readline() # entries
                 mode = self.readp.readline()[:-1]
                 count = int(self.readp.readline()[:-1])
                 data = self.readp.read(count)
             elif line.startswith(" "):
                 data += line[1:]
             elif line.startswith("M "):
                 pass
             elif line.startswith("Mbinary "):
                 count = int(self.readp.readline()[:-1])
                 data = self.readp.read(count)
             else:
                 if line == "ok\n":
                     return (data, "x" in mode and "x" or "")
                 elif line.startswith("E "):
                     warn("cvs server: %s\n" % line[2:])
                 elif line.startswith("Remove"):
                     l = self.readp.readline()
                     l = self.readp.readline()
                     if l != "ok\n":
                         abort("unknown CVS response: %s\n" % l)
                 else:
                     abort("unknown CVS response: %s\n" % line)
 
     def getfile(self, file, rev):
         data, mode = self._getfile(file, rev)
         self.modecache[(file, rev)] = mode
         return data
 
     def getmode(self, file, rev):
         return self.modecache[(file, rev)]
 
     def getchanges(self, rev):
         self.modecache = {}
         files = self.files[rev]
         cl = files.items()
         cl.sort()
         return cl
 
     def recode(self, text):
         return text.decode(self.encoding, "replace").encode("utf-8")
 
     def getcommit(self, rev):
         return self.changeset[rev]
 
     def gettags(self):
         return self.tags
 
 class convert_git:
     def __init__(self, path):
         if os.path.isdir(path + "/.git"):
             path += "/.git"
         self.path = path
         if not os.path.exists(path + "/objects"):
             raise NoRepo("couldn't open GIT repo %s" % path)
 
     def getheads(self):
         fh = os.popen("GIT_DIR=%s git-rev-parse --verify HEAD" % self.path)
         return [fh.read()[:-1]]
 
     def catfile(self, rev, type):
         if rev == "0" * 40: raise IOError()
         fh = os.popen("GIT_DIR=%s git-cat-file %s %s 2>/dev/null" % (self.path, type, rev))
         return fh.read()
 
     def getfile(self, name, rev):
         return self.catfile(rev, "blob")
 
     def getmode(self, name, rev):
         return self.modecache[(name, rev)]
 
     def getchanges(self, version):
         self.modecache = {}
         fh = os.popen("GIT_DIR=%s git-diff-tree --root -m -r %s" % (self.path, version))
         changes = []
         for l in fh:
             if "\t" not in l: continue
             m, f = l[:-1].split("\t")
             m = m.split()
             h = m[3]
             p = (m[1] == "100755")
             s = (m[1] == "120000")
             self.modecache[(f, h)] = (p and "x") or (s and "l") or ""
             changes.append((f, h))
         return changes
 
     def getcommit(self, version):
         c = self.catfile(version, "commit") # read the commit hash
         end = c.find("\n\n")
         message = c[end+2:]
         message = recode(message)
         l = c[:end].splitlines()
         manifest = l[0].split()[1]
         parents = []
         for e in l[1:]:
             n,v = e.split(" ", 1)
             if n == "author":
                 p = v.split()
                 tm, tz = p[-2:]
                 author = " ".join(p[:-2])
                 if author[0] == "<": author = author[1:-1]
                 author = recode(author)
             if n == "committer":
                 p = v.split()
                 tm, tz = p[-2:]
                 committer = " ".join(p[:-2])
                 if committer[0] == "<": committer = committer[1:-1]
                 committer = recode(committer)
                 message += "\ncommitter: %s\n" % committer
             if n == "parent": parents.append(v)
 
         tzs, tzh, tzm = tz[-5:-4] + "1", tz[-4:-2], tz[-2:]
         tz = -int(tzs) * (int(tzh) * 3600 + int(tzm))
         date = tm + " " + str(tz)
 
         c = commit(parents=parents, date=date, author=author, desc=message)
         return c
 
     def gettags(self):
         tags = {}
         fh = os.popen('git-ls-remote --tags "%s" 2>/dev/null' % self.path)
         prefix = 'refs/tags/'
         for line in fh:
             line = line.strip()
             if not line.endswith("^{}"):
                 continue
             node, tag = line.split(None, 1)
             if not tag.startswith(prefix):
                 continue
             tag = tag[len(prefix):-3]
             tags[tag] = node
 
         return tags
 
 class convert_mercurial:
     def __init__(self, path):
         self.path = path
         u = ui.ui()
         try:
             self.repo = hg.repository(u, path)
         except:
             raise NoRepo("could open hg repo %s" % path)
 
     def mapfile(self):
         return os.path.join(self.path, ".hg", "shamap")
 
     def getheads(self):
         h = self.repo.changelog.heads()
         return [ hg.hex(x) for x in h ]
 
     def putfile(self, f, e, data):
         self.repo.wwrite(f, data, e)
         if self.repo.dirstate.state(f) == '?':
             self.repo.dirstate.update([f], "a")
 
     def delfile(self, f):
         try:
             os.unlink(self.repo.wjoin(f))
             #self.repo.remove([f])
         except:
             pass
 
     def putcommit(self, files, parents, commit):
         seen = {}
         pl = []
         for p in parents:
             if p not in seen:
                 pl.append(p)
                 seen[p] = 1
         parents = pl
 
         if len(parents) < 2: parents.append("0" * 40)
         if len(parents) < 2: parents.append("0" * 40)
         p2 = parents.pop(0)
 
         text = commit.desc
         extra = {}
         try:
             extra["branch"] = commit.branch
         except AttributeError:
             pass
 
         while parents:
             p1 = p2
             p2 = parents.pop(0)
             a = self.repo.rawcommit(files, text, commit.author, commit.date,
                                     hg.bin(p1), hg.bin(p2), extra=extra)
             text = "(octopus merge fixup)\n"
             p2 = hg.hex(self.repo.changelog.tip())
 
         return p2
 
     def puttags(self, tags):
         try:
             old = self.repo.wfile(".hgtags").read()
             oldlines = old.splitlines(1)
             oldlines.sort()
         except:
             oldlines = []
 
         k = tags.keys()
         k.sort()
         newlines = []
         for tag in k:
             newlines.append("%s %s\n" % (tags[tag], tag))
 
         newlines.sort()
 
         if newlines != oldlines:
             status("updating tags\n")
             f = self.repo.wfile(".hgtags", "w")
             f.write("".join(newlines))
             f.close()
             if not oldlines: self.repo.add([".hgtags"])
             date = "%s 0" % int(time.mktime(time.gmtime()))
             self.repo.rawcommit([".hgtags"], "update tags", "convert-repo",
                                 date, self.repo.changelog.tip(), hg.nullid)
             return hg.hex(self.repo.changelog.tip())
 
 converters = [convert_cvs, convert_git, convert_mercurial]
 
 def converter(path):
     if not os.path.isdir(path):
         abort("%s: not a directory\n" % path)
     for c in converters:
         try:
             return c(path)
         except NoRepo:
             pass
     abort("%s: unknown repository type\n" % path)
 
 class convert:
     def __init__(self, source, dest, mapfile, opts):
 
         self.source = source
         self.dest = dest
         self.mapfile = mapfile
         self.opts = opts
         self.commitcache = {}
 
         self.map = {}
         try:
             for l in file(self.mapfile):
                 sv, dv = l[:-1].split()
                 self.map[sv] = dv
         except IOError:
             pass
 
     def walktree(self, heads):
         visit = heads
         known = {}
         parents = {}
         while visit:
             n = visit.pop(0)
             if n in known or n in self.map: continue
             known[n] = 1
             self.commitcache[n] = self.source.getcommit(n)
             cp = self.commitcache[n].parents
             for p in cp:
                 parents.setdefault(n, []).append(p)
                 visit.append(p)
 
         return parents
 
     def toposort(self, parents):
         visit = parents.keys()
         seen = {}
         children = {}
 
         while visit:
             n = visit.pop(0)
             if n in seen: continue
             seen[n] = 1
             pc = 0
             if n in parents:
                 for p in parents[n]:
                     if p not in self.map: pc += 1
                     visit.append(p)
                     children.setdefault(p, []).append(n)
             if not pc: root = n
 
         s = []
         removed = {}
         visit = children.keys()
         while visit:
             n = visit.pop(0)
             if n in removed: continue
             dep = 0
             if n in parents:
                 for p in parents[n]:
                     if p in self.map: continue
                     if p not in removed:
                         # we're still dependent
                         visit.append(n)
                         dep = 1
                         break
 
             if not dep:
                 # all n's parents are in the list
                 removed[n] = 1
                 if n not in self.map:
                     s.append(n)
                 if n in children:
                     for c in children[n]:
                         visit.insert(0, c)
 
         if opts.get('datesort'):
             depth = {}
             for n in s:
                 depth[n] = 0
                 pl = [p for p in self.commitcache[n].parents if p not in self.map]
                 if pl:
                     depth[n] = max([depth[p] for p in pl]) + 1
 
             s = [(depth[n], self.commitcache[n].date, n) for n in s]
             s.sort()
             s = [e[2] for e in s]
 
         return s
 
     def copy(self, rev):
         c = self.commitcache[rev]
         files = self.source.getchanges(rev)
 
         for f,v in files:
             try:
                 data = self.source.getfile(f, v)
             except IOError, inst:
                 self.dest.delfile(f)
             else:
                 e = self.source.getmode(f, v)
                 self.dest.putfile(f, e, data)
 
         r = [self.map[v] for v in c.parents]
         f = [f for f,v in files]
         self.map[rev] = self.dest.putcommit(f, r, c)
         file(self.mapfile, "a").write("%s %s\n" % (rev, self.map[rev]))
 
     def convert(self):
         status("scanning source...\n")
         heads = self.source.getheads()
         parents = self.walktree(heads)
         status("sorting...\n")
         t = self.toposort(parents)
         num = len(t)
         c = None
 
         status("converting...\n")
         for c in t:
             num -= 1
             desc = self.commitcache[c].desc
             if "\n" in desc:
                 desc = desc.splitlines()[0]
             status("%d %s\n" % (num, desc))
             self.copy(c)
 
         tags = self.source.gettags()
         ctags = {}
         for k in tags:
             v = tags[k]
             if v in self.map:
                 ctags[k] = self.map[v]
 
         if c and ctags:
             nrev = self.dest.puttags(ctags)
             # write another hash correspondence to override the previous
             # one so we don't end up with extra tag heads
             if nrev:
                 file(self.mapfile, "a").write("%s %s\n" % (c, nrev))
 
 def command(src, dest=None, mapfile=None, **opts):
     srcc = converter(src)
     if not hasattr(srcc, "getcommit"):
         abort("%s: can't read from this repo type\n" % src)
 
     if not dest:
         dest = src + "-hg"
         status("assuming destination %s\n" % dest)
         if not os.path.isdir(dest):
             status("creating repository %s\n" % dest)
             os.system("hg init " + dest)
     destc = converter(dest)
     if not hasattr(destc, "putcommit"):
         abort("%s: can't write to this repo type\n" % src)
 
     if not mapfile:
         try:
             mapfile = destc.mapfile()
         except:
             mapfile = os.path.join(destc, "map")
 
     c = convert(srcc, destc, mapfile, opts)
     c.convert()
 
 options = [('q', 'quiet', None, 'suppress output'),
            ('', 'datesort', None, 'try to sort changesets by date')]
 opts = {}
 args = fancyopts.fancyopts(sys.argv[1:], options, opts)
 
 if opts['quiet']:
     quiet = 1
 
 try:
     command(*args, **opts)
 except Abort, inst:
     warn(inst)
 except KeyboardInterrupt:
     status("interrupted\n")