convert: Perforce source for conversion to Mercurial

hgext/convert/p4.py

@@ -0,0 +1,176 b''
+#
+# Perforce source for convert extension.
+#
+# Copyright 2009, Frank Kingswood <[e-mail]>
+#
+# This software may be used and distributed according to the terms
+# of the GNU General Public License, incorporated herein by reference.
+#
+
+from mercurial import util
+from mercurial.i18n import _
+
+from common import commit, converter_source, checktool
+import marshal
+
+def loaditer(f):
+    "Yield the dictionary objects generated by p4"
+    try:
+        while True:
+            d = marshal.load(f)
+            if not d:
+                break
+            yield d
+    except EOFError:
+        pass
+
+class p4_source(converter_source):
+    def __init__(self, ui, path, rev=None):
+        super(p4_source, self).__init__(ui, path, rev=rev)
+
+        checktool('p4')
+
+        self.p4changes = {}
+        self.heads = {}
+        self.changeset = {}
+        self.files = {}
+        self.tags = {}
+        self.lastbranch = {}
+        self.parent = {}
+        self.encoding = "latin_1"
+        self.depotname = {}           # mapping from local name to depot name
+        self.modecache = {}
+
+        self._parse(ui, path)
+
+    def _parse_view(self, path):
+        "Read changes affecting the path"
+        cmd = "p4 -G changes -s submitted '%s'" % path
+        stdout = util.popen(cmd)
+        for d in loaditer(stdout):
+            c = d.get("change", None)
+            if c:
+                self.p4changes[c] = True
+
+    def _parse(self, ui, path):
+        "Prepare list of P4 filenames and revisions to import"
+        ui.status(_('reading p4 views\n'))
+
+        # read client spec or view
+        if "/" in path:
+            self._parse_view(path)
+            if path.startswith("//") and path.endswith("/..."):
+                views = {path[:-3]:""}
+            else:
+                views = {"//": ""}
+        else:
+            cmd = "p4 -G client -o '%s'" % path
+            clientspec = marshal.load(util.popen(cmd))
+            
+            views = {}
+            for client in clientspec:
+                if client.startswith("View"):
+                    sview, cview = clientspec[client].split()
+                    self._parse_view(sview)
+                    if sview.endswith("...") and cview.endswith("..."):
+                        sview = sview[:-3]
+                        cview = cview[:-3]
+                    cview = cview[2:]
+                    cview = cview[cview.find("/") + 1:]
+                    views[sview] = cview
+
+        # list of changes that affect our source files
+        self.p4changes = self.p4changes.keys()
+        self.p4changes.sort(key=int)
+
+        # list with depot pathnames, longest first
+        vieworder = views.keys()
+        vieworder.sort(key=lambda x: -len(x))
+
+        # handle revision limiting
+        startrev = self.ui.config('convert', 'p4.startrev', default=0)
+        self.p4changes = [x for x in self.p4changes 
+                          if ((not startrev or int(x) >= int(startrev)) and 
+                              (not self.rev or int(x) <= int(self.rev)))]
+
+        # now read the full changelists to get the list of file revisions
+        ui.status(_('collecting p4 changelists\n'))
+        lastid = None
+        for change in self.p4changes:
+            cmd = "p4 -G describe %s" % change
+            stdout = util.popen(cmd)
+            d = marshal.load(stdout)
+
+            desc = self.recode(d["desc"])
+            shortdesc = desc.split("\n", 1)[0]
+            t = '%s %s' % (d["change"], repr(shortdesc)[1:-1])
+            ui.status(util.ellipsis(t, 80) + '\n')
+
+            if lastid:
+                parents = [lastid]
+            else:
+                parents = []
+            
+            date = (int(d["time"]), 0)     # timezone not set
+            c = commit(author=self.recode(d["user"]), date=util.datestr(date),
+                        parents=parents, desc=desc, branch='', extra={"p4": change})
+
+            files = []
+            i = 0
+            while ("depotFile%d" % i) in d and ("rev%d" % i) in d:
+                oldname = d["depotFile%d" % i]
+                filename = None
+                for v in vieworder:
+                    if oldname.startswith(v):
+                        filename = views[v] + oldname[len(v):]
+                        break
+                if filename:
+                    files.append((filename, d["rev%d" % i]))
+                    self.depotname[filename] = oldname
+                i += 1
+            self.changeset[change] = c
+            self.files[change] = files
+            lastid = change
+        
+        if lastid:
+            self.heads = [lastid]
+
+    def getheads(self):
+        return self.heads
+
+    def getfile(self, name, rev):
+        cmd = "p4 -G print '%s#%s'" % (self.depotname[name], rev)
+        stdout = util.popen(cmd)
+
+        mode = None
+        data = ""
+
+        for d in loaditer(stdout):
+            if d["code"] == "stat":
+                if "+x" in d["type"]:
+                    mode = "x"
+                else:
+                    mode = ""
+            elif d["code"] == "text":
+                data += d["data"]
+
+        if mode is None:
+            raise IOError()
+
+        self.modecache[(name, rev)] = mode
+        return data
+
+    def getmode(self, name, rev):
+        return self.modecache[(name, rev)]
+
+    def getchanges(self, rev):
+        return self.files[rev], {}
+
+    def getcommit(self, rev):
+        return self.changeset[rev]
+
+    def gettags(self):
+        return self.tags
+
+    def getchangedfiles(self, rev, i):
+        return util.sort([x[0] for x in self.files[rev]])

tests/test-convert-p4

@@ -0,0 +1,75 b''
+#!/bin/sh
+
+"$TESTDIR/hghave" p4 || exit 80
+
+echo "[extensions]" >> $HGRCPATH
+echo "convert = " >> $HGRCPATH
+
+echo % create p4 depot
+export P4ROOT=$PWD/depot
+export P4AUDIT=$P4ROOT/audit
+export P4JOURNAL=$P4ROOT/journal
+export P4LOG=$P4ROOT/log
+export P4PORT=localhost:16661
+export P4DEBUG=1
+
+echo % start the p4 server
+[ ! -d $P4ROOT ] && mkdir $P4ROOT
+p4d -f -J off >$P4ROOT/stdout 2>$P4ROOT/stderr &
+trap "echo % stop the p4 server ; p4 admin stop" EXIT
+
+# wait for the server to initialize
+while ! p4 ; do
+   sleep 1
+done >/dev/null 2>/dev/null
+
+echo % create a client spec
+export P4CLIENT=hg-p4-import
+DEPOTPATH=//depot/test-mercurial-import/...
+p4 client -o | sed '/^View:/,$ d' >p4client
+echo View: >>p4client
+echo " $DEPOTPATH //$P4CLIENT/..." >>p4client
+p4 client -i <p4client
+
+echo % populate the depot
+echo a > a
+mkdir b
+echo c > b/c
+p4 add a b/c
+p4 submit -d initial
+
+echo % change some files
+p4 edit a
+echo aa >> a
+p4 submit -d "change a"
+
+p4 edit b/c
+echo cc >> b/c
+p4 submit -d "change b/c"
+
+echo % convert
+hg convert -s p4 $DEPOTPATH dst
+hg -R dst log --template 'rev=#rev# desc="#desc#" tags="#tags#" files="#files#"\n'
+
+echo % change some files
+p4 edit a b/c
+echo aaa >> a
+echo ccc >> b/c
+p4 submit -d "change a b/c"
+
+echo % convert again
+hg convert -s p4 $DEPOTPATH dst
+hg -R dst log --template 'rev=#rev# desc="#desc#" tags="#tags#" files="#files#"\n'
+
+echo % interesting names
+echo dddd > "d d"
+mkdir " e "
+echo fff >" e /f "
+p4 add "d d" " e /f "
+p4 submit -d "add d e f"
+
+echo % convert again
+hg convert -s p4 $DEPOTPATH dst
+hg -R dst log --template 'rev=#rev# desc="#desc#" tags="#tags#" files="#files#"\n'
+
+

tests/test-convert-p4.out

@@ -0,0 +1,88 b''
+% create p4 depot
+% start the p4 server
+% create a client spec
+Client hg-p4-import saved.
+% populate the depot
+//depot/test-mercurial-import/a#1 - opened for add
+//depot/test-mercurial-import/b/c#1 - opened for add
+Submitting change 1.
+Locking 2 files ...
+add //depot/test-mercurial-import/a#1
+add //depot/test-mercurial-import/b/c#1
+Change 1 submitted.
+% change some files
+//depot/test-mercurial-import/a#1 - opened for edit
+Submitting change 2.
+Locking 1 files ...
+edit //depot/test-mercurial-import/a#2
+Change 2 submitted.
+//depot/test-mercurial-import/b/c#1 - opened for edit
+Submitting change 3.
+Locking 1 files ...
+edit //depot/test-mercurial-import/b/c#2
+Change 3 submitted.
+% convert
+initializing destination dst repository
+reading p4 views
+collecting p4 changelists
+1 initial
+2 change a
+3 change b/c
+scanning source...
+sorting...
+converting...
+2 initial
+1 change a
+0 change b/c
+rev=2 desc="change b/c" tags="tip" files="b/c"
+rev=1 desc="change a" tags="" files="a"
+rev=0 desc="initial" tags="" files="a b/c"
+% change some files
+//depot/test-mercurial-import/a#2 - opened for edit
+//depot/test-mercurial-import/b/c#2 - opened for edit
+Submitting change 4.
+Locking 2 files ...
+edit //depot/test-mercurial-import/a#3
+edit //depot/test-mercurial-import/b/c#3
+Change 4 submitted.
+% convert again
+reading p4 views
+collecting p4 changelists
+1 initial
+2 change a
+3 change b/c
+4 change a b/c
+scanning source...
+sorting...
+converting...
+0 change a b/c
+rev=3 desc="change a b/c" tags="tip" files="a b/c"
+rev=2 desc="change b/c" tags="" files="b/c"
+rev=1 desc="change a" tags="" files="a"
+rev=0 desc="initial" tags="" files="a b/c"
+% interesting names
+//depot/test-mercurial-import/d d#1 - opened for add
+//depot/test-mercurial-import/ e /f #1 - opened for add
+Submitting change 5.
+Locking 2 files ...
+add //depot/test-mercurial-import/ e /f #1
+add //depot/test-mercurial-import/d d#1
+Change 5 submitted.
+% convert again
+reading p4 views
+collecting p4 changelists
+1 initial
+2 change a
+3 change b/c
+4 change a b/c
+5 add d e f
+scanning source...
+sorting...
+converting...
+0 add d e f
+rev=4 desc="add d e f" tags="tip" files=" e /f  d d"
+rev=3 desc="change a b/c" tags="" files="a b/c"
+rev=2 desc="change b/c" tags="" files="b/c"
+rev=1 desc="change a" tags="" files="a"
+rev=0 desc="initial" tags="" files="a b/c"
+% stop the p4 server

hgext/convert/__init__.py

@@ -25,6 +25,7 b' def convert(ui, src, dest=None, revmapfi'
     - Monotone [mtn]
     - GNU Arch [gnuarch]
     - Bazaar [bzr]
+    - Perforce [p4]
 
     Accepted destination formats [identifiers]:
     - Mercurial [hg]
@@ -168,6 +169,22 b' def convert(ui, src, dest=None, revmapfi'
     --config convert.svn.startrev=0           (svn revision number)
         specify start Subversion revision.
 
+    Perforce Source
+    ---------------
+
+    The Perforce (P4) importer can be given a p4 depot path or a client
+    specification as source. It will convert all files in the source to
+    a flat Mercurial repository, ignoring labels, branches and integrations.
+    Note that when a depot path is given you then usually should specify a
+    target directory, because otherwise the target may be named ...-hg.
+
+    It is possible to limit the amount of source history to be converted
+    by specifying an initial Perforce revision.
+
+    --config convert.p4.startrev=0           (perforce changelist number)
+        specify initial Perforce revision.
+
+
     Mercurial Destination
     ---------------------
 

hgext/convert/convcmd.py

@@ -14,6 +14,7 b' from subversion import debugsvnlog, svn_'
 from monotone import monotone_source
 from gnuarch import gnuarch_source
 from bzr import bzr_source
+from p4 import p4_source
 import filemap
 
 import os, shutil
@@ -37,6 +38,7 b' source_converters = ['
     ('mtn', monotone_source),
     ('gnuarch', gnuarch_source),
     ('bzr', bzr_source),
+    ('p4', p4_source),
     ]
 
 sink_converters = [

tests/hghave

@@ -125,6 +125,9 b' def has_svn_bindings():'
     except ImportError:
         return False
 
+def has_p4():
+    return matchoutput('p4 -V', r'Rev\. P4/') and matchoutput('p4d -V', r'Rev\. P4D/')
+
 def has_symlink():
     return hasattr(os, "symlink")
 
@@ -173,6 +176,7 b' checks = {'
     "lsprof": (has_lsprof, "python lsprof module"),
     "mtn": (has_mtn, "monotone client (> 0.31)"),
     "outer-repo": (has_outer_repo, "outer repo"),
+    "p4": (has_p4, "Perforce server and client"),
     "pygments": (has_pygments, "Pygments source highlighting library"),
     "svn": (has_svn, "subversion client and admin tools"),
     "svn-bindings": (has_svn_bindings, "subversion python bindings"),

tests/test-convert.out

@@ -11,6 +11,7 b' convert a foreign SCM repository to a Me'
     - Monotone [mtn]
     - GNU Arch [gnuarch]
     - Bazaar [bzr]
+    - Perforce [p4]
 
     Accepted destination formats [identifiers]:
     - Mercurial [hg]
@@ -154,6 +155,22 b' convert a foreign SCM repository to a Me'
     --config convert.svn.startrev=0           (svn revision number)
         specify start Subversion revision.
 
+    Perforce Source
+    ---------------
+
+    The Perforce (P4) importer can be given a p4 depot path or a client
+    specification as source. It will convert all files in the source to
+    a flat Mercurial repository, ignoring labels, branches and integrations.
+    Note that when a depot path is given you then usually should specify a
+    target directory, because otherwise the target may be named ...-hg.
+
+    It is possible to limit the amount of source history to be converted
+    by specifying an initial Perforce revision.
+
+    --config convert.p4.startrev=0           (perforce changelist number)
+        specify initial Perforce revision.
+
+
     Mercurial Destination
     ---------------------