convert/cvs: delay CVS log parsing after initialization (issue1581/2)

Source and destination constructors should be fast so configurations issues are
hit quickly, including authentication and filemap/authormap/splicemap issues.
Delaying might be a problem if the remove side disconnects idle connections
while the log is being read. It did not happen when converting openafs
repository, where log retrieval took at least 10mn.

hgext/convert/cvs.py

@@ -1,353 +1,359 b''
 # CVS conversion code inspired by hg-cvs-import and git-cvsimport
 
 import os, locale, re, socket, errno
 from cStringIO import StringIO
 from mercurial import util
 from mercurial.i18n import _
 
 from common import NoRepo, commit, converter_source, checktool
 import cvsps
 
 class convert_cvs(converter_source):
     def __init__(self, ui, path, rev=None):
         super(convert_cvs, self).__init__(ui, path, rev=rev)
 
         cvs = os.path.join(path, "CVS")
         if not os.path.exists(cvs):
             raise NoRepo("%s does not look like a CVS checkout" % path)
 
         checktool('cvs')
         self.cmd = ui.config('convert', 'cvsps', 'builtin')
         cvspsexe = self.cmd.split(None, 1)[0]
         self.builtin = cvspsexe == 'builtin'
 
         if not self.builtin:
             checktool(cvspsexe)
 
-        self.changeset = {}
+        self.changeset = None
         self.files = {}
         self.tags = {}
         self.lastbranch = {}
         self.parent = {}
         self.socket = None
         self.cvsroot = file(os.path.join(cvs, "Root")).read()[:-1]
         self.cvsrepo = file(os.path.join(cvs, "Repository")).read()[:-1]
         self.encoding = locale.getpreferredencoding()
 
-        self._parse(ui)
         self._connect()
 
-    def _parse(self, ui):
-        if self.changeset:
+    def _parse(self):
+        if self.changeset is not None:
             return
+        self.changeset = {}
 
         maxrev = 0
         cmd = self.cmd
         if self.rev:
             # TODO: handle tags
             try:
                 # patchset number?
                 maxrev = int(self.rev)
             except ValueError:
                 try:
                     # date
                     util.parsedate(self.rev, ['%Y/%m/%d %H:%M:%S'])
                     cmd = '%s -d "1970/01/01 00:00:01" -d "%s"' % (cmd, self.rev)
                 except util.Abort:
                     raise util.Abort(_('revision %s is not a patchset number or date') % self.rev)
 
         d = os.getcwd()
         try:
             os.chdir(self.path)
             id = None
             state = 0
             filerevids = {}
 
             if self.builtin:
                 # builtin cvsps code
-                ui.status(_('using builtin cvsps\n'))
+                self.ui.status(_('using builtin cvsps\n'))
 
-                db = cvsps.createlog(ui, cache='update')
-                db = cvsps.createchangeset(ui, db,
-                      fuzz=int(ui.config('convert', 'cvsps.fuzz', 60)),
-                      mergeto=ui.config('convert', 'cvsps.mergeto', None),
-                      mergefrom=ui.config('convert', 'cvsps.mergefrom', None))
+                db = cvsps.createlog(self.ui, cache='update')
+                db = cvsps.createchangeset(self.ui, db,
+                      fuzz=int(self.ui.config('convert', 'cvsps.fuzz', 60)),
+                      mergeto=self.ui.config('convert', 'cvsps.mergeto', None),
+                      mergefrom=self.ui.config('convert', 'cvsps.mergefrom', None))
 
                 for cs in db:
                     if maxrev and cs.id>maxrev:
                         break
                     id = str(cs.id)
                     cs.author = self.recode(cs.author)
                     self.lastbranch[cs.branch] = id
                     cs.comment = self.recode(cs.comment)
                     date = util.datestr(cs.date)
                     self.tags.update(dict.fromkeys(cs.tags, id))
 
                     files = {}
                     for f in cs.entries:
                         files[f.file] = "%s%s" % ('.'.join([str(x) for x in f.revision]),
                                                   ['', '(DEAD)'][f.dead])
 
                     # add current commit to set
                     c = commit(author=cs.author, date=date,
                              parents=[str(p.id) for p in cs.parents],
                              desc=cs.comment, branch=cs.branch or '')
                     self.changeset[id] = c
                     self.files[id] = files
             else:
                 # external cvsps
                 for l in util.popen(cmd):
                     if state == 0: # header
                         if l.startswith("PatchSet"):
                             id = l[9:-2]
                             if maxrev and int(id) > maxrev:
                                 # ignore everything
                                 state = 3
                         elif l.startswith("Date:"):
                             date = util.parsedate(l[6:-1], ["%Y/%m/%d %H:%M:%S"])
                             date = util.datestr(date)
                         elif l.startswith("Branch:"):
                             branch = l[8:-1]
                             self.parent[id] = self.lastbranch.get(branch, 'bad')
                             self.lastbranch[branch] = id
                         elif l.startswith("Ancestor branch:"):
                             ancestor = l[17:-1]
                             # figure out the parent later
                             self.parent[id] = self.lastbranch[ancestor]
                         elif l.startswith("Author:"):
                             author = self.recode(l[8:-1])
                         elif l.startswith("Tag:") or l.startswith("Tags:"):
                             t = l[l.index(':')+1:]
                             t = [ut.strip() for ut in t.split(',')]
                             if (len(t) > 1) or (t[0] and (t[0] != "(none)")):
                                 self.tags.update(dict.fromkeys(t, id))
                         elif l.startswith("Log:"):
                             # switch to gathering log
                             state = 1
                             log = ""
                     elif state == 1: # log
                         if l == "Members: \n":
                             # switch to gathering members
                             files = {}
                             oldrevs = []
                             log = self.recode(log[:-1])
                             state = 2
                         else:
                             # gather log
                             log += l
                     elif state == 2: # members
                         if l == "\n": # start of next entry
                             state = 0
                             p = [self.parent[id]]
                             if id == "1":
                                 p = []
                             if branch == "HEAD":
                                 branch = ""
                             if branch:
                                 latest = 0
                                 # the last changeset that contains a base
                                 # file is our parent
                                 for r in oldrevs:
                                     latest = max(filerevids.get(r, 0), latest)
                                 if latest:
                                     p = [latest]
 
                             # add current commit to set
                             c = commit(author=author, date=date, parents=p,
                                        desc=log, branch=branch)
                             self.changeset[id] = c
                             self.files[id] = files
                         else:
                             colon = l.rfind(':')
                             file = l[1:colon]
                             rev = l[colon+1:-2]
                             oldrev, rev = rev.split("->")
                             files[file] = rev
 
                             # save some information for identifying branch points
                             oldrevs.append("%s:%s" % (oldrev, file))
                             filerevids["%s:%s" % (rev, file)] = id
                     elif state == 3:
                         # swallow all input
                         continue
 
             self.heads = self.lastbranch.values()
         finally:
             os.chdir(d)
 
     def _connect(self):
         root = self.cvsroot
         conntype = None
         user, host = None, None
         cmd = ['cvs', 'server']
 
         self.ui.status(_("connecting to %s\n") % root)
 
         if root.startswith(":pserver:"):
             root = root[9:]
             m = re.match(r'(?:(.*?)(?::(.*?))?@)?([^:\/]*)(?::(\d*))?(.*)',
                          root)
             if m:
                 conntype = "pserver"
                 user, passw, serv, port, root = m.groups()
                 if not user:
                     user = "anonymous"
                 if not port:
                     port = 2401
                 else:
                     port = int(port)
                 format0 = ":pserver:%s@%s:%s" % (user, serv, root)
                 format1 = ":pserver:%s@%s:%d%s" % (user, serv, port, root)
 
                 if not passw:
                     passw = "A"
                     cvspass = os.path.expanduser("~/.cvspass")
                     try:
                         pf = open(cvspass)
                         for line in pf.read().splitlines():
                             part1, part2 = line.split(' ', 1)
                             if part1 == '/1':
                                 # /1 :pserver:[e-mail]:2401/cvsroot/foo Ah<Z
                                 part1, part2 = part2.split(' ', 1)
                                 format = format1
                             else:
                                 # :pserver:[e-mail]:/cvsroot/foo Ah<Z
                                 format = format0
                             if part1 == format:
                                 passw = part2
                                 break
                         pf.close()
                     except IOError, inst:
                         if inst.errno != errno.ENOENT:
                             if not getattr(inst, 'filename', None):
                                 inst.filename = cvspass
                             raise
 
                 sck = socket.socket()
                 sck.connect((serv, port))
                 sck.send("\n".join(["BEGIN AUTH REQUEST", root, user, passw,
                                     "END AUTH REQUEST", ""]))
                 if sck.recv(128) != "I LOVE YOU\n":
                     raise util.Abort(_("CVS pserver authentication failed"))
 
                 self.writep = self.readp = sck.makefile('r+')
 
         if not conntype and root.startswith(":local:"):
             conntype = "local"
             root = root[7:]
 
         if not conntype:
             # :ext:user@host/home/user/path/to/cvsroot
             if root.startswith(":ext:"):
                 root = root[5:]
             m = re.match(r'(?:([^@:/]+)@)?([^:/]+):?(.*)', root)
             # Do not take Windows path "c:\foo\bar" for a connection strings
             if os.path.isdir(root) or not m:
                 conntype = "local"
             else:
                 conntype = "rsh"
                 user, host, root = m.group(1), m.group(2), m.group(3)
 
         if conntype != "pserver":
             if conntype == "rsh":
                 rsh = os.environ.get("CVS_RSH") or "ssh"
                 if user:
                     cmd = [rsh, '-l', user, host] + cmd
                 else:
                     cmd = [rsh, host] + cmd
 
             # popen2 does not support argument lists under Windows
             cmd = [util.shellquote(arg) for arg in cmd]
             cmd = util.quotecommand(' '.join(cmd))
             self.writep, self.readp = util.popen2(cmd, 'b')
 
         self.realroot = root
 
         self.writep.write("Root %s\n" % root)
         self.writep.write("Valid-responses ok error Valid-requests Mode"
                           " M Mbinary E Checked-in Created Updated"
                           " Merged Removed\n")
         self.writep.write("valid-requests\n")
         self.writep.flush()
         r = self.readp.readline()
         if not r.startswith("Valid-requests"):
             raise util.Abort(_("server sucks"))
         if "UseUnchanged" in r:
             self.writep.write("UseUnchanged\n")
             self.writep.flush()
             r = self.readp.readline()
 
     def getheads(self):
+        self._parse()
         return self.heads
 
     def _getfile(self, name, rev):
 
         def chunkedread(fp, count):
             # file-objects returned by socked.makefile() do not handle
             # large read() requests very well.
             chunksize = 65536
             output = StringIO()
             while count > 0:
                 data = fp.read(min(count, chunksize))
                 if not data:
                     raise util.Abort(_("%d bytes missing from remote file") % count)
                 count -= len(data)
                 output.write(data)
             return output.getvalue()
 
         if rev.endswith("(DEAD)"):
             raise IOError
 
         args = ("-N -P -kk -r %s --" % rev).split()
         args.append(self.cvsrepo + '/' + name)
         for x in args:
             self.writep.write("Argument %s\n" % x)
         self.writep.write("Directory .\n%s\nco\n" % self.realroot)
         self.writep.flush()
 
         data = ""
         while 1:
             line = self.readp.readline()
             if line.startswith("Created ") or line.startswith("Updated "):
                 self.readp.readline() # path
                 self.readp.readline() # entries
                 mode = self.readp.readline()[:-1]
                 count = int(self.readp.readline()[:-1])
                 data = chunkedread(self.readp, count)
             elif line.startswith(" "):
                 data += line[1:]
             elif line.startswith("M "):
                 pass
             elif line.startswith("Mbinary "):
                 count = int(self.readp.readline()[:-1])
                 data = chunkedread(self.readp, count)
             else:
                 if line == "ok\n":
                     return (data, "x" in mode and "x" or "")
                 elif line.startswith("E "):
                     self.ui.warn(_("cvs server: %s\n") % line[2:])
                 elif line.startswith("Remove"):
                     self.readp.readline()
                 else:
                     raise util.Abort(_("unknown CVS response: %s") % line)
 
     def getfile(self, file, rev):
+        self._parse()
         data, mode = self._getfile(file, rev)
         self.modecache[(file, rev)] = mode
         return data
 
     def getmode(self, file, rev):
         return self.modecache[(file, rev)]
 
     def getchanges(self, rev):
+        self._parse()
         self.modecache = {}
         return util.sort(self.files[rev].items()), {}
 
     def getcommit(self, rev):
+        self._parse()
         return self.changeset[rev]
 
     def gettags(self):
+        self._parse()
         return self.tags
 
     def getchangedfiles(self, rev, i):
+        self._parse()
         return util.sort(self.files[rev].keys())

tests/test-convert-cvs-branch.out

@@ -1,53 +1,53 b''
 % create cvs repository
 % Create a new project
 
 
 N src/a
 N src/b
 No conflicts created by this import
 cvs checkout: Updating src
 U src/a
 U src/b
 % Branch the project
 cvs tag: Tagging .
 T a
 T b
 cvs update: Updating .
 % Modify file a, then b, then a
 cvs commit: Examining .
 checking in src/a,v
 cvs commit: Examining .
 checking in src/b,v
 cvs commit: Examining .
 checking in src/a,v
 % Convert
 assuming destination src-hg
 initializing destination src-hg repository
+connecting to cvsrepo
+scanning source...
 using builtin cvsps
 collecting CVS rlog
 7 log entries
 creating changesets
 5 changeset entries
-connecting to cvsrepo
-scanning source...
 sorting...
 converting...
 4 Initial revision
 3 init
 2 mod a
 1 mod b
 0 mod a again
 updating tags
 % Check the result
 o  5 () update tags files: .hgtags
 |
 | o  4 (BRANCH) mod a again files: a
 | |
 | o  3 (BRANCH) mod b files: b
 | |
 | o  2 (BRANCH) mod a files: a
 | |
 | o  1 (v0) init files:
 |/
 o  0 () Initial revision files: a b
 

tests/test-convert-cvs-builtincvsps.out

@@ -1,236 +1,236 b''
 % create cvs repository
 % create source directory
 % import source directory
 N src/a
 N src/b/c
 
 No conflicts created by this import
 
 % checkout source directory
 U src/a
 U src/b/c
 % commit a new revision changing b/c
 checking in src/b/c,v
 % convert fresh repo
 initializing destination src-hg repository
+connecting to cvsrepo
+scanning source...
 using builtin cvsps
 collecting CVS rlog
 5 log entries
 creating changesets
 3 changeset entries
-connecting to cvsrepo
-scanning source...
 sorting...
 converting...
 2 Initial revision
 1 import
 0 ci0
 updating tags
 a
 c
 c
 % convert fresh repo with --filemap
 initializing destination src-filemap repository
+connecting to cvsrepo
+scanning source...
 using builtin cvsps
 collecting CVS rlog
 5 log entries
 creating changesets
 3 changeset entries
-connecting to cvsrepo
-scanning source...
 sorting...
 converting...
 2 Initial revision
 1 import
 rolling back last transaction
 0 ci0
 updating tags
 c
 c
 2 update tags files: .hgtags
 1 ci0 files: b/c
 0 Initial revision files: b/c
 % commit new file revisions
 checking in src/a,v
 checking in src/b/c,v
 % convert again
+connecting to cvsrepo
+scanning source...
 using builtin cvsps
 collecting CVS rlog
 7 log entries
 creating changesets
 4 changeset entries
-connecting to cvsrepo
-scanning source...
 sorting...
 converting...
 0 ci1
 a
 a
 c
 c
 c
 % convert again with --filemap
+connecting to cvsrepo
+scanning source...
 using builtin cvsps
 collecting CVS rlog
 7 log entries
 creating changesets
 4 changeset entries
-connecting to cvsrepo
-scanning source...
 sorting...
 converting...
 0 ci1
 c
 c
 c
 3 ci1 files: b/c
 2 update tags files: .hgtags
 1 ci0 files: b/c
 0 Initial revision files: b/c
 % commit branch
 U b/c
 T a
 T b/c
 checking in src/b/c,v
 % convert again
+connecting to cvsrepo
+scanning source...
 using builtin cvsps
 collecting CVS rlog
 8 log entries
 creating changesets
 5 changeset entries
-connecting to cvsrepo
-scanning source...
 sorting...
 converting...
 0 ci2
 a
 c
 d
 % convert again with --filemap
+connecting to cvsrepo
+scanning source...
 using builtin cvsps
 collecting CVS rlog
 8 log entries
 creating changesets
 5 changeset entries
-connecting to cvsrepo
-scanning source...
 sorting...
 converting...
 0 ci2
 c
 d
 4 ci2 files: b/c
 3 ci1 files: b/c
 2 update tags files: .hgtags
 1 ci0 files: b/c
 0 Initial revision files: b/c
 % commit a new revision with funny log message
 checking in src/a,v
 % convert again
+connecting to cvsrepo
+scanning source...
 using builtin cvsps
 collecting CVS rlog
 9 log entries
 creating changesets
 6 changeset entries
-connecting to cvsrepo
-scanning source...
 sorting...
 converting...
 0 funny
 o  6 (branch) funny
 |  ----------------------------
 |  log message files: a
 o  5 (branch) ci2 files: b/c
 |
 | o  4 () ci1 files: a b/c
 | |
 | o  3 () update tags files: .hgtags
 | |
 | o  2 () ci0 files: b/c
 |/
 | o  1 (INITIAL) import files:
 |/
 o  0 () Initial revision files: a b/c
 
 % testing debugcvsps
 collecting CVS rlog
 9 log entries
 creating changesets
 6 changeset entries
 ---------------------
 PatchSet 1 
 Date:
 Author:
 Branch: HEAD
 Tag: (none) 
 Log:
 Initial revision
 
 Members: 
 	a:INITIAL->1.1 
 	b/c:INITIAL->1.1 
 
 ---------------------
 PatchSet 2 
 Date:
 Author:
 Branch: INITIAL
 Tag: start 
 Log:
 import
 
 Members: 
 	a:1.1->1.1.1.1 
 	b/c:1.1->1.1.1.1 
 
 ---------------------
 PatchSet 3 
 Date:
 Author:
 Branch: HEAD
 Tag: (none) 
 Log:
 ci0
 
 Members: 
 	b/c:1.1->1.2 
 
 ---------------------
 PatchSet 4 
 Date:
 Author:
 Branch: HEAD
 Tag: (none) 
 Log:
 ci1
 
 Members: 
 	a:1.1->1.2 
 	b/c:1.2->1.3 
 
 ---------------------
 PatchSet 5 
 Date:
 Author:
 Branch: branch
 Tag: (none) 
 Log:
 ci2
 
 Members: 
 	b/c:1.1->1.1.2.1 
 
 ---------------------
 PatchSet 6 
 Date:
 Author:
 Branch: branch
 Tag: (none) 
 Log:
 funny
 ----------------------------
 log message
 
 Members: 
 	a:1.2->1.2.2.1 
 

tests/test-convert-cvs-synthetic.out

@@ -1,54 +1,54 b''
 % create cvs repository with one project
 cvs checkout: Updating proj
 % create file1 on the trunk
 cvs add: scheduling file `file1' for addition
 cvs add: use `cvs commit' to add this file permanently
 *REPO*/proj/file1,v  <--  file1
 initial revision: 1.1
 % create two branches
 cvs tag: Tagging .
 T file1
 cvs tag: Tagging .
 T file1
 % create file2 on branch v1_0
 cvs update: Updating .
 cvs add: scheduling file `file2' for addition on branch `v1_0'
 cvs add: use `cvs commit' to add this file permanently
 *REPO*/proj/Attic/file2,v  <--  file2
 new revision: 1.1.2.1; previous revision: 1.1
 % create file3, file4 on branch v1_1
 cvs update: Updating .
 cvs update: `file2' is no longer in the repository
 cvs add: scheduling file `file3' for addition on branch `v1_1'
 cvs add: scheduling file `file4' for addition on branch `v1_1'
 cvs add: use `cvs commit' to add these files permanently
 *REPO*/proj/Attic/file3,v  <--  file3
 new revision: 1.1.2.1; previous revision: 1.1
 *REPO*/proj/Attic/file4,v  <--  file4
 new revision: 1.1.2.1; previous revision: 1.1
 % merge file2 from v1_0 to v1_1
 cvs update: Updating .
 U file2
 cvs commit: Examining .
 *REPO*/proj/Attic/file2,v  <--  file2
 new revision: 1.1.4.2; previous revision: 1.1.4.1
 % convert to hg
 initializing destination proj.hg repository
+connecting to *REPO*
+scanning source...
 using builtin cvsps
 collecting CVS rlog
 9 log entries
 creating changesets
 4 changeset entries
-connecting to *REPO*
-scanning source...
 sorting...
 converting...
 3 add file1 on trunk
 2 add file2 on branch v1_0
 1 add file3, file4 on branch v1_1
 0 merge file2 from v1_0 to v1_1
 % hg log output
 3 merge file2 from v1_0 to v1_1
 2 add file3, file4 on branch v1_1
 1 add file2 on branch v1_0
 0 add file1 on trunk