##// END OF EJS Templates
convert: lowercase status and abort messages
Martin Geisler -
r16925:eaf6a6d7 default
parent child Browse files
Show More
@@ -1,470 +1,470 b''
1 1 # convcmd - convert extension commands definition
2 2 #
3 3 # Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
4 4 #
5 5 # This software may be used and distributed according to the terms of the
6 6 # GNU General Public License version 2 or any later version.
7 7
8 8 from common import NoRepo, MissingTool, SKIPREV, mapfile
9 9 from cvs import convert_cvs
10 10 from darcs import darcs_source
11 11 from git import convert_git
12 12 from hg import mercurial_source, mercurial_sink
13 13 from subversion import svn_source, svn_sink
14 14 from monotone import monotone_source
15 15 from gnuarch import gnuarch_source
16 16 from bzr import bzr_source
17 17 from p4 import p4_source
18 18 import filemap, common
19 19
20 20 import os, shutil
21 21 from mercurial import hg, util, encoding
22 22 from mercurial.i18n import _
23 23
24 24 orig_encoding = 'ascii'
25 25
26 26 def recode(s):
27 27 if isinstance(s, unicode):
28 28 return s.encode(orig_encoding, 'replace')
29 29 else:
30 30 return s.decode('utf-8').encode(orig_encoding, 'replace')
31 31
32 32 source_converters = [
33 33 ('cvs', convert_cvs, 'branchsort'),
34 34 ('git', convert_git, 'branchsort'),
35 35 ('svn', svn_source, 'branchsort'),
36 36 ('hg', mercurial_source, 'sourcesort'),
37 37 ('darcs', darcs_source, 'branchsort'),
38 38 ('mtn', monotone_source, 'branchsort'),
39 39 ('gnuarch', gnuarch_source, 'branchsort'),
40 40 ('bzr', bzr_source, 'branchsort'),
41 41 ('p4', p4_source, 'branchsort'),
42 42 ]
43 43
44 44 sink_converters = [
45 45 ('hg', mercurial_sink),
46 46 ('svn', svn_sink),
47 47 ]
48 48
49 49 def convertsource(ui, path, type, rev):
50 50 exceptions = []
51 51 if type and type not in [s[0] for s in source_converters]:
52 52 raise util.Abort(_('%s: invalid source repository type') % type)
53 53 for name, source, sortmode in source_converters:
54 54 try:
55 55 if not type or name == type:
56 56 return source(ui, path, rev), sortmode
57 57 except (NoRepo, MissingTool), inst:
58 58 exceptions.append(inst)
59 59 if not ui.quiet:
60 60 for inst in exceptions:
61 61 ui.write("%s\n" % inst)
62 62 raise util.Abort(_('%s: missing or unsupported repository') % path)
63 63
64 64 def convertsink(ui, path, type):
65 65 if type and type not in [s[0] for s in sink_converters]:
66 66 raise util.Abort(_('%s: invalid destination repository type') % type)
67 67 for name, sink in sink_converters:
68 68 try:
69 69 if not type or name == type:
70 70 return sink(ui, path)
71 71 except NoRepo, inst:
72 72 ui.note(_("convert: %s\n") % inst)
73 73 except MissingTool, inst:
74 74 raise util.Abort('%s\n' % inst)
75 75 raise util.Abort(_('%s: unknown repository type') % path)
76 76
77 77 class progresssource(object):
78 78 def __init__(self, ui, source, filecount):
79 79 self.ui = ui
80 80 self.source = source
81 81 self.filecount = filecount
82 82 self.retrieved = 0
83 83
84 84 def getfile(self, file, rev):
85 85 self.retrieved += 1
86 86 self.ui.progress(_('getting files'), self.retrieved,
87 87 item=file, total=self.filecount)
88 88 return self.source.getfile(file, rev)
89 89
90 90 def lookuprev(self, rev):
91 91 return self.source.lookuprev(rev)
92 92
93 93 def close(self):
94 94 self.ui.progress(_('getting files'), None)
95 95
96 96 class converter(object):
97 97 def __init__(self, ui, source, dest, revmapfile, opts):
98 98
99 99 self.source = source
100 100 self.dest = dest
101 101 self.ui = ui
102 102 self.opts = opts
103 103 self.commitcache = {}
104 104 self.authors = {}
105 105 self.authorfile = None
106 106
107 107 # Record converted revisions persistently: maps source revision
108 108 # ID to target revision ID (both strings). (This is how
109 109 # incremental conversions work.)
110 110 self.map = mapfile(ui, revmapfile)
111 111
112 112 # Read first the dst author map if any
113 113 authorfile = self.dest.authorfile()
114 114 if authorfile and os.path.exists(authorfile):
115 115 self.readauthormap(authorfile)
116 116 # Extend/Override with new author map if necessary
117 117 if opts.get('authormap'):
118 118 self.readauthormap(opts.get('authormap'))
119 119 self.authorfile = self.dest.authorfile()
120 120
121 121 self.splicemap = common.parsesplicemap(opts.get('splicemap'))
122 122 self.branchmap = mapfile(ui, opts.get('branchmap'))
123 123
124 124 def walktree(self, heads):
125 125 '''Return a mapping that identifies the uncommitted parents of every
126 126 uncommitted changeset.'''
127 127 visit = heads
128 128 known = set()
129 129 parents = {}
130 130 while visit:
131 131 n = visit.pop(0)
132 132 if n in known or n in self.map:
133 133 continue
134 134 known.add(n)
135 135 self.ui.progress(_('scanning'), len(known), unit=_('revisions'))
136 136 commit = self.cachecommit(n)
137 137 parents[n] = []
138 138 for p in commit.parents:
139 139 parents[n].append(p)
140 140 visit.append(p)
141 141 self.ui.progress(_('scanning'), None)
142 142
143 143 return parents
144 144
145 145 def mergesplicemap(self, parents, splicemap):
146 146 """A splicemap redefines child/parent relationships. Check the
147 147 map contains valid revision identifiers and merge the new
148 148 links in the source graph.
149 149 """
150 150 for c in splicemap:
151 151 if c not in parents:
152 152 if not self.dest.hascommit(self.map.get(c, c)):
153 153 # Could be in source but not converted during this run
154 154 self.ui.warn(_('splice map revision %s is not being '
155 155 'converted, ignoring\n') % c)
156 156 continue
157 157 pc = []
158 158 for p in splicemap[c]:
159 159 # We do not have to wait for nodes already in dest.
160 160 if self.dest.hascommit(self.map.get(p, p)):
161 161 continue
162 162 # Parent is not in dest and not being converted, not good
163 163 if p not in parents:
164 164 raise util.Abort(_('unknown splice map parent: %s') % p)
165 165 pc.append(p)
166 166 parents[c] = pc
167 167
168 168 def toposort(self, parents, sortmode):
169 169 '''Return an ordering such that every uncommitted changeset is
170 170 preceeded by all its uncommitted ancestors.'''
171 171
172 172 def mapchildren(parents):
173 173 """Return a (children, roots) tuple where 'children' maps parent
174 174 revision identifiers to children ones, and 'roots' is the list of
175 175 revisions without parents. 'parents' must be a mapping of revision
176 176 identifier to its parents ones.
177 177 """
178 178 visit = parents.keys()
179 179 seen = set()
180 180 children = {}
181 181 roots = []
182 182
183 183 while visit:
184 184 n = visit.pop(0)
185 185 if n in seen:
186 186 continue
187 187 seen.add(n)
188 188 # Ensure that nodes without parents are present in the
189 189 # 'children' mapping.
190 190 children.setdefault(n, [])
191 191 hasparent = False
192 192 for p in parents[n]:
193 193 if p not in self.map:
194 194 visit.append(p)
195 195 hasparent = True
196 196 children.setdefault(p, []).append(n)
197 197 if not hasparent:
198 198 roots.append(n)
199 199
200 200 return children, roots
201 201
202 202 # Sort functions are supposed to take a list of revisions which
203 203 # can be converted immediately and pick one
204 204
205 205 def makebranchsorter():
206 206 """If the previously converted revision has a child in the
207 207 eligible revisions list, pick it. Return the list head
208 208 otherwise. Branch sort attempts to minimize branch
209 209 switching, which is harmful for Mercurial backend
210 210 compression.
211 211 """
212 212 prev = [None]
213 213 def picknext(nodes):
214 214 next = nodes[0]
215 215 for n in nodes:
216 216 if prev[0] in parents[n]:
217 217 next = n
218 218 break
219 219 prev[0] = next
220 220 return next
221 221 return picknext
222 222
223 223 def makesourcesorter():
224 224 """Source specific sort."""
225 225 keyfn = lambda n: self.commitcache[n].sortkey
226 226 def picknext(nodes):
227 227 return sorted(nodes, key=keyfn)[0]
228 228 return picknext
229 229
230 230 def makedatesorter():
231 231 """Sort revisions by date."""
232 232 dates = {}
233 233 def getdate(n):
234 234 if n not in dates:
235 235 dates[n] = util.parsedate(self.commitcache[n].date)
236 236 return dates[n]
237 237
238 238 def picknext(nodes):
239 239 return min([(getdate(n), n) for n in nodes])[1]
240 240
241 241 return picknext
242 242
243 243 if sortmode == 'branchsort':
244 244 picknext = makebranchsorter()
245 245 elif sortmode == 'datesort':
246 246 picknext = makedatesorter()
247 247 elif sortmode == 'sourcesort':
248 248 picknext = makesourcesorter()
249 249 else:
250 250 raise util.Abort(_('unknown sort mode: %s') % sortmode)
251 251
252 252 children, actives = mapchildren(parents)
253 253
254 254 s = []
255 255 pendings = {}
256 256 while actives:
257 257 n = picknext(actives)
258 258 actives.remove(n)
259 259 s.append(n)
260 260
261 261 # Update dependents list
262 262 for c in children.get(n, []):
263 263 if c not in pendings:
264 264 pendings[c] = [p for p in parents[c] if p not in self.map]
265 265 try:
266 266 pendings[c].remove(n)
267 267 except ValueError:
268 268 raise util.Abort(_('cycle detected between %s and %s')
269 269 % (recode(c), recode(n)))
270 270 if not pendings[c]:
271 271 # Parents are converted, node is eligible
272 272 actives.insert(0, c)
273 273 pendings[c] = None
274 274
275 275 if len(s) != len(parents):
276 276 raise util.Abort(_("not all revisions were sorted"))
277 277
278 278 return s
279 279
280 280 def writeauthormap(self):
281 281 authorfile = self.authorfile
282 282 if authorfile:
283 self.ui.status(_('Writing author map file %s\n') % authorfile)
283 self.ui.status(_('writing author map file %s\n') % authorfile)
284 284 ofile = open(authorfile, 'w+')
285 285 for author in self.authors:
286 286 ofile.write("%s=%s\n" % (author, self.authors[author]))
287 287 ofile.close()
288 288
289 289 def readauthormap(self, authorfile):
290 290 afile = open(authorfile, 'r')
291 291 for line in afile:
292 292
293 293 line = line.strip()
294 294 if not line or line.startswith('#'):
295 295 continue
296 296
297 297 try:
298 298 srcauthor, dstauthor = line.split('=', 1)
299 299 except ValueError:
300 msg = _('Ignoring bad line in author map file %s: %s\n')
300 msg = _('ignoring bad line in author map file %s: %s\n')
301 301 self.ui.warn(msg % (authorfile, line.rstrip()))
302 302 continue
303 303
304 304 srcauthor = srcauthor.strip()
305 305 dstauthor = dstauthor.strip()
306 306 if self.authors.get(srcauthor) in (None, dstauthor):
307 307 msg = _('mapping author %s to %s\n')
308 308 self.ui.debug(msg % (srcauthor, dstauthor))
309 309 self.authors[srcauthor] = dstauthor
310 310 continue
311 311
312 312 m = _('overriding mapping for author %s, was %s, will be %s\n')
313 313 self.ui.status(m % (srcauthor, self.authors[srcauthor], dstauthor))
314 314
315 315 afile.close()
316 316
317 317 def cachecommit(self, rev):
318 318 commit = self.source.getcommit(rev)
319 319 commit.author = self.authors.get(commit.author, commit.author)
320 320 commit.branch = self.branchmap.get(commit.branch, commit.branch)
321 321 self.commitcache[rev] = commit
322 322 return commit
323 323
324 324 def copy(self, rev):
325 325 commit = self.commitcache[rev]
326 326
327 327 changes = self.source.getchanges(rev)
328 328 if isinstance(changes, basestring):
329 329 if changes == SKIPREV:
330 330 dest = SKIPREV
331 331 else:
332 332 dest = self.map[changes]
333 333 self.map[rev] = dest
334 334 return
335 335 files, copies = changes
336 336 pbranches = []
337 337 if commit.parents:
338 338 for prev in commit.parents:
339 339 if prev not in self.commitcache:
340 340 self.cachecommit(prev)
341 341 pbranches.append((self.map[prev],
342 342 self.commitcache[prev].branch))
343 343 self.dest.setbranch(commit.branch, pbranches)
344 344 try:
345 345 parents = self.splicemap[rev]
346 346 self.ui.status(_('spliced in %s as parents of %s\n') %
347 347 (parents, rev))
348 348 parents = [self.map.get(p, p) for p in parents]
349 349 except KeyError:
350 350 parents = [b[0] for b in pbranches]
351 351 source = progresssource(self.ui, self.source, len(files))
352 352 newnode = self.dest.putcommit(files, copies, parents, commit,
353 353 source, self.map)
354 354 source.close()
355 355 self.source.converted(rev, newnode)
356 356 self.map[rev] = newnode
357 357
358 358 def convert(self, sortmode):
359 359 try:
360 360 self.source.before()
361 361 self.dest.before()
362 362 self.source.setrevmap(self.map)
363 363 self.ui.status(_("scanning source...\n"))
364 364 heads = self.source.getheads()
365 365 parents = self.walktree(heads)
366 366 self.mergesplicemap(parents, self.splicemap)
367 367 self.ui.status(_("sorting...\n"))
368 368 t = self.toposort(parents, sortmode)
369 369 num = len(t)
370 370 c = None
371 371
372 372 self.ui.status(_("converting...\n"))
373 373 for i, c in enumerate(t):
374 374 num -= 1
375 375 desc = self.commitcache[c].desc
376 376 if "\n" in desc:
377 377 desc = desc.splitlines()[0]
378 378 # convert log message to local encoding without using
379 379 # tolocal() because the encoding.encoding convert()
380 380 # uses is 'utf-8'
381 381 self.ui.status("%d %s\n" % (num, recode(desc)))
382 382 self.ui.note(_("source: %s\n") % recode(c))
383 383 self.ui.progress(_('converting'), i, unit=_('revisions'),
384 384 total=len(t))
385 385 self.copy(c)
386 386 self.ui.progress(_('converting'), None)
387 387
388 388 tags = self.source.gettags()
389 389 ctags = {}
390 390 for k in tags:
391 391 v = tags[k]
392 392 if self.map.get(v, SKIPREV) != SKIPREV:
393 393 ctags[k] = self.map[v]
394 394
395 395 if c and ctags:
396 396 nrev, tagsparent = self.dest.puttags(ctags)
397 397 if nrev and tagsparent:
398 398 # write another hash correspondence to override the previous
399 399 # one so we don't end up with extra tag heads
400 400 tagsparents = [e for e in self.map.iteritems()
401 401 if e[1] == tagsparent]
402 402 if tagsparents:
403 403 self.map[tagsparents[0][0]] = nrev
404 404
405 405 bookmarks = self.source.getbookmarks()
406 406 cbookmarks = {}
407 407 for k in bookmarks:
408 408 v = bookmarks[k]
409 409 if self.map.get(v, SKIPREV) != SKIPREV:
410 410 cbookmarks[k] = self.map[v]
411 411
412 412 if c and cbookmarks:
413 413 self.dest.putbookmarks(cbookmarks)
414 414
415 415 self.writeauthormap()
416 416 finally:
417 417 self.cleanup()
418 418
419 419 def cleanup(self):
420 420 try:
421 421 self.dest.after()
422 422 finally:
423 423 self.source.after()
424 424 self.map.close()
425 425
426 426 def convert(ui, src, dest=None, revmapfile=None, **opts):
427 427 global orig_encoding
428 428 orig_encoding = encoding.encoding
429 429 encoding.encoding = 'UTF-8'
430 430
431 431 # support --authors as an alias for --authormap
432 432 if not opts.get('authormap'):
433 433 opts['authormap'] = opts.get('authors')
434 434
435 435 if not dest:
436 436 dest = hg.defaultdest(src) + "-hg"
437 437 ui.status(_("assuming destination %s\n") % dest)
438 438
439 439 destc = convertsink(ui, dest, opts.get('dest_type'))
440 440
441 441 try:
442 442 srcc, defaultsort = convertsource(ui, src, opts.get('source_type'),
443 443 opts.get('rev'))
444 444 except Exception:
445 445 for path in destc.created:
446 446 shutil.rmtree(path, True)
447 447 raise
448 448
449 449 sortmodes = ('branchsort', 'datesort', 'sourcesort')
450 450 sortmode = [m for m in sortmodes if opts.get(m)]
451 451 if len(sortmode) > 1:
452 452 raise util.Abort(_('more than one sort mode specified'))
453 453 sortmode = sortmode and sortmode[0] or defaultsort
454 454 if sortmode == 'sourcesort' and not srcc.hasnativeorder():
455 455 raise util.Abort(_('--sourcesort is not supported by this data source'))
456 456
457 457 fmap = opts.get('filemap')
458 458 if fmap:
459 459 srcc = filemap.filemap_source(ui, srcc, fmap)
460 460 destc.setfilemapmode(True)
461 461
462 462 if not revmapfile:
463 463 try:
464 464 revmapfile = destc.revmapfile()
465 465 except Exception:
466 466 revmapfile = os.path.join(destc, "map")
467 467
468 468 c = converter(ui, srcc, destc, revmapfile, opts)
469 469 c.convert(sortmode)
470 470
@@ -1,1252 +1,1252 b''
1 1 # Subversion 1.4/1.5 Python API backend
2 2 #
3 3 # Copyright(C) 2007 Daniel Holth et al
4 4
5 5 import os, re, sys, tempfile, urllib, urllib2, xml.dom.minidom
6 6 import cPickle as pickle
7 7
8 8 from mercurial import strutil, scmutil, util, encoding
9 9 from mercurial.i18n import _
10 10
11 11 propertycache = util.propertycache
12 12
13 13 # Subversion stuff. Works best with very recent Python SVN bindings
14 14 # e.g. SVN 1.5 or backports. Thanks to the bzr folks for enhancing
15 15 # these bindings.
16 16
17 17 from cStringIO import StringIO
18 18
19 19 from common import NoRepo, MissingTool, commit, encodeargs, decodeargs
20 20 from common import commandline, converter_source, converter_sink, mapfile
21 21
22 22 try:
23 23 from svn.core import SubversionException, Pool
24 24 import svn
25 25 import svn.client
26 26 import svn.core
27 27 import svn.ra
28 28 import svn.delta
29 29 import transport
30 30 import warnings
31 31 warnings.filterwarnings('ignore',
32 32 module='svn.core',
33 33 category=DeprecationWarning)
34 34
35 35 except ImportError:
36 36 svn = None
37 37
38 38 class SvnPathNotFound(Exception):
39 39 pass
40 40
41 41 def revsplit(rev):
42 42 """Parse a revision string and return (uuid, path, revnum)."""
43 43 url, revnum = rev.rsplit('@', 1)
44 44 parts = url.split('/', 1)
45 45 mod = ''
46 46 if len(parts) > 1:
47 47 mod = '/' + parts[1]
48 48 return parts[0][4:], mod, int(revnum)
49 49
50 50 def quote(s):
51 51 # As of svn 1.7, many svn calls expect "canonical" paths. In
52 52 # theory, we should call svn.core.*canonicalize() on all paths
53 53 # before passing them to the API. Instead, we assume the base url
54 54 # is canonical and copy the behaviour of svn URL encoding function
55 55 # so we can extend it safely with new components. The "safe"
56 56 # characters were taken from the "svn_uri__char_validity" table in
57 57 # libsvn_subr/path.c.
58 58 return urllib.quote(s, "!$&'()*+,-./:=@_~")
59 59
60 60 def geturl(path):
61 61 try:
62 62 return svn.client.url_from_path(svn.core.svn_path_canonicalize(path))
63 63 except SubversionException:
64 64 # svn.client.url_from_path() fails with local repositories
65 65 pass
66 66 if os.path.isdir(path):
67 67 path = os.path.normpath(os.path.abspath(path))
68 68 if os.name == 'nt':
69 69 path = '/' + util.normpath(path)
70 70 # Module URL is later compared with the repository URL returned
71 71 # by svn API, which is UTF-8.
72 72 path = encoding.tolocal(path)
73 73 path = 'file://%s' % quote(path)
74 74 return svn.core.svn_path_canonicalize(path)
75 75
76 76 def optrev(number):
77 77 optrev = svn.core.svn_opt_revision_t()
78 78 optrev.kind = svn.core.svn_opt_revision_number
79 79 optrev.value.number = number
80 80 return optrev
81 81
82 82 class changedpath(object):
83 83 def __init__(self, p):
84 84 self.copyfrom_path = p.copyfrom_path
85 85 self.copyfrom_rev = p.copyfrom_rev
86 86 self.action = p.action
87 87
88 88 def get_log_child(fp, url, paths, start, end, limit=0,
89 89 discover_changed_paths=True, strict_node_history=False):
90 90 protocol = -1
91 91 def receiver(orig_paths, revnum, author, date, message, pool):
92 92 if orig_paths is not None:
93 93 for k, v in orig_paths.iteritems():
94 94 orig_paths[k] = changedpath(v)
95 95 pickle.dump((orig_paths, revnum, author, date, message),
96 96 fp, protocol)
97 97
98 98 try:
99 99 # Use an ra of our own so that our parent can consume
100 100 # our results without confusing the server.
101 101 t = transport.SvnRaTransport(url=url)
102 102 svn.ra.get_log(t.ra, paths, start, end, limit,
103 103 discover_changed_paths,
104 104 strict_node_history,
105 105 receiver)
106 106 except IOError:
107 107 # Caller may interrupt the iteration
108 108 pickle.dump(None, fp, protocol)
109 109 except Exception, inst:
110 110 pickle.dump(str(inst), fp, protocol)
111 111 else:
112 112 pickle.dump(None, fp, protocol)
113 113 fp.close()
114 114 # With large history, cleanup process goes crazy and suddenly
115 115 # consumes *huge* amount of memory. The output file being closed,
116 116 # there is no need for clean termination.
117 117 os._exit(0)
118 118
119 119 def debugsvnlog(ui, **opts):
120 120 """Fetch SVN log in a subprocess and channel them back to parent to
121 121 avoid memory collection issues.
122 122 """
123 123 util.setbinary(sys.stdin)
124 124 util.setbinary(sys.stdout)
125 125 args = decodeargs(sys.stdin.read())
126 126 get_log_child(sys.stdout, *args)
127 127
128 128 class logstream(object):
129 129 """Interruptible revision log iterator."""
130 130 def __init__(self, stdout):
131 131 self._stdout = stdout
132 132
133 133 def __iter__(self):
134 134 while True:
135 135 try:
136 136 entry = pickle.load(self._stdout)
137 137 except EOFError:
138 138 raise util.Abort(_('Mercurial failed to run itself, check'
139 139 ' hg executable is in PATH'))
140 140 try:
141 141 orig_paths, revnum, author, date, message = entry
142 142 except (TypeError, ValueError):
143 143 if entry is None:
144 144 break
145 145 raise util.Abort(_("log stream exception '%s'") % entry)
146 146 yield entry
147 147
148 148 def close(self):
149 149 if self._stdout:
150 150 self._stdout.close()
151 151 self._stdout = None
152 152
153 153
154 154 # Check to see if the given path is a local Subversion repo. Verify this by
155 155 # looking for several svn-specific files and directories in the given
156 156 # directory.
157 157 def filecheck(ui, path, proto):
158 158 for x in ('locks', 'hooks', 'format', 'db'):
159 159 if not os.path.exists(os.path.join(path, x)):
160 160 return False
161 161 return True
162 162
163 163 # Check to see if a given path is the root of an svn repo over http. We verify
164 164 # this by requesting a version-controlled URL we know can't exist and looking
165 165 # for the svn-specific "not found" XML.
166 166 def httpcheck(ui, path, proto):
167 167 try:
168 168 opener = urllib2.build_opener()
169 169 rsp = opener.open('%s://%s/!svn/ver/0/.svn' % (proto, path))
170 170 data = rsp.read()
171 171 except urllib2.HTTPError, inst:
172 172 if inst.code != 404:
173 173 # Except for 404 we cannot know for sure this is not an svn repo
174 174 ui.warn(_('svn: cannot probe remote repository, assume it could '
175 175 'be a subversion repository. Use --source-type if you '
176 176 'know better.\n'))
177 177 return True
178 178 data = inst.fp.read()
179 179 except Exception:
180 180 # Could be urllib2.URLError if the URL is invalid or anything else.
181 181 return False
182 182 return '<m:human-readable errcode="160013">' in data
183 183
184 184 protomap = {'http': httpcheck,
185 185 'https': httpcheck,
186 186 'file': filecheck,
187 187 }
188 188 def issvnurl(ui, url):
189 189 try:
190 190 proto, path = url.split('://', 1)
191 191 if proto == 'file':
192 192 path = urllib.url2pathname(path)
193 193 except ValueError:
194 194 proto = 'file'
195 195 path = os.path.abspath(url)
196 196 if proto == 'file':
197 197 path = util.pconvert(path)
198 198 check = protomap.get(proto, lambda *args: False)
199 199 while '/' in path:
200 200 if check(ui, path, proto):
201 201 return True
202 202 path = path.rsplit('/', 1)[0]
203 203 return False
204 204
205 205 # SVN conversion code stolen from bzr-svn and tailor
206 206 #
207 207 # Subversion looks like a versioned filesystem, branches structures
208 208 # are defined by conventions and not enforced by the tool. First,
209 209 # we define the potential branches (modules) as "trunk" and "branches"
210 210 # children directories. Revisions are then identified by their
211 211 # module and revision number (and a repository identifier).
212 212 #
213 213 # The revision graph is really a tree (or a forest). By default, a
214 214 # revision parent is the previous revision in the same module. If the
215 215 # module directory is copied/moved from another module then the
216 216 # revision is the module root and its parent the source revision in
217 217 # the parent module. A revision has at most one parent.
218 218 #
219 219 class svn_source(converter_source):
220 220 def __init__(self, ui, url, rev=None):
221 221 super(svn_source, self).__init__(ui, url, rev=rev)
222 222
223 223 if not (url.startswith('svn://') or url.startswith('svn+ssh://') or
224 224 (os.path.exists(url) and
225 225 os.path.exists(os.path.join(url, '.svn'))) or
226 226 issvnurl(ui, url)):
227 227 raise NoRepo(_("%s does not look like a Subversion repository")
228 228 % url)
229 229 if svn is None:
230 raise MissingTool(_('Could not load Subversion python bindings'))
230 raise MissingTool(_('could not load Subversion python bindings'))
231 231
232 232 try:
233 233 version = svn.core.SVN_VER_MAJOR, svn.core.SVN_VER_MINOR
234 234 if version < (1, 4):
235 235 raise MissingTool(_('Subversion python bindings %d.%d found, '
236 236 '1.4 or later required') % version)
237 237 except AttributeError:
238 238 raise MissingTool(_('Subversion python bindings are too old, 1.4 '
239 239 'or later required'))
240 240
241 241 self.lastrevs = {}
242 242
243 243 latest = None
244 244 try:
245 245 # Support file://path@rev syntax. Useful e.g. to convert
246 246 # deleted branches.
247 247 at = url.rfind('@')
248 248 if at >= 0:
249 249 latest = int(url[at + 1:])
250 250 url = url[:at]
251 251 except ValueError:
252 252 pass
253 253 self.url = geturl(url)
254 254 self.encoding = 'UTF-8' # Subversion is always nominal UTF-8
255 255 try:
256 256 self.transport = transport.SvnRaTransport(url=self.url)
257 257 self.ra = self.transport.ra
258 258 self.ctx = self.transport.client
259 259 self.baseurl = svn.ra.get_repos_root(self.ra)
260 260 # Module is either empty or a repository path starting with
261 261 # a slash and not ending with a slash.
262 262 self.module = urllib.unquote(self.url[len(self.baseurl):])
263 263 self.prevmodule = None
264 264 self.rootmodule = self.module
265 265 self.commits = {}
266 266 self.paths = {}
267 267 self.uuid = svn.ra.get_uuid(self.ra)
268 268 except SubversionException:
269 269 ui.traceback()
270 270 raise NoRepo(_("%s does not look like a Subversion repository")
271 271 % self.url)
272 272
273 273 if rev:
274 274 try:
275 275 latest = int(rev)
276 276 except ValueError:
277 277 raise util.Abort(_('svn: revision %s is not an integer') % rev)
278 278
279 279 self.trunkname = self.ui.config('convert', 'svn.trunk',
280 280 'trunk').strip('/')
281 281 self.startrev = self.ui.config('convert', 'svn.startrev', default=0)
282 282 try:
283 283 self.startrev = int(self.startrev)
284 284 if self.startrev < 0:
285 285 self.startrev = 0
286 286 except ValueError:
287 287 raise util.Abort(_('svn: start revision %s is not an integer')
288 288 % self.startrev)
289 289
290 290 try:
291 291 self.head = self.latest(self.module, latest)
292 292 except SvnPathNotFound:
293 293 self.head = None
294 294 if not self.head:
295 295 raise util.Abort(_('no revision found in module %s')
296 296 % self.module)
297 297 self.last_changed = self.revnum(self.head)
298 298
299 299 self._changescache = None
300 300
301 301 if os.path.exists(os.path.join(url, '.svn/entries')):
302 302 self.wc = url
303 303 else:
304 304 self.wc = None
305 305 self.convertfp = None
306 306
307 307 def setrevmap(self, revmap):
308 308 lastrevs = {}
309 309 for revid in revmap.iterkeys():
310 310 uuid, module, revnum = revsplit(revid)
311 311 lastrevnum = lastrevs.setdefault(module, revnum)
312 312 if revnum > lastrevnum:
313 313 lastrevs[module] = revnum
314 314 self.lastrevs = lastrevs
315 315
316 316 def exists(self, path, optrev):
317 317 try:
318 318 svn.client.ls(self.url.rstrip('/') + '/' + quote(path),
319 319 optrev, False, self.ctx)
320 320 return True
321 321 except SubversionException:
322 322 return False
323 323
324 324 def getheads(self):
325 325
326 326 def isdir(path, revnum):
327 327 kind = self._checkpath(path, revnum)
328 328 return kind == svn.core.svn_node_dir
329 329
330 330 def getcfgpath(name, rev):
331 331 cfgpath = self.ui.config('convert', 'svn.' + name)
332 332 if cfgpath is not None and cfgpath.strip() == '':
333 333 return None
334 334 path = (cfgpath or name).strip('/')
335 335 if not self.exists(path, rev):
336 336 if self.module.endswith(path) and name == 'trunk':
337 337 # we are converting from inside this directory
338 338 return None
339 339 if cfgpath:
340 340 raise util.Abort(_('expected %s to be at %r, but not found')
341 341 % (name, path))
342 342 return None
343 343 self.ui.note(_('found %s at %r\n') % (name, path))
344 344 return path
345 345
346 346 rev = optrev(self.last_changed)
347 347 oldmodule = ''
348 348 trunk = getcfgpath('trunk', rev)
349 349 self.tags = getcfgpath('tags', rev)
350 350 branches = getcfgpath('branches', rev)
351 351
352 352 # If the project has a trunk or branches, we will extract heads
353 353 # from them. We keep the project root otherwise.
354 354 if trunk:
355 355 oldmodule = self.module or ''
356 356 self.module += '/' + trunk
357 357 self.head = self.latest(self.module, self.last_changed)
358 358 if not self.head:
359 359 raise util.Abort(_('no revision found in module %s')
360 360 % self.module)
361 361
362 362 # First head in the list is the module's head
363 363 self.heads = [self.head]
364 364 if self.tags is not None:
365 365 self.tags = '%s/%s' % (oldmodule , (self.tags or 'tags'))
366 366
367 367 # Check if branches bring a few more heads to the list
368 368 if branches:
369 369 rpath = self.url.strip('/')
370 370 branchnames = svn.client.ls(rpath + '/' + quote(branches),
371 371 rev, False, self.ctx)
372 372 for branch in branchnames.keys():
373 373 module = '%s/%s/%s' % (oldmodule, branches, branch)
374 374 if not isdir(module, self.last_changed):
375 375 continue
376 376 brevid = self.latest(module, self.last_changed)
377 377 if not brevid:
378 378 self.ui.note(_('ignoring empty branch %s\n') % branch)
379 379 continue
380 380 self.ui.note(_('found branch %s at %d\n') %
381 381 (branch, self.revnum(brevid)))
382 382 self.heads.append(brevid)
383 383
384 384 if self.startrev and self.heads:
385 385 if len(self.heads) > 1:
386 386 raise util.Abort(_('svn: start revision is not supported '
387 387 'with more than one branch'))
388 388 revnum = self.revnum(self.heads[0])
389 389 if revnum < self.startrev:
390 390 raise util.Abort(
391 391 _('svn: no revision found after start revision %d')
392 392 % self.startrev)
393 393
394 394 return self.heads
395 395
396 396 def getchanges(self, rev):
397 397 if self._changescache and self._changescache[0] == rev:
398 398 return self._changescache[1]
399 399 self._changescache = None
400 400 (paths, parents) = self.paths[rev]
401 401 if parents:
402 402 files, self.removed, copies = self.expandpaths(rev, paths, parents)
403 403 else:
404 404 # Perform a full checkout on roots
405 405 uuid, module, revnum = revsplit(rev)
406 406 entries = svn.client.ls(self.baseurl + quote(module),
407 407 optrev(revnum), True, self.ctx)
408 408 files = [n for n, e in entries.iteritems()
409 409 if e.kind == svn.core.svn_node_file]
410 410 copies = {}
411 411 self.removed = set()
412 412
413 413 files.sort()
414 414 files = zip(files, [rev] * len(files))
415 415
416 416 # caller caches the result, so free it here to release memory
417 417 del self.paths[rev]
418 418 return (files, copies)
419 419
420 420 def getchangedfiles(self, rev, i):
421 421 changes = self.getchanges(rev)
422 422 self._changescache = (rev, changes)
423 423 return [f[0] for f in changes[0]]
424 424
425 425 def getcommit(self, rev):
426 426 if rev not in self.commits:
427 427 uuid, module, revnum = revsplit(rev)
428 428 self.module = module
429 429 self.reparent(module)
430 430 # We assume that:
431 431 # - requests for revisions after "stop" come from the
432 432 # revision graph backward traversal. Cache all of them
433 433 # down to stop, they will be used eventually.
434 434 # - requests for revisions before "stop" come to get
435 435 # isolated branches parents. Just fetch what is needed.
436 436 stop = self.lastrevs.get(module, 0)
437 437 if revnum < stop:
438 438 stop = revnum + 1
439 439 self._fetch_revisions(revnum, stop)
440 440 if rev not in self.commits:
441 441 raise util.Abort(_('svn: revision %s not found') % revnum)
442 442 commit = self.commits[rev]
443 443 # caller caches the result, so free it here to release memory
444 444 del self.commits[rev]
445 445 return commit
446 446
447 447 def gettags(self):
448 448 tags = {}
449 449 if self.tags is None:
450 450 return tags
451 451
452 452 # svn tags are just a convention, project branches left in a
453 453 # 'tags' directory. There is no other relationship than
454 454 # ancestry, which is expensive to discover and makes them hard
455 455 # to update incrementally. Worse, past revisions may be
456 456 # referenced by tags far away in the future, requiring a deep
457 457 # history traversal on every calculation. Current code
458 458 # performs a single backward traversal, tracking moves within
459 459 # the tags directory (tag renaming) and recording a new tag
460 460 # everytime a project is copied from outside the tags
461 461 # directory. It also lists deleted tags, this behaviour may
462 462 # change in the future.
463 463 pendings = []
464 464 tagspath = self.tags
465 465 start = svn.ra.get_latest_revnum(self.ra)
466 466 stream = self._getlog([self.tags], start, self.startrev)
467 467 try:
468 468 for entry in stream:
469 469 origpaths, revnum, author, date, message = entry
470 470 copies = [(e.copyfrom_path, e.copyfrom_rev, p) for p, e
471 471 in origpaths.iteritems() if e.copyfrom_path]
472 472 # Apply moves/copies from more specific to general
473 473 copies.sort(reverse=True)
474 474
475 475 srctagspath = tagspath
476 476 if copies and copies[-1][2] == tagspath:
477 477 # Track tags directory moves
478 478 srctagspath = copies.pop()[0]
479 479
480 480 for source, sourcerev, dest in copies:
481 481 if not dest.startswith(tagspath + '/'):
482 482 continue
483 483 for tag in pendings:
484 484 if tag[0].startswith(dest):
485 485 tagpath = source + tag[0][len(dest):]
486 486 tag[:2] = [tagpath, sourcerev]
487 487 break
488 488 else:
489 489 pendings.append([source, sourcerev, dest])
490 490
491 491 # Filter out tags with children coming from different
492 492 # parts of the repository like:
493 493 # /tags/tag.1 (from /trunk:10)
494 494 # /tags/tag.1/foo (from /branches/foo:12)
495 495 # Here/tags/tag.1 discarded as well as its children.
496 496 # It happens with tools like cvs2svn. Such tags cannot
497 497 # be represented in mercurial.
498 498 addeds = dict((p, e.copyfrom_path) for p, e
499 499 in origpaths.iteritems()
500 500 if e.action == 'A' and e.copyfrom_path)
501 501 badroots = set()
502 502 for destroot in addeds:
503 503 for source, sourcerev, dest in pendings:
504 504 if (not dest.startswith(destroot + '/')
505 505 or source.startswith(addeds[destroot] + '/')):
506 506 continue
507 507 badroots.add(destroot)
508 508 break
509 509
510 510 for badroot in badroots:
511 511 pendings = [p for p in pendings if p[2] != badroot
512 512 and not p[2].startswith(badroot + '/')]
513 513
514 514 # Tell tag renamings from tag creations
515 515 renamings = []
516 516 for source, sourcerev, dest in pendings:
517 517 tagname = dest.split('/')[-1]
518 518 if source.startswith(srctagspath):
519 519 renamings.append([source, sourcerev, tagname])
520 520 continue
521 521 if tagname in tags:
522 522 # Keep the latest tag value
523 523 continue
524 524 # From revision may be fake, get one with changes
525 525 try:
526 526 tagid = self.latest(source, sourcerev)
527 527 if tagid and tagname not in tags:
528 528 tags[tagname] = tagid
529 529 except SvnPathNotFound:
530 530 # It happens when we are following directories
531 531 # we assumed were copied with their parents
532 532 # but were really created in the tag
533 533 # directory.
534 534 pass
535 535 pendings = renamings
536 536 tagspath = srctagspath
537 537 finally:
538 538 stream.close()
539 539 return tags
540 540
541 541 def converted(self, rev, destrev):
542 542 if not self.wc:
543 543 return
544 544 if self.convertfp is None:
545 545 self.convertfp = open(os.path.join(self.wc, '.svn', 'hg-shamap'),
546 546 'a')
547 547 self.convertfp.write('%s %d\n' % (destrev, self.revnum(rev)))
548 548 self.convertfp.flush()
549 549
550 550 def revid(self, revnum, module=None):
551 551 return 'svn:%s%s@%s' % (self.uuid, module or self.module, revnum)
552 552
553 553 def revnum(self, rev):
554 554 return int(rev.split('@')[-1])
555 555
556 556 def latest(self, path, stop=None):
557 557 """Find the latest revid affecting path, up to stop revision
558 558 number. If stop is None, default to repository latest
559 559 revision. It may return a revision in a different module,
560 560 since a branch may be moved without a change being
561 561 reported. Return None if computed module does not belong to
562 562 rootmodule subtree.
563 563 """
564 564 def findchanges(path, start, stop=None):
565 565 stream = self._getlog([path], start, stop or 1)
566 566 try:
567 567 for entry in stream:
568 568 paths, revnum, author, date, message = entry
569 569 if stop is None and paths:
570 570 # We do not know the latest changed revision,
571 571 # keep the first one with changed paths.
572 572 break
573 573 if revnum <= stop:
574 574 break
575 575
576 576 for p in paths:
577 577 if (not path.startswith(p) or
578 578 not paths[p].copyfrom_path):
579 579 continue
580 580 newpath = paths[p].copyfrom_path + path[len(p):]
581 581 self.ui.debug("branch renamed from %s to %s at %d\n" %
582 582 (path, newpath, revnum))
583 583 path = newpath
584 584 break
585 585 if not paths:
586 586 revnum = None
587 587 return revnum, path
588 588 finally:
589 589 stream.close()
590 590
591 591 if not path.startswith(self.rootmodule):
592 592 # Requests on foreign branches may be forbidden at server level
593 593 self.ui.debug('ignoring foreign branch %r\n' % path)
594 594 return None
595 595
596 596 if stop is None:
597 597 stop = svn.ra.get_latest_revnum(self.ra)
598 598 try:
599 599 prevmodule = self.reparent('')
600 600 dirent = svn.ra.stat(self.ra, path.strip('/'), stop)
601 601 self.reparent(prevmodule)
602 602 except SubversionException:
603 603 dirent = None
604 604 if not dirent:
605 605 raise SvnPathNotFound(_('%s not found up to revision %d')
606 606 % (path, stop))
607 607
608 608 # stat() gives us the previous revision on this line of
609 609 # development, but it might be in *another module*. Fetch the
610 610 # log and detect renames down to the latest revision.
611 611 revnum, realpath = findchanges(path, stop, dirent.created_rev)
612 612 if revnum is None:
613 613 # Tools like svnsync can create empty revision, when
614 614 # synchronizing only a subtree for instance. These empty
615 615 # revisions created_rev still have their original values
616 616 # despite all changes having disappeared and can be
617 617 # returned by ra.stat(), at least when stating the root
618 618 # module. In that case, do not trust created_rev and scan
619 619 # the whole history.
620 620 revnum, realpath = findchanges(path, stop)
621 621 if revnum is None:
622 622 self.ui.debug('ignoring empty branch %r\n' % realpath)
623 623 return None
624 624
625 625 if not realpath.startswith(self.rootmodule):
626 626 self.ui.debug('ignoring foreign branch %r\n' % realpath)
627 627 return None
628 628 return self.revid(revnum, realpath)
629 629
630 630 def reparent(self, module):
631 631 """Reparent the svn transport and return the previous parent."""
632 632 if self.prevmodule == module:
633 633 return module
634 634 svnurl = self.baseurl + quote(module)
635 635 prevmodule = self.prevmodule
636 636 if prevmodule is None:
637 637 prevmodule = ''
638 638 self.ui.debug("reparent to %s\n" % svnurl)
639 639 svn.ra.reparent(self.ra, svnurl)
640 640 self.prevmodule = module
641 641 return prevmodule
642 642
643 643 def expandpaths(self, rev, paths, parents):
644 644 changed, removed = set(), set()
645 645 copies = {}
646 646
647 647 new_module, revnum = revsplit(rev)[1:]
648 648 if new_module != self.module:
649 649 self.module = new_module
650 650 self.reparent(self.module)
651 651
652 652 for i, (path, ent) in enumerate(paths):
653 653 self.ui.progress(_('scanning paths'), i, item=path,
654 654 total=len(paths))
655 655 entrypath = self.getrelpath(path)
656 656
657 657 kind = self._checkpath(entrypath, revnum)
658 658 if kind == svn.core.svn_node_file:
659 659 changed.add(self.recode(entrypath))
660 660 if not ent.copyfrom_path or not parents:
661 661 continue
662 662 # Copy sources not in parent revisions cannot be
663 663 # represented, ignore their origin for now
664 664 pmodule, prevnum = revsplit(parents[0])[1:]
665 665 if ent.copyfrom_rev < prevnum:
666 666 continue
667 667 copyfrom_path = self.getrelpath(ent.copyfrom_path, pmodule)
668 668 if not copyfrom_path:
669 669 continue
670 670 self.ui.debug("copied to %s from %s@%s\n" %
671 671 (entrypath, copyfrom_path, ent.copyfrom_rev))
672 672 copies[self.recode(entrypath)] = self.recode(copyfrom_path)
673 673 elif kind == 0: # gone, but had better be a deleted *file*
674 674 self.ui.debug("gone from %s\n" % ent.copyfrom_rev)
675 675 pmodule, prevnum = revsplit(parents[0])[1:]
676 676 parentpath = pmodule + "/" + entrypath
677 677 fromkind = self._checkpath(entrypath, prevnum, pmodule)
678 678
679 679 if fromkind == svn.core.svn_node_file:
680 680 removed.add(self.recode(entrypath))
681 681 elif fromkind == svn.core.svn_node_dir:
682 682 oroot = parentpath.strip('/')
683 683 nroot = path.strip('/')
684 684 children = self._iterfiles(oroot, prevnum)
685 685 for childpath in children:
686 686 childpath = childpath.replace(oroot, nroot)
687 687 childpath = self.getrelpath("/" + childpath, pmodule)
688 688 if childpath:
689 689 removed.add(self.recode(childpath))
690 690 else:
691 691 self.ui.debug('unknown path in revision %d: %s\n' % \
692 692 (revnum, path))
693 693 elif kind == svn.core.svn_node_dir:
694 694 if ent.action == 'M':
695 695 # If the directory just had a prop change,
696 696 # then we shouldn't need to look for its children.
697 697 continue
698 698 if ent.action == 'R' and parents:
699 699 # If a directory is replacing a file, mark the previous
700 700 # file as deleted
701 701 pmodule, prevnum = revsplit(parents[0])[1:]
702 702 pkind = self._checkpath(entrypath, prevnum, pmodule)
703 703 if pkind == svn.core.svn_node_file:
704 704 removed.add(self.recode(entrypath))
705 705 elif pkind == svn.core.svn_node_dir:
706 706 # We do not know what files were kept or removed,
707 707 # mark them all as changed.
708 708 for childpath in self._iterfiles(pmodule, prevnum):
709 709 childpath = self.getrelpath("/" + childpath)
710 710 if childpath:
711 711 changed.add(self.recode(childpath))
712 712
713 713 for childpath in self._iterfiles(path, revnum):
714 714 childpath = self.getrelpath("/" + childpath)
715 715 if childpath:
716 716 changed.add(self.recode(childpath))
717 717
718 718 # Handle directory copies
719 719 if not ent.copyfrom_path or not parents:
720 720 continue
721 721 # Copy sources not in parent revisions cannot be
722 722 # represented, ignore their origin for now
723 723 pmodule, prevnum = revsplit(parents[0])[1:]
724 724 if ent.copyfrom_rev < prevnum:
725 725 continue
726 726 copyfrompath = self.getrelpath(ent.copyfrom_path, pmodule)
727 727 if not copyfrompath:
728 728 continue
729 729 self.ui.debug("mark %s came from %s:%d\n"
730 730 % (path, copyfrompath, ent.copyfrom_rev))
731 731 children = self._iterfiles(ent.copyfrom_path, ent.copyfrom_rev)
732 732 for childpath in children:
733 733 childpath = self.getrelpath("/" + childpath, pmodule)
734 734 if not childpath:
735 735 continue
736 736 copytopath = path + childpath[len(copyfrompath):]
737 737 copytopath = self.getrelpath(copytopath)
738 738 copies[self.recode(copytopath)] = self.recode(childpath)
739 739
740 740 self.ui.progress(_('scanning paths'), None)
741 741 changed.update(removed)
742 742 return (list(changed), removed, copies)
743 743
744 744 def _fetch_revisions(self, from_revnum, to_revnum):
745 745 if from_revnum < to_revnum:
746 746 from_revnum, to_revnum = to_revnum, from_revnum
747 747
748 748 self.child_cset = None
749 749
750 750 def parselogentry(orig_paths, revnum, author, date, message):
751 751 """Return the parsed commit object or None, and True if
752 752 the revision is a branch root.
753 753 """
754 754 self.ui.debug("parsing revision %d (%d changes)\n" %
755 755 (revnum, len(orig_paths)))
756 756
757 757 branched = False
758 758 rev = self.revid(revnum)
759 759 # branch log might return entries for a parent we already have
760 760
761 761 if rev in self.commits or revnum < to_revnum:
762 762 return None, branched
763 763
764 764 parents = []
765 765 # check whether this revision is the start of a branch or part
766 766 # of a branch renaming
767 767 orig_paths = sorted(orig_paths.iteritems())
768 768 root_paths = [(p, e) for p, e in orig_paths
769 769 if self.module.startswith(p)]
770 770 if root_paths:
771 771 path, ent = root_paths[-1]
772 772 if ent.copyfrom_path:
773 773 branched = True
774 774 newpath = ent.copyfrom_path + self.module[len(path):]
775 775 # ent.copyfrom_rev may not be the actual last revision
776 776 previd = self.latest(newpath, ent.copyfrom_rev)
777 777 if previd is not None:
778 778 prevmodule, prevnum = revsplit(previd)[1:]
779 779 if prevnum >= self.startrev:
780 780 parents = [previd]
781 781 self.ui.note(
782 782 _('found parent of branch %s at %d: %s\n') %
783 783 (self.module, prevnum, prevmodule))
784 784 else:
785 785 self.ui.debug("no copyfrom path, don't know what to do.\n")
786 786
787 787 paths = []
788 788 # filter out unrelated paths
789 789 for path, ent in orig_paths:
790 790 if self.getrelpath(path) is None:
791 791 continue
792 792 paths.append((path, ent))
793 793
794 794 # Example SVN datetime. Includes microseconds.
795 795 # ISO-8601 conformant
796 796 # '2007-01-04T17:35:00.902377Z'
797 797 date = util.parsedate(date[:19] + " UTC", ["%Y-%m-%dT%H:%M:%S"])
798 798
799 799 log = message and self.recode(message) or ''
800 800 author = author and self.recode(author) or ''
801 801 try:
802 802 branch = self.module.split("/")[-1]
803 803 if branch == self.trunkname:
804 804 branch = None
805 805 except IndexError:
806 806 branch = None
807 807
808 808 cset = commit(author=author,
809 809 date=util.datestr(date, '%Y-%m-%d %H:%M:%S %1%2'),
810 810 desc=log,
811 811 parents=parents,
812 812 branch=branch,
813 813 rev=rev)
814 814
815 815 self.commits[rev] = cset
816 816 # The parents list is *shared* among self.paths and the
817 817 # commit object. Both will be updated below.
818 818 self.paths[rev] = (paths, cset.parents)
819 819 if self.child_cset and not self.child_cset.parents:
820 820 self.child_cset.parents[:] = [rev]
821 821 self.child_cset = cset
822 822 return cset, branched
823 823
824 824 self.ui.note(_('fetching revision log for "%s" from %d to %d\n') %
825 825 (self.module, from_revnum, to_revnum))
826 826
827 827 try:
828 828 firstcset = None
829 829 lastonbranch = False
830 830 stream = self._getlog([self.module], from_revnum, to_revnum)
831 831 try:
832 832 for entry in stream:
833 833 paths, revnum, author, date, message = entry
834 834 if revnum < self.startrev:
835 835 lastonbranch = True
836 836 break
837 837 if not paths:
838 838 self.ui.debug('revision %d has no entries\n' % revnum)
839 839 # If we ever leave the loop on an empty
840 840 # revision, do not try to get a parent branch
841 841 lastonbranch = lastonbranch or revnum == 0
842 842 continue
843 843 cset, lastonbranch = parselogentry(paths, revnum, author,
844 844 date, message)
845 845 if cset:
846 846 firstcset = cset
847 847 if lastonbranch:
848 848 break
849 849 finally:
850 850 stream.close()
851 851
852 852 if not lastonbranch and firstcset and not firstcset.parents:
853 853 # The first revision of the sequence (the last fetched one)
854 854 # has invalid parents if not a branch root. Find the parent
855 855 # revision now, if any.
856 856 try:
857 857 firstrevnum = self.revnum(firstcset.rev)
858 858 if firstrevnum > 1:
859 859 latest = self.latest(self.module, firstrevnum - 1)
860 860 if latest:
861 861 firstcset.parents.append(latest)
862 862 except SvnPathNotFound:
863 863 pass
864 864 except SubversionException, (inst, num):
865 865 if num == svn.core.SVN_ERR_FS_NO_SUCH_REVISION:
866 866 raise util.Abort(_('svn: branch has no revision %s')
867 867 % to_revnum)
868 868 raise
869 869
870 870 def getfile(self, file, rev):
871 871 # TODO: ra.get_file transmits the whole file instead of diffs.
872 872 if file in self.removed:
873 873 raise IOError
874 874 mode = ''
875 875 try:
876 876 new_module, revnum = revsplit(rev)[1:]
877 877 if self.module != new_module:
878 878 self.module = new_module
879 879 self.reparent(self.module)
880 880 io = StringIO()
881 881 info = svn.ra.get_file(self.ra, file, revnum, io)
882 882 data = io.getvalue()
883 883 # ra.get_files() seems to keep a reference on the input buffer
884 884 # preventing collection. Release it explicitely.
885 885 io.close()
886 886 if isinstance(info, list):
887 887 info = info[-1]
888 888 mode = ("svn:executable" in info) and 'x' or ''
889 889 mode = ("svn:special" in info) and 'l' or mode
890 890 except SubversionException, e:
891 891 notfound = (svn.core.SVN_ERR_FS_NOT_FOUND,
892 892 svn.core.SVN_ERR_RA_DAV_PATH_NOT_FOUND)
893 893 if e.apr_err in notfound: # File not found
894 894 raise IOError
895 895 raise
896 896 if mode == 'l':
897 897 link_prefix = "link "
898 898 if data.startswith(link_prefix):
899 899 data = data[len(link_prefix):]
900 900 return data, mode
901 901
902 902 def _iterfiles(self, path, revnum):
903 903 """Enumerate all files in path at revnum, recursively."""
904 904 path = path.strip('/')
905 905 pool = Pool()
906 906 rpath = '/'.join([self.baseurl, quote(path)]).strip('/')
907 907 entries = svn.client.ls(rpath, optrev(revnum), True, self.ctx, pool)
908 908 if path:
909 909 path += '/'
910 910 return ((path + p) for p, e in entries.iteritems()
911 911 if e.kind == svn.core.svn_node_file)
912 912
913 913 def getrelpath(self, path, module=None):
914 914 if module is None:
915 915 module = self.module
916 916 # Given the repository url of this wc, say
917 917 # "http://server/plone/CMFPlone/branches/Plone-2_0-branch"
918 918 # extract the "entry" portion (a relative path) from what
919 919 # svn log --xml says, ie
920 920 # "/CMFPlone/branches/Plone-2_0-branch/tests/PloneTestCase.py"
921 921 # that is to say "tests/PloneTestCase.py"
922 922 if path.startswith(module):
923 923 relative = path.rstrip('/')[len(module):]
924 924 if relative.startswith('/'):
925 925 return relative[1:]
926 926 elif relative == '':
927 927 return relative
928 928
929 929 # The path is outside our tracked tree...
930 930 self.ui.debug('%r is not under %r, ignoring\n' % (path, module))
931 931 return None
932 932
933 933 def _checkpath(self, path, revnum, module=None):
934 934 if module is not None:
935 935 prevmodule = self.reparent('')
936 936 path = module + '/' + path
937 937 try:
938 938 # ra.check_path does not like leading slashes very much, it leads
939 939 # to PROPFIND subversion errors
940 940 return svn.ra.check_path(self.ra, path.strip('/'), revnum)
941 941 finally:
942 942 if module is not None:
943 943 self.reparent(prevmodule)
944 944
945 945 def _getlog(self, paths, start, end, limit=0, discover_changed_paths=True,
946 946 strict_node_history=False):
947 947 # Normalize path names, svn >= 1.5 only wants paths relative to
948 948 # supplied URL
949 949 relpaths = []
950 950 for p in paths:
951 951 if not p.startswith('/'):
952 952 p = self.module + '/' + p
953 953 relpaths.append(p.strip('/'))
954 954 args = [self.baseurl, relpaths, start, end, limit,
955 955 discover_changed_paths, strict_node_history]
956 956 arg = encodeargs(args)
957 957 hgexe = util.hgexecutable()
958 958 cmd = '%s debugsvnlog' % util.shellquote(hgexe)
959 959 stdin, stdout = util.popen2(util.quotecommand(cmd))
960 960 stdin.write(arg)
961 961 try:
962 962 stdin.close()
963 963 except IOError:
964 964 raise util.Abort(_('Mercurial failed to run itself, check'
965 965 ' hg executable is in PATH'))
966 966 return logstream(stdout)
967 967
968 968 pre_revprop_change = '''#!/bin/sh
969 969
970 970 REPOS="$1"
971 971 REV="$2"
972 972 USER="$3"
973 973 PROPNAME="$4"
974 974 ACTION="$5"
975 975
976 976 if [ "$ACTION" = "M" -a "$PROPNAME" = "svn:log" ]; then exit 0; fi
977 977 if [ "$ACTION" = "A" -a "$PROPNAME" = "hg:convert-branch" ]; then exit 0; fi
978 978 if [ "$ACTION" = "A" -a "$PROPNAME" = "hg:convert-rev" ]; then exit 0; fi
979 979
980 980 echo "Changing prohibited revision property" >&2
981 981 exit 1
982 982 '''
983 983
984 984 class svn_sink(converter_sink, commandline):
985 985 commit_re = re.compile(r'Committed revision (\d+).', re.M)
986 986 uuid_re = re.compile(r'Repository UUID:\s*(\S+)', re.M)
987 987
988 988 def prerun(self):
989 989 if self.wc:
990 990 os.chdir(self.wc)
991 991
992 992 def postrun(self):
993 993 if self.wc:
994 994 os.chdir(self.cwd)
995 995
996 996 def join(self, name):
997 997 return os.path.join(self.wc, '.svn', name)
998 998
999 999 def revmapfile(self):
1000 1000 return self.join('hg-shamap')
1001 1001
1002 1002 def authorfile(self):
1003 1003 return self.join('hg-authormap')
1004 1004
1005 1005 def __init__(self, ui, path):
1006 1006
1007 1007 converter_sink.__init__(self, ui, path)
1008 1008 commandline.__init__(self, ui, 'svn')
1009 1009 self.delete = []
1010 1010 self.setexec = []
1011 1011 self.delexec = []
1012 1012 self.copies = []
1013 1013 self.wc = None
1014 1014 self.cwd = os.getcwd()
1015 1015
1016 1016 path = os.path.realpath(path)
1017 1017
1018 1018 created = False
1019 1019 if os.path.isfile(os.path.join(path, '.svn', 'entries')):
1020 1020 self.wc = path
1021 1021 self.run0('update')
1022 1022 else:
1023 1023 wcpath = os.path.join(os.getcwd(), os.path.basename(path) + '-wc')
1024 1024
1025 1025 if os.path.isdir(os.path.dirname(path)):
1026 1026 if not os.path.exists(os.path.join(path, 'db', 'fs-type')):
1027 1027 ui.status(_('initializing svn repository %r\n') %
1028 1028 os.path.basename(path))
1029 1029 commandline(ui, 'svnadmin').run0('create', path)
1030 1030 created = path
1031 1031 path = util.normpath(path)
1032 1032 if not path.startswith('/'):
1033 1033 path = '/' + path
1034 1034 path = 'file://' + path
1035 1035
1036 1036 ui.status(_('initializing svn working copy %r\n')
1037 1037 % os.path.basename(wcpath))
1038 1038 self.run0('checkout', path, wcpath)
1039 1039
1040 1040 self.wc = wcpath
1041 1041 self.opener = scmutil.opener(self.wc)
1042 1042 self.wopener = scmutil.opener(self.wc)
1043 1043 self.childmap = mapfile(ui, self.join('hg-childmap'))
1044 1044 self.is_exec = util.checkexec(self.wc) and util.isexec or None
1045 1045
1046 1046 if created:
1047 1047 hook = os.path.join(created, 'hooks', 'pre-revprop-change')
1048 1048 fp = open(hook, 'w')
1049 1049 fp.write(pre_revprop_change)
1050 1050 fp.close()
1051 1051 util.setflags(hook, False, True)
1052 1052
1053 1053 output = self.run0('info')
1054 1054 self.uuid = self.uuid_re.search(output).group(1).strip()
1055 1055
1056 1056 def wjoin(self, *names):
1057 1057 return os.path.join(self.wc, *names)
1058 1058
1059 1059 @propertycache
1060 1060 def manifest(self):
1061 1061 # As of svn 1.7, the "add" command fails when receiving
1062 1062 # already tracked entries, so we have to track and filter them
1063 1063 # ourselves.
1064 1064 m = set()
1065 1065 output = self.run0('ls', recursive=True, xml=True)
1066 1066 doc = xml.dom.minidom.parseString(output)
1067 1067 for e in doc.getElementsByTagName('entry'):
1068 1068 for n in e.childNodes:
1069 1069 if n.nodeType != n.ELEMENT_NODE or n.tagName != 'name':
1070 1070 continue
1071 1071 name = ''.join(c.data for c in n.childNodes
1072 1072 if c.nodeType == c.TEXT_NODE)
1073 1073 # Entries are compared with names coming from
1074 1074 # mercurial, so bytes with undefined encoding. Our
1075 1075 # best bet is to assume they are in local
1076 1076 # encoding. They will be passed to command line calls
1077 1077 # later anyway, so they better be.
1078 1078 m.add(encoding.tolocal(name.encode('utf-8')))
1079 1079 break
1080 1080 return m
1081 1081
1082 1082 def putfile(self, filename, flags, data):
1083 1083 if 'l' in flags:
1084 1084 self.wopener.symlink(data, filename)
1085 1085 else:
1086 1086 try:
1087 1087 if os.path.islink(self.wjoin(filename)):
1088 1088 os.unlink(filename)
1089 1089 except OSError:
1090 1090 pass
1091 1091 self.wopener.write(filename, data)
1092 1092
1093 1093 if self.is_exec:
1094 1094 was_exec = self.is_exec(self.wjoin(filename))
1095 1095 else:
1096 1096 # On filesystems not supporting execute-bit, there is no way
1097 1097 # to know if it is set but asking subversion. Setting it
1098 1098 # systematically is just as expensive and much simpler.
1099 1099 was_exec = 'x' not in flags
1100 1100
1101 1101 util.setflags(self.wjoin(filename), False, 'x' in flags)
1102 1102 if was_exec:
1103 1103 if 'x' not in flags:
1104 1104 self.delexec.append(filename)
1105 1105 else:
1106 1106 if 'x' in flags:
1107 1107 self.setexec.append(filename)
1108 1108
1109 1109 def _copyfile(self, source, dest):
1110 1110 # SVN's copy command pukes if the destination file exists, but
1111 1111 # our copyfile method expects to record a copy that has
1112 1112 # already occurred. Cross the semantic gap.
1113 1113 wdest = self.wjoin(dest)
1114 1114 exists = os.path.lexists(wdest)
1115 1115 if exists:
1116 1116 fd, tempname = tempfile.mkstemp(
1117 1117 prefix='hg-copy-', dir=os.path.dirname(wdest))
1118 1118 os.close(fd)
1119 1119 os.unlink(tempname)
1120 1120 os.rename(wdest, tempname)
1121 1121 try:
1122 1122 self.run0('copy', source, dest)
1123 1123 finally:
1124 1124 self.manifest.add(dest)
1125 1125 if exists:
1126 1126 try:
1127 1127 os.unlink(wdest)
1128 1128 except OSError:
1129 1129 pass
1130 1130 os.rename(tempname, wdest)
1131 1131
1132 1132 def dirs_of(self, files):
1133 1133 dirs = set()
1134 1134 for f in files:
1135 1135 if os.path.isdir(self.wjoin(f)):
1136 1136 dirs.add(f)
1137 1137 for i in strutil.rfindall(f, '/'):
1138 1138 dirs.add(f[:i])
1139 1139 return dirs
1140 1140
1141 1141 def add_dirs(self, files):
1142 1142 add_dirs = [d for d in sorted(self.dirs_of(files))
1143 1143 if d not in self.manifest]
1144 1144 if add_dirs:
1145 1145 self.manifest.update(add_dirs)
1146 1146 self.xargs(add_dirs, 'add', non_recursive=True, quiet=True)
1147 1147 return add_dirs
1148 1148
1149 1149 def add_files(self, files):
1150 1150 files = [f for f in files if f not in self.manifest]
1151 1151 if files:
1152 1152 self.manifest.update(files)
1153 1153 self.xargs(files, 'add', quiet=True)
1154 1154 return files
1155 1155
1156 1156 def tidy_dirs(self, names):
1157 1157 deleted = []
1158 1158 for d in sorted(self.dirs_of(names), reverse=True):
1159 1159 wd = self.wjoin(d)
1160 1160 if os.listdir(wd) == '.svn':
1161 1161 self.run0('delete', d)
1162 1162 self.manifest.remove(d)
1163 1163 deleted.append(d)
1164 1164 return deleted
1165 1165
1166 1166 def addchild(self, parent, child):
1167 1167 self.childmap[parent] = child
1168 1168
1169 1169 def revid(self, rev):
1170 1170 return u"svn:%s@%s" % (self.uuid, rev)
1171 1171
1172 1172 def putcommit(self, files, copies, parents, commit, source, revmap):
1173 1173 for parent in parents:
1174 1174 try:
1175 1175 return self.revid(self.childmap[parent])
1176 1176 except KeyError:
1177 1177 pass
1178 1178
1179 1179 # Apply changes to working copy
1180 1180 for f, v in files:
1181 1181 try:
1182 1182 data, mode = source.getfile(f, v)
1183 1183 except IOError:
1184 1184 self.delete.append(f)
1185 1185 else:
1186 1186 self.putfile(f, mode, data)
1187 1187 if f in copies:
1188 1188 self.copies.append([copies[f], f])
1189 1189 files = [f[0] for f in files]
1190 1190
1191 1191 entries = set(self.delete)
1192 1192 files = frozenset(files)
1193 1193 entries.update(self.add_dirs(files.difference(entries)))
1194 1194 if self.copies:
1195 1195 for s, d in self.copies:
1196 1196 self._copyfile(s, d)
1197 1197 self.copies = []
1198 1198 if self.delete:
1199 1199 self.xargs(self.delete, 'delete')
1200 1200 for f in self.delete:
1201 1201 self.manifest.remove(f)
1202 1202 self.delete = []
1203 1203 entries.update(self.add_files(files.difference(entries)))
1204 1204 entries.update(self.tidy_dirs(entries))
1205 1205 if self.delexec:
1206 1206 self.xargs(self.delexec, 'propdel', 'svn:executable')
1207 1207 self.delexec = []
1208 1208 if self.setexec:
1209 1209 self.xargs(self.setexec, 'propset', 'svn:executable', '*')
1210 1210 self.setexec = []
1211 1211
1212 1212 fd, messagefile = tempfile.mkstemp(prefix='hg-convert-')
1213 1213 fp = os.fdopen(fd, 'w')
1214 1214 fp.write(commit.desc)
1215 1215 fp.close()
1216 1216 try:
1217 1217 output = self.run0('commit',
1218 1218 username=util.shortuser(commit.author),
1219 1219 file=messagefile,
1220 1220 encoding='utf-8')
1221 1221 try:
1222 1222 rev = self.commit_re.search(output).group(1)
1223 1223 except AttributeError:
1224 1224 if not files:
1225 1225 return parents[0]
1226 1226 self.ui.warn(_('unexpected svn output:\n'))
1227 1227 self.ui.warn(output)
1228 1228 raise util.Abort(_('unable to cope with svn output'))
1229 1229 if commit.rev:
1230 1230 self.run('propset', 'hg:convert-rev', commit.rev,
1231 1231 revprop=True, revision=rev)
1232 1232 if commit.branch and commit.branch != 'default':
1233 1233 self.run('propset', 'hg:convert-branch', commit.branch,
1234 1234 revprop=True, revision=rev)
1235 1235 for parent in parents:
1236 1236 self.addchild(parent, rev)
1237 1237 return self.revid(rev)
1238 1238 finally:
1239 1239 os.unlink(messagefile)
1240 1240
1241 1241 def puttags(self, tags):
1242 1242 self.ui.warn(_('writing Subversion tags is not yet implemented\n'))
1243 1243 return None, None
1244 1244
1245 1245 def hascommit(self, rev):
1246 1246 # This is not correct as one can convert to an existing subversion
1247 1247 # repository and childmap would not list all revisions. Too bad.
1248 1248 if rev in self.childmap:
1249 1249 return True
1250 1250 raise util.Abort(_('splice map revision %s not found in subversion '
1251 1251 'child map (revision lookups are not implemented)')
1252 1252 % rev)
@@ -1,58 +1,58 b''
1 1
2 2 $ cat >> $HGRCPATH <<EOF
3 3 > [extensions]
4 4 > convert=
5 5 > EOF
6 6
7 7 Prepare orig repo
8 8
9 9 $ hg init orig
10 10 $ cd orig
11 11 $ echo foo > foo
12 12 $ HGUSER='user name' hg ci -qAm 'foo'
13 13 $ cd ..
14 14
15 15 Explicit --authors
16 16
17 17 $ cat > authormap.txt <<EOF
18 18 > user name = Long User Name
19 19 >
20 20 > # comment
21 21 > this line is ignored
22 22 > EOF
23 23 $ hg convert --authors authormap.txt orig new
24 24 initializing destination new repository
25 Ignoring bad line in author map file authormap.txt: this line is ignored
25 ignoring bad line in author map file authormap.txt: this line is ignored
26 26 scanning source...
27 27 sorting...
28 28 converting...
29 29 0 foo
30 Writing author map file $TESTTMP/new/.hg/authormap (glob)
30 writing author map file $TESTTMP/new/.hg/authormap
31 31 $ cat new/.hg/authormap
32 32 user name=Long User Name
33 33 $ hg -Rnew log
34 34 changeset: 0:d89716e88087
35 35 tag: tip
36 36 user: Long User Name
37 37 date: Thu Jan 01 00:00:00 1970 +0000
38 38 summary: foo
39 39
40 40 $ rm -rf new
41 41
42 42 Implicit .hg/authormap
43 43
44 44 $ hg init new
45 45 $ mv authormap.txt new/.hg/authormap
46 46 $ hg convert orig new
47 Ignoring bad line in author map file $TESTTMP/new/.hg/authormap: this line is ignored (glob)
47 ignoring bad line in author map file $TESTTMP/new/.hg/authormap: this line is ignored
48 48 scanning source...
49 49 sorting...
50 50 converting...
51 51 0 foo
52 52 $ hg -Rnew log
53 53 changeset: 0:d89716e88087
54 54 tag: tip
55 55 user: Long User Name
56 56 date: Thu Jan 01 00:00:00 1970 +0000
57 57 summary: foo
58 58
General Comments 0
You need to be logged in to leave comments. Login now