##// END OF EJS Templates
convert: svn: move some debug output to verbose
Brendan Cully -
r4792:2b5723a5 default
parent child Browse files
Show More
@@ -1,561 +1,562 b''
1 1 # Subversion 1.4/1.5 Python API backend
2 2 #
3 3 # Copyright(C) 2007 Daniel Holth et al
4 4
5 5 import pprint
6 6 import locale
7 7
8 8 from mercurial import util
9 9
10 10 # Subversion stuff. Works best with very recent Python SVN bindings
11 11 # e.g. SVN 1.5 or backports. Thanks to the bzr folks for enhancing
12 12 # these bindings.
13 13
14 14 from cStringIO import StringIO
15 15
16 16 from common import NoRepo, commit, converter_source
17 17
18 18 try:
19 19 from svn.core import SubversionException, Pool
20 20 import svn.core
21 21 import svn.ra
22 22 import svn.delta
23 23 import svn
24 24 import transport
25 25 except ImportError:
26 26 pass
27 27
28 28 class CompatibilityException(Exception): pass
29 29
30 30 LOG_BATCH_SIZE = 50
31 31
32 32 class svn_entry(object):
33 33 """Emulate a Subversion path change."""
34 34 __slots__ = ['path', 'copyfrom_path', 'copyfrom_rev', 'action']
35 35 def __init__(self, entry):
36 36 self.copyfrom_path = entry.copyfrom_path
37 37 self.copyfrom_rev = entry.copyfrom_rev
38 38 self.action = entry.action
39 39
40 40 def __str__(self):
41 41 return "%s %s %s" % (self.action, self.copyfrom_path, self.copyfrom_rev)
42 42
43 43 def __repr__(self):
44 44 return self.__str__()
45 45
46 46 class svn_paths(object):
47 47 """Emulate a Subversion ordered dictionary of changed paths."""
48 48 __slots__ = ['values', 'order']
49 49 def __init__(self, orig_paths):
50 50 self.order = []
51 51 self.values = {}
52 52 if hasattr(orig_paths, 'keys'):
53 53 self.order = sorted(orig_paths.keys())
54 54 self.values.update(orig_paths)
55 55 return
56 56 if not orig_paths:
57 57 return
58 58 for path in orig_paths:
59 59 self.order.append(path)
60 60 self.values[path] = svn_entry(orig_paths[path])
61 61 self.order.sort() # maybe the order it came in isn't so great...
62 62
63 63 def __iter__(self):
64 64 return iter(self.order)
65 65
66 66 def __getitem__(self, key):
67 67 return self.values[key]
68 68
69 69 def __str__(self):
70 70 s = "{\n"
71 71 for path in self.order:
72 72 s += "'%s': %s,\n" % (path, self.values[path])
73 73 s += "}"
74 74 return s
75 75
76 76 def __repr__(self):
77 77 return self.__str__()
78 78
79 79 # SVN conversion code stolen from bzr-svn and tailor
80 80 class convert_svn(converter_source):
81 81 def __init__(self, ui, url, rev=None):
82 82 try:
83 83 SubversionException
84 84 except NameError:
85 85 msg = 'subversion python bindings could not be loaded\n'
86 86 ui.warn(msg)
87 87 raise NoRepo(msg)
88 88
89 89 self.ui = ui
90 90 self.encoding = locale.getpreferredencoding()
91 91 latest = None
92 92 if rev:
93 93 try:
94 94 latest = int(rev)
95 95 except ValueError:
96 96 raise util.Abort('svn: revision %s is not an integer' % rev)
97 97 try:
98 98 # Support file://path@rev syntax. Useful e.g. to convert
99 99 # deleted branches.
100 100 url, latest = url.rsplit("@", 1)
101 101 latest = int(latest)
102 102 except ValueError, e:
103 103 pass
104 104 self.url = url
105 105 self.encoding = 'UTF-8' # Subversion is always nominal UTF-8
106 106 try:
107 107 self.transport = transport.SvnRaTransport(url = url)
108 108 self.ra = self.transport.ra
109 109 self.base = svn.ra.get_repos_root(self.ra)
110 110 self.module = self.url[len(self.base):]
111 111 self.modulemap = {} # revision, module
112 112 self.commits = {}
113 113 self.files = {}
114 114 self.uuid = svn.ra.get_uuid(self.ra).decode(self.encoding)
115 115 except SubversionException, e:
116 116 raise NoRepo("couldn't open SVN repo %s" % url)
117 117
118 118 try:
119 119 self.get_blacklist()
120 120 except IOError, e:
121 121 pass
122 122
123 123 self.last_changed = self.latest(self.module, latest)
124 124
125 125 self.head = self.rev(self.last_changed)
126 126
127 127 def rev(self, revnum):
128 128 return (u"svn:%s%s@%s" % (self.uuid, self.module, revnum)).decode(self.encoding)
129 129
130 130 def revnum(self, rev):
131 131 return int(rev.split('@')[-1])
132 132
133 133 def latest(self, path, stop=0):
134 134 'find the latest revision affecting path, up to stop'
135 135 if not stop:
136 136 stop = svn.ra.get_latest_revnum(self.ra)
137 137 try:
138 138 self.reparent('')
139 139 dirent = svn.ra.stat(self.ra, path.strip('/'), stop)
140 140 self.reparent(self.module)
141 141 except SubversionException:
142 142 dirent = None
143 143 if not dirent:
144 144 raise util.Abort('%s not found up to revision %d' \
145 145 % (path, stop))
146 146
147 147 return dirent.created_rev
148 148
149 149 def get_blacklist(self):
150 150 """Avoid certain revision numbers.
151 151 It is not uncommon for two nearby revisions to cancel each other
152 152 out, e.g. 'I copied trunk into a subdirectory of itself instead
153 153 of making a branch'. The converted repository is significantly
154 154 smaller if we ignore such revisions."""
155 155 self.blacklist = set()
156 156 blacklist = self.blacklist
157 157 for line in file("blacklist.txt", "r"):
158 158 if not line.startswith("#"):
159 159 try:
160 160 svn_rev = int(line.strip())
161 161 blacklist.add(svn_rev)
162 162 except ValueError, e:
163 163 pass # not an integer or a comment
164 164
165 165 def is_blacklisted(self, svn_rev):
166 166 return svn_rev in self.blacklist
167 167
168 168 def reparent(self, module):
169 169 svn_url = self.base + module
170 170 self.ui.debug("reparent to %s\n" % svn_url.encode(self.encoding))
171 171 svn.ra.reparent(self.ra, svn_url.encode(self.encoding))
172 172
173 173 def _fetch_revisions(self, from_revnum = 0, to_revnum = 347, pb=None):
174 174 # batching is broken for branches
175 175 to_revnum = 0
176 176 if not hasattr(self, 'child_rev'):
177 177 self.child_rev = from_revnum
178 178 self.child_cset = self.commits.get(self.child_rev)
179 179 else:
180 180 self.commits[self.child_rev] = self.child_cset
181 181 # batching broken
182 182 return
183 183 # if the branch was created in the middle of the last batch,
184 184 # svn log will complain that the path doesn't exist in this batch
185 185 # so we roll the parser back to the last revision where this branch appeared
186 186 revnum = self.revnum(self.child_rev)
187 187 if revnum > from_revnum:
188 188 from_revnum = revnum
189 189
190 self.ui.debug('Fetching revisions %d to %d\n' % (from_revnum, to_revnum))
190 self.ui.note('fetching revision log from %d to %d\n' % \
191 (from_revnum, to_revnum))
191 192
192 193 def get_entry_from_path(path, module=self.module):
193 194 # Given the repository url of this wc, say
194 195 # "http://server/plone/CMFPlone/branches/Plone-2_0-branch"
195 196 # extract the "entry" portion (a relative path) from what
196 197 # svn log --xml says, ie
197 198 # "/CMFPlone/branches/Plone-2_0-branch/tests/PloneTestCase.py"
198 199 # that is to say "tests/PloneTestCase.py"
199 200
200 201 if path.startswith(module):
201 202 relative = path[len(module):]
202 203 if relative.startswith('/'):
203 204 return relative[1:]
204 205 else:
205 206 return relative
206 207
207 208 # The path is outside our tracked tree...
208 209 self.ui.debug('Ignoring %r since it is not under %r\n' % (path, module))
209 210 return None
210 211
211 212 received = []
212 213 def rcvr(*arg, **args):
213 214 orig_paths, revnum, author, date, message, pool = arg
214 215 new_orig_paths = svn_paths(orig_paths)
215 216 rcvr2(new_orig_paths, revnum, author, date, message, pool)
216 217
217 218 def rcvr2(orig_paths, revnum, author, date, message, pool, better_paths = None):
218 219 if not self.is_blacklisted(revnum):
219 220 received.append((orig_paths, revnum, author, date, message))
220 221
221 222 def after_received(orig_paths, revnum, author, date, message):
223 self.ui.note("parsing revision %d\n" % revnum)
224 if orig_paths is None:
225 return
226
222 227 if revnum in self.modulemap:
223 228 new_module = self.modulemap[revnum]
224 229 if new_module != self.module:
225 230 self.module = new_module
226 231 self.reparent(self.module)
227 232
228 233 copyfrom = {} # Map of entrypath, revision for finding source of deleted revisions.
229 234 copies = {}
230 235 entries = []
231 self.ui.debug("Parsing revision %d\n" % revnum)
232 if orig_paths is None:
233 return
234
235 236 rev = self.rev(revnum)
236 237 try:
237 238 branch = self.module.split("/")[-1]
238 239 if branch == 'trunk':
239 240 branch = ''
240 241 except IndexError:
241 242 branch = None
242 243
243 244 for path in orig_paths:
244 245 # self.ui.write("path %s\n" % path)
245 246 if path == self.module: # Follow branching back in history
246 247 ent = orig_paths[path]
247 248 if ent:
248 249 if ent.copyfrom_path:
249 250 self.modulemap[ent.copyfrom_rev] = ent.copyfrom_path
250 251 else:
251 252 self.ui.debug("No copyfrom path, don't know what to do.\n")
252 253 # Maybe it was added and there is no more history.
253 254 entrypath = get_entry_from_path(path, module=self.module)
254 255 # self.ui.write("entrypath %s\n" % entrypath)
255 256 if entrypath is None:
256 257 # Outside our area of interest
257 258 self.ui.debug("boring@%s: %s\n" % (revnum, path))
258 259 continue
259 260 entry = entrypath.decode(self.encoding)
260 261 ent = orig_paths[path]
261 262 if not entrypath:
262 263 # TODO: branch creation event
263 264 pass
264 265
265 266 kind = svn.ra.check_path(self.ra, entrypath, revnum)
266 267 if kind == svn.core.svn_node_file:
267 268 if ent.copyfrom_path:
268 269 copyfrom_path = get_entry_from_path(ent.copyfrom_path)
269 270 if copyfrom_path:
270 271 self.ui.debug("Copied to %s from %s@%s\n" % (entry, copyfrom_path, ent.copyfrom_rev))
271 272 # It's probably important for hg that the source
272 273 # exists in the revision's parent, not just the
273 274 # ent.copyfrom_rev
274 275 fromkind = svn.ra.check_path(self.ra, copyfrom_path, ent.copyfrom_rev)
275 276 if fromkind != 0:
276 277 copies[self.recode(entry)] = self.recode(copyfrom_path)
277 278 entries.append(self.recode(entry))
278 279 elif kind == 0: # gone, but had better be a deleted *file*
279 280 self.ui.debug("gone from %s\n" % ent.copyfrom_rev)
280 281
281 282 fromrev = revnum - 1
282 283 # might always need to be revnum - 1 in these 3 lines?
283 284 old_module = self.modulemap.get(fromrev, self.module)
284 285 basepath = old_module + "/" + get_entry_from_path(path, module=self.module)
285 286 entrypath = old_module + "/" + get_entry_from_path(path, module=self.module)
286 287
287 288 def lookup_parts(p):
288 289 rc = None
289 290 parts = p.split("/")
290 291 for i in range(len(parts)):
291 292 part = "/".join(parts[:i])
292 293 info = part, copyfrom.get(part, None)
293 294 if info[1] is not None:
294 295 self.ui.debug("Found parent directory %s\n" % info)
295 296 rc = info
296 297 return rc
297 298
298 299 self.ui.debug("base, entry %s %s\n" % (basepath, entrypath))
299 300
300 301 frompath, froment = lookup_parts(entrypath) or (None, revnum - 1)
301 302
302 303 # need to remove fragment from lookup_parts and replace with copyfrom_path
303 304 if frompath is not None:
304 305 self.ui.debug("munge-o-matic\n")
305 306 self.ui.debug(entrypath + '\n')
306 307 self.ui.debug(entrypath[len(frompath):] + '\n')
307 308 entrypath = froment.copyfrom_path + entrypath[len(frompath):]
308 309 fromrev = froment.copyfrom_rev
309 310 self.ui.debug("Info: %s %s %s %s\n" % (frompath, froment, ent, entrypath))
310 311
311 312 fromkind = svn.ra.check_path(self.ra, entrypath, fromrev)
312 313 if fromkind == svn.core.svn_node_file: # a deleted file
313 314 entries.append(self.recode(entry))
314 315 else:
315 316 # print "Deleted/moved non-file:", revnum, path, ent
316 317 # children = self._find_children(path, revnum - 1)
317 318 # print "find children %s@%d from %d action %s" % (path, revnum, ent.copyfrom_rev, ent.action)
318 319 # Sometimes this is tricky. For example: in
319 320 # The Subversion Repository revision 6940 a dir
320 321 # was copied and one of its files was deleted
321 322 # from the new location in the same commit. This
322 323 # code can't deal with that yet.
323 324 if ent.action == 'C':
324 325 children = self._find_children(path, fromrev)
325 326 else:
326 327 oroot = entrypath.strip('/')
327 328 nroot = path.strip('/')
328 329 children = self._find_children(oroot, fromrev)
329 330 children = [s.replace(oroot,nroot) for s in children]
330 331 # Mark all [files, not directories] as deleted.
331 332 for child in children:
332 333 # Can we move a child directory and its
333 334 # parent in the same commit? (probably can). Could
334 335 # cause problems if instead of revnum -1,
335 336 # we have to look in (copyfrom_path, revnum - 1)
336 337 entrypath = get_entry_from_path("/" + child, module=old_module)
337 338 if entrypath:
338 339 entry = self.recode(entrypath.decode(self.encoding))
339 340 if entry in copies:
340 341 # deleted file within a copy
341 342 del copies[entry]
342 343 else:
343 344 entries.append(entry)
344 345 elif kind == svn.core.svn_node_dir:
345 346 # Should probably synthesize normal file entries
346 347 # and handle as above to clean up copy/rename handling.
347 348
348 349 # If the directory just had a prop change,
349 350 # then we shouldn't need to look for its children.
350 351 # Also this could create duplicate entries. Not sure
351 352 # whether this will matter. Maybe should make entries a set.
352 353 # print "Changed directory", revnum, path, ent.action, ent.copyfrom_path, ent.copyfrom_rev
353 354 # This will fail if a directory was copied
354 355 # from another branch and then some of its files
355 356 # were deleted in the same transaction.
356 357 children = self._find_children(path, revnum)
357 358 children.sort()
358 359 for child in children:
359 360 # Can we move a child directory and its
360 361 # parent in the same commit? (probably can). Could
361 362 # cause problems if instead of revnum -1,
362 363 # we have to look in (copyfrom_path, revnum - 1)
363 364 entrypath = get_entry_from_path("/" + child, module=self.module)
364 365 # print child, self.module, entrypath
365 366 if entrypath:
366 367 # Need to filter out directories here...
367 368 kind = svn.ra.check_path(self.ra, entrypath, revnum)
368 369 if kind != svn.core.svn_node_dir:
369 370 entries.append(self.recode(entrypath))
370 371
371 372 # Copies here (must copy all from source)
372 373 # Probably not a real problem for us if
373 374 # source does not exist
374 375
375 376 # Can do this with the copy command "hg copy"
376 377 # if ent.copyfrom_path:
377 378 # copyfrom_entry = get_entry_from_path(ent.copyfrom_path.decode(self.encoding),
378 379 # module=self.module)
379 380 # copyto_entry = entrypath
380 381 #
381 382 # print "copy directory", copyfrom_entry, 'to', copyto_entry
382 383 #
383 384 # copies.append((copyfrom_entry, copyto_entry))
384 385
385 386 if ent.copyfrom_path:
386 387 copyfrom_path = ent.copyfrom_path.decode(self.encoding)
387 388 copyfrom_entry = get_entry_from_path(copyfrom_path, module=self.module)
388 389 if copyfrom_entry:
389 390 copyfrom[path] = ent
390 391 self.ui.debug("mark %s came from %s\n" % (path, copyfrom[path]))
391 392
392 393 # Good, /probably/ a regular copy. Really should check
393 394 # to see whether the parent revision actually contains
394 395 # the directory in question.
395 396 children = self._find_children(self.recode(copyfrom_path), ent.copyfrom_rev)
396 397 children.sort()
397 398 for child in children:
398 399 entrypath = get_entry_from_path("/" + child, module=self.module)
399 400 if entrypath:
400 401 entry = entrypath.decode(self.encoding)
401 402 # print "COPY COPY From", copyfrom_entry, entry
402 403 copyto_path = path + entry[len(copyfrom_entry):]
403 404 copyto_entry = get_entry_from_path(copyto_path, module=self.module)
404 405 # print "COPY", entry, "COPY To", copyto_entry
405 406 copies[self.recode(copyto_entry)] = self.recode(entry)
406 407 # copy from quux splort/quuxfile
407 408
408 409 self.modulemap[revnum] = self.module # track backwards in time
409 410 # a list of (filename, id) where id lets us retrieve the file.
410 411 # eg in git, id is the object hash. for svn it'll be the
411 412 self.files[rev] = zip(entries, [rev] * len(entries))
412 413 if not entries:
413 414 return
414 415
415 416 # Example SVN datetime. Includes microseconds.
416 417 # ISO-8601 conformant
417 418 # '2007-01-04T17:35:00.902377Z'
418 419 date = util.parsedate(date[:18] + " UTC", ["%Y-%m-%dT%H:%M:%S"])
419 420
420 421 log = message and self.recode(message)
421 422 author = author and self.recode(author) or ''
422 423
423 424 cset = commit(author=author,
424 425 date=util.datestr(date),
425 426 desc=log,
426 427 parents=[],
427 428 copies=copies,
428 429 branch=branch)
429 430
430 431 if self.child_cset and self.child_rev != rev:
431 432 self.child_cset.parents = [rev]
432 433 self.commits[self.child_rev] = self.child_cset
433 434 self.child_cset = cset
434 435 self.child_rev = rev
435 436
436 437 try:
437 438 discover_changed_paths = True
438 439 strict_node_history = False
439 440 svn.ra.get_log(self.ra, [self.module], from_revnum, to_revnum,
440 441 0, discover_changed_paths, strict_node_history, rcvr)
441 442 for args in received:
442 443 after_received(*args)
443 444 self.last_revnum = to_revnum
444 445 except SubversionException, (_, num):
445 446 if num == svn.core.SVN_ERR_FS_NO_SUCH_REVISION:
446 447 raise NoSuchRevision(branch=self,
447 448 revision="Revision number %d" % to_revnum)
448 449 raise
449 450
450 451 def getheads(self):
451 452 # svn-url@rev
452 453 # Not safe if someone committed:
453 454 self.heads = [self.head]
454 455 # print self.commits.keys()
455 456 return self.heads
456 457
457 458 def _getfile(self, file, rev):
458 459 io = StringIO()
459 460 # TODO: ra.get_file transmits the whole file instead of diffs.
460 461 mode = ''
461 462 try:
462 463 revnum = self.revnum(rev)
463 464 if self.module != self.modulemap[revnum]:
464 465 self.module = self.modulemap[revnum]
465 466 self.reparent(self.module)
466 467 info = svn.ra.get_file(self.ra, file, revnum, io)
467 468 if isinstance(info, list):
468 469 info = info[-1]
469 470 mode = ("svn:executable" in info) and 'x' or ''
470 471 mode = ("svn:special" in info) and 'l' or mode
471 472 except SubversionException, e:
472 473 notfound = (svn.core.SVN_ERR_FS_NOT_FOUND,
473 474 svn.core.SVN_ERR_RA_DAV_PATH_NOT_FOUND)
474 475 if e.apr_err in notfound: # File not found
475 476 raise IOError()
476 477 raise
477 478 data = io.getvalue()
478 479 if mode == 'l':
479 480 link_prefix = "link "
480 481 if data.startswith(link_prefix):
481 482 data = data[len(link_prefix):]
482 483 return data, mode
483 484
484 485 def getfile(self, file, rev):
485 486 data, mode = self._getfile(file, rev)
486 487 self.modecache[(file, rev)] = mode
487 488 return data
488 489
489 490 def getmode(self, file, rev):
490 491 return self.modecache[(file, rev)]
491 492
492 493 def getchanges(self, rev):
493 494 self.modecache = {}
494 495 files = self.files[rev]
495 496 cl = files
496 497 cl.sort()
497 498 return cl
498 499
499 500 def getcommit(self, rev):
500 501 if rev not in self.commits:
501 502 revnum = self.revnum(rev)
502 503 minrev = revnum - LOG_BATCH_SIZE > 0 and revnum - LOG_BATCH_SIZE or 0
503 504 self._fetch_revisions(from_revnum=revnum, to_revnum=minrev)
504 505 return self.commits[rev]
505 506
506 507 def gettags(self):
507 508 return []
508 509
509 510 def _find_children(self, path, revnum):
510 511 path = path.strip("/")
511 512
512 513 def _find_children_fallback(path, revnum):
513 514 # SWIG python bindings for getdir are broken up to at least 1.4.3
514 515 if not hasattr(self, 'client_ctx'):
515 516 self.client_ctx = svn.client.create_context()
516 517 pool = Pool()
517 518 optrev = svn.core.svn_opt_revision_t()
518 519 optrev.kind = svn.core.svn_opt_revision_number
519 520 optrev.value.number = revnum
520 521 rpath = '/'.join([self.base, path]).strip('/')
521 522 return ['%s/%s' % (path, x) for x in svn.client.ls(rpath, optrev, True, self.client_ctx, pool).keys()]
522 523
523 524 if hasattr(self, '_find_children_fallback'):
524 525 return _find_children_fallback(path, revnum)
525 526
526 527 self.reparent("/" + path)
527 528 pool = Pool()
528 529
529 530 children = []
530 531 def find_children_inner(children, path, revnum = revnum):
531 532 if hasattr(svn.ra, 'get_dir2'): # Since SVN 1.4
532 533 fields = 0xffffffff # Binding does not provide SVN_DIRENT_ALL
533 534 getdir = svn.ra.get_dir2(self.ra, path, revnum, fields, pool)
534 535 else:
535 536 getdir = svn.ra.get_dir(self.ra, path, revnum, pool)
536 537 if type(getdir) == dict:
537 538 # python binding for getdir is broken up to at least 1.4.3
538 539 raise CompatibilityException()
539 540 dirents = getdir[0]
540 541 if type(dirents) == int:
541 542 # got here once due to infinite recursion bug
542 543 # pprint.pprint(getdir)
543 544 return
544 545 c = dirents.keys()
545 546 c.sort()
546 547 for child in c:
547 548 dirent = dirents[child]
548 549 if dirent.kind == svn.core.svn_node_dir:
549 550 find_children_inner(children, (path + "/" + child).strip("/"))
550 551 else:
551 552 children.append((path + "/" + child).strip("/"))
552 553
553 554 try:
554 555 find_children_inner(children, "")
555 556 except CompatibilityException:
556 557 self._find_children_fallback = True
557 558 self.reparent(self.module)
558 559 return _find_children_fallback(path, revnum)
559 560
560 561 self.reparent(self.module)
561 562 return [path + "/" + c for c in children]
General Comments 0
You need to be logged in to leave comments. Login now