##// END OF EJS Templates
svn: fix subprocess problems on some of the calls for file checking.
milka -
r926:8b3be25b default
parent child Browse files
Show More
@@ -1,856 +1,855 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2020 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 from __future__ import absolute_import
19 19
20 20 import os
21 21 import subprocess
22 22 import time
23 23 from urllib2 import URLError
24 24 import urlparse
25 25 import logging
26 26 import posixpath as vcspath
27 27 import StringIO
28 28 import urllib
29 29 import traceback
30 30
31 31 import svn.client
32 32 import svn.core
33 33 import svn.delta
34 34 import svn.diff
35 35 import svn.fs
36 36 import svn.repos
37 37
38 38 from vcsserver import svn_diff, exceptions, subprocessio, settings
39 39 from vcsserver.base import RepoFactory, raise_from_original, ArchiveNode, archive_repo
40 40 from vcsserver.exceptions import NoContentException
41 from vcsserver.utils import safe_str
41 42 from vcsserver.vcs_base import RemoteBase
42 43
43 44 log = logging.getLogger(__name__)
44 45
45 46
46 47 svn_compatible_versions_map = {
47 48 'pre-1.4-compatible': '1.3',
48 49 'pre-1.5-compatible': '1.4',
49 50 'pre-1.6-compatible': '1.5',
50 51 'pre-1.8-compatible': '1.7',
51 52 'pre-1.9-compatible': '1.8',
52 53 }
53 54
54 55 current_compatible_version = '1.12'
55 56
56 57
57 58 def reraise_safe_exceptions(func):
58 59 """Decorator for converting svn exceptions to something neutral."""
59 60 def wrapper(*args, **kwargs):
60 61 try:
61 62 return func(*args, **kwargs)
62 63 except Exception as e:
63 64 if not hasattr(e, '_vcs_kind'):
64 65 log.exception("Unhandled exception in svn remote call")
65 66 raise_from_original(exceptions.UnhandledException(e))
66 67 raise
67 68 return wrapper
68 69
69 70
70 71 class SubversionFactory(RepoFactory):
71 72 repo_type = 'svn'
72 73
73 74 def _create_repo(self, wire, create, compatible_version):
74 75 path = svn.core.svn_path_canonicalize(wire['path'])
75 76 if create:
76 77 fs_config = {'compatible-version': current_compatible_version}
77 78 if compatible_version:
78 79
79 80 compatible_version_string = \
80 81 svn_compatible_versions_map.get(compatible_version) \
81 82 or compatible_version
82 83 fs_config['compatible-version'] = compatible_version_string
83 84
84 85 log.debug('Create SVN repo with config "%s"', fs_config)
85 86 repo = svn.repos.create(path, "", "", None, fs_config)
86 87 else:
87 88 repo = svn.repos.open(path)
88 89
89 90 log.debug('Got SVN object: %s', repo)
90 91 return repo
91 92
92 93 def repo(self, wire, create=False, compatible_version=None):
93 94 """
94 95 Get a repository instance for the given path.
95 96 """
96 97 return self._create_repo(wire, create, compatible_version)
97 98
98 99
99 100 NODE_TYPE_MAPPING = {
100 101 svn.core.svn_node_file: 'file',
101 102 svn.core.svn_node_dir: 'dir',
102 103 }
103 104
104 105
105 106 class SvnRemote(RemoteBase):
106 107
107 108 def __init__(self, factory, hg_factory=None):
108 109 self._factory = factory
109 110 # TODO: Remove once we do not use internal Mercurial objects anymore
110 111 # for subversion
111 112 self._hg_factory = hg_factory
112 113
113 114 @reraise_safe_exceptions
114 115 def discover_svn_version(self):
115 116 try:
116 117 import svn.core
117 118 svn_ver = svn.core.SVN_VERSION
118 119 except ImportError:
119 120 svn_ver = None
120 121 return svn_ver
121 122
122 123 @reraise_safe_exceptions
123 124 def is_empty(self, wire):
124 125
125 126 try:
126 127 return self.lookup(wire, -1) == 0
127 128 except Exception:
128 129 log.exception("failed to read object_store")
129 130 return False
130 131
131 132 def check_url(self, url, config_items):
132 133 # this can throw exception if not installed, but we detect this
133 134 from hgsubversion import svnrepo
134 135
135 136 baseui = self._hg_factory._create_config(config_items)
136 137 # uuid function get's only valid UUID from proper repo, else
137 138 # throws exception
138 139 try:
139 140 svnrepo.svnremoterepo(baseui, url).svn.uuid
140 141 except Exception:
141 142 tb = traceback.format_exc()
142 143 log.debug("Invalid Subversion url: `%s`, tb: %s", url, tb)
143 144 raise URLError(
144 145 '"%s" is not a valid Subversion source url.' % (url, ))
145 146 return True
146 147
147 148 def is_path_valid_repository(self, wire, path):
148 149
149 150 # NOTE(marcink): short circuit the check for SVN repo
150 151 # the repos.open might be expensive to check, but we have one cheap
151 152 # pre condition that we can use, to check for 'format' file
152 153
153 154 if not os.path.isfile(os.path.join(path, 'format')):
154 155 return False
155 156
156 157 try:
157 158 svn.repos.open(path)
158 159 except svn.core.SubversionException:
159 160 tb = traceback.format_exc()
160 161 log.debug("Invalid Subversion path `%s`, tb: %s", path, tb)
161 162 return False
162 163 return True
163 164
164 165 @reraise_safe_exceptions
165 166 def verify(self, wire,):
166 167 repo_path = wire['path']
167 168 if not self.is_path_valid_repository(wire, repo_path):
168 169 raise Exception(
169 170 "Path %s is not a valid Subversion repository." % repo_path)
170 171
171 172 cmd = ['svnadmin', 'info', repo_path]
172 173 stdout, stderr = subprocessio.run_command(cmd)
173 174 return stdout
174 175
175 176 def lookup(self, wire, revision):
176 177 if revision not in [-1, None, 'HEAD']:
177 178 raise NotImplementedError
178 179 repo = self._factory.repo(wire)
179 180 fs_ptr = svn.repos.fs(repo)
180 181 head = svn.fs.youngest_rev(fs_ptr)
181 182 return head
182 183
183 184 def lookup_interval(self, wire, start_ts, end_ts):
184 185 repo = self._factory.repo(wire)
185 186 fsobj = svn.repos.fs(repo)
186 187 start_rev = None
187 188 end_rev = None
188 189 if start_ts:
189 190 start_ts_svn = apr_time_t(start_ts)
190 191 start_rev = svn.repos.dated_revision(repo, start_ts_svn) + 1
191 192 else:
192 193 start_rev = 1
193 194 if end_ts:
194 195 end_ts_svn = apr_time_t(end_ts)
195 196 end_rev = svn.repos.dated_revision(repo, end_ts_svn)
196 197 else:
197 198 end_rev = svn.fs.youngest_rev(fsobj)
198 199 return start_rev, end_rev
199 200
200 201 def revision_properties(self, wire, revision):
201 202
202 203 cache_on, context_uid, repo_id = self._cache_on(wire)
203 204 @self.region.conditional_cache_on_arguments(condition=cache_on)
204 205 def _revision_properties(_repo_id, _revision):
205 206 repo = self._factory.repo(wire)
206 207 fs_ptr = svn.repos.fs(repo)
207 208 return svn.fs.revision_proplist(fs_ptr, revision)
208 209 return _revision_properties(repo_id, revision)
209 210
210 211 def revision_changes(self, wire, revision):
211 212
212 213 repo = self._factory.repo(wire)
213 214 fsobj = svn.repos.fs(repo)
214 215 rev_root = svn.fs.revision_root(fsobj, revision)
215 216
216 217 editor = svn.repos.ChangeCollector(fsobj, rev_root)
217 218 editor_ptr, editor_baton = svn.delta.make_editor(editor)
218 219 base_dir = ""
219 220 send_deltas = False
220 221 svn.repos.replay2(
221 222 rev_root, base_dir, svn.core.SVN_INVALID_REVNUM, send_deltas,
222 223 editor_ptr, editor_baton, None)
223 224
224 225 added = []
225 226 changed = []
226 227 removed = []
227 228
228 229 # TODO: CHANGE_ACTION_REPLACE: Figure out where it belongs
229 230 for path, change in editor.changes.iteritems():
230 231 # TODO: Decide what to do with directory nodes. Subversion can add
231 232 # empty directories.
232 233
233 234 if change.item_kind == svn.core.svn_node_dir:
234 235 continue
235 236 if change.action in [svn.repos.CHANGE_ACTION_ADD]:
236 237 added.append(path)
237 238 elif change.action in [svn.repos.CHANGE_ACTION_MODIFY,
238 239 svn.repos.CHANGE_ACTION_REPLACE]:
239 240 changed.append(path)
240 241 elif change.action in [svn.repos.CHANGE_ACTION_DELETE]:
241 242 removed.append(path)
242 243 else:
243 244 raise NotImplementedError(
244 245 "Action %s not supported on path %s" % (
245 246 change.action, path))
246 247
247 248 changes = {
248 249 'added': added,
249 250 'changed': changed,
250 251 'removed': removed,
251 252 }
252 253 return changes
253 254
254 255 @reraise_safe_exceptions
255 256 def node_history(self, wire, path, revision, limit):
256 257 cache_on, context_uid, repo_id = self._cache_on(wire)
257 258 @self.region.conditional_cache_on_arguments(condition=cache_on)
258 259 def _assert_correct_path(_context_uid, _repo_id, _path, _revision, _limit):
259 260 cross_copies = False
260 261 repo = self._factory.repo(wire)
261 262 fsobj = svn.repos.fs(repo)
262 263 rev_root = svn.fs.revision_root(fsobj, revision)
263 264
264 265 history_revisions = []
265 266 history = svn.fs.node_history(rev_root, path)
266 267 history = svn.fs.history_prev(history, cross_copies)
267 268 while history:
268 269 __, node_revision = svn.fs.history_location(history)
269 270 history_revisions.append(node_revision)
270 271 if limit and len(history_revisions) >= limit:
271 272 break
272 273 history = svn.fs.history_prev(history, cross_copies)
273 274 return history_revisions
274 275 return _assert_correct_path(context_uid, repo_id, path, revision, limit)
275 276
276 277 def node_properties(self, wire, path, revision):
277 278 cache_on, context_uid, repo_id = self._cache_on(wire)
278 279 @self.region.conditional_cache_on_arguments(condition=cache_on)
279 280 def _node_properties(_repo_id, _path, _revision):
280 281 repo = self._factory.repo(wire)
281 282 fsobj = svn.repos.fs(repo)
282 283 rev_root = svn.fs.revision_root(fsobj, revision)
283 284 return svn.fs.node_proplist(rev_root, path)
284 285 return _node_properties(repo_id, path, revision)
285 286
286 287 def file_annotate(self, wire, path, revision):
287 288 abs_path = 'file://' + urllib.pathname2url(
288 289 vcspath.join(wire['path'], path))
289 290 file_uri = svn.core.svn_path_canonicalize(abs_path)
290 291
291 292 start_rev = svn_opt_revision_value_t(0)
292 293 peg_rev = svn_opt_revision_value_t(revision)
293 294 end_rev = peg_rev
294 295
295 296 annotations = []
296 297
297 298 def receiver(line_no, revision, author, date, line, pool):
298 299 annotations.append((line_no, revision, line))
299 300
300 301 # TODO: Cannot use blame5, missing typemap function in the swig code
301 302 try:
302 303 svn.client.blame2(
303 304 file_uri, peg_rev, start_rev, end_rev,
304 305 receiver, svn.client.create_context())
305 306 except svn.core.SubversionException as exc:
306 307 log.exception("Error during blame operation.")
307 308 raise Exception(
308 309 "Blame not supported or file does not exist at path %s. "
309 310 "Error %s." % (path, exc))
310 311
311 312 return annotations
312 313
313 314 def get_node_type(self, wire, path, revision=None):
314 315
315 316 cache_on, context_uid, repo_id = self._cache_on(wire)
316 317 @self.region.conditional_cache_on_arguments(condition=cache_on)
317 318 def _get_node_type(_repo_id, _path, _revision):
318 319 repo = self._factory.repo(wire)
319 320 fs_ptr = svn.repos.fs(repo)
320 321 if _revision is None:
321 322 _revision = svn.fs.youngest_rev(fs_ptr)
322 323 root = svn.fs.revision_root(fs_ptr, _revision)
323 324 node = svn.fs.check_path(root, path)
324 325 return NODE_TYPE_MAPPING.get(node, None)
325 326 return _get_node_type(repo_id, path, revision)
326 327
327 328 def get_nodes(self, wire, path, revision=None):
328 329
329 330 cache_on, context_uid, repo_id = self._cache_on(wire)
330 331 @self.region.conditional_cache_on_arguments(condition=cache_on)
331 332 def _get_nodes(_repo_id, _path, _revision):
332 333 repo = self._factory.repo(wire)
333 334 fsobj = svn.repos.fs(repo)
334 335 if _revision is None:
335 336 _revision = svn.fs.youngest_rev(fsobj)
336 337 root = svn.fs.revision_root(fsobj, _revision)
337 338 entries = svn.fs.dir_entries(root, path)
338 339 result = []
339 340 for entry_path, entry_info in entries.iteritems():
340 341 result.append(
341 342 (entry_path, NODE_TYPE_MAPPING.get(entry_info.kind, None)))
342 343 return result
343 344 return _get_nodes(repo_id, path, revision)
344 345
345 346 def get_file_content(self, wire, path, rev=None):
346 347 repo = self._factory.repo(wire)
347 348 fsobj = svn.repos.fs(repo)
348 349 if rev is None:
349 350 rev = svn.fs.youngest_revision(fsobj)
350 351 root = svn.fs.revision_root(fsobj, rev)
351 352 content = svn.core.Stream(svn.fs.file_contents(root, path))
352 353 return content.read()
353 354
354 355 def get_file_size(self, wire, path, revision=None):
355 356
356 357 cache_on, context_uid, repo_id = self._cache_on(wire)
357 358 @self.region.conditional_cache_on_arguments(condition=cache_on)
358 359 def _get_file_size(_repo_id, _path, _revision):
359 360 repo = self._factory.repo(wire)
360 361 fsobj = svn.repos.fs(repo)
361 362 if _revision is None:
362 363 _revision = svn.fs.youngest_revision(fsobj)
363 364 root = svn.fs.revision_root(fsobj, _revision)
364 365 size = svn.fs.file_length(root, path)
365 366 return size
366 367 return _get_file_size(repo_id, path, revision)
367 368
368 369 def create_repository(self, wire, compatible_version=None):
369 370 log.info('Creating Subversion repository in path "%s"', wire['path'])
370 371 self._factory.repo(wire, create=True,
371 372 compatible_version=compatible_version)
372 373
373 374 def get_url_and_credentials(self, src_url):
374 375 obj = urlparse.urlparse(src_url)
375 376 username = obj.username or None
376 377 password = obj.password or None
377 378 return username, password, src_url
378 379
379 380 def import_remote_repository(self, wire, src_url):
380 381 repo_path = wire['path']
381 382 if not self.is_path_valid_repository(wire, repo_path):
382 383 raise Exception(
383 384 "Path %s is not a valid Subversion repository." % repo_path)
384 385
385 386 username, password, src_url = self.get_url_and_credentials(src_url)
386 387 rdump_cmd = ['svnrdump', 'dump', '--non-interactive',
387 388 '--trust-server-cert-failures=unknown-ca']
388 389 if username and password:
389 390 rdump_cmd += ['--username', username, '--password', password]
390 391 rdump_cmd += [src_url]
391 392
392 393 rdump = subprocess.Popen(
393 394 rdump_cmd,
394 395 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
395 396 load = subprocess.Popen(
396 397 ['svnadmin', 'load', repo_path], stdin=rdump.stdout)
397 398
398 399 # TODO: johbo: This can be a very long operation, might be better
399 400 # to track some kind of status and provide an api to check if the
400 401 # import is done.
401 402 rdump.wait()
402 403 load.wait()
403 404
404 405 log.debug('Return process ended with code: %s', rdump.returncode)
405 406 if rdump.returncode != 0:
406 407 errors = rdump.stderr.read()
407 408 log.error('svnrdump dump failed: statuscode %s: message: %s',
408 409 rdump.returncode, errors)
409 410 reason = 'UNKNOWN'
410 411 if 'svnrdump: E230001:' in errors:
411 412 reason = 'INVALID_CERTIFICATE'
412 413
413 414 if reason == 'UNKNOWN':
414 415 reason = 'UNKNOWN:{}'.format(errors)
415 416 raise Exception(
416 417 'Failed to dump the remote repository from %s. Reason:%s' % (
417 418 src_url, reason))
418 419 if load.returncode != 0:
419 420 raise Exception(
420 421 'Failed to load the dump of remote repository from %s.' %
421 422 (src_url, ))
422 423
423 424 def commit(self, wire, message, author, timestamp, updated, removed):
424 425 assert isinstance(message, str)
425 426 assert isinstance(author, str)
426 427
427 428 repo = self._factory.repo(wire)
428 429 fsobj = svn.repos.fs(repo)
429 430
430 431 rev = svn.fs.youngest_rev(fsobj)
431 432 txn = svn.repos.fs_begin_txn_for_commit(repo, rev, author, message)
432 433 txn_root = svn.fs.txn_root(txn)
433 434
434 435 for node in updated:
435 436 TxnNodeProcessor(node, txn_root).update()
436 437 for node in removed:
437 438 TxnNodeProcessor(node, txn_root).remove()
438 439
439 440 commit_id = svn.repos.fs_commit_txn(repo, txn)
440 441
441 442 if timestamp:
442 443 apr_time = apr_time_t(timestamp)
443 444 ts_formatted = svn.core.svn_time_to_cstring(apr_time)
444 445 svn.fs.change_rev_prop(fsobj, commit_id, 'svn:date', ts_formatted)
445 446
446 447 log.debug('Committed revision "%s" to "%s".', commit_id, wire['path'])
447 448 return commit_id
448 449
449 450 def diff(self, wire, rev1, rev2, path1=None, path2=None,
450 451 ignore_whitespace=False, context=3):
451 452
452 453 wire.update(cache=False)
453 454 repo = self._factory.repo(wire)
454 455 diff_creator = SvnDiffer(
455 456 repo, rev1, path1, rev2, path2, ignore_whitespace, context)
456 457 try:
457 458 return diff_creator.generate_diff()
458 459 except svn.core.SubversionException as e:
459 460 log.exception(
460 461 "Error during diff operation operation. "
461 462 "Path might not exist %s, %s" % (path1, path2))
462 463 return ""
463 464
464 465 @reraise_safe_exceptions
465 466 def is_large_file(self, wire, path):
466 467 return False
467 468
468 469 @reraise_safe_exceptions
469 470 def is_binary(self, wire, rev, path):
470 471 cache_on, context_uid, repo_id = self._cache_on(wire)
471 472
472 473 @self.region.conditional_cache_on_arguments(condition=cache_on)
473 474 def _is_binary(_repo_id, _rev, _path):
474 475 raw_bytes = self.get_file_content(wire, path, rev)
475 476 return raw_bytes and '\0' in raw_bytes
476 477
477 478 return _is_binary(repo_id, rev, path)
478 479
479 480 @reraise_safe_exceptions
480 481 def run_svn_command(self, wire, cmd, **opts):
481 482 path = wire.get('path', None)
482 483
483 484 if path and os.path.isdir(path):
484 485 opts['cwd'] = path
485 486
486 safe_call = False
487 if '_safe' in opts:
488 safe_call = True
487 safe_call = opts.pop('_safe', False)
489 488
490 489 svnenv = os.environ.copy()
491 490 svnenv.update(opts.pop('extra_env', {}))
492 491
493 492 _opts = {'env': svnenv, 'shell': False}
494 493
495 494 try:
496 495 _opts.update(opts)
497 496 p = subprocessio.SubprocessIOChunker(cmd, **_opts)
498 497
499 498 return ''.join(p), ''.join(p.error)
500 499 except (EnvironmentError, OSError) as err:
501 500 if safe_call:
502 return '', err
501 return '', safe_str(err).strip()
503 502 else:
504 503 cmd = ' '.join(cmd) # human friendly CMD
505 504 tb_err = ("Couldn't run svn command (%s).\n"
506 505 "Original error was:%s\n"
507 506 "Call options:%s\n"
508 507 % (cmd, err, _opts))
509 508 log.exception(tb_err)
510 509 raise exceptions.VcsException()(tb_err)
511 510
512 511 @reraise_safe_exceptions
513 512 def install_hooks(self, wire, force=False):
514 513 from vcsserver.hook_utils import install_svn_hooks
515 514 repo_path = wire['path']
516 515 binary_dir = settings.BINARY_DIR
517 516 executable = None
518 517 if binary_dir:
519 518 executable = os.path.join(binary_dir, 'python')
520 519 return install_svn_hooks(
521 520 repo_path, executable=executable, force_create=force)
522 521
523 522 @reraise_safe_exceptions
524 523 def get_hooks_info(self, wire):
525 524 from vcsserver.hook_utils import (
526 525 get_svn_pre_hook_version, get_svn_post_hook_version)
527 526 repo_path = wire['path']
528 527 return {
529 528 'pre_version': get_svn_pre_hook_version(repo_path),
530 529 'post_version': get_svn_post_hook_version(repo_path),
531 530 }
532 531
533 532 @reraise_safe_exceptions
534 533 def archive_repo(self, wire, archive_dest_path, kind, mtime, archive_at_path,
535 534 archive_dir_name, commit_id):
536 535
537 536 def walk_tree(root, root_dir, _commit_id):
538 537 """
539 538 Special recursive svn repo walker
540 539 """
541 540
542 541 filemode_default = 0o100644
543 542 filemode_executable = 0o100755
544 543
545 544 file_iter = svn.fs.dir_entries(root, root_dir)
546 545 for f_name in file_iter:
547 546 f_type = NODE_TYPE_MAPPING.get(file_iter[f_name].kind, None)
548 547
549 548 if f_type == 'dir':
550 549 # return only DIR, and then all entries in that dir
551 550 yield os.path.join(root_dir, f_name), {'mode': filemode_default}, f_type
552 551 new_root = os.path.join(root_dir, f_name)
553 552 for _f_name, _f_data, _f_type in walk_tree(root, new_root, _commit_id):
554 553 yield _f_name, _f_data, _f_type
555 554 else:
556 555 f_path = os.path.join(root_dir, f_name).rstrip('/')
557 556 prop_list = svn.fs.node_proplist(root, f_path)
558 557
559 558 f_mode = filemode_default
560 559 if prop_list.get('svn:executable'):
561 560 f_mode = filemode_executable
562 561
563 562 f_is_link = False
564 563 if prop_list.get('svn:special'):
565 564 f_is_link = True
566 565
567 566 data = {
568 567 'is_link': f_is_link,
569 568 'mode': f_mode,
570 569 'content_stream': svn.core.Stream(svn.fs.file_contents(root, f_path)).read
571 570 }
572 571
573 572 yield f_path, data, f_type
574 573
575 574 def file_walker(_commit_id, path):
576 575 repo = self._factory.repo(wire)
577 576 root = svn.fs.revision_root(svn.repos.fs(repo), int(commit_id))
578 577
579 578 def no_content():
580 579 raise NoContentException()
581 580
582 581 for f_name, f_data, f_type in walk_tree(root, path, _commit_id):
583 582 file_path = f_name
584 583
585 584 if f_type == 'dir':
586 585 mode = f_data['mode']
587 586 yield ArchiveNode(file_path, mode, False, no_content)
588 587 else:
589 588 mode = f_data['mode']
590 589 is_link = f_data['is_link']
591 590 data_stream = f_data['content_stream']
592 591 yield ArchiveNode(file_path, mode, is_link, data_stream)
593 592
594 593 return archive_repo(file_walker, archive_dest_path, kind, mtime, archive_at_path,
595 594 archive_dir_name, commit_id)
596 595
597 596
598 597 class SvnDiffer(object):
599 598 """
600 599 Utility to create diffs based on difflib and the Subversion api
601 600 """
602 601
603 602 binary_content = False
604 603
605 604 def __init__(
606 605 self, repo, src_rev, src_path, tgt_rev, tgt_path,
607 606 ignore_whitespace, context):
608 607 self.repo = repo
609 608 self.ignore_whitespace = ignore_whitespace
610 609 self.context = context
611 610
612 611 fsobj = svn.repos.fs(repo)
613 612
614 613 self.tgt_rev = tgt_rev
615 614 self.tgt_path = tgt_path or ''
616 615 self.tgt_root = svn.fs.revision_root(fsobj, tgt_rev)
617 616 self.tgt_kind = svn.fs.check_path(self.tgt_root, self.tgt_path)
618 617
619 618 self.src_rev = src_rev
620 619 self.src_path = src_path or self.tgt_path
621 620 self.src_root = svn.fs.revision_root(fsobj, src_rev)
622 621 self.src_kind = svn.fs.check_path(self.src_root, self.src_path)
623 622
624 623 self._validate()
625 624
626 625 def _validate(self):
627 626 if (self.tgt_kind != svn.core.svn_node_none and
628 627 self.src_kind != svn.core.svn_node_none and
629 628 self.src_kind != self.tgt_kind):
630 629 # TODO: johbo: proper error handling
631 630 raise Exception(
632 631 "Source and target are not compatible for diff generation. "
633 632 "Source type: %s, target type: %s" %
634 633 (self.src_kind, self.tgt_kind))
635 634
636 635 def generate_diff(self):
637 636 buf = StringIO.StringIO()
638 637 if self.tgt_kind == svn.core.svn_node_dir:
639 638 self._generate_dir_diff(buf)
640 639 else:
641 640 self._generate_file_diff(buf)
642 641 return buf.getvalue()
643 642
644 643 def _generate_dir_diff(self, buf):
645 644 editor = DiffChangeEditor()
646 645 editor_ptr, editor_baton = svn.delta.make_editor(editor)
647 646 svn.repos.dir_delta2(
648 647 self.src_root,
649 648 self.src_path,
650 649 '', # src_entry
651 650 self.tgt_root,
652 651 self.tgt_path,
653 652 editor_ptr, editor_baton,
654 653 authorization_callback_allow_all,
655 654 False, # text_deltas
656 655 svn.core.svn_depth_infinity, # depth
657 656 False, # entry_props
658 657 False, # ignore_ancestry
659 658 )
660 659
661 660 for path, __, change in sorted(editor.changes):
662 661 self._generate_node_diff(
663 662 buf, change, path, self.tgt_path, path, self.src_path)
664 663
665 664 def _generate_file_diff(self, buf):
666 665 change = None
667 666 if self.src_kind == svn.core.svn_node_none:
668 667 change = "add"
669 668 elif self.tgt_kind == svn.core.svn_node_none:
670 669 change = "delete"
671 670 tgt_base, tgt_path = vcspath.split(self.tgt_path)
672 671 src_base, src_path = vcspath.split(self.src_path)
673 672 self._generate_node_diff(
674 673 buf, change, tgt_path, tgt_base, src_path, src_base)
675 674
676 675 def _generate_node_diff(
677 676 self, buf, change, tgt_path, tgt_base, src_path, src_base):
678 677
679 678 if self.src_rev == self.tgt_rev and tgt_base == src_base:
680 679 # makes consistent behaviour with git/hg to return empty diff if
681 680 # we compare same revisions
682 681 return
683 682
684 683 tgt_full_path = vcspath.join(tgt_base, tgt_path)
685 684 src_full_path = vcspath.join(src_base, src_path)
686 685
687 686 self.binary_content = False
688 687 mime_type = self._get_mime_type(tgt_full_path)
689 688
690 689 if mime_type and not mime_type.startswith('text'):
691 690 self.binary_content = True
692 691 buf.write("=" * 67 + '\n')
693 692 buf.write("Cannot display: file marked as a binary type.\n")
694 693 buf.write("svn:mime-type = %s\n" % mime_type)
695 694 buf.write("Index: %s\n" % (tgt_path, ))
696 695 buf.write("=" * 67 + '\n')
697 696 buf.write("diff --git a/%(tgt_path)s b/%(tgt_path)s\n" % {
698 697 'tgt_path': tgt_path})
699 698
700 699 if change == 'add':
701 700 # TODO: johbo: SVN is missing a zero here compared to git
702 701 buf.write("new file mode 10644\n")
703 702
704 703 #TODO(marcink): intro to binary detection of svn patches
705 704 # if self.binary_content:
706 705 # buf.write('GIT binary patch\n')
707 706
708 707 buf.write("--- /dev/null\t(revision 0)\n")
709 708 src_lines = []
710 709 else:
711 710 if change == 'delete':
712 711 buf.write("deleted file mode 10644\n")
713 712
714 713 #TODO(marcink): intro to binary detection of svn patches
715 714 # if self.binary_content:
716 715 # buf.write('GIT binary patch\n')
717 716
718 717 buf.write("--- a/%s\t(revision %s)\n" % (
719 718 src_path, self.src_rev))
720 719 src_lines = self._svn_readlines(self.src_root, src_full_path)
721 720
722 721 if change == 'delete':
723 722 buf.write("+++ /dev/null\t(revision %s)\n" % (self.tgt_rev, ))
724 723 tgt_lines = []
725 724 else:
726 725 buf.write("+++ b/%s\t(revision %s)\n" % (
727 726 tgt_path, self.tgt_rev))
728 727 tgt_lines = self._svn_readlines(self.tgt_root, tgt_full_path)
729 728
730 729 if not self.binary_content:
731 730 udiff = svn_diff.unified_diff(
732 731 src_lines, tgt_lines, context=self.context,
733 732 ignore_blank_lines=self.ignore_whitespace,
734 733 ignore_case=False,
735 734 ignore_space_changes=self.ignore_whitespace)
736 735 buf.writelines(udiff)
737 736
738 737 def _get_mime_type(self, path):
739 738 try:
740 739 mime_type = svn.fs.node_prop(
741 740 self.tgt_root, path, svn.core.SVN_PROP_MIME_TYPE)
742 741 except svn.core.SubversionException:
743 742 mime_type = svn.fs.node_prop(
744 743 self.src_root, path, svn.core.SVN_PROP_MIME_TYPE)
745 744 return mime_type
746 745
747 746 def _svn_readlines(self, fs_root, node_path):
748 747 if self.binary_content:
749 748 return []
750 749 node_kind = svn.fs.check_path(fs_root, node_path)
751 750 if node_kind not in (
752 751 svn.core.svn_node_file, svn.core.svn_node_symlink):
753 752 return []
754 753 content = svn.core.Stream(svn.fs.file_contents(fs_root, node_path)).read()
755 754 return content.splitlines(True)
756 755
757 756
758 757 class DiffChangeEditor(svn.delta.Editor):
759 758 """
760 759 Records changes between two given revisions
761 760 """
762 761
763 762 def __init__(self):
764 763 self.changes = []
765 764
766 765 def delete_entry(self, path, revision, parent_baton, pool=None):
767 766 self.changes.append((path, None, 'delete'))
768 767
769 768 def add_file(
770 769 self, path, parent_baton, copyfrom_path, copyfrom_revision,
771 770 file_pool=None):
772 771 self.changes.append((path, 'file', 'add'))
773 772
774 773 def open_file(self, path, parent_baton, base_revision, file_pool=None):
775 774 self.changes.append((path, 'file', 'change'))
776 775
777 776
778 777 def authorization_callback_allow_all(root, path, pool):
779 778 return True
780 779
781 780
782 781 class TxnNodeProcessor(object):
783 782 """
784 783 Utility to process the change of one node within a transaction root.
785 784
786 785 It encapsulates the knowledge of how to add, update or remove
787 786 a node for a given transaction root. The purpose is to support the method
788 787 `SvnRemote.commit`.
789 788 """
790 789
791 790 def __init__(self, node, txn_root):
792 791 assert isinstance(node['path'], str)
793 792
794 793 self.node = node
795 794 self.txn_root = txn_root
796 795
797 796 def update(self):
798 797 self._ensure_parent_dirs()
799 798 self._add_file_if_node_does_not_exist()
800 799 self._update_file_content()
801 800 self._update_file_properties()
802 801
803 802 def remove(self):
804 803 svn.fs.delete(self.txn_root, self.node['path'])
805 804 # TODO: Clean up directory if empty
806 805
807 806 def _ensure_parent_dirs(self):
808 807 curdir = vcspath.dirname(self.node['path'])
809 808 dirs_to_create = []
810 809 while not self._svn_path_exists(curdir):
811 810 dirs_to_create.append(curdir)
812 811 curdir = vcspath.dirname(curdir)
813 812
814 813 for curdir in reversed(dirs_to_create):
815 814 log.debug('Creating missing directory "%s"', curdir)
816 815 svn.fs.make_dir(self.txn_root, curdir)
817 816
818 817 def _svn_path_exists(self, path):
819 818 path_status = svn.fs.check_path(self.txn_root, path)
820 819 return path_status != svn.core.svn_node_none
821 820
822 821 def _add_file_if_node_does_not_exist(self):
823 822 kind = svn.fs.check_path(self.txn_root, self.node['path'])
824 823 if kind == svn.core.svn_node_none:
825 824 svn.fs.make_file(self.txn_root, self.node['path'])
826 825
827 826 def _update_file_content(self):
828 827 assert isinstance(self.node['content'], str)
829 828 handler, baton = svn.fs.apply_textdelta(
830 829 self.txn_root, self.node['path'], None, None)
831 830 svn.delta.svn_txdelta_send_string(self.node['content'], handler, baton)
832 831
833 832 def _update_file_properties(self):
834 833 properties = self.node.get('properties', {})
835 834 for key, value in properties.iteritems():
836 835 svn.fs.change_node_prop(
837 836 self.txn_root, self.node['path'], key, value)
838 837
839 838
840 839 def apr_time_t(timestamp):
841 840 """
842 841 Convert a Python timestamp into APR timestamp type apr_time_t
843 842 """
844 843 return timestamp * 1E6
845 844
846 845
847 846 def svn_opt_revision_value_t(num):
848 847 """
849 848 Put `num` into a `svn_opt_revision_value_t` structure.
850 849 """
851 850 value = svn.core.svn_opt_revision_value_t()
852 851 value.number = num
853 852 revision = svn.core.svn_opt_revision_t()
854 853 revision.kind = svn.core.svn_opt_revision_number
855 854 revision.value = value
856 855 return revision
General Comments 0
You need to be logged in to leave comments. Login now