##// END OF EJS Templates
repositories: implemented faster dedicated checks for empty repositories
marcink -
r698:65b1b84c default
parent child Browse files
Show More
@@ -1,742 +1,751 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2019 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17 import collections
18 18 import logging
19 19 import os
20 20 import posixpath as vcspath
21 21 import re
22 22 import stat
23 23 import traceback
24 24 import urllib
25 25 import urllib2
26 26 from functools import wraps
27 27
28 28 import more_itertools
29 29 from dulwich import index, objects
30 30 from dulwich.client import HttpGitClient, LocalGitClient
31 31 from dulwich.errors import (
32 32 NotGitRepository, ChecksumMismatch, WrongObjectException,
33 33 MissingCommitError, ObjectMissing, HangupException,
34 34 UnexpectedCommandError)
35 35 from dulwich.repo import Repo as DulwichRepo, Tag
36 36 from dulwich.server import update_server_info
37 37
38 38 from vcsserver import exceptions, settings, subprocessio
39 39 from vcsserver.utils import safe_str
40 40 from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original
41 41 from vcsserver.hgcompat import (
42 42 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
43 43 from vcsserver.git_lfs.lib import LFSOidStore
44 44
45 45 DIR_STAT = stat.S_IFDIR
46 46 FILE_MODE = stat.S_IFMT
47 47 GIT_LINK = objects.S_IFGITLINK
48 48
49 49 log = logging.getLogger(__name__)
50 50
51 51
52 52 def reraise_safe_exceptions(func):
53 53 """Converts Dulwich exceptions to something neutral."""
54 54 @wraps(func)
55 55 def wrapper(*args, **kwargs):
56 56 try:
57 57 return func(*args, **kwargs)
58 58 except (ChecksumMismatch, WrongObjectException, MissingCommitError,
59 59 ObjectMissing) as e:
60 60 exc = exceptions.LookupException(e)
61 61 raise exc(e)
62 62 except (HangupException, UnexpectedCommandError) as e:
63 63 exc = exceptions.VcsException(e)
64 64 raise exc(e)
65 65 except Exception as e:
66 66 # NOTE(marcink): becuase of how dulwich handles some exceptions
67 67 # (KeyError on empty repos), we cannot track this and catch all
68 68 # exceptions, it's an exceptions from other handlers
69 69 #if not hasattr(e, '_vcs_kind'):
70 70 #log.exception("Unhandled exception in git remote call")
71 71 #raise_from_original(exceptions.UnhandledException)
72 72 raise
73 73 return wrapper
74 74
75 75
76 76 class Repo(DulwichRepo):
77 77 """
78 78 A wrapper for dulwich Repo class.
79 79
80 80 Since dulwich is sometimes keeping .idx file descriptors open, it leads to
81 81 "Too many open files" error. We need to close all opened file descriptors
82 82 once the repo object is destroyed.
83 83
84 84 TODO: mikhail: please check if we need this wrapper after updating dulwich
85 85 to 0.12.0 +
86 86 """
87 87 def __del__(self):
88 88 if hasattr(self, 'object_store'):
89 89 self.close()
90 90
91 91
92 92 class GitFactory(RepoFactory):
93 93 repo_type = 'git'
94 94
95 95 def _create_repo(self, wire, create):
96 96 repo_path = str_to_dulwich(wire['path'])
97 97 return Repo(repo_path)
98 98
99 99
100 100 class GitRemote(object):
101 101
102 102 def __init__(self, factory):
103 103 self._factory = factory
104 104 self.peeled_ref_marker = '^{}'
105 105 self._bulk_methods = {
106 106 "author": self.commit_attribute,
107 107 "date": self.get_object_attrs,
108 108 "message": self.commit_attribute,
109 109 "parents": self.commit_attribute,
110 110 "_commit": self.revision,
111 111 }
112 112
113 113 def _wire_to_config(self, wire):
114 114 if 'config' in wire:
115 115 return dict([(x[0] + '_' + x[1], x[2]) for x in wire['config']])
116 116 return {}
117 117
118 118 def _assign_ref(self, wire, ref, commit_id):
119 119 repo = self._factory.repo(wire)
120 120 repo[ref] = commit_id
121 121
122 122 def _remote_conf(self, config):
123 123 params = [
124 124 '-c', 'core.askpass=""',
125 125 ]
126 126 ssl_cert_dir = config.get('vcs_ssl_dir')
127 127 if ssl_cert_dir:
128 128 params.extend(['-c', 'http.sslCAinfo={}'.format(ssl_cert_dir)])
129 129 return params
130 130
131 131 @reraise_safe_exceptions
132 def is_empty(self, wire):
133 repo = self._factory.repo(wire)
134 try:
135 return not repo.head()
136 except Exception:
137 log.exception("failed to read object_store")
138 return True
139
140 @reraise_safe_exceptions
132 141 def add_object(self, wire, content):
133 142 repo = self._factory.repo(wire)
134 143 blob = objects.Blob()
135 144 blob.set_raw_string(content)
136 145 repo.object_store.add_object(blob)
137 146 return blob.id
138 147
139 148 @reraise_safe_exceptions
140 149 def assert_correct_path(self, wire):
141 150 path = wire.get('path')
142 151 try:
143 152 self._factory.repo(wire)
144 153 except NotGitRepository as e:
145 154 tb = traceback.format_exc()
146 155 log.debug("Invalid Git path `%s`, tb: %s", path, tb)
147 156 return False
148 157
149 158 return True
150 159
151 160 @reraise_safe_exceptions
152 161 def bare(self, wire):
153 162 repo = self._factory.repo(wire)
154 163 return repo.bare
155 164
156 165 @reraise_safe_exceptions
157 166 def blob_as_pretty_string(self, wire, sha):
158 167 repo = self._factory.repo(wire)
159 168 return repo[sha].as_pretty_string()
160 169
161 170 @reraise_safe_exceptions
162 171 def blob_raw_length(self, wire, sha):
163 172 repo = self._factory.repo(wire)
164 173 blob = repo[sha]
165 174 return blob.raw_length()
166 175
167 176 def _parse_lfs_pointer(self, raw_content):
168 177
169 178 spec_string = 'version https://git-lfs.github.com/spec'
170 179 if raw_content and raw_content.startswith(spec_string):
171 180 pattern = re.compile(r"""
172 181 (?:\n)?
173 182 ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
174 183 ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
175 184 ^size[ ](?P<oid_size>[0-9]+)\n
176 185 (?:\n)?
177 186 """, re.VERBOSE | re.MULTILINE)
178 187 match = pattern.match(raw_content)
179 188 if match:
180 189 return match.groupdict()
181 190
182 191 return {}
183 192
184 193 @reraise_safe_exceptions
185 194 def is_large_file(self, wire, sha):
186 195 repo = self._factory.repo(wire)
187 196 blob = repo[sha]
188 197 return self._parse_lfs_pointer(blob.as_raw_string())
189 198
190 199 @reraise_safe_exceptions
191 200 def in_largefiles_store(self, wire, oid):
192 201 repo = self._factory.repo(wire)
193 202 conf = self._wire_to_config(wire)
194 203
195 204 store_location = conf.get('vcs_git_lfs_store_location')
196 205 if store_location:
197 206 repo_name = repo.path
198 207 store = LFSOidStore(
199 208 oid=oid, repo=repo_name, store_location=store_location)
200 209 return store.has_oid()
201 210
202 211 return False
203 212
204 213 @reraise_safe_exceptions
205 214 def store_path(self, wire, oid):
206 215 repo = self._factory.repo(wire)
207 216 conf = self._wire_to_config(wire)
208 217
209 218 store_location = conf.get('vcs_git_lfs_store_location')
210 219 if store_location:
211 220 repo_name = repo.path
212 221 store = LFSOidStore(
213 222 oid=oid, repo=repo_name, store_location=store_location)
214 223 return store.oid_path
215 224 raise ValueError('Unable to fetch oid with path {}'.format(oid))
216 225
217 226 @reraise_safe_exceptions
218 227 def bulk_request(self, wire, rev, pre_load):
219 228 result = {}
220 229 for attr in pre_load:
221 230 try:
222 231 method = self._bulk_methods[attr]
223 232 args = [wire, rev]
224 233 if attr == "date":
225 234 args.extend(["commit_time", "commit_timezone"])
226 235 elif attr in ["author", "message", "parents"]:
227 236 args.append(attr)
228 237 result[attr] = method(*args)
229 238 except KeyError as e:
230 239 raise exceptions.VcsException(e)(
231 240 "Unknown bulk attribute: %s" % attr)
232 241 return result
233 242
234 243 def _build_opener(self, url):
235 244 handlers = []
236 245 url_obj = url_parser(url)
237 246 _, authinfo = url_obj.authinfo()
238 247
239 248 if authinfo:
240 249 # create a password manager
241 250 passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
242 251 passmgr.add_password(*authinfo)
243 252
244 253 handlers.extend((httpbasicauthhandler(passmgr),
245 254 httpdigestauthhandler(passmgr)))
246 255
247 256 return urllib2.build_opener(*handlers)
248 257
249 258 @reraise_safe_exceptions
250 259 def check_url(self, url, config):
251 260 url_obj = url_parser(url)
252 261 test_uri, _ = url_obj.authinfo()
253 262 url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
254 263 url_obj.query = obfuscate_qs(url_obj.query)
255 264 cleaned_uri = str(url_obj)
256 265 log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
257 266
258 267 if not test_uri.endswith('info/refs'):
259 268 test_uri = test_uri.rstrip('/') + '/info/refs'
260 269
261 270 o = self._build_opener(url)
262 271 o.addheaders = [('User-Agent', 'git/1.7.8.0')] # fake some git
263 272
264 273 q = {"service": 'git-upload-pack'}
265 274 qs = '?%s' % urllib.urlencode(q)
266 275 cu = "%s%s" % (test_uri, qs)
267 276 req = urllib2.Request(cu, None, {})
268 277
269 278 try:
270 279 log.debug("Trying to open URL %s", cleaned_uri)
271 280 resp = o.open(req)
272 281 if resp.code != 200:
273 282 raise exceptions.URLError()('Return Code is not 200')
274 283 except Exception as e:
275 284 log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
276 285 # means it cannot be cloned
277 286 raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e))
278 287
279 288 # now detect if it's proper git repo
280 289 gitdata = resp.read()
281 290 if 'service=git-upload-pack' in gitdata:
282 291 pass
283 292 elif re.findall(r'[0-9a-fA-F]{40}\s+refs', gitdata):
284 293 # old style git can return some other format !
285 294 pass
286 295 else:
287 296 raise exceptions.URLError()(
288 297 "url [%s] does not look like an git" % (cleaned_uri,))
289 298
290 299 return True
291 300
292 301 @reraise_safe_exceptions
293 302 def clone(self, wire, url, deferred, valid_refs, update_after_clone):
294 303 # TODO(marcink): deprecate this method. Last i checked we don't use it anymore
295 304 remote_refs = self.pull(wire, url, apply_refs=False)
296 305 repo = self._factory.repo(wire)
297 306 if isinstance(valid_refs, list):
298 307 valid_refs = tuple(valid_refs)
299 308
300 309 for k in remote_refs:
301 310 # only parse heads/tags and skip so called deferred tags
302 311 if k.startswith(valid_refs) and not k.endswith(deferred):
303 312 repo[k] = remote_refs[k]
304 313
305 314 if update_after_clone:
306 315 # we want to checkout HEAD
307 316 repo["HEAD"] = remote_refs["HEAD"]
308 317 index.build_index_from_tree(repo.path, repo.index_path(),
309 318 repo.object_store, repo["HEAD"].tree)
310 319
311 320 # TODO: this is quite complex, check if that can be simplified
312 321 @reraise_safe_exceptions
313 322 def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
314 323 repo = self._factory.repo(wire)
315 324 object_store = repo.object_store
316 325
317 326 # Create tree and populates it with blobs
318 327 commit_tree = commit_tree and repo[commit_tree] or objects.Tree()
319 328
320 329 for node in updated:
321 330 # Compute subdirs if needed
322 331 dirpath, nodename = vcspath.split(node['path'])
323 332 dirnames = map(safe_str, dirpath and dirpath.split('/') or [])
324 333 parent = commit_tree
325 334 ancestors = [('', parent)]
326 335
327 336 # Tries to dig for the deepest existing tree
328 337 while dirnames:
329 338 curdir = dirnames.pop(0)
330 339 try:
331 340 dir_id = parent[curdir][1]
332 341 except KeyError:
333 342 # put curdir back into dirnames and stops
334 343 dirnames.insert(0, curdir)
335 344 break
336 345 else:
337 346 # If found, updates parent
338 347 parent = repo[dir_id]
339 348 ancestors.append((curdir, parent))
340 349 # Now parent is deepest existing tree and we need to create
341 350 # subtrees for dirnames (in reverse order)
342 351 # [this only applies for nodes from added]
343 352 new_trees = []
344 353
345 354 blob = objects.Blob.from_string(node['content'])
346 355
347 356 if dirnames:
348 357 # If there are trees which should be created we need to build
349 358 # them now (in reverse order)
350 359 reversed_dirnames = list(reversed(dirnames))
351 360 curtree = objects.Tree()
352 361 curtree[node['node_path']] = node['mode'], blob.id
353 362 new_trees.append(curtree)
354 363 for dirname in reversed_dirnames[:-1]:
355 364 newtree = objects.Tree()
356 365 newtree[dirname] = (DIR_STAT, curtree.id)
357 366 new_trees.append(newtree)
358 367 curtree = newtree
359 368 parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
360 369 else:
361 370 parent.add(
362 371 name=node['node_path'], mode=node['mode'], hexsha=blob.id)
363 372
364 373 new_trees.append(parent)
365 374 # Update ancestors
366 375 reversed_ancestors = reversed(
367 376 [(a[1], b[1], b[0]) for a, b in zip(ancestors, ancestors[1:])])
368 377 for parent, tree, path in reversed_ancestors:
369 378 parent[path] = (DIR_STAT, tree.id)
370 379 object_store.add_object(tree)
371 380
372 381 object_store.add_object(blob)
373 382 for tree in new_trees:
374 383 object_store.add_object(tree)
375 384
376 385 for node_path in removed:
377 386 paths = node_path.split('/')
378 387 tree = commit_tree
379 388 trees = [tree]
380 389 # Traverse deep into the forest...
381 390 for path in paths:
382 391 try:
383 392 obj = repo[tree[path][1]]
384 393 if isinstance(obj, objects.Tree):
385 394 trees.append(obj)
386 395 tree = obj
387 396 except KeyError:
388 397 break
389 398 # Cut down the blob and all rotten trees on the way back...
390 399 for path, tree in reversed(zip(paths, trees)):
391 400 del tree[path]
392 401 if tree:
393 402 # This tree still has elements - don't remove it or any
394 403 # of it's parents
395 404 break
396 405
397 406 object_store.add_object(commit_tree)
398 407
399 408 # Create commit
400 409 commit = objects.Commit()
401 410 commit.tree = commit_tree.id
402 411 for k, v in commit_data.iteritems():
403 412 setattr(commit, k, v)
404 413 object_store.add_object(commit)
405 414
406 415 ref = 'refs/heads/%s' % branch
407 416 repo.refs[ref] = commit.id
408 417
409 418 return commit.id
410 419
411 420 @reraise_safe_exceptions
412 421 def pull(self, wire, url, apply_refs=True, refs=None, update_after=False):
413 422 if url != 'default' and '://' not in url:
414 423 client = LocalGitClient(url)
415 424 else:
416 425 url_obj = url_parser(url)
417 426 o = self._build_opener(url)
418 427 url, _ = url_obj.authinfo()
419 428 client = HttpGitClient(base_url=url, opener=o)
420 429 repo = self._factory.repo(wire)
421 430
422 431 determine_wants = repo.object_store.determine_wants_all
423 432 if refs:
424 433 def determine_wants_requested(references):
425 434 return [references[r] for r in references if r in refs]
426 435 determine_wants = determine_wants_requested
427 436
428 437 try:
429 438 remote_refs = client.fetch(
430 439 path=url, target=repo, determine_wants=determine_wants)
431 440 except NotGitRepository as e:
432 441 log.warning(
433 442 'Trying to fetch from "%s" failed, not a Git repository.', url)
434 443 # Exception can contain unicode which we convert
435 444 raise exceptions.AbortException(e)(repr(e))
436 445
437 446 # mikhail: client.fetch() returns all the remote refs, but fetches only
438 447 # refs filtered by `determine_wants` function. We need to filter result
439 448 # as well
440 449 if refs:
441 450 remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
442 451
443 452 if apply_refs:
444 453 # TODO: johbo: Needs proper test coverage with a git repository
445 454 # that contains a tag object, so that we would end up with
446 455 # a peeled ref at this point.
447 456 for k in remote_refs:
448 457 if k.endswith(self.peeled_ref_marker):
449 458 log.debug("Skipping peeled reference %s", k)
450 459 continue
451 460 repo[k] = remote_refs[k]
452 461
453 462 if refs and not update_after:
454 463 # mikhail: explicitly set the head to the last ref.
455 464 repo['HEAD'] = remote_refs[refs[-1]]
456 465
457 466 if update_after:
458 467 # we want to checkout HEAD
459 468 repo["HEAD"] = remote_refs["HEAD"]
460 469 index.build_index_from_tree(repo.path, repo.index_path(),
461 470 repo.object_store, repo["HEAD"].tree)
462 471 return remote_refs
463 472
464 473 @reraise_safe_exceptions
465 474 def sync_fetch(self, wire, url, refs=None):
466 475 repo = self._factory.repo(wire)
467 476 if refs and not isinstance(refs, (list, tuple)):
468 477 refs = [refs]
469 478 config = self._wire_to_config(wire)
470 479 # get all remote refs we'll use to fetch later
471 480 output, __ = self.run_git_command(
472 481 wire, ['ls-remote', url], fail_on_stderr=False,
473 482 _copts=self._remote_conf(config),
474 483 extra_env={'GIT_TERMINAL_PROMPT': '0'})
475 484
476 485 remote_refs = collections.OrderedDict()
477 486 fetch_refs = []
478 487
479 488 for ref_line in output.splitlines():
480 489 sha, ref = ref_line.split('\t')
481 490 sha = sha.strip()
482 491 if ref in remote_refs:
483 492 # duplicate, skip
484 493 continue
485 494 if ref.endswith(self.peeled_ref_marker):
486 495 log.debug("Skipping peeled reference %s", ref)
487 496 continue
488 497 # don't sync HEAD
489 498 if ref in ['HEAD']:
490 499 continue
491 500
492 501 remote_refs[ref] = sha
493 502
494 503 if refs and sha in refs:
495 504 # we filter fetch using our specified refs
496 505 fetch_refs.append('{}:{}'.format(ref, ref))
497 506 elif not refs:
498 507 fetch_refs.append('{}:{}'.format(ref, ref))
499 508 log.debug('Finished obtaining fetch refs, total: %s', len(fetch_refs))
500 509 if fetch_refs:
501 510 for chunk in more_itertools.chunked(fetch_refs, 1024 * 4):
502 511 fetch_refs_chunks = list(chunk)
503 512 log.debug('Fetching %s refs from import url', len(fetch_refs_chunks))
504 513 _out, _err = self.run_git_command(
505 514 wire, ['fetch', url, '--force', '--prune', '--'] + fetch_refs_chunks,
506 515 fail_on_stderr=False,
507 516 _copts=self._remote_conf(config),
508 517 extra_env={'GIT_TERMINAL_PROMPT': '0'})
509 518
510 519 return remote_refs
511 520
512 521 @reraise_safe_exceptions
513 522 def sync_push(self, wire, url, refs=None):
514 523 if not self.check_url(url, wire):
515 524 return
516 525 config = self._wire_to_config(wire)
517 526 repo = self._factory.repo(wire)
518 527 self.run_git_command(
519 528 wire, ['push', url, '--mirror'], fail_on_stderr=False,
520 529 _copts=self._remote_conf(config),
521 530 extra_env={'GIT_TERMINAL_PROMPT': '0'})
522 531
523 532 @reraise_safe_exceptions
524 533 def get_remote_refs(self, wire, url):
525 534 repo = Repo(url)
526 535 return repo.get_refs()
527 536
528 537 @reraise_safe_exceptions
529 538 def get_description(self, wire):
530 539 repo = self._factory.repo(wire)
531 540 return repo.get_description()
532 541
533 542 @reraise_safe_exceptions
534 543 def get_missing_revs(self, wire, rev1, rev2, path2):
535 544 repo = self._factory.repo(wire)
536 545 LocalGitClient(thin_packs=False).fetch(path2, repo)
537 546
538 547 wire_remote = wire.copy()
539 548 wire_remote['path'] = path2
540 549 repo_remote = self._factory.repo(wire_remote)
541 550 LocalGitClient(thin_packs=False).fetch(wire["path"], repo_remote)
542 551
543 552 revs = [
544 553 x.commit.id
545 554 for x in repo_remote.get_walker(include=[rev2], exclude=[rev1])]
546 555 return revs
547 556
548 557 @reraise_safe_exceptions
549 558 def get_object(self, wire, sha):
550 559 repo = self._factory.repo(wire)
551 560 obj = repo.get_object(sha)
552 561 commit_id = obj.id
553 562
554 563 if isinstance(obj, Tag):
555 564 commit_id = obj.object[1]
556 565
557 566 return {
558 567 'id': obj.id,
559 568 'type': obj.type_name,
560 569 'commit_id': commit_id
561 570 }
562 571
563 572 @reraise_safe_exceptions
564 573 def get_object_attrs(self, wire, sha, *attrs):
565 574 repo = self._factory.repo(wire)
566 575 obj = repo.get_object(sha)
567 576 return list(getattr(obj, a) for a in attrs)
568 577
569 578 @reraise_safe_exceptions
570 579 def get_refs(self, wire):
571 580 repo = self._factory.repo(wire)
572 581 result = {}
573 582 for ref, sha in repo.refs.as_dict().items():
574 583 peeled_sha = repo.get_peeled(ref)
575 584 result[ref] = peeled_sha
576 585 return result
577 586
578 587 @reraise_safe_exceptions
579 588 def get_refs_path(self, wire):
580 589 repo = self._factory.repo(wire)
581 590 return repo.refs.path
582 591
583 592 @reraise_safe_exceptions
584 593 def head(self, wire, show_exc=True):
585 594 repo = self._factory.repo(wire)
586 595 try:
587 596 return repo.head()
588 597 except Exception:
589 598 if show_exc:
590 599 raise
591 600
592 601 @reraise_safe_exceptions
593 602 def init(self, wire):
594 603 repo_path = str_to_dulwich(wire['path'])
595 604 self.repo = Repo.init(repo_path)
596 605
597 606 @reraise_safe_exceptions
598 607 def init_bare(self, wire):
599 608 repo_path = str_to_dulwich(wire['path'])
600 609 self.repo = Repo.init_bare(repo_path)
601 610
602 611 @reraise_safe_exceptions
603 612 def revision(self, wire, rev):
604 613 repo = self._factory.repo(wire)
605 614 obj = repo[rev]
606 615 obj_data = {
607 616 'id': obj.id,
608 617 }
609 618 try:
610 619 obj_data['tree'] = obj.tree
611 620 except AttributeError:
612 621 pass
613 622 return obj_data
614 623
615 624 @reraise_safe_exceptions
616 625 def commit_attribute(self, wire, rev, attr):
617 626 repo = self._factory.repo(wire)
618 627 obj = repo[rev]
619 628 return getattr(obj, attr)
620 629
621 630 @reraise_safe_exceptions
622 631 def set_refs(self, wire, key, value):
623 632 repo = self._factory.repo(wire)
624 633 repo.refs[key] = value
625 634
626 635 @reraise_safe_exceptions
627 636 def remove_ref(self, wire, key):
628 637 repo = self._factory.repo(wire)
629 638 del repo.refs[key]
630 639
631 640 @reraise_safe_exceptions
632 641 def tree_changes(self, wire, source_id, target_id):
633 642 repo = self._factory.repo(wire)
634 643 source = repo[source_id].tree if source_id else None
635 644 target = repo[target_id].tree
636 645 result = repo.object_store.tree_changes(source, target)
637 646 return list(result)
638 647
639 648 @reraise_safe_exceptions
640 649 def tree_items(self, wire, tree_id):
641 650 repo = self._factory.repo(wire)
642 651 tree = repo[tree_id]
643 652
644 653 result = []
645 654 for item in tree.iteritems():
646 655 item_sha = item.sha
647 656 item_mode = item.mode
648 657
649 658 if FILE_MODE(item_mode) == GIT_LINK:
650 659 item_type = "link"
651 660 else:
652 661 item_type = repo[item_sha].type_name
653 662
654 663 result.append((item.path, item_mode, item_sha, item_type))
655 664 return result
656 665
657 666 @reraise_safe_exceptions
658 667 def update_server_info(self, wire):
659 668 repo = self._factory.repo(wire)
660 669 update_server_info(repo)
661 670
662 671 @reraise_safe_exceptions
663 672 def discover_git_version(self):
664 673 stdout, _ = self.run_git_command(
665 674 {}, ['--version'], _bare=True, _safe=True)
666 675 prefix = 'git version'
667 676 if stdout.startswith(prefix):
668 677 stdout = stdout[len(prefix):]
669 678 return stdout.strip()
670 679
671 680 @reraise_safe_exceptions
672 681 def run_git_command(self, wire, cmd, **opts):
673 682 path = wire.get('path', None)
674 683
675 684 if path and os.path.isdir(path):
676 685 opts['cwd'] = path
677 686
678 687 if '_bare' in opts:
679 688 _copts = []
680 689 del opts['_bare']
681 690 else:
682 691 _copts = ['-c', 'core.quotepath=false', ]
683 692 safe_call = False
684 693 if '_safe' in opts:
685 694 # no exc on failure
686 695 del opts['_safe']
687 696 safe_call = True
688 697
689 698 if '_copts' in opts:
690 699 _copts.extend(opts['_copts'] or [])
691 700 del opts['_copts']
692 701
693 702 gitenv = os.environ.copy()
694 703 gitenv.update(opts.pop('extra_env', {}))
695 704 # need to clean fix GIT_DIR !
696 705 if 'GIT_DIR' in gitenv:
697 706 del gitenv['GIT_DIR']
698 707 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
699 708 gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1'
700 709
701 710 cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
702 711 _opts = {'env': gitenv, 'shell': False}
703 712
704 713 try:
705 714 _opts.update(opts)
706 715 p = subprocessio.SubprocessIOChunker(cmd, **_opts)
707 716
708 717 return ''.join(p), ''.join(p.error)
709 718 except (EnvironmentError, OSError) as err:
710 719 cmd = ' '.join(cmd) # human friendly CMD
711 720 tb_err = ("Couldn't run git command (%s).\n"
712 721 "Original error was:%s\n"
713 722 "Call options:%s\n"
714 723 % (cmd, err, _opts))
715 724 log.exception(tb_err)
716 725 if safe_call:
717 726 return '', err
718 727 else:
719 728 raise exceptions.VcsException()(tb_err)
720 729
721 730 @reraise_safe_exceptions
722 731 def install_hooks(self, wire, force=False):
723 732 from vcsserver.hook_utils import install_git_hooks
724 733 repo = self._factory.repo(wire)
725 734 return install_git_hooks(repo.path, repo.bare, force_create=force)
726 735
727 736 @reraise_safe_exceptions
728 737 def get_hooks_info(self, wire):
729 738 from vcsserver.hook_utils import (
730 739 get_git_pre_hook_version, get_git_post_hook_version)
731 740 repo = self._factory.repo(wire)
732 741 return {
733 742 'pre_version': get_git_pre_hook_version(repo.path, repo.bare),
734 743 'post_version': get_git_post_hook_version(repo.path, repo.bare),
735 744 }
736 745
737 746
738 747 def str_to_dulwich(value):
739 748 """
740 749 Dulwich 0.10.1a requires `unicode` objects to be passed in.
741 750 """
742 751 return value.decode(settings.WIRE_ENCODING)
@@ -1,846 +1,856 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2019 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import io
19 19 import logging
20 20 import stat
21 21 import urllib
22 22 import urllib2
23 23 import traceback
24 24
25 25 from hgext import largefiles, rebase
26 26 from hgext.strip import strip as hgext_strip
27 27 from mercurial import commands
28 28 from mercurial import unionrepo
29 29 from mercurial import verify
30 30
31 31 import vcsserver
32 32 from vcsserver import exceptions
33 33 from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original
34 34 from vcsserver.hgcompat import (
35 35 archival, bin, clone, config as hgconfig, diffopts, hex, get_ctx,
36 36 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler,
37 37 makepeer, instance, match, memctx, exchange, memfilectx, nullrev, hg_merge,
38 38 patch, peer, revrange, ui, hg_tag, Abort, LookupError, RepoError,
39 39 RepoLookupError, InterventionRequired, RequirementError)
40 40
41 41 log = logging.getLogger(__name__)
42 42
43 43
44 44 def make_ui_from_config(repo_config):
45 45
46 46 class LoggingUI(ui.ui):
47 47 def status(self, *msg, **opts):
48 48 log.info(' '.join(msg).rstrip('\n'))
49 49 super(LoggingUI, self).status(*msg, **opts)
50 50
51 51 def warn(self, *msg, **opts):
52 52 log.warn(' '.join(msg).rstrip('\n'))
53 53 super(LoggingUI, self).warn(*msg, **opts)
54 54
55 55 def error(self, *msg, **opts):
56 56 log.error(' '.join(msg).rstrip('\n'))
57 57 super(LoggingUI, self).error(*msg, **opts)
58 58
59 59 def note(self, *msg, **opts):
60 60 log.info(' '.join(msg).rstrip('\n'))
61 61 super(LoggingUI, self).note(*msg, **opts)
62 62
63 63 def debug(self, *msg, **opts):
64 64 log.debug(' '.join(msg).rstrip('\n'))
65 65 super(LoggingUI, self).debug(*msg, **opts)
66 66
67 67 baseui = LoggingUI()
68 68
69 69 # clean the baseui object
70 70 baseui._ocfg = hgconfig.config()
71 71 baseui._ucfg = hgconfig.config()
72 72 baseui._tcfg = hgconfig.config()
73 73
74 74 for section, option, value in repo_config:
75 75 baseui.setconfig(section, option, value)
76 76
77 77 # make our hgweb quiet so it doesn't print output
78 78 baseui.setconfig('ui', 'quiet', 'true')
79 79
80 80 baseui.setconfig('ui', 'paginate', 'never')
81 81 # for better Error reporting of Mercurial
82 82 baseui.setconfig('ui', 'message-output', 'stderr')
83 83
84 84 # force mercurial to only use 1 thread, otherwise it may try to set a
85 85 # signal in a non-main thread, thus generating a ValueError.
86 86 baseui.setconfig('worker', 'numcpus', 1)
87 87
88 88 # If there is no config for the largefiles extension, we explicitly disable
89 89 # it here. This overrides settings from repositories hgrc file. Recent
90 90 # mercurial versions enable largefiles in hgrc on clone from largefile
91 91 # repo.
92 92 if not baseui.hasconfig('extensions', 'largefiles'):
93 93 log.debug('Explicitly disable largefiles extension for repo.')
94 94 baseui.setconfig('extensions', 'largefiles', '!')
95 95
96 96 return baseui
97 97
98 98
99 99 def reraise_safe_exceptions(func):
100 100 """Decorator for converting mercurial exceptions to something neutral."""
101 101 def wrapper(*args, **kwargs):
102 102 try:
103 103 return func(*args, **kwargs)
104 104 except (Abort, InterventionRequired) as e:
105 105 raise_from_original(exceptions.AbortException(e))
106 106 except RepoLookupError as e:
107 107 raise_from_original(exceptions.LookupException(e))
108 108 except RequirementError as e:
109 109 raise_from_original(exceptions.RequirementException(e))
110 110 except RepoError as e:
111 111 raise_from_original(exceptions.VcsException(e))
112 112 except LookupError as e:
113 113 raise_from_original(exceptions.LookupException(e))
114 114 except Exception as e:
115 115 if not hasattr(e, '_vcs_kind'):
116 116 log.exception("Unhandled exception in hg remote call")
117 117 raise_from_original(exceptions.UnhandledException(e))
118 118
119 119 raise
120 120 return wrapper
121 121
122 122
123 123 class MercurialFactory(RepoFactory):
124 124 repo_type = 'hg'
125 125
126 126 def _create_config(self, config, hooks=True):
127 127 if not hooks:
128 128 hooks_to_clean = frozenset((
129 129 'changegroup.repo_size', 'preoutgoing.pre_pull',
130 130 'outgoing.pull_logger', 'prechangegroup.pre_push'))
131 131 new_config = []
132 132 for section, option, value in config:
133 133 if section == 'hooks' and option in hooks_to_clean:
134 134 continue
135 135 new_config.append((section, option, value))
136 136 config = new_config
137 137
138 138 baseui = make_ui_from_config(config)
139 139 return baseui
140 140
141 141 def _create_repo(self, wire, create):
142 142 baseui = self._create_config(wire["config"])
143 143 return instance(baseui, wire["path"], create)
144 144
145 145
146 146 class HgRemote(object):
147 147
148 148 def __init__(self, factory):
149 149 self._factory = factory
150 150
151 151 self._bulk_methods = {
152 152 "affected_files": self.ctx_files,
153 153 "author": self.ctx_user,
154 154 "branch": self.ctx_branch,
155 155 "children": self.ctx_children,
156 156 "date": self.ctx_date,
157 157 "message": self.ctx_description,
158 158 "parents": self.ctx_parents,
159 159 "status": self.ctx_status,
160 160 "obsolete": self.ctx_obsolete,
161 161 "phase": self.ctx_phase,
162 162 "hidden": self.ctx_hidden,
163 163 "_file_paths": self.ctx_list,
164 164 }
165 165
166 166 def _get_ctx(self, repo, ref):
167 167 return get_ctx(repo, ref)
168 168
169 169 @reraise_safe_exceptions
170 170 def discover_hg_version(self):
171 171 from mercurial import util
172 172 return util.version()
173 173
174 174 @reraise_safe_exceptions
175 def is_empty(self, wire):
176 repo = self._factory.repo(wire)
177
178 try:
179 return len(repo) == 0
180 except Exception:
181 log.exception("failed to read object_store")
182 return False
183
184 @reraise_safe_exceptions
175 185 def archive_repo(self, archive_path, mtime, file_info, kind):
176 186 if kind == "tgz":
177 187 archiver = archival.tarit(archive_path, mtime, "gz")
178 188 elif kind == "tbz2":
179 189 archiver = archival.tarit(archive_path, mtime, "bz2")
180 190 elif kind == 'zip':
181 191 archiver = archival.zipit(archive_path, mtime)
182 192 else:
183 193 raise exceptions.ArchiveException()(
184 194 'Remote does not support: "%s".' % kind)
185 195
186 196 for f_path, f_mode, f_is_link, f_content in file_info:
187 197 archiver.addfile(f_path, f_mode, f_is_link, f_content)
188 198 archiver.done()
189 199
190 200 @reraise_safe_exceptions
191 201 def bookmarks(self, wire):
192 202 repo = self._factory.repo(wire)
193 203 return dict(repo._bookmarks)
194 204
195 205 @reraise_safe_exceptions
196 206 def branches(self, wire, normal, closed):
197 207 repo = self._factory.repo(wire)
198 208 iter_branches = repo.branchmap().iterbranches()
199 209 bt = {}
200 210 for branch_name, _heads, tip, is_closed in iter_branches:
201 211 if normal and not is_closed:
202 212 bt[branch_name] = tip
203 213 if closed and is_closed:
204 214 bt[branch_name] = tip
205 215
206 216 return bt
207 217
208 218 @reraise_safe_exceptions
209 219 def bulk_request(self, wire, rev, pre_load):
210 220 result = {}
211 221 for attr in pre_load:
212 222 try:
213 223 method = self._bulk_methods[attr]
214 224 result[attr] = method(wire, rev)
215 225 except KeyError as e:
216 226 raise exceptions.VcsException(e)(
217 227 'Unknown bulk attribute: "%s"' % attr)
218 228 return result
219 229
220 230 @reraise_safe_exceptions
221 231 def clone(self, wire, source, dest, update_after_clone=False, hooks=True):
222 232 baseui = self._factory._create_config(wire["config"], hooks=hooks)
223 233 clone(baseui, source, dest, noupdate=not update_after_clone)
224 234
225 235 @reraise_safe_exceptions
226 236 def commitctx(
227 237 self, wire, message, parents, commit_time, commit_timezone,
228 238 user, files, extra, removed, updated):
229 239
230 240 repo = self._factory.repo(wire)
231 241 baseui = self._factory._create_config(wire['config'])
232 242 publishing = baseui.configbool('phases', 'publish')
233 243 if publishing:
234 244 new_commit = 'public'
235 245 else:
236 246 new_commit = 'draft'
237 247
238 248 def _filectxfn(_repo, ctx, path):
239 249 """
240 250 Marks given path as added/changed/removed in a given _repo. This is
241 251 for internal mercurial commit function.
242 252 """
243 253
244 254 # check if this path is removed
245 255 if path in removed:
246 256 # returning None is a way to mark node for removal
247 257 return None
248 258
249 259 # check if this path is added
250 260 for node in updated:
251 261 if node['path'] == path:
252 262 return memfilectx(
253 263 _repo,
254 264 changectx=ctx,
255 265 path=node['path'],
256 266 data=node['content'],
257 267 islink=False,
258 268 isexec=bool(node['mode'] & stat.S_IXUSR),
259 269 copied=False)
260 270
261 271 raise exceptions.AbortException()(
262 272 "Given path haven't been marked as added, "
263 273 "changed or removed (%s)" % path)
264 274
265 275 with repo.ui.configoverride({('phases', 'new-commit'): new_commit}):
266 276
267 277 commit_ctx = memctx(
268 278 repo=repo,
269 279 parents=parents,
270 280 text=message,
271 281 files=files,
272 282 filectxfn=_filectxfn,
273 283 user=user,
274 284 date=(commit_time, commit_timezone),
275 285 extra=extra)
276 286
277 287 n = repo.commitctx(commit_ctx)
278 288 new_id = hex(n)
279 289
280 290 return new_id
281 291
282 292 @reraise_safe_exceptions
283 293 def ctx_branch(self, wire, revision):
284 294 repo = self._factory.repo(wire)
285 295 ctx = self._get_ctx(repo, revision)
286 296 return ctx.branch()
287 297
288 298 @reraise_safe_exceptions
289 299 def ctx_children(self, wire, revision):
290 300 repo = self._factory.repo(wire)
291 301 ctx = self._get_ctx(repo, revision)
292 302 return [child.rev() for child in ctx.children()]
293 303
294 304 @reraise_safe_exceptions
295 305 def ctx_date(self, wire, revision):
296 306 repo = self._factory.repo(wire)
297 307 ctx = self._get_ctx(repo, revision)
298 308 return ctx.date()
299 309
300 310 @reraise_safe_exceptions
301 311 def ctx_description(self, wire, revision):
302 312 repo = self._factory.repo(wire)
303 313 ctx = self._get_ctx(repo, revision)
304 314 return ctx.description()
305 315
306 316 @reraise_safe_exceptions
307 317 def ctx_files(self, wire, revision):
308 318 repo = self._factory.repo(wire)
309 319 ctx = self._get_ctx(repo, revision)
310 320 return ctx.files()
311 321
312 322 @reraise_safe_exceptions
313 323 def ctx_list(self, path, revision):
314 324 repo = self._factory.repo(path)
315 325 ctx = self._get_ctx(repo, revision)
316 326 return list(ctx)
317 327
318 328 @reraise_safe_exceptions
319 329 def ctx_parents(self, wire, revision):
320 330 repo = self._factory.repo(wire)
321 331 ctx = self._get_ctx(repo, revision)
322 332 return [parent.rev() for parent in ctx.parents()]
323 333
324 334 @reraise_safe_exceptions
325 335 def ctx_phase(self, wire, revision):
326 336 repo = self._factory.repo(wire)
327 337 ctx = self._get_ctx(repo, revision)
328 338 # public=0, draft=1, secret=3
329 339 return ctx.phase()
330 340
331 341 @reraise_safe_exceptions
332 342 def ctx_obsolete(self, wire, revision):
333 343 repo = self._factory.repo(wire)
334 344 ctx = self._get_ctx(repo, revision)
335 345 return ctx.obsolete()
336 346
337 347 @reraise_safe_exceptions
338 348 def ctx_hidden(self, wire, revision):
339 349 repo = self._factory.repo(wire)
340 350 ctx = self._get_ctx(repo, revision)
341 351 return ctx.hidden()
342 352
343 353 @reraise_safe_exceptions
344 354 def ctx_substate(self, wire, revision):
345 355 repo = self._factory.repo(wire)
346 356 ctx = self._get_ctx(repo, revision)
347 357 return ctx.substate
348 358
349 359 @reraise_safe_exceptions
350 360 def ctx_status(self, wire, revision):
351 361 repo = self._factory.repo(wire)
352 362 ctx = self._get_ctx(repo, revision)
353 363 status = repo[ctx.p1().node()].status(other=ctx.node())
354 364 # object of status (odd, custom named tuple in mercurial) is not
355 365 # correctly serializable, we make it a list, as the underling
356 366 # API expects this to be a list
357 367 return list(status)
358 368
359 369 @reraise_safe_exceptions
360 370 def ctx_user(self, wire, revision):
361 371 repo = self._factory.repo(wire)
362 372 ctx = self._get_ctx(repo, revision)
363 373 return ctx.user()
364 374
365 375 @reraise_safe_exceptions
366 376 def check_url(self, url, config):
367 377 _proto = None
368 378 if '+' in url[:url.find('://')]:
369 379 _proto = url[0:url.find('+')]
370 380 url = url[url.find('+') + 1:]
371 381 handlers = []
372 382 url_obj = url_parser(url)
373 383 test_uri, authinfo = url_obj.authinfo()
374 384 url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
375 385 url_obj.query = obfuscate_qs(url_obj.query)
376 386
377 387 cleaned_uri = str(url_obj)
378 388 log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
379 389
380 390 if authinfo:
381 391 # create a password manager
382 392 passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
383 393 passmgr.add_password(*authinfo)
384 394
385 395 handlers.extend((httpbasicauthhandler(passmgr),
386 396 httpdigestauthhandler(passmgr)))
387 397
388 398 o = urllib2.build_opener(*handlers)
389 399 o.addheaders = [('Content-Type', 'application/mercurial-0.1'),
390 400 ('Accept', 'application/mercurial-0.1')]
391 401
392 402 q = {"cmd": 'between'}
393 403 q.update({'pairs': "%s-%s" % ('0' * 40, '0' * 40)})
394 404 qs = '?%s' % urllib.urlencode(q)
395 405 cu = "%s%s" % (test_uri, qs)
396 406 req = urllib2.Request(cu, None, {})
397 407
398 408 try:
399 409 log.debug("Trying to open URL %s", cleaned_uri)
400 410 resp = o.open(req)
401 411 if resp.code != 200:
402 412 raise exceptions.URLError()('Return Code is not 200')
403 413 except Exception as e:
404 414 log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
405 415 # means it cannot be cloned
406 416 raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e))
407 417
408 418 # now check if it's a proper hg repo, but don't do it for svn
409 419 try:
410 420 if _proto == 'svn':
411 421 pass
412 422 else:
413 423 # check for pure hg repos
414 424 log.debug(
415 425 "Verifying if URL is a Mercurial repository: %s",
416 426 cleaned_uri)
417 427 ui = make_ui_from_config(config)
418 428 peer_checker = makepeer(ui, url)
419 429 peer_checker.lookup('tip')
420 430 except Exception as e:
421 431 log.warning("URL is not a valid Mercurial repository: %s",
422 432 cleaned_uri)
423 433 raise exceptions.URLError(e)(
424 434 "url [%s] does not look like an hg repo org_exc: %s"
425 435 % (cleaned_uri, e))
426 436
427 437 log.info("URL is a valid Mercurial repository: %s", cleaned_uri)
428 438 return True
429 439
430 440 @reraise_safe_exceptions
431 441 def diff(
432 442 self, wire, rev1, rev2, file_filter, opt_git, opt_ignorews,
433 443 context):
434 444 repo = self._factory.repo(wire)
435 445
436 446 if file_filter:
437 447 match_filter = match(file_filter[0], '', [file_filter[1]])
438 448 else:
439 449 match_filter = file_filter
440 450 opts = diffopts(git=opt_git, ignorews=opt_ignorews, context=context)
441 451
442 452 try:
443 453 return "".join(patch.diff(
444 454 repo, node1=rev1, node2=rev2, match=match_filter, opts=opts))
445 455 except RepoLookupError as e:
446 456 raise exceptions.LookupException(e)()
447 457
448 458 @reraise_safe_exceptions
449 459 def node_history(self, wire, revision, path, limit):
450 460 repo = self._factory.repo(wire)
451 461
452 462 ctx = self._get_ctx(repo, revision)
453 463 fctx = ctx.filectx(path)
454 464
455 465 def history_iter():
456 466 limit_rev = fctx.rev()
457 467 for obj in reversed(list(fctx.filelog())):
458 468 obj = fctx.filectx(obj)
459 469 ctx = obj.changectx()
460 470 if ctx.hidden() or ctx.obsolete():
461 471 continue
462 472
463 473 if limit_rev >= obj.rev():
464 474 yield obj
465 475
466 476 history = []
467 477 for cnt, obj in enumerate(history_iter()):
468 478 if limit and cnt >= limit:
469 479 break
470 480 history.append(hex(obj.node()))
471 481
472 482 return [x for x in history]
473 483
474 484 @reraise_safe_exceptions
475 485 def node_history_untill(self, wire, revision, path, limit):
476 486 repo = self._factory.repo(wire)
477 487 ctx = self._get_ctx(repo, revision)
478 488 fctx = ctx.filectx(path)
479 489
480 490 file_log = list(fctx.filelog())
481 491 if limit:
482 492 # Limit to the last n items
483 493 file_log = file_log[-limit:]
484 494
485 495 return [hex(fctx.filectx(cs).node()) for cs in reversed(file_log)]
486 496
487 497 @reraise_safe_exceptions
488 498 def fctx_annotate(self, wire, revision, path):
489 499 repo = self._factory.repo(wire)
490 500 ctx = self._get_ctx(repo, revision)
491 501 fctx = ctx.filectx(path)
492 502
493 503 result = []
494 504 for i, annotate_obj in enumerate(fctx.annotate(), 1):
495 505 ln_no = i
496 506 sha = hex(annotate_obj.fctx.node())
497 507 content = annotate_obj.text
498 508 result.append((ln_no, sha, content))
499 509 return result
500 510
501 511 @reraise_safe_exceptions
502 512 def fctx_data(self, wire, revision, path):
503 513 repo = self._factory.repo(wire)
504 514 ctx = self._get_ctx(repo, revision)
505 515 fctx = ctx.filectx(path)
506 516 return fctx.data()
507 517
508 518 @reraise_safe_exceptions
509 519 def fctx_flags(self, wire, revision, path):
510 520 repo = self._factory.repo(wire)
511 521 ctx = self._get_ctx(repo, revision)
512 522 fctx = ctx.filectx(path)
513 523 return fctx.flags()
514 524
515 525 @reraise_safe_exceptions
516 526 def fctx_size(self, wire, revision, path):
517 527 repo = self._factory.repo(wire)
518 528 ctx = self._get_ctx(repo, revision)
519 529 fctx = ctx.filectx(path)
520 530 return fctx.size()
521 531
522 532 @reraise_safe_exceptions
523 533 def get_all_commit_ids(self, wire, name):
524 534 repo = self._factory.repo(wire)
525 535 repo = repo.filtered(name)
526 536 revs = map(lambda x: hex(x[7]), repo.changelog.index)
527 537 return revs
528 538
529 539 @reraise_safe_exceptions
530 540 def get_config_value(self, wire, section, name, untrusted=False):
531 541 repo = self._factory.repo(wire)
532 542 return repo.ui.config(section, name, untrusted=untrusted)
533 543
534 544 @reraise_safe_exceptions
535 545 def get_config_bool(self, wire, section, name, untrusted=False):
536 546 repo = self._factory.repo(wire)
537 547 return repo.ui.configbool(section, name, untrusted=untrusted)
538 548
539 549 @reraise_safe_exceptions
540 550 def get_config_list(self, wire, section, name, untrusted=False):
541 551 repo = self._factory.repo(wire)
542 552 return repo.ui.configlist(section, name, untrusted=untrusted)
543 553
544 554 @reraise_safe_exceptions
545 555 def is_large_file(self, wire, path):
546 556 return largefiles.lfutil.isstandin(path)
547 557
548 558 @reraise_safe_exceptions
549 559 def in_largefiles_store(self, wire, sha):
550 560 repo = self._factory.repo(wire)
551 561 return largefiles.lfutil.instore(repo, sha)
552 562
553 563 @reraise_safe_exceptions
554 564 def in_user_cache(self, wire, sha):
555 565 repo = self._factory.repo(wire)
556 566 return largefiles.lfutil.inusercache(repo.ui, sha)
557 567
558 568 @reraise_safe_exceptions
559 569 def store_path(self, wire, sha):
560 570 repo = self._factory.repo(wire)
561 571 return largefiles.lfutil.storepath(repo, sha)
562 572
563 573 @reraise_safe_exceptions
564 574 def link(self, wire, sha, path):
565 575 repo = self._factory.repo(wire)
566 576 largefiles.lfutil.link(
567 577 largefiles.lfutil.usercachepath(repo.ui, sha), path)
568 578
569 579 @reraise_safe_exceptions
570 580 def localrepository(self, wire, create=False):
571 581 self._factory.repo(wire, create=create)
572 582
573 583 @reraise_safe_exceptions
574 584 def lookup(self, wire, revision, both):
575 585
576 586 repo = self._factory.repo(wire)
577 587
578 588 if isinstance(revision, int):
579 589 # NOTE(marcink):
580 590 # since Mercurial doesn't support negative indexes properly
581 591 # we need to shift accordingly by one to get proper index, e.g
582 592 # repo[-1] => repo[-2]
583 593 # repo[0] => repo[-1]
584 594 if revision <= 0:
585 595 revision = revision + -1
586 596 try:
587 597 ctx = self._get_ctx(repo, revision)
588 598 except (TypeError, RepoLookupError) as e:
589 599 e._org_exc_tb = traceback.format_exc()
590 600 raise exceptions.LookupException(e)(revision)
591 601 except LookupError as e:
592 602 e._org_exc_tb = traceback.format_exc()
593 603 raise exceptions.LookupException(e)(e.name)
594 604
595 605 if not both:
596 606 return ctx.hex()
597 607
598 608 ctx = repo[ctx.hex()]
599 609 return ctx.hex(), ctx.rev()
600 610
601 611 @reraise_safe_exceptions
602 612 def pull(self, wire, url, commit_ids=None):
603 613 repo = self._factory.repo(wire)
604 614 # Disable any prompts for this repo
605 615 repo.ui.setconfig('ui', 'interactive', 'off', '-y')
606 616
607 617 remote = peer(repo, {}, url)
608 618 # Disable any prompts for this remote
609 619 remote.ui.setconfig('ui', 'interactive', 'off', '-y')
610 620
611 621 if commit_ids:
612 622 commit_ids = [bin(commit_id) for commit_id in commit_ids]
613 623
614 624 return exchange.pull(
615 625 repo, remote, heads=commit_ids, force=None).cgresult
616 626
617 627 @reraise_safe_exceptions
618 628 def sync_push(self, wire, url):
619 629 if not self.check_url(url, wire['config']):
620 630 return
621 631
622 632 repo = self._factory.repo(wire)
623 633
624 634 # Disable any prompts for this repo
625 635 repo.ui.setconfig('ui', 'interactive', 'off', '-y')
626 636
627 637 bookmarks = dict(repo._bookmarks).keys()
628 638 remote = peer(repo, {}, url)
629 639 # Disable any prompts for this remote
630 640 remote.ui.setconfig('ui', 'interactive', 'off', '-y')
631 641
632 642 return exchange.push(
633 643 repo, remote, newbranch=True, bookmarks=bookmarks).cgresult
634 644
635 645 @reraise_safe_exceptions
636 646 def revision(self, wire, rev):
637 647 repo = self._factory.repo(wire)
638 648 ctx = self._get_ctx(repo, rev)
639 649 return ctx.rev()
640 650
641 651 @reraise_safe_exceptions
642 652 def rev_range(self, wire, filter):
643 653 repo = self._factory.repo(wire)
644 654 revisions = [rev for rev in revrange(repo, filter)]
645 655 return revisions
646 656
647 657 @reraise_safe_exceptions
648 658 def rev_range_hash(self, wire, node):
649 659 repo = self._factory.repo(wire)
650 660
651 661 def get_revs(repo, rev_opt):
652 662 if rev_opt:
653 663 revs = revrange(repo, rev_opt)
654 664 if len(revs) == 0:
655 665 return (nullrev, nullrev)
656 666 return max(revs), min(revs)
657 667 else:
658 668 return len(repo) - 1, 0
659 669
660 670 stop, start = get_revs(repo, [node + ':'])
661 671 revs = [hex(repo[r].node()) for r in xrange(start, stop + 1)]
662 672 return revs
663 673
664 674 @reraise_safe_exceptions
665 675 def revs_from_revspec(self, wire, rev_spec, *args, **kwargs):
666 676 other_path = kwargs.pop('other_path', None)
667 677
668 678 # case when we want to compare two independent repositories
669 679 if other_path and other_path != wire["path"]:
670 680 baseui = self._factory._create_config(wire["config"])
671 681 repo = unionrepo.makeunionrepository(baseui, other_path, wire["path"])
672 682 else:
673 683 repo = self._factory.repo(wire)
674 684 return list(repo.revs(rev_spec, *args))
675 685
676 686 @reraise_safe_exceptions
677 687 def strip(self, wire, revision, update, backup):
678 688 repo = self._factory.repo(wire)
679 689 ctx = self._get_ctx(repo, revision)
680 690 hgext_strip(
681 691 repo.baseui, repo, ctx.node(), update=update, backup=backup)
682 692
683 693 @reraise_safe_exceptions
684 694 def verify(self, wire,):
685 695 repo = self._factory.repo(wire)
686 696 baseui = self._factory._create_config(wire['config'])
687 697 baseui.setconfig('ui', 'quiet', 'false')
688 698 output = io.BytesIO()
689 699
690 700 def write(data, **unused_kwargs):
691 701 output.write(data)
692 702 baseui.write = write
693 703
694 704 repo.ui = baseui
695 705 verify.verify(repo)
696 706 return output.getvalue()
697 707
698 708 @reraise_safe_exceptions
699 709 def tag(self, wire, name, revision, message, local, user,
700 710 tag_time, tag_timezone):
701 711 repo = self._factory.repo(wire)
702 712 ctx = self._get_ctx(repo, revision)
703 713 node = ctx.node()
704 714
705 715 date = (tag_time, tag_timezone)
706 716 try:
707 717 hg_tag.tag(repo, name, node, message, local, user, date)
708 718 except Abort as e:
709 719 log.exception("Tag operation aborted")
710 720 # Exception can contain unicode which we convert
711 721 raise exceptions.AbortException(e)(repr(e))
712 722
713 723 @reraise_safe_exceptions
714 724 def tags(self, wire):
715 725 repo = self._factory.repo(wire)
716 726 return repo.tags()
717 727
718 728 @reraise_safe_exceptions
719 729 def update(self, wire, node=None, clean=False):
720 730 repo = self._factory.repo(wire)
721 731 baseui = self._factory._create_config(wire['config'])
722 732 commands.update(baseui, repo, node=node, clean=clean)
723 733
724 734 @reraise_safe_exceptions
725 735 def identify(self, wire):
726 736 repo = self._factory.repo(wire)
727 737 baseui = self._factory._create_config(wire['config'])
728 738 output = io.BytesIO()
729 739 baseui.write = output.write
730 740 # This is required to get a full node id
731 741 baseui.debugflag = True
732 742 commands.identify(baseui, repo, id=True)
733 743
734 744 return output.getvalue()
735 745
736 746 @reraise_safe_exceptions
737 747 def pull_cmd(self, wire, source, bookmark=None, branch=None, revision=None,
738 748 hooks=True):
739 749 repo = self._factory.repo(wire)
740 750 baseui = self._factory._create_config(wire['config'], hooks=hooks)
741 751
742 752 # Mercurial internally has a lot of logic that checks ONLY if
743 753 # option is defined, we just pass those if they are defined then
744 754 opts = {}
745 755 if bookmark:
746 756 opts['bookmark'] = bookmark
747 757 if branch:
748 758 opts['branch'] = branch
749 759 if revision:
750 760 opts['rev'] = revision
751 761
752 762 commands.pull(baseui, repo, source, **opts)
753 763
754 764 @reraise_safe_exceptions
755 765 def heads(self, wire, branch=None):
756 766 repo = self._factory.repo(wire)
757 767 baseui = self._factory._create_config(wire['config'])
758 768 output = io.BytesIO()
759 769
760 770 def write(data, **unused_kwargs):
761 771 output.write(data)
762 772
763 773 baseui.write = write
764 774 if branch:
765 775 args = [branch]
766 776 else:
767 777 args = []
768 778 commands.heads(baseui, repo, template='{node} ', *args)
769 779
770 780 return output.getvalue()
771 781
772 782 @reraise_safe_exceptions
773 783 def ancestor(self, wire, revision1, revision2):
774 784 repo = self._factory.repo(wire)
775 785 changelog = repo.changelog
776 786 lookup = repo.lookup
777 787 a = changelog.ancestor(lookup(revision1), lookup(revision2))
778 788 return hex(a)
779 789
780 790 @reraise_safe_exceptions
781 791 def push(self, wire, revisions, dest_path, hooks=True,
782 792 push_branches=False):
783 793 repo = self._factory.repo(wire)
784 794 baseui = self._factory._create_config(wire['config'], hooks=hooks)
785 795 commands.push(baseui, repo, dest=dest_path, rev=revisions,
786 796 new_branch=push_branches)
787 797
788 798 @reraise_safe_exceptions
789 799 def merge(self, wire, revision):
790 800 repo = self._factory.repo(wire)
791 801 baseui = self._factory._create_config(wire['config'])
792 802 repo.ui.setconfig('ui', 'merge', 'internal:dump')
793 803
794 804 # In case of sub repositories are used mercurial prompts the user in
795 805 # case of merge conflicts or different sub repository sources. By
796 806 # setting the interactive flag to `False` mercurial doesn't prompt the
797 807 # used but instead uses a default value.
798 808 repo.ui.setconfig('ui', 'interactive', False)
799 809 commands.merge(baseui, repo, rev=revision)
800 810
801 811 @reraise_safe_exceptions
802 812 def merge_state(self, wire):
803 813 repo = self._factory.repo(wire)
804 814 repo.ui.setconfig('ui', 'merge', 'internal:dump')
805 815
806 816 # In case of sub repositories are used mercurial prompts the user in
807 817 # case of merge conflicts or different sub repository sources. By
808 818 # setting the interactive flag to `False` mercurial doesn't prompt the
809 819 # used but instead uses a default value.
810 820 repo.ui.setconfig('ui', 'interactive', False)
811 821 ms = hg_merge.mergestate(repo)
812 822 return [x for x in ms.unresolved()]
813 823
814 824 @reraise_safe_exceptions
815 825 def commit(self, wire, message, username, close_branch=False):
816 826 repo = self._factory.repo(wire)
817 827 baseui = self._factory._create_config(wire['config'])
818 828 repo.ui.setconfig('ui', 'username', username)
819 829 commands.commit(baseui, repo, message=message, close_branch=close_branch)
820 830
821 831
822 832 @reraise_safe_exceptions
823 833 def rebase(self, wire, source=None, dest=None, abort=False):
824 834 repo = self._factory.repo(wire)
825 835 baseui = self._factory._create_config(wire['config'])
826 836 repo.ui.setconfig('ui', 'merge', 'internal:dump')
827 837 rebase.rebase(
828 838 baseui, repo, base=source, dest=dest, abort=abort, keep=not abort)
829 839
830 840 @reraise_safe_exceptions
831 841 def bookmark(self, wire, bookmark, revision=None):
832 842 repo = self._factory.repo(wire)
833 843 baseui = self._factory._create_config(wire['config'])
834 844 commands.bookmark(baseui, repo, bookmark, rev=revision, force=True)
835 845
836 846 @reraise_safe_exceptions
837 847 def install_hooks(self, wire, force=False):
838 848 # we don't need any special hooks for Mercurial
839 849 pass
840 850
841 851 @reraise_safe_exceptions
842 852 def get_hooks_info(self, wire):
843 853 return {
844 854 'pre_version': vcsserver.__version__,
845 855 'post_version': vcsserver.__version__,
846 856 }
@@ -1,765 +1,775 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2019 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 from __future__ import absolute_import
19 19
20 20 import os
21 21 import subprocess
22 22 from urllib2 import URLError
23 23 import urlparse
24 24 import logging
25 25 import posixpath as vcspath
26 26 import StringIO
27 27 import urllib
28 28 import traceback
29 29
30 30 import svn.client
31 31 import svn.core
32 32 import svn.delta
33 33 import svn.diff
34 34 import svn.fs
35 35 import svn.repos
36 36
37 37 from vcsserver import svn_diff, exceptions, subprocessio, settings
38 38 from vcsserver.base import RepoFactory, raise_from_original
39 39
40 40 log = logging.getLogger(__name__)
41 41
42 42
43 43 # Set of svn compatible version flags.
44 44 # Compare with subversion/svnadmin/svnadmin.c
45 45 svn_compatible_versions = {
46 46 'pre-1.4-compatible',
47 47 'pre-1.5-compatible',
48 48 'pre-1.6-compatible',
49 49 'pre-1.8-compatible',
50 50 'pre-1.9-compatible'
51 51 }
52 52
53 53 svn_compatible_versions_map = {
54 54 'pre-1.4-compatible': '1.3',
55 55 'pre-1.5-compatible': '1.4',
56 56 'pre-1.6-compatible': '1.5',
57 57 'pre-1.8-compatible': '1.7',
58 58 'pre-1.9-compatible': '1.8',
59 59 }
60 60
61 61
62 62 def reraise_safe_exceptions(func):
63 63 """Decorator for converting svn exceptions to something neutral."""
64 64 def wrapper(*args, **kwargs):
65 65 try:
66 66 return func(*args, **kwargs)
67 67 except Exception as e:
68 68 if not hasattr(e, '_vcs_kind'):
69 69 log.exception("Unhandled exception in svn remote call")
70 70 raise_from_original(exceptions.UnhandledException(e))
71 71 raise
72 72 return wrapper
73 73
74 74
75 75 class SubversionFactory(RepoFactory):
76 76 repo_type = 'svn'
77 77
78 78 def _create_repo(self, wire, create, compatible_version):
79 79 path = svn.core.svn_path_canonicalize(wire['path'])
80 80 if create:
81 81 fs_config = {'compatible-version': '1.9'}
82 82 if compatible_version:
83 83 if compatible_version not in svn_compatible_versions:
84 84 raise Exception('Unknown SVN compatible version "{}"'
85 85 .format(compatible_version))
86 86 fs_config['compatible-version'] = \
87 87 svn_compatible_versions_map[compatible_version]
88 88
89 89 log.debug('Create SVN repo with config "%s"', fs_config)
90 90 repo = svn.repos.create(path, "", "", None, fs_config)
91 91 else:
92 92 repo = svn.repos.open(path)
93 93
94 94 log.debug('Got SVN object: %s', repo)
95 95 return repo
96 96
97 97 def repo(self, wire, create=False, compatible_version=None):
98 98 """
99 99 Get a repository instance for the given path.
100 100
101 101 Uses internally the low level beaker API since the decorators introduce
102 102 significant overhead.
103 103 """
104 104 region = self._cache_region
105 105 context = wire.get('context', None)
106 106 repo_path = wire.get('path', '')
107 107 context_uid = '{}'.format(context)
108 108 cache = wire.get('cache', True)
109 109 cache_on = context and cache
110 110
111 111 @region.conditional_cache_on_arguments(condition=cache_on)
112 112 def create_new_repo(_repo_type, _repo_path, _context_uid, compatible_version_id):
113 113 return self._create_repo(wire, create, compatible_version)
114 114
115 115 return create_new_repo(self.repo_type, repo_path, context_uid,
116 116 compatible_version)
117 117
118 118
119 119 NODE_TYPE_MAPPING = {
120 120 svn.core.svn_node_file: 'file',
121 121 svn.core.svn_node_dir: 'dir',
122 122 }
123 123
124 124
125 125 class SvnRemote(object):
126 126
127 127 def __init__(self, factory, hg_factory=None):
128 128 self._factory = factory
129 129 # TODO: Remove once we do not use internal Mercurial objects anymore
130 130 # for subversion
131 131 self._hg_factory = hg_factory
132 132
133 133 @reraise_safe_exceptions
134 134 def discover_svn_version(self):
135 135 try:
136 136 import svn.core
137 137 svn_ver = svn.core.SVN_VERSION
138 138 except ImportError:
139 139 svn_ver = None
140 140 return svn_ver
141 141
142 @reraise_safe_exceptions
143 def is_empty(self, wire):
144 repo = self._factory.repo(wire)
145
146 try:
147 return self.lookup(wire, -1) == 0
148 except Exception:
149 log.exception("failed to read object_store")
150 return False
151
142 152 def check_url(self, url, config_items):
143 153 # this can throw exception if not installed, but we detect this
144 154 from hgsubversion import svnrepo
145 155
146 156 baseui = self._hg_factory._create_config(config_items)
147 157 # uuid function get's only valid UUID from proper repo, else
148 158 # throws exception
149 159 try:
150 160 svnrepo.svnremoterepo(baseui, url).svn.uuid
151 161 except Exception:
152 162 tb = traceback.format_exc()
153 163 log.debug("Invalid Subversion url: `%s`, tb: %s", url, tb)
154 164 raise URLError(
155 165 '"%s" is not a valid Subversion source url.' % (url, ))
156 166 return True
157 167
158 168 def is_path_valid_repository(self, wire, path):
159 169
160 170 # NOTE(marcink): short circuit the check for SVN repo
161 171 # the repos.open might be expensive to check, but we have one cheap
162 172 # pre condition that we can use, to check for 'format' file
163 173
164 174 if not os.path.isfile(os.path.join(path, 'format')):
165 175 return False
166 176
167 177 try:
168 178 svn.repos.open(path)
169 179 except svn.core.SubversionException:
170 180 tb = traceback.format_exc()
171 181 log.debug("Invalid Subversion path `%s`, tb: %s", path, tb)
172 182 return False
173 183 return True
174 184
175 185 @reraise_safe_exceptions
176 186 def verify(self, wire,):
177 187 repo_path = wire['path']
178 188 if not self.is_path_valid_repository(wire, repo_path):
179 189 raise Exception(
180 190 "Path %s is not a valid Subversion repository." % repo_path)
181 191
182 192 cmd = ['svnadmin', 'info', repo_path]
183 193 stdout, stderr = subprocessio.run_command(cmd)
184 194 return stdout
185 195
186 196 def lookup(self, wire, revision):
187 197 if revision not in [-1, None, 'HEAD']:
188 198 raise NotImplementedError
189 199 repo = self._factory.repo(wire)
190 200 fs_ptr = svn.repos.fs(repo)
191 201 head = svn.fs.youngest_rev(fs_ptr)
192 202 return head
193 203
194 204 def lookup_interval(self, wire, start_ts, end_ts):
195 205 repo = self._factory.repo(wire)
196 206 fsobj = svn.repos.fs(repo)
197 207 start_rev = None
198 208 end_rev = None
199 209 if start_ts:
200 210 start_ts_svn = apr_time_t(start_ts)
201 211 start_rev = svn.repos.dated_revision(repo, start_ts_svn) + 1
202 212 else:
203 213 start_rev = 1
204 214 if end_ts:
205 215 end_ts_svn = apr_time_t(end_ts)
206 216 end_rev = svn.repos.dated_revision(repo, end_ts_svn)
207 217 else:
208 218 end_rev = svn.fs.youngest_rev(fsobj)
209 219 return start_rev, end_rev
210 220
211 221 def revision_properties(self, wire, revision):
212 222 repo = self._factory.repo(wire)
213 223 fs_ptr = svn.repos.fs(repo)
214 224 return svn.fs.revision_proplist(fs_ptr, revision)
215 225
216 226 def revision_changes(self, wire, revision):
217 227
218 228 repo = self._factory.repo(wire)
219 229 fsobj = svn.repos.fs(repo)
220 230 rev_root = svn.fs.revision_root(fsobj, revision)
221 231
222 232 editor = svn.repos.ChangeCollector(fsobj, rev_root)
223 233 editor_ptr, editor_baton = svn.delta.make_editor(editor)
224 234 base_dir = ""
225 235 send_deltas = False
226 236 svn.repos.replay2(
227 237 rev_root, base_dir, svn.core.SVN_INVALID_REVNUM, send_deltas,
228 238 editor_ptr, editor_baton, None)
229 239
230 240 added = []
231 241 changed = []
232 242 removed = []
233 243
234 244 # TODO: CHANGE_ACTION_REPLACE: Figure out where it belongs
235 245 for path, change in editor.changes.iteritems():
236 246 # TODO: Decide what to do with directory nodes. Subversion can add
237 247 # empty directories.
238 248
239 249 if change.item_kind == svn.core.svn_node_dir:
240 250 continue
241 251 if change.action in [svn.repos.CHANGE_ACTION_ADD]:
242 252 added.append(path)
243 253 elif change.action in [svn.repos.CHANGE_ACTION_MODIFY,
244 254 svn.repos.CHANGE_ACTION_REPLACE]:
245 255 changed.append(path)
246 256 elif change.action in [svn.repos.CHANGE_ACTION_DELETE]:
247 257 removed.append(path)
248 258 else:
249 259 raise NotImplementedError(
250 260 "Action %s not supported on path %s" % (
251 261 change.action, path))
252 262
253 263 changes = {
254 264 'added': added,
255 265 'changed': changed,
256 266 'removed': removed,
257 267 }
258 268 return changes
259 269
260 270 def node_history(self, wire, path, revision, limit):
261 271 cross_copies = False
262 272 repo = self._factory.repo(wire)
263 273 fsobj = svn.repos.fs(repo)
264 274 rev_root = svn.fs.revision_root(fsobj, revision)
265 275
266 276 history_revisions = []
267 277 history = svn.fs.node_history(rev_root, path)
268 278 history = svn.fs.history_prev(history, cross_copies)
269 279 while history:
270 280 __, node_revision = svn.fs.history_location(history)
271 281 history_revisions.append(node_revision)
272 282 if limit and len(history_revisions) >= limit:
273 283 break
274 284 history = svn.fs.history_prev(history, cross_copies)
275 285 return history_revisions
276 286
277 287 def node_properties(self, wire, path, revision):
278 288 repo = self._factory.repo(wire)
279 289 fsobj = svn.repos.fs(repo)
280 290 rev_root = svn.fs.revision_root(fsobj, revision)
281 291 return svn.fs.node_proplist(rev_root, path)
282 292
283 293 def file_annotate(self, wire, path, revision):
284 294 abs_path = 'file://' + urllib.pathname2url(
285 295 vcspath.join(wire['path'], path))
286 296 file_uri = svn.core.svn_path_canonicalize(abs_path)
287 297
288 298 start_rev = svn_opt_revision_value_t(0)
289 299 peg_rev = svn_opt_revision_value_t(revision)
290 300 end_rev = peg_rev
291 301
292 302 annotations = []
293 303
294 304 def receiver(line_no, revision, author, date, line, pool):
295 305 annotations.append((line_no, revision, line))
296 306
297 307 # TODO: Cannot use blame5, missing typemap function in the swig code
298 308 try:
299 309 svn.client.blame2(
300 310 file_uri, peg_rev, start_rev, end_rev,
301 311 receiver, svn.client.create_context())
302 312 except svn.core.SubversionException as exc:
303 313 log.exception("Error during blame operation.")
304 314 raise Exception(
305 315 "Blame not supported or file does not exist at path %s. "
306 316 "Error %s." % (path, exc))
307 317
308 318 return annotations
309 319
310 320 def get_node_type(self, wire, path, rev=None):
311 321 repo = self._factory.repo(wire)
312 322 fs_ptr = svn.repos.fs(repo)
313 323 if rev is None:
314 324 rev = svn.fs.youngest_rev(fs_ptr)
315 325 root = svn.fs.revision_root(fs_ptr, rev)
316 326 node = svn.fs.check_path(root, path)
317 327 return NODE_TYPE_MAPPING.get(node, None)
318 328
319 329 def get_nodes(self, wire, path, revision=None):
320 330 repo = self._factory.repo(wire)
321 331 fsobj = svn.repos.fs(repo)
322 332 if revision is None:
323 333 revision = svn.fs.youngest_rev(fsobj)
324 334 root = svn.fs.revision_root(fsobj, revision)
325 335 entries = svn.fs.dir_entries(root, path)
326 336 result = []
327 337 for entry_path, entry_info in entries.iteritems():
328 338 result.append(
329 339 (entry_path, NODE_TYPE_MAPPING.get(entry_info.kind, None)))
330 340 return result
331 341
332 342 def get_file_content(self, wire, path, rev=None):
333 343 repo = self._factory.repo(wire)
334 344 fsobj = svn.repos.fs(repo)
335 345 if rev is None:
336 346 rev = svn.fs.youngest_revision(fsobj)
337 347 root = svn.fs.revision_root(fsobj, rev)
338 348 content = svn.core.Stream(svn.fs.file_contents(root, path))
339 349 return content.read()
340 350
341 351 def get_file_size(self, wire, path, revision=None):
342 352 repo = self._factory.repo(wire)
343 353 fsobj = svn.repos.fs(repo)
344 354 if revision is None:
345 355 revision = svn.fs.youngest_revision(fsobj)
346 356 root = svn.fs.revision_root(fsobj, revision)
347 357 size = svn.fs.file_length(root, path)
348 358 return size
349 359
350 360 def create_repository(self, wire, compatible_version=None):
351 361 log.info('Creating Subversion repository in path "%s"', wire['path'])
352 362 self._factory.repo(wire, create=True,
353 363 compatible_version=compatible_version)
354 364
355 365 def get_url_and_credentials(self, src_url):
356 366 obj = urlparse.urlparse(src_url)
357 367 username = obj.username or None
358 368 password = obj.password or None
359 369 return username, password, src_url
360 370
361 371 def import_remote_repository(self, wire, src_url):
362 372 repo_path = wire['path']
363 373 if not self.is_path_valid_repository(wire, repo_path):
364 374 raise Exception(
365 375 "Path %s is not a valid Subversion repository." % repo_path)
366 376
367 377 username, password, src_url = self.get_url_and_credentials(src_url)
368 378 rdump_cmd = ['svnrdump', 'dump', '--non-interactive',
369 379 '--trust-server-cert-failures=unknown-ca']
370 380 if username and password:
371 381 rdump_cmd += ['--username', username, '--password', password]
372 382 rdump_cmd += [src_url]
373 383
374 384 rdump = subprocess.Popen(
375 385 rdump_cmd,
376 386 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
377 387 load = subprocess.Popen(
378 388 ['svnadmin', 'load', repo_path], stdin=rdump.stdout)
379 389
380 390 # TODO: johbo: This can be a very long operation, might be better
381 391 # to track some kind of status and provide an api to check if the
382 392 # import is done.
383 393 rdump.wait()
384 394 load.wait()
385 395
386 396 log.debug('Return process ended with code: %s', rdump.returncode)
387 397 if rdump.returncode != 0:
388 398 errors = rdump.stderr.read()
389 399 log.error('svnrdump dump failed: statuscode %s: message: %s',
390 400 rdump.returncode, errors)
391 401 reason = 'UNKNOWN'
392 402 if 'svnrdump: E230001:' in errors:
393 403 reason = 'INVALID_CERTIFICATE'
394 404
395 405 if reason == 'UNKNOWN':
396 406 reason = 'UNKNOWN:{}'.format(errors)
397 407 raise Exception(
398 408 'Failed to dump the remote repository from %s. Reason:%s' % (
399 409 src_url, reason))
400 410 if load.returncode != 0:
401 411 raise Exception(
402 412 'Failed to load the dump of remote repository from %s.' %
403 413 (src_url, ))
404 414
405 415 def commit(self, wire, message, author, timestamp, updated, removed):
406 416 assert isinstance(message, str)
407 417 assert isinstance(author, str)
408 418
409 419 repo = self._factory.repo(wire)
410 420 fsobj = svn.repos.fs(repo)
411 421
412 422 rev = svn.fs.youngest_rev(fsobj)
413 423 txn = svn.repos.fs_begin_txn_for_commit(repo, rev, author, message)
414 424 txn_root = svn.fs.txn_root(txn)
415 425
416 426 for node in updated:
417 427 TxnNodeProcessor(node, txn_root).update()
418 428 for node in removed:
419 429 TxnNodeProcessor(node, txn_root).remove()
420 430
421 431 commit_id = svn.repos.fs_commit_txn(repo, txn)
422 432
423 433 if timestamp:
424 434 apr_time = apr_time_t(timestamp)
425 435 ts_formatted = svn.core.svn_time_to_cstring(apr_time)
426 436 svn.fs.change_rev_prop(fsobj, commit_id, 'svn:date', ts_formatted)
427 437
428 438 log.debug('Committed revision "%s" to "%s".', commit_id, wire['path'])
429 439 return commit_id
430 440
431 441 def diff(self, wire, rev1, rev2, path1=None, path2=None,
432 442 ignore_whitespace=False, context=3):
433 443
434 444 wire.update(cache=False)
435 445 repo = self._factory.repo(wire)
436 446 diff_creator = SvnDiffer(
437 447 repo, rev1, path1, rev2, path2, ignore_whitespace, context)
438 448 try:
439 449 return diff_creator.generate_diff()
440 450 except svn.core.SubversionException as e:
441 451 log.exception(
442 452 "Error during diff operation operation. "
443 453 "Path might not exist %s, %s" % (path1, path2))
444 454 return ""
445 455
446 456 @reraise_safe_exceptions
447 457 def is_large_file(self, wire, path):
448 458 return False
449 459
450 460 @reraise_safe_exceptions
451 461 def run_svn_command(self, wire, cmd, **opts):
452 462 path = wire.get('path', None)
453 463
454 464 if path and os.path.isdir(path):
455 465 opts['cwd'] = path
456 466
457 467 safe_call = False
458 468 if '_safe' in opts:
459 469 safe_call = True
460 470
461 471 svnenv = os.environ.copy()
462 472 svnenv.update(opts.pop('extra_env', {}))
463 473
464 474 _opts = {'env': svnenv, 'shell': False}
465 475
466 476 try:
467 477 _opts.update(opts)
468 478 p = subprocessio.SubprocessIOChunker(cmd, **_opts)
469 479
470 480 return ''.join(p), ''.join(p.error)
471 481 except (EnvironmentError, OSError) as err:
472 482 cmd = ' '.join(cmd) # human friendly CMD
473 483 tb_err = ("Couldn't run svn command (%s).\n"
474 484 "Original error was:%s\n"
475 485 "Call options:%s\n"
476 486 % (cmd, err, _opts))
477 487 log.exception(tb_err)
478 488 if safe_call:
479 489 return '', err
480 490 else:
481 491 raise exceptions.VcsException()(tb_err)
482 492
483 493 @reraise_safe_exceptions
484 494 def install_hooks(self, wire, force=False):
485 495 from vcsserver.hook_utils import install_svn_hooks
486 496 repo_path = wire['path']
487 497 binary_dir = settings.BINARY_DIR
488 498 executable = None
489 499 if binary_dir:
490 500 executable = os.path.join(binary_dir, 'python')
491 501 return install_svn_hooks(
492 502 repo_path, executable=executable, force_create=force)
493 503
494 504 @reraise_safe_exceptions
495 505 def get_hooks_info(self, wire):
496 506 from vcsserver.hook_utils import (
497 507 get_svn_pre_hook_version, get_svn_post_hook_version)
498 508 repo_path = wire['path']
499 509 return {
500 510 'pre_version': get_svn_pre_hook_version(repo_path),
501 511 'post_version': get_svn_post_hook_version(repo_path),
502 512 }
503 513
504 514
505 515 class SvnDiffer(object):
506 516 """
507 517 Utility to create diffs based on difflib and the Subversion api
508 518 """
509 519
510 520 binary_content = False
511 521
512 522 def __init__(
513 523 self, repo, src_rev, src_path, tgt_rev, tgt_path,
514 524 ignore_whitespace, context):
515 525 self.repo = repo
516 526 self.ignore_whitespace = ignore_whitespace
517 527 self.context = context
518 528
519 529 fsobj = svn.repos.fs(repo)
520 530
521 531 self.tgt_rev = tgt_rev
522 532 self.tgt_path = tgt_path or ''
523 533 self.tgt_root = svn.fs.revision_root(fsobj, tgt_rev)
524 534 self.tgt_kind = svn.fs.check_path(self.tgt_root, self.tgt_path)
525 535
526 536 self.src_rev = src_rev
527 537 self.src_path = src_path or self.tgt_path
528 538 self.src_root = svn.fs.revision_root(fsobj, src_rev)
529 539 self.src_kind = svn.fs.check_path(self.src_root, self.src_path)
530 540
531 541 self._validate()
532 542
533 543 def _validate(self):
534 544 if (self.tgt_kind != svn.core.svn_node_none and
535 545 self.src_kind != svn.core.svn_node_none and
536 546 self.src_kind != self.tgt_kind):
537 547 # TODO: johbo: proper error handling
538 548 raise Exception(
539 549 "Source and target are not compatible for diff generation. "
540 550 "Source type: %s, target type: %s" %
541 551 (self.src_kind, self.tgt_kind))
542 552
543 553 def generate_diff(self):
544 554 buf = StringIO.StringIO()
545 555 if self.tgt_kind == svn.core.svn_node_dir:
546 556 self._generate_dir_diff(buf)
547 557 else:
548 558 self._generate_file_diff(buf)
549 559 return buf.getvalue()
550 560
551 561 def _generate_dir_diff(self, buf):
552 562 editor = DiffChangeEditor()
553 563 editor_ptr, editor_baton = svn.delta.make_editor(editor)
554 564 svn.repos.dir_delta2(
555 565 self.src_root,
556 566 self.src_path,
557 567 '', # src_entry
558 568 self.tgt_root,
559 569 self.tgt_path,
560 570 editor_ptr, editor_baton,
561 571 authorization_callback_allow_all,
562 572 False, # text_deltas
563 573 svn.core.svn_depth_infinity, # depth
564 574 False, # entry_props
565 575 False, # ignore_ancestry
566 576 )
567 577
568 578 for path, __, change in sorted(editor.changes):
569 579 self._generate_node_diff(
570 580 buf, change, path, self.tgt_path, path, self.src_path)
571 581
572 582 def _generate_file_diff(self, buf):
573 583 change = None
574 584 if self.src_kind == svn.core.svn_node_none:
575 585 change = "add"
576 586 elif self.tgt_kind == svn.core.svn_node_none:
577 587 change = "delete"
578 588 tgt_base, tgt_path = vcspath.split(self.tgt_path)
579 589 src_base, src_path = vcspath.split(self.src_path)
580 590 self._generate_node_diff(
581 591 buf, change, tgt_path, tgt_base, src_path, src_base)
582 592
583 593 def _generate_node_diff(
584 594 self, buf, change, tgt_path, tgt_base, src_path, src_base):
585 595
586 596 if self.src_rev == self.tgt_rev and tgt_base == src_base:
587 597 # makes consistent behaviour with git/hg to return empty diff if
588 598 # we compare same revisions
589 599 return
590 600
591 601 tgt_full_path = vcspath.join(tgt_base, tgt_path)
592 602 src_full_path = vcspath.join(src_base, src_path)
593 603
594 604 self.binary_content = False
595 605 mime_type = self._get_mime_type(tgt_full_path)
596 606
597 607 if mime_type and not mime_type.startswith('text'):
598 608 self.binary_content = True
599 609 buf.write("=" * 67 + '\n')
600 610 buf.write("Cannot display: file marked as a binary type.\n")
601 611 buf.write("svn:mime-type = %s\n" % mime_type)
602 612 buf.write("Index: %s\n" % (tgt_path, ))
603 613 buf.write("=" * 67 + '\n')
604 614 buf.write("diff --git a/%(tgt_path)s b/%(tgt_path)s\n" % {
605 615 'tgt_path': tgt_path})
606 616
607 617 if change == 'add':
608 618 # TODO: johbo: SVN is missing a zero here compared to git
609 619 buf.write("new file mode 10644\n")
610 620
611 621 #TODO(marcink): intro to binary detection of svn patches
612 622 # if self.binary_content:
613 623 # buf.write('GIT binary patch\n')
614 624
615 625 buf.write("--- /dev/null\t(revision 0)\n")
616 626 src_lines = []
617 627 else:
618 628 if change == 'delete':
619 629 buf.write("deleted file mode 10644\n")
620 630
621 631 #TODO(marcink): intro to binary detection of svn patches
622 632 # if self.binary_content:
623 633 # buf.write('GIT binary patch\n')
624 634
625 635 buf.write("--- a/%s\t(revision %s)\n" % (
626 636 src_path, self.src_rev))
627 637 src_lines = self._svn_readlines(self.src_root, src_full_path)
628 638
629 639 if change == 'delete':
630 640 buf.write("+++ /dev/null\t(revision %s)\n" % (self.tgt_rev, ))
631 641 tgt_lines = []
632 642 else:
633 643 buf.write("+++ b/%s\t(revision %s)\n" % (
634 644 tgt_path, self.tgt_rev))
635 645 tgt_lines = self._svn_readlines(self.tgt_root, tgt_full_path)
636 646
637 647 if not self.binary_content:
638 648 udiff = svn_diff.unified_diff(
639 649 src_lines, tgt_lines, context=self.context,
640 650 ignore_blank_lines=self.ignore_whitespace,
641 651 ignore_case=False,
642 652 ignore_space_changes=self.ignore_whitespace)
643 653 buf.writelines(udiff)
644 654
645 655 def _get_mime_type(self, path):
646 656 try:
647 657 mime_type = svn.fs.node_prop(
648 658 self.tgt_root, path, svn.core.SVN_PROP_MIME_TYPE)
649 659 except svn.core.SubversionException:
650 660 mime_type = svn.fs.node_prop(
651 661 self.src_root, path, svn.core.SVN_PROP_MIME_TYPE)
652 662 return mime_type
653 663
654 664 def _svn_readlines(self, fs_root, node_path):
655 665 if self.binary_content:
656 666 return []
657 667 node_kind = svn.fs.check_path(fs_root, node_path)
658 668 if node_kind not in (
659 669 svn.core.svn_node_file, svn.core.svn_node_symlink):
660 670 return []
661 671 content = svn.core.Stream(
662 672 svn.fs.file_contents(fs_root, node_path)).read()
663 673 return content.splitlines(True)
664 674
665 675
666 676
667 677 class DiffChangeEditor(svn.delta.Editor):
668 678 """
669 679 Records changes between two given revisions
670 680 """
671 681
672 682 def __init__(self):
673 683 self.changes = []
674 684
675 685 def delete_entry(self, path, revision, parent_baton, pool=None):
676 686 self.changes.append((path, None, 'delete'))
677 687
678 688 def add_file(
679 689 self, path, parent_baton, copyfrom_path, copyfrom_revision,
680 690 file_pool=None):
681 691 self.changes.append((path, 'file', 'add'))
682 692
683 693 def open_file(self, path, parent_baton, base_revision, file_pool=None):
684 694 self.changes.append((path, 'file', 'change'))
685 695
686 696
687 697 def authorization_callback_allow_all(root, path, pool):
688 698 return True
689 699
690 700
691 701 class TxnNodeProcessor(object):
692 702 """
693 703 Utility to process the change of one node within a transaction root.
694 704
695 705 It encapsulates the knowledge of how to add, update or remove
696 706 a node for a given transaction root. The purpose is to support the method
697 707 `SvnRemote.commit`.
698 708 """
699 709
700 710 def __init__(self, node, txn_root):
701 711 assert isinstance(node['path'], str)
702 712
703 713 self.node = node
704 714 self.txn_root = txn_root
705 715
706 716 def update(self):
707 717 self._ensure_parent_dirs()
708 718 self._add_file_if_node_does_not_exist()
709 719 self._update_file_content()
710 720 self._update_file_properties()
711 721
712 722 def remove(self):
713 723 svn.fs.delete(self.txn_root, self.node['path'])
714 724 # TODO: Clean up directory if empty
715 725
716 726 def _ensure_parent_dirs(self):
717 727 curdir = vcspath.dirname(self.node['path'])
718 728 dirs_to_create = []
719 729 while not self._svn_path_exists(curdir):
720 730 dirs_to_create.append(curdir)
721 731 curdir = vcspath.dirname(curdir)
722 732
723 733 for curdir in reversed(dirs_to_create):
724 734 log.debug('Creating missing directory "%s"', curdir)
725 735 svn.fs.make_dir(self.txn_root, curdir)
726 736
727 737 def _svn_path_exists(self, path):
728 738 path_status = svn.fs.check_path(self.txn_root, path)
729 739 return path_status != svn.core.svn_node_none
730 740
731 741 def _add_file_if_node_does_not_exist(self):
732 742 kind = svn.fs.check_path(self.txn_root, self.node['path'])
733 743 if kind == svn.core.svn_node_none:
734 744 svn.fs.make_file(self.txn_root, self.node['path'])
735 745
736 746 def _update_file_content(self):
737 747 assert isinstance(self.node['content'], str)
738 748 handler, baton = svn.fs.apply_textdelta(
739 749 self.txn_root, self.node['path'], None, None)
740 750 svn.delta.svn_txdelta_send_string(self.node['content'], handler, baton)
741 751
742 752 def _update_file_properties(self):
743 753 properties = self.node.get('properties', {})
744 754 for key, value in properties.iteritems():
745 755 svn.fs.change_node_prop(
746 756 self.txn_root, self.node['path'], key, value)
747 757
748 758
749 759 def apr_time_t(timestamp):
750 760 """
751 761 Convert a Python timestamp into APR timestamp type apr_time_t
752 762 """
753 763 return timestamp * 1E6
754 764
755 765
756 766 def svn_opt_revision_value_t(num):
757 767 """
758 768 Put `num` into a `svn_opt_revision_value_t` structure.
759 769 """
760 770 value = svn.core.svn_opt_revision_value_t()
761 771 value.number = num
762 772 revision = svn.core.svn_opt_revision_t()
763 773 revision.kind = svn.core.svn_opt_revision_number
764 774 revision.value = value
765 775 return revision
General Comments 0
You need to be logged in to leave comments. Login now