##// END OF EJS Templates
py3: compat and code fixes
super-admin -
r1042:b2b1b1e7 python3
parent child Browse files
Show More
@@ -1,1281 +1,1281 b''
1 RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2020 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import collections
19 19 import logging
20 20 import os
21 21 import posixpath as vcspath
22 22 import re
23 23 import stat
24 24 import traceback
25 25 import urllib.request, urllib.parse, urllib.error
26 26 import urllib.request, urllib.error, urllib.parse
27 27 from functools import wraps
28 28
29 29 import more_itertools
30 30 import pygit2
31 31 from pygit2 import Repository as LibGit2Repo
32 32 from pygit2 import index as LibGit2Index
33 33 from dulwich import index, objects
34 34 from dulwich.client import HttpGitClient, LocalGitClient
35 35 from dulwich.errors import (
36 36 NotGitRepository, ChecksumMismatch, WrongObjectException,
37 37 MissingCommitError, ObjectMissing, HangupException,
38 38 UnexpectedCommandError)
39 39 from dulwich.repo import Repo as DulwichRepo
40 40 from dulwich.server import update_server_info
41 41
42 42 from vcsserver import exceptions, settings, subprocessio
43 43 from vcsserver.utils import safe_str, safe_int, safe_unicode
44 44 from vcsserver.base import RepoFactory, obfuscate_qs, ArchiveNode, archive_repo
45 45 from vcsserver.hgcompat import (
46 46 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
47 47 from vcsserver.git_lfs.lib import LFSOidStore
48 48 from vcsserver.vcs_base import RemoteBase
49 49
50 50 DIR_STAT = stat.S_IFDIR
51 51 FILE_MODE = stat.S_IFMT
52 52 GIT_LINK = objects.S_IFGITLINK
53 53 PEELED_REF_MARKER = '^{}'
54 54
55 55
56 56 log = logging.getLogger(__name__)
57 57
58 58
59 59 def str_to_dulwich(value):
60 60 """
61 61 Dulwich 0.10.1a requires `unicode` objects to be passed in.
62 62 """
63 63 return value.decode(settings.WIRE_ENCODING)
64 64
65 65
66 66 def reraise_safe_exceptions(func):
67 67 """Converts Dulwich exceptions to something neutral."""
68 68
69 69 @wraps(func)
70 70 def wrapper(*args, **kwargs):
71 71 try:
72 72 return func(*args, **kwargs)
73 73 except (ChecksumMismatch, WrongObjectException, MissingCommitError, ObjectMissing,) as e:
74 74 exc = exceptions.LookupException(org_exc=e)
75 75 raise exc(safe_str(e))
76 76 except (HangupException, UnexpectedCommandError) as e:
77 77 exc = exceptions.VcsException(org_exc=e)
78 78 raise exc(safe_str(e))
79 79 except Exception as e:
80 80 # NOTE(marcink): becuase of how dulwich handles some exceptions
81 81 # (KeyError on empty repos), we cannot track this and catch all
82 82 # exceptions, it's an exceptions from other handlers
83 83 #if not hasattr(e, '_vcs_kind'):
84 84 #log.exception("Unhandled exception in git remote call")
85 85 #raise_from_original(exceptions.UnhandledException)
86 86 raise
87 87 return wrapper
88 88
89 89
90 90 class Repo(DulwichRepo):
91 91 """
92 92 A wrapper for dulwich Repo class.
93 93
94 94 Since dulwich is sometimes keeping .idx file descriptors open, it leads to
95 95 "Too many open files" error. We need to close all opened file descriptors
96 96 once the repo object is destroyed.
97 97 """
98 98 def __del__(self):
99 99 if hasattr(self, 'object_store'):
100 100 self.close()
101 101
102 102
103 103 class Repository(LibGit2Repo):
104 104
105 105 def __enter__(self):
106 106 return self
107 107
108 108 def __exit__(self, exc_type, exc_val, exc_tb):
109 109 self.free()
110 110
111 111
112 112 class GitFactory(RepoFactory):
113 113 repo_type = 'git'
114 114
115 115 def _create_repo(self, wire, create, use_libgit2=False):
116 116 if use_libgit2:
117 117 return Repository(wire['path'])
118 118 else:
119 119 repo_path = str_to_dulwich(wire['path'])
120 120 return Repo(repo_path)
121 121
122 122 def repo(self, wire, create=False, use_libgit2=False):
123 123 """
124 124 Get a repository instance for the given path.
125 125 """
126 126 return self._create_repo(wire, create, use_libgit2)
127 127
128 128 def repo_libgit2(self, wire):
129 129 return self.repo(wire, use_libgit2=True)
130 130
131 131
132 132 class GitRemote(RemoteBase):
133 133
134 134 def __init__(self, factory):
135 135 self._factory = factory
136 136 self._bulk_methods = {
137 137 "date": self.date,
138 138 "author": self.author,
139 139 "branch": self.branch,
140 140 "message": self.message,
141 141 "parents": self.parents,
142 142 "_commit": self.revision,
143 143 }
144 144
145 145 def _wire_to_config(self, wire):
146 146 if 'config' in wire:
147 147 return dict([(x[0] + '_' + x[1], x[2]) for x in wire['config']])
148 148 return {}
149 149
150 150 def _remote_conf(self, config):
151 151 params = [
152 152 '-c', 'core.askpass=""',
153 153 ]
154 154 ssl_cert_dir = config.get('vcs_ssl_dir')
155 155 if ssl_cert_dir:
156 156 params.extend(['-c', 'http.sslCAinfo={}'.format(ssl_cert_dir)])
157 157 return params
158 158
159 159 @reraise_safe_exceptions
160 160 def discover_git_version(self):
161 161 stdout, _ = self.run_git_command(
162 162 {}, ['--version'], _bare=True, _safe=True)
163 163 prefix = 'git version'
164 164 if stdout.startswith(prefix):
165 165 stdout = stdout[len(prefix):]
166 166 return stdout.strip()
167 167
168 168 @reraise_safe_exceptions
169 169 def is_empty(self, wire):
170 170 repo_init = self._factory.repo_libgit2(wire)
171 171 with repo_init as repo:
172 172
173 173 try:
174 174 has_head = repo.head.name
175 175 if has_head:
176 176 return False
177 177
178 178 # NOTE(marcink): check again using more expensive method
179 179 return repo.is_empty
180 180 except Exception:
181 181 pass
182 182
183 183 return True
184 184
185 185 @reraise_safe_exceptions
186 186 def assert_correct_path(self, wire):
187 187 cache_on, context_uid, repo_id = self._cache_on(wire)
188 188 region = self._region(wire)
189 189 @region.conditional_cache_on_arguments(condition=cache_on)
190 190 def _assert_correct_path(_context_uid, _repo_id):
191 191 try:
192 192 repo_init = self._factory.repo_libgit2(wire)
193 193 with repo_init as repo:
194 194 pass
195 195 except pygit2.GitError:
196 196 path = wire.get('path')
197 197 tb = traceback.format_exc()
198 198 log.debug("Invalid Git path `%s`, tb: %s", path, tb)
199 199 return False
200 200
201 201 return True
202 202 return _assert_correct_path(context_uid, repo_id)
203 203
204 204 @reraise_safe_exceptions
205 205 def bare(self, wire):
206 206 repo_init = self._factory.repo_libgit2(wire)
207 207 with repo_init as repo:
208 208 return repo.is_bare
209 209
210 210 @reraise_safe_exceptions
211 211 def blob_as_pretty_string(self, wire, sha):
212 212 repo_init = self._factory.repo_libgit2(wire)
213 213 with repo_init as repo:
214 214 blob_obj = repo[sha]
215 215 blob = blob_obj.data
216 216 return blob
217 217
218 218 @reraise_safe_exceptions
219 219 def blob_raw_length(self, wire, sha):
220 220 cache_on, context_uid, repo_id = self._cache_on(wire)
221 221 region = self._region(wire)
222 222 @region.conditional_cache_on_arguments(condition=cache_on)
223 223 def _blob_raw_length(_repo_id, _sha):
224 224
225 225 repo_init = self._factory.repo_libgit2(wire)
226 226 with repo_init as repo:
227 227 blob = repo[sha]
228 228 return blob.size
229 229
230 230 return _blob_raw_length(repo_id, sha)
231 231
232 232 def _parse_lfs_pointer(self, raw_content):
233 233
234 234 spec_string = 'version https://git-lfs.github.com/spec'
235 235 if raw_content and raw_content.startswith(spec_string):
236 236 pattern = re.compile(r"""
237 237 (?:\n)?
238 238 ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
239 239 ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
240 240 ^size[ ](?P<oid_size>[0-9]+)\n
241 241 (?:\n)?
242 242 """, re.VERBOSE | re.MULTILINE)
243 243 match = pattern.match(raw_content)
244 244 if match:
245 245 return match.groupdict()
246 246
247 247 return {}
248 248
249 249 @reraise_safe_exceptions
250 250 def is_large_file(self, wire, commit_id):
251 251 cache_on, context_uid, repo_id = self._cache_on(wire)
252 252
253 253 region = self._region(wire)
254 254 @region.conditional_cache_on_arguments(condition=cache_on)
255 255 def _is_large_file(_repo_id, _sha):
256 256 repo_init = self._factory.repo_libgit2(wire)
257 257 with repo_init as repo:
258 258 blob = repo[commit_id]
259 259 if blob.is_binary:
260 260 return {}
261 261
262 262 return self._parse_lfs_pointer(blob.data)
263 263
264 264 return _is_large_file(repo_id, commit_id)
265 265
266 266 @reraise_safe_exceptions
267 267 def is_binary(self, wire, tree_id):
268 268 cache_on, context_uid, repo_id = self._cache_on(wire)
269 269
270 270 region = self._region(wire)
271 271 @region.conditional_cache_on_arguments(condition=cache_on)
272 272 def _is_binary(_repo_id, _tree_id):
273 273 repo_init = self._factory.repo_libgit2(wire)
274 274 with repo_init as repo:
275 275 blob_obj = repo[tree_id]
276 276 return blob_obj.is_binary
277 277
278 278 return _is_binary(repo_id, tree_id)
279 279
280 280 @reraise_safe_exceptions
281 281 def in_largefiles_store(self, wire, oid):
282 282 conf = self._wire_to_config(wire)
283 283 repo_init = self._factory.repo_libgit2(wire)
284 284 with repo_init as repo:
285 285 repo_name = repo.path
286 286
287 287 store_location = conf.get('vcs_git_lfs_store_location')
288 288 if store_location:
289 289
290 290 store = LFSOidStore(
291 291 oid=oid, repo=repo_name, store_location=store_location)
292 292 return store.has_oid()
293 293
294 294 return False
295 295
296 296 @reraise_safe_exceptions
297 297 def store_path(self, wire, oid):
298 298 conf = self._wire_to_config(wire)
299 299 repo_init = self._factory.repo_libgit2(wire)
300 300 with repo_init as repo:
301 301 repo_name = repo.path
302 302
303 303 store_location = conf.get('vcs_git_lfs_store_location')
304 304 if store_location:
305 305 store = LFSOidStore(
306 306 oid=oid, repo=repo_name, store_location=store_location)
307 307 return store.oid_path
308 308 raise ValueError('Unable to fetch oid with path {}'.format(oid))
309 309
310 310 @reraise_safe_exceptions
311 311 def bulk_request(self, wire, rev, pre_load):
312 312 cache_on, context_uid, repo_id = self._cache_on(wire)
313 313 region = self._region(wire)
314 314 @region.conditional_cache_on_arguments(condition=cache_on)
315 315 def _bulk_request(_repo_id, _rev, _pre_load):
316 316 result = {}
317 317 for attr in pre_load:
318 318 try:
319 319 method = self._bulk_methods[attr]
320 320 args = [wire, rev]
321 321 result[attr] = method(*args)
322 322 except KeyError as e:
323 323 raise exceptions.VcsException(e)(
324 324 "Unknown bulk attribute: %s" % attr)
325 325 return result
326 326
327 327 return _bulk_request(repo_id, rev, sorted(pre_load))
328 328
329 329 def _build_opener(self, url):
330 330 handlers = []
331 331 url_obj = url_parser(url)
332 332 _, authinfo = url_obj.authinfo()
333 333
334 334 if authinfo:
335 335 # create a password manager
336 336 passmgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
337 337 passmgr.add_password(*authinfo)
338 338
339 339 handlers.extend((httpbasicauthhandler(passmgr),
340 340 httpdigestauthhandler(passmgr)))
341 341
342 342 return urllib.request.build_opener(*handlers)
343 343
344 344 def _type_id_to_name(self, type_id):
345 345 return {
346 346 1: b'commit',
347 347 2: b'tree',
348 348 3: b'blob',
349 349 4: b'tag'
350 350 }[type_id]
351 351
352 352 @reraise_safe_exceptions
353 353 def check_url(self, url, config):
354 354 url_obj = url_parser(url)
355 355 test_uri, _ = url_obj.authinfo()
356 356 url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
357 357 url_obj.query = obfuscate_qs(url_obj.query)
358 358 cleaned_uri = str(url_obj)
359 359 log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
360 360
361 361 if not test_uri.endswith('info/refs'):
362 362 test_uri = test_uri.rstrip('/') + '/info/refs'
363 363
364 364 o = self._build_opener(url)
365 365 o.addheaders = [('User-Agent', 'git/1.7.8.0')] # fake some git
366 366
367 367 q = {"service": 'git-upload-pack'}
368 368 qs = '?%s' % urllib.parse.urlencode(q)
369 369 cu = "%s%s" % (test_uri, qs)
370 370 req = urllib.request.Request(cu, None, {})
371 371
372 372 try:
373 373 log.debug("Trying to open URL %s", cleaned_uri)
374 374 resp = o.open(req)
375 375 if resp.code != 200:
376 376 raise exceptions.URLError()('Return Code is not 200')
377 377 except Exception as e:
378 378 log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
379 379 # means it cannot be cloned
380 380 raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e))
381 381
382 382 # now detect if it's proper git repo
383 383 gitdata = resp.read()
384 384 if 'service=git-upload-pack' in gitdata:
385 385 pass
386 386 elif re.findall(r'[0-9a-fA-F]{40}\s+refs', gitdata):
387 387 # old style git can return some other format !
388 388 pass
389 389 else:
390 390 raise exceptions.URLError()(
391 391 "url [%s] does not look like an git" % (cleaned_uri,))
392 392
393 393 return True
394 394
395 395 @reraise_safe_exceptions
396 396 def clone(self, wire, url, deferred, valid_refs, update_after_clone):
397 397 # TODO(marcink): deprecate this method. Last i checked we don't use it anymore
398 398 remote_refs = self.pull(wire, url, apply_refs=False)
399 399 repo = self._factory.repo(wire)
400 400 if isinstance(valid_refs, list):
401 401 valid_refs = tuple(valid_refs)
402 402
403 403 for k in remote_refs:
404 404 # only parse heads/tags and skip so called deferred tags
405 405 if k.startswith(valid_refs) and not k.endswith(deferred):
406 406 repo[k] = remote_refs[k]
407 407
408 408 if update_after_clone:
409 409 # we want to checkout HEAD
410 410 repo["HEAD"] = remote_refs["HEAD"]
411 411 index.build_index_from_tree(repo.path, repo.index_path(),
412 412 repo.object_store, repo["HEAD"].tree)
413 413
414 414 @reraise_safe_exceptions
415 415 def branch(self, wire, commit_id):
416 416 cache_on, context_uid, repo_id = self._cache_on(wire)
417 417 region = self._region(wire)
418 418 @region.conditional_cache_on_arguments(condition=cache_on)
419 419 def _branch(_context_uid, _repo_id, _commit_id):
420 420 regex = re.compile('^refs/heads')
421 421
422 422 def filter_with(ref):
423 423 return regex.match(ref[0]) and ref[1] == _commit_id
424 424
425 425 branches = list(filter(filter_with, list(self.get_refs(wire).items())))
426 426 return [x[0].split('refs/heads/')[-1] for x in branches]
427 427
428 428 return _branch(context_uid, repo_id, commit_id)
429 429
430 430 @reraise_safe_exceptions
431 431 def commit_branches(self, wire, commit_id):
432 432 cache_on, context_uid, repo_id = self._cache_on(wire)
433 433 region = self._region(wire)
434 434 @region.conditional_cache_on_arguments(condition=cache_on)
435 435 def _commit_branches(_context_uid, _repo_id, _commit_id):
436 436 repo_init = self._factory.repo_libgit2(wire)
437 437 with repo_init as repo:
438 438 branches = [x for x in repo.branches.with_commit(_commit_id)]
439 439 return branches
440 440
441 441 return _commit_branches(context_uid, repo_id, commit_id)
442 442
443 443 @reraise_safe_exceptions
444 444 def add_object(self, wire, content):
445 445 repo_init = self._factory.repo_libgit2(wire)
446 446 with repo_init as repo:
447 447 blob = objects.Blob()
448 448 blob.set_raw_string(content)
449 449 repo.object_store.add_object(blob)
450 450 return blob.id
451 451
452 452 # TODO: this is quite complex, check if that can be simplified
453 453 @reraise_safe_exceptions
454 454 def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
455 455 # Defines the root tree
456 456 class _Root(object):
457 457 def __repr__(self):
458 458 return 'ROOT TREE'
459 459 ROOT = _Root()
460 460
461 461 repo = self._factory.repo(wire)
462 462 object_store = repo.object_store
463 463
464 464 # Create tree and populates it with blobs
465 465
466 466 if commit_tree and repo[commit_tree]:
467 467 git_commit = repo[commit_data['parents'][0]]
468 468 commit_tree = repo[git_commit.tree] # root tree
469 469 else:
470 470 commit_tree = objects.Tree()
471 471
472 472 for node in updated:
473 473 # Compute subdirs if needed
474 474 dirpath, nodename = vcspath.split(node['path'])
475 475 dirnames = list(map(safe_str, dirpath and dirpath.split('/') or []))
476 476 parent = commit_tree
477 477 ancestors = [('', parent)]
478 478
479 479 # Tries to dig for the deepest existing tree
480 480 while dirnames:
481 481 curdir = dirnames.pop(0)
482 482 try:
483 483 dir_id = parent[curdir][1]
484 484 except KeyError:
485 485 # put curdir back into dirnames and stops
486 486 dirnames.insert(0, curdir)
487 487 break
488 488 else:
489 489 # If found, updates parent
490 490 parent = repo[dir_id]
491 491 ancestors.append((curdir, parent))
492 492 # Now parent is deepest existing tree and we need to create
493 493 # subtrees for dirnames (in reverse order)
494 494 # [this only applies for nodes from added]
495 495 new_trees = []
496 496
497 497 blob = objects.Blob.from_string(node['content'])
498 498
499 499 if dirnames:
500 500 # If there are trees which should be created we need to build
501 501 # them now (in reverse order)
502 502 reversed_dirnames = list(reversed(dirnames))
503 503 curtree = objects.Tree()
504 504 curtree[node['node_path']] = node['mode'], blob.id
505 505 new_trees.append(curtree)
506 506 for dirname in reversed_dirnames[:-1]:
507 507 newtree = objects.Tree()
508 508 newtree[dirname] = (DIR_STAT, curtree.id)
509 509 new_trees.append(newtree)
510 510 curtree = newtree
511 511 parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
512 512 else:
513 513 parent.add(name=node['node_path'], mode=node['mode'], hexsha=blob.id)
514 514
515 515 new_trees.append(parent)
516 516 # Update ancestors
517 517 reversed_ancestors = reversed(
518 518 [(a[1], b[1], b[0]) for a, b in zip(ancestors, ancestors[1:])])
519 519 for parent, tree, path in reversed_ancestors:
520 520 parent[path] = (DIR_STAT, tree.id)
521 521 object_store.add_object(tree)
522 522
523 523 object_store.add_object(blob)
524 524 for tree in new_trees:
525 525 object_store.add_object(tree)
526 526
527 527 for node_path in removed:
528 528 paths = node_path.split('/')
529 529 tree = commit_tree # start with top-level
530 530 trees = [{'tree': tree, 'path': ROOT}]
531 531 # Traverse deep into the forest...
532 532 # resolve final tree by iterating the path.
533 533 # e.g a/b/c.txt will get
534 534 # - root as tree then
535 535 # - 'a' as tree,
536 536 # - 'b' as tree,
537 537 # - stop at c as blob.
538 538 for path in paths:
539 539 try:
540 540 obj = repo[tree[path][1]]
541 541 if isinstance(obj, objects.Tree):
542 542 trees.append({'tree': obj, 'path': path})
543 543 tree = obj
544 544 except KeyError:
545 545 break
546 546 #PROBLEM:
547 547 """
548 548 We're not editing same reference tree object
549 549 """
550 550 # Cut down the blob and all rotten trees on the way back...
551 551 for path, tree_data in reversed(list(zip(paths, trees))):
552 552 tree = tree_data['tree']
553 553 tree.__delitem__(path)
554 554 # This operation edits the tree, we need to mark new commit back
555 555
556 556 if len(tree) > 0:
557 557 # This tree still has elements - don't remove it or any
558 558 # of it's parents
559 559 break
560 560
561 561 object_store.add_object(commit_tree)
562 562
563 563 # Create commit
564 564 commit = objects.Commit()
565 565 commit.tree = commit_tree.id
566 566 for k, v in commit_data.items():
567 567 setattr(commit, k, v)
568 568 object_store.add_object(commit)
569 569
570 570 self.create_branch(wire, branch, commit.id)
571 571
572 572 # dulwich set-ref
573 573 ref = 'refs/heads/%s' % branch
574 574 repo.refs[ref] = commit.id
575 575
576 576 return commit.id
577 577
578 578 @reraise_safe_exceptions
579 579 def pull(self, wire, url, apply_refs=True, refs=None, update_after=False):
580 580 if url != 'default' and '://' not in url:
581 581 client = LocalGitClient(url)
582 582 else:
583 583 url_obj = url_parser(url)
584 584 o = self._build_opener(url)
585 585 url, _ = url_obj.authinfo()
586 586 client = HttpGitClient(base_url=url, opener=o)
587 587 repo = self._factory.repo(wire)
588 588
589 589 determine_wants = repo.object_store.determine_wants_all
590 590 if refs:
591 591 def determine_wants_requested(references):
592 592 return [references[r] for r in references if r in refs]
593 593 determine_wants = determine_wants_requested
594 594
595 595 try:
596 596 remote_refs = client.fetch(
597 597 path=url, target=repo, determine_wants=determine_wants)
598 598 except NotGitRepository as e:
599 599 log.warning(
600 600 'Trying to fetch from "%s" failed, not a Git repository.', url)
601 601 # Exception can contain unicode which we convert
602 602 raise exceptions.AbortException(e)(repr(e))
603 603
604 604 # mikhail: client.fetch() returns all the remote refs, but fetches only
605 605 # refs filtered by `determine_wants` function. We need to filter result
606 606 # as well
607 607 if refs:
608 608 remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
609 609
610 610 if apply_refs:
611 611 # TODO: johbo: Needs proper test coverage with a git repository
612 612 # that contains a tag object, so that we would end up with
613 613 # a peeled ref at this point.
614 614 for k in remote_refs:
615 615 if k.endswith(PEELED_REF_MARKER):
616 616 log.debug("Skipping peeled reference %s", k)
617 617 continue
618 618 repo[k] = remote_refs[k]
619 619
620 620 if refs and not update_after:
621 621 # mikhail: explicitly set the head to the last ref.
622 622 repo["HEAD"] = remote_refs[refs[-1]]
623 623
624 624 if update_after:
625 625 # we want to checkout HEAD
626 626 repo["HEAD"] = remote_refs["HEAD"]
627 627 index.build_index_from_tree(repo.path, repo.index_path(),
628 628 repo.object_store, repo["HEAD"].tree)
629 629 return remote_refs
630 630
631 631 @reraise_safe_exceptions
632 632 def sync_fetch(self, wire, url, refs=None, all_refs=False):
633 633 repo = self._factory.repo(wire)
634 634 if refs and not isinstance(refs, (list, tuple)):
635 635 refs = [refs]
636 636
637 637 config = self._wire_to_config(wire)
638 638 # get all remote refs we'll use to fetch later
639 639 cmd = ['ls-remote']
640 640 if not all_refs:
641 641 cmd += ['--heads', '--tags']
642 642 cmd += [url]
643 643 output, __ = self.run_git_command(
644 644 wire, cmd, fail_on_stderr=False,
645 645 _copts=self._remote_conf(config),
646 646 extra_env={'GIT_TERMINAL_PROMPT': '0'})
647 647
648 648 remote_refs = collections.OrderedDict()
649 649 fetch_refs = []
650 650
651 651 for ref_line in output.splitlines():
652 652 sha, ref = ref_line.split('\t')
653 653 sha = sha.strip()
654 654 if ref in remote_refs:
655 655 # duplicate, skip
656 656 continue
657 657 if ref.endswith(PEELED_REF_MARKER):
658 658 log.debug("Skipping peeled reference %s", ref)
659 659 continue
660 660 # don't sync HEAD
661 661 if ref in ['HEAD']:
662 662 continue
663 663
664 664 remote_refs[ref] = sha
665 665
666 666 if refs and sha in refs:
667 667 # we filter fetch using our specified refs
668 668 fetch_refs.append('{}:{}'.format(ref, ref))
669 669 elif not refs:
670 670 fetch_refs.append('{}:{}'.format(ref, ref))
671 671 log.debug('Finished obtaining fetch refs, total: %s', len(fetch_refs))
672 672
673 673 if fetch_refs:
674 674 for chunk in more_itertools.chunked(fetch_refs, 1024 * 4):
675 675 fetch_refs_chunks = list(chunk)
676 676 log.debug('Fetching %s refs from import url', len(fetch_refs_chunks))
677 677 _out, _err = self.run_git_command(
678 678 wire, ['fetch', url, '--force', '--prune', '--'] + fetch_refs_chunks,
679 679 fail_on_stderr=False,
680 680 _copts=self._remote_conf(config),
681 681 extra_env={'GIT_TERMINAL_PROMPT': '0'})
682 682
683 683 return remote_refs
684 684
685 685 @reraise_safe_exceptions
686 686 def sync_push(self, wire, url, refs=None):
687 687 if not self.check_url(url, wire):
688 688 return
689 689 config = self._wire_to_config(wire)
690 690 self._factory.repo(wire)
691 691 self.run_git_command(
692 692 wire, ['push', url, '--mirror'], fail_on_stderr=False,
693 693 _copts=self._remote_conf(config),
694 694 extra_env={'GIT_TERMINAL_PROMPT': '0'})
695 695
696 696 @reraise_safe_exceptions
697 697 def get_remote_refs(self, wire, url):
698 698 repo = Repo(url)
699 699 return repo.get_refs()
700 700
701 701 @reraise_safe_exceptions
702 702 def get_description(self, wire):
703 703 repo = self._factory.repo(wire)
704 704 return repo.get_description()
705 705
706 706 @reraise_safe_exceptions
707 707 def get_missing_revs(self, wire, rev1, rev2, path2):
708 708 repo = self._factory.repo(wire)
709 709 LocalGitClient(thin_packs=False).fetch(path2, repo)
710 710
711 711 wire_remote = wire.copy()
712 712 wire_remote['path'] = path2
713 713 repo_remote = self._factory.repo(wire_remote)
714 714 LocalGitClient(thin_packs=False).fetch(wire["path"], repo_remote)
715 715
716 716 revs = [
717 717 x.commit.id
718 718 for x in repo_remote.get_walker(include=[rev2], exclude=[rev1])]
719 719 return revs
720 720
721 721 @reraise_safe_exceptions
722 722 def get_object(self, wire, sha, maybe_unreachable=False):
723 723 cache_on, context_uid, repo_id = self._cache_on(wire)
724 724 region = self._region(wire)
725 725 @region.conditional_cache_on_arguments(condition=cache_on)
726 726 def _get_object(_context_uid, _repo_id, _sha):
727 727 repo_init = self._factory.repo_libgit2(wire)
728 728 with repo_init as repo:
729 729
730 730 missing_commit_err = 'Commit {} does not exist for `{}`'.format(sha, wire['path'])
731 731 try:
732 732 commit = repo.revparse_single(sha)
733 733 except KeyError:
734 734 # NOTE(marcink): KeyError doesn't give us any meaningful information
735 735 # here, we instead give something more explicit
736 736 e = exceptions.RefNotFoundException('SHA: %s not found', sha)
737 737 raise exceptions.LookupException(e)(missing_commit_err)
738 738 except ValueError as e:
739 739 raise exceptions.LookupException(e)(missing_commit_err)
740 740
741 741 is_tag = False
742 742 if isinstance(commit, pygit2.Tag):
743 743 commit = repo.get(commit.target)
744 744 is_tag = True
745 745
746 746 check_dangling = True
747 747 if is_tag:
748 748 check_dangling = False
749 749
750 750 if check_dangling and maybe_unreachable:
751 751 check_dangling = False
752 752
753 753 # we used a reference and it parsed means we're not having a dangling commit
754 754 if sha != commit.hex:
755 755 check_dangling = False
756 756
757 757 if check_dangling:
758 758 # check for dangling commit
759 759 for branch in repo.branches.with_commit(commit.hex):
760 760 if branch:
761 761 break
762 762 else:
763 763 # NOTE(marcink): Empty error doesn't give us any meaningful information
764 764 # here, we instead give something more explicit
765 765 e = exceptions.RefNotFoundException('SHA: %s not found in branches', sha)
766 766 raise exceptions.LookupException(e)(missing_commit_err)
767 767
768 768 commit_id = commit.hex
769 769 type_id = commit.type_str
770 770
771 771 return {
772 772 'id': commit_id,
773 773 'type': self._type_id_to_name(type_id),
774 774 'commit_id': commit_id,
775 775 'idx': 0
776 776 }
777 777
778 778 return _get_object(context_uid, repo_id, sha)
779 779
780 780 @reraise_safe_exceptions
781 781 def get_refs(self, wire):
782 782 cache_on, context_uid, repo_id = self._cache_on(wire)
783 783 region = self._region(wire)
784 784 @region.conditional_cache_on_arguments(condition=cache_on)
785 785 def _get_refs(_context_uid, _repo_id):
786 786
787 787 repo_init = self._factory.repo_libgit2(wire)
788 788 with repo_init as repo:
789 789 regex = re.compile('^refs/(heads|tags)/')
790 790 return {x.name: x.target.hex for x in
791 791 [ref for ref in repo.listall_reference_objects() if regex.match(ref.name)]}
792 792
793 793 return _get_refs(context_uid, repo_id)
794 794
795 795 @reraise_safe_exceptions
796 796 def get_branch_pointers(self, wire):
797 797 cache_on, context_uid, repo_id = self._cache_on(wire)
798 798 region = self._region(wire)
799 799 @region.conditional_cache_on_arguments(condition=cache_on)
800 800 def _get_branch_pointers(_context_uid, _repo_id):
801 801
802 802 repo_init = self._factory.repo_libgit2(wire)
803 803 regex = re.compile('^refs/heads')
804 804 with repo_init as repo:
805 805 branches = [ref for ref in repo.listall_reference_objects() if regex.match(ref.name)]
806 806 return {x.target.hex: x.shorthand for x in branches}
807 807
808 808 return _get_branch_pointers(context_uid, repo_id)
809 809
810 810 @reraise_safe_exceptions
811 811 def head(self, wire, show_exc=True):
812 812 cache_on, context_uid, repo_id = self._cache_on(wire)
813 813 region = self._region(wire)
814 814 @region.conditional_cache_on_arguments(condition=cache_on)
815 815 def _head(_context_uid, _repo_id, _show_exc):
816 816 repo_init = self._factory.repo_libgit2(wire)
817 817 with repo_init as repo:
818 818 try:
819 819 return repo.head.peel().hex
820 820 except Exception:
821 821 if show_exc:
822 822 raise
823 823 return _head(context_uid, repo_id, show_exc)
824 824
825 825 @reraise_safe_exceptions
826 826 def init(self, wire):
827 827 repo_path = str_to_dulwich(wire['path'])
828 828 self.repo = Repo.init(repo_path)
829 829
830 830 @reraise_safe_exceptions
831 831 def init_bare(self, wire):
832 832 repo_path = str_to_dulwich(wire['path'])
833 833 self.repo = Repo.init_bare(repo_path)
834 834
835 835 @reraise_safe_exceptions
836 836 def revision(self, wire, rev):
837 837
838 838 cache_on, context_uid, repo_id = self._cache_on(wire)
839 839 region = self._region(wire)
840 840 @region.conditional_cache_on_arguments(condition=cache_on)
841 841 def _revision(_context_uid, _repo_id, _rev):
842 842 repo_init = self._factory.repo_libgit2(wire)
843 843 with repo_init as repo:
844 844 commit = repo[rev]
845 845 obj_data = {
846 846 'id': commit.id.hex,
847 847 }
848 848 # tree objects itself don't have tree_id attribute
849 849 if hasattr(commit, 'tree_id'):
850 850 obj_data['tree'] = commit.tree_id.hex
851 851
852 852 return obj_data
853 853 return _revision(context_uid, repo_id, rev)
854 854
855 855 @reraise_safe_exceptions
856 856 def date(self, wire, commit_id):
857 857 cache_on, context_uid, repo_id = self._cache_on(wire)
858 858 region = self._region(wire)
859 859 @region.conditional_cache_on_arguments(condition=cache_on)
860 860 def _date(_repo_id, _commit_id):
861 861 repo_init = self._factory.repo_libgit2(wire)
862 862 with repo_init as repo:
863 863 commit = repo[commit_id]
864 864
865 865 if hasattr(commit, 'commit_time'):
866 866 commit_time, commit_time_offset = commit.commit_time, commit.commit_time_offset
867 867 else:
868 868 commit = commit.get_object()
869 869 commit_time, commit_time_offset = commit.commit_time, commit.commit_time_offset
870 870
871 871 # TODO(marcink): check dulwich difference of offset vs timezone
872 872 return [commit_time, commit_time_offset]
873 873 return _date(repo_id, commit_id)
874 874
875 875 @reraise_safe_exceptions
876 876 def author(self, wire, commit_id):
877 877 cache_on, context_uid, repo_id = self._cache_on(wire)
878 878 region = self._region(wire)
879 879 @region.conditional_cache_on_arguments(condition=cache_on)
880 880 def _author(_repo_id, _commit_id):
881 881 repo_init = self._factory.repo_libgit2(wire)
882 882 with repo_init as repo:
883 883 commit = repo[commit_id]
884 884
885 885 if hasattr(commit, 'author'):
886 886 author = commit.author
887 887 else:
888 888 author = commit.get_object().author
889 889
890 890 if author.email:
891 891 return "{} <{}>".format(author.name, author.email)
892 892
893 893 try:
894 894 return "{}".format(author.name)
895 895 except Exception:
896 896 return "{}".format(safe_unicode(author.raw_name))
897 897
898 898 return _author(repo_id, commit_id)
899 899
900 900 @reraise_safe_exceptions
901 901 def message(self, wire, commit_id):
902 902 cache_on, context_uid, repo_id = self._cache_on(wire)
903 903 region = self._region(wire)
904 904 @region.conditional_cache_on_arguments(condition=cache_on)
905 905 def _message(_repo_id, _commit_id):
906 906 repo_init = self._factory.repo_libgit2(wire)
907 907 with repo_init as repo:
908 908 commit = repo[commit_id]
909 909 return commit.message
910 910 return _message(repo_id, commit_id)
911 911
912 912 @reraise_safe_exceptions
913 913 def parents(self, wire, commit_id):
914 914 cache_on, context_uid, repo_id = self._cache_on(wire)
915 915 region = self._region(wire)
916 916 @region.conditional_cache_on_arguments(condition=cache_on)
917 917 def _parents(_repo_id, _commit_id):
918 918 repo_init = self._factory.repo_libgit2(wire)
919 919 with repo_init as repo:
920 920 commit = repo[commit_id]
921 921 if hasattr(commit, 'parent_ids'):
922 922 parent_ids = commit.parent_ids
923 923 else:
924 924 parent_ids = commit.get_object().parent_ids
925 925
926 926 return [x.hex for x in parent_ids]
927 927 return _parents(repo_id, commit_id)
928 928
929 929 @reraise_safe_exceptions
930 930 def children(self, wire, commit_id):
931 931 cache_on, context_uid, repo_id = self._cache_on(wire)
932 932 region = self._region(wire)
933 933 @region.conditional_cache_on_arguments(condition=cache_on)
934 934 def _children(_repo_id, _commit_id):
935 935 output, __ = self.run_git_command(
936 936 wire, ['rev-list', '--all', '--children'])
937 937
938 938 child_ids = []
939 939 pat = re.compile(r'^%s' % commit_id)
940 940 for l in output.splitlines():
941 941 if pat.match(l):
942 942 found_ids = l.split(' ')[1:]
943 943 child_ids.extend(found_ids)
944 944
945 945 return child_ids
946 946 return _children(repo_id, commit_id)
947 947
948 948 @reraise_safe_exceptions
949 949 def set_refs(self, wire, key, value):
950 950 repo_init = self._factory.repo_libgit2(wire)
951 951 with repo_init as repo:
952 952 repo.references.create(key, value, force=True)
953 953
954 954 @reraise_safe_exceptions
955 955 def create_branch(self, wire, branch_name, commit_id, force=False):
956 956 repo_init = self._factory.repo_libgit2(wire)
957 957 with repo_init as repo:
958 958 commit = repo[commit_id]
959 959
960 960 if force:
961 961 repo.branches.local.create(branch_name, commit, force=force)
962 962 elif not repo.branches.get(branch_name):
963 963 # create only if that branch isn't existing
964 964 repo.branches.local.create(branch_name, commit, force=force)
965 965
966 966 @reraise_safe_exceptions
967 967 def remove_ref(self, wire, key):
968 968 repo_init = self._factory.repo_libgit2(wire)
969 969 with repo_init as repo:
970 970 repo.references.delete(key)
971 971
972 972 @reraise_safe_exceptions
973 973 def tag_remove(self, wire, tag_name):
974 974 repo_init = self._factory.repo_libgit2(wire)
975 975 with repo_init as repo:
976 976 key = 'refs/tags/{}'.format(tag_name)
977 977 repo.references.delete(key)
978 978
979 979 @reraise_safe_exceptions
980 980 def tree_changes(self, wire, source_id, target_id):
981 981 # TODO(marcink): remove this seems it's only used by tests
982 982 repo = self._factory.repo(wire)
983 983 source = repo[source_id].tree if source_id else None
984 984 target = repo[target_id].tree
985 985 result = repo.object_store.tree_changes(source, target)
986 986 return list(result)
987 987
988 988 @reraise_safe_exceptions
989 989 def tree_and_type_for_path(self, wire, commit_id, path):
990 990
991 991 cache_on, context_uid, repo_id = self._cache_on(wire)
992 992 region = self._region(wire)
993 993 @region.conditional_cache_on_arguments(condition=cache_on)
994 994 def _tree_and_type_for_path(_context_uid, _repo_id, _commit_id, _path):
995 995 repo_init = self._factory.repo_libgit2(wire)
996 996
997 997 with repo_init as repo:
998 998 commit = repo[commit_id]
999 999 try:
1000 1000 tree = commit.tree[path]
1001 1001 except KeyError:
1002 1002 return None, None, None
1003 1003
1004 1004 return tree.id.hex, tree.type_str, tree.filemode
1005 1005 return _tree_and_type_for_path(context_uid, repo_id, commit_id, path)
1006 1006
1007 1007 @reraise_safe_exceptions
1008 1008 def tree_items(self, wire, tree_id):
1009 1009 cache_on, context_uid, repo_id = self._cache_on(wire)
1010 1010 region = self._region(wire)
1011 1011 @region.conditional_cache_on_arguments(condition=cache_on)
1012 1012 def _tree_items(_repo_id, _tree_id):
1013 1013
1014 1014 repo_init = self._factory.repo_libgit2(wire)
1015 1015 with repo_init as repo:
1016 1016 try:
1017 1017 tree = repo[tree_id]
1018 1018 except KeyError:
1019 1019 raise ObjectMissing('No tree with id: {}'.format(tree_id))
1020 1020
1021 1021 result = []
1022 1022 for item in tree:
1023 1023 item_sha = item.hex
1024 1024 item_mode = item.filemode
1025 1025 item_type = item.type_str
1026 1026
1027 1027 if item_type == 'commit':
1028 1028 # NOTE(marcink): submodules we translate to 'link' for backward compat
1029 1029 item_type = 'link'
1030 1030
1031 1031 result.append((item.name, item_mode, item_sha, item_type))
1032 1032 return result
1033 1033 return _tree_items(repo_id, tree_id)
1034 1034
1035 1035 @reraise_safe_exceptions
1036 1036 def diff_2(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
1037 1037 """
1038 1038 Old version that uses subprocess to call diff
1039 1039 """
1040 1040
1041 1041 flags = [
1042 1042 '-U%s' % context, '--patch',
1043 1043 '--binary',
1044 1044 '--find-renames',
1045 1045 '--no-indent-heuristic',
1046 1046 # '--indent-heuristic',
1047 1047 #'--full-index',
1048 1048 #'--abbrev=40'
1049 1049 ]
1050 1050
1051 1051 if opt_ignorews:
1052 1052 flags.append('--ignore-all-space')
1053 1053
1054 1054 if commit_id_1 == self.EMPTY_COMMIT:
1055 1055 cmd = ['show'] + flags + [commit_id_2]
1056 1056 else:
1057 1057 cmd = ['diff'] + flags + [commit_id_1, commit_id_2]
1058 1058
1059 1059 if file_filter:
1060 1060 cmd.extend(['--', file_filter])
1061 1061
1062 1062 diff, __ = self.run_git_command(wire, cmd)
1063 1063 # If we used 'show' command, strip first few lines (until actual diff
1064 1064 # starts)
1065 1065 if commit_id_1 == self.EMPTY_COMMIT:
1066 1066 lines = diff.splitlines()
1067 1067 x = 0
1068 1068 for line in lines:
1069 1069 if line.startswith('diff'):
1070 1070 break
1071 1071 x += 1
1072 1072 # Append new line just like 'diff' command do
1073 1073 diff = '\n'.join(lines[x:]) + '\n'
1074 1074 return diff
1075 1075
1076 1076 @reraise_safe_exceptions
1077 1077 def diff(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
1078 1078 repo_init = self._factory.repo_libgit2(wire)
1079 1079 with repo_init as repo:
1080 1080 swap = True
1081 1081 flags = 0
1082 1082 flags |= pygit2.GIT_DIFF_SHOW_BINARY
1083 1083
1084 1084 if opt_ignorews:
1085 1085 flags |= pygit2.GIT_DIFF_IGNORE_WHITESPACE
1086 1086
1087 1087 if commit_id_1 == self.EMPTY_COMMIT:
1088 1088 comm1 = repo[commit_id_2]
1089 1089 diff_obj = comm1.tree.diff_to_tree(
1090 1090 flags=flags, context_lines=context, swap=swap)
1091 1091
1092 1092 else:
1093 1093 comm1 = repo[commit_id_2]
1094 1094 comm2 = repo[commit_id_1]
1095 1095 diff_obj = comm1.tree.diff_to_tree(
1096 1096 comm2.tree, flags=flags, context_lines=context, swap=swap)
1097 1097 similar_flags = 0
1098 1098 similar_flags |= pygit2.GIT_DIFF_FIND_RENAMES
1099 1099 diff_obj.find_similar(flags=similar_flags)
1100 1100
1101 1101 if file_filter:
1102 1102 for p in diff_obj:
1103 1103 if p.delta.old_file.path == file_filter:
1104 1104 return p.patch or ''
1105 1105 # fo matching path == no diff
1106 1106 return ''
1107 1107 return diff_obj.patch or ''
1108 1108
1109 1109 @reraise_safe_exceptions
1110 1110 def node_history(self, wire, commit_id, path, limit):
1111 1111 cache_on, context_uid, repo_id = self._cache_on(wire)
1112 1112 region = self._region(wire)
1113 1113 @region.conditional_cache_on_arguments(condition=cache_on)
1114 1114 def _node_history(_context_uid, _repo_id, _commit_id, _path, _limit):
1115 1115 # optimize for n==1, rev-list is much faster for that use-case
1116 1116 if limit == 1:
1117 1117 cmd = ['rev-list', '-1', commit_id, '--', path]
1118 1118 else:
1119 1119 cmd = ['log']
1120 1120 if limit:
1121 1121 cmd.extend(['-n', str(safe_int(limit, 0))])
1122 1122 cmd.extend(['--pretty=format: %H', '-s', commit_id, '--', path])
1123 1123
1124 1124 output, __ = self.run_git_command(wire, cmd)
1125 1125 commit_ids = re.findall(r'[0-9a-fA-F]{40}', output)
1126 1126
1127 1127 return [x for x in commit_ids]
1128 1128 return _node_history(context_uid, repo_id, commit_id, path, limit)
1129 1129
1130 1130 @reraise_safe_exceptions
1131 1131 def node_annotate(self, wire, commit_id, path):
1132 1132
1133 1133 cmd = ['blame', '-l', '--root', '-r', commit_id, '--', path]
1134 1134 # -l ==> outputs long shas (and we need all 40 characters)
1135 1135 # --root ==> doesn't put '^' character for boundaries
1136 1136 # -r commit_id ==> blames for the given commit
1137 1137 output, __ = self.run_git_command(wire, cmd)
1138 1138
1139 1139 result = []
1140 1140 for i, blame_line in enumerate(output.split('\n')[:-1]):
1141 1141 line_no = i + 1
1142 1142 commit_id, line = re.split(r' ', blame_line, 1)
1143 1143 result.append((line_no, commit_id, line))
1144 1144 return result
1145 1145
1146 1146 @reraise_safe_exceptions
1147 1147 def update_server_info(self, wire):
1148 1148 repo = self._factory.repo(wire)
1149 1149 update_server_info(repo)
1150 1150
1151 1151 @reraise_safe_exceptions
1152 1152 def get_all_commit_ids(self, wire):
1153 1153
1154 1154 cache_on, context_uid, repo_id = self._cache_on(wire)
1155 1155 region = self._region(wire)
1156 1156 @region.conditional_cache_on_arguments(condition=cache_on)
1157 1157 def _get_all_commit_ids(_context_uid, _repo_id):
1158 1158
1159 1159 cmd = ['rev-list', '--reverse', '--date-order', '--branches', '--tags']
1160 1160 try:
1161 1161 output, __ = self.run_git_command(wire, cmd)
1162 1162 return output.splitlines()
1163 1163 except Exception:
1164 1164 # Can be raised for empty repositories
1165 1165 return []
1166 1166 return _get_all_commit_ids(context_uid, repo_id)
1167 1167
1168 1168 @reraise_safe_exceptions
1169 1169 def run_git_command(self, wire, cmd, **opts):
1170 1170 path = wire.get('path', None)
1171 1171
1172 1172 if path and os.path.isdir(path):
1173 1173 opts['cwd'] = path
1174 1174
1175 1175 if '_bare' in opts:
1176 1176 _copts = []
1177 1177 del opts['_bare']
1178 1178 else:
1179 1179 _copts = ['-c', 'core.quotepath=false', ]
1180 1180 safe_call = False
1181 1181 if '_safe' in opts:
1182 1182 # no exc on failure
1183 1183 del opts['_safe']
1184 1184 safe_call = True
1185 1185
1186 1186 if '_copts' in opts:
1187 1187 _copts.extend(opts['_copts'] or [])
1188 1188 del opts['_copts']
1189 1189
1190 1190 gitenv = os.environ.copy()
1191 1191 gitenv.update(opts.pop('extra_env', {}))
1192 1192 # need to clean fix GIT_DIR !
1193 1193 if 'GIT_DIR' in gitenv:
1194 1194 del gitenv['GIT_DIR']
1195 1195 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
1196 1196 gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1'
1197 1197
1198 1198 cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
1199 1199 _opts = {'env': gitenv, 'shell': False}
1200 1200
1201 1201 proc = None
1202 1202 try:
1203 1203 _opts.update(opts)
1204 1204 proc = subprocessio.SubprocessIOChunker(cmd, **_opts)
1205 1205
1206 1206 return ''.join(proc), ''.join(proc.error)
1207 1207 except (EnvironmentError, OSError) as err:
1208 1208 cmd = ' '.join(cmd) # human friendly CMD
1209 1209 tb_err = ("Couldn't run git command (%s).\n"
1210 1210 "Original error was:%s\n"
1211 1211 "Call options:%s\n"
1212 1212 % (cmd, err, _opts))
1213 1213 log.exception(tb_err)
1214 1214 if safe_call:
1215 1215 return '', err
1216 1216 else:
1217 1217 raise exceptions.VcsException()(tb_err)
1218 1218 finally:
1219 1219 if proc:
1220 1220 proc.close()
1221 1221
1222 1222 @reraise_safe_exceptions
1223 1223 def install_hooks(self, wire, force=False):
1224 1224 from vcsserver.hook_utils import install_git_hooks
1225 1225 bare = self.bare(wire)
1226 1226 path = wire['path']
1227 1227 return install_git_hooks(path, bare, force_create=force)
1228 1228
1229 1229 @reraise_safe_exceptions
1230 1230 def get_hooks_info(self, wire):
1231 1231 from vcsserver.hook_utils import (
1232 1232 get_git_pre_hook_version, get_git_post_hook_version)
1233 1233 bare = self.bare(wire)
1234 1234 path = wire['path']
1235 1235 return {
1236 1236 'pre_version': get_git_pre_hook_version(path, bare),
1237 1237 'post_version': get_git_post_hook_version(path, bare),
1238 1238 }
1239 1239
1240 1240 @reraise_safe_exceptions
1241 1241 def set_head_ref(self, wire, head_name):
1242 1242 log.debug('Setting refs/head to `%s`', head_name)
1243 1243 cmd = ['symbolic-ref', '"HEAD"', '"refs/heads/%s"' % head_name]
1244 1244 output, __ = self.run_git_command(wire, cmd)
1245 1245 return [head_name] + output.splitlines()
1246 1246
1247 1247 @reraise_safe_exceptions
1248 1248 def archive_repo(self, wire, archive_dest_path, kind, mtime, archive_at_path,
1249 1249 archive_dir_name, commit_id):
1250 1250
1251 1251 def file_walker(_commit_id, path):
1252 1252 repo_init = self._factory.repo_libgit2(wire)
1253 1253
1254 1254 with repo_init as repo:
1255 1255 commit = repo[commit_id]
1256 1256
1257 1257 if path in ['', '/']:
1258 1258 tree = commit.tree
1259 1259 else:
1260 1260 tree = commit.tree[path.rstrip('/')]
1261 1261 tree_id = tree.id.hex
1262 1262 try:
1263 1263 tree = repo[tree_id]
1264 1264 except KeyError:
1265 1265 raise ObjectMissing('No tree with id: {}'.format(tree_id))
1266 1266
1267 1267 index = LibGit2Index.Index()
1268 1268 index.read_tree(tree)
1269 1269 file_iter = index
1270 1270
1271 1271 for fn in file_iter:
1272 1272 file_path = fn.path
1273 1273 mode = fn.mode
1274 1274 is_link = stat.S_ISLNK(mode)
1275 1275 if mode == pygit2.GIT_FILEMODE_COMMIT:
1276 1276 log.debug('Skipping path %s as a commit node', file_path)
1277 1277 continue
1278 1278 yield ArchiveNode(file_path, mode, is_link, repo[fn.hex].read_raw)
1279 1279
1280 1280 return archive_repo(file_walker, archive_dest_path, kind, mtime, archive_at_path,
1281 1281 archive_dir_name, commit_id)
@@ -1,1047 +1,1046 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2020 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 import functools
17
18 18 import io
19 19 import logging
20 import os
21 20 import stat
22 21 import urllib.request, urllib.parse, urllib.error
23 22 import urllib.request, urllib.error, urllib.parse
24 23 import traceback
25 24
26 25 from hgext import largefiles, rebase, purge
27 from hgext.strip import strip as hgext_strip
26
28 27 from mercurial import commands
29 28 from mercurial import unionrepo
30 29 from mercurial import verify
31 30 from mercurial import repair
32 31
33 32 import vcsserver
34 33 from vcsserver import exceptions
35 34 from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original, archive_repo, ArchiveNode
36 35 from vcsserver.hgcompat import (
37 36 archival, bin, clone, config as hgconfig, diffopts, hex, get_ctx,
38 37 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler,
39 38 makepeer, instance, match, memctx, exchange, memfilectx, nullrev, hg_merge,
40 39 patch, peer, revrange, ui, hg_tag, Abort, LookupError, RepoError,
41 40 RepoLookupError, InterventionRequired, RequirementError,
42 alwaysmatcher, patternmatcher, hgutil)
41 alwaysmatcher, patternmatcher, hgutil, hgext_strip)
43 42 from vcsserver.vcs_base import RemoteBase
44 43
45 44 log = logging.getLogger(__name__)
46 45
47 46
48 47 def make_ui_from_config(repo_config):
49 48
50 49 class LoggingUI(ui.ui):
51 50 def status(self, *msg, **opts):
52 51 log.info(' '.join(msg).rstrip('\n'))
53 52 super(LoggingUI, self).status(*msg, **opts)
54 53
55 54 def warn(self, *msg, **opts):
56 55 log.warn(' '.join(msg).rstrip('\n'))
57 56 super(LoggingUI, self).warn(*msg, **opts)
58 57
59 58 def error(self, *msg, **opts):
60 59 log.error(' '.join(msg).rstrip('\n'))
61 60 super(LoggingUI, self).error(*msg, **opts)
62 61
63 62 def note(self, *msg, **opts):
64 63 log.info(' '.join(msg).rstrip('\n'))
65 64 super(LoggingUI, self).note(*msg, **opts)
66 65
67 66 def debug(self, *msg, **opts):
68 67 log.debug(' '.join(msg).rstrip('\n'))
69 68 super(LoggingUI, self).debug(*msg, **opts)
70 69
71 70 baseui = LoggingUI()
72 71
73 72 # clean the baseui object
74 73 baseui._ocfg = hgconfig.config()
75 74 baseui._ucfg = hgconfig.config()
76 75 baseui._tcfg = hgconfig.config()
77 76
78 77 for section, option, value in repo_config:
79 78 baseui.setconfig(section, option, value)
80 79
81 80 # make our hgweb quiet so it doesn't print output
82 81 baseui.setconfig('ui', 'quiet', 'true')
83 82
84 83 baseui.setconfig('ui', 'paginate', 'never')
85 84 # for better Error reporting of Mercurial
86 85 baseui.setconfig('ui', 'message-output', 'stderr')
87 86
88 87 # force mercurial to only use 1 thread, otherwise it may try to set a
89 88 # signal in a non-main thread, thus generating a ValueError.
90 89 baseui.setconfig('worker', 'numcpus', 1)
91 90
92 91 # If there is no config for the largefiles extension, we explicitly disable
93 92 # it here. This overrides settings from repositories hgrc file. Recent
94 93 # mercurial versions enable largefiles in hgrc on clone from largefile
95 94 # repo.
96 95 if not baseui.hasconfig('extensions', 'largefiles'):
97 96 log.debug('Explicitly disable largefiles extension for repo.')
98 97 baseui.setconfig('extensions', 'largefiles', '!')
99 98
100 99 return baseui
101 100
102 101
103 102 def reraise_safe_exceptions(func):
104 103 """Decorator for converting mercurial exceptions to something neutral."""
105 104
106 105 def wrapper(*args, **kwargs):
107 106 try:
108 107 return func(*args, **kwargs)
109 108 except (Abort, InterventionRequired) as e:
110 109 raise_from_original(exceptions.AbortException(e))
111 110 except RepoLookupError as e:
112 111 raise_from_original(exceptions.LookupException(e))
113 112 except RequirementError as e:
114 113 raise_from_original(exceptions.RequirementException(e))
115 114 except RepoError as e:
116 115 raise_from_original(exceptions.VcsException(e))
117 116 except LookupError as e:
118 117 raise_from_original(exceptions.LookupException(e))
119 118 except Exception as e:
120 119 if not hasattr(e, '_vcs_kind'):
121 120 log.exception("Unhandled exception in hg remote call")
122 121 raise_from_original(exceptions.UnhandledException(e))
123 122
124 123 raise
125 124 return wrapper
126 125
127 126
128 127 class MercurialFactory(RepoFactory):
129 128 repo_type = 'hg'
130 129
131 130 def _create_config(self, config, hooks=True):
132 131 if not hooks:
133 132 hooks_to_clean = frozenset((
134 133 'changegroup.repo_size', 'preoutgoing.pre_pull',
135 134 'outgoing.pull_logger', 'prechangegroup.pre_push'))
136 135 new_config = []
137 136 for section, option, value in config:
138 137 if section == 'hooks' and option in hooks_to_clean:
139 138 continue
140 139 new_config.append((section, option, value))
141 140 config = new_config
142 141
143 142 baseui = make_ui_from_config(config)
144 143 return baseui
145 144
146 145 def _create_repo(self, wire, create):
147 146 baseui = self._create_config(wire["config"])
148 147 return instance(baseui, wire["path"], create)
149 148
150 149 def repo(self, wire, create=False):
151 150 """
152 151 Get a repository instance for the given path.
153 152 """
154 153 return self._create_repo(wire, create)
155 154
156 155
157 156 def patch_ui_message_output(baseui):
158 157 baseui.setconfig('ui', 'quiet', 'false')
159 158 output = io.BytesIO()
160 159
161 160 def write(data, **unused_kwargs):
162 161 output.write(data)
163 162
164 163 baseui.status = write
165 164 baseui.write = write
166 165 baseui.warn = write
167 166 baseui.debug = write
168 167
169 168 return baseui, output
170 169
171 170
172 171 class HgRemote(RemoteBase):
173 172
174 173 def __init__(self, factory):
175 174 self._factory = factory
176 175 self._bulk_methods = {
177 176 "affected_files": self.ctx_files,
178 177 "author": self.ctx_user,
179 178 "branch": self.ctx_branch,
180 179 "children": self.ctx_children,
181 180 "date": self.ctx_date,
182 181 "message": self.ctx_description,
183 182 "parents": self.ctx_parents,
184 183 "status": self.ctx_status,
185 184 "obsolete": self.ctx_obsolete,
186 185 "phase": self.ctx_phase,
187 186 "hidden": self.ctx_hidden,
188 187 "_file_paths": self.ctx_list,
189 188 }
190 189
191 190 def _get_ctx(self, repo, ref):
192 191 return get_ctx(repo, ref)
193 192
194 193 @reraise_safe_exceptions
195 194 def discover_hg_version(self):
196 195 from mercurial import util
197 196 return util.version()
198 197
199 198 @reraise_safe_exceptions
200 199 def is_empty(self, wire):
201 200 repo = self._factory.repo(wire)
202 201
203 202 try:
204 203 return len(repo) == 0
205 204 except Exception:
206 205 log.exception("failed to read object_store")
207 206 return False
208 207
209 208 @reraise_safe_exceptions
210 209 def bookmarks(self, wire):
211 210 cache_on, context_uid, repo_id = self._cache_on(wire)
212 211 region = self._region(wire)
213 212 @region.conditional_cache_on_arguments(condition=cache_on)
214 213 def _bookmarks(_context_uid, _repo_id):
215 214 repo = self._factory.repo(wire)
216 215 return dict(repo._bookmarks)
217 216
218 217 return _bookmarks(context_uid, repo_id)
219 218
220 219 @reraise_safe_exceptions
221 220 def branches(self, wire, normal, closed):
222 221 cache_on, context_uid, repo_id = self._cache_on(wire)
223 222 region = self._region(wire)
224 223 @region.conditional_cache_on_arguments(condition=cache_on)
225 224 def _branches(_context_uid, _repo_id, _normal, _closed):
226 225 repo = self._factory.repo(wire)
227 226 iter_branches = repo.branchmap().iterbranches()
228 227 bt = {}
229 228 for branch_name, _heads, tip, is_closed in iter_branches:
230 229 if normal and not is_closed:
231 230 bt[branch_name] = tip
232 231 if closed and is_closed:
233 232 bt[branch_name] = tip
234 233
235 234 return bt
236 235
237 236 return _branches(context_uid, repo_id, normal, closed)
238 237
239 238 @reraise_safe_exceptions
240 239 def bulk_request(self, wire, commit_id, pre_load):
241 240 cache_on, context_uid, repo_id = self._cache_on(wire)
242 241 region = self._region(wire)
243 242 @region.conditional_cache_on_arguments(condition=cache_on)
244 243 def _bulk_request(_repo_id, _commit_id, _pre_load):
245 244 result = {}
246 245 for attr in pre_load:
247 246 try:
248 247 method = self._bulk_methods[attr]
249 248 result[attr] = method(wire, commit_id)
250 249 except KeyError as e:
251 250 raise exceptions.VcsException(e)(
252 251 'Unknown bulk attribute: "%s"' % attr)
253 252 return result
254 253
255 254 return _bulk_request(repo_id, commit_id, sorted(pre_load))
256 255
257 256 @reraise_safe_exceptions
258 257 def ctx_branch(self, wire, commit_id):
259 258 cache_on, context_uid, repo_id = self._cache_on(wire)
260 259 region = self._region(wire)
261 260 @region.conditional_cache_on_arguments(condition=cache_on)
262 261 def _ctx_branch(_repo_id, _commit_id):
263 262 repo = self._factory.repo(wire)
264 263 ctx = self._get_ctx(repo, commit_id)
265 264 return ctx.branch()
266 265 return _ctx_branch(repo_id, commit_id)
267 266
268 267 @reraise_safe_exceptions
269 268 def ctx_date(self, wire, commit_id):
270 269 cache_on, context_uid, repo_id = self._cache_on(wire)
271 270 region = self._region(wire)
272 271 @region.conditional_cache_on_arguments(condition=cache_on)
273 272 def _ctx_date(_repo_id, _commit_id):
274 273 repo = self._factory.repo(wire)
275 274 ctx = self._get_ctx(repo, commit_id)
276 275 return ctx.date()
277 276 return _ctx_date(repo_id, commit_id)
278 277
279 278 @reraise_safe_exceptions
280 279 def ctx_description(self, wire, revision):
281 280 repo = self._factory.repo(wire)
282 281 ctx = self._get_ctx(repo, revision)
283 282 return ctx.description()
284 283
285 284 @reraise_safe_exceptions
286 285 def ctx_files(self, wire, commit_id):
287 286 cache_on, context_uid, repo_id = self._cache_on(wire)
288 287 region = self._region(wire)
289 288 @region.conditional_cache_on_arguments(condition=cache_on)
290 289 def _ctx_files(_repo_id, _commit_id):
291 290 repo = self._factory.repo(wire)
292 291 ctx = self._get_ctx(repo, commit_id)
293 292 return ctx.files()
294 293
295 294 return _ctx_files(repo_id, commit_id)
296 295
297 296 @reraise_safe_exceptions
298 297 def ctx_list(self, path, revision):
299 298 repo = self._factory.repo(path)
300 299 ctx = self._get_ctx(repo, revision)
301 300 return list(ctx)
302 301
303 302 @reraise_safe_exceptions
304 303 def ctx_parents(self, wire, commit_id):
305 304 cache_on, context_uid, repo_id = self._cache_on(wire)
306 305 region = self._region(wire)
307 306 @region.conditional_cache_on_arguments(condition=cache_on)
308 307 def _ctx_parents(_repo_id, _commit_id):
309 308 repo = self._factory.repo(wire)
310 309 ctx = self._get_ctx(repo, commit_id)
311 310 return [parent.hex() for parent in ctx.parents()
312 311 if not (parent.hidden() or parent.obsolete())]
313 312
314 313 return _ctx_parents(repo_id, commit_id)
315 314
316 315 @reraise_safe_exceptions
317 316 def ctx_children(self, wire, commit_id):
318 317 cache_on, context_uid, repo_id = self._cache_on(wire)
319 318 region = self._region(wire)
320 319 @region.conditional_cache_on_arguments(condition=cache_on)
321 320 def _ctx_children(_repo_id, _commit_id):
322 321 repo = self._factory.repo(wire)
323 322 ctx = self._get_ctx(repo, commit_id)
324 323 return [child.hex() for child in ctx.children()
325 324 if not (child.hidden() or child.obsolete())]
326 325
327 326 return _ctx_children(repo_id, commit_id)
328 327
329 328 @reraise_safe_exceptions
330 329 def ctx_phase(self, wire, commit_id):
331 330 cache_on, context_uid, repo_id = self._cache_on(wire)
332 331 region = self._region(wire)
333 332 @region.conditional_cache_on_arguments(condition=cache_on)
334 333 def _ctx_phase(_context_uid, _repo_id, _commit_id):
335 334 repo = self._factory.repo(wire)
336 335 ctx = self._get_ctx(repo, commit_id)
337 336 # public=0, draft=1, secret=3
338 337 return ctx.phase()
339 338 return _ctx_phase(context_uid, repo_id, commit_id)
340 339
341 340 @reraise_safe_exceptions
342 341 def ctx_obsolete(self, wire, commit_id):
343 342 cache_on, context_uid, repo_id = self._cache_on(wire)
344 343 region = self._region(wire)
345 344 @region.conditional_cache_on_arguments(condition=cache_on)
346 345 def _ctx_obsolete(_context_uid, _repo_id, _commit_id):
347 346 repo = self._factory.repo(wire)
348 347 ctx = self._get_ctx(repo, commit_id)
349 348 return ctx.obsolete()
350 349 return _ctx_obsolete(context_uid, repo_id, commit_id)
351 350
352 351 @reraise_safe_exceptions
353 352 def ctx_hidden(self, wire, commit_id):
354 353 cache_on, context_uid, repo_id = self._cache_on(wire)
355 354 region = self._region(wire)
356 355 @region.conditional_cache_on_arguments(condition=cache_on)
357 356 def _ctx_hidden(_context_uid, _repo_id, _commit_id):
358 357 repo = self._factory.repo(wire)
359 358 ctx = self._get_ctx(repo, commit_id)
360 359 return ctx.hidden()
361 360 return _ctx_hidden(context_uid, repo_id, commit_id)
362 361
363 362 @reraise_safe_exceptions
364 363 def ctx_substate(self, wire, revision):
365 364 repo = self._factory.repo(wire)
366 365 ctx = self._get_ctx(repo, revision)
367 366 return ctx.substate
368 367
369 368 @reraise_safe_exceptions
370 369 def ctx_status(self, wire, revision):
371 370 repo = self._factory.repo(wire)
372 371 ctx = self._get_ctx(repo, revision)
373 372 status = repo[ctx.p1().node()].status(other=ctx.node())
374 373 # object of status (odd, custom named tuple in mercurial) is not
375 374 # correctly serializable, we make it a list, as the underling
376 375 # API expects this to be a list
377 376 return list(status)
378 377
379 378 @reraise_safe_exceptions
380 379 def ctx_user(self, wire, revision):
381 380 repo = self._factory.repo(wire)
382 381 ctx = self._get_ctx(repo, revision)
383 382 return ctx.user()
384 383
385 384 @reraise_safe_exceptions
386 385 def check_url(self, url, config):
387 386 _proto = None
388 387 if '+' in url[:url.find('://')]:
389 388 _proto = url[0:url.find('+')]
390 389 url = url[url.find('+') + 1:]
391 390 handlers = []
392 391 url_obj = url_parser(url)
393 392 test_uri, authinfo = url_obj.authinfo()
394 393 url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
395 394 url_obj.query = obfuscate_qs(url_obj.query)
396 395
397 396 cleaned_uri = str(url_obj)
398 397 log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
399 398
400 399 if authinfo:
401 400 # create a password manager
402 401 passmgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
403 402 passmgr.add_password(*authinfo)
404 403
405 404 handlers.extend((httpbasicauthhandler(passmgr),
406 405 httpdigestauthhandler(passmgr)))
407 406
408 407 o = urllib.request.build_opener(*handlers)
409 408 o.addheaders = [('Content-Type', 'application/mercurial-0.1'),
410 409 ('Accept', 'application/mercurial-0.1')]
411 410
412 411 q = {"cmd": 'between'}
413 412 q.update({'pairs': "%s-%s" % ('0' * 40, '0' * 40)})
414 413 qs = '?%s' % urllib.parse.urlencode(q)
415 414 cu = "%s%s" % (test_uri, qs)
416 415 req = urllib.request.Request(cu, None, {})
417 416
418 417 try:
419 418 log.debug("Trying to open URL %s", cleaned_uri)
420 419 resp = o.open(req)
421 420 if resp.code != 200:
422 421 raise exceptions.URLError()('Return Code is not 200')
423 422 except Exception as e:
424 423 log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
425 424 # means it cannot be cloned
426 425 raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e))
427 426
428 427 # now check if it's a proper hg repo, but don't do it for svn
429 428 try:
430 429 if _proto == 'svn':
431 430 pass
432 431 else:
433 432 # check for pure hg repos
434 433 log.debug(
435 434 "Verifying if URL is a Mercurial repository: %s",
436 435 cleaned_uri)
437 436 ui = make_ui_from_config(config)
438 437 peer_checker = makepeer(ui, url)
439 438 peer_checker.lookup('tip')
440 439 except Exception as e:
441 440 log.warning("URL is not a valid Mercurial repository: %s",
442 441 cleaned_uri)
443 442 raise exceptions.URLError(e)(
444 443 "url [%s] does not look like an hg repo org_exc: %s"
445 444 % (cleaned_uri, e))
446 445
447 446 log.info("URL is a valid Mercurial repository: %s", cleaned_uri)
448 447 return True
449 448
450 449 @reraise_safe_exceptions
451 450 def diff(self, wire, commit_id_1, commit_id_2, file_filter, opt_git, opt_ignorews, context):
452 451 repo = self._factory.repo(wire)
453 452
454 453 if file_filter:
455 454 match_filter = match(file_filter[0], '', [file_filter[1]])
456 455 else:
457 456 match_filter = file_filter
458 457 opts = diffopts(git=opt_git, ignorews=opt_ignorews, context=context, showfunc=1)
459 458
460 459 try:
461 460 return "".join(patch.diff(
462 461 repo, node1=commit_id_1, node2=commit_id_2, match=match_filter, opts=opts))
463 462 except RepoLookupError as e:
464 463 raise exceptions.LookupException(e)()
465 464
466 465 @reraise_safe_exceptions
467 466 def node_history(self, wire, revision, path, limit):
468 467 cache_on, context_uid, repo_id = self._cache_on(wire)
469 468 region = self._region(wire)
470 469 @region.conditional_cache_on_arguments(condition=cache_on)
471 470 def _node_history(_context_uid, _repo_id, _revision, _path, _limit):
472 471 repo = self._factory.repo(wire)
473 472
474 473 ctx = self._get_ctx(repo, revision)
475 474 fctx = ctx.filectx(path)
476 475
477 476 def history_iter():
478 477 limit_rev = fctx.rev()
479 478 for obj in reversed(list(fctx.filelog())):
480 479 obj = fctx.filectx(obj)
481 480 ctx = obj.changectx()
482 481 if ctx.hidden() or ctx.obsolete():
483 482 continue
484 483
485 484 if limit_rev >= obj.rev():
486 485 yield obj
487 486
488 487 history = []
489 488 for cnt, obj in enumerate(history_iter()):
490 489 if limit and cnt >= limit:
491 490 break
492 491 history.append(hex(obj.node()))
493 492
494 493 return [x for x in history]
495 494 return _node_history(context_uid, repo_id, revision, path, limit)
496 495
497 496 @reraise_safe_exceptions
498 497 def node_history_untill(self, wire, revision, path, limit):
499 498 cache_on, context_uid, repo_id = self._cache_on(wire)
500 499 region = self._region(wire)
501 500 @region.conditional_cache_on_arguments(condition=cache_on)
502 501 def _node_history_until(_context_uid, _repo_id):
503 502 repo = self._factory.repo(wire)
504 503 ctx = self._get_ctx(repo, revision)
505 504 fctx = ctx.filectx(path)
506 505
507 506 file_log = list(fctx.filelog())
508 507 if limit:
509 508 # Limit to the last n items
510 509 file_log = file_log[-limit:]
511 510
512 511 return [hex(fctx.filectx(cs).node()) for cs in reversed(file_log)]
513 512 return _node_history_until(context_uid, repo_id, revision, path, limit)
514 513
515 514 @reraise_safe_exceptions
516 515 def fctx_annotate(self, wire, revision, path):
517 516 repo = self._factory.repo(wire)
518 517 ctx = self._get_ctx(repo, revision)
519 518 fctx = ctx.filectx(path)
520 519
521 520 result = []
522 521 for i, annotate_obj in enumerate(fctx.annotate(), 1):
523 522 ln_no = i
524 523 sha = hex(annotate_obj.fctx.node())
525 524 content = annotate_obj.text
526 525 result.append((ln_no, sha, content))
527 526 return result
528 527
529 528 @reraise_safe_exceptions
530 529 def fctx_node_data(self, wire, revision, path):
531 530 repo = self._factory.repo(wire)
532 531 ctx = self._get_ctx(repo, revision)
533 532 fctx = ctx.filectx(path)
534 533 return fctx.data()
535 534
536 535 @reraise_safe_exceptions
537 536 def fctx_flags(self, wire, commit_id, path):
538 537 cache_on, context_uid, repo_id = self._cache_on(wire)
539 538 region = self._region(wire)
540 539 @region.conditional_cache_on_arguments(condition=cache_on)
541 540 def _fctx_flags(_repo_id, _commit_id, _path):
542 541 repo = self._factory.repo(wire)
543 542 ctx = self._get_ctx(repo, commit_id)
544 543 fctx = ctx.filectx(path)
545 544 return fctx.flags()
546 545
547 546 return _fctx_flags(repo_id, commit_id, path)
548 547
549 548 @reraise_safe_exceptions
550 549 def fctx_size(self, wire, commit_id, path):
551 550 cache_on, context_uid, repo_id = self._cache_on(wire)
552 551 region = self._region(wire)
553 552 @region.conditional_cache_on_arguments(condition=cache_on)
554 553 def _fctx_size(_repo_id, _revision, _path):
555 554 repo = self._factory.repo(wire)
556 555 ctx = self._get_ctx(repo, commit_id)
557 556 fctx = ctx.filectx(path)
558 557 return fctx.size()
559 558 return _fctx_size(repo_id, commit_id, path)
560 559
561 560 @reraise_safe_exceptions
562 561 def get_all_commit_ids(self, wire, name):
563 562 cache_on, context_uid, repo_id = self._cache_on(wire)
564 563 region = self._region(wire)
565 564 @region.conditional_cache_on_arguments(condition=cache_on)
566 565 def _get_all_commit_ids(_context_uid, _repo_id, _name):
567 566 repo = self._factory.repo(wire)
568 567 repo = repo.filtered(name)
569 568 revs = [hex(x[7]) for x in repo.changelog.index]
570 569 return revs
571 570 return _get_all_commit_ids(context_uid, repo_id, name)
572 571
573 572 @reraise_safe_exceptions
574 573 def get_config_value(self, wire, section, name, untrusted=False):
575 574 repo = self._factory.repo(wire)
576 575 return repo.ui.config(section, name, untrusted=untrusted)
577 576
578 577 @reraise_safe_exceptions
579 578 def is_large_file(self, wire, commit_id, path):
580 579 cache_on, context_uid, repo_id = self._cache_on(wire)
581 580 region = self._region(wire)
582 581 @region.conditional_cache_on_arguments(condition=cache_on)
583 582 def _is_large_file(_context_uid, _repo_id, _commit_id, _path):
584 583 return largefiles.lfutil.isstandin(path)
585 584
586 585 return _is_large_file(context_uid, repo_id, commit_id, path)
587 586
588 587 @reraise_safe_exceptions
589 588 def is_binary(self, wire, revision, path):
590 589 cache_on, context_uid, repo_id = self._cache_on(wire)
591 590
592 591 region = self._region(wire)
593 592 @region.conditional_cache_on_arguments(condition=cache_on)
594 593 def _is_binary(_repo_id, _sha, _path):
595 594 repo = self._factory.repo(wire)
596 595 ctx = self._get_ctx(repo, revision)
597 596 fctx = ctx.filectx(path)
598 597 return fctx.isbinary()
599 598
600 599 return _is_binary(repo_id, revision, path)
601 600
602 601 @reraise_safe_exceptions
603 602 def in_largefiles_store(self, wire, sha):
604 603 repo = self._factory.repo(wire)
605 604 return largefiles.lfutil.instore(repo, sha)
606 605
607 606 @reraise_safe_exceptions
608 607 def in_user_cache(self, wire, sha):
609 608 repo = self._factory.repo(wire)
610 609 return largefiles.lfutil.inusercache(repo.ui, sha)
611 610
612 611 @reraise_safe_exceptions
613 612 def store_path(self, wire, sha):
614 613 repo = self._factory.repo(wire)
615 614 return largefiles.lfutil.storepath(repo, sha)
616 615
617 616 @reraise_safe_exceptions
618 617 def link(self, wire, sha, path):
619 618 repo = self._factory.repo(wire)
620 619 largefiles.lfutil.link(
621 620 largefiles.lfutil.usercachepath(repo.ui, sha), path)
622 621
623 622 @reraise_safe_exceptions
624 623 def localrepository(self, wire, create=False):
625 624 self._factory.repo(wire, create=create)
626 625
627 626 @reraise_safe_exceptions
628 627 def lookup(self, wire, revision, both):
629 628 cache_on, context_uid, repo_id = self._cache_on(wire)
630 629
631 630 region = self._region(wire)
632 631 @region.conditional_cache_on_arguments(condition=cache_on)
633 632 def _lookup(_context_uid, _repo_id, _revision, _both):
634 633
635 634 repo = self._factory.repo(wire)
636 635 rev = _revision
637 636 if isinstance(rev, int):
638 637 # NOTE(marcink):
639 638 # since Mercurial doesn't support negative indexes properly
640 639 # we need to shift accordingly by one to get proper index, e.g
641 640 # repo[-1] => repo[-2]
642 641 # repo[0] => repo[-1]
643 642 if rev <= 0:
644 643 rev = rev + -1
645 644 try:
646 645 ctx = self._get_ctx(repo, rev)
647 646 except (TypeError, RepoLookupError) as e:
648 647 e._org_exc_tb = traceback.format_exc()
649 648 raise exceptions.LookupException(e)(rev)
650 649 except LookupError as e:
651 650 e._org_exc_tb = traceback.format_exc()
652 651 raise exceptions.LookupException(e)(e.name)
653 652
654 653 if not both:
655 654 return ctx.hex()
656 655
657 656 ctx = repo[ctx.hex()]
658 657 return ctx.hex(), ctx.rev()
659 658
660 659 return _lookup(context_uid, repo_id, revision, both)
661 660
662 661 @reraise_safe_exceptions
663 662 def sync_push(self, wire, url):
664 663 if not self.check_url(url, wire['config']):
665 664 return
666 665
667 666 repo = self._factory.repo(wire)
668 667
669 668 # Disable any prompts for this repo
670 669 repo.ui.setconfig('ui', 'interactive', 'off', '-y')
671 670
672 671 bookmarks = list(dict(repo._bookmarks).keys())
673 672 remote = peer(repo, {}, url)
674 673 # Disable any prompts for this remote
675 674 remote.ui.setconfig('ui', 'interactive', 'off', '-y')
676 675
677 676 return exchange.push(
678 677 repo, remote, newbranch=True, bookmarks=bookmarks).cgresult
679 678
680 679 @reraise_safe_exceptions
681 680 def revision(self, wire, rev):
682 681 repo = self._factory.repo(wire)
683 682 ctx = self._get_ctx(repo, rev)
684 683 return ctx.rev()
685 684
686 685 @reraise_safe_exceptions
687 686 def rev_range(self, wire, commit_filter):
688 687 cache_on, context_uid, repo_id = self._cache_on(wire)
689 688
690 689 region = self._region(wire)
691 690 @region.conditional_cache_on_arguments(condition=cache_on)
692 691 def _rev_range(_context_uid, _repo_id, _filter):
693 692 repo = self._factory.repo(wire)
694 693 revisions = [rev for rev in revrange(repo, commit_filter)]
695 694 return revisions
696 695
697 696 return _rev_range(context_uid, repo_id, sorted(commit_filter))
698 697
699 698 @reraise_safe_exceptions
700 699 def rev_range_hash(self, wire, node):
701 700 repo = self._factory.repo(wire)
702 701
703 702 def get_revs(repo, rev_opt):
704 703 if rev_opt:
705 704 revs = revrange(repo, rev_opt)
706 705 if len(revs) == 0:
707 706 return (nullrev, nullrev)
708 707 return max(revs), min(revs)
709 708 else:
710 709 return len(repo) - 1, 0
711 710
712 711 stop, start = get_revs(repo, [node + ':'])
713 712 revs = [hex(repo[r].node()) for r in range(start, stop + 1)]
714 713 return revs
715 714
716 715 @reraise_safe_exceptions
717 716 def revs_from_revspec(self, wire, rev_spec, *args, **kwargs):
718 717 other_path = kwargs.pop('other_path', None)
719 718
720 719 # case when we want to compare two independent repositories
721 720 if other_path and other_path != wire["path"]:
722 721 baseui = self._factory._create_config(wire["config"])
723 722 repo = unionrepo.makeunionrepository(baseui, other_path, wire["path"])
724 723 else:
725 724 repo = self._factory.repo(wire)
726 725 return list(repo.revs(rev_spec, *args))
727 726
728 727 @reraise_safe_exceptions
729 728 def verify(self, wire,):
730 729 repo = self._factory.repo(wire)
731 730 baseui = self._factory._create_config(wire['config'])
732 731
733 732 baseui, output = patch_ui_message_output(baseui)
734 733
735 734 repo.ui = baseui
736 735 verify.verify(repo)
737 736 return output.getvalue()
738 737
739 738 @reraise_safe_exceptions
740 739 def hg_update_cache(self, wire,):
741 740 repo = self._factory.repo(wire)
742 741 baseui = self._factory._create_config(wire['config'])
743 742 baseui, output = patch_ui_message_output(baseui)
744 743
745 744 repo.ui = baseui
746 745 with repo.wlock(), repo.lock():
747 746 repo.updatecaches(full=True)
748 747
749 748 return output.getvalue()
750 749
751 750 @reraise_safe_exceptions
752 751 def hg_rebuild_fn_cache(self, wire,):
753 752 repo = self._factory.repo(wire)
754 753 baseui = self._factory._create_config(wire['config'])
755 754 baseui, output = patch_ui_message_output(baseui)
756 755
757 756 repo.ui = baseui
758 757
759 758 repair.rebuildfncache(baseui, repo)
760 759
761 760 return output.getvalue()
762 761
763 762 @reraise_safe_exceptions
764 763 def tags(self, wire):
765 764 cache_on, context_uid, repo_id = self._cache_on(wire)
766 765 region = self._region(wire)
767 766 @region.conditional_cache_on_arguments(condition=cache_on)
768 767 def _tags(_context_uid, _repo_id):
769 768 repo = self._factory.repo(wire)
770 769 return repo.tags()
771 770
772 771 return _tags(context_uid, repo_id)
773 772
774 773 @reraise_safe_exceptions
775 774 def update(self, wire, node=None, clean=False):
776 775 repo = self._factory.repo(wire)
777 776 baseui = self._factory._create_config(wire['config'])
778 777 commands.update(baseui, repo, node=node, clean=clean)
779 778
780 779 @reraise_safe_exceptions
781 780 def identify(self, wire):
782 781 repo = self._factory.repo(wire)
783 782 baseui = self._factory._create_config(wire['config'])
784 783 output = io.BytesIO()
785 784 baseui.write = output.write
786 785 # This is required to get a full node id
787 786 baseui.debugflag = True
788 787 commands.identify(baseui, repo, id=True)
789 788
790 789 return output.getvalue()
791 790
792 791 @reraise_safe_exceptions
793 792 def heads(self, wire, branch=None):
794 793 repo = self._factory.repo(wire)
795 794 baseui = self._factory._create_config(wire['config'])
796 795 output = io.BytesIO()
797 796
798 797 def write(data, **unused_kwargs):
799 798 output.write(data)
800 799
801 800 baseui.write = write
802 801 if branch:
803 802 args = [branch]
804 803 else:
805 804 args = []
806 805 commands.heads(baseui, repo, template='{node} ', *args)
807 806
808 807 return output.getvalue()
809 808
810 809 @reraise_safe_exceptions
811 810 def ancestor(self, wire, revision1, revision2):
812 811 repo = self._factory.repo(wire)
813 812 changelog = repo.changelog
814 813 lookup = repo.lookup
815 814 a = changelog.ancestor(lookup(revision1), lookup(revision2))
816 815 return hex(a)
817 816
818 817 @reraise_safe_exceptions
819 818 def clone(self, wire, source, dest, update_after_clone=False, hooks=True):
820 819 baseui = self._factory._create_config(wire["config"], hooks=hooks)
821 820 clone(baseui, source, dest, noupdate=not update_after_clone)
822 821
823 822 @reraise_safe_exceptions
824 823 def commitctx(self, wire, message, parents, commit_time, commit_timezone, user, files, extra, removed, updated):
825 824
826 825 repo = self._factory.repo(wire)
827 826 baseui = self._factory._create_config(wire['config'])
828 827 publishing = baseui.configbool('phases', 'publish')
829 828 if publishing:
830 829 new_commit = 'public'
831 830 else:
832 831 new_commit = 'draft'
833 832
834 833 def _filectxfn(_repo, ctx, path):
835 834 """
836 835 Marks given path as added/changed/removed in a given _repo. This is
837 836 for internal mercurial commit function.
838 837 """
839 838
840 839 # check if this path is removed
841 840 if path in removed:
842 841 # returning None is a way to mark node for removal
843 842 return None
844 843
845 844 # check if this path is added
846 845 for node in updated:
847 846 if node['path'] == path:
848 847 return memfilectx(
849 848 _repo,
850 849 changectx=ctx,
851 850 path=node['path'],
852 851 data=node['content'],
853 852 islink=False,
854 853 isexec=bool(node['mode'] & stat.S_IXUSR),
855 854 copysource=False)
856 855
857 856 raise exceptions.AbortException()(
858 857 "Given path haven't been marked as added, "
859 858 "changed or removed (%s)" % path)
860 859
861 860 with repo.ui.configoverride({('phases', 'new-commit'): new_commit}):
862 861
863 862 commit_ctx = memctx(
864 863 repo=repo,
865 864 parents=parents,
866 865 text=message,
867 866 files=files,
868 867 filectxfn=_filectxfn,
869 868 user=user,
870 869 date=(commit_time, commit_timezone),
871 870 extra=extra)
872 871
873 872 n = repo.commitctx(commit_ctx)
874 873 new_id = hex(n)
875 874
876 875 return new_id
877 876
878 877 @reraise_safe_exceptions
879 878 def pull(self, wire, url, commit_ids=None):
880 879 repo = self._factory.repo(wire)
881 880 # Disable any prompts for this repo
882 881 repo.ui.setconfig('ui', 'interactive', 'off', '-y')
883 882
884 883 remote = peer(repo, {}, url)
885 884 # Disable any prompts for this remote
886 885 remote.ui.setconfig('ui', 'interactive', 'off', '-y')
887 886
888 887 if commit_ids:
889 888 commit_ids = [bin(commit_id) for commit_id in commit_ids]
890 889
891 890 return exchange.pull(
892 891 repo, remote, heads=commit_ids, force=None).cgresult
893 892
894 893 @reraise_safe_exceptions
895 894 def pull_cmd(self, wire, source, bookmark=None, branch=None, revision=None, hooks=True):
896 895 repo = self._factory.repo(wire)
897 896 baseui = self._factory._create_config(wire['config'], hooks=hooks)
898 897
899 898 # Mercurial internally has a lot of logic that checks ONLY if
900 899 # option is defined, we just pass those if they are defined then
901 900 opts = {}
902 901 if bookmark:
903 902 opts['bookmark'] = bookmark
904 903 if branch:
905 904 opts['branch'] = branch
906 905 if revision:
907 906 opts['rev'] = revision
908 907
909 908 commands.pull(baseui, repo, source, **opts)
910 909
911 910 @reraise_safe_exceptions
912 911 def push(self, wire, revisions, dest_path, hooks=True, push_branches=False):
913 912 repo = self._factory.repo(wire)
914 913 baseui = self._factory._create_config(wire['config'], hooks=hooks)
915 914 commands.push(baseui, repo, dest=dest_path, rev=revisions,
916 915 new_branch=push_branches)
917 916
918 917 @reraise_safe_exceptions
919 918 def strip(self, wire, revision, update, backup):
920 919 repo = self._factory.repo(wire)
921 920 ctx = self._get_ctx(repo, revision)
922 921 hgext_strip(
923 922 repo.baseui, repo, ctx.node(), update=update, backup=backup)
924 923
925 924 @reraise_safe_exceptions
926 925 def get_unresolved_files(self, wire):
927 926 repo = self._factory.repo(wire)
928 927
929 928 log.debug('Calculating unresolved files for repo: %s', repo)
930 929 output = io.BytesIO()
931 930
932 931 def write(data, **unused_kwargs):
933 932 output.write(data)
934 933
935 934 baseui = self._factory._create_config(wire['config'])
936 935 baseui.write = write
937 936
938 937 commands.resolve(baseui, repo, list=True)
939 938 unresolved = output.getvalue().splitlines(0)
940 939 return unresolved
941 940
942 941 @reraise_safe_exceptions
943 942 def merge(self, wire, revision):
944 943 repo = self._factory.repo(wire)
945 944 baseui = self._factory._create_config(wire['config'])
946 945 repo.ui.setconfig('ui', 'merge', 'internal:dump')
947 946
948 947 # In case of sub repositories are used mercurial prompts the user in
949 948 # case of merge conflicts or different sub repository sources. By
950 949 # setting the interactive flag to `False` mercurial doesn't prompt the
951 950 # used but instead uses a default value.
952 951 repo.ui.setconfig('ui', 'interactive', False)
953 952 commands.merge(baseui, repo, rev=revision)
954 953
955 954 @reraise_safe_exceptions
956 955 def merge_state(self, wire):
957 956 repo = self._factory.repo(wire)
958 957 repo.ui.setconfig('ui', 'merge', 'internal:dump')
959 958
960 959 # In case of sub repositories are used mercurial prompts the user in
961 960 # case of merge conflicts or different sub repository sources. By
962 961 # setting the interactive flag to `False` mercurial doesn't prompt the
963 962 # used but instead uses a default value.
964 963 repo.ui.setconfig('ui', 'interactive', False)
965 964 ms = hg_merge.mergestate(repo)
966 965 return [x for x in ms.unresolved()]
967 966
968 967 @reraise_safe_exceptions
969 968 def commit(self, wire, message, username, close_branch=False):
970 969 repo = self._factory.repo(wire)
971 970 baseui = self._factory._create_config(wire['config'])
972 971 repo.ui.setconfig('ui', 'username', username)
973 972 commands.commit(baseui, repo, message=message, close_branch=close_branch)
974 973
975 974 @reraise_safe_exceptions
976 975 def rebase(self, wire, source=None, dest=None, abort=False):
977 976 repo = self._factory.repo(wire)
978 977 baseui = self._factory._create_config(wire['config'])
979 978 repo.ui.setconfig('ui', 'merge', 'internal:dump')
980 979 # In case of sub repositories are used mercurial prompts the user in
981 980 # case of merge conflicts or different sub repository sources. By
982 981 # setting the interactive flag to `False` mercurial doesn't prompt the
983 982 # used but instead uses a default value.
984 983 repo.ui.setconfig('ui', 'interactive', False)
985 984 rebase.rebase(baseui, repo, base=source, dest=dest, abort=abort, keep=not abort)
986 985
987 986 @reraise_safe_exceptions
988 987 def tag(self, wire, name, revision, message, local, user, tag_time, tag_timezone):
989 988 repo = self._factory.repo(wire)
990 989 ctx = self._get_ctx(repo, revision)
991 990 node = ctx.node()
992 991
993 992 date = (tag_time, tag_timezone)
994 993 try:
995 994 hg_tag.tag(repo, name, node, message, local, user, date)
996 995 except Abort as e:
997 996 log.exception("Tag operation aborted")
998 997 # Exception can contain unicode which we convert
999 998 raise exceptions.AbortException(e)(repr(e))
1000 999
1001 1000 @reraise_safe_exceptions
1002 1001 def bookmark(self, wire, bookmark, revision=None):
1003 1002 repo = self._factory.repo(wire)
1004 1003 baseui = self._factory._create_config(wire['config'])
1005 1004 commands.bookmark(baseui, repo, bookmark, rev=revision, force=True)
1006 1005
1007 1006 @reraise_safe_exceptions
1008 1007 def install_hooks(self, wire, force=False):
1009 1008 # we don't need any special hooks for Mercurial
1010 1009 pass
1011 1010
1012 1011 @reraise_safe_exceptions
1013 1012 def get_hooks_info(self, wire):
1014 1013 return {
1015 1014 'pre_version': vcsserver.__version__,
1016 1015 'post_version': vcsserver.__version__,
1017 1016 }
1018 1017
1019 1018 @reraise_safe_exceptions
1020 1019 def set_head_ref(self, wire, head_name):
1021 1020 pass
1022 1021
1023 1022 @reraise_safe_exceptions
1024 1023 def archive_repo(self, wire, archive_dest_path, kind, mtime, archive_at_path,
1025 1024 archive_dir_name, commit_id):
1026 1025
1027 1026 def file_walker(_commit_id, path):
1028 1027 repo = self._factory.repo(wire)
1029 1028 ctx = repo[_commit_id]
1030 1029 is_root = path in ['', '/']
1031 1030 if is_root:
1032 1031 matcher = alwaysmatcher(badfn=None)
1033 1032 else:
1034 1033 matcher = patternmatcher('', [(b'glob', path+'/**', b'')], badfn=None)
1035 1034 file_iter = ctx.manifest().walk(matcher)
1036 1035
1037 1036 for fn in file_iter:
1038 1037 file_path = fn
1039 1038 flags = ctx.flags(fn)
1040 1039 mode = b'x' in flags and 0o755 or 0o644
1041 1040 is_link = b'l' in flags
1042 1041
1043 1042 yield ArchiveNode(file_path, mode, is_link, ctx[fn].data)
1044 1043
1045 1044 return archive_repo(file_walker, archive_dest_path, kind, mtime, archive_at_path,
1046 1045 archive_dir_name, commit_id)
1047 1046
@@ -1,79 +1,84 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2020 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 """
19 19 Mercurial libs compatibility
20 20 """
21 21
22 22 import mercurial
23 23 from mercurial import demandimport
24
24 25 # patch demandimport, due to bug in mercurial when it always triggers
25 26 # demandimport.enable()
26 27 demandimport.enable = lambda *args, **kwargs: 1
27 28
28 29 from mercurial import ui
29 30 from mercurial import patch
30 31 from mercurial import config
31 32 from mercurial import extensions
32 33 from mercurial import scmutil
33 34 from mercurial import archival
34 35 from mercurial import discovery
35 36 from mercurial import unionrepo
36 37 from mercurial import localrepo
37 38 from mercurial import merge as hg_merge
38 39 from mercurial import subrepo
39 40 from mercurial import subrepoutil
40 41 from mercurial import tags as hg_tag
41 42 from mercurial import util as hgutil
42 from mercurial.commands import clone, nullid, pull
43 from mercurial.commands import clone, pull
44 from mercurial.node import nullid
43 45 from mercurial.context import memctx, memfilectx
44 46 from mercurial.error import (
45 47 LookupError, RepoError, RepoLookupError, Abort, InterventionRequired,
46 48 RequirementError, ProgrammingError)
47 49 from mercurial.hgweb import hgweb_mod
48 50 from mercurial.localrepo import instance
49 51 from mercurial.match import match, alwaysmatcher, patternmatcher
50 52 from mercurial.mdiff import diffopts
51 53 from mercurial.node import bin, hex
52 54 from mercurial.encoding import tolocal
53 55 from mercurial.discovery import findcommonoutgoing
54 56 from mercurial.hg import peer
55 57 from mercurial.httppeer import makepeer
56 from mercurial.util import url as hg_url
58 from mercurial.utils.urlutil import url as hg_url
57 59 from mercurial.scmutil import revrange, revsymbol
58 60 from mercurial.node import nullrev
59 61 from mercurial import exchange
60 62 from hgext import largefiles
61 63
62 64 # those authnadlers are patched for python 2.6.5 bug an
63 65 # infinit looping when given invalid resources
64 66 from mercurial.url import httpbasicauthhandler, httpdigestauthhandler
65 67
68 # hg strip is in core now
69 from mercurial import strip as hgext_strip
70
66 71
67 72 def get_ctx(repo, ref):
68 73 try:
69 74 ctx = repo[ref]
70 75 except (ProgrammingError, TypeError):
71 76 # we're unable to find the rev using a regular lookup, we fallback
72 77 # to slower, but backward compat revsymbol usage
73 78 ctx = revsymbol(repo, ref)
74 79 except (LookupError, RepoLookupError):
75 80 # Similar case as above but only for refs that are not numeric
76 81 if isinstance(ref, (int, long)):
77 82 raise
78 83 ctx = revsymbol(repo, ref)
79 84 return ctx
@@ -1,263 +1,261 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2020 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import os
19 19 import time
20 20 import logging
21 21 import functools
22 22
23 23 from dogpile.cache import CacheRegion
24 from dogpile.cache.util import compat
25 24
26 25 from vcsserver.utils import safe_str, sha1
27
28 26 from vcsserver.lib.rc_cache import region_meta
29 27
30 28 log = logging.getLogger(__name__)
31 29
32 30
33 31 class RhodeCodeCacheRegion(CacheRegion):
34 32
35 33 def conditional_cache_on_arguments(
36 34 self, namespace=None,
37 35 expiration_time=None,
38 36 should_cache_fn=None,
39 to_str=compat.string_type,
37 to_str=str,
40 38 function_key_generator=None,
41 39 condition=True):
42 40 """
43 41 Custom conditional decorator, that will not touch any dogpile internals if
44 42 condition isn't meet. This works a bit different than should_cache_fn
45 43 And it's faster in cases we don't ever want to compute cached values
46 44 """
47 expiration_time_is_callable = compat.callable(expiration_time)
45 expiration_time_is_callable = callable(expiration_time)
48 46
49 47 if function_key_generator is None:
50 48 function_key_generator = self.function_key_generator
51 49
52 50 # workaround for py2 and cython problems, this block should be removed
53 51 # once we've migrated to py3
54 52 if 'cython' == 'cython':
55 53 def decorator(fn):
56 if to_str is compat.string_type:
54 if to_str is str:
57 55 # backwards compatible
58 56 key_generator = function_key_generator(namespace, fn)
59 57 else:
60 58 key_generator = function_key_generator(namespace, fn, to_str=to_str)
61 59
62 60 @functools.wraps(fn)
63 61 def decorate(*arg, **kw):
64 62 key = key_generator(*arg, **kw)
65 63
66 64 @functools.wraps(fn)
67 65 def creator():
68 66 return fn(*arg, **kw)
69 67
70 68 if not condition:
71 69 return creator()
72 70
73 71 timeout = expiration_time() if expiration_time_is_callable \
74 72 else expiration_time
75 73
76 74 return self.get_or_create(key, creator, timeout, should_cache_fn)
77 75
78 76 def invalidate(*arg, **kw):
79 77 key = key_generator(*arg, **kw)
80 78 self.delete(key)
81 79
82 80 def set_(value, *arg, **kw):
83 81 key = key_generator(*arg, **kw)
84 82 self.set(key, value)
85 83
86 84 def get(*arg, **kw):
87 85 key = key_generator(*arg, **kw)
88 86 return self.get(key)
89 87
90 88 def refresh(*arg, **kw):
91 89 key = key_generator(*arg, **kw)
92 90 value = fn(*arg, **kw)
93 91 self.set(key, value)
94 92 return value
95 93
96 94 decorate.set = set_
97 95 decorate.invalidate = invalidate
98 96 decorate.refresh = refresh
99 97 decorate.get = get
100 98 decorate.original = fn
101 99 decorate.key_generator = key_generator
102 100 decorate.__wrapped__ = fn
103 101
104 102 return decorate
105 103 return decorator
106 104
107 105 def get_or_create_for_user_func(key_generator, user_func, *arg, **kw):
108 106
109 107 if not condition:
110 108 log.debug('Calling un-cached method:%s', user_func.func_name)
111 109 start = time.time()
112 110 result = user_func(*arg, **kw)
113 111 total = time.time() - start
114 112 log.debug('un-cached method:%s took %.4fs', user_func.func_name, total)
115 113 return result
116 114
117 115 key = key_generator(*arg, **kw)
118 116
119 117 timeout = expiration_time() if expiration_time_is_callable \
120 118 else expiration_time
121 119
122 120 log.debug('Calling cached method:`%s`', user_func.func_name)
123 121 return self.get_or_create(key, user_func, timeout, should_cache_fn, (arg, kw))
124 122
125 123 def cache_decorator(user_func):
126 if to_str is compat.string_type:
124 if to_str is str:
127 125 # backwards compatible
128 126 key_generator = function_key_generator(namespace, user_func)
129 127 else:
130 128 key_generator = function_key_generator(namespace, user_func, to_str=to_str)
131 129
132 130 def refresh(*arg, **kw):
133 131 """
134 132 Like invalidate, but regenerates the value instead
135 133 """
136 134 key = key_generator(*arg, **kw)
137 135 value = user_func(*arg, **kw)
138 136 self.set(key, value)
139 137 return value
140 138
141 139 def invalidate(*arg, **kw):
142 140 key = key_generator(*arg, **kw)
143 141 self.delete(key)
144 142
145 143 def set_(value, *arg, **kw):
146 144 key = key_generator(*arg, **kw)
147 145 self.set(key, value)
148 146
149 147 def get(*arg, **kw):
150 148 key = key_generator(*arg, **kw)
151 149 return self.get(key)
152 150
153 151 user_func.set = set_
154 152 user_func.invalidate = invalidate
155 153 user_func.get = get
156 154 user_func.refresh = refresh
157 155 user_func.key_generator = key_generator
158 156 user_func.original = user_func
159 157
160 158 # Use `decorate` to preserve the signature of :param:`user_func`.
161 159 return decorator.decorate(user_func, functools.partial(
162 160 get_or_create_for_user_func, key_generator))
163 161
164 162 return cache_decorator
165 163
166 164
167 165 def make_region(*arg, **kw):
168 166 return RhodeCodeCacheRegion(*arg, **kw)
169 167
170 168
171 169 def get_default_cache_settings(settings, prefixes=None):
172 170 prefixes = prefixes or []
173 171 cache_settings = {}
174 172 for key in settings.keys():
175 173 for prefix in prefixes:
176 174 if key.startswith(prefix):
177 175 name = key.split(prefix)[1].strip()
178 176 val = settings[key]
179 if isinstance(val, compat.string_types):
177 if isinstance(val, str):
180 178 val = val.strip()
181 179 cache_settings[name] = val
182 180 return cache_settings
183 181
184 182
185 183 def compute_key_from_params(*args):
186 184 """
187 185 Helper to compute key from given params to be used in cache manager
188 186 """
189 187 return sha1("_".join(map(safe_str, args)))
190 188
191 189
192 190 def backend_key_generator(backend):
193 191 """
194 192 Special wrapper that also sends over the backend to the key generator
195 193 """
196 194 def wrapper(namespace, fn):
197 195 return key_generator(backend, namespace, fn)
198 196 return wrapper
199 197
200 198
201 199 def key_generator(backend, namespace, fn):
202 200 fname = fn.__name__
203 201
204 202 def generate_key(*args):
205 203 backend_prefix = getattr(backend, 'key_prefix', None) or 'backend_prefix'
206 204 namespace_pref = namespace or 'default_namespace'
207 205 arg_key = compute_key_from_params(*args)
208 206 final_key = "{}:{}:{}_{}".format(backend_prefix, namespace_pref, fname, arg_key)
209 207
210 208 return final_key
211 209
212 210 return generate_key
213 211
214 212
215 213 def get_or_create_region(region_name, region_namespace=None):
216 214 from vcsserver.lib.rc_cache.backends import FileNamespaceBackend
217 215 region_obj = region_meta.dogpile_cache_regions.get(region_name)
218 216 if not region_obj:
219 217 raise EnvironmentError(
220 218 'Region `{}` not in configured: {}.'.format(
221 219 region_name, region_meta.dogpile_cache_regions.keys()))
222 220
223 221 region_uid_name = '{}:{}'.format(region_name, region_namespace)
224 222 if isinstance(region_obj.actual_backend, FileNamespaceBackend):
225 223 region_exist = region_meta.dogpile_cache_regions.get(region_namespace)
226 224 if region_exist:
227 225 log.debug('Using already configured region: %s', region_namespace)
228 226 return region_exist
229 227 cache_dir = region_meta.dogpile_config_defaults['cache_dir']
230 228 expiration_time = region_obj.expiration_time
231 229
232 230 if not os.path.isdir(cache_dir):
233 231 os.makedirs(cache_dir)
234 232 new_region = make_region(
235 233 name=region_uid_name,
236 234 function_key_generator=backend_key_generator(region_obj.actual_backend)
237 235 )
238 236 namespace_filename = os.path.join(
239 237 cache_dir, "{}.cache.dbm".format(region_namespace))
240 238 # special type that allows 1db per namespace
241 239 new_region.configure(
242 240 backend='dogpile.cache.rc.file_namespace',
243 241 expiration_time=expiration_time,
244 242 arguments={"filename": namespace_filename}
245 243 )
246 244
247 245 # create and save in region caches
248 246 log.debug('configuring new region: %s', region_uid_name)
249 247 region_obj = region_meta.dogpile_cache_regions[region_namespace] = new_region
250 248
251 249 return region_obj
252 250
253 251
254 252 def clear_cache_namespace(cache_region, cache_namespace_uid, invalidate=False):
255 253 region = get_or_create_region(cache_region, cache_namespace_uid)
256 254 cache_keys = region.backend.list_keys(prefix=cache_namespace_uid)
257 255 num_delete_keys = len(cache_keys)
258 256 if invalidate:
259 257 region.invalidate(hard=False)
260 258 else:
261 259 if num_delete_keys:
262 260 region.delete_multi(cache_keys)
263 261 return num_delete_keys
@@ -1,867 +1,865 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2020 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18
19
20 19 import os
21 20 import subprocess
22 import time
23 21 from urllib.error import URLError
24 22 import urllib.parse
25 23 import logging
26 24 import posixpath as vcspath
27 25 import io
28 26 import urllib.request, urllib.parse, urllib.error
29 27 import traceback
30 28
31 29 import svn.client
32 30 import svn.core
33 31 import svn.delta
34 32 import svn.diff
35 33 import svn.fs
36 34 import svn.repos
37 35
38 36 from vcsserver import svn_diff, exceptions, subprocessio, settings
39 37 from vcsserver.base import RepoFactory, raise_from_original, ArchiveNode, archive_repo
40 38 from vcsserver.exceptions import NoContentException
41 39 from vcsserver.utils import safe_str
42 40 from vcsserver.vcs_base import RemoteBase
43 41
44 42 log = logging.getLogger(__name__)
45 43
46 44
47 45 svn_compatible_versions_map = {
48 46 'pre-1.4-compatible': '1.3',
49 47 'pre-1.5-compatible': '1.4',
50 48 'pre-1.6-compatible': '1.5',
51 49 'pre-1.8-compatible': '1.7',
52 50 'pre-1.9-compatible': '1.8',
53 51 }
54 52
55 53 current_compatible_version = '1.12'
56 54
57 55
58 56 def reraise_safe_exceptions(func):
59 57 """Decorator for converting svn exceptions to something neutral."""
60 58 def wrapper(*args, **kwargs):
61 59 try:
62 60 return func(*args, **kwargs)
63 61 except Exception as e:
64 62 if not hasattr(e, '_vcs_kind'):
65 63 log.exception("Unhandled exception in svn remote call")
66 64 raise_from_original(exceptions.UnhandledException(e))
67 65 raise
68 66 return wrapper
69 67
70 68
71 69 class SubversionFactory(RepoFactory):
72 70 repo_type = 'svn'
73 71
74 72 def _create_repo(self, wire, create, compatible_version):
75 73 path = svn.core.svn_path_canonicalize(wire['path'])
76 74 if create:
77 75 fs_config = {'compatible-version': current_compatible_version}
78 76 if compatible_version:
79 77
80 78 compatible_version_string = \
81 79 svn_compatible_versions_map.get(compatible_version) \
82 80 or compatible_version
83 81 fs_config['compatible-version'] = compatible_version_string
84 82
85 83 log.debug('Create SVN repo with config "%s"', fs_config)
86 84 repo = svn.repos.create(path, "", "", None, fs_config)
87 85 else:
88 86 repo = svn.repos.open(path)
89 87
90 88 log.debug('Got SVN object: %s', repo)
91 89 return repo
92 90
93 91 def repo(self, wire, create=False, compatible_version=None):
94 92 """
95 93 Get a repository instance for the given path.
96 94 """
97 95 return self._create_repo(wire, create, compatible_version)
98 96
99 97
100 98 NODE_TYPE_MAPPING = {
101 99 svn.core.svn_node_file: 'file',
102 100 svn.core.svn_node_dir: 'dir',
103 101 }
104 102
105 103
106 104 class SvnRemote(RemoteBase):
107 105
108 106 def __init__(self, factory, hg_factory=None):
109 107 self._factory = factory
110 108 # TODO: Remove once we do not use internal Mercurial objects anymore
111 109 # for subversion
112 110 self._hg_factory = hg_factory
113 111
114 112 @reraise_safe_exceptions
115 113 def discover_svn_version(self):
116 114 try:
117 115 import svn.core
118 116 svn_ver = svn.core.SVN_VERSION
119 117 except ImportError:
120 118 svn_ver = None
121 119 return svn_ver
122 120
123 121 @reraise_safe_exceptions
124 122 def is_empty(self, wire):
125 123
126 124 try:
127 125 return self.lookup(wire, -1) == 0
128 126 except Exception:
129 127 log.exception("failed to read object_store")
130 128 return False
131 129
132 130 def check_url(self, url, config_items):
133 131 # this can throw exception if not installed, but we detect this
134 132 from hgsubversion import svnrepo
135 133
136 134 baseui = self._hg_factory._create_config(config_items)
137 135 # uuid function get's only valid UUID from proper repo, else
138 136 # throws exception
139 137 try:
140 138 svnrepo.svnremoterepo(baseui, url).svn.uuid
141 139 except Exception:
142 140 tb = traceback.format_exc()
143 141 log.debug("Invalid Subversion url: `%s`, tb: %s", url, tb)
144 142 raise URLError(
145 143 '"%s" is not a valid Subversion source url.' % (url, ))
146 144 return True
147 145
148 146 def is_path_valid_repository(self, wire, path):
149 147
150 148 # NOTE(marcink): short circuit the check for SVN repo
151 149 # the repos.open might be expensive to check, but we have one cheap
152 150 # pre condition that we can use, to check for 'format' file
153 151
154 152 if not os.path.isfile(os.path.join(path, 'format')):
155 153 return False
156 154
157 155 try:
158 156 svn.repos.open(path)
159 157 except svn.core.SubversionException:
160 158 tb = traceback.format_exc()
161 159 log.debug("Invalid Subversion path `%s`, tb: %s", path, tb)
162 160 return False
163 161 return True
164 162
165 163 @reraise_safe_exceptions
166 164 def verify(self, wire,):
167 165 repo_path = wire['path']
168 166 if not self.is_path_valid_repository(wire, repo_path):
169 167 raise Exception(
170 168 "Path %s is not a valid Subversion repository." % repo_path)
171 169
172 170 cmd = ['svnadmin', 'info', repo_path]
173 171 stdout, stderr = subprocessio.run_command(cmd)
174 172 return stdout
175 173
176 174 def lookup(self, wire, revision):
177 175 if revision not in [-1, None, 'HEAD']:
178 176 raise NotImplementedError
179 177 repo = self._factory.repo(wire)
180 178 fs_ptr = svn.repos.fs(repo)
181 179 head = svn.fs.youngest_rev(fs_ptr)
182 180 return head
183 181
184 182 def lookup_interval(self, wire, start_ts, end_ts):
185 183 repo = self._factory.repo(wire)
186 184 fsobj = svn.repos.fs(repo)
187 185 start_rev = None
188 186 end_rev = None
189 187 if start_ts:
190 188 start_ts_svn = apr_time_t(start_ts)
191 189 start_rev = svn.repos.dated_revision(repo, start_ts_svn) + 1
192 190 else:
193 191 start_rev = 1
194 192 if end_ts:
195 193 end_ts_svn = apr_time_t(end_ts)
196 194 end_rev = svn.repos.dated_revision(repo, end_ts_svn)
197 195 else:
198 196 end_rev = svn.fs.youngest_rev(fsobj)
199 197 return start_rev, end_rev
200 198
201 199 def revision_properties(self, wire, revision):
202 200
203 201 cache_on, context_uid, repo_id = self._cache_on(wire)
204 202 region = self._region(wire)
205 203 @region.conditional_cache_on_arguments(condition=cache_on)
206 204 def _revision_properties(_repo_id, _revision):
207 205 repo = self._factory.repo(wire)
208 206 fs_ptr = svn.repos.fs(repo)
209 207 return svn.fs.revision_proplist(fs_ptr, revision)
210 208 return _revision_properties(repo_id, revision)
211 209
212 210 def revision_changes(self, wire, revision):
213 211
214 212 repo = self._factory.repo(wire)
215 213 fsobj = svn.repos.fs(repo)
216 214 rev_root = svn.fs.revision_root(fsobj, revision)
217 215
218 216 editor = svn.repos.ChangeCollector(fsobj, rev_root)
219 217 editor_ptr, editor_baton = svn.delta.make_editor(editor)
220 218 base_dir = ""
221 219 send_deltas = False
222 220 svn.repos.replay2(
223 221 rev_root, base_dir, svn.core.SVN_INVALID_REVNUM, send_deltas,
224 222 editor_ptr, editor_baton, None)
225 223
226 224 added = []
227 225 changed = []
228 226 removed = []
229 227
230 228 # TODO: CHANGE_ACTION_REPLACE: Figure out where it belongs
231 229 for path, change in editor.changes.items():
232 230 # TODO: Decide what to do with directory nodes. Subversion can add
233 231 # empty directories.
234 232
235 233 if change.item_kind == svn.core.svn_node_dir:
236 234 continue
237 235 if change.action in [svn.repos.CHANGE_ACTION_ADD]:
238 236 added.append(path)
239 237 elif change.action in [svn.repos.CHANGE_ACTION_MODIFY,
240 238 svn.repos.CHANGE_ACTION_REPLACE]:
241 239 changed.append(path)
242 240 elif change.action in [svn.repos.CHANGE_ACTION_DELETE]:
243 241 removed.append(path)
244 242 else:
245 243 raise NotImplementedError(
246 244 "Action %s not supported on path %s" % (
247 245 change.action, path))
248 246
249 247 changes = {
250 248 'added': added,
251 249 'changed': changed,
252 250 'removed': removed,
253 251 }
254 252 return changes
255 253
256 254 @reraise_safe_exceptions
257 255 def node_history(self, wire, path, revision, limit):
258 256 cache_on, context_uid, repo_id = self._cache_on(wire)
259 257 region = self._region(wire)
260 258 @region.conditional_cache_on_arguments(condition=cache_on)
261 259 def _assert_correct_path(_context_uid, _repo_id, _path, _revision, _limit):
262 260 cross_copies = False
263 261 repo = self._factory.repo(wire)
264 262 fsobj = svn.repos.fs(repo)
265 263 rev_root = svn.fs.revision_root(fsobj, revision)
266 264
267 265 history_revisions = []
268 266 history = svn.fs.node_history(rev_root, path)
269 267 history = svn.fs.history_prev(history, cross_copies)
270 268 while history:
271 269 __, node_revision = svn.fs.history_location(history)
272 270 history_revisions.append(node_revision)
273 271 if limit and len(history_revisions) >= limit:
274 272 break
275 273 history = svn.fs.history_prev(history, cross_copies)
276 274 return history_revisions
277 275 return _assert_correct_path(context_uid, repo_id, path, revision, limit)
278 276
279 277 def node_properties(self, wire, path, revision):
280 278 cache_on, context_uid, repo_id = self._cache_on(wire)
281 279 region = self._region(wire)
282 280 @region.conditional_cache_on_arguments(condition=cache_on)
283 281 def _node_properties(_repo_id, _path, _revision):
284 282 repo = self._factory.repo(wire)
285 283 fsobj = svn.repos.fs(repo)
286 284 rev_root = svn.fs.revision_root(fsobj, revision)
287 285 return svn.fs.node_proplist(rev_root, path)
288 286 return _node_properties(repo_id, path, revision)
289 287
290 288 def file_annotate(self, wire, path, revision):
291 289 abs_path = 'file://' + urllib.pathname2url(
292 290 vcspath.join(wire['path'], path))
293 291 file_uri = svn.core.svn_path_canonicalize(abs_path)
294 292
295 293 start_rev = svn_opt_revision_value_t(0)
296 294 peg_rev = svn_opt_revision_value_t(revision)
297 295 end_rev = peg_rev
298 296
299 297 annotations = []
300 298
301 299 def receiver(line_no, revision, author, date, line, pool):
302 300 annotations.append((line_no, revision, line))
303 301
304 302 # TODO: Cannot use blame5, missing typemap function in the swig code
305 303 try:
306 304 svn.client.blame2(
307 305 file_uri, peg_rev, start_rev, end_rev,
308 306 receiver, svn.client.create_context())
309 307 except svn.core.SubversionException as exc:
310 308 log.exception("Error during blame operation.")
311 309 raise Exception(
312 310 "Blame not supported or file does not exist at path %s. "
313 311 "Error %s." % (path, exc))
314 312
315 313 return annotations
316 314
317 315 def get_node_type(self, wire, path, revision=None):
318 316
319 317 cache_on, context_uid, repo_id = self._cache_on(wire)
320 318 region = self._region(wire)
321 319 @region.conditional_cache_on_arguments(condition=cache_on)
322 320 def _get_node_type(_repo_id, _path, _revision):
323 321 repo = self._factory.repo(wire)
324 322 fs_ptr = svn.repos.fs(repo)
325 323 if _revision is None:
326 324 _revision = svn.fs.youngest_rev(fs_ptr)
327 325 root = svn.fs.revision_root(fs_ptr, _revision)
328 326 node = svn.fs.check_path(root, path)
329 327 return NODE_TYPE_MAPPING.get(node, None)
330 328 return _get_node_type(repo_id, path, revision)
331 329
332 330 def get_nodes(self, wire, path, revision=None):
333 331
334 332 cache_on, context_uid, repo_id = self._cache_on(wire)
335 333 region = self._region(wire)
336 334 @region.conditional_cache_on_arguments(condition=cache_on)
337 335 def _get_nodes(_repo_id, _path, _revision):
338 336 repo = self._factory.repo(wire)
339 337 fsobj = svn.repos.fs(repo)
340 338 if _revision is None:
341 339 _revision = svn.fs.youngest_rev(fsobj)
342 340 root = svn.fs.revision_root(fsobj, _revision)
343 341 entries = svn.fs.dir_entries(root, path)
344 342 result = []
345 343 for entry_path, entry_info in entries.items():
346 344 result.append(
347 345 (entry_path, NODE_TYPE_MAPPING.get(entry_info.kind, None)))
348 346 return result
349 347 return _get_nodes(repo_id, path, revision)
350 348
351 349 def get_file_content(self, wire, path, rev=None):
352 350 repo = self._factory.repo(wire)
353 351 fsobj = svn.repos.fs(repo)
354 352 if rev is None:
355 353 rev = svn.fs.youngest_revision(fsobj)
356 354 root = svn.fs.revision_root(fsobj, rev)
357 355 content = svn.core.Stream(svn.fs.file_contents(root, path))
358 356 return content.read()
359 357
360 358 def get_file_size(self, wire, path, revision=None):
361 359
362 360 cache_on, context_uid, repo_id = self._cache_on(wire)
363 361 region = self._region(wire)
364 362 @region.conditional_cache_on_arguments(condition=cache_on)
365 363 def _get_file_size(_repo_id, _path, _revision):
366 364 repo = self._factory.repo(wire)
367 365 fsobj = svn.repos.fs(repo)
368 366 if _revision is None:
369 367 _revision = svn.fs.youngest_revision(fsobj)
370 368 root = svn.fs.revision_root(fsobj, _revision)
371 369 size = svn.fs.file_length(root, path)
372 370 return size
373 371 return _get_file_size(repo_id, path, revision)
374 372
375 373 def create_repository(self, wire, compatible_version=None):
376 374 log.info('Creating Subversion repository in path "%s"', wire['path'])
377 375 self._factory.repo(wire, create=True,
378 376 compatible_version=compatible_version)
379 377
380 378 def get_url_and_credentials(self, src_url):
381 379 obj = urllib.parse.urlparse(src_url)
382 380 username = obj.username or None
383 381 password = obj.password or None
384 382 return username, password, src_url
385 383
386 384 def import_remote_repository(self, wire, src_url):
387 385 repo_path = wire['path']
388 386 if not self.is_path_valid_repository(wire, repo_path):
389 387 raise Exception(
390 388 "Path %s is not a valid Subversion repository." % repo_path)
391 389
392 390 username, password, src_url = self.get_url_and_credentials(src_url)
393 391 rdump_cmd = ['svnrdump', 'dump', '--non-interactive',
394 392 '--trust-server-cert-failures=unknown-ca']
395 393 if username and password:
396 394 rdump_cmd += ['--username', username, '--password', password]
397 395 rdump_cmd += [src_url]
398 396
399 397 rdump = subprocess.Popen(
400 398 rdump_cmd,
401 399 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
402 400 load = subprocess.Popen(
403 401 ['svnadmin', 'load', repo_path], stdin=rdump.stdout)
404 402
405 403 # TODO: johbo: This can be a very long operation, might be better
406 404 # to track some kind of status and provide an api to check if the
407 405 # import is done.
408 406 rdump.wait()
409 407 load.wait()
410 408
411 409 log.debug('Return process ended with code: %s', rdump.returncode)
412 410 if rdump.returncode != 0:
413 411 errors = rdump.stderr.read()
414 412 log.error('svnrdump dump failed: statuscode %s: message: %s',
415 413 rdump.returncode, errors)
416 414 reason = 'UNKNOWN'
417 415 if 'svnrdump: E230001:' in errors:
418 416 reason = 'INVALID_CERTIFICATE'
419 417
420 418 if reason == 'UNKNOWN':
421 419 reason = 'UNKNOWN:{}'.format(errors)
422 420 raise Exception(
423 421 'Failed to dump the remote repository from %s. Reason:%s' % (
424 422 src_url, reason))
425 423 if load.returncode != 0:
426 424 raise Exception(
427 425 'Failed to load the dump of remote repository from %s.' %
428 426 (src_url, ))
429 427
430 428 def commit(self, wire, message, author, timestamp, updated, removed):
431 429 assert isinstance(message, str)
432 430 assert isinstance(author, str)
433 431
434 432 repo = self._factory.repo(wire)
435 433 fsobj = svn.repos.fs(repo)
436 434
437 435 rev = svn.fs.youngest_rev(fsobj)
438 436 txn = svn.repos.fs_begin_txn_for_commit(repo, rev, author, message)
439 437 txn_root = svn.fs.txn_root(txn)
440 438
441 439 for node in updated:
442 440 TxnNodeProcessor(node, txn_root).update()
443 441 for node in removed:
444 442 TxnNodeProcessor(node, txn_root).remove()
445 443
446 444 commit_id = svn.repos.fs_commit_txn(repo, txn)
447 445
448 446 if timestamp:
449 447 apr_time = apr_time_t(timestamp)
450 448 ts_formatted = svn.core.svn_time_to_cstring(apr_time)
451 449 svn.fs.change_rev_prop(fsobj, commit_id, 'svn:date', ts_formatted)
452 450
453 451 log.debug('Committed revision "%s" to "%s".', commit_id, wire['path'])
454 452 return commit_id
455 453
456 454 def diff(self, wire, rev1, rev2, path1=None, path2=None,
457 455 ignore_whitespace=False, context=3):
458 456
459 457 wire.update(cache=False)
460 458 repo = self._factory.repo(wire)
461 459 diff_creator = SvnDiffer(
462 460 repo, rev1, path1, rev2, path2, ignore_whitespace, context)
463 461 try:
464 462 return diff_creator.generate_diff()
465 463 except svn.core.SubversionException as e:
466 464 log.exception(
467 465 "Error during diff operation operation. "
468 466 "Path might not exist %s, %s" % (path1, path2))
469 467 return ""
470 468
471 469 @reraise_safe_exceptions
472 470 def is_large_file(self, wire, path):
473 471 return False
474 472
475 473 @reraise_safe_exceptions
476 474 def is_binary(self, wire, rev, path):
477 475 cache_on, context_uid, repo_id = self._cache_on(wire)
478 476
479 477 region = self._region(wire)
480 478 @region.conditional_cache_on_arguments(condition=cache_on)
481 479 def _is_binary(_repo_id, _rev, _path):
482 480 raw_bytes = self.get_file_content(wire, path, rev)
483 481 return raw_bytes and '\0' in raw_bytes
484 482
485 483 return _is_binary(repo_id, rev, path)
486 484
487 485 @reraise_safe_exceptions
488 486 def run_svn_command(self, wire, cmd, **opts):
489 487 path = wire.get('path', None)
490 488
491 489 if path and os.path.isdir(path):
492 490 opts['cwd'] = path
493 491
494 492 safe_call = opts.pop('_safe', False)
495 493
496 494 svnenv = os.environ.copy()
497 495 svnenv.update(opts.pop('extra_env', {}))
498 496
499 497 _opts = {'env': svnenv, 'shell': False}
500 498
501 499 try:
502 500 _opts.update(opts)
503 501 p = subprocessio.SubprocessIOChunker(cmd, **_opts)
504 502
505 503 return ''.join(p), ''.join(p.error)
506 504 except (EnvironmentError, OSError) as err:
507 505 if safe_call:
508 506 return '', safe_str(err).strip()
509 507 else:
510 508 cmd = ' '.join(cmd) # human friendly CMD
511 509 tb_err = ("Couldn't run svn command (%s).\n"
512 510 "Original error was:%s\n"
513 511 "Call options:%s\n"
514 512 % (cmd, err, _opts))
515 513 log.exception(tb_err)
516 514 raise exceptions.VcsException()(tb_err)
517 515
518 516 @reraise_safe_exceptions
519 517 def install_hooks(self, wire, force=False):
520 518 from vcsserver.hook_utils import install_svn_hooks
521 519 repo_path = wire['path']
522 520 binary_dir = settings.BINARY_DIR
523 521 executable = None
524 522 if binary_dir:
525 523 executable = os.path.join(binary_dir, 'python')
526 524 return install_svn_hooks(
527 525 repo_path, executable=executable, force_create=force)
528 526
529 527 @reraise_safe_exceptions
530 528 def get_hooks_info(self, wire):
531 529 from vcsserver.hook_utils import (
532 530 get_svn_pre_hook_version, get_svn_post_hook_version)
533 531 repo_path = wire['path']
534 532 return {
535 533 'pre_version': get_svn_pre_hook_version(repo_path),
536 534 'post_version': get_svn_post_hook_version(repo_path),
537 535 }
538 536
539 537 @reraise_safe_exceptions
540 538 def set_head_ref(self, wire, head_name):
541 539 pass
542 540
543 541 @reraise_safe_exceptions
544 542 def archive_repo(self, wire, archive_dest_path, kind, mtime, archive_at_path,
545 543 archive_dir_name, commit_id):
546 544
547 545 def walk_tree(root, root_dir, _commit_id):
548 546 """
549 547 Special recursive svn repo walker
550 548 """
551 549
552 550 filemode_default = 0o100644
553 551 filemode_executable = 0o100755
554 552
555 553 file_iter = svn.fs.dir_entries(root, root_dir)
556 554 for f_name in file_iter:
557 555 f_type = NODE_TYPE_MAPPING.get(file_iter[f_name].kind, None)
558 556
559 557 if f_type == 'dir':
560 558 # return only DIR, and then all entries in that dir
561 559 yield os.path.join(root_dir, f_name), {'mode': filemode_default}, f_type
562 560 new_root = os.path.join(root_dir, f_name)
563 561 for _f_name, _f_data, _f_type in walk_tree(root, new_root, _commit_id):
564 562 yield _f_name, _f_data, _f_type
565 563 else:
566 564 f_path = os.path.join(root_dir, f_name).rstrip('/')
567 565 prop_list = svn.fs.node_proplist(root, f_path)
568 566
569 567 f_mode = filemode_default
570 568 if prop_list.get('svn:executable'):
571 569 f_mode = filemode_executable
572 570
573 571 f_is_link = False
574 572 if prop_list.get('svn:special'):
575 573 f_is_link = True
576 574
577 575 data = {
578 576 'is_link': f_is_link,
579 577 'mode': f_mode,
580 578 'content_stream': svn.core.Stream(svn.fs.file_contents(root, f_path)).read
581 579 }
582 580
583 581 yield f_path, data, f_type
584 582
585 583 def file_walker(_commit_id, path):
586 584 repo = self._factory.repo(wire)
587 585 root = svn.fs.revision_root(svn.repos.fs(repo), int(commit_id))
588 586
589 587 def no_content():
590 588 raise NoContentException()
591 589
592 590 for f_name, f_data, f_type in walk_tree(root, path, _commit_id):
593 591 file_path = f_name
594 592
595 593 if f_type == 'dir':
596 594 mode = f_data['mode']
597 595 yield ArchiveNode(file_path, mode, False, no_content)
598 596 else:
599 597 mode = f_data['mode']
600 598 is_link = f_data['is_link']
601 599 data_stream = f_data['content_stream']
602 600 yield ArchiveNode(file_path, mode, is_link, data_stream)
603 601
604 602 return archive_repo(file_walker, archive_dest_path, kind, mtime, archive_at_path,
605 603 archive_dir_name, commit_id)
606 604
607 605
608 606 class SvnDiffer(object):
609 607 """
610 608 Utility to create diffs based on difflib and the Subversion api
611 609 """
612 610
613 611 binary_content = False
614 612
615 613 def __init__(
616 614 self, repo, src_rev, src_path, tgt_rev, tgt_path,
617 615 ignore_whitespace, context):
618 616 self.repo = repo
619 617 self.ignore_whitespace = ignore_whitespace
620 618 self.context = context
621 619
622 620 fsobj = svn.repos.fs(repo)
623 621
624 622 self.tgt_rev = tgt_rev
625 623 self.tgt_path = tgt_path or ''
626 624 self.tgt_root = svn.fs.revision_root(fsobj, tgt_rev)
627 625 self.tgt_kind = svn.fs.check_path(self.tgt_root, self.tgt_path)
628 626
629 627 self.src_rev = src_rev
630 628 self.src_path = src_path or self.tgt_path
631 629 self.src_root = svn.fs.revision_root(fsobj, src_rev)
632 630 self.src_kind = svn.fs.check_path(self.src_root, self.src_path)
633 631
634 632 self._validate()
635 633
636 634 def _validate(self):
637 635 if (self.tgt_kind != svn.core.svn_node_none and
638 636 self.src_kind != svn.core.svn_node_none and
639 637 self.src_kind != self.tgt_kind):
640 638 # TODO: johbo: proper error handling
641 639 raise Exception(
642 640 "Source and target are not compatible for diff generation. "
643 641 "Source type: %s, target type: %s" %
644 642 (self.src_kind, self.tgt_kind))
645 643
646 644 def generate_diff(self):
647 645 buf = io.StringIO()
648 646 if self.tgt_kind == svn.core.svn_node_dir:
649 647 self._generate_dir_diff(buf)
650 648 else:
651 649 self._generate_file_diff(buf)
652 650 return buf.getvalue()
653 651
654 652 def _generate_dir_diff(self, buf):
655 653 editor = DiffChangeEditor()
656 654 editor_ptr, editor_baton = svn.delta.make_editor(editor)
657 655 svn.repos.dir_delta2(
658 656 self.src_root,
659 657 self.src_path,
660 658 '', # src_entry
661 659 self.tgt_root,
662 660 self.tgt_path,
663 661 editor_ptr, editor_baton,
664 662 authorization_callback_allow_all,
665 663 False, # text_deltas
666 664 svn.core.svn_depth_infinity, # depth
667 665 False, # entry_props
668 666 False, # ignore_ancestry
669 667 )
670 668
671 669 for path, __, change in sorted(editor.changes):
672 670 self._generate_node_diff(
673 671 buf, change, path, self.tgt_path, path, self.src_path)
674 672
675 673 def _generate_file_diff(self, buf):
676 674 change = None
677 675 if self.src_kind == svn.core.svn_node_none:
678 676 change = "add"
679 677 elif self.tgt_kind == svn.core.svn_node_none:
680 678 change = "delete"
681 679 tgt_base, tgt_path = vcspath.split(self.tgt_path)
682 680 src_base, src_path = vcspath.split(self.src_path)
683 681 self._generate_node_diff(
684 682 buf, change, tgt_path, tgt_base, src_path, src_base)
685 683
686 684 def _generate_node_diff(
687 685 self, buf, change, tgt_path, tgt_base, src_path, src_base):
688 686
689 687 if self.src_rev == self.tgt_rev and tgt_base == src_base:
690 688 # makes consistent behaviour with git/hg to return empty diff if
691 689 # we compare same revisions
692 690 return
693 691
694 692 tgt_full_path = vcspath.join(tgt_base, tgt_path)
695 693 src_full_path = vcspath.join(src_base, src_path)
696 694
697 695 self.binary_content = False
698 696 mime_type = self._get_mime_type(tgt_full_path)
699 697
700 698 if mime_type and not mime_type.startswith('text'):
701 699 self.binary_content = True
702 700 buf.write("=" * 67 + '\n')
703 701 buf.write("Cannot display: file marked as a binary type.\n")
704 702 buf.write("svn:mime-type = %s\n" % mime_type)
705 703 buf.write("Index: %s\n" % (tgt_path, ))
706 704 buf.write("=" * 67 + '\n')
707 705 buf.write("diff --git a/%(tgt_path)s b/%(tgt_path)s\n" % {
708 706 'tgt_path': tgt_path})
709 707
710 708 if change == 'add':
711 709 # TODO: johbo: SVN is missing a zero here compared to git
712 710 buf.write("new file mode 10644\n")
713 711
714 712 #TODO(marcink): intro to binary detection of svn patches
715 713 # if self.binary_content:
716 714 # buf.write('GIT binary patch\n')
717 715
718 716 buf.write("--- /dev/null\t(revision 0)\n")
719 717 src_lines = []
720 718 else:
721 719 if change == 'delete':
722 720 buf.write("deleted file mode 10644\n")
723 721
724 722 #TODO(marcink): intro to binary detection of svn patches
725 723 # if self.binary_content:
726 724 # buf.write('GIT binary patch\n')
727 725
728 726 buf.write("--- a/%s\t(revision %s)\n" % (
729 727 src_path, self.src_rev))
730 728 src_lines = self._svn_readlines(self.src_root, src_full_path)
731 729
732 730 if change == 'delete':
733 731 buf.write("+++ /dev/null\t(revision %s)\n" % (self.tgt_rev, ))
734 732 tgt_lines = []
735 733 else:
736 734 buf.write("+++ b/%s\t(revision %s)\n" % (
737 735 tgt_path, self.tgt_rev))
738 736 tgt_lines = self._svn_readlines(self.tgt_root, tgt_full_path)
739 737
740 738 if not self.binary_content:
741 739 udiff = svn_diff.unified_diff(
742 740 src_lines, tgt_lines, context=self.context,
743 741 ignore_blank_lines=self.ignore_whitespace,
744 742 ignore_case=False,
745 743 ignore_space_changes=self.ignore_whitespace)
746 744 buf.writelines(udiff)
747 745
748 746 def _get_mime_type(self, path):
749 747 try:
750 748 mime_type = svn.fs.node_prop(
751 749 self.tgt_root, path, svn.core.SVN_PROP_MIME_TYPE)
752 750 except svn.core.SubversionException:
753 751 mime_type = svn.fs.node_prop(
754 752 self.src_root, path, svn.core.SVN_PROP_MIME_TYPE)
755 753 return mime_type
756 754
757 755 def _svn_readlines(self, fs_root, node_path):
758 756 if self.binary_content:
759 757 return []
760 758 node_kind = svn.fs.check_path(fs_root, node_path)
761 759 if node_kind not in (
762 760 svn.core.svn_node_file, svn.core.svn_node_symlink):
763 761 return []
764 762 content = svn.core.Stream(
765 763 svn.fs.file_contents(fs_root, node_path)).read()
766 764 return content.splitlines(True)
767 765
768 766
769 767 class DiffChangeEditor(svn.delta.Editor):
770 768 """
771 769 Records changes between two given revisions
772 770 """
773 771
774 772 def __init__(self):
775 773 self.changes = []
776 774
777 775 def delete_entry(self, path, revision, parent_baton, pool=None):
778 776 self.changes.append((path, None, 'delete'))
779 777
780 778 def add_file(
781 779 self, path, parent_baton, copyfrom_path, copyfrom_revision,
782 780 file_pool=None):
783 781 self.changes.append((path, 'file', 'add'))
784 782
785 783 def open_file(self, path, parent_baton, base_revision, file_pool=None):
786 784 self.changes.append((path, 'file', 'change'))
787 785
788 786
789 787 def authorization_callback_allow_all(root, path, pool):
790 788 return True
791 789
792 790
793 791 class TxnNodeProcessor(object):
794 792 """
795 793 Utility to process the change of one node within a transaction root.
796 794
797 795 It encapsulates the knowledge of how to add, update or remove
798 796 a node for a given transaction root. The purpose is to support the method
799 797 `SvnRemote.commit`.
800 798 """
801 799
802 800 def __init__(self, node, txn_root):
803 801 assert isinstance(node['path'], str)
804 802
805 803 self.node = node
806 804 self.txn_root = txn_root
807 805
808 806 def update(self):
809 807 self._ensure_parent_dirs()
810 808 self._add_file_if_node_does_not_exist()
811 809 self._update_file_content()
812 810 self._update_file_properties()
813 811
814 812 def remove(self):
815 813 svn.fs.delete(self.txn_root, self.node['path'])
816 814 # TODO: Clean up directory if empty
817 815
818 816 def _ensure_parent_dirs(self):
819 817 curdir = vcspath.dirname(self.node['path'])
820 818 dirs_to_create = []
821 819 while not self._svn_path_exists(curdir):
822 820 dirs_to_create.append(curdir)
823 821 curdir = vcspath.dirname(curdir)
824 822
825 823 for curdir in reversed(dirs_to_create):
826 824 log.debug('Creating missing directory "%s"', curdir)
827 825 svn.fs.make_dir(self.txn_root, curdir)
828 826
829 827 def _svn_path_exists(self, path):
830 828 path_status = svn.fs.check_path(self.txn_root, path)
831 829 return path_status != svn.core.svn_node_none
832 830
833 831 def _add_file_if_node_does_not_exist(self):
834 832 kind = svn.fs.check_path(self.txn_root, self.node['path'])
835 833 if kind == svn.core.svn_node_none:
836 834 svn.fs.make_file(self.txn_root, self.node['path'])
837 835
838 836 def _update_file_content(self):
839 837 assert isinstance(self.node['content'], str)
840 838 handler, baton = svn.fs.apply_textdelta(
841 839 self.txn_root, self.node['path'], None, None)
842 840 svn.delta.svn_txdelta_send_string(self.node['content'], handler, baton)
843 841
844 842 def _update_file_properties(self):
845 843 properties = self.node.get('properties', {})
846 844 for key, value in properties.items():
847 845 svn.fs.change_node_prop(
848 846 self.txn_root, self.node['path'], key, value)
849 847
850 848
851 849 def apr_time_t(timestamp):
852 850 """
853 851 Convert a Python timestamp into APR timestamp type apr_time_t
854 852 """
855 853 return timestamp * 1E6
856 854
857 855
858 856 def svn_opt_revision_value_t(num):
859 857 """
860 858 Put `num` into a `svn_opt_revision_value_t` structure.
861 859 """
862 860 value = svn.core.svn_opt_revision_value_t()
863 861 value.number = num
864 862 revision = svn.core.svn_opt_revision_t()
865 863 revision.kind = svn.core.svn_opt_revision_number
866 864 revision.value = value
867 865 return revision
General Comments 0
You need to be logged in to leave comments. Login now