##// END OF EJS Templates
git: rename fetch into pull because this is what it actually does.
marcink -
r550:964721d2 default
parent child Browse files
Show More
@@ -1,711 +1,716 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2018 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17 import collections
18 18 import logging
19 19 import os
20 20 import posixpath as vcspath
21 21 import re
22 22 import stat
23 23 import traceback
24 24 import urllib
25 25 import urllib2
26 26 from functools import wraps
27 27
28 28 from dulwich import index, objects
29 29 from dulwich.client import HttpGitClient, LocalGitClient
30 30 from dulwich.errors import (
31 31 NotGitRepository, ChecksumMismatch, WrongObjectException,
32 32 MissingCommitError, ObjectMissing, HangupException,
33 33 UnexpectedCommandError)
34 34 from dulwich.repo import Repo as DulwichRepo, Tag
35 35 from dulwich.server import update_server_info
36 36
37 37 from vcsserver import exceptions, settings, subprocessio
38 38 from vcsserver.utils import safe_str
39 39 from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original
40 40 from vcsserver.hgcompat import (
41 41 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
42 42 from vcsserver.git_lfs.lib import LFSOidStore
43 43
44 44 DIR_STAT = stat.S_IFDIR
45 45 FILE_MODE = stat.S_IFMT
46 46 GIT_LINK = objects.S_IFGITLINK
47 47
48 48 log = logging.getLogger(__name__)
49 49
50 50
51 51 def reraise_safe_exceptions(func):
52 52 """Converts Dulwich exceptions to something neutral."""
53 53 @wraps(func)
54 54 def wrapper(*args, **kwargs):
55 55 try:
56 56 return func(*args, **kwargs)
57 57 except (ChecksumMismatch, WrongObjectException, MissingCommitError,
58 58 ObjectMissing) as e:
59 59 raise exceptions.LookupException(e)(e.message)
60 60 except (HangupException, UnexpectedCommandError) as e:
61 61 raise exceptions.VcsException(e)(e.message)
62 62 except Exception as e:
63 63 # NOTE(marcink): becuase of how dulwich handles some exceptions
64 64 # (KeyError on empty repos), we cannot track this and catch all
65 65 # exceptions, it's an exceptions from other handlers
66 66 #if not hasattr(e, '_vcs_kind'):
67 67 #log.exception("Unhandled exception in git remote call")
68 68 #raise_from_original(exceptions.UnhandledException)
69 69 raise
70 70 return wrapper
71 71
72 72
73 73 class Repo(DulwichRepo):
74 74 """
75 75 A wrapper for dulwich Repo class.
76 76
77 77 Since dulwich is sometimes keeping .idx file descriptors open, it leads to
78 78 "Too many open files" error. We need to close all opened file descriptors
79 79 once the repo object is destroyed.
80 80
81 81 TODO: mikhail: please check if we need this wrapper after updating dulwich
82 82 to 0.12.0 +
83 83 """
84 84 def __del__(self):
85 85 if hasattr(self, 'object_store'):
86 86 self.close()
87 87
88 88
89 89 class GitFactory(RepoFactory):
90 90 repo_type = 'git'
91 91
92 92 def _create_repo(self, wire, create):
93 93 repo_path = str_to_dulwich(wire['path'])
94 94 return Repo(repo_path)
95 95
96 96
97 97 class GitRemote(object):
98 98
99 99 def __init__(self, factory):
100 100 self._factory = factory
101
101 self.peeled_ref_marker = '^{}'
102 102 self._bulk_methods = {
103 103 "author": self.commit_attribute,
104 104 "date": self.get_object_attrs,
105 105 "message": self.commit_attribute,
106 106 "parents": self.commit_attribute,
107 107 "_commit": self.revision,
108 108 }
109 109
110 110 def _wire_to_config(self, wire):
111 111 if 'config' in wire:
112 112 return dict([(x[0] + '_' + x[1], x[2]) for x in wire['config']])
113 113 return {}
114 114
115 115 def _assign_ref(self, wire, ref, commit_id):
116 116 repo = self._factory.repo(wire)
117 117 repo[ref] = commit_id
118 118
119 119 @reraise_safe_exceptions
120 120 def add_object(self, wire, content):
121 121 repo = self._factory.repo(wire)
122 122 blob = objects.Blob()
123 123 blob.set_raw_string(content)
124 124 repo.object_store.add_object(blob)
125 125 return blob.id
126 126
127 127 @reraise_safe_exceptions
128 128 def assert_correct_path(self, wire):
129 129 path = wire.get('path')
130 130 try:
131 131 self._factory.repo(wire)
132 132 except NotGitRepository as e:
133 133 tb = traceback.format_exc()
134 134 log.debug("Invalid Git path `%s`, tb: %s", path, tb)
135 135 return False
136 136
137 137 return True
138 138
139 139 @reraise_safe_exceptions
140 140 def bare(self, wire):
141 141 repo = self._factory.repo(wire)
142 142 return repo.bare
143 143
144 144 @reraise_safe_exceptions
145 145 def blob_as_pretty_string(self, wire, sha):
146 146 repo = self._factory.repo(wire)
147 147 return repo[sha].as_pretty_string()
148 148
149 149 @reraise_safe_exceptions
150 150 def blob_raw_length(self, wire, sha):
151 151 repo = self._factory.repo(wire)
152 152 blob = repo[sha]
153 153 return blob.raw_length()
154 154
155 155 def _parse_lfs_pointer(self, raw_content):
156 156
157 157 spec_string = 'version https://git-lfs.github.com/spec'
158 158 if raw_content and raw_content.startswith(spec_string):
159 159 pattern = re.compile(r"""
160 160 (?:\n)?
161 161 ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
162 162 ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
163 163 ^size[ ](?P<oid_size>[0-9]+)\n
164 164 (?:\n)?
165 165 """, re.VERBOSE | re.MULTILINE)
166 166 match = pattern.match(raw_content)
167 167 if match:
168 168 return match.groupdict()
169 169
170 170 return {}
171 171
172 172 @reraise_safe_exceptions
173 173 def is_large_file(self, wire, sha):
174 174 repo = self._factory.repo(wire)
175 175 blob = repo[sha]
176 176 return self._parse_lfs_pointer(blob.as_raw_string())
177 177
178 178 @reraise_safe_exceptions
179 179 def in_largefiles_store(self, wire, oid):
180 180 repo = self._factory.repo(wire)
181 181 conf = self._wire_to_config(wire)
182 182
183 183 store_location = conf.get('vcs_git_lfs_store_location')
184 184 if store_location:
185 185 repo_name = repo.path
186 186 store = LFSOidStore(
187 187 oid=oid, repo=repo_name, store_location=store_location)
188 188 return store.has_oid()
189 189
190 190 return False
191 191
192 192 @reraise_safe_exceptions
193 193 def store_path(self, wire, oid):
194 194 repo = self._factory.repo(wire)
195 195 conf = self._wire_to_config(wire)
196 196
197 197 store_location = conf.get('vcs_git_lfs_store_location')
198 198 if store_location:
199 199 repo_name = repo.path
200 200 store = LFSOidStore(
201 201 oid=oid, repo=repo_name, store_location=store_location)
202 202 return store.oid_path
203 203 raise ValueError('Unable to fetch oid with path {}'.format(oid))
204 204
205 205 @reraise_safe_exceptions
206 206 def bulk_request(self, wire, rev, pre_load):
207 207 result = {}
208 208 for attr in pre_load:
209 209 try:
210 210 method = self._bulk_methods[attr]
211 211 args = [wire, rev]
212 212 if attr == "date":
213 213 args.extend(["commit_time", "commit_timezone"])
214 214 elif attr in ["author", "message", "parents"]:
215 215 args.append(attr)
216 216 result[attr] = method(*args)
217 217 except KeyError as e:
218 218 raise exceptions.VcsException(e)(
219 219 "Unknown bulk attribute: %s" % attr)
220 220 return result
221 221
222 222 def _build_opener(self, url):
223 223 handlers = []
224 224 url_obj = url_parser(url)
225 225 _, authinfo = url_obj.authinfo()
226 226
227 227 if authinfo:
228 228 # create a password manager
229 229 passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
230 230 passmgr.add_password(*authinfo)
231 231
232 232 handlers.extend((httpbasicauthhandler(passmgr),
233 233 httpdigestauthhandler(passmgr)))
234 234
235 235 return urllib2.build_opener(*handlers)
236 236
237 237 @reraise_safe_exceptions
238 238 def check_url(self, url, config):
239 239 url_obj = url_parser(url)
240 240 test_uri, _ = url_obj.authinfo()
241 241 url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
242 242 url_obj.query = obfuscate_qs(url_obj.query)
243 243 cleaned_uri = str(url_obj)
244 244 log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
245 245
246 246 if not test_uri.endswith('info/refs'):
247 247 test_uri = test_uri.rstrip('/') + '/info/refs'
248 248
249 249 o = self._build_opener(url)
250 250 o.addheaders = [('User-Agent', 'git/1.7.8.0')] # fake some git
251 251
252 252 q = {"service": 'git-upload-pack'}
253 253 qs = '?%s' % urllib.urlencode(q)
254 254 cu = "%s%s" % (test_uri, qs)
255 255 req = urllib2.Request(cu, None, {})
256 256
257 257 try:
258 258 log.debug("Trying to open URL %s", cleaned_uri)
259 259 resp = o.open(req)
260 260 if resp.code != 200:
261 261 raise exceptions.URLError()('Return Code is not 200')
262 262 except Exception as e:
263 263 log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
264 264 # means it cannot be cloned
265 265 raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e))
266 266
267 267 # now detect if it's proper git repo
268 268 gitdata = resp.read()
269 269 if 'service=git-upload-pack' in gitdata:
270 270 pass
271 271 elif re.findall(r'[0-9a-fA-F]{40}\s+refs', gitdata):
272 272 # old style git can return some other format !
273 273 pass
274 274 else:
275 275 raise exceptions.URLError()(
276 276 "url [%s] does not look like an git" % (cleaned_uri,))
277 277
278 278 return True
279 279
280 280 @reraise_safe_exceptions
281 281 def clone(self, wire, url, deferred, valid_refs, update_after_clone):
282 remote_refs = self.fetch(wire, url, apply_refs=False)
282 # TODO(marcink): deprecate this method. Last i checked we don't use it anymore
283 remote_refs = self.pull(wire, url, apply_refs=False)
283 284 repo = self._factory.repo(wire)
284 285 if isinstance(valid_refs, list):
285 286 valid_refs = tuple(valid_refs)
286 287
287 288 for k in remote_refs:
288 289 # only parse heads/tags and skip so called deferred tags
289 290 if k.startswith(valid_refs) and not k.endswith(deferred):
290 291 repo[k] = remote_refs[k]
291 292
292 293 if update_after_clone:
293 294 # we want to checkout HEAD
294 295 repo["HEAD"] = remote_refs["HEAD"]
295 296 index.build_index_from_tree(repo.path, repo.index_path(),
296 297 repo.object_store, repo["HEAD"].tree)
297 298
298 299 # TODO: this is quite complex, check if that can be simplified
299 300 @reraise_safe_exceptions
300 301 def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
301 302 repo = self._factory.repo(wire)
302 303 object_store = repo.object_store
303 304
304 305 # Create tree and populates it with blobs
305 306 commit_tree = commit_tree and repo[commit_tree] or objects.Tree()
306 307
307 308 for node in updated:
308 309 # Compute subdirs if needed
309 310 dirpath, nodename = vcspath.split(node['path'])
310 311 dirnames = map(safe_str, dirpath and dirpath.split('/') or [])
311 312 parent = commit_tree
312 313 ancestors = [('', parent)]
313 314
314 315 # Tries to dig for the deepest existing tree
315 316 while dirnames:
316 317 curdir = dirnames.pop(0)
317 318 try:
318 319 dir_id = parent[curdir][1]
319 320 except KeyError:
320 321 # put curdir back into dirnames and stops
321 322 dirnames.insert(0, curdir)
322 323 break
323 324 else:
324 325 # If found, updates parent
325 326 parent = repo[dir_id]
326 327 ancestors.append((curdir, parent))
327 328 # Now parent is deepest existing tree and we need to create
328 329 # subtrees for dirnames (in reverse order)
329 330 # [this only applies for nodes from added]
330 331 new_trees = []
331 332
332 333 blob = objects.Blob.from_string(node['content'])
333 334
334 335 if dirnames:
335 336 # If there are trees which should be created we need to build
336 337 # them now (in reverse order)
337 338 reversed_dirnames = list(reversed(dirnames))
338 339 curtree = objects.Tree()
339 340 curtree[node['node_path']] = node['mode'], blob.id
340 341 new_trees.append(curtree)
341 342 for dirname in reversed_dirnames[:-1]:
342 343 newtree = objects.Tree()
343 344 newtree[dirname] = (DIR_STAT, curtree.id)
344 345 new_trees.append(newtree)
345 346 curtree = newtree
346 347 parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
347 348 else:
348 349 parent.add(
349 350 name=node['node_path'], mode=node['mode'], hexsha=blob.id)
350 351
351 352 new_trees.append(parent)
352 353 # Update ancestors
353 354 reversed_ancestors = reversed(
354 355 [(a[1], b[1], b[0]) for a, b in zip(ancestors, ancestors[1:])])
355 356 for parent, tree, path in reversed_ancestors:
356 357 parent[path] = (DIR_STAT, tree.id)
357 358 object_store.add_object(tree)
358 359
359 360 object_store.add_object(blob)
360 361 for tree in new_trees:
361 362 object_store.add_object(tree)
362 363
363 364 for node_path in removed:
364 365 paths = node_path.split('/')
365 366 tree = commit_tree
366 367 trees = [tree]
367 368 # Traverse deep into the forest...
368 369 for path in paths:
369 370 try:
370 371 obj = repo[tree[path][1]]
371 372 if isinstance(obj, objects.Tree):
372 373 trees.append(obj)
373 374 tree = obj
374 375 except KeyError:
375 376 break
376 377 # Cut down the blob and all rotten trees on the way back...
377 378 for path, tree in reversed(zip(paths, trees)):
378 379 del tree[path]
379 380 if tree:
380 381 # This tree still has elements - don't remove it or any
381 382 # of it's parents
382 383 break
383 384
384 385 object_store.add_object(commit_tree)
385 386
386 387 # Create commit
387 388 commit = objects.Commit()
388 389 commit.tree = commit_tree.id
389 390 for k, v in commit_data.iteritems():
390 391 setattr(commit, k, v)
391 392 object_store.add_object(commit)
392 393
393 394 ref = 'refs/heads/%s' % branch
394 395 repo.refs[ref] = commit.id
395 396
396 397 return commit.id
397 398
398 399 @reraise_safe_exceptions
399 def fetch(self, wire, url, apply_refs=True, refs=None):
400 def pull(self, wire, url, apply_refs=True, refs=None):
400 401 if url != 'default' and '://' not in url:
401 402 client = LocalGitClient(url)
402 403 else:
403 404 url_obj = url_parser(url)
404 405 o = self._build_opener(url)
405 406 url, _ = url_obj.authinfo()
406 407 client = HttpGitClient(base_url=url, opener=o)
407 408 repo = self._factory.repo(wire)
408 409
409 410 determine_wants = repo.object_store.determine_wants_all
410 411 if refs:
411 412 def determine_wants_requested(references):
412 413 return [references[r] for r in references if r in refs]
413 414 determine_wants = determine_wants_requested
414 415
415 416 try:
416 417 remote_refs = client.fetch(
417 418 path=url, target=repo, determine_wants=determine_wants)
418 419 except NotGitRepository as e:
419 420 log.warning(
420 421 'Trying to fetch from "%s" failed, not a Git repository.', url)
421 422 # Exception can contain unicode which we convert
422 423 raise exceptions.AbortException(e)(repr(e))
423 424
424 425 # mikhail: client.fetch() returns all the remote refs, but fetches only
425 426 # refs filtered by `determine_wants` function. We need to filter result
426 427 # as well
427 428 if refs:
428 429 remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
429 430
430 431 if apply_refs:
431 432 # TODO: johbo: Needs proper test coverage with a git repository
432 433 # that contains a tag object, so that we would end up with
433 434 # a peeled ref at this point.
434 PEELED_REF_MARKER = '^{}'
435 435 for k in remote_refs:
436 if k.endswith(PEELED_REF_MARKER):
437 log.info("Skipping peeled reference %s", k)
436 if k.endswith(self.peeled_ref_marker):
437 log.debug("Skipping peeled reference %s", k)
438 438 continue
439 439 repo[k] = remote_refs[k]
440 440
441 441 if refs:
442 442 # mikhail: explicitly set the head to the last ref.
443 443 repo['HEAD'] = remote_refs[refs[-1]]
444 444
445 # TODO: mikhail: should we return remote_refs here to be
446 # consistent?
447 445 else:
448 446 return remote_refs
449 447
450 448 @reraise_safe_exceptions
451 449 def sync_fetch(self, wire, url, refs=None):
452 450 repo = self._factory.repo(wire)
453 451 if refs and not isinstance(refs, (list, tuple)):
454 452 refs = [refs]
455 453
456 # get remote heads
454 # get all remote refs we'll use to fetch later
457 455 output, __ = self.run_git_command(
458 456 wire, ['ls-remote', url], fail_on_stderr=False,
459 457 _copts=['-c', 'core.askpass=""'],
460 458 extra_env={'GIT_TERMINAL_PROMPT': '0'})
461 459
462 460 remote_refs = collections.OrderedDict()
463 461 fetch_refs = []
462
464 463 for ref_line in output.splitlines():
465 464 sha, ref = ref_line.split('\t')
466 465 sha = sha.strip()
466 if ref in remote_refs:
467 # duplicate, skip
468 continue
469 if ref.endswith(self.peeled_ref_marker):
470 log.debug("Skipping peeled reference %s", ref)
471 continue
467 472 remote_refs[ref] = sha
468 473
469 474 if refs and sha in refs:
470 475 # we filter fetch using our specified refs
471 476 fetch_refs.append('{}:{}'.format(ref, ref))
472 477 elif not refs:
473 478 fetch_refs.append('{}:{}'.format(ref, ref))
474 479
475 fetch_refs.append('{}:{}'.format(ref, ref))
476
477 _out, _err = self.run_git_command(
478 wire, ['fetch', url, '--'] + fetch_refs, fail_on_stderr=False,
479 _copts=['-c', 'core.askpass=""'],
480 extra_env={'GIT_TERMINAL_PROMPT': '0'})
480 if fetch_refs:
481 _out, _err = self.run_git_command(
482 wire, ['fetch', url, '--prune', '--'] + fetch_refs,
483 fail_on_stderr=False,
484 _copts=['-c', 'core.askpass=""'],
485 extra_env={'GIT_TERMINAL_PROMPT': '0'})
481 486
482 487 return remote_refs
483 488
484 489 @reraise_safe_exceptions
485 490 def sync_push(self, wire, url, refs=None):
486 491 if not self.check_url(url, wire):
487 492 return
488 493
489 494 repo = self._factory.repo(wire)
490 495 self.run_git_command(
491 496 wire, ['push', url, '--mirror'], fail_on_stderr=False,
492 497 _copts=['-c', 'core.askpass=""'],
493 498 extra_env={'GIT_TERMINAL_PROMPT': '0'})
494 499
495 500 @reraise_safe_exceptions
496 501 def get_remote_refs(self, wire, url):
497 502 repo = Repo(url)
498 503 return repo.get_refs()
499 504
500 505 @reraise_safe_exceptions
501 506 def get_description(self, wire):
502 507 repo = self._factory.repo(wire)
503 508 return repo.get_description()
504 509
505 510 @reraise_safe_exceptions
506 511 def get_file_history(self, wire, file_path, commit_id, limit):
507 512 repo = self._factory.repo(wire)
508 513 include = [commit_id]
509 514 paths = [file_path]
510 515
511 516 walker = repo.get_walker(include, paths=paths, max_entries=limit)
512 517 return [x.commit.id for x in walker]
513 518
514 519 @reraise_safe_exceptions
515 520 def get_missing_revs(self, wire, rev1, rev2, path2):
516 521 repo = self._factory.repo(wire)
517 522 LocalGitClient(thin_packs=False).fetch(path2, repo)
518 523
519 524 wire_remote = wire.copy()
520 525 wire_remote['path'] = path2
521 526 repo_remote = self._factory.repo(wire_remote)
522 527 LocalGitClient(thin_packs=False).fetch(wire["path"], repo_remote)
523 528
524 529 revs = [
525 530 x.commit.id
526 531 for x in repo_remote.get_walker(include=[rev2], exclude=[rev1])]
527 532 return revs
528 533
529 534 @reraise_safe_exceptions
530 535 def get_object(self, wire, sha):
531 536 repo = self._factory.repo(wire)
532 537 obj = repo.get_object(sha)
533 538 commit_id = obj.id
534 539
535 540 if isinstance(obj, Tag):
536 541 commit_id = obj.object[1]
537 542
538 543 return {
539 544 'id': obj.id,
540 545 'type': obj.type_name,
541 546 'commit_id': commit_id
542 547 }
543 548
544 549 @reraise_safe_exceptions
545 550 def get_object_attrs(self, wire, sha, *attrs):
546 551 repo = self._factory.repo(wire)
547 552 obj = repo.get_object(sha)
548 553 return list(getattr(obj, a) for a in attrs)
549 554
550 555 @reraise_safe_exceptions
551 556 def get_refs(self, wire):
552 557 repo = self._factory.repo(wire)
553 558 result = {}
554 559 for ref, sha in repo.refs.as_dict().items():
555 560 peeled_sha = repo.get_peeled(ref)
556 561 result[ref] = peeled_sha
557 562 return result
558 563
559 564 @reraise_safe_exceptions
560 565 def get_refs_path(self, wire):
561 566 repo = self._factory.repo(wire)
562 567 return repo.refs.path
563 568
564 569 @reraise_safe_exceptions
565 570 def head(self, wire, show_exc=True):
566 571 repo = self._factory.repo(wire)
567 572 try:
568 573 return repo.head()
569 574 except Exception:
570 575 if show_exc:
571 576 raise
572 577
573 578 @reraise_safe_exceptions
574 579 def init(self, wire):
575 580 repo_path = str_to_dulwich(wire['path'])
576 581 self.repo = Repo.init(repo_path)
577 582
578 583 @reraise_safe_exceptions
579 584 def init_bare(self, wire):
580 585 repo_path = str_to_dulwich(wire['path'])
581 586 self.repo = Repo.init_bare(repo_path)
582 587
583 588 @reraise_safe_exceptions
584 589 def revision(self, wire, rev):
585 590 repo = self._factory.repo(wire)
586 591 obj = repo[rev]
587 592 obj_data = {
588 593 'id': obj.id,
589 594 }
590 595 try:
591 596 obj_data['tree'] = obj.tree
592 597 except AttributeError:
593 598 pass
594 599 return obj_data
595 600
596 601 @reraise_safe_exceptions
597 602 def commit_attribute(self, wire, rev, attr):
598 603 repo = self._factory.repo(wire)
599 604 obj = repo[rev]
600 605 return getattr(obj, attr)
601 606
602 607 @reraise_safe_exceptions
603 608 def set_refs(self, wire, key, value):
604 609 repo = self._factory.repo(wire)
605 610 repo.refs[key] = value
606 611
607 612 @reraise_safe_exceptions
608 613 def remove_ref(self, wire, key):
609 614 repo = self._factory.repo(wire)
610 615 del repo.refs[key]
611 616
612 617 @reraise_safe_exceptions
613 618 def tree_changes(self, wire, source_id, target_id):
614 619 repo = self._factory.repo(wire)
615 620 source = repo[source_id].tree if source_id else None
616 621 target = repo[target_id].tree
617 622 result = repo.object_store.tree_changes(source, target)
618 623 return list(result)
619 624
620 625 @reraise_safe_exceptions
621 626 def tree_items(self, wire, tree_id):
622 627 repo = self._factory.repo(wire)
623 628 tree = repo[tree_id]
624 629
625 630 result = []
626 631 for item in tree.iteritems():
627 632 item_sha = item.sha
628 633 item_mode = item.mode
629 634
630 635 if FILE_MODE(item_mode) == GIT_LINK:
631 636 item_type = "link"
632 637 else:
633 638 item_type = repo[item_sha].type_name
634 639
635 640 result.append((item.path, item_mode, item_sha, item_type))
636 641 return result
637 642
638 643 @reraise_safe_exceptions
639 644 def update_server_info(self, wire):
640 645 repo = self._factory.repo(wire)
641 646 update_server_info(repo)
642 647
643 648 @reraise_safe_exceptions
644 649 def discover_git_version(self):
645 650 stdout, _ = self.run_git_command(
646 651 {}, ['--version'], _bare=True, _safe=True)
647 652 prefix = 'git version'
648 653 if stdout.startswith(prefix):
649 654 stdout = stdout[len(prefix):]
650 655 return stdout.strip()
651 656
652 657 @reraise_safe_exceptions
653 658 def run_git_command(self, wire, cmd, **opts):
654 659 path = wire.get('path', None)
655 660
656 661 if path and os.path.isdir(path):
657 662 opts['cwd'] = path
658 663
659 664 if '_bare' in opts:
660 665 _copts = []
661 666 del opts['_bare']
662 667 else:
663 668 _copts = ['-c', 'core.quotepath=false', ]
664 669 safe_call = False
665 670 if '_safe' in opts:
666 671 # no exc on failure
667 672 del opts['_safe']
668 673 safe_call = True
669 674
670 675 if '_copts' in opts:
671 676 _copts.extend(opts['_copts'] or [])
672 677 del opts['_copts']
673 678
674 679 gitenv = os.environ.copy()
675 680 gitenv.update(opts.pop('extra_env', {}))
676 681 # need to clean fix GIT_DIR !
677 682 if 'GIT_DIR' in gitenv:
678 683 del gitenv['GIT_DIR']
679 684 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
680 685 gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1'
681 686
682 687 cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
683 688
684 689 try:
685 690 _opts = {'env': gitenv, 'shell': False}
686 691 _opts.update(opts)
687 692 p = subprocessio.SubprocessIOChunker(cmd, **_opts)
688 693
689 694 return ''.join(p), ''.join(p.error)
690 695 except (EnvironmentError, OSError) as err:
691 696 cmd = ' '.join(cmd) # human friendly CMD
692 697 tb_err = ("Couldn't run git command (%s).\n"
693 698 "Original error was:%s\n" % (cmd, err))
694 699 log.exception(tb_err)
695 700 if safe_call:
696 701 return '', err
697 702 else:
698 703 raise exceptions.VcsException()(tb_err)
699 704
700 705 @reraise_safe_exceptions
701 706 def install_hooks(self, wire, force=False):
702 707 from vcsserver.hook_utils import install_git_hooks
703 708 repo = self._factory.repo(wire)
704 709 return install_git_hooks(repo.path, repo.bare, force_create=force)
705 710
706 711
707 712 def str_to_dulwich(value):
708 713 """
709 714 Dulwich 0.10.1a requires `unicode` objects to be passed in.
710 715 """
711 716 return value.decode(settings.WIRE_ENCODING)
@@ -1,165 +1,165 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2018 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import inspect
19 19
20 20 import pytest
21 21 import dulwich.errors
22 22 from mock import Mock, patch
23 23
24 24 from vcsserver import git
25 25
26 26
27 27 SAMPLE_REFS = {
28 28 'HEAD': 'fd627b9e0dd80b47be81af07c4a98518244ed2f7',
29 29 'refs/tags/v0.1.9': '341d28f0eec5ddf0b6b77871e13c2bbd6bec685c',
30 30 'refs/tags/v0.1.8': '74ebce002c088b8a5ecf40073db09375515ecd68',
31 31 'refs/tags/v0.1.1': 'e6ea6d16e2f26250124a1f4b4fe37a912f9d86a0',
32 32 'refs/tags/v0.1.3': '5a3a8fb005554692b16e21dee62bf02667d8dc3e',
33 33 }
34 34
35 35
36 36 @pytest.fixture
37 37 def git_remote():
38 38 """
39 39 A GitRemote instance with a mock factory.
40 40 """
41 41 factory = Mock()
42 42 remote = git.GitRemote(factory)
43 43 return remote
44 44
45 45
46 46 def test_discover_git_version(git_remote):
47 47 version = git_remote.discover_git_version()
48 48 assert version
49 49
50 50
51 51 class TestGitFetch(object):
52 52 def setup(self):
53 53 self.mock_repo = Mock()
54 54 factory = Mock()
55 55 factory.repo = Mock(return_value=self.mock_repo)
56 56 self.remote_git = git.GitRemote(factory)
57 57
58 58 def test_fetches_all_when_no_commit_ids_specified(self):
59 59 def side_effect(determine_wants, *args, **kwargs):
60 60 determine_wants(SAMPLE_REFS)
61 61
62 62 with patch('dulwich.client.LocalGitClient.fetch') as mock_fetch:
63 63 mock_fetch.side_effect = side_effect
64 self.remote_git.fetch(wire=None, url='/tmp/', apply_refs=False)
64 self.remote_git.pull(wire=None, url='/tmp/', apply_refs=False)
65 65 determine_wants = self.mock_repo.object_store.determine_wants_all
66 66 determine_wants.assert_called_once_with(SAMPLE_REFS)
67 67
68 68 def test_fetches_specified_commits(self):
69 69 selected_refs = {
70 70 'refs/tags/v0.1.8': '74ebce002c088b8a5ecf40073db09375515ecd68',
71 71 'refs/tags/v0.1.3': '5a3a8fb005554692b16e21dee62bf02667d8dc3e',
72 72 }
73 73
74 74 def side_effect(determine_wants, *args, **kwargs):
75 75 result = determine_wants(SAMPLE_REFS)
76 76 assert sorted(result) == sorted(selected_refs.values())
77 77 return result
78 78
79 79 with patch('dulwich.client.LocalGitClient.fetch') as mock_fetch:
80 80 mock_fetch.side_effect = side_effect
81 self.remote_git.fetch(
81 self.remote_git.pull(
82 82 wire=None, url='/tmp/', apply_refs=False,
83 83 refs=selected_refs.keys())
84 84 determine_wants = self.mock_repo.object_store.determine_wants_all
85 85 assert determine_wants.call_count == 0
86 86
87 87 def test_get_remote_refs(self):
88 88 factory = Mock()
89 89 remote_git = git.GitRemote(factory)
90 90 url = 'http://example.com/test/test.git'
91 91 sample_refs = {
92 92 'refs/tags/v0.1.8': '74ebce002c088b8a5ecf40073db09375515ecd68',
93 93 'refs/tags/v0.1.3': '5a3a8fb005554692b16e21dee62bf02667d8dc3e',
94 94 }
95 95
96 96 with patch('vcsserver.git.Repo', create=False) as mock_repo:
97 97 mock_repo().get_refs.return_value = sample_refs
98 98 remote_refs = remote_git.get_remote_refs(wire=None, url=url)
99 99 mock_repo().get_refs.assert_called_once_with()
100 100 assert remote_refs == sample_refs
101 101
102 102 def test_remove_ref(self):
103 103 ref_to_remove = 'refs/tags/v0.1.9'
104 104 self.mock_repo.refs = SAMPLE_REFS.copy()
105 105 self.remote_git.remove_ref(None, ref_to_remove)
106 106 assert ref_to_remove not in self.mock_repo.refs
107 107
108 108
109 109 class TestReraiseSafeExceptions(object):
110 110 def test_method_decorated_with_reraise_safe_exceptions(self):
111 111 factory = Mock()
112 112 git_remote = git.GitRemote(factory)
113 113
114 114 def fake_function():
115 115 return None
116 116
117 117 decorator = git.reraise_safe_exceptions(fake_function)
118 118
119 119 methods = inspect.getmembers(git_remote, predicate=inspect.ismethod)
120 120 for method_name, method in methods:
121 121 if not method_name.startswith('_'):
122 122 assert method.im_func.__code__ == decorator.__code__
123 123
124 124 @pytest.mark.parametrize('side_effect, expected_type', [
125 125 (dulwich.errors.ChecksumMismatch('0000000', 'deadbeef'), 'lookup'),
126 126 (dulwich.errors.NotCommitError('deadbeef'), 'lookup'),
127 127 (dulwich.errors.MissingCommitError('deadbeef'), 'lookup'),
128 128 (dulwich.errors.ObjectMissing('deadbeef'), 'lookup'),
129 129 (dulwich.errors.HangupException(), 'error'),
130 130 (dulwich.errors.UnexpectedCommandError('test-cmd'), 'error'),
131 131 ])
132 132 def test_safe_exceptions_reraised(self, side_effect, expected_type):
133 133 @git.reraise_safe_exceptions
134 134 def fake_method():
135 135 raise side_effect
136 136
137 137 with pytest.raises(Exception) as exc_info:
138 138 fake_method()
139 139 assert type(exc_info.value) == Exception
140 140 assert exc_info.value._vcs_kind == expected_type
141 141
142 142
143 143 class TestDulwichRepoWrapper(object):
144 144 def test_calls_close_on_delete(self):
145 145 isdir_patcher = patch('dulwich.repo.os.path.isdir', return_value=True)
146 146 with isdir_patcher:
147 147 repo = git.Repo('/tmp/abcde')
148 148 with patch.object(git.DulwichRepo, 'close') as close_mock:
149 149 del repo
150 150 close_mock.assert_called_once_with()
151 151
152 152
153 153 class TestGitFactory(object):
154 154 def test_create_repo_returns_dulwich_wrapper(self):
155 155
156 156 with patch('vcsserver.lib.rc_cache.region_meta.dogpile_cache_regions') as mock:
157 157 mock.side_effect = {'repo_objects': ''}
158 158 factory = git.GitFactory()
159 159 wire = {
160 160 'path': '/tmp/abcde'
161 161 }
162 162 isdir_patcher = patch('dulwich.repo.os.path.isdir', return_value=True)
163 163 with isdir_patcher:
164 164 result = factory._create_repo(wire, True)
165 165 assert isinstance(result, git.Repo)
General Comments 0
You need to be logged in to leave comments. Login now