##// END OF EJS Templates
libgit2: updated code for tests....
marcink -
r730:ef69988e default
parent child Browse files
Show More
@@ -1,868 +1,873 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2019 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17 import collections
18 18 import logging
19 19 import os
20 20 import posixpath as vcspath
21 21 import re
22 22 import stat
23 23 import traceback
24 24 import urllib
25 25 import urllib2
26 26 from functools import wraps
27 27
28 28 import more_itertools
29 29 import pygit2
30 30 from pygit2 import Repository as LibGit2Repo
31 31 from dulwich import index, objects
32 32 from dulwich.client import HttpGitClient, LocalGitClient
33 33 from dulwich.errors import (
34 34 NotGitRepository, ChecksumMismatch, WrongObjectException,
35 35 MissingCommitError, ObjectMissing, HangupException,
36 36 UnexpectedCommandError)
37 37 from dulwich.repo import Repo as DulwichRepo
38 38 from dulwich.server import update_server_info
39 39
40 40 from vcsserver import exceptions, settings, subprocessio
41 41 from vcsserver.utils import safe_str
42 42 from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original
43 43 from vcsserver.hgcompat import (
44 44 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
45 45 from vcsserver.git_lfs.lib import LFSOidStore
46 46
47 47 DIR_STAT = stat.S_IFDIR
48 48 FILE_MODE = stat.S_IFMT
49 49 GIT_LINK = objects.S_IFGITLINK
50 50
51 51 log = logging.getLogger(__name__)
52 52
53 53
54 54 def reraise_safe_exceptions(func):
55 55 """Converts Dulwich exceptions to something neutral."""
56 56
57 57 @wraps(func)
58 58 def wrapper(*args, **kwargs):
59 59 try:
60 60 return func(*args, **kwargs)
61 61 except (ChecksumMismatch, WrongObjectException, MissingCommitError, ObjectMissing,) as e:
62 62 exc = exceptions.LookupException(org_exc=e)
63 63 raise exc(safe_str(e))
64 64 except (HangupException, UnexpectedCommandError) as e:
65 65 exc = exceptions.VcsException(org_exc=e)
66 66 raise exc(safe_str(e))
67 67 except Exception as e:
68 68 # NOTE(marcink): becuase of how dulwich handles some exceptions
69 69 # (KeyError on empty repos), we cannot track this and catch all
70 70 # exceptions, it's an exceptions from other handlers
71 71 #if not hasattr(e, '_vcs_kind'):
72 72 #log.exception("Unhandled exception in git remote call")
73 73 #raise_from_original(exceptions.UnhandledException)
74 74 raise
75 75 return wrapper
76 76
77 77
78 78 class Repo(DulwichRepo):
79 79 """
80 80 A wrapper for dulwich Repo class.
81 81
82 82 Since dulwich is sometimes keeping .idx file descriptors open, it leads to
83 83 "Too many open files" error. We need to close all opened file descriptors
84 84 once the repo object is destroyed.
85 85 """
86 86 def __del__(self):
87 87 if hasattr(self, 'object_store'):
88 88 self.close()
89 89
90 90
91 91 class Repository(LibGit2Repo):
92 92
93 93 def __enter__(self):
94 94 return self
95 95
96 96 def __exit__(self, exc_type, exc_val, exc_tb):
97 97 self.free()
98 98
99 99
100 100 class GitFactory(RepoFactory):
101 101 repo_type = 'git'
102 102
103 103 def _create_repo(self, wire, create, use_libgit2=False):
104 104 if use_libgit2:
105 105 return Repository(wire['path'])
106 106 else:
107 107 repo_path = str_to_dulwich(wire['path'])
108 108 return Repo(repo_path)
109 109
110 110 def repo(self, wire, create=False, use_libgit2=False):
111 111 """
112 112 Get a repository instance for the given path.
113 113 """
114 114 region = self._cache_region
115 115 context = wire.get('context', None)
116 116 repo_path = wire.get('path', '')
117 117 context_uid = '{}'.format(context)
118 118 cache = wire.get('cache', True)
119 119 cache_on = context and cache
120 120
121 121 @region.conditional_cache_on_arguments(condition=cache_on)
122 122 def create_new_repo(_repo_type, _repo_path, _context_uid, _use_libgit2):
123 123 return self._create_repo(wire, create, use_libgit2)
124 124
125 125 repo = create_new_repo(self.repo_type, repo_path, context_uid, use_libgit2)
126 126 return repo
127 127
128 128 def repo_libgit2(self, wire):
129 129 return self.repo(wire, use_libgit2=True)
130 130
131 131
132 132 class GitRemote(object):
133 133
134 134 def __init__(self, factory):
135 135 self._factory = factory
136 136 self.peeled_ref_marker = '^{}'
137 137 self._bulk_methods = {
138 138 "date": self.date,
139 139 "author": self.author,
140 140 "message": self.message,
141 141 "parents": self.parents,
142 142 "_commit": self.revision,
143 143 }
144 144
145 145 def _wire_to_config(self, wire):
146 146 if 'config' in wire:
147 147 return dict([(x[0] + '_' + x[1], x[2]) for x in wire['config']])
148 148 return {}
149 149
150 150 def _remote_conf(self, config):
151 151 params = [
152 152 '-c', 'core.askpass=""',
153 153 ]
154 154 ssl_cert_dir = config.get('vcs_ssl_dir')
155 155 if ssl_cert_dir:
156 156 params.extend(['-c', 'http.sslCAinfo={}'.format(ssl_cert_dir)])
157 157 return params
158 158
159 159 @reraise_safe_exceptions
160 160 def is_empty(self, wire):
161 161 repo = self._factory.repo_libgit2(wire)
162 162
163 163 try:
164 return not repo.head.name
164 has_head = repo.head.name
165 if has_head:
166 return False
167
168 # NOTE(marcink): check again using more expensive method
169 return repo.is_empty
165 170 except Exception:
166 return True
171 pass
172
173 return True
167 174
168 175 @reraise_safe_exceptions
169 176 def add_object(self, wire, content):
170 177 repo = self._factory.repo(wire)
171 178 blob = objects.Blob()
172 179 blob.set_raw_string(content)
173 180 repo.object_store.add_object(blob)
174 181 return blob.id
175 182
176 183 @reraise_safe_exceptions
177 184 def assert_correct_path(self, wire):
178 185 try:
179 186 self._factory.repo_libgit2(wire)
180 187 except pygit2.GitError:
181 188 path = wire.get('path')
182 189 tb = traceback.format_exc()
183 190 log.debug("Invalid Git path `%s`, tb: %s", path, tb)
184 191 return False
185 192
186 193 return True
187 194
188 195 @reraise_safe_exceptions
189 196 def bare(self, wire):
190 197 repo = self._factory.repo_libgit2(wire)
191 198 return repo.is_bare
192 199
193 200 @reraise_safe_exceptions
194 201 def blob_as_pretty_string(self, wire, sha):
195 202 repo_init = self._factory.repo_libgit2(wire)
196 203 with repo_init as repo:
197 204 blob_obj = repo[sha]
198 205 blob = blob_obj.data
199 206 return blob
200 207
201 208 @reraise_safe_exceptions
202 209 def blob_raw_length(self, wire, sha):
203 210 repo_init = self._factory.repo_libgit2(wire)
204 211 with repo_init as repo:
205 212 blob = repo[sha]
206 213 return blob.size
207 214
208 215 def _parse_lfs_pointer(self, raw_content):
209 216
210 217 spec_string = 'version https://git-lfs.github.com/spec'
211 218 if raw_content and raw_content.startswith(spec_string):
212 219 pattern = re.compile(r"""
213 220 (?:\n)?
214 221 ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
215 222 ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
216 223 ^size[ ](?P<oid_size>[0-9]+)\n
217 224 (?:\n)?
218 225 """, re.VERBOSE | re.MULTILINE)
219 226 match = pattern.match(raw_content)
220 227 if match:
221 228 return match.groupdict()
222 229
223 230 return {}
224 231
225 232 @reraise_safe_exceptions
226 233 def is_large_file(self, wire, sha):
227 234 repo_init = self._factory.repo_libgit2(wire)
228 235
229 236 with repo_init as repo:
230 237 blob = repo[sha]
231 238 if blob.is_binary:
232 239 return {}
233 240
234 241 return self._parse_lfs_pointer(blob.data)
235 242
236 243 @reraise_safe_exceptions
237 244 def in_largefiles_store(self, wire, oid):
238 245 repo = self._factory.repo_libgit2(wire)
239 246 conf = self._wire_to_config(wire)
240 247
241 248 store_location = conf.get('vcs_git_lfs_store_location')
242 249 if store_location:
243 250 repo_name = repo.path
244 251 store = LFSOidStore(
245 252 oid=oid, repo=repo_name, store_location=store_location)
246 253 return store.has_oid()
247 254
248 255 return False
249 256
250 257 @reraise_safe_exceptions
251 258 def store_path(self, wire, oid):
252 259 repo = self._factory.repo_libgit2(wire)
253 260 conf = self._wire_to_config(wire)
254 261
255 262 store_location = conf.get('vcs_git_lfs_store_location')
256 263 if store_location:
257 264 repo_name = repo.path
258 265 store = LFSOidStore(
259 266 oid=oid, repo=repo_name, store_location=store_location)
260 267 return store.oid_path
261 268 raise ValueError('Unable to fetch oid with path {}'.format(oid))
262 269
263 270 @reraise_safe_exceptions
264 271 def bulk_request(self, wire, rev, pre_load):
265 272 result = {}
266 273 for attr in pre_load:
267 274 try:
268 275 method = self._bulk_methods[attr]
269 276 args = [wire, rev]
270 277 result[attr] = method(*args)
271 278 except KeyError as e:
272 279 raise exceptions.VcsException(e)("Unknown bulk attribute: %s" % attr)
273 280 return result
274 281
275 282 def _build_opener(self, url):
276 283 handlers = []
277 284 url_obj = url_parser(url)
278 285 _, authinfo = url_obj.authinfo()
279 286
280 287 if authinfo:
281 288 # create a password manager
282 289 passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
283 290 passmgr.add_password(*authinfo)
284 291
285 292 handlers.extend((httpbasicauthhandler(passmgr),
286 293 httpdigestauthhandler(passmgr)))
287 294
288 295 return urllib2.build_opener(*handlers)
289 296
290 297 def _type_id_to_name(self, type_id):
291 298 return {
292 299 1: b'commit',
293 300 2: b'tree',
294 301 3: b'blob',
295 302 4: b'tag'
296 303 }[type_id]
297 304
298 305 @reraise_safe_exceptions
299 306 def check_url(self, url, config):
300 307 url_obj = url_parser(url)
301 308 test_uri, _ = url_obj.authinfo()
302 309 url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
303 310 url_obj.query = obfuscate_qs(url_obj.query)
304 311 cleaned_uri = str(url_obj)
305 312 log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
306 313
307 314 if not test_uri.endswith('info/refs'):
308 315 test_uri = test_uri.rstrip('/') + '/info/refs'
309 316
310 317 o = self._build_opener(url)
311 318 o.addheaders = [('User-Agent', 'git/1.7.8.0')] # fake some git
312 319
313 320 q = {"service": 'git-upload-pack'}
314 321 qs = '?%s' % urllib.urlencode(q)
315 322 cu = "%s%s" % (test_uri, qs)
316 323 req = urllib2.Request(cu, None, {})
317 324
318 325 try:
319 326 log.debug("Trying to open URL %s", cleaned_uri)
320 327 resp = o.open(req)
321 328 if resp.code != 200:
322 329 raise exceptions.URLError()('Return Code is not 200')
323 330 except Exception as e:
324 331 log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
325 332 # means it cannot be cloned
326 333 raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e))
327 334
328 335 # now detect if it's proper git repo
329 336 gitdata = resp.read()
330 337 if 'service=git-upload-pack' in gitdata:
331 338 pass
332 339 elif re.findall(r'[0-9a-fA-F]{40}\s+refs', gitdata):
333 340 # old style git can return some other format !
334 341 pass
335 342 else:
336 343 raise exceptions.URLError()(
337 344 "url [%s] does not look like an git" % (cleaned_uri,))
338 345
339 346 return True
340 347
341 348 @reraise_safe_exceptions
342 349 def clone(self, wire, url, deferred, valid_refs, update_after_clone):
343 350 # TODO(marcink): deprecate this method. Last i checked we don't use it anymore
344 351 remote_refs = self.pull(wire, url, apply_refs=False)
345 352 repo = self._factory.repo(wire)
346 353 if isinstance(valid_refs, list):
347 354 valid_refs = tuple(valid_refs)
348 355
349 356 for k in remote_refs:
350 357 # only parse heads/tags and skip so called deferred tags
351 358 if k.startswith(valid_refs) and not k.endswith(deferred):
352 359 repo[k] = remote_refs[k]
353 360
354 361 if update_after_clone:
355 362 # we want to checkout HEAD
356 363 repo["HEAD"] = remote_refs["HEAD"]
357 364 index.build_index_from_tree(repo.path, repo.index_path(),
358 365 repo.object_store, repo["HEAD"].tree)
359 366
360 367 # TODO: this is quite complex, check if that can be simplified
361 368 @reraise_safe_exceptions
362 369 def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
363 370 repo = self._factory.repo(wire)
364 371 object_store = repo.object_store
365 372
366 373 # Create tree and populates it with blobs
367 374 commit_tree = commit_tree and repo[commit_tree] or objects.Tree()
368 375
369 376 for node in updated:
370 377 # Compute subdirs if needed
371 378 dirpath, nodename = vcspath.split(node['path'])
372 379 dirnames = map(safe_str, dirpath and dirpath.split('/') or [])
373 380 parent = commit_tree
374 381 ancestors = [('', parent)]
375 382
376 383 # Tries to dig for the deepest existing tree
377 384 while dirnames:
378 385 curdir = dirnames.pop(0)
379 386 try:
380 387 dir_id = parent[curdir][1]
381 388 except KeyError:
382 389 # put curdir back into dirnames and stops
383 390 dirnames.insert(0, curdir)
384 391 break
385 392 else:
386 393 # If found, updates parent
387 394 parent = repo[dir_id]
388 395 ancestors.append((curdir, parent))
389 396 # Now parent is deepest existing tree and we need to create
390 397 # subtrees for dirnames (in reverse order)
391 398 # [this only applies for nodes from added]
392 399 new_trees = []
393 400
394 401 blob = objects.Blob.from_string(node['content'])
395 402
396 403 if dirnames:
397 404 # If there are trees which should be created we need to build
398 405 # them now (in reverse order)
399 406 reversed_dirnames = list(reversed(dirnames))
400 407 curtree = objects.Tree()
401 408 curtree[node['node_path']] = node['mode'], blob.id
402 409 new_trees.append(curtree)
403 410 for dirname in reversed_dirnames[:-1]:
404 411 newtree = objects.Tree()
405 412 newtree[dirname] = (DIR_STAT, curtree.id)
406 413 new_trees.append(newtree)
407 414 curtree = newtree
408 415 parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
409 416 else:
410 417 parent.add(name=node['node_path'], mode=node['mode'], hexsha=blob.id)
411 418
412 419 new_trees.append(parent)
413 420 # Update ancestors
414 421 reversed_ancestors = reversed(
415 422 [(a[1], b[1], b[0]) for a, b in zip(ancestors, ancestors[1:])])
416 423 for parent, tree, path in reversed_ancestors:
417 424 parent[path] = (DIR_STAT, tree.id)
418 425 object_store.add_object(tree)
419 426
420 427 object_store.add_object(blob)
421 428 for tree in new_trees:
422 429 object_store.add_object(tree)
423 430
424 431 for node_path in removed:
425 432 paths = node_path.split('/')
426 433 tree = commit_tree
427 434 trees = [tree]
428 435 # Traverse deep into the forest...
429 436 for path in paths:
430 437 try:
431 438 obj = repo[tree[path][1]]
432 439 if isinstance(obj, objects.Tree):
433 440 trees.append(obj)
434 441 tree = obj
435 442 except KeyError:
436 443 break
437 444 # Cut down the blob and all rotten trees on the way back...
438 445 for path, tree in reversed(zip(paths, trees)):
439 446 del tree[path]
440 447 if tree:
441 448 # This tree still has elements - don't remove it or any
442 449 # of it's parents
443 450 break
444 451
445 452 object_store.add_object(commit_tree)
446 453
447 454 # Create commit
448 455 commit = objects.Commit()
449 456 commit.tree = commit_tree.id
450 457 for k, v in commit_data.iteritems():
451 458 setattr(commit, k, v)
452 459 object_store.add_object(commit)
453 460
454 461 self.create_branch(wire, branch, commit.id)
455 462
456 463 # dulwich set-ref
457 464 ref = 'refs/heads/%s' % branch
458 465 repo.refs[ref] = commit.id
459 466
460 467 return commit.id
461 468
462 469 @reraise_safe_exceptions
463 470 def pull(self, wire, url, apply_refs=True, refs=None, update_after=False):
464 471 if url != 'default' and '://' not in url:
465 472 client = LocalGitClient(url)
466 473 else:
467 474 url_obj = url_parser(url)
468 475 o = self._build_opener(url)
469 476 url, _ = url_obj.authinfo()
470 477 client = HttpGitClient(base_url=url, opener=o)
471 478 repo = self._factory.repo(wire)
472 479
473 480 determine_wants = repo.object_store.determine_wants_all
474 481 if refs:
475 482 def determine_wants_requested(references):
476 483 return [references[r] for r in references if r in refs]
477 484 determine_wants = determine_wants_requested
478 485
479 486 try:
480 487 remote_refs = client.fetch(
481 488 path=url, target=repo, determine_wants=determine_wants)
482 489 except NotGitRepository as e:
483 490 log.warning(
484 491 'Trying to fetch from "%s" failed, not a Git repository.', url)
485 492 # Exception can contain unicode which we convert
486 493 raise exceptions.AbortException(e)(repr(e))
487 494
488 495 # mikhail: client.fetch() returns all the remote refs, but fetches only
489 496 # refs filtered by `determine_wants` function. We need to filter result
490 497 # as well
491 498 if refs:
492 499 remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
493 500
494 501 if apply_refs:
495 502 # TODO: johbo: Needs proper test coverage with a git repository
496 503 # that contains a tag object, so that we would end up with
497 504 # a peeled ref at this point.
498 505 for k in remote_refs:
499 506 if k.endswith(self.peeled_ref_marker):
500 507 log.debug("Skipping peeled reference %s", k)
501 508 continue
502 509 repo[k] = remote_refs[k]
503 510
504 511 if refs and not update_after:
505 512 # mikhail: explicitly set the head to the last ref.
506 513 repo['HEAD'] = remote_refs[refs[-1]]
507 514
508 515 if update_after:
509 516 # we want to checkout HEAD
510 517 repo["HEAD"] = remote_refs["HEAD"]
511 518 index.build_index_from_tree(repo.path, repo.index_path(),
512 519 repo.object_store, repo["HEAD"].tree)
513 520 return remote_refs
514 521
515 522 @reraise_safe_exceptions
516 523 def sync_fetch(self, wire, url, refs=None):
517 524 repo = self._factory.repo(wire)
518 525 if refs and not isinstance(refs, (list, tuple)):
519 526 refs = [refs]
520 527 config = self._wire_to_config(wire)
521 528 # get all remote refs we'll use to fetch later
522 529 output, __ = self.run_git_command(
523 530 wire, ['ls-remote', url], fail_on_stderr=False,
524 531 _copts=self._remote_conf(config),
525 532 extra_env={'GIT_TERMINAL_PROMPT': '0'})
526 533
527 534 remote_refs = collections.OrderedDict()
528 535 fetch_refs = []
529 536
530 537 for ref_line in output.splitlines():
531 538 sha, ref = ref_line.split('\t')
532 539 sha = sha.strip()
533 540 if ref in remote_refs:
534 541 # duplicate, skip
535 542 continue
536 543 if ref.endswith(self.peeled_ref_marker):
537 544 log.debug("Skipping peeled reference %s", ref)
538 545 continue
539 546 # don't sync HEAD
540 547 if ref in ['HEAD']:
541 548 continue
542 549
543 550 remote_refs[ref] = sha
544 551
545 552 if refs and sha in refs:
546 553 # we filter fetch using our specified refs
547 554 fetch_refs.append('{}:{}'.format(ref, ref))
548 555 elif not refs:
549 556 fetch_refs.append('{}:{}'.format(ref, ref))
550 557 log.debug('Finished obtaining fetch refs, total: %s', len(fetch_refs))
551 558 if fetch_refs:
552 559 for chunk in more_itertools.chunked(fetch_refs, 1024 * 4):
553 560 fetch_refs_chunks = list(chunk)
554 561 log.debug('Fetching %s refs from import url', len(fetch_refs_chunks))
555 562 _out, _err = self.run_git_command(
556 563 wire, ['fetch', url, '--force', '--prune', '--'] + fetch_refs_chunks,
557 564 fail_on_stderr=False,
558 565 _copts=self._remote_conf(config),
559 566 extra_env={'GIT_TERMINAL_PROMPT': '0'})
560 567
561 568 return remote_refs
562 569
563 570 @reraise_safe_exceptions
564 571 def sync_push(self, wire, url, refs=None):
565 572 if not self.check_url(url, wire):
566 573 return
567 574 config = self._wire_to_config(wire)
568 575 repo = self._factory.repo(wire)
569 576 self.run_git_command(
570 577 wire, ['push', url, '--mirror'], fail_on_stderr=False,
571 578 _copts=self._remote_conf(config),
572 579 extra_env={'GIT_TERMINAL_PROMPT': '0'})
573 580
574 581 @reraise_safe_exceptions
575 582 def get_remote_refs(self, wire, url):
576 583 repo = Repo(url)
577 584 return repo.get_refs()
578 585
579 586 @reraise_safe_exceptions
580 587 def get_description(self, wire):
581 588 repo = self._factory.repo(wire)
582 589 return repo.get_description()
583 590
584 591 @reraise_safe_exceptions
585 592 def get_missing_revs(self, wire, rev1, rev2, path2):
586 593 repo = self._factory.repo(wire)
587 594 LocalGitClient(thin_packs=False).fetch(path2, repo)
588 595
589 596 wire_remote = wire.copy()
590 597 wire_remote['path'] = path2
591 598 repo_remote = self._factory.repo(wire_remote)
592 599 LocalGitClient(thin_packs=False).fetch(wire["path"], repo_remote)
593 600
594 601 revs = [
595 602 x.commit.id
596 603 for x in repo_remote.get_walker(include=[rev2], exclude=[rev1])]
597 604 return revs
598 605
599 606 @reraise_safe_exceptions
600 607 def get_object(self, wire, sha):
601 608 repo = self._factory.repo_libgit2(wire)
602 609
603 610 missing_commit_err = 'Commit {} does not exist for `{}`'.format(sha, wire['path'])
604 611 try:
605 612 commit = repo.revparse_single(sha)
606 613 except (KeyError, ValueError) as e:
607 614 raise exceptions.LookupException(e)(missing_commit_err)
608 615
609 616 if isinstance(commit, pygit2.Tag):
610 617 commit = repo.get(commit.target)
611 618
612 619 # check for dangling commit
613 620 branches = [x for x in repo.branches.with_commit(commit.hex)]
614 621 if not branches:
615 622 raise exceptions.LookupException(None)(missing_commit_err)
616 623
617 624 commit_id = commit.hex
618 625 type_id = commit.type
619 626
620 627 return {
621 628 'id': commit_id,
622 629 'type': self._type_id_to_name(type_id),
623 630 'commit_id': commit_id,
624 631 'idx': 0
625 632 }
626 633
627 634 @reraise_safe_exceptions
628 635 def get_refs(self, wire):
629 636 repo = self._factory.repo_libgit2(wire)
630 637
631 638 result = {}
632 639 for ref in repo.references:
633 640 peeled_sha = repo.lookup_reference(ref).peel()
634 641 result[ref] = peeled_sha.hex
635 642
636 643 return result
637 644
638 645 @reraise_safe_exceptions
639 646 def head(self, wire, show_exc=True):
640 647 repo = self._factory.repo_libgit2(wire)
641 648 try:
642 649 return repo.head.peel().hex
643 650 except Exception:
644 651 if show_exc:
645 652 raise
646 653
647 654 @reraise_safe_exceptions
648 655 def init(self, wire):
649 656 repo_path = str_to_dulwich(wire['path'])
650 657 self.repo = Repo.init(repo_path)
651 658
652 659 @reraise_safe_exceptions
653 660 def init_bare(self, wire):
654 661 repo_path = str_to_dulwich(wire['path'])
655 662 self.repo = Repo.init_bare(repo_path)
656 663
657 664 @reraise_safe_exceptions
658 665 def revision(self, wire, rev):
659 666 repo = self._factory.repo_libgit2(wire)
660 667 commit = repo[rev]
661 668 obj_data = {
662 669 'id': commit.id.hex,
663 670 }
664 671 # tree objects itself don't have tree_id attribute
665 672 if hasattr(commit, 'tree_id'):
666 673 obj_data['tree'] = commit.tree_id.hex
667 674
668 675 return obj_data
669 676
670 677 @reraise_safe_exceptions
671 678 def date(self, wire, rev):
672 679 repo = self._factory.repo_libgit2(wire)
673 680 commit = repo[rev]
674 681 # TODO(marcink): check dulwich difference of offset vs timezone
675 682 return [commit.commit_time, commit.commit_time_offset]
676 683
677 684 @reraise_safe_exceptions
678 685 def author(self, wire, rev):
679 686 repo = self._factory.repo_libgit2(wire)
680 687 commit = repo[rev]
681 688 if commit.author.email:
682 689 return u"{} <{}>".format(commit.author.name, commit.author.email)
683 690
684 691 return u"{}".format(commit.author.raw_name)
685 692
686 693 @reraise_safe_exceptions
687 694 def message(self, wire, rev):
688 695 repo = self._factory.repo_libgit2(wire)
689 696 commit = repo[rev]
690 697 return commit.message
691 698
692 699 @reraise_safe_exceptions
693 700 def parents(self, wire, rev):
694 701 repo = self._factory.repo_libgit2(wire)
695 702 commit = repo[rev]
696 703 return [x.hex for x in commit.parent_ids]
697 704
698 705 @reraise_safe_exceptions
699 706 def set_refs(self, wire, key, value):
700 707 repo = self._factory.repo_libgit2(wire)
701 708 repo.references.create(key, value, force=True)
702 709
703 710 @reraise_safe_exceptions
704 711 def create_branch(self, wire, branch_name, commit_id, force=False):
705 712 repo = self._factory.repo_libgit2(wire)
706 713 commit = repo[commit_id]
707 714
708 715 if force:
709 716 repo.branches.local.create(branch_name, commit, force=force)
710 717 elif not repo.branches.get(branch_name):
711 718 # create only if that branch isn't existing
712 719 repo.branches.local.create(branch_name, commit, force=force)
713 720
714 721 @reraise_safe_exceptions
715 722 def remove_ref(self, wire, key):
716 723 repo = self._factory.repo_libgit2(wire)
717 724 repo.references.delete(key)
718 725
719 726 @reraise_safe_exceptions
720 727 def tag_remove(self, wire, tag_name):
721 728 repo = self._factory.repo_libgit2(wire)
722 729 key = 'refs/tags/{}'.format(tag_name)
723 730 repo.references.delete(key)
724 731
725 732 @reraise_safe_exceptions
726 733 def tree_changes(self, wire, source_id, target_id):
727 734 # TODO(marcink): remove this seems it's only used by tests
728 735 repo = self._factory.repo(wire)
729 736 source = repo[source_id].tree if source_id else None
730 737 target = repo[target_id].tree
731 738 result = repo.object_store.tree_changes(source, target)
732 739 return list(result)
733 740
734 741 @reraise_safe_exceptions
735 742 def tree_and_type_for_path(self, wire, commit_id, path):
736 743 repo_init = self._factory.repo_libgit2(wire)
737 744
738 745 with repo_init as repo:
739 746 commit = repo[commit_id]
740 747 try:
741 748 tree = commit.tree[path]
742 749 except KeyError:
743 750 return None, None, None
744 751
745 752 return tree.id.hex, tree.type, tree.filemode
746 753
747 754 @reraise_safe_exceptions
748 755 def tree_items(self, wire, tree_id):
749 756 repo_init = self._factory.repo_libgit2(wire)
750 757
751 758 with repo_init as repo:
752 759 try:
753 760 tree = repo[tree_id]
754 761 except KeyError:
755 762 raise ObjectMissing('No tree with id: {}'.format(tree_id))
756 763
757 764 result = []
758 765 for item in tree:
759 766 item_sha = item.hex
760 767 item_mode = item.filemode
761 768 item_type = item.type
762 769
763 770 if item_type == 'commit':
764 771 # NOTE(marcink): submodules we translate to 'link' for backward compat
765 772 item_type = 'link'
766 773
767 774 result.append((item.name, item_mode, item_sha, item_type))
768 775 return result
769 776
770 777 @reraise_safe_exceptions
771 778 def update_server_info(self, wire):
772 779 repo = self._factory.repo(wire)
773 780 update_server_info(repo)
774 781
775 782 @reraise_safe_exceptions
776 783 def discover_git_version(self):
777 784 stdout, _ = self.run_git_command(
778 785 {}, ['--version'], _bare=True, _safe=True)
779 786 prefix = 'git version'
780 787 if stdout.startswith(prefix):
781 788 stdout = stdout[len(prefix):]
782 789 return stdout.strip()
783 790
784 791 @reraise_safe_exceptions
785 792 def get_all_commit_ids(self, wire):
786 if self.is_empty(wire):
787 return []
788 793
789 794 cmd = ['rev-list', '--reverse', '--date-order', '--branches', '--tags']
790 795 try:
791 796 output, __ = self.run_git_command(wire, cmd)
792 797 return output.splitlines()
793 798 except Exception:
794 799 # Can be raised for empty repositories
795 800 return []
796 801
797 802 @reraise_safe_exceptions
798 803 def run_git_command(self, wire, cmd, **opts):
799 804 path = wire.get('path', None)
800 805
801 806 if path and os.path.isdir(path):
802 807 opts['cwd'] = path
803 808
804 809 if '_bare' in opts:
805 810 _copts = []
806 811 del opts['_bare']
807 812 else:
808 813 _copts = ['-c', 'core.quotepath=false', ]
809 814 safe_call = False
810 815 if '_safe' in opts:
811 816 # no exc on failure
812 817 del opts['_safe']
813 818 safe_call = True
814 819
815 820 if '_copts' in opts:
816 821 _copts.extend(opts['_copts'] or [])
817 822 del opts['_copts']
818 823
819 824 gitenv = os.environ.copy()
820 825 gitenv.update(opts.pop('extra_env', {}))
821 826 # need to clean fix GIT_DIR !
822 827 if 'GIT_DIR' in gitenv:
823 828 del gitenv['GIT_DIR']
824 829 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
825 830 gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1'
826 831
827 832 cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
828 833 _opts = {'env': gitenv, 'shell': False}
829 834
830 835 try:
831 836 _opts.update(opts)
832 837 p = subprocessio.SubprocessIOChunker(cmd, **_opts)
833 838
834 839 return ''.join(p), ''.join(p.error)
835 840 except (EnvironmentError, OSError) as err:
836 841 cmd = ' '.join(cmd) # human friendly CMD
837 842 tb_err = ("Couldn't run git command (%s).\n"
838 843 "Original error was:%s\n"
839 844 "Call options:%s\n"
840 845 % (cmd, err, _opts))
841 846 log.exception(tb_err)
842 847 if safe_call:
843 848 return '', err
844 849 else:
845 850 raise exceptions.VcsException()(tb_err)
846 851
847 852 @reraise_safe_exceptions
848 853 def install_hooks(self, wire, force=False):
849 854 from vcsserver.hook_utils import install_git_hooks
850 855 repo = self._factory.repo(wire)
851 856 return install_git_hooks(repo.path, repo.bare, force_create=force)
852 857
853 858 @reraise_safe_exceptions
854 859 def get_hooks_info(self, wire):
855 860 from vcsserver.hook_utils import (
856 861 get_git_pre_hook_version, get_git_post_hook_version)
857 862 repo = self._factory.repo(wire)
858 863 return {
859 864 'pre_version': get_git_pre_hook_version(repo.path, repo.bare),
860 865 'post_version': get_git_post_hook_version(repo.path, repo.bare),
861 866 }
862 867
863 868
864 869 def str_to_dulwich(value):
865 870 """
866 871 Dulwich 0.10.1a requires `unicode` objects to be passed in.
867 872 """
868 873 return value.decode(settings.WIRE_ENCODING)
General Comments 0
You need to be logged in to leave comments. Login now