##// END OF EJS Templates
git: switched most git operations to libgit2
marcink -
r725:d64a71b9 default
parent child Browse files
Show More
@@ -44,25 +44,7 b' class RepoFactory(object):'
44 44 raise NotImplementedError()
45 45
46 46 def repo(self, wire, create=False):
47 """
48 Get a repository instance for the given path.
49
50 Uses internally the low level beaker API since the decorators introduce
51 significant overhead.
52 """
53 region = self._cache_region
54 context = wire.get('context', None)
55 repo_path = wire.get('path', '')
56 context_uid = '{}'.format(context)
57 cache = wire.get('cache', True)
58 cache_on = context and cache
59
60 @region.conditional_cache_on_arguments(condition=cache_on)
61 def create_new_repo(_repo_type, _repo_path, _context_uid):
62 return self._create_repo(wire, create)
63
64 repo = create_new_repo(self.repo_type, repo_path, context_uid)
65 return repo
47 raise NotImplementedError()
66 48
67 49
68 50 def obfuscate_qs(query_string):
@@ -26,13 +26,15 b' import urllib2'
26 26 from functools import wraps
27 27
28 28 import more_itertools
29 import pygit2
30 from pygit2 import Repository as LibGit2Repo
29 31 from dulwich import index, objects
30 32 from dulwich.client import HttpGitClient, LocalGitClient
31 33 from dulwich.errors import (
32 34 NotGitRepository, ChecksumMismatch, WrongObjectException,
33 35 MissingCommitError, ObjectMissing, HangupException,
34 36 UnexpectedCommandError)
35 from dulwich.repo import Repo as DulwichRepo, Tag
37 from dulwich.repo import Repo as DulwichRepo
36 38 from dulwich.server import update_server_info
37 39
38 40 from vcsserver import exceptions, settings, subprocessio
@@ -51,17 +53,17 b' log = logging.getLogger(__name__)'
51 53
52 54 def reraise_safe_exceptions(func):
53 55 """Converts Dulwich exceptions to something neutral."""
56
54 57 @wraps(func)
55 58 def wrapper(*args, **kwargs):
56 59 try:
57 60 return func(*args, **kwargs)
58 except (ChecksumMismatch, WrongObjectException, MissingCommitError,
59 ObjectMissing) as e:
60 exc = exceptions.LookupException(e)
61 raise exc(e)
61 except (ChecksumMismatch, WrongObjectException, MissingCommitError, ObjectMissing,) as e:
62 exc = exceptions.LookupException(org_exc=e)
63 raise exc(safe_str(e))
62 64 except (HangupException, UnexpectedCommandError) as e:
63 exc = exceptions.VcsException(e)
64 raise exc(e)
65 exc = exceptions.VcsException(org_exc=e)
66 raise exc(safe_str(e))
65 67 except Exception as e:
66 68 # NOTE(marcink): becuase of how dulwich handles some exceptions
67 69 # (KeyError on empty repos), we cannot track this and catch all
@@ -80,22 +82,52 b' class Repo(DulwichRepo):'
80 82 Since dulwich is sometimes keeping .idx file descriptors open, it leads to
81 83 "Too many open files" error. We need to close all opened file descriptors
82 84 once the repo object is destroyed.
83
84 TODO: mikhail: please check if we need this wrapper after updating dulwich
85 to 0.12.0 +
86 85 """
87 86 def __del__(self):
88 87 if hasattr(self, 'object_store'):
89 88 self.close()
90 89
91 90
91 class Repository(LibGit2Repo):
92
93 def __enter__(self):
94 return self
95
96 def __exit__(self, exc_type, exc_val, exc_tb):
97 self.free()
98
99
92 100 class GitFactory(RepoFactory):
93 101 repo_type = 'git'
94 102
95 def _create_repo(self, wire, create):
103 def _create_repo(self, wire, create, use_libgit2=False):
104 if use_libgit2:
105 return Repository(wire['path'])
106 else:
96 107 repo_path = str_to_dulwich(wire['path'])
97 108 return Repo(repo_path)
98 109
110 def repo(self, wire, create=False, use_libgit2=False):
111 """
112 Get a repository instance for the given path.
113 """
114 region = self._cache_region
115 context = wire.get('context', None)
116 repo_path = wire.get('path', '')
117 context_uid = '{}'.format(context)
118 cache = wire.get('cache', True)
119 cache_on = context and cache
120
121 @region.conditional_cache_on_arguments(condition=cache_on)
122 def create_new_repo(_repo_type, _repo_path, _context_uid, _use_libgit2):
123 return self._create_repo(wire, create, use_libgit2)
124
125 repo = create_new_repo(self.repo_type, repo_path, context_uid, use_libgit2)
126 return repo
127
128 def repo_libgit2(self, wire):
129 return self.repo(wire, use_libgit2=True)
130
99 131
100 132 class GitRemote(object):
101 133
@@ -103,10 +135,10 b' class GitRemote(object):'
103 135 self._factory = factory
104 136 self.peeled_ref_marker = '^{}'
105 137 self._bulk_methods = {
106 "author": self.commit_attribute,
107 "date": self.get_object_attrs,
108 "message": self.commit_attribute,
109 "parents": self.commit_attribute,
138 "date": self.date,
139 "author": self.author,
140 "message": self.message,
141 "parents": self.parents,
110 142 "_commit": self.revision,
111 143 }
112 144
@@ -115,10 +147,6 b' class GitRemote(object):'
115 147 return dict([(x[0] + '_' + x[1], x[2]) for x in wire['config']])
116 148 return {}
117 149
118 def _assign_ref(self, wire, ref, commit_id):
119 repo = self._factory.repo(wire)
120 repo[ref] = commit_id
121
122 150 def _remote_conf(self, config):
123 151 params = [
124 152 '-c', 'core.askpass=""',
@@ -130,12 +158,15 b' class GitRemote(object):'
130 158
131 159 @reraise_safe_exceptions
132 160 def is_empty(self, wire):
133 repo = self._factory.repo(wire)
134 try:
135 return not repo.head()
136 except Exception:
137 log.exception("failed to read object_store")
138 return True
161 repo = self._factory.repo_libgit2(wire)
162
163 # NOTE(marcink): old solution as an alternative
164 # try:
165 # return not repo.head.name
166 # except Exception:
167 # return True
168
169 return repo.is_empty
139 170
140 171 @reraise_safe_exceptions
141 172 def add_object(self, wire, content):
@@ -147,10 +178,10 b' class GitRemote(object):'
147 178
148 179 @reraise_safe_exceptions
149 180 def assert_correct_path(self, wire):
181 try:
182 self._factory.repo_libgit2(wire)
183 except pygit2.GitError:
150 184 path = wire.get('path')
151 try:
152 self._factory.repo(wire)
153 except NotGitRepository as e:
154 185 tb = traceback.format_exc()
155 186 log.debug("Invalid Git path `%s`, tb: %s", path, tb)
156 187 return False
@@ -159,19 +190,23 b' class GitRemote(object):'
159 190
160 191 @reraise_safe_exceptions
161 192 def bare(self, wire):
162 repo = self._factory.repo(wire)
163 return repo.bare
193 repo = self._factory.repo_libgit2(wire)
194 return repo.is_bare
164 195
165 196 @reraise_safe_exceptions
166 197 def blob_as_pretty_string(self, wire, sha):
167 repo = self._factory.repo(wire)
168 return repo[sha].as_pretty_string()
198 repo_init = self._factory.repo_libgit2(wire)
199 with repo_init as repo:
200 blob_obj = repo[sha]
201 blob = blob_obj.data
202 return blob
169 203
170 204 @reraise_safe_exceptions
171 205 def blob_raw_length(self, wire, sha):
172 repo = self._factory.repo(wire)
206 repo_init = self._factory.repo_libgit2(wire)
207 with repo_init as repo:
173 208 blob = repo[sha]
174 return blob.raw_length()
209 return blob.size
175 210
176 211 def _parse_lfs_pointer(self, raw_content):
177 212
@@ -230,14 +265,9 b' class GitRemote(object):'
230 265 try:
231 266 method = self._bulk_methods[attr]
232 267 args = [wire, rev]
233 if attr == "date":
234 args.extend(["commit_time", "commit_timezone"])
235 elif attr in ["author", "message", "parents"]:
236 args.append(attr)
237 268 result[attr] = method(*args)
238 269 except KeyError as e:
239 raise exceptions.VcsException(e)(
240 "Unknown bulk attribute: %s" % attr)
270 raise exceptions.VcsException(e)("Unknown bulk attribute: %s" % attr)
241 271 return result
242 272
243 273 def _build_opener(self, url):
@@ -255,6 +285,14 b' class GitRemote(object):'
255 285
256 286 return urllib2.build_opener(*handlers)
257 287
288 def _type_id_to_name(self, type_id):
289 return {
290 1: b'commit',
291 2: b'tree',
292 3: b'blob',
293 4: b'tag'
294 }[type_id]
295
258 296 @reraise_safe_exceptions
259 297 def check_url(self, url, config):
260 298 url_obj = url_parser(url)
@@ -367,8 +405,7 b' class GitRemote(object):'
367 405 curtree = newtree
368 406 parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
369 407 else:
370 parent.add(
371 name=node['node_path'], mode=node['mode'], hexsha=blob.id)
408 parent.add(name=node['node_path'], mode=node['mode'], hexsha=blob.id)
372 409
373 410 new_trees.append(parent)
374 411 # Update ancestors
@@ -412,6 +449,9 b' class GitRemote(object):'
412 449 setattr(commit, k, v)
413 450 object_store.add_object(commit)
414 451
452 self.create_branch(wire, branch, commit.id)
453
454 # dulwich set-ref
415 455 ref = 'refs/heads/%s' % branch
416 456 repo.refs[ref] = commit.id
417 457
@@ -556,45 +596,43 b' class GitRemote(object):'
556 596
557 597 @reraise_safe_exceptions
558 598 def get_object(self, wire, sha):
559 repo = self._factory.repo(wire)
560 obj = repo.get_object(sha)
561 commit_id = obj.id
599 repo = self._factory.repo_libgit2(wire)
562 600
563 if isinstance(obj, Tag):
564 commit_id = obj.object[1]
601 try:
602 commit = repo.revparse_single(sha)
603 except (KeyError, ValueError) as e:
604 msg = 'Commit {} does not exist for `{}`'.format(sha, wire['path'])
605 raise exceptions.LookupException(e)(msg)
606
607 if isinstance(commit, pygit2.Tag):
608 commit = repo.get(commit.target)
609
610 commit_id = commit.hex
611 type_id = commit.type
565 612
566 613 return {
567 'id': obj.id,
568 'type': obj.type_name,
614 'id': commit_id,
615 'type': self._type_id_to_name(type_id),
569 616 'commit_id': commit_id,
570 617 'idx': 0
571 618 }
572 619
573 620 @reraise_safe_exceptions
574 def get_object_attrs(self, wire, sha, *attrs):
575 repo = self._factory.repo(wire)
576 obj = repo.get_object(sha)
577 return list(getattr(obj, a) for a in attrs)
621 def get_refs(self, wire):
622 repo = self._factory.repo_libgit2(wire)
578 623
579 @reraise_safe_exceptions
580 def get_refs(self, wire):
581 repo = self._factory.repo(wire)
582 624 result = {}
583 for ref, sha in repo.refs.as_dict().items():
584 peeled_sha = repo.get_peeled(ref)
585 result[ref] = peeled_sha
625 for ref in repo.references:
626 peeled_sha = repo.lookup_reference(ref).peel()
627 result[ref] = peeled_sha.hex
628
586 629 return result
587 630
588 631 @reraise_safe_exceptions
589 def get_refs_path(self, wire):
590 repo = self._factory.repo(wire)
591 return repo.refs.path
592
593 @reraise_safe_exceptions
594 632 def head(self, wire, show_exc=True):
595 repo = self._factory.repo(wire)
633 repo = self._factory.repo_libgit2(wire)
596 634 try:
597 return repo.head()
635 return repo.head.peel().hex
598 636 except Exception:
599 637 if show_exc:
600 638 raise
@@ -611,35 +649,75 b' class GitRemote(object):'
611 649
612 650 @reraise_safe_exceptions
613 651 def revision(self, wire, rev):
614 repo = self._factory.repo(wire)
615 obj = repo[rev]
652 repo = self._factory.repo_libgit2(wire)
653 commit = repo[rev]
616 654 obj_data = {
617 'id': obj.id,
655 'id': commit.id.hex,
618 656 }
619 try:
620 obj_data['tree'] = obj.tree
621 except AttributeError:
622 pass
657 # tree objects itself don't have tree_id attribute
658 if hasattr(commit, 'tree_id'):
659 obj_data['tree'] = commit.tree_id.hex
660
623 661 return obj_data
624 662
625 663 @reraise_safe_exceptions
626 def commit_attribute(self, wire, rev, attr):
627 repo = self._factory.repo(wire)
628 obj = repo[rev]
629 return getattr(obj, attr)
664 def date(self, wire, rev):
665 repo = self._factory.repo_libgit2(wire)
666 commit = repo[rev]
667 # TODO(marcink): check dulwich difference of offset vs timezone
668 return [commit.commit_time, commit.commit_time_offset]
669
670 @reraise_safe_exceptions
671 def author(self, wire, rev):
672 repo = self._factory.repo_libgit2(wire)
673 commit = repo[rev]
674 if commit.author.email:
675 return u"{} <{}>".format(commit.author.name, commit.author.email)
676
677 return u"{}".format(commit.author.raw_name)
678
679 @reraise_safe_exceptions
680 def message(self, wire, rev):
681 repo = self._factory.repo_libgit2(wire)
682 commit = repo[rev]
683 return commit.message
684
685 @reraise_safe_exceptions
686 def parents(self, wire, rev):
687 repo = self._factory.repo_libgit2(wire)
688 commit = repo[rev]
689 return [x.hex for x in commit.parent_ids]
630 690
631 691 @reraise_safe_exceptions
632 692 def set_refs(self, wire, key, value):
633 repo = self._factory.repo(wire)
634 repo.refs[key] = value
693 repo = self._factory.repo_libgit2(wire)
694 repo.references.create(key, value, force=True)
695
696 @reraise_safe_exceptions
697 def create_branch(self, wire, branch_name, commit_id, force=False):
698 repo = self._factory.repo_libgit2(wire)
699 commit = repo[commit_id]
700
701 if force:
702 repo.branches.local.create(branch_name, commit, force=force)
703 elif not repo.branches.get(branch_name):
704 # create only if that branch isn't existing
705 repo.branches.local.create(branch_name, commit, force=force)
635 706
636 707 @reraise_safe_exceptions
637 708 def remove_ref(self, wire, key):
638 repo = self._factory.repo(wire)
639 del repo.refs[key]
709 repo = self._factory.repo_libgit2(wire)
710 repo.references.delete(key)
711
712 @reraise_safe_exceptions
713 def tag_remove(self, wire, tag_name):
714 repo = self._factory.repo_libgit2(wire)
715 key = 'refs/tags/{}'.format(tag_name)
716 repo.references.delete(key)
640 717
641 718 @reraise_safe_exceptions
642 719 def tree_changes(self, wire, source_id, target_id):
720 # TODO(marcink): remove this seems it's only used by tests
643 721 repo = self._factory.repo(wire)
644 722 source = repo[source_id].tree if source_id else None
645 723 target = repo[target_id].tree
@@ -648,20 +726,22 b' class GitRemote(object):'
648 726
649 727 @reraise_safe_exceptions
650 728 def tree_items(self, wire, tree_id):
651 repo = self._factory.repo(wire)
729 repo_init = self._factory.repo_libgit2(wire)
730
731 with repo_init as repo:
652 732 tree = repo[tree_id]
653 733
654 734 result = []
655 for item in tree.iteritems():
656 item_sha = item.sha
657 item_mode = item.mode
735 for item in tree:
736 item_sha = item.hex
737 item_mode = item.filemode
738 item_type = item.type
658 739
659 if FILE_MODE(item_mode) == GIT_LINK:
660 item_type = "link"
661 else:
662 item_type = repo[item_sha].type_name
740 if item_type == 'commit':
741 # NOTE(marcink): submodules we translate to 'link' for backward compat
742 item_type = 'link'
663 743
664 result.append((item.path, item_mode, item_sha, item_type))
744 result.append((item.name, item_mode, item_sha, item_type))
665 745 return result
666 746
667 747 @reraise_safe_exceptions
@@ -679,6 +759,19 b' class GitRemote(object):'
679 759 return stdout.strip()
680 760
681 761 @reraise_safe_exceptions
762 def get_all_commit_ids(self, wire):
763 if self.is_empty(wire):
764 return []
765
766 cmd = ['rev-list', '--reverse', '--date-order', '--branches', '--tags']
767 try:
768 output, __ = self.run_git_command(wire, cmd)
769 return output.splitlines()
770 except Exception:
771 # Can be raised for empty repositories
772 return []
773
774 @reraise_safe_exceptions
682 775 def run_git_command(self, wire, cmd, **opts):
683 776 path = wire.get('path', None)
684 777
@@ -98,6 +98,7 b' def make_ui_from_config(repo_config):'
98 98
99 99 def reraise_safe_exceptions(func):
100 100 """Decorator for converting mercurial exceptions to something neutral."""
101
101 102 def wrapper(*args, **kwargs):
102 103 try:
103 104 return func(*args, **kwargs)
@@ -142,6 +143,23 b' class MercurialFactory(RepoFactory):'
142 143 baseui = self._create_config(wire["config"])
143 144 return instance(baseui, wire["path"], create)
144 145
146 def repo(self, wire, create=False):
147 """
148 Get a repository instance for the given path.
149 """
150 region = self._cache_region
151 context = wire.get('context', None)
152 repo_path = wire.get('path', '')
153 context_uid = '{}'.format(context)
154 cache = wire.get('cache', True)
155 cache_on = context and cache
156
157 @region.conditional_cache_on_arguments(condition=cache_on)
158 def create_new_repo(_repo_type, _repo_path, _context_uid):
159 return self._create_repo(wire, create)
160
161 return create_new_repo(self.repo_type, repo_path, context_uid)
162
145 163
146 164 class HgRemote(object):
147 165
@@ -354,6 +354,7 b' class HTTPApplication(object):'
354 354
355 355 log.debug('method called:%s with kwargs:%s context_uid: %s',
356 356 method, kwargs, context_uid)
357
357 358 try:
358 359 resp = getattr(remote, method)(*args, **kwargs)
359 360 except Exception as e:
@@ -97,9 +97,6 b' class SubversionFactory(RepoFactory):'
97 97 def repo(self, wire, create=False, compatible_version=None):
98 98 """
99 99 Get a repository instance for the given path.
100
101 Uses internally the low level beaker API since the decorators introduce
102 significant overhead.
103 100 """
104 101 region = self._cache_region
105 102 context = wire.get('context', None)
@@ -99,14 +99,9 b' class TestGitFetch(object):'
99 99 mock_repo().get_refs.assert_called_once_with()
100 100 assert remote_refs == sample_refs
101 101
102 def test_remove_ref(self):
103 ref_to_remove = 'refs/tags/v0.1.9'
104 self.mock_repo.refs = SAMPLE_REFS.copy()
105 self.remote_git.remove_ref(None, ref_to_remove)
106 assert ref_to_remove not in self.mock_repo.refs
107
108 102
109 103 class TestReraiseSafeExceptions(object):
104
110 105 def test_method_decorated_with_reraise_safe_exceptions(self):
111 106 factory = Mock()
112 107 git_remote = git.GitRemote(factory)
General Comments 0
You need to be logged in to leave comments. Login now