##// END OF EJS Templates
git: switched most git operations to libgit2
marcink -
r725:d64a71b9 default
parent child
Show More
@@ -44,25 +44,7 class RepoFactory(object):
44 raise NotImplementedError()
44 raise NotImplementedError()
45
45
46 def repo(self, wire, create=False):
46 def repo(self, wire, create=False):
47 """
47 raise NotImplementedError()
48 Get a repository instance for the given path.
49
50 Uses internally the low level beaker API since the decorators introduce
51 significant overhead.
52 """
53 region = self._cache_region
54 context = wire.get('context', None)
55 repo_path = wire.get('path', '')
56 context_uid = '{}'.format(context)
57 cache = wire.get('cache', True)
58 cache_on = context and cache
59
60 @region.conditional_cache_on_arguments(condition=cache_on)
61 def create_new_repo(_repo_type, _repo_path, _context_uid):
62 return self._create_repo(wire, create)
63
64 repo = create_new_repo(self.repo_type, repo_path, context_uid)
65 return repo
66
48
67
49
68 def obfuscate_qs(query_string):
50 def obfuscate_qs(query_string):
@@ -26,13 +26,15 import urllib2
26 from functools import wraps
26 from functools import wraps
27
27
28 import more_itertools
28 import more_itertools
29 import pygit2
30 from pygit2 import Repository as LibGit2Repo
29 from dulwich import index, objects
31 from dulwich import index, objects
30 from dulwich.client import HttpGitClient, LocalGitClient
32 from dulwich.client import HttpGitClient, LocalGitClient
31 from dulwich.errors import (
33 from dulwich.errors import (
32 NotGitRepository, ChecksumMismatch, WrongObjectException,
34 NotGitRepository, ChecksumMismatch, WrongObjectException,
33 MissingCommitError, ObjectMissing, HangupException,
35 MissingCommitError, ObjectMissing, HangupException,
34 UnexpectedCommandError)
36 UnexpectedCommandError)
35 from dulwich.repo import Repo as DulwichRepo, Tag
37 from dulwich.repo import Repo as DulwichRepo
36 from dulwich.server import update_server_info
38 from dulwich.server import update_server_info
37
39
38 from vcsserver import exceptions, settings, subprocessio
40 from vcsserver import exceptions, settings, subprocessio
@@ -51,17 +53,17 log = logging.getLogger(__name__)
51
53
52 def reraise_safe_exceptions(func):
54 def reraise_safe_exceptions(func):
53 """Converts Dulwich exceptions to something neutral."""
55 """Converts Dulwich exceptions to something neutral."""
56
54 @wraps(func)
57 @wraps(func)
55 def wrapper(*args, **kwargs):
58 def wrapper(*args, **kwargs):
56 try:
59 try:
57 return func(*args, **kwargs)
60 return func(*args, **kwargs)
58 except (ChecksumMismatch, WrongObjectException, MissingCommitError,
61 except (ChecksumMismatch, WrongObjectException, MissingCommitError, ObjectMissing,) as e:
59 ObjectMissing) as e:
62 exc = exceptions.LookupException(org_exc=e)
60 exc = exceptions.LookupException(e)
63 raise exc(safe_str(e))
61 raise exc(e)
62 except (HangupException, UnexpectedCommandError) as e:
64 except (HangupException, UnexpectedCommandError) as e:
63 exc = exceptions.VcsException(e)
65 exc = exceptions.VcsException(org_exc=e)
64 raise exc(e)
66 raise exc(safe_str(e))
65 except Exception as e:
67 except Exception as e:
66 # NOTE(marcink): becuase of how dulwich handles some exceptions
68 # NOTE(marcink): becuase of how dulwich handles some exceptions
67 # (KeyError on empty repos), we cannot track this and catch all
69 # (KeyError on empty repos), we cannot track this and catch all
@@ -80,21 +82,51 class Repo(DulwichRepo):
80 Since dulwich is sometimes keeping .idx file descriptors open, it leads to
82 Since dulwich is sometimes keeping .idx file descriptors open, it leads to
81 "Too many open files" error. We need to close all opened file descriptors
83 "Too many open files" error. We need to close all opened file descriptors
82 once the repo object is destroyed.
84 once the repo object is destroyed.
83
84 TODO: mikhail: please check if we need this wrapper after updating dulwich
85 to 0.12.0 +
86 """
85 """
87 def __del__(self):
86 def __del__(self):
88 if hasattr(self, 'object_store'):
87 if hasattr(self, 'object_store'):
89 self.close()
88 self.close()
90
89
91
90
91 class Repository(LibGit2Repo):
92
93 def __enter__(self):
94 return self
95
96 def __exit__(self, exc_type, exc_val, exc_tb):
97 self.free()
98
99
92 class GitFactory(RepoFactory):
100 class GitFactory(RepoFactory):
93 repo_type = 'git'
101 repo_type = 'git'
94
102
95 def _create_repo(self, wire, create):
103 def _create_repo(self, wire, create, use_libgit2=False):
96 repo_path = str_to_dulwich(wire['path'])
104 if use_libgit2:
97 return Repo(repo_path)
105 return Repository(wire['path'])
106 else:
107 repo_path = str_to_dulwich(wire['path'])
108 return Repo(repo_path)
109
110 def repo(self, wire, create=False, use_libgit2=False):
111 """
112 Get a repository instance for the given path.
113 """
114 region = self._cache_region
115 context = wire.get('context', None)
116 repo_path = wire.get('path', '')
117 context_uid = '{}'.format(context)
118 cache = wire.get('cache', True)
119 cache_on = context and cache
120
121 @region.conditional_cache_on_arguments(condition=cache_on)
122 def create_new_repo(_repo_type, _repo_path, _context_uid, _use_libgit2):
123 return self._create_repo(wire, create, use_libgit2)
124
125 repo = create_new_repo(self.repo_type, repo_path, context_uid, use_libgit2)
126 return repo
127
128 def repo_libgit2(self, wire):
129 return self.repo(wire, use_libgit2=True)
98
130
99
131
100 class GitRemote(object):
132 class GitRemote(object):
@@ -103,10 +135,10 class GitRemote(object):
103 self._factory = factory
135 self._factory = factory
104 self.peeled_ref_marker = '^{}'
136 self.peeled_ref_marker = '^{}'
105 self._bulk_methods = {
137 self._bulk_methods = {
106 "author": self.commit_attribute,
138 "date": self.date,
107 "date": self.get_object_attrs,
139 "author": self.author,
108 "message": self.commit_attribute,
140 "message": self.message,
109 "parents": self.commit_attribute,
141 "parents": self.parents,
110 "_commit": self.revision,
142 "_commit": self.revision,
111 }
143 }
112
144
@@ -115,10 +147,6 class GitRemote(object):
115 return dict([(x[0] + '_' + x[1], x[2]) for x in wire['config']])
147 return dict([(x[0] + '_' + x[1], x[2]) for x in wire['config']])
116 return {}
148 return {}
117
149
118 def _assign_ref(self, wire, ref, commit_id):
119 repo = self._factory.repo(wire)
120 repo[ref] = commit_id
121
122 def _remote_conf(self, config):
150 def _remote_conf(self, config):
123 params = [
151 params = [
124 '-c', 'core.askpass=""',
152 '-c', 'core.askpass=""',
@@ -130,12 +158,15 class GitRemote(object):
130
158
131 @reraise_safe_exceptions
159 @reraise_safe_exceptions
132 def is_empty(self, wire):
160 def is_empty(self, wire):
133 repo = self._factory.repo(wire)
161 repo = self._factory.repo_libgit2(wire)
134 try:
162
135 return not repo.head()
163 # NOTE(marcink): old solution as an alternative
136 except Exception:
164 # try:
137 log.exception("failed to read object_store")
165 # return not repo.head.name
138 return True
166 # except Exception:
167 # return True
168
169 return repo.is_empty
139
170
140 @reraise_safe_exceptions
171 @reraise_safe_exceptions
141 def add_object(self, wire, content):
172 def add_object(self, wire, content):
@@ -147,10 +178,10 class GitRemote(object):
147
178
148 @reraise_safe_exceptions
179 @reraise_safe_exceptions
149 def assert_correct_path(self, wire):
180 def assert_correct_path(self, wire):
150 path = wire.get('path')
151 try:
181 try:
152 self._factory.repo(wire)
182 self._factory.repo_libgit2(wire)
153 except NotGitRepository as e:
183 except pygit2.GitError:
184 path = wire.get('path')
154 tb = traceback.format_exc()
185 tb = traceback.format_exc()
155 log.debug("Invalid Git path `%s`, tb: %s", path, tb)
186 log.debug("Invalid Git path `%s`, tb: %s", path, tb)
156 return False
187 return False
@@ -159,19 +190,23 class GitRemote(object):
159
190
160 @reraise_safe_exceptions
191 @reraise_safe_exceptions
161 def bare(self, wire):
192 def bare(self, wire):
162 repo = self._factory.repo(wire)
193 repo = self._factory.repo_libgit2(wire)
163 return repo.bare
194 return repo.is_bare
164
195
165 @reraise_safe_exceptions
196 @reraise_safe_exceptions
166 def blob_as_pretty_string(self, wire, sha):
197 def blob_as_pretty_string(self, wire, sha):
167 repo = self._factory.repo(wire)
198 repo_init = self._factory.repo_libgit2(wire)
168 return repo[sha].as_pretty_string()
199 with repo_init as repo:
200 blob_obj = repo[sha]
201 blob = blob_obj.data
202 return blob
169
203
170 @reraise_safe_exceptions
204 @reraise_safe_exceptions
171 def blob_raw_length(self, wire, sha):
205 def blob_raw_length(self, wire, sha):
172 repo = self._factory.repo(wire)
206 repo_init = self._factory.repo_libgit2(wire)
173 blob = repo[sha]
207 with repo_init as repo:
174 return blob.raw_length()
208 blob = repo[sha]
209 return blob.size
175
210
176 def _parse_lfs_pointer(self, raw_content):
211 def _parse_lfs_pointer(self, raw_content):
177
212
@@ -230,14 +265,9 class GitRemote(object):
230 try:
265 try:
231 method = self._bulk_methods[attr]
266 method = self._bulk_methods[attr]
232 args = [wire, rev]
267 args = [wire, rev]
233 if attr == "date":
234 args.extend(["commit_time", "commit_timezone"])
235 elif attr in ["author", "message", "parents"]:
236 args.append(attr)
237 result[attr] = method(*args)
268 result[attr] = method(*args)
238 except KeyError as e:
269 except KeyError as e:
239 raise exceptions.VcsException(e)(
270 raise exceptions.VcsException(e)("Unknown bulk attribute: %s" % attr)
240 "Unknown bulk attribute: %s" % attr)
241 return result
271 return result
242
272
243 def _build_opener(self, url):
273 def _build_opener(self, url):
@@ -255,6 +285,14 class GitRemote(object):
255
285
256 return urllib2.build_opener(*handlers)
286 return urllib2.build_opener(*handlers)
257
287
288 def _type_id_to_name(self, type_id):
289 return {
290 1: b'commit',
291 2: b'tree',
292 3: b'blob',
293 4: b'tag'
294 }[type_id]
295
258 @reraise_safe_exceptions
296 @reraise_safe_exceptions
259 def check_url(self, url, config):
297 def check_url(self, url, config):
260 url_obj = url_parser(url)
298 url_obj = url_parser(url)
@@ -367,8 +405,7 class GitRemote(object):
367 curtree = newtree
405 curtree = newtree
368 parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
406 parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
369 else:
407 else:
370 parent.add(
408 parent.add(name=node['node_path'], mode=node['mode'], hexsha=blob.id)
371 name=node['node_path'], mode=node['mode'], hexsha=blob.id)
372
409
373 new_trees.append(parent)
410 new_trees.append(parent)
374 # Update ancestors
411 # Update ancestors
@@ -412,6 +449,9 class GitRemote(object):
412 setattr(commit, k, v)
449 setattr(commit, k, v)
413 object_store.add_object(commit)
450 object_store.add_object(commit)
414
451
452 self.create_branch(wire, branch, commit.id)
453
454 # dulwich set-ref
415 ref = 'refs/heads/%s' % branch
455 ref = 'refs/heads/%s' % branch
416 repo.refs[ref] = commit.id
456 repo.refs[ref] = commit.id
417
457
@@ -556,45 +596,43 class GitRemote(object):
556
596
557 @reraise_safe_exceptions
597 @reraise_safe_exceptions
558 def get_object(self, wire, sha):
598 def get_object(self, wire, sha):
559 repo = self._factory.repo(wire)
599 repo = self._factory.repo_libgit2(wire)
560 obj = repo.get_object(sha)
561 commit_id = obj.id
562
600
563 if isinstance(obj, Tag):
601 try:
564 commit_id = obj.object[1]
602 commit = repo.revparse_single(sha)
603 except (KeyError, ValueError) as e:
604 msg = 'Commit {} does not exist for `{}`'.format(sha, wire['path'])
605 raise exceptions.LookupException(e)(msg)
606
607 if isinstance(commit, pygit2.Tag):
608 commit = repo.get(commit.target)
609
610 commit_id = commit.hex
611 type_id = commit.type
565
612
566 return {
613 return {
567 'id': obj.id,
614 'id': commit_id,
568 'type': obj.type_name,
615 'type': self._type_id_to_name(type_id),
569 'commit_id': commit_id,
616 'commit_id': commit_id,
570 'idx': 0
617 'idx': 0
571 }
618 }
572
619
573 @reraise_safe_exceptions
620 @reraise_safe_exceptions
574 def get_object_attrs(self, wire, sha, *attrs):
621 def get_refs(self, wire):
575 repo = self._factory.repo(wire)
622 repo = self._factory.repo_libgit2(wire)
576 obj = repo.get_object(sha)
577 return list(getattr(obj, a) for a in attrs)
578
623
579 @reraise_safe_exceptions
580 def get_refs(self, wire):
581 repo = self._factory.repo(wire)
582 result = {}
624 result = {}
583 for ref, sha in repo.refs.as_dict().items():
625 for ref in repo.references:
584 peeled_sha = repo.get_peeled(ref)
626 peeled_sha = repo.lookup_reference(ref).peel()
585 result[ref] = peeled_sha
627 result[ref] = peeled_sha.hex
628
586 return result
629 return result
587
630
588 @reraise_safe_exceptions
631 @reraise_safe_exceptions
589 def get_refs_path(self, wire):
590 repo = self._factory.repo(wire)
591 return repo.refs.path
592
593 @reraise_safe_exceptions
594 def head(self, wire, show_exc=True):
632 def head(self, wire, show_exc=True):
595 repo = self._factory.repo(wire)
633 repo = self._factory.repo_libgit2(wire)
596 try:
634 try:
597 return repo.head()
635 return repo.head.peel().hex
598 except Exception:
636 except Exception:
599 if show_exc:
637 if show_exc:
600 raise
638 raise
@@ -611,35 +649,75 class GitRemote(object):
611
649
612 @reraise_safe_exceptions
650 @reraise_safe_exceptions
613 def revision(self, wire, rev):
651 def revision(self, wire, rev):
614 repo = self._factory.repo(wire)
652 repo = self._factory.repo_libgit2(wire)
615 obj = repo[rev]
653 commit = repo[rev]
616 obj_data = {
654 obj_data = {
617 'id': obj.id,
655 'id': commit.id.hex,
618 }
656 }
619 try:
657 # tree objects itself don't have tree_id attribute
620 obj_data['tree'] = obj.tree
658 if hasattr(commit, 'tree_id'):
621 except AttributeError:
659 obj_data['tree'] = commit.tree_id.hex
622 pass
660
623 return obj_data
661 return obj_data
624
662
625 @reraise_safe_exceptions
663 @reraise_safe_exceptions
626 def commit_attribute(self, wire, rev, attr):
664 def date(self, wire, rev):
627 repo = self._factory.repo(wire)
665 repo = self._factory.repo_libgit2(wire)
628 obj = repo[rev]
666 commit = repo[rev]
629 return getattr(obj, attr)
667 # TODO(marcink): check dulwich difference of offset vs timezone
668 return [commit.commit_time, commit.commit_time_offset]
669
670 @reraise_safe_exceptions
671 def author(self, wire, rev):
672 repo = self._factory.repo_libgit2(wire)
673 commit = repo[rev]
674 if commit.author.email:
675 return u"{} <{}>".format(commit.author.name, commit.author.email)
676
677 return u"{}".format(commit.author.raw_name)
678
679 @reraise_safe_exceptions
680 def message(self, wire, rev):
681 repo = self._factory.repo_libgit2(wire)
682 commit = repo[rev]
683 return commit.message
684
685 @reraise_safe_exceptions
686 def parents(self, wire, rev):
687 repo = self._factory.repo_libgit2(wire)
688 commit = repo[rev]
689 return [x.hex for x in commit.parent_ids]
630
690
631 @reraise_safe_exceptions
691 @reraise_safe_exceptions
632 def set_refs(self, wire, key, value):
692 def set_refs(self, wire, key, value):
633 repo = self._factory.repo(wire)
693 repo = self._factory.repo_libgit2(wire)
634 repo.refs[key] = value
694 repo.references.create(key, value, force=True)
695
696 @reraise_safe_exceptions
697 def create_branch(self, wire, branch_name, commit_id, force=False):
698 repo = self._factory.repo_libgit2(wire)
699 commit = repo[commit_id]
700
701 if force:
702 repo.branches.local.create(branch_name, commit, force=force)
703 elif not repo.branches.get(branch_name):
704 # create only if that branch isn't existing
705 repo.branches.local.create(branch_name, commit, force=force)
635
706
636 @reraise_safe_exceptions
707 @reraise_safe_exceptions
637 def remove_ref(self, wire, key):
708 def remove_ref(self, wire, key):
638 repo = self._factory.repo(wire)
709 repo = self._factory.repo_libgit2(wire)
639 del repo.refs[key]
710 repo.references.delete(key)
711
712 @reraise_safe_exceptions
713 def tag_remove(self, wire, tag_name):
714 repo = self._factory.repo_libgit2(wire)
715 key = 'refs/tags/{}'.format(tag_name)
716 repo.references.delete(key)
640
717
641 @reraise_safe_exceptions
718 @reraise_safe_exceptions
642 def tree_changes(self, wire, source_id, target_id):
719 def tree_changes(self, wire, source_id, target_id):
720 # TODO(marcink): remove this seems it's only used by tests
643 repo = self._factory.repo(wire)
721 repo = self._factory.repo(wire)
644 source = repo[source_id].tree if source_id else None
722 source = repo[source_id].tree if source_id else None
645 target = repo[target_id].tree
723 target = repo[target_id].tree
@@ -648,21 +726,23 class GitRemote(object):
648
726
649 @reraise_safe_exceptions
727 @reraise_safe_exceptions
650 def tree_items(self, wire, tree_id):
728 def tree_items(self, wire, tree_id):
651 repo = self._factory.repo(wire)
729 repo_init = self._factory.repo_libgit2(wire)
652 tree = repo[tree_id]
653
730
654 result = []
731 with repo_init as repo:
655 for item in tree.iteritems():
732 tree = repo[tree_id]
656 item_sha = item.sha
657 item_mode = item.mode
658
733
659 if FILE_MODE(item_mode) == GIT_LINK:
734 result = []
660 item_type = "link"
735 for item in tree:
661 else:
736 item_sha = item.hex
662 item_type = repo[item_sha].type_name
737 item_mode = item.filemode
738 item_type = item.type
663
739
664 result.append((item.path, item_mode, item_sha, item_type))
740 if item_type == 'commit':
665 return result
741 # NOTE(marcink): submodules we translate to 'link' for backward compat
742 item_type = 'link'
743
744 result.append((item.name, item_mode, item_sha, item_type))
745 return result
666
746
667 @reraise_safe_exceptions
747 @reraise_safe_exceptions
668 def update_server_info(self, wire):
748 def update_server_info(self, wire):
@@ -679,6 +759,19 class GitRemote(object):
679 return stdout.strip()
759 return stdout.strip()
680
760
681 @reraise_safe_exceptions
761 @reraise_safe_exceptions
762 def get_all_commit_ids(self, wire):
763 if self.is_empty(wire):
764 return []
765
766 cmd = ['rev-list', '--reverse', '--date-order', '--branches', '--tags']
767 try:
768 output, __ = self.run_git_command(wire, cmd)
769 return output.splitlines()
770 except Exception:
771 # Can be raised for empty repositories
772 return []
773
774 @reraise_safe_exceptions
682 def run_git_command(self, wire, cmd, **opts):
775 def run_git_command(self, wire, cmd, **opts):
683 path = wire.get('path', None)
776 path = wire.get('path', None)
684
777
@@ -98,6 +98,7 def make_ui_from_config(repo_config):
98
98
99 def reraise_safe_exceptions(func):
99 def reraise_safe_exceptions(func):
100 """Decorator for converting mercurial exceptions to something neutral."""
100 """Decorator for converting mercurial exceptions to something neutral."""
101
101 def wrapper(*args, **kwargs):
102 def wrapper(*args, **kwargs):
102 try:
103 try:
103 return func(*args, **kwargs)
104 return func(*args, **kwargs)
@@ -142,6 +143,23 class MercurialFactory(RepoFactory):
142 baseui = self._create_config(wire["config"])
143 baseui = self._create_config(wire["config"])
143 return instance(baseui, wire["path"], create)
144 return instance(baseui, wire["path"], create)
144
145
146 def repo(self, wire, create=False):
147 """
148 Get a repository instance for the given path.
149 """
150 region = self._cache_region
151 context = wire.get('context', None)
152 repo_path = wire.get('path', '')
153 context_uid = '{}'.format(context)
154 cache = wire.get('cache', True)
155 cache_on = context and cache
156
157 @region.conditional_cache_on_arguments(condition=cache_on)
158 def create_new_repo(_repo_type, _repo_path, _context_uid):
159 return self._create_repo(wire, create)
160
161 return create_new_repo(self.repo_type, repo_path, context_uid)
162
145
163
146 class HgRemote(object):
164 class HgRemote(object):
147
165
@@ -354,6 +354,7 class HTTPApplication(object):
354
354
355 log.debug('method called:%s with kwargs:%s context_uid: %s',
355 log.debug('method called:%s with kwargs:%s context_uid: %s',
356 method, kwargs, context_uid)
356 method, kwargs, context_uid)
357
357 try:
358 try:
358 resp = getattr(remote, method)(*args, **kwargs)
359 resp = getattr(remote, method)(*args, **kwargs)
359 except Exception as e:
360 except Exception as e:
@@ -97,9 +97,6 class SubversionFactory(RepoFactory):
97 def repo(self, wire, create=False, compatible_version=None):
97 def repo(self, wire, create=False, compatible_version=None):
98 """
98 """
99 Get a repository instance for the given path.
99 Get a repository instance for the given path.
100
101 Uses internally the low level beaker API since the decorators introduce
102 significant overhead.
103 """
100 """
104 region = self._cache_region
101 region = self._cache_region
105 context = wire.get('context', None)
102 context = wire.get('context', None)
@@ -99,14 +99,9 class TestGitFetch(object):
99 mock_repo().get_refs.assert_called_once_with()
99 mock_repo().get_refs.assert_called_once_with()
100 assert remote_refs == sample_refs
100 assert remote_refs == sample_refs
101
101
102 def test_remove_ref(self):
103 ref_to_remove = 'refs/tags/v0.1.9'
104 self.mock_repo.refs = SAMPLE_REFS.copy()
105 self.remote_git.remove_ref(None, ref_to_remove)
106 assert ref_to_remove not in self.mock_repo.refs
107
108
102
109 class TestReraiseSafeExceptions(object):
103 class TestReraiseSafeExceptions(object):
104
110 def test_method_decorated_with_reraise_safe_exceptions(self):
105 def test_method_decorated_with_reraise_safe_exceptions(self):
111 factory = Mock()
106 factory = Mock()
112 git_remote = git.GitRemote(factory)
107 git_remote = git.GitRemote(factory)
General Comments 0
You need to be logged in to leave comments. Login now