##// END OF EJS Templates
libgit2: updated code for tests....
marcink -
r730:ef69988e default
parent child Browse files
Show More
@@ -1,868 +1,873 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2019 RhodeCode GmbH
2 # Copyright (C) 2014-2019 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 import collections
17 import collections
18 import logging
18 import logging
19 import os
19 import os
20 import posixpath as vcspath
20 import posixpath as vcspath
21 import re
21 import re
22 import stat
22 import stat
23 import traceback
23 import traceback
24 import urllib
24 import urllib
25 import urllib2
25 import urllib2
26 from functools import wraps
26 from functools import wraps
27
27
28 import more_itertools
28 import more_itertools
29 import pygit2
29 import pygit2
30 from pygit2 import Repository as LibGit2Repo
30 from pygit2 import Repository as LibGit2Repo
31 from dulwich import index, objects
31 from dulwich import index, objects
32 from dulwich.client import HttpGitClient, LocalGitClient
32 from dulwich.client import HttpGitClient, LocalGitClient
33 from dulwich.errors import (
33 from dulwich.errors import (
34 NotGitRepository, ChecksumMismatch, WrongObjectException,
34 NotGitRepository, ChecksumMismatch, WrongObjectException,
35 MissingCommitError, ObjectMissing, HangupException,
35 MissingCommitError, ObjectMissing, HangupException,
36 UnexpectedCommandError)
36 UnexpectedCommandError)
37 from dulwich.repo import Repo as DulwichRepo
37 from dulwich.repo import Repo as DulwichRepo
38 from dulwich.server import update_server_info
38 from dulwich.server import update_server_info
39
39
40 from vcsserver import exceptions, settings, subprocessio
40 from vcsserver import exceptions, settings, subprocessio
41 from vcsserver.utils import safe_str
41 from vcsserver.utils import safe_str
42 from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original
42 from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original
43 from vcsserver.hgcompat import (
43 from vcsserver.hgcompat import (
44 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
44 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
45 from vcsserver.git_lfs.lib import LFSOidStore
45 from vcsserver.git_lfs.lib import LFSOidStore
46
46
47 DIR_STAT = stat.S_IFDIR
47 DIR_STAT = stat.S_IFDIR
48 FILE_MODE = stat.S_IFMT
48 FILE_MODE = stat.S_IFMT
49 GIT_LINK = objects.S_IFGITLINK
49 GIT_LINK = objects.S_IFGITLINK
50
50
51 log = logging.getLogger(__name__)
51 log = logging.getLogger(__name__)
52
52
53
53
54 def reraise_safe_exceptions(func):
54 def reraise_safe_exceptions(func):
55 """Converts Dulwich exceptions to something neutral."""
55 """Converts Dulwich exceptions to something neutral."""
56
56
57 @wraps(func)
57 @wraps(func)
58 def wrapper(*args, **kwargs):
58 def wrapper(*args, **kwargs):
59 try:
59 try:
60 return func(*args, **kwargs)
60 return func(*args, **kwargs)
61 except (ChecksumMismatch, WrongObjectException, MissingCommitError, ObjectMissing,) as e:
61 except (ChecksumMismatch, WrongObjectException, MissingCommitError, ObjectMissing,) as e:
62 exc = exceptions.LookupException(org_exc=e)
62 exc = exceptions.LookupException(org_exc=e)
63 raise exc(safe_str(e))
63 raise exc(safe_str(e))
64 except (HangupException, UnexpectedCommandError) as e:
64 except (HangupException, UnexpectedCommandError) as e:
65 exc = exceptions.VcsException(org_exc=e)
65 exc = exceptions.VcsException(org_exc=e)
66 raise exc(safe_str(e))
66 raise exc(safe_str(e))
67 except Exception as e:
67 except Exception as e:
68 # NOTE(marcink): becuase of how dulwich handles some exceptions
68 # NOTE(marcink): becuase of how dulwich handles some exceptions
69 # (KeyError on empty repos), we cannot track this and catch all
69 # (KeyError on empty repos), we cannot track this and catch all
70 # exceptions, it's an exceptions from other handlers
70 # exceptions, it's an exceptions from other handlers
71 #if not hasattr(e, '_vcs_kind'):
71 #if not hasattr(e, '_vcs_kind'):
72 #log.exception("Unhandled exception in git remote call")
72 #log.exception("Unhandled exception in git remote call")
73 #raise_from_original(exceptions.UnhandledException)
73 #raise_from_original(exceptions.UnhandledException)
74 raise
74 raise
75 return wrapper
75 return wrapper
76
76
77
77
78 class Repo(DulwichRepo):
78 class Repo(DulwichRepo):
79 """
79 """
80 A wrapper for dulwich Repo class.
80 A wrapper for dulwich Repo class.
81
81
82 Since dulwich is sometimes keeping .idx file descriptors open, it leads to
82 Since dulwich is sometimes keeping .idx file descriptors open, it leads to
83 "Too many open files" error. We need to close all opened file descriptors
83 "Too many open files" error. We need to close all opened file descriptors
84 once the repo object is destroyed.
84 once the repo object is destroyed.
85 """
85 """
86 def __del__(self):
86 def __del__(self):
87 if hasattr(self, 'object_store'):
87 if hasattr(self, 'object_store'):
88 self.close()
88 self.close()
89
89
90
90
91 class Repository(LibGit2Repo):
91 class Repository(LibGit2Repo):
92
92
93 def __enter__(self):
93 def __enter__(self):
94 return self
94 return self
95
95
96 def __exit__(self, exc_type, exc_val, exc_tb):
96 def __exit__(self, exc_type, exc_val, exc_tb):
97 self.free()
97 self.free()
98
98
99
99
100 class GitFactory(RepoFactory):
100 class GitFactory(RepoFactory):
101 repo_type = 'git'
101 repo_type = 'git'
102
102
103 def _create_repo(self, wire, create, use_libgit2=False):
103 def _create_repo(self, wire, create, use_libgit2=False):
104 if use_libgit2:
104 if use_libgit2:
105 return Repository(wire['path'])
105 return Repository(wire['path'])
106 else:
106 else:
107 repo_path = str_to_dulwich(wire['path'])
107 repo_path = str_to_dulwich(wire['path'])
108 return Repo(repo_path)
108 return Repo(repo_path)
109
109
110 def repo(self, wire, create=False, use_libgit2=False):
110 def repo(self, wire, create=False, use_libgit2=False):
111 """
111 """
112 Get a repository instance for the given path.
112 Get a repository instance for the given path.
113 """
113 """
114 region = self._cache_region
114 region = self._cache_region
115 context = wire.get('context', None)
115 context = wire.get('context', None)
116 repo_path = wire.get('path', '')
116 repo_path = wire.get('path', '')
117 context_uid = '{}'.format(context)
117 context_uid = '{}'.format(context)
118 cache = wire.get('cache', True)
118 cache = wire.get('cache', True)
119 cache_on = context and cache
119 cache_on = context and cache
120
120
121 @region.conditional_cache_on_arguments(condition=cache_on)
121 @region.conditional_cache_on_arguments(condition=cache_on)
122 def create_new_repo(_repo_type, _repo_path, _context_uid, _use_libgit2):
122 def create_new_repo(_repo_type, _repo_path, _context_uid, _use_libgit2):
123 return self._create_repo(wire, create, use_libgit2)
123 return self._create_repo(wire, create, use_libgit2)
124
124
125 repo = create_new_repo(self.repo_type, repo_path, context_uid, use_libgit2)
125 repo = create_new_repo(self.repo_type, repo_path, context_uid, use_libgit2)
126 return repo
126 return repo
127
127
128 def repo_libgit2(self, wire):
128 def repo_libgit2(self, wire):
129 return self.repo(wire, use_libgit2=True)
129 return self.repo(wire, use_libgit2=True)
130
130
131
131
132 class GitRemote(object):
132 class GitRemote(object):
133
133
134 def __init__(self, factory):
134 def __init__(self, factory):
135 self._factory = factory
135 self._factory = factory
136 self.peeled_ref_marker = '^{}'
136 self.peeled_ref_marker = '^{}'
137 self._bulk_methods = {
137 self._bulk_methods = {
138 "date": self.date,
138 "date": self.date,
139 "author": self.author,
139 "author": self.author,
140 "message": self.message,
140 "message": self.message,
141 "parents": self.parents,
141 "parents": self.parents,
142 "_commit": self.revision,
142 "_commit": self.revision,
143 }
143 }
144
144
145 def _wire_to_config(self, wire):
145 def _wire_to_config(self, wire):
146 if 'config' in wire:
146 if 'config' in wire:
147 return dict([(x[0] + '_' + x[1], x[2]) for x in wire['config']])
147 return dict([(x[0] + '_' + x[1], x[2]) for x in wire['config']])
148 return {}
148 return {}
149
149
150 def _remote_conf(self, config):
150 def _remote_conf(self, config):
151 params = [
151 params = [
152 '-c', 'core.askpass=""',
152 '-c', 'core.askpass=""',
153 ]
153 ]
154 ssl_cert_dir = config.get('vcs_ssl_dir')
154 ssl_cert_dir = config.get('vcs_ssl_dir')
155 if ssl_cert_dir:
155 if ssl_cert_dir:
156 params.extend(['-c', 'http.sslCAinfo={}'.format(ssl_cert_dir)])
156 params.extend(['-c', 'http.sslCAinfo={}'.format(ssl_cert_dir)])
157 return params
157 return params
158
158
159 @reraise_safe_exceptions
159 @reraise_safe_exceptions
160 def is_empty(self, wire):
160 def is_empty(self, wire):
161 repo = self._factory.repo_libgit2(wire)
161 repo = self._factory.repo_libgit2(wire)
162
162
163 try:
163 try:
164 return not repo.head.name
164 has_head = repo.head.name
165 if has_head:
166 return False
167
168 # NOTE(marcink): check again using more expensive method
169 return repo.is_empty
165 except Exception:
170 except Exception:
166 return True
171 pass
172
173 return True
167
174
168 @reraise_safe_exceptions
175 @reraise_safe_exceptions
169 def add_object(self, wire, content):
176 def add_object(self, wire, content):
170 repo = self._factory.repo(wire)
177 repo = self._factory.repo(wire)
171 blob = objects.Blob()
178 blob = objects.Blob()
172 blob.set_raw_string(content)
179 blob.set_raw_string(content)
173 repo.object_store.add_object(blob)
180 repo.object_store.add_object(blob)
174 return blob.id
181 return blob.id
175
182
176 @reraise_safe_exceptions
183 @reraise_safe_exceptions
177 def assert_correct_path(self, wire):
184 def assert_correct_path(self, wire):
178 try:
185 try:
179 self._factory.repo_libgit2(wire)
186 self._factory.repo_libgit2(wire)
180 except pygit2.GitError:
187 except pygit2.GitError:
181 path = wire.get('path')
188 path = wire.get('path')
182 tb = traceback.format_exc()
189 tb = traceback.format_exc()
183 log.debug("Invalid Git path `%s`, tb: %s", path, tb)
190 log.debug("Invalid Git path `%s`, tb: %s", path, tb)
184 return False
191 return False
185
192
186 return True
193 return True
187
194
188 @reraise_safe_exceptions
195 @reraise_safe_exceptions
189 def bare(self, wire):
196 def bare(self, wire):
190 repo = self._factory.repo_libgit2(wire)
197 repo = self._factory.repo_libgit2(wire)
191 return repo.is_bare
198 return repo.is_bare
192
199
193 @reraise_safe_exceptions
200 @reraise_safe_exceptions
194 def blob_as_pretty_string(self, wire, sha):
201 def blob_as_pretty_string(self, wire, sha):
195 repo_init = self._factory.repo_libgit2(wire)
202 repo_init = self._factory.repo_libgit2(wire)
196 with repo_init as repo:
203 with repo_init as repo:
197 blob_obj = repo[sha]
204 blob_obj = repo[sha]
198 blob = blob_obj.data
205 blob = blob_obj.data
199 return blob
206 return blob
200
207
201 @reraise_safe_exceptions
208 @reraise_safe_exceptions
202 def blob_raw_length(self, wire, sha):
209 def blob_raw_length(self, wire, sha):
203 repo_init = self._factory.repo_libgit2(wire)
210 repo_init = self._factory.repo_libgit2(wire)
204 with repo_init as repo:
211 with repo_init as repo:
205 blob = repo[sha]
212 blob = repo[sha]
206 return blob.size
213 return blob.size
207
214
208 def _parse_lfs_pointer(self, raw_content):
215 def _parse_lfs_pointer(self, raw_content):
209
216
210 spec_string = 'version https://git-lfs.github.com/spec'
217 spec_string = 'version https://git-lfs.github.com/spec'
211 if raw_content and raw_content.startswith(spec_string):
218 if raw_content and raw_content.startswith(spec_string):
212 pattern = re.compile(r"""
219 pattern = re.compile(r"""
213 (?:\n)?
220 (?:\n)?
214 ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
221 ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
215 ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
222 ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
216 ^size[ ](?P<oid_size>[0-9]+)\n
223 ^size[ ](?P<oid_size>[0-9]+)\n
217 (?:\n)?
224 (?:\n)?
218 """, re.VERBOSE | re.MULTILINE)
225 """, re.VERBOSE | re.MULTILINE)
219 match = pattern.match(raw_content)
226 match = pattern.match(raw_content)
220 if match:
227 if match:
221 return match.groupdict()
228 return match.groupdict()
222
229
223 return {}
230 return {}
224
231
225 @reraise_safe_exceptions
232 @reraise_safe_exceptions
226 def is_large_file(self, wire, sha):
233 def is_large_file(self, wire, sha):
227 repo_init = self._factory.repo_libgit2(wire)
234 repo_init = self._factory.repo_libgit2(wire)
228
235
229 with repo_init as repo:
236 with repo_init as repo:
230 blob = repo[sha]
237 blob = repo[sha]
231 if blob.is_binary:
238 if blob.is_binary:
232 return {}
239 return {}
233
240
234 return self._parse_lfs_pointer(blob.data)
241 return self._parse_lfs_pointer(blob.data)
235
242
236 @reraise_safe_exceptions
243 @reraise_safe_exceptions
237 def in_largefiles_store(self, wire, oid):
244 def in_largefiles_store(self, wire, oid):
238 repo = self._factory.repo_libgit2(wire)
245 repo = self._factory.repo_libgit2(wire)
239 conf = self._wire_to_config(wire)
246 conf = self._wire_to_config(wire)
240
247
241 store_location = conf.get('vcs_git_lfs_store_location')
248 store_location = conf.get('vcs_git_lfs_store_location')
242 if store_location:
249 if store_location:
243 repo_name = repo.path
250 repo_name = repo.path
244 store = LFSOidStore(
251 store = LFSOidStore(
245 oid=oid, repo=repo_name, store_location=store_location)
252 oid=oid, repo=repo_name, store_location=store_location)
246 return store.has_oid()
253 return store.has_oid()
247
254
248 return False
255 return False
249
256
250 @reraise_safe_exceptions
257 @reraise_safe_exceptions
251 def store_path(self, wire, oid):
258 def store_path(self, wire, oid):
252 repo = self._factory.repo_libgit2(wire)
259 repo = self._factory.repo_libgit2(wire)
253 conf = self._wire_to_config(wire)
260 conf = self._wire_to_config(wire)
254
261
255 store_location = conf.get('vcs_git_lfs_store_location')
262 store_location = conf.get('vcs_git_lfs_store_location')
256 if store_location:
263 if store_location:
257 repo_name = repo.path
264 repo_name = repo.path
258 store = LFSOidStore(
265 store = LFSOidStore(
259 oid=oid, repo=repo_name, store_location=store_location)
266 oid=oid, repo=repo_name, store_location=store_location)
260 return store.oid_path
267 return store.oid_path
261 raise ValueError('Unable to fetch oid with path {}'.format(oid))
268 raise ValueError('Unable to fetch oid with path {}'.format(oid))
262
269
263 @reraise_safe_exceptions
270 @reraise_safe_exceptions
264 def bulk_request(self, wire, rev, pre_load):
271 def bulk_request(self, wire, rev, pre_load):
265 result = {}
272 result = {}
266 for attr in pre_load:
273 for attr in pre_load:
267 try:
274 try:
268 method = self._bulk_methods[attr]
275 method = self._bulk_methods[attr]
269 args = [wire, rev]
276 args = [wire, rev]
270 result[attr] = method(*args)
277 result[attr] = method(*args)
271 except KeyError as e:
278 except KeyError as e:
272 raise exceptions.VcsException(e)("Unknown bulk attribute: %s" % attr)
279 raise exceptions.VcsException(e)("Unknown bulk attribute: %s" % attr)
273 return result
280 return result
274
281
275 def _build_opener(self, url):
282 def _build_opener(self, url):
276 handlers = []
283 handlers = []
277 url_obj = url_parser(url)
284 url_obj = url_parser(url)
278 _, authinfo = url_obj.authinfo()
285 _, authinfo = url_obj.authinfo()
279
286
280 if authinfo:
287 if authinfo:
281 # create a password manager
288 # create a password manager
282 passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
289 passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
283 passmgr.add_password(*authinfo)
290 passmgr.add_password(*authinfo)
284
291
285 handlers.extend((httpbasicauthhandler(passmgr),
292 handlers.extend((httpbasicauthhandler(passmgr),
286 httpdigestauthhandler(passmgr)))
293 httpdigestauthhandler(passmgr)))
287
294
288 return urllib2.build_opener(*handlers)
295 return urllib2.build_opener(*handlers)
289
296
290 def _type_id_to_name(self, type_id):
297 def _type_id_to_name(self, type_id):
291 return {
298 return {
292 1: b'commit',
299 1: b'commit',
293 2: b'tree',
300 2: b'tree',
294 3: b'blob',
301 3: b'blob',
295 4: b'tag'
302 4: b'tag'
296 }[type_id]
303 }[type_id]
297
304
298 @reraise_safe_exceptions
305 @reraise_safe_exceptions
299 def check_url(self, url, config):
306 def check_url(self, url, config):
300 url_obj = url_parser(url)
307 url_obj = url_parser(url)
301 test_uri, _ = url_obj.authinfo()
308 test_uri, _ = url_obj.authinfo()
302 url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
309 url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
303 url_obj.query = obfuscate_qs(url_obj.query)
310 url_obj.query = obfuscate_qs(url_obj.query)
304 cleaned_uri = str(url_obj)
311 cleaned_uri = str(url_obj)
305 log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
312 log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
306
313
307 if not test_uri.endswith('info/refs'):
314 if not test_uri.endswith('info/refs'):
308 test_uri = test_uri.rstrip('/') + '/info/refs'
315 test_uri = test_uri.rstrip('/') + '/info/refs'
309
316
310 o = self._build_opener(url)
317 o = self._build_opener(url)
311 o.addheaders = [('User-Agent', 'git/1.7.8.0')] # fake some git
318 o.addheaders = [('User-Agent', 'git/1.7.8.0')] # fake some git
312
319
313 q = {"service": 'git-upload-pack'}
320 q = {"service": 'git-upload-pack'}
314 qs = '?%s' % urllib.urlencode(q)
321 qs = '?%s' % urllib.urlencode(q)
315 cu = "%s%s" % (test_uri, qs)
322 cu = "%s%s" % (test_uri, qs)
316 req = urllib2.Request(cu, None, {})
323 req = urllib2.Request(cu, None, {})
317
324
318 try:
325 try:
319 log.debug("Trying to open URL %s", cleaned_uri)
326 log.debug("Trying to open URL %s", cleaned_uri)
320 resp = o.open(req)
327 resp = o.open(req)
321 if resp.code != 200:
328 if resp.code != 200:
322 raise exceptions.URLError()('Return Code is not 200')
329 raise exceptions.URLError()('Return Code is not 200')
323 except Exception as e:
330 except Exception as e:
324 log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
331 log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
325 # means it cannot be cloned
332 # means it cannot be cloned
326 raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e))
333 raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e))
327
334
328 # now detect if it's proper git repo
335 # now detect if it's proper git repo
329 gitdata = resp.read()
336 gitdata = resp.read()
330 if 'service=git-upload-pack' in gitdata:
337 if 'service=git-upload-pack' in gitdata:
331 pass
338 pass
332 elif re.findall(r'[0-9a-fA-F]{40}\s+refs', gitdata):
339 elif re.findall(r'[0-9a-fA-F]{40}\s+refs', gitdata):
333 # old style git can return some other format !
340 # old style git can return some other format !
334 pass
341 pass
335 else:
342 else:
336 raise exceptions.URLError()(
343 raise exceptions.URLError()(
337 "url [%s] does not look like an git" % (cleaned_uri,))
344 "url [%s] does not look like an git" % (cleaned_uri,))
338
345
339 return True
346 return True
340
347
341 @reraise_safe_exceptions
348 @reraise_safe_exceptions
342 def clone(self, wire, url, deferred, valid_refs, update_after_clone):
349 def clone(self, wire, url, deferred, valid_refs, update_after_clone):
343 # TODO(marcink): deprecate this method. Last i checked we don't use it anymore
350 # TODO(marcink): deprecate this method. Last i checked we don't use it anymore
344 remote_refs = self.pull(wire, url, apply_refs=False)
351 remote_refs = self.pull(wire, url, apply_refs=False)
345 repo = self._factory.repo(wire)
352 repo = self._factory.repo(wire)
346 if isinstance(valid_refs, list):
353 if isinstance(valid_refs, list):
347 valid_refs = tuple(valid_refs)
354 valid_refs = tuple(valid_refs)
348
355
349 for k in remote_refs:
356 for k in remote_refs:
350 # only parse heads/tags and skip so called deferred tags
357 # only parse heads/tags and skip so called deferred tags
351 if k.startswith(valid_refs) and not k.endswith(deferred):
358 if k.startswith(valid_refs) and not k.endswith(deferred):
352 repo[k] = remote_refs[k]
359 repo[k] = remote_refs[k]
353
360
354 if update_after_clone:
361 if update_after_clone:
355 # we want to checkout HEAD
362 # we want to checkout HEAD
356 repo["HEAD"] = remote_refs["HEAD"]
363 repo["HEAD"] = remote_refs["HEAD"]
357 index.build_index_from_tree(repo.path, repo.index_path(),
364 index.build_index_from_tree(repo.path, repo.index_path(),
358 repo.object_store, repo["HEAD"].tree)
365 repo.object_store, repo["HEAD"].tree)
359
366
360 # TODO: this is quite complex, check if that can be simplified
367 # TODO: this is quite complex, check if that can be simplified
361 @reraise_safe_exceptions
368 @reraise_safe_exceptions
362 def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
369 def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
363 repo = self._factory.repo(wire)
370 repo = self._factory.repo(wire)
364 object_store = repo.object_store
371 object_store = repo.object_store
365
372
366 # Create tree and populates it with blobs
373 # Create tree and populates it with blobs
367 commit_tree = commit_tree and repo[commit_tree] or objects.Tree()
374 commit_tree = commit_tree and repo[commit_tree] or objects.Tree()
368
375
369 for node in updated:
376 for node in updated:
370 # Compute subdirs if needed
377 # Compute subdirs if needed
371 dirpath, nodename = vcspath.split(node['path'])
378 dirpath, nodename = vcspath.split(node['path'])
372 dirnames = map(safe_str, dirpath and dirpath.split('/') or [])
379 dirnames = map(safe_str, dirpath and dirpath.split('/') or [])
373 parent = commit_tree
380 parent = commit_tree
374 ancestors = [('', parent)]
381 ancestors = [('', parent)]
375
382
376 # Tries to dig for the deepest existing tree
383 # Tries to dig for the deepest existing tree
377 while dirnames:
384 while dirnames:
378 curdir = dirnames.pop(0)
385 curdir = dirnames.pop(0)
379 try:
386 try:
380 dir_id = parent[curdir][1]
387 dir_id = parent[curdir][1]
381 except KeyError:
388 except KeyError:
382 # put curdir back into dirnames and stops
389 # put curdir back into dirnames and stops
383 dirnames.insert(0, curdir)
390 dirnames.insert(0, curdir)
384 break
391 break
385 else:
392 else:
386 # If found, updates parent
393 # If found, updates parent
387 parent = repo[dir_id]
394 parent = repo[dir_id]
388 ancestors.append((curdir, parent))
395 ancestors.append((curdir, parent))
389 # Now parent is deepest existing tree and we need to create
396 # Now parent is deepest existing tree and we need to create
390 # subtrees for dirnames (in reverse order)
397 # subtrees for dirnames (in reverse order)
391 # [this only applies for nodes from added]
398 # [this only applies for nodes from added]
392 new_trees = []
399 new_trees = []
393
400
394 blob = objects.Blob.from_string(node['content'])
401 blob = objects.Blob.from_string(node['content'])
395
402
396 if dirnames:
403 if dirnames:
397 # If there are trees which should be created we need to build
404 # If there are trees which should be created we need to build
398 # them now (in reverse order)
405 # them now (in reverse order)
399 reversed_dirnames = list(reversed(dirnames))
406 reversed_dirnames = list(reversed(dirnames))
400 curtree = objects.Tree()
407 curtree = objects.Tree()
401 curtree[node['node_path']] = node['mode'], blob.id
408 curtree[node['node_path']] = node['mode'], blob.id
402 new_trees.append(curtree)
409 new_trees.append(curtree)
403 for dirname in reversed_dirnames[:-1]:
410 for dirname in reversed_dirnames[:-1]:
404 newtree = objects.Tree()
411 newtree = objects.Tree()
405 newtree[dirname] = (DIR_STAT, curtree.id)
412 newtree[dirname] = (DIR_STAT, curtree.id)
406 new_trees.append(newtree)
413 new_trees.append(newtree)
407 curtree = newtree
414 curtree = newtree
408 parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
415 parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
409 else:
416 else:
410 parent.add(name=node['node_path'], mode=node['mode'], hexsha=blob.id)
417 parent.add(name=node['node_path'], mode=node['mode'], hexsha=blob.id)
411
418
412 new_trees.append(parent)
419 new_trees.append(parent)
413 # Update ancestors
420 # Update ancestors
414 reversed_ancestors = reversed(
421 reversed_ancestors = reversed(
415 [(a[1], b[1], b[0]) for a, b in zip(ancestors, ancestors[1:])])
422 [(a[1], b[1], b[0]) for a, b in zip(ancestors, ancestors[1:])])
416 for parent, tree, path in reversed_ancestors:
423 for parent, tree, path in reversed_ancestors:
417 parent[path] = (DIR_STAT, tree.id)
424 parent[path] = (DIR_STAT, tree.id)
418 object_store.add_object(tree)
425 object_store.add_object(tree)
419
426
420 object_store.add_object(blob)
427 object_store.add_object(blob)
421 for tree in new_trees:
428 for tree in new_trees:
422 object_store.add_object(tree)
429 object_store.add_object(tree)
423
430
424 for node_path in removed:
431 for node_path in removed:
425 paths = node_path.split('/')
432 paths = node_path.split('/')
426 tree = commit_tree
433 tree = commit_tree
427 trees = [tree]
434 trees = [tree]
428 # Traverse deep into the forest...
435 # Traverse deep into the forest...
429 for path in paths:
436 for path in paths:
430 try:
437 try:
431 obj = repo[tree[path][1]]
438 obj = repo[tree[path][1]]
432 if isinstance(obj, objects.Tree):
439 if isinstance(obj, objects.Tree):
433 trees.append(obj)
440 trees.append(obj)
434 tree = obj
441 tree = obj
435 except KeyError:
442 except KeyError:
436 break
443 break
437 # Cut down the blob and all rotten trees on the way back...
444 # Cut down the blob and all rotten trees on the way back...
438 for path, tree in reversed(zip(paths, trees)):
445 for path, tree in reversed(zip(paths, trees)):
439 del tree[path]
446 del tree[path]
440 if tree:
447 if tree:
441 # This tree still has elements - don't remove it or any
448 # This tree still has elements - don't remove it or any
442 # of it's parents
449 # of it's parents
443 break
450 break
444
451
445 object_store.add_object(commit_tree)
452 object_store.add_object(commit_tree)
446
453
447 # Create commit
454 # Create commit
448 commit = objects.Commit()
455 commit = objects.Commit()
449 commit.tree = commit_tree.id
456 commit.tree = commit_tree.id
450 for k, v in commit_data.iteritems():
457 for k, v in commit_data.iteritems():
451 setattr(commit, k, v)
458 setattr(commit, k, v)
452 object_store.add_object(commit)
459 object_store.add_object(commit)
453
460
454 self.create_branch(wire, branch, commit.id)
461 self.create_branch(wire, branch, commit.id)
455
462
456 # dulwich set-ref
463 # dulwich set-ref
457 ref = 'refs/heads/%s' % branch
464 ref = 'refs/heads/%s' % branch
458 repo.refs[ref] = commit.id
465 repo.refs[ref] = commit.id
459
466
460 return commit.id
467 return commit.id
461
468
462 @reraise_safe_exceptions
469 @reraise_safe_exceptions
463 def pull(self, wire, url, apply_refs=True, refs=None, update_after=False):
470 def pull(self, wire, url, apply_refs=True, refs=None, update_after=False):
464 if url != 'default' and '://' not in url:
471 if url != 'default' and '://' not in url:
465 client = LocalGitClient(url)
472 client = LocalGitClient(url)
466 else:
473 else:
467 url_obj = url_parser(url)
474 url_obj = url_parser(url)
468 o = self._build_opener(url)
475 o = self._build_opener(url)
469 url, _ = url_obj.authinfo()
476 url, _ = url_obj.authinfo()
470 client = HttpGitClient(base_url=url, opener=o)
477 client = HttpGitClient(base_url=url, opener=o)
471 repo = self._factory.repo(wire)
478 repo = self._factory.repo(wire)
472
479
473 determine_wants = repo.object_store.determine_wants_all
480 determine_wants = repo.object_store.determine_wants_all
474 if refs:
481 if refs:
475 def determine_wants_requested(references):
482 def determine_wants_requested(references):
476 return [references[r] for r in references if r in refs]
483 return [references[r] for r in references if r in refs]
477 determine_wants = determine_wants_requested
484 determine_wants = determine_wants_requested
478
485
479 try:
486 try:
480 remote_refs = client.fetch(
487 remote_refs = client.fetch(
481 path=url, target=repo, determine_wants=determine_wants)
488 path=url, target=repo, determine_wants=determine_wants)
482 except NotGitRepository as e:
489 except NotGitRepository as e:
483 log.warning(
490 log.warning(
484 'Trying to fetch from "%s" failed, not a Git repository.', url)
491 'Trying to fetch from "%s" failed, not a Git repository.', url)
485 # Exception can contain unicode which we convert
492 # Exception can contain unicode which we convert
486 raise exceptions.AbortException(e)(repr(e))
493 raise exceptions.AbortException(e)(repr(e))
487
494
488 # mikhail: client.fetch() returns all the remote refs, but fetches only
495 # mikhail: client.fetch() returns all the remote refs, but fetches only
489 # refs filtered by `determine_wants` function. We need to filter result
496 # refs filtered by `determine_wants` function. We need to filter result
490 # as well
497 # as well
491 if refs:
498 if refs:
492 remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
499 remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
493
500
494 if apply_refs:
501 if apply_refs:
495 # TODO: johbo: Needs proper test coverage with a git repository
502 # TODO: johbo: Needs proper test coverage with a git repository
496 # that contains a tag object, so that we would end up with
503 # that contains a tag object, so that we would end up with
497 # a peeled ref at this point.
504 # a peeled ref at this point.
498 for k in remote_refs:
505 for k in remote_refs:
499 if k.endswith(self.peeled_ref_marker):
506 if k.endswith(self.peeled_ref_marker):
500 log.debug("Skipping peeled reference %s", k)
507 log.debug("Skipping peeled reference %s", k)
501 continue
508 continue
502 repo[k] = remote_refs[k]
509 repo[k] = remote_refs[k]
503
510
504 if refs and not update_after:
511 if refs and not update_after:
505 # mikhail: explicitly set the head to the last ref.
512 # mikhail: explicitly set the head to the last ref.
506 repo['HEAD'] = remote_refs[refs[-1]]
513 repo['HEAD'] = remote_refs[refs[-1]]
507
514
508 if update_after:
515 if update_after:
509 # we want to checkout HEAD
516 # we want to checkout HEAD
510 repo["HEAD"] = remote_refs["HEAD"]
517 repo["HEAD"] = remote_refs["HEAD"]
511 index.build_index_from_tree(repo.path, repo.index_path(),
518 index.build_index_from_tree(repo.path, repo.index_path(),
512 repo.object_store, repo["HEAD"].tree)
519 repo.object_store, repo["HEAD"].tree)
513 return remote_refs
520 return remote_refs
514
521
515 @reraise_safe_exceptions
522 @reraise_safe_exceptions
516 def sync_fetch(self, wire, url, refs=None):
523 def sync_fetch(self, wire, url, refs=None):
517 repo = self._factory.repo(wire)
524 repo = self._factory.repo(wire)
518 if refs and not isinstance(refs, (list, tuple)):
525 if refs and not isinstance(refs, (list, tuple)):
519 refs = [refs]
526 refs = [refs]
520 config = self._wire_to_config(wire)
527 config = self._wire_to_config(wire)
521 # get all remote refs we'll use to fetch later
528 # get all remote refs we'll use to fetch later
522 output, __ = self.run_git_command(
529 output, __ = self.run_git_command(
523 wire, ['ls-remote', url], fail_on_stderr=False,
530 wire, ['ls-remote', url], fail_on_stderr=False,
524 _copts=self._remote_conf(config),
531 _copts=self._remote_conf(config),
525 extra_env={'GIT_TERMINAL_PROMPT': '0'})
532 extra_env={'GIT_TERMINAL_PROMPT': '0'})
526
533
527 remote_refs = collections.OrderedDict()
534 remote_refs = collections.OrderedDict()
528 fetch_refs = []
535 fetch_refs = []
529
536
530 for ref_line in output.splitlines():
537 for ref_line in output.splitlines():
531 sha, ref = ref_line.split('\t')
538 sha, ref = ref_line.split('\t')
532 sha = sha.strip()
539 sha = sha.strip()
533 if ref in remote_refs:
540 if ref in remote_refs:
534 # duplicate, skip
541 # duplicate, skip
535 continue
542 continue
536 if ref.endswith(self.peeled_ref_marker):
543 if ref.endswith(self.peeled_ref_marker):
537 log.debug("Skipping peeled reference %s", ref)
544 log.debug("Skipping peeled reference %s", ref)
538 continue
545 continue
539 # don't sync HEAD
546 # don't sync HEAD
540 if ref in ['HEAD']:
547 if ref in ['HEAD']:
541 continue
548 continue
542
549
543 remote_refs[ref] = sha
550 remote_refs[ref] = sha
544
551
545 if refs and sha in refs:
552 if refs and sha in refs:
546 # we filter fetch using our specified refs
553 # we filter fetch using our specified refs
547 fetch_refs.append('{}:{}'.format(ref, ref))
554 fetch_refs.append('{}:{}'.format(ref, ref))
548 elif not refs:
555 elif not refs:
549 fetch_refs.append('{}:{}'.format(ref, ref))
556 fetch_refs.append('{}:{}'.format(ref, ref))
550 log.debug('Finished obtaining fetch refs, total: %s', len(fetch_refs))
557 log.debug('Finished obtaining fetch refs, total: %s', len(fetch_refs))
551 if fetch_refs:
558 if fetch_refs:
552 for chunk in more_itertools.chunked(fetch_refs, 1024 * 4):
559 for chunk in more_itertools.chunked(fetch_refs, 1024 * 4):
553 fetch_refs_chunks = list(chunk)
560 fetch_refs_chunks = list(chunk)
554 log.debug('Fetching %s refs from import url', len(fetch_refs_chunks))
561 log.debug('Fetching %s refs from import url', len(fetch_refs_chunks))
555 _out, _err = self.run_git_command(
562 _out, _err = self.run_git_command(
556 wire, ['fetch', url, '--force', '--prune', '--'] + fetch_refs_chunks,
563 wire, ['fetch', url, '--force', '--prune', '--'] + fetch_refs_chunks,
557 fail_on_stderr=False,
564 fail_on_stderr=False,
558 _copts=self._remote_conf(config),
565 _copts=self._remote_conf(config),
559 extra_env={'GIT_TERMINAL_PROMPT': '0'})
566 extra_env={'GIT_TERMINAL_PROMPT': '0'})
560
567
561 return remote_refs
568 return remote_refs
562
569
563 @reraise_safe_exceptions
570 @reraise_safe_exceptions
564 def sync_push(self, wire, url, refs=None):
571 def sync_push(self, wire, url, refs=None):
565 if not self.check_url(url, wire):
572 if not self.check_url(url, wire):
566 return
573 return
567 config = self._wire_to_config(wire)
574 config = self._wire_to_config(wire)
568 repo = self._factory.repo(wire)
575 repo = self._factory.repo(wire)
569 self.run_git_command(
576 self.run_git_command(
570 wire, ['push', url, '--mirror'], fail_on_stderr=False,
577 wire, ['push', url, '--mirror'], fail_on_stderr=False,
571 _copts=self._remote_conf(config),
578 _copts=self._remote_conf(config),
572 extra_env={'GIT_TERMINAL_PROMPT': '0'})
579 extra_env={'GIT_TERMINAL_PROMPT': '0'})
573
580
574 @reraise_safe_exceptions
581 @reraise_safe_exceptions
575 def get_remote_refs(self, wire, url):
582 def get_remote_refs(self, wire, url):
576 repo = Repo(url)
583 repo = Repo(url)
577 return repo.get_refs()
584 return repo.get_refs()
578
585
579 @reraise_safe_exceptions
586 @reraise_safe_exceptions
580 def get_description(self, wire):
587 def get_description(self, wire):
581 repo = self._factory.repo(wire)
588 repo = self._factory.repo(wire)
582 return repo.get_description()
589 return repo.get_description()
583
590
584 @reraise_safe_exceptions
591 @reraise_safe_exceptions
585 def get_missing_revs(self, wire, rev1, rev2, path2):
592 def get_missing_revs(self, wire, rev1, rev2, path2):
586 repo = self._factory.repo(wire)
593 repo = self._factory.repo(wire)
587 LocalGitClient(thin_packs=False).fetch(path2, repo)
594 LocalGitClient(thin_packs=False).fetch(path2, repo)
588
595
589 wire_remote = wire.copy()
596 wire_remote = wire.copy()
590 wire_remote['path'] = path2
597 wire_remote['path'] = path2
591 repo_remote = self._factory.repo(wire_remote)
598 repo_remote = self._factory.repo(wire_remote)
592 LocalGitClient(thin_packs=False).fetch(wire["path"], repo_remote)
599 LocalGitClient(thin_packs=False).fetch(wire["path"], repo_remote)
593
600
594 revs = [
601 revs = [
595 x.commit.id
602 x.commit.id
596 for x in repo_remote.get_walker(include=[rev2], exclude=[rev1])]
603 for x in repo_remote.get_walker(include=[rev2], exclude=[rev1])]
597 return revs
604 return revs
598
605
599 @reraise_safe_exceptions
606 @reraise_safe_exceptions
600 def get_object(self, wire, sha):
607 def get_object(self, wire, sha):
601 repo = self._factory.repo_libgit2(wire)
608 repo = self._factory.repo_libgit2(wire)
602
609
603 missing_commit_err = 'Commit {} does not exist for `{}`'.format(sha, wire['path'])
610 missing_commit_err = 'Commit {} does not exist for `{}`'.format(sha, wire['path'])
604 try:
611 try:
605 commit = repo.revparse_single(sha)
612 commit = repo.revparse_single(sha)
606 except (KeyError, ValueError) as e:
613 except (KeyError, ValueError) as e:
607 raise exceptions.LookupException(e)(missing_commit_err)
614 raise exceptions.LookupException(e)(missing_commit_err)
608
615
609 if isinstance(commit, pygit2.Tag):
616 if isinstance(commit, pygit2.Tag):
610 commit = repo.get(commit.target)
617 commit = repo.get(commit.target)
611
618
612 # check for dangling commit
619 # check for dangling commit
613 branches = [x for x in repo.branches.with_commit(commit.hex)]
620 branches = [x for x in repo.branches.with_commit(commit.hex)]
614 if not branches:
621 if not branches:
615 raise exceptions.LookupException(None)(missing_commit_err)
622 raise exceptions.LookupException(None)(missing_commit_err)
616
623
617 commit_id = commit.hex
624 commit_id = commit.hex
618 type_id = commit.type
625 type_id = commit.type
619
626
620 return {
627 return {
621 'id': commit_id,
628 'id': commit_id,
622 'type': self._type_id_to_name(type_id),
629 'type': self._type_id_to_name(type_id),
623 'commit_id': commit_id,
630 'commit_id': commit_id,
624 'idx': 0
631 'idx': 0
625 }
632 }
626
633
627 @reraise_safe_exceptions
634 @reraise_safe_exceptions
628 def get_refs(self, wire):
635 def get_refs(self, wire):
629 repo = self._factory.repo_libgit2(wire)
636 repo = self._factory.repo_libgit2(wire)
630
637
631 result = {}
638 result = {}
632 for ref in repo.references:
639 for ref in repo.references:
633 peeled_sha = repo.lookup_reference(ref).peel()
640 peeled_sha = repo.lookup_reference(ref).peel()
634 result[ref] = peeled_sha.hex
641 result[ref] = peeled_sha.hex
635
642
636 return result
643 return result
637
644
638 @reraise_safe_exceptions
645 @reraise_safe_exceptions
639 def head(self, wire, show_exc=True):
646 def head(self, wire, show_exc=True):
640 repo = self._factory.repo_libgit2(wire)
647 repo = self._factory.repo_libgit2(wire)
641 try:
648 try:
642 return repo.head.peel().hex
649 return repo.head.peel().hex
643 except Exception:
650 except Exception:
644 if show_exc:
651 if show_exc:
645 raise
652 raise
646
653
647 @reraise_safe_exceptions
654 @reraise_safe_exceptions
648 def init(self, wire):
655 def init(self, wire):
649 repo_path = str_to_dulwich(wire['path'])
656 repo_path = str_to_dulwich(wire['path'])
650 self.repo = Repo.init(repo_path)
657 self.repo = Repo.init(repo_path)
651
658
652 @reraise_safe_exceptions
659 @reraise_safe_exceptions
653 def init_bare(self, wire):
660 def init_bare(self, wire):
654 repo_path = str_to_dulwich(wire['path'])
661 repo_path = str_to_dulwich(wire['path'])
655 self.repo = Repo.init_bare(repo_path)
662 self.repo = Repo.init_bare(repo_path)
656
663
657 @reraise_safe_exceptions
664 @reraise_safe_exceptions
658 def revision(self, wire, rev):
665 def revision(self, wire, rev):
659 repo = self._factory.repo_libgit2(wire)
666 repo = self._factory.repo_libgit2(wire)
660 commit = repo[rev]
667 commit = repo[rev]
661 obj_data = {
668 obj_data = {
662 'id': commit.id.hex,
669 'id': commit.id.hex,
663 }
670 }
664 # tree objects itself don't have tree_id attribute
671 # tree objects itself don't have tree_id attribute
665 if hasattr(commit, 'tree_id'):
672 if hasattr(commit, 'tree_id'):
666 obj_data['tree'] = commit.tree_id.hex
673 obj_data['tree'] = commit.tree_id.hex
667
674
668 return obj_data
675 return obj_data
669
676
670 @reraise_safe_exceptions
677 @reraise_safe_exceptions
671 def date(self, wire, rev):
678 def date(self, wire, rev):
672 repo = self._factory.repo_libgit2(wire)
679 repo = self._factory.repo_libgit2(wire)
673 commit = repo[rev]
680 commit = repo[rev]
674 # TODO(marcink): check dulwich difference of offset vs timezone
681 # TODO(marcink): check dulwich difference of offset vs timezone
675 return [commit.commit_time, commit.commit_time_offset]
682 return [commit.commit_time, commit.commit_time_offset]
676
683
677 @reraise_safe_exceptions
684 @reraise_safe_exceptions
678 def author(self, wire, rev):
685 def author(self, wire, rev):
679 repo = self._factory.repo_libgit2(wire)
686 repo = self._factory.repo_libgit2(wire)
680 commit = repo[rev]
687 commit = repo[rev]
681 if commit.author.email:
688 if commit.author.email:
682 return u"{} <{}>".format(commit.author.name, commit.author.email)
689 return u"{} <{}>".format(commit.author.name, commit.author.email)
683
690
684 return u"{}".format(commit.author.raw_name)
691 return u"{}".format(commit.author.raw_name)
685
692
686 @reraise_safe_exceptions
693 @reraise_safe_exceptions
687 def message(self, wire, rev):
694 def message(self, wire, rev):
688 repo = self._factory.repo_libgit2(wire)
695 repo = self._factory.repo_libgit2(wire)
689 commit = repo[rev]
696 commit = repo[rev]
690 return commit.message
697 return commit.message
691
698
692 @reraise_safe_exceptions
699 @reraise_safe_exceptions
693 def parents(self, wire, rev):
700 def parents(self, wire, rev):
694 repo = self._factory.repo_libgit2(wire)
701 repo = self._factory.repo_libgit2(wire)
695 commit = repo[rev]
702 commit = repo[rev]
696 return [x.hex for x in commit.parent_ids]
703 return [x.hex for x in commit.parent_ids]
697
704
698 @reraise_safe_exceptions
705 @reraise_safe_exceptions
699 def set_refs(self, wire, key, value):
706 def set_refs(self, wire, key, value):
700 repo = self._factory.repo_libgit2(wire)
707 repo = self._factory.repo_libgit2(wire)
701 repo.references.create(key, value, force=True)
708 repo.references.create(key, value, force=True)
702
709
703 @reraise_safe_exceptions
710 @reraise_safe_exceptions
704 def create_branch(self, wire, branch_name, commit_id, force=False):
711 def create_branch(self, wire, branch_name, commit_id, force=False):
705 repo = self._factory.repo_libgit2(wire)
712 repo = self._factory.repo_libgit2(wire)
706 commit = repo[commit_id]
713 commit = repo[commit_id]
707
714
708 if force:
715 if force:
709 repo.branches.local.create(branch_name, commit, force=force)
716 repo.branches.local.create(branch_name, commit, force=force)
710 elif not repo.branches.get(branch_name):
717 elif not repo.branches.get(branch_name):
711 # create only if that branch isn't existing
718 # create only if that branch isn't existing
712 repo.branches.local.create(branch_name, commit, force=force)
719 repo.branches.local.create(branch_name, commit, force=force)
713
720
714 @reraise_safe_exceptions
721 @reraise_safe_exceptions
715 def remove_ref(self, wire, key):
722 def remove_ref(self, wire, key):
716 repo = self._factory.repo_libgit2(wire)
723 repo = self._factory.repo_libgit2(wire)
717 repo.references.delete(key)
724 repo.references.delete(key)
718
725
719 @reraise_safe_exceptions
726 @reraise_safe_exceptions
720 def tag_remove(self, wire, tag_name):
727 def tag_remove(self, wire, tag_name):
721 repo = self._factory.repo_libgit2(wire)
728 repo = self._factory.repo_libgit2(wire)
722 key = 'refs/tags/{}'.format(tag_name)
729 key = 'refs/tags/{}'.format(tag_name)
723 repo.references.delete(key)
730 repo.references.delete(key)
724
731
725 @reraise_safe_exceptions
732 @reraise_safe_exceptions
726 def tree_changes(self, wire, source_id, target_id):
733 def tree_changes(self, wire, source_id, target_id):
727 # TODO(marcink): remove this seems it's only used by tests
734 # TODO(marcink): remove this seems it's only used by tests
728 repo = self._factory.repo(wire)
735 repo = self._factory.repo(wire)
729 source = repo[source_id].tree if source_id else None
736 source = repo[source_id].tree if source_id else None
730 target = repo[target_id].tree
737 target = repo[target_id].tree
731 result = repo.object_store.tree_changes(source, target)
738 result = repo.object_store.tree_changes(source, target)
732 return list(result)
739 return list(result)
733
740
734 @reraise_safe_exceptions
741 @reraise_safe_exceptions
735 def tree_and_type_for_path(self, wire, commit_id, path):
742 def tree_and_type_for_path(self, wire, commit_id, path):
736 repo_init = self._factory.repo_libgit2(wire)
743 repo_init = self._factory.repo_libgit2(wire)
737
744
738 with repo_init as repo:
745 with repo_init as repo:
739 commit = repo[commit_id]
746 commit = repo[commit_id]
740 try:
747 try:
741 tree = commit.tree[path]
748 tree = commit.tree[path]
742 except KeyError:
749 except KeyError:
743 return None, None, None
750 return None, None, None
744
751
745 return tree.id.hex, tree.type, tree.filemode
752 return tree.id.hex, tree.type, tree.filemode
746
753
747 @reraise_safe_exceptions
754 @reraise_safe_exceptions
748 def tree_items(self, wire, tree_id):
755 def tree_items(self, wire, tree_id):
749 repo_init = self._factory.repo_libgit2(wire)
756 repo_init = self._factory.repo_libgit2(wire)
750
757
751 with repo_init as repo:
758 with repo_init as repo:
752 try:
759 try:
753 tree = repo[tree_id]
760 tree = repo[tree_id]
754 except KeyError:
761 except KeyError:
755 raise ObjectMissing('No tree with id: {}'.format(tree_id))
762 raise ObjectMissing('No tree with id: {}'.format(tree_id))
756
763
757 result = []
764 result = []
758 for item in tree:
765 for item in tree:
759 item_sha = item.hex
766 item_sha = item.hex
760 item_mode = item.filemode
767 item_mode = item.filemode
761 item_type = item.type
768 item_type = item.type
762
769
763 if item_type == 'commit':
770 if item_type == 'commit':
764 # NOTE(marcink): submodules we translate to 'link' for backward compat
771 # NOTE(marcink): submodules we translate to 'link' for backward compat
765 item_type = 'link'
772 item_type = 'link'
766
773
767 result.append((item.name, item_mode, item_sha, item_type))
774 result.append((item.name, item_mode, item_sha, item_type))
768 return result
775 return result
769
776
770 @reraise_safe_exceptions
777 @reraise_safe_exceptions
771 def update_server_info(self, wire):
778 def update_server_info(self, wire):
772 repo = self._factory.repo(wire)
779 repo = self._factory.repo(wire)
773 update_server_info(repo)
780 update_server_info(repo)
774
781
775 @reraise_safe_exceptions
782 @reraise_safe_exceptions
776 def discover_git_version(self):
783 def discover_git_version(self):
777 stdout, _ = self.run_git_command(
784 stdout, _ = self.run_git_command(
778 {}, ['--version'], _bare=True, _safe=True)
785 {}, ['--version'], _bare=True, _safe=True)
779 prefix = 'git version'
786 prefix = 'git version'
780 if stdout.startswith(prefix):
787 if stdout.startswith(prefix):
781 stdout = stdout[len(prefix):]
788 stdout = stdout[len(prefix):]
782 return stdout.strip()
789 return stdout.strip()
783
790
784 @reraise_safe_exceptions
791 @reraise_safe_exceptions
785 def get_all_commit_ids(self, wire):
792 def get_all_commit_ids(self, wire):
786 if self.is_empty(wire):
787 return []
788
793
789 cmd = ['rev-list', '--reverse', '--date-order', '--branches', '--tags']
794 cmd = ['rev-list', '--reverse', '--date-order', '--branches', '--tags']
790 try:
795 try:
791 output, __ = self.run_git_command(wire, cmd)
796 output, __ = self.run_git_command(wire, cmd)
792 return output.splitlines()
797 return output.splitlines()
793 except Exception:
798 except Exception:
794 # Can be raised for empty repositories
799 # Can be raised for empty repositories
795 return []
800 return []
796
801
797 @reraise_safe_exceptions
802 @reraise_safe_exceptions
798 def run_git_command(self, wire, cmd, **opts):
803 def run_git_command(self, wire, cmd, **opts):
799 path = wire.get('path', None)
804 path = wire.get('path', None)
800
805
801 if path and os.path.isdir(path):
806 if path and os.path.isdir(path):
802 opts['cwd'] = path
807 opts['cwd'] = path
803
808
804 if '_bare' in opts:
809 if '_bare' in opts:
805 _copts = []
810 _copts = []
806 del opts['_bare']
811 del opts['_bare']
807 else:
812 else:
808 _copts = ['-c', 'core.quotepath=false', ]
813 _copts = ['-c', 'core.quotepath=false', ]
809 safe_call = False
814 safe_call = False
810 if '_safe' in opts:
815 if '_safe' in opts:
811 # no exc on failure
816 # no exc on failure
812 del opts['_safe']
817 del opts['_safe']
813 safe_call = True
818 safe_call = True
814
819
815 if '_copts' in opts:
820 if '_copts' in opts:
816 _copts.extend(opts['_copts'] or [])
821 _copts.extend(opts['_copts'] or [])
817 del opts['_copts']
822 del opts['_copts']
818
823
819 gitenv = os.environ.copy()
824 gitenv = os.environ.copy()
820 gitenv.update(opts.pop('extra_env', {}))
825 gitenv.update(opts.pop('extra_env', {}))
821 # need to clean fix GIT_DIR !
826 # need to clean fix GIT_DIR !
822 if 'GIT_DIR' in gitenv:
827 if 'GIT_DIR' in gitenv:
823 del gitenv['GIT_DIR']
828 del gitenv['GIT_DIR']
824 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
829 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
825 gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1'
830 gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1'
826
831
827 cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
832 cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
828 _opts = {'env': gitenv, 'shell': False}
833 _opts = {'env': gitenv, 'shell': False}
829
834
830 try:
835 try:
831 _opts.update(opts)
836 _opts.update(opts)
832 p = subprocessio.SubprocessIOChunker(cmd, **_opts)
837 p = subprocessio.SubprocessIOChunker(cmd, **_opts)
833
838
834 return ''.join(p), ''.join(p.error)
839 return ''.join(p), ''.join(p.error)
835 except (EnvironmentError, OSError) as err:
840 except (EnvironmentError, OSError) as err:
836 cmd = ' '.join(cmd) # human friendly CMD
841 cmd = ' '.join(cmd) # human friendly CMD
837 tb_err = ("Couldn't run git command (%s).\n"
842 tb_err = ("Couldn't run git command (%s).\n"
838 "Original error was:%s\n"
843 "Original error was:%s\n"
839 "Call options:%s\n"
844 "Call options:%s\n"
840 % (cmd, err, _opts))
845 % (cmd, err, _opts))
841 log.exception(tb_err)
846 log.exception(tb_err)
842 if safe_call:
847 if safe_call:
843 return '', err
848 return '', err
844 else:
849 else:
845 raise exceptions.VcsException()(tb_err)
850 raise exceptions.VcsException()(tb_err)
846
851
847 @reraise_safe_exceptions
852 @reraise_safe_exceptions
848 def install_hooks(self, wire, force=False):
853 def install_hooks(self, wire, force=False):
849 from vcsserver.hook_utils import install_git_hooks
854 from vcsserver.hook_utils import install_git_hooks
850 repo = self._factory.repo(wire)
855 repo = self._factory.repo(wire)
851 return install_git_hooks(repo.path, repo.bare, force_create=force)
856 return install_git_hooks(repo.path, repo.bare, force_create=force)
852
857
853 @reraise_safe_exceptions
858 @reraise_safe_exceptions
854 def get_hooks_info(self, wire):
859 def get_hooks_info(self, wire):
855 from vcsserver.hook_utils import (
860 from vcsserver.hook_utils import (
856 get_git_pre_hook_version, get_git_post_hook_version)
861 get_git_pre_hook_version, get_git_post_hook_version)
857 repo = self._factory.repo(wire)
862 repo = self._factory.repo(wire)
858 return {
863 return {
859 'pre_version': get_git_pre_hook_version(repo.path, repo.bare),
864 'pre_version': get_git_pre_hook_version(repo.path, repo.bare),
860 'post_version': get_git_post_hook_version(repo.path, repo.bare),
865 'post_version': get_git_post_hook_version(repo.path, repo.bare),
861 }
866 }
862
867
863
868
864 def str_to_dulwich(value):
869 def str_to_dulwich(value):
865 """
870 """
866 Dulwich 0.10.1a requires `unicode` objects to be passed in.
871 Dulwich 0.10.1a requires `unicode` objects to be passed in.
867 """
872 """
868 return value.decode(settings.WIRE_ENCODING)
873 return value.decode(settings.WIRE_ENCODING)
General Comments 0
You need to be logged in to leave comments. Login now