##// END OF EJS Templates
git: switched diff to libgit2
marcink -
r755:114e4e43 default
parent child Browse files
Show More
@@ -1,1085 +1,1123 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2019 RhodeCode GmbH
2 # Copyright (C) 2014-2019 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import collections
18 import collections
19 import logging
19 import logging
20 import os
20 import os
21 import posixpath as vcspath
21 import posixpath as vcspath
22 import re
22 import re
23 import stat
23 import stat
24 import traceback
24 import traceback
25 import urllib
25 import urllib
26 import urllib2
26 import urllib2
27 from functools import wraps
27 from functools import wraps
28
28
29 import more_itertools
29 import more_itertools
30 import pygit2
30 import pygit2
31 from pygit2 import Repository as LibGit2Repo
31 from pygit2 import Repository as LibGit2Repo
32 from dulwich import index, objects
32 from dulwich import index, objects
33 from dulwich.client import HttpGitClient, LocalGitClient
33 from dulwich.client import HttpGitClient, LocalGitClient
34 from dulwich.errors import (
34 from dulwich.errors import (
35 NotGitRepository, ChecksumMismatch, WrongObjectException,
35 NotGitRepository, ChecksumMismatch, WrongObjectException,
36 MissingCommitError, ObjectMissing, HangupException,
36 MissingCommitError, ObjectMissing, HangupException,
37 UnexpectedCommandError)
37 UnexpectedCommandError)
38 from dulwich.repo import Repo as DulwichRepo
38 from dulwich.repo import Repo as DulwichRepo
39 from dulwich.server import update_server_info
39 from dulwich.server import update_server_info
40
40
41 from vcsserver import exceptions, settings, subprocessio
41 from vcsserver import exceptions, settings, subprocessio
42 from vcsserver.utils import safe_str, safe_int
42 from vcsserver.utils import safe_str, safe_int
43 from vcsserver.base import RepoFactory, obfuscate_qs
43 from vcsserver.base import RepoFactory, obfuscate_qs
44 from vcsserver.hgcompat import (
44 from vcsserver.hgcompat import (
45 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
45 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
46 from vcsserver.git_lfs.lib import LFSOidStore
46 from vcsserver.git_lfs.lib import LFSOidStore
47 from vcsserver.vcs_base import RemoteBase
47 from vcsserver.vcs_base import RemoteBase
48
48
49 DIR_STAT = stat.S_IFDIR
49 DIR_STAT = stat.S_IFDIR
50 FILE_MODE = stat.S_IFMT
50 FILE_MODE = stat.S_IFMT
51 GIT_LINK = objects.S_IFGITLINK
51 GIT_LINK = objects.S_IFGITLINK
52 PEELED_REF_MARKER = '^{}'
52 PEELED_REF_MARKER = '^{}'
53
53
54
54
55 log = logging.getLogger(__name__)
55 log = logging.getLogger(__name__)
56
56
57
57
58 def str_to_dulwich(value):
58 def str_to_dulwich(value):
59 """
59 """
60 Dulwich 0.10.1a requires `unicode` objects to be passed in.
60 Dulwich 0.10.1a requires `unicode` objects to be passed in.
61 """
61 """
62 return value.decode(settings.WIRE_ENCODING)
62 return value.decode(settings.WIRE_ENCODING)
63
63
64
64
65 def reraise_safe_exceptions(func):
65 def reraise_safe_exceptions(func):
66 """Converts Dulwich exceptions to something neutral."""
66 """Converts Dulwich exceptions to something neutral."""
67
67
68 @wraps(func)
68 @wraps(func)
69 def wrapper(*args, **kwargs):
69 def wrapper(*args, **kwargs):
70 try:
70 try:
71 return func(*args, **kwargs)
71 return func(*args, **kwargs)
72 except (ChecksumMismatch, WrongObjectException, MissingCommitError, ObjectMissing,) as e:
72 except (ChecksumMismatch, WrongObjectException, MissingCommitError, ObjectMissing,) as e:
73 exc = exceptions.LookupException(org_exc=e)
73 exc = exceptions.LookupException(org_exc=e)
74 raise exc(safe_str(e))
74 raise exc(safe_str(e))
75 except (HangupException, UnexpectedCommandError) as e:
75 except (HangupException, UnexpectedCommandError) as e:
76 exc = exceptions.VcsException(org_exc=e)
76 exc = exceptions.VcsException(org_exc=e)
77 raise exc(safe_str(e))
77 raise exc(safe_str(e))
78 except Exception as e:
78 except Exception as e:
79 # NOTE(marcink): becuase of how dulwich handles some exceptions
79 # NOTE(marcink): becuase of how dulwich handles some exceptions
80 # (KeyError on empty repos), we cannot track this and catch all
80 # (KeyError on empty repos), we cannot track this and catch all
81 # exceptions, it's an exceptions from other handlers
81 # exceptions, it's an exceptions from other handlers
82 #if not hasattr(e, '_vcs_kind'):
82 #if not hasattr(e, '_vcs_kind'):
83 #log.exception("Unhandled exception in git remote call")
83 #log.exception("Unhandled exception in git remote call")
84 #raise_from_original(exceptions.UnhandledException)
84 #raise_from_original(exceptions.UnhandledException)
85 raise
85 raise
86 return wrapper
86 return wrapper
87
87
88
88
89 class Repo(DulwichRepo):
89 class Repo(DulwichRepo):
90 """
90 """
91 A wrapper for dulwich Repo class.
91 A wrapper for dulwich Repo class.
92
92
93 Since dulwich is sometimes keeping .idx file descriptors open, it leads to
93 Since dulwich is sometimes keeping .idx file descriptors open, it leads to
94 "Too many open files" error. We need to close all opened file descriptors
94 "Too many open files" error. We need to close all opened file descriptors
95 once the repo object is destroyed.
95 once the repo object is destroyed.
96 """
96 """
97 def __del__(self):
97 def __del__(self):
98 if hasattr(self, 'object_store'):
98 if hasattr(self, 'object_store'):
99 self.close()
99 self.close()
100
100
101
101
102 class Repository(LibGit2Repo):
102 class Repository(LibGit2Repo):
103
103
104 def __enter__(self):
104 def __enter__(self):
105 return self
105 return self
106
106
107 def __exit__(self, exc_type, exc_val, exc_tb):
107 def __exit__(self, exc_type, exc_val, exc_tb):
108 self.free()
108 self.free()
109
109
110
110
111 class GitFactory(RepoFactory):
111 class GitFactory(RepoFactory):
112 repo_type = 'git'
112 repo_type = 'git'
113
113
114 def _create_repo(self, wire, create, use_libgit2=False):
114 def _create_repo(self, wire, create, use_libgit2=False):
115 if use_libgit2:
115 if use_libgit2:
116 return Repository(wire['path'])
116 return Repository(wire['path'])
117 else:
117 else:
118 repo_path = str_to_dulwich(wire['path'])
118 repo_path = str_to_dulwich(wire['path'])
119 return Repo(repo_path)
119 return Repo(repo_path)
120
120
121 def repo(self, wire, create=False, use_libgit2=False):
121 def repo(self, wire, create=False, use_libgit2=False):
122 """
122 """
123 Get a repository instance for the given path.
123 Get a repository instance for the given path.
124 """
124 """
125 return self._create_repo(wire, create, use_libgit2)
125 return self._create_repo(wire, create, use_libgit2)
126
126
127 def repo_libgit2(self, wire):
127 def repo_libgit2(self, wire):
128 return self.repo(wire, use_libgit2=True)
128 return self.repo(wire, use_libgit2=True)
129
129
130
130
131 class GitRemote(RemoteBase):
131 class GitRemote(RemoteBase):
132
132
133 def __init__(self, factory):
133 def __init__(self, factory):
134 self._factory = factory
134 self._factory = factory
135 self._bulk_methods = {
135 self._bulk_methods = {
136 "date": self.date,
136 "date": self.date,
137 "author": self.author,
137 "author": self.author,
138 "branch": self.branch,
138 "branch": self.branch,
139 "message": self.message,
139 "message": self.message,
140 "parents": self.parents,
140 "parents": self.parents,
141 "_commit": self.revision,
141 "_commit": self.revision,
142 }
142 }
143
143
144 def _wire_to_config(self, wire):
144 def _wire_to_config(self, wire):
145 if 'config' in wire:
145 if 'config' in wire:
146 return dict([(x[0] + '_' + x[1], x[2]) for x in wire['config']])
146 return dict([(x[0] + '_' + x[1], x[2]) for x in wire['config']])
147 return {}
147 return {}
148
148
149 def _remote_conf(self, config):
149 def _remote_conf(self, config):
150 params = [
150 params = [
151 '-c', 'core.askpass=""',
151 '-c', 'core.askpass=""',
152 ]
152 ]
153 ssl_cert_dir = config.get('vcs_ssl_dir')
153 ssl_cert_dir = config.get('vcs_ssl_dir')
154 if ssl_cert_dir:
154 if ssl_cert_dir:
155 params.extend(['-c', 'http.sslCAinfo={}'.format(ssl_cert_dir)])
155 params.extend(['-c', 'http.sslCAinfo={}'.format(ssl_cert_dir)])
156 return params
156 return params
157
157
158 @reraise_safe_exceptions
158 @reraise_safe_exceptions
159 def discover_git_version(self):
159 def discover_git_version(self):
160 stdout, _ = self.run_git_command(
160 stdout, _ = self.run_git_command(
161 {}, ['--version'], _bare=True, _safe=True)
161 {}, ['--version'], _bare=True, _safe=True)
162 prefix = 'git version'
162 prefix = 'git version'
163 if stdout.startswith(prefix):
163 if stdout.startswith(prefix):
164 stdout = stdout[len(prefix):]
164 stdout = stdout[len(prefix):]
165 return stdout.strip()
165 return stdout.strip()
166
166
167 @reraise_safe_exceptions
167 @reraise_safe_exceptions
168 def is_empty(self, wire):
168 def is_empty(self, wire):
169 repo_init = self._factory.repo_libgit2(wire)
169 repo_init = self._factory.repo_libgit2(wire)
170 with repo_init as repo:
170 with repo_init as repo:
171
171
172 try:
172 try:
173 has_head = repo.head.name
173 has_head = repo.head.name
174 if has_head:
174 if has_head:
175 return False
175 return False
176
176
177 # NOTE(marcink): check again using more expensive method
177 # NOTE(marcink): check again using more expensive method
178 return repo.is_empty
178 return repo.is_empty
179 except Exception:
179 except Exception:
180 pass
180 pass
181
181
182 return True
182 return True
183
183
184 @reraise_safe_exceptions
184 @reraise_safe_exceptions
185 def assert_correct_path(self, wire):
185 def assert_correct_path(self, wire):
186 cache_on, context_uid, repo_id = self._cache_on(wire)
186 cache_on, context_uid, repo_id = self._cache_on(wire)
187 @self.region.conditional_cache_on_arguments(condition=cache_on)
187 @self.region.conditional_cache_on_arguments(condition=cache_on)
188 def _assert_correct_path(_context_uid, _repo_id):
188 def _assert_correct_path(_context_uid, _repo_id):
189 try:
189 try:
190 repo_init = self._factory.repo_libgit2(wire)
190 repo_init = self._factory.repo_libgit2(wire)
191 with repo_init as repo:
191 with repo_init as repo:
192 pass
192 pass
193 except pygit2.GitError:
193 except pygit2.GitError:
194 path = wire.get('path')
194 path = wire.get('path')
195 tb = traceback.format_exc()
195 tb = traceback.format_exc()
196 log.debug("Invalid Git path `%s`, tb: %s", path, tb)
196 log.debug("Invalid Git path `%s`, tb: %s", path, tb)
197 return False
197 return False
198
198
199 return True
199 return True
200 return _assert_correct_path(context_uid, repo_id)
200 return _assert_correct_path(context_uid, repo_id)
201
201
202 @reraise_safe_exceptions
202 @reraise_safe_exceptions
203 def bare(self, wire):
203 def bare(self, wire):
204 repo_init = self._factory.repo_libgit2(wire)
204 repo_init = self._factory.repo_libgit2(wire)
205 with repo_init as repo:
205 with repo_init as repo:
206 return repo.is_bare
206 return repo.is_bare
207
207
208 @reraise_safe_exceptions
208 @reraise_safe_exceptions
209 def blob_as_pretty_string(self, wire, sha):
209 def blob_as_pretty_string(self, wire, sha):
210 repo_init = self._factory.repo_libgit2(wire)
210 repo_init = self._factory.repo_libgit2(wire)
211 with repo_init as repo:
211 with repo_init as repo:
212 blob_obj = repo[sha]
212 blob_obj = repo[sha]
213 blob = blob_obj.data
213 blob = blob_obj.data
214 return blob
214 return blob
215
215
216 @reraise_safe_exceptions
216 @reraise_safe_exceptions
217 def blob_raw_length(self, wire, sha):
217 def blob_raw_length(self, wire, sha):
218 cache_on, context_uid, repo_id = self._cache_on(wire)
218 cache_on, context_uid, repo_id = self._cache_on(wire)
219 @self.region.conditional_cache_on_arguments(condition=cache_on)
219 @self.region.conditional_cache_on_arguments(condition=cache_on)
220 def _blob_raw_length(_repo_id, _sha):
220 def _blob_raw_length(_repo_id, _sha):
221
221
222 repo_init = self._factory.repo_libgit2(wire)
222 repo_init = self._factory.repo_libgit2(wire)
223 with repo_init as repo:
223 with repo_init as repo:
224 blob = repo[sha]
224 blob = repo[sha]
225 return blob.size
225 return blob.size
226
226
227 return _blob_raw_length(repo_id, sha)
227 return _blob_raw_length(repo_id, sha)
228
228
229 def _parse_lfs_pointer(self, raw_content):
229 def _parse_lfs_pointer(self, raw_content):
230
230
231 spec_string = 'version https://git-lfs.github.com/spec'
231 spec_string = 'version https://git-lfs.github.com/spec'
232 if raw_content and raw_content.startswith(spec_string):
232 if raw_content and raw_content.startswith(spec_string):
233 pattern = re.compile(r"""
233 pattern = re.compile(r"""
234 (?:\n)?
234 (?:\n)?
235 ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
235 ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
236 ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
236 ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
237 ^size[ ](?P<oid_size>[0-9]+)\n
237 ^size[ ](?P<oid_size>[0-9]+)\n
238 (?:\n)?
238 (?:\n)?
239 """, re.VERBOSE | re.MULTILINE)
239 """, re.VERBOSE | re.MULTILINE)
240 match = pattern.match(raw_content)
240 match = pattern.match(raw_content)
241 if match:
241 if match:
242 return match.groupdict()
242 return match.groupdict()
243
243
244 return {}
244 return {}
245
245
246 @reraise_safe_exceptions
246 @reraise_safe_exceptions
247 def is_large_file(self, wire, commit_id):
247 def is_large_file(self, wire, commit_id):
248
248
249 cache_on, context_uid, repo_id = self._cache_on(wire)
249 cache_on, context_uid, repo_id = self._cache_on(wire)
250 @self.region.conditional_cache_on_arguments(condition=cache_on)
250 @self.region.conditional_cache_on_arguments(condition=cache_on)
251 def _is_large_file(_repo_id, _sha):
251 def _is_large_file(_repo_id, _sha):
252 repo_init = self._factory.repo_libgit2(wire)
252 repo_init = self._factory.repo_libgit2(wire)
253 with repo_init as repo:
253 with repo_init as repo:
254 blob = repo[commit_id]
254 blob = repo[commit_id]
255 if blob.is_binary:
255 if blob.is_binary:
256 return {}
256 return {}
257
257
258 return self._parse_lfs_pointer(blob.data)
258 return self._parse_lfs_pointer(blob.data)
259
259
260 return _is_large_file(repo_id, commit_id)
260 return _is_large_file(repo_id, commit_id)
261
261
262 @reraise_safe_exceptions
262 @reraise_safe_exceptions
263 def in_largefiles_store(self, wire, oid):
263 def in_largefiles_store(self, wire, oid):
264 conf = self._wire_to_config(wire)
264 conf = self._wire_to_config(wire)
265 repo_init = self._factory.repo_libgit2(wire)
265 repo_init = self._factory.repo_libgit2(wire)
266 with repo_init as repo:
266 with repo_init as repo:
267 repo_name = repo.path
267 repo_name = repo.path
268
268
269 store_location = conf.get('vcs_git_lfs_store_location')
269 store_location = conf.get('vcs_git_lfs_store_location')
270 if store_location:
270 if store_location:
271
271
272 store = LFSOidStore(
272 store = LFSOidStore(
273 oid=oid, repo=repo_name, store_location=store_location)
273 oid=oid, repo=repo_name, store_location=store_location)
274 return store.has_oid()
274 return store.has_oid()
275
275
276 return False
276 return False
277
277
278 @reraise_safe_exceptions
278 @reraise_safe_exceptions
279 def store_path(self, wire, oid):
279 def store_path(self, wire, oid):
280 conf = self._wire_to_config(wire)
280 conf = self._wire_to_config(wire)
281 repo_init = self._factory.repo_libgit2(wire)
281 repo_init = self._factory.repo_libgit2(wire)
282 with repo_init as repo:
282 with repo_init as repo:
283 repo_name = repo.path
283 repo_name = repo.path
284
284
285 store_location = conf.get('vcs_git_lfs_store_location')
285 store_location = conf.get('vcs_git_lfs_store_location')
286 if store_location:
286 if store_location:
287 store = LFSOidStore(
287 store = LFSOidStore(
288 oid=oid, repo=repo_name, store_location=store_location)
288 oid=oid, repo=repo_name, store_location=store_location)
289 return store.oid_path
289 return store.oid_path
290 raise ValueError('Unable to fetch oid with path {}'.format(oid))
290 raise ValueError('Unable to fetch oid with path {}'.format(oid))
291
291
292 @reraise_safe_exceptions
292 @reraise_safe_exceptions
293 def bulk_request(self, wire, rev, pre_load):
293 def bulk_request(self, wire, rev, pre_load):
294 cache_on, context_uid, repo_id = self._cache_on(wire)
294 cache_on, context_uid, repo_id = self._cache_on(wire)
295 @self.region.conditional_cache_on_arguments(condition=cache_on)
295 @self.region.conditional_cache_on_arguments(condition=cache_on)
296 def _bulk_request(_repo_id, _rev, _pre_load):
296 def _bulk_request(_repo_id, _rev, _pre_load):
297 result = {}
297 result = {}
298 for attr in pre_load:
298 for attr in pre_load:
299 try:
299 try:
300 method = self._bulk_methods[attr]
300 method = self._bulk_methods[attr]
301 args = [wire, rev]
301 args = [wire, rev]
302 result[attr] = method(*args)
302 result[attr] = method(*args)
303 except KeyError as e:
303 except KeyError as e:
304 raise exceptions.VcsException(e)(
304 raise exceptions.VcsException(e)(
305 "Unknown bulk attribute: %s" % attr)
305 "Unknown bulk attribute: %s" % attr)
306 return result
306 return result
307
307
308 return _bulk_request(repo_id, rev, sorted(pre_load))
308 return _bulk_request(repo_id, rev, sorted(pre_load))
309
309
310 def _build_opener(self, url):
310 def _build_opener(self, url):
311 handlers = []
311 handlers = []
312 url_obj = url_parser(url)
312 url_obj = url_parser(url)
313 _, authinfo = url_obj.authinfo()
313 _, authinfo = url_obj.authinfo()
314
314
315 if authinfo:
315 if authinfo:
316 # create a password manager
316 # create a password manager
317 passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
317 passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
318 passmgr.add_password(*authinfo)
318 passmgr.add_password(*authinfo)
319
319
320 handlers.extend((httpbasicauthhandler(passmgr),
320 handlers.extend((httpbasicauthhandler(passmgr),
321 httpdigestauthhandler(passmgr)))
321 httpdigestauthhandler(passmgr)))
322
322
323 return urllib2.build_opener(*handlers)
323 return urllib2.build_opener(*handlers)
324
324
325 def _type_id_to_name(self, type_id):
325 def _type_id_to_name(self, type_id):
326 return {
326 return {
327 1: b'commit',
327 1: b'commit',
328 2: b'tree',
328 2: b'tree',
329 3: b'blob',
329 3: b'blob',
330 4: b'tag'
330 4: b'tag'
331 }[type_id]
331 }[type_id]
332
332
333 @reraise_safe_exceptions
333 @reraise_safe_exceptions
334 def check_url(self, url, config):
334 def check_url(self, url, config):
335 url_obj = url_parser(url)
335 url_obj = url_parser(url)
336 test_uri, _ = url_obj.authinfo()
336 test_uri, _ = url_obj.authinfo()
337 url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
337 url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
338 url_obj.query = obfuscate_qs(url_obj.query)
338 url_obj.query = obfuscate_qs(url_obj.query)
339 cleaned_uri = str(url_obj)
339 cleaned_uri = str(url_obj)
340 log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
340 log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
341
341
342 if not test_uri.endswith('info/refs'):
342 if not test_uri.endswith('info/refs'):
343 test_uri = test_uri.rstrip('/') + '/info/refs'
343 test_uri = test_uri.rstrip('/') + '/info/refs'
344
344
345 o = self._build_opener(url)
345 o = self._build_opener(url)
346 o.addheaders = [('User-Agent', 'git/1.7.8.0')] # fake some git
346 o.addheaders = [('User-Agent', 'git/1.7.8.0')] # fake some git
347
347
348 q = {"service": 'git-upload-pack'}
348 q = {"service": 'git-upload-pack'}
349 qs = '?%s' % urllib.urlencode(q)
349 qs = '?%s' % urllib.urlencode(q)
350 cu = "%s%s" % (test_uri, qs)
350 cu = "%s%s" % (test_uri, qs)
351 req = urllib2.Request(cu, None, {})
351 req = urllib2.Request(cu, None, {})
352
352
353 try:
353 try:
354 log.debug("Trying to open URL %s", cleaned_uri)
354 log.debug("Trying to open URL %s", cleaned_uri)
355 resp = o.open(req)
355 resp = o.open(req)
356 if resp.code != 200:
356 if resp.code != 200:
357 raise exceptions.URLError()('Return Code is not 200')
357 raise exceptions.URLError()('Return Code is not 200')
358 except Exception as e:
358 except Exception as e:
359 log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
359 log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
360 # means it cannot be cloned
360 # means it cannot be cloned
361 raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e))
361 raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e))
362
362
363 # now detect if it's proper git repo
363 # now detect if it's proper git repo
364 gitdata = resp.read()
364 gitdata = resp.read()
365 if 'service=git-upload-pack' in gitdata:
365 if 'service=git-upload-pack' in gitdata:
366 pass
366 pass
367 elif re.findall(r'[0-9a-fA-F]{40}\s+refs', gitdata):
367 elif re.findall(r'[0-9a-fA-F]{40}\s+refs', gitdata):
368 # old style git can return some other format !
368 # old style git can return some other format !
369 pass
369 pass
370 else:
370 else:
371 raise exceptions.URLError()(
371 raise exceptions.URLError()(
372 "url [%s] does not look like an git" % (cleaned_uri,))
372 "url [%s] does not look like an git" % (cleaned_uri,))
373
373
374 return True
374 return True
375
375
376 @reraise_safe_exceptions
376 @reraise_safe_exceptions
377 def clone(self, wire, url, deferred, valid_refs, update_after_clone):
377 def clone(self, wire, url, deferred, valid_refs, update_after_clone):
378 # TODO(marcink): deprecate this method. Last i checked we don't use it anymore
378 # TODO(marcink): deprecate this method. Last i checked we don't use it anymore
379 remote_refs = self.pull(wire, url, apply_refs=False)
379 remote_refs = self.pull(wire, url, apply_refs=False)
380 repo = self._factory.repo(wire)
380 repo = self._factory.repo(wire)
381 if isinstance(valid_refs, list):
381 if isinstance(valid_refs, list):
382 valid_refs = tuple(valid_refs)
382 valid_refs = tuple(valid_refs)
383
383
384 for k in remote_refs:
384 for k in remote_refs:
385 # only parse heads/tags and skip so called deferred tags
385 # only parse heads/tags and skip so called deferred tags
386 if k.startswith(valid_refs) and not k.endswith(deferred):
386 if k.startswith(valid_refs) and not k.endswith(deferred):
387 repo[k] = remote_refs[k]
387 repo[k] = remote_refs[k]
388
388
389 if update_after_clone:
389 if update_after_clone:
390 # we want to checkout HEAD
390 # we want to checkout HEAD
391 repo["HEAD"] = remote_refs["HEAD"]
391 repo["HEAD"] = remote_refs["HEAD"]
392 index.build_index_from_tree(repo.path, repo.index_path(),
392 index.build_index_from_tree(repo.path, repo.index_path(),
393 repo.object_store, repo["HEAD"].tree)
393 repo.object_store, repo["HEAD"].tree)
394
394
395 @reraise_safe_exceptions
395 @reraise_safe_exceptions
396 def branch(self, wire, commit_id):
396 def branch(self, wire, commit_id):
397 cache_on, context_uid, repo_id = self._cache_on(wire)
397 cache_on, context_uid, repo_id = self._cache_on(wire)
398 @self.region.conditional_cache_on_arguments(condition=cache_on)
398 @self.region.conditional_cache_on_arguments(condition=cache_on)
399 def _branch(_context_uid, _repo_id, _commit_id):
399 def _branch(_context_uid, _repo_id, _commit_id):
400 regex = re.compile('^refs/heads')
400 regex = re.compile('^refs/heads')
401
401
402 def filter_with(ref):
402 def filter_with(ref):
403 return regex.match(ref[0]) and ref[1] == _commit_id
403 return regex.match(ref[0]) and ref[1] == _commit_id
404
404
405 branches = filter(filter_with, self.get_refs(wire).items())
405 branches = filter(filter_with, self.get_refs(wire).items())
406 return [x[0].split('refs/heads/')[-1] for x in branches]
406 return [x[0].split('refs/heads/')[-1] for x in branches]
407
407
408 return _branch(context_uid, repo_id, commit_id)
408 return _branch(context_uid, repo_id, commit_id)
409
409
410 @reraise_safe_exceptions
410 @reraise_safe_exceptions
411 def commit_branches(self, wire, commit_id):
411 def commit_branches(self, wire, commit_id):
412 cache_on, context_uid, repo_id = self._cache_on(wire)
412 cache_on, context_uid, repo_id = self._cache_on(wire)
413 @self.region.conditional_cache_on_arguments(condition=cache_on)
413 @self.region.conditional_cache_on_arguments(condition=cache_on)
414 def _commit_branches(_context_uid, _repo_id, _commit_id):
414 def _commit_branches(_context_uid, _repo_id, _commit_id):
415 repo_init = self._factory.repo_libgit2(wire)
415 repo_init = self._factory.repo_libgit2(wire)
416 with repo_init as repo:
416 with repo_init as repo:
417 branches = [x for x in repo.branches.with_commit(_commit_id)]
417 branches = [x for x in repo.branches.with_commit(_commit_id)]
418 return branches
418 return branches
419
419
420 return _commit_branches(context_uid, repo_id, commit_id)
420 return _commit_branches(context_uid, repo_id, commit_id)
421
421
422 @reraise_safe_exceptions
422 @reraise_safe_exceptions
423 def add_object(self, wire, content):
423 def add_object(self, wire, content):
424 repo_init = self._factory.repo_libgit2(wire)
424 repo_init = self._factory.repo_libgit2(wire)
425 with repo_init as repo:
425 with repo_init as repo:
426 blob = objects.Blob()
426 blob = objects.Blob()
427 blob.set_raw_string(content)
427 blob.set_raw_string(content)
428 repo.object_store.add_object(blob)
428 repo.object_store.add_object(blob)
429 return blob.id
429 return blob.id
430
430
431 # TODO: this is quite complex, check if that can be simplified
431 # TODO: this is quite complex, check if that can be simplified
432 @reraise_safe_exceptions
432 @reraise_safe_exceptions
433 def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
433 def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
434 repo = self._factory.repo(wire)
434 repo = self._factory.repo(wire)
435 object_store = repo.object_store
435 object_store = repo.object_store
436
436
437 # Create tree and populates it with blobs
437 # Create tree and populates it with blobs
438 commit_tree = commit_tree and repo[commit_tree] or objects.Tree()
438 commit_tree = commit_tree and repo[commit_tree] or objects.Tree()
439
439
440 for node in updated:
440 for node in updated:
441 # Compute subdirs if needed
441 # Compute subdirs if needed
442 dirpath, nodename = vcspath.split(node['path'])
442 dirpath, nodename = vcspath.split(node['path'])
443 dirnames = map(safe_str, dirpath and dirpath.split('/') or [])
443 dirnames = map(safe_str, dirpath and dirpath.split('/') or [])
444 parent = commit_tree
444 parent = commit_tree
445 ancestors = [('', parent)]
445 ancestors = [('', parent)]
446
446
447 # Tries to dig for the deepest existing tree
447 # Tries to dig for the deepest existing tree
448 while dirnames:
448 while dirnames:
449 curdir = dirnames.pop(0)
449 curdir = dirnames.pop(0)
450 try:
450 try:
451 dir_id = parent[curdir][1]
451 dir_id = parent[curdir][1]
452 except KeyError:
452 except KeyError:
453 # put curdir back into dirnames and stops
453 # put curdir back into dirnames and stops
454 dirnames.insert(0, curdir)
454 dirnames.insert(0, curdir)
455 break
455 break
456 else:
456 else:
457 # If found, updates parent
457 # If found, updates parent
458 parent = repo[dir_id]
458 parent = repo[dir_id]
459 ancestors.append((curdir, parent))
459 ancestors.append((curdir, parent))
460 # Now parent is deepest existing tree and we need to create
460 # Now parent is deepest existing tree and we need to create
461 # subtrees for dirnames (in reverse order)
461 # subtrees for dirnames (in reverse order)
462 # [this only applies for nodes from added]
462 # [this only applies for nodes from added]
463 new_trees = []
463 new_trees = []
464
464
465 blob = objects.Blob.from_string(node['content'])
465 blob = objects.Blob.from_string(node['content'])
466
466
467 if dirnames:
467 if dirnames:
468 # If there are trees which should be created we need to build
468 # If there are trees which should be created we need to build
469 # them now (in reverse order)
469 # them now (in reverse order)
470 reversed_dirnames = list(reversed(dirnames))
470 reversed_dirnames = list(reversed(dirnames))
471 curtree = objects.Tree()
471 curtree = objects.Tree()
472 curtree[node['node_path']] = node['mode'], blob.id
472 curtree[node['node_path']] = node['mode'], blob.id
473 new_trees.append(curtree)
473 new_trees.append(curtree)
474 for dirname in reversed_dirnames[:-1]:
474 for dirname in reversed_dirnames[:-1]:
475 newtree = objects.Tree()
475 newtree = objects.Tree()
476 newtree[dirname] = (DIR_STAT, curtree.id)
476 newtree[dirname] = (DIR_STAT, curtree.id)
477 new_trees.append(newtree)
477 new_trees.append(newtree)
478 curtree = newtree
478 curtree = newtree
479 parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
479 parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
480 else:
480 else:
481 parent.add(name=node['node_path'], mode=node['mode'], hexsha=blob.id)
481 parent.add(name=node['node_path'], mode=node['mode'], hexsha=blob.id)
482
482
483 new_trees.append(parent)
483 new_trees.append(parent)
484 # Update ancestors
484 # Update ancestors
485 reversed_ancestors = reversed(
485 reversed_ancestors = reversed(
486 [(a[1], b[1], b[0]) for a, b in zip(ancestors, ancestors[1:])])
486 [(a[1], b[1], b[0]) for a, b in zip(ancestors, ancestors[1:])])
487 for parent, tree, path in reversed_ancestors:
487 for parent, tree, path in reversed_ancestors:
488 parent[path] = (DIR_STAT, tree.id)
488 parent[path] = (DIR_STAT, tree.id)
489 object_store.add_object(tree)
489 object_store.add_object(tree)
490
490
491 object_store.add_object(blob)
491 object_store.add_object(blob)
492 for tree in new_trees:
492 for tree in new_trees:
493 object_store.add_object(tree)
493 object_store.add_object(tree)
494
494
495 for node_path in removed:
495 for node_path in removed:
496 paths = node_path.split('/')
496 paths = node_path.split('/')
497 tree = commit_tree
497 tree = commit_tree
498 trees = [tree]
498 trees = [tree]
499 # Traverse deep into the forest...
499 # Traverse deep into the forest...
500 for path in paths:
500 for path in paths:
501 try:
501 try:
502 obj = repo[tree[path][1]]
502 obj = repo[tree[path][1]]
503 if isinstance(obj, objects.Tree):
503 if isinstance(obj, objects.Tree):
504 trees.append(obj)
504 trees.append(obj)
505 tree = obj
505 tree = obj
506 except KeyError:
506 except KeyError:
507 break
507 break
508 # Cut down the blob and all rotten trees on the way back...
508 # Cut down the blob and all rotten trees on the way back...
509 for path, tree in reversed(zip(paths, trees)):
509 for path, tree in reversed(zip(paths, trees)):
510 del tree[path]
510 del tree[path]
511 if tree:
511 if tree:
512 # This tree still has elements - don't remove it or any
512 # This tree still has elements - don't remove it or any
513 # of it's parents
513 # of it's parents
514 break
514 break
515
515
516 object_store.add_object(commit_tree)
516 object_store.add_object(commit_tree)
517
517
518 # Create commit
518 # Create commit
519 commit = objects.Commit()
519 commit = objects.Commit()
520 commit.tree = commit_tree.id
520 commit.tree = commit_tree.id
521 for k, v in commit_data.iteritems():
521 for k, v in commit_data.iteritems():
522 setattr(commit, k, v)
522 setattr(commit, k, v)
523 object_store.add_object(commit)
523 object_store.add_object(commit)
524
524
525 self.create_branch(wire, branch, commit.id)
525 self.create_branch(wire, branch, commit.id)
526
526
527 # dulwich set-ref
527 # dulwich set-ref
528 ref = 'refs/heads/%s' % branch
528 ref = 'refs/heads/%s' % branch
529 repo.refs[ref] = commit.id
529 repo.refs[ref] = commit.id
530
530
531 return commit.id
531 return commit.id
532
532
533 @reraise_safe_exceptions
533 @reraise_safe_exceptions
534 def pull(self, wire, url, apply_refs=True, refs=None, update_after=False):
534 def pull(self, wire, url, apply_refs=True, refs=None, update_after=False):
535 if url != 'default' and '://' not in url:
535 if url != 'default' and '://' not in url:
536 client = LocalGitClient(url)
536 client = LocalGitClient(url)
537 else:
537 else:
538 url_obj = url_parser(url)
538 url_obj = url_parser(url)
539 o = self._build_opener(url)
539 o = self._build_opener(url)
540 url, _ = url_obj.authinfo()
540 url, _ = url_obj.authinfo()
541 client = HttpGitClient(base_url=url, opener=o)
541 client = HttpGitClient(base_url=url, opener=o)
542 repo = self._factory.repo(wire)
542 repo = self._factory.repo(wire)
543
543
544 determine_wants = repo.object_store.determine_wants_all
544 determine_wants = repo.object_store.determine_wants_all
545 if refs:
545 if refs:
546 def determine_wants_requested(references):
546 def determine_wants_requested(references):
547 return [references[r] for r in references if r in refs]
547 return [references[r] for r in references if r in refs]
548 determine_wants = determine_wants_requested
548 determine_wants = determine_wants_requested
549
549
550 try:
550 try:
551 remote_refs = client.fetch(
551 remote_refs = client.fetch(
552 path=url, target=repo, determine_wants=determine_wants)
552 path=url, target=repo, determine_wants=determine_wants)
553 except NotGitRepository as e:
553 except NotGitRepository as e:
554 log.warning(
554 log.warning(
555 'Trying to fetch from "%s" failed, not a Git repository.', url)
555 'Trying to fetch from "%s" failed, not a Git repository.', url)
556 # Exception can contain unicode which we convert
556 # Exception can contain unicode which we convert
557 raise exceptions.AbortException(e)(repr(e))
557 raise exceptions.AbortException(e)(repr(e))
558
558
559 # mikhail: client.fetch() returns all the remote refs, but fetches only
559 # mikhail: client.fetch() returns all the remote refs, but fetches only
560 # refs filtered by `determine_wants` function. We need to filter result
560 # refs filtered by `determine_wants` function. We need to filter result
561 # as well
561 # as well
562 if refs:
562 if refs:
563 remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
563 remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
564
564
565 if apply_refs:
565 if apply_refs:
566 # TODO: johbo: Needs proper test coverage with a git repository
566 # TODO: johbo: Needs proper test coverage with a git repository
567 # that contains a tag object, so that we would end up with
567 # that contains a tag object, so that we would end up with
568 # a peeled ref at this point.
568 # a peeled ref at this point.
569 for k in remote_refs:
569 for k in remote_refs:
570 if k.endswith(PEELED_REF_MARKER):
570 if k.endswith(PEELED_REF_MARKER):
571 log.debug("Skipping peeled reference %s", k)
571 log.debug("Skipping peeled reference %s", k)
572 continue
572 continue
573 repo[k] = remote_refs[k]
573 repo[k] = remote_refs[k]
574
574
575 if refs and not update_after:
575 if refs and not update_after:
576 # mikhail: explicitly set the head to the last ref.
576 # mikhail: explicitly set the head to the last ref.
577 repo['HEAD'] = remote_refs[refs[-1]]
577 repo['HEAD'] = remote_refs[refs[-1]]
578
578
579 if update_after:
579 if update_after:
580 # we want to checkout HEAD
580 # we want to checkout HEAD
581 repo["HEAD"] = remote_refs["HEAD"]
581 repo["HEAD"] = remote_refs["HEAD"]
582 index.build_index_from_tree(repo.path, repo.index_path(),
582 index.build_index_from_tree(repo.path, repo.index_path(),
583 repo.object_store, repo["HEAD"].tree)
583 repo.object_store, repo["HEAD"].tree)
584 return remote_refs
584 return remote_refs
585
585
586 @reraise_safe_exceptions
586 @reraise_safe_exceptions
587 def sync_fetch(self, wire, url, refs=None):
587 def sync_fetch(self, wire, url, refs=None):
588 repo = self._factory.repo(wire)
588 repo = self._factory.repo(wire)
589 if refs and not isinstance(refs, (list, tuple)):
589 if refs and not isinstance(refs, (list, tuple)):
590 refs = [refs]
590 refs = [refs]
591 config = self._wire_to_config(wire)
591 config = self._wire_to_config(wire)
592 # get all remote refs we'll use to fetch later
592 # get all remote refs we'll use to fetch later
593 output, __ = self.run_git_command(
593 output, __ = self.run_git_command(
594 wire, ['ls-remote', url], fail_on_stderr=False,
594 wire, ['ls-remote', url], fail_on_stderr=False,
595 _copts=self._remote_conf(config),
595 _copts=self._remote_conf(config),
596 extra_env={'GIT_TERMINAL_PROMPT': '0'})
596 extra_env={'GIT_TERMINAL_PROMPT': '0'})
597
597
598 remote_refs = collections.OrderedDict()
598 remote_refs = collections.OrderedDict()
599 fetch_refs = []
599 fetch_refs = []
600
600
601 for ref_line in output.splitlines():
601 for ref_line in output.splitlines():
602 sha, ref = ref_line.split('\t')
602 sha, ref = ref_line.split('\t')
603 sha = sha.strip()
603 sha = sha.strip()
604 if ref in remote_refs:
604 if ref in remote_refs:
605 # duplicate, skip
605 # duplicate, skip
606 continue
606 continue
607 if ref.endswith(PEELED_REF_MARKER):
607 if ref.endswith(PEELED_REF_MARKER):
608 log.debug("Skipping peeled reference %s", ref)
608 log.debug("Skipping peeled reference %s", ref)
609 continue
609 continue
610 # don't sync HEAD
610 # don't sync HEAD
611 if ref in ['HEAD']:
611 if ref in ['HEAD']:
612 continue
612 continue
613
613
614 remote_refs[ref] = sha
614 remote_refs[ref] = sha
615
615
616 if refs and sha in refs:
616 if refs and sha in refs:
617 # we filter fetch using our specified refs
617 # we filter fetch using our specified refs
618 fetch_refs.append('{}:{}'.format(ref, ref))
618 fetch_refs.append('{}:{}'.format(ref, ref))
619 elif not refs:
619 elif not refs:
620 fetch_refs.append('{}:{}'.format(ref, ref))
620 fetch_refs.append('{}:{}'.format(ref, ref))
621 log.debug('Finished obtaining fetch refs, total: %s', len(fetch_refs))
621 log.debug('Finished obtaining fetch refs, total: %s', len(fetch_refs))
622 if fetch_refs:
622 if fetch_refs:
623 for chunk in more_itertools.chunked(fetch_refs, 1024 * 4):
623 for chunk in more_itertools.chunked(fetch_refs, 1024 * 4):
624 fetch_refs_chunks = list(chunk)
624 fetch_refs_chunks = list(chunk)
625 log.debug('Fetching %s refs from import url', len(fetch_refs_chunks))
625 log.debug('Fetching %s refs from import url', len(fetch_refs_chunks))
626 _out, _err = self.run_git_command(
626 _out, _err = self.run_git_command(
627 wire, ['fetch', url, '--force', '--prune', '--'] + fetch_refs_chunks,
627 wire, ['fetch', url, '--force', '--prune', '--'] + fetch_refs_chunks,
628 fail_on_stderr=False,
628 fail_on_stderr=False,
629 _copts=self._remote_conf(config),
629 _copts=self._remote_conf(config),
630 extra_env={'GIT_TERMINAL_PROMPT': '0'})
630 extra_env={'GIT_TERMINAL_PROMPT': '0'})
631
631
632 return remote_refs
632 return remote_refs
633
633
634 @reraise_safe_exceptions
634 @reraise_safe_exceptions
635 def sync_push(self, wire, url, refs=None):
635 def sync_push(self, wire, url, refs=None):
636 if not self.check_url(url, wire):
636 if not self.check_url(url, wire):
637 return
637 return
638 config = self._wire_to_config(wire)
638 config = self._wire_to_config(wire)
639 self._factory.repo(wire)
639 self._factory.repo(wire)
640 self.run_git_command(
640 self.run_git_command(
641 wire, ['push', url, '--mirror'], fail_on_stderr=False,
641 wire, ['push', url, '--mirror'], fail_on_stderr=False,
642 _copts=self._remote_conf(config),
642 _copts=self._remote_conf(config),
643 extra_env={'GIT_TERMINAL_PROMPT': '0'})
643 extra_env={'GIT_TERMINAL_PROMPT': '0'})
644
644
645 @reraise_safe_exceptions
645 @reraise_safe_exceptions
646 def get_remote_refs(self, wire, url):
646 def get_remote_refs(self, wire, url):
647 repo = Repo(url)
647 repo = Repo(url)
648 return repo.get_refs()
648 return repo.get_refs()
649
649
650 @reraise_safe_exceptions
650 @reraise_safe_exceptions
651 def get_description(self, wire):
651 def get_description(self, wire):
652 repo = self._factory.repo(wire)
652 repo = self._factory.repo(wire)
653 return repo.get_description()
653 return repo.get_description()
654
654
655 @reraise_safe_exceptions
655 @reraise_safe_exceptions
656 def get_missing_revs(self, wire, rev1, rev2, path2):
656 def get_missing_revs(self, wire, rev1, rev2, path2):
657 repo = self._factory.repo(wire)
657 repo = self._factory.repo(wire)
658 LocalGitClient(thin_packs=False).fetch(path2, repo)
658 LocalGitClient(thin_packs=False).fetch(path2, repo)
659
659
660 wire_remote = wire.copy()
660 wire_remote = wire.copy()
661 wire_remote['path'] = path2
661 wire_remote['path'] = path2
662 repo_remote = self._factory.repo(wire_remote)
662 repo_remote = self._factory.repo(wire_remote)
663 LocalGitClient(thin_packs=False).fetch(wire["path"], repo_remote)
663 LocalGitClient(thin_packs=False).fetch(wire["path"], repo_remote)
664
664
665 revs = [
665 revs = [
666 x.commit.id
666 x.commit.id
667 for x in repo_remote.get_walker(include=[rev2], exclude=[rev1])]
667 for x in repo_remote.get_walker(include=[rev2], exclude=[rev1])]
668 return revs
668 return revs
669
669
670 @reraise_safe_exceptions
670 @reraise_safe_exceptions
671 def get_object(self, wire, sha):
671 def get_object(self, wire, sha):
672 cache_on, context_uid, repo_id = self._cache_on(wire)
672 cache_on, context_uid, repo_id = self._cache_on(wire)
673 @self.region.conditional_cache_on_arguments(condition=cache_on)
673 @self.region.conditional_cache_on_arguments(condition=cache_on)
674 def _get_object(_context_uid, _repo_id, _sha):
674 def _get_object(_context_uid, _repo_id, _sha):
675 repo_init = self._factory.repo_libgit2(wire)
675 repo_init = self._factory.repo_libgit2(wire)
676 with repo_init as repo:
676 with repo_init as repo:
677
677
678 missing_commit_err = 'Commit {} does not exist for `{}`'.format(sha, wire['path'])
678 missing_commit_err = 'Commit {} does not exist for `{}`'.format(sha, wire['path'])
679 try:
679 try:
680 commit = repo.revparse_single(sha)
680 commit = repo.revparse_single(sha)
681 except (KeyError, ValueError) as e:
681 except (KeyError, ValueError) as e:
682 raise exceptions.LookupException(e)(missing_commit_err)
682 raise exceptions.LookupException(e)(missing_commit_err)
683
683
684 is_tag = False
684 is_tag = False
685 if isinstance(commit, pygit2.Tag):
685 if isinstance(commit, pygit2.Tag):
686 commit = repo.get(commit.target)
686 commit = repo.get(commit.target)
687 is_tag = True
687 is_tag = True
688
688
689 if not is_tag:
689 if not is_tag:
690 # check for dangling commit
690 # check for dangling commit
691 branches = [x for x in repo.branches.with_commit(commit.hex)]
691 branches = [x for x in repo.branches.with_commit(commit.hex)]
692 if not branches:
692 if not branches:
693 raise exceptions.LookupException(None)(missing_commit_err)
693 raise exceptions.LookupException(None)(missing_commit_err)
694
694
695 commit_id = commit.hex
695 commit_id = commit.hex
696 type_id = commit.type
696 type_id = commit.type
697
697
698 return {
698 return {
699 'id': commit_id,
699 'id': commit_id,
700 'type': self._type_id_to_name(type_id),
700 'type': self._type_id_to_name(type_id),
701 'commit_id': commit_id,
701 'commit_id': commit_id,
702 'idx': 0
702 'idx': 0
703 }
703 }
704
704
705 return _get_object(context_uid, repo_id, sha)
705 return _get_object(context_uid, repo_id, sha)
706
706
707 @reraise_safe_exceptions
707 @reraise_safe_exceptions
708 def get_refs(self, wire):
708 def get_refs(self, wire):
709 cache_on, context_uid, repo_id = self._cache_on(wire)
709 cache_on, context_uid, repo_id = self._cache_on(wire)
710 @self.region.conditional_cache_on_arguments(condition=cache_on)
710 @self.region.conditional_cache_on_arguments(condition=cache_on)
711 def _get_refs(_context_uid, _repo_id):
711 def _get_refs(_context_uid, _repo_id):
712
712
713 repo_init = self._factory.repo_libgit2(wire)
713 repo_init = self._factory.repo_libgit2(wire)
714 with repo_init as repo:
714 with repo_init as repo:
715 regex = re.compile('^refs/(heads|tags)/')
715 regex = re.compile('^refs/(heads|tags)/')
716 return {x.name: x.target.hex for x in
716 return {x.name: x.target.hex for x in
717 filter(lambda ref: regex.match(ref.name) ,repo.listall_reference_objects())}
717 filter(lambda ref: regex.match(ref.name) ,repo.listall_reference_objects())}
718
718
719 return _get_refs(context_uid, repo_id)
719 return _get_refs(context_uid, repo_id)
720
720
721 @reraise_safe_exceptions
721 @reraise_safe_exceptions
722 def get_branch_pointers(self, wire):
722 def get_branch_pointers(self, wire):
723 cache_on, context_uid, repo_id = self._cache_on(wire)
723 cache_on, context_uid, repo_id = self._cache_on(wire)
724 @self.region.conditional_cache_on_arguments(condition=cache_on)
724 @self.region.conditional_cache_on_arguments(condition=cache_on)
725 def _get_branch_pointers(_context_uid, _repo_id):
725 def _get_branch_pointers(_context_uid, _repo_id):
726
726
727 repo_init = self._factory.repo_libgit2(wire)
727 repo_init = self._factory.repo_libgit2(wire)
728 regex = re.compile('^refs/heads')
728 regex = re.compile('^refs/heads')
729 with repo_init as repo:
729 with repo_init as repo:
730 branches = filter(lambda ref: regex.match(ref.name), repo.listall_reference_objects())
730 branches = filter(lambda ref: regex.match(ref.name), repo.listall_reference_objects())
731 return {x.target.hex: x.shorthand for x in branches}
731 return {x.target.hex: x.shorthand for x in branches}
732
732
733 return _get_branch_pointers(context_uid, repo_id)
733 return _get_branch_pointers(context_uid, repo_id)
734
734
735 @reraise_safe_exceptions
735 @reraise_safe_exceptions
736 def head(self, wire, show_exc=True):
736 def head(self, wire, show_exc=True):
737 cache_on, context_uid, repo_id = self._cache_on(wire)
737 cache_on, context_uid, repo_id = self._cache_on(wire)
738 @self.region.conditional_cache_on_arguments(condition=cache_on)
738 @self.region.conditional_cache_on_arguments(condition=cache_on)
739 def _head(_context_uid, _repo_id, _show_exc):
739 def _head(_context_uid, _repo_id, _show_exc):
740 repo_init = self._factory.repo_libgit2(wire)
740 repo_init = self._factory.repo_libgit2(wire)
741 with repo_init as repo:
741 with repo_init as repo:
742 try:
742 try:
743 return repo.head.peel().hex
743 return repo.head.peel().hex
744 except Exception:
744 except Exception:
745 if show_exc:
745 if show_exc:
746 raise
746 raise
747 return _head(context_uid, repo_id, show_exc)
747 return _head(context_uid, repo_id, show_exc)
748
748
749 @reraise_safe_exceptions
749 @reraise_safe_exceptions
750 def init(self, wire):
750 def init(self, wire):
751 repo_path = str_to_dulwich(wire['path'])
751 repo_path = str_to_dulwich(wire['path'])
752 self.repo = Repo.init(repo_path)
752 self.repo = Repo.init(repo_path)
753
753
754 @reraise_safe_exceptions
754 @reraise_safe_exceptions
755 def init_bare(self, wire):
755 def init_bare(self, wire):
756 repo_path = str_to_dulwich(wire['path'])
756 repo_path = str_to_dulwich(wire['path'])
757 self.repo = Repo.init_bare(repo_path)
757 self.repo = Repo.init_bare(repo_path)
758
758
759 @reraise_safe_exceptions
759 @reraise_safe_exceptions
760 def revision(self, wire, rev):
760 def revision(self, wire, rev):
761
761
762 cache_on, context_uid, repo_id = self._cache_on(wire)
762 cache_on, context_uid, repo_id = self._cache_on(wire)
763 @self.region.conditional_cache_on_arguments(condition=cache_on)
763 @self.region.conditional_cache_on_arguments(condition=cache_on)
764 def _revision(_context_uid, _repo_id, _rev):
764 def _revision(_context_uid, _repo_id, _rev):
765 repo_init = self._factory.repo_libgit2(wire)
765 repo_init = self._factory.repo_libgit2(wire)
766 with repo_init as repo:
766 with repo_init as repo:
767 commit = repo[rev]
767 commit = repo[rev]
768 obj_data = {
768 obj_data = {
769 'id': commit.id.hex,
769 'id': commit.id.hex,
770 }
770 }
771 # tree objects itself don't have tree_id attribute
771 # tree objects itself don't have tree_id attribute
772 if hasattr(commit, 'tree_id'):
772 if hasattr(commit, 'tree_id'):
773 obj_data['tree'] = commit.tree_id.hex
773 obj_data['tree'] = commit.tree_id.hex
774
774
775 return obj_data
775 return obj_data
776 return _revision(context_uid, repo_id, rev)
776 return _revision(context_uid, repo_id, rev)
777
777
778 @reraise_safe_exceptions
778 @reraise_safe_exceptions
779 def date(self, wire, commit_id):
779 def date(self, wire, commit_id):
780 cache_on, context_uid, repo_id = self._cache_on(wire)
780 cache_on, context_uid, repo_id = self._cache_on(wire)
781 @self.region.conditional_cache_on_arguments(condition=cache_on)
781 @self.region.conditional_cache_on_arguments(condition=cache_on)
782 def _date(_repo_id, _commit_id):
782 def _date(_repo_id, _commit_id):
783 repo_init = self._factory.repo_libgit2(wire)
783 repo_init = self._factory.repo_libgit2(wire)
784 with repo_init as repo:
784 with repo_init as repo:
785 commit = repo[commit_id]
785 commit = repo[commit_id]
786 # TODO(marcink): check dulwich difference of offset vs timezone
786 # TODO(marcink): check dulwich difference of offset vs timezone
787 return [commit.commit_time, commit.commit_time_offset]
787 return [commit.commit_time, commit.commit_time_offset]
788 return _date(repo_id, commit_id)
788 return _date(repo_id, commit_id)
789
789
790 @reraise_safe_exceptions
790 @reraise_safe_exceptions
791 def author(self, wire, commit_id):
791 def author(self, wire, commit_id):
792 cache_on, context_uid, repo_id = self._cache_on(wire)
792 cache_on, context_uid, repo_id = self._cache_on(wire)
793 @self.region.conditional_cache_on_arguments(condition=cache_on)
793 @self.region.conditional_cache_on_arguments(condition=cache_on)
794 def _author(_repo_id, _commit_id):
794 def _author(_repo_id, _commit_id):
795 repo_init = self._factory.repo_libgit2(wire)
795 repo_init = self._factory.repo_libgit2(wire)
796 with repo_init as repo:
796 with repo_init as repo:
797 commit = repo[commit_id]
797 commit = repo[commit_id]
798 if commit.author.email:
798 if commit.author.email:
799 return u"{} <{}>".format(commit.author.name, commit.author.email)
799 return u"{} <{}>".format(commit.author.name, commit.author.email)
800
800
801 return u"{}".format(commit.author.raw_name)
801 return u"{}".format(commit.author.raw_name)
802 return _author(repo_id, commit_id)
802 return _author(repo_id, commit_id)
803
803
804 @reraise_safe_exceptions
804 @reraise_safe_exceptions
805 def message(self, wire, commit_id):
805 def message(self, wire, commit_id):
806 cache_on, context_uid, repo_id = self._cache_on(wire)
806 cache_on, context_uid, repo_id = self._cache_on(wire)
807 @self.region.conditional_cache_on_arguments(condition=cache_on)
807 @self.region.conditional_cache_on_arguments(condition=cache_on)
808 def _message(_repo_id, _commit_id):
808 def _message(_repo_id, _commit_id):
809 repo_init = self._factory.repo_libgit2(wire)
809 repo_init = self._factory.repo_libgit2(wire)
810 with repo_init as repo:
810 with repo_init as repo:
811 commit = repo[commit_id]
811 commit = repo[commit_id]
812 return commit.message
812 return commit.message
813 return _message(repo_id, commit_id)
813 return _message(repo_id, commit_id)
814
814
815 @reraise_safe_exceptions
815 @reraise_safe_exceptions
816 def parents(self, wire, commit_id):
816 def parents(self, wire, commit_id):
817 cache_on, context_uid, repo_id = self._cache_on(wire)
817 cache_on, context_uid, repo_id = self._cache_on(wire)
818 @self.region.conditional_cache_on_arguments(condition=cache_on)
818 @self.region.conditional_cache_on_arguments(condition=cache_on)
819 def _parents(_repo_id, _commit_id):
819 def _parents(_repo_id, _commit_id):
820 repo_init = self._factory.repo_libgit2(wire)
820 repo_init = self._factory.repo_libgit2(wire)
821 with repo_init as repo:
821 with repo_init as repo:
822 commit = repo[commit_id]
822 commit = repo[commit_id]
823 return [x.hex for x in commit.parent_ids]
823 return [x.hex for x in commit.parent_ids]
824 return _parents(repo_id, commit_id)
824 return _parents(repo_id, commit_id)
825
825
826 @reraise_safe_exceptions
826 @reraise_safe_exceptions
827 def children(self, wire, commit_id):
827 def children(self, wire, commit_id):
828 cache_on, context_uid, repo_id = self._cache_on(wire)
828 cache_on, context_uid, repo_id = self._cache_on(wire)
829 @self.region.conditional_cache_on_arguments(condition=cache_on)
829 @self.region.conditional_cache_on_arguments(condition=cache_on)
830 def _children(_repo_id, _commit_id):
830 def _children(_repo_id, _commit_id):
831 output, __ = self.run_git_command(
831 output, __ = self.run_git_command(
832 wire, ['rev-list', '--all', '--children'])
832 wire, ['rev-list', '--all', '--children'])
833
833
834 child_ids = []
834 child_ids = []
835 pat = re.compile(r'^%s' % commit_id)
835 pat = re.compile(r'^%s' % commit_id)
836 for l in output.splitlines():
836 for l in output.splitlines():
837 if pat.match(l):
837 if pat.match(l):
838 found_ids = l.split(' ')[1:]
838 found_ids = l.split(' ')[1:]
839 child_ids.extend(found_ids)
839 child_ids.extend(found_ids)
840
840
841 return child_ids
841 return child_ids
842 return _children(repo_id, commit_id)
842 return _children(repo_id, commit_id)
843
843
844 @reraise_safe_exceptions
844 @reraise_safe_exceptions
845 def set_refs(self, wire, key, value):
845 def set_refs(self, wire, key, value):
846 repo_init = self._factory.repo_libgit2(wire)
846 repo_init = self._factory.repo_libgit2(wire)
847 with repo_init as repo:
847 with repo_init as repo:
848 repo.references.create(key, value, force=True)
848 repo.references.create(key, value, force=True)
849
849
850 @reraise_safe_exceptions
850 @reraise_safe_exceptions
851 def create_branch(self, wire, branch_name, commit_id, force=False):
851 def create_branch(self, wire, branch_name, commit_id, force=False):
852 repo_init = self._factory.repo_libgit2(wire)
852 repo_init = self._factory.repo_libgit2(wire)
853 with repo_init as repo:
853 with repo_init as repo:
854 commit = repo[commit_id]
854 commit = repo[commit_id]
855
855
856 if force:
856 if force:
857 repo.branches.local.create(branch_name, commit, force=force)
857 repo.branches.local.create(branch_name, commit, force=force)
858 elif not repo.branches.get(branch_name):
858 elif not repo.branches.get(branch_name):
859 # create only if that branch isn't existing
859 # create only if that branch isn't existing
860 repo.branches.local.create(branch_name, commit, force=force)
860 repo.branches.local.create(branch_name, commit, force=force)
861
861
862 @reraise_safe_exceptions
862 @reraise_safe_exceptions
863 def remove_ref(self, wire, key):
863 def remove_ref(self, wire, key):
864 repo_init = self._factory.repo_libgit2(wire)
864 repo_init = self._factory.repo_libgit2(wire)
865 with repo_init as repo:
865 with repo_init as repo:
866 repo.references.delete(key)
866 repo.references.delete(key)
867
867
868 @reraise_safe_exceptions
868 @reraise_safe_exceptions
869 def tag_remove(self, wire, tag_name):
869 def tag_remove(self, wire, tag_name):
870 repo_init = self._factory.repo_libgit2(wire)
870 repo_init = self._factory.repo_libgit2(wire)
871 with repo_init as repo:
871 with repo_init as repo:
872 key = 'refs/tags/{}'.format(tag_name)
872 key = 'refs/tags/{}'.format(tag_name)
873 repo.references.delete(key)
873 repo.references.delete(key)
874
874
875 @reraise_safe_exceptions
875 @reraise_safe_exceptions
876 def tree_changes(self, wire, source_id, target_id):
876 def tree_changes(self, wire, source_id, target_id):
877 # TODO(marcink): remove this seems it's only used by tests
877 # TODO(marcink): remove this seems it's only used by tests
878 repo = self._factory.repo(wire)
878 repo = self._factory.repo(wire)
879 source = repo[source_id].tree if source_id else None
879 source = repo[source_id].tree if source_id else None
880 target = repo[target_id].tree
880 target = repo[target_id].tree
881 result = repo.object_store.tree_changes(source, target)
881 result = repo.object_store.tree_changes(source, target)
882 return list(result)
882 return list(result)
883
883
884 @reraise_safe_exceptions
884 @reraise_safe_exceptions
885 def tree_and_type_for_path(self, wire, commit_id, path):
885 def tree_and_type_for_path(self, wire, commit_id, path):
886
886
887 cache_on, context_uid, repo_id = self._cache_on(wire)
887 cache_on, context_uid, repo_id = self._cache_on(wire)
888 @self.region.conditional_cache_on_arguments(condition=cache_on)
888 @self.region.conditional_cache_on_arguments(condition=cache_on)
889 def _tree_and_type_for_path(_context_uid, _repo_id, _commit_id, _path):
889 def _tree_and_type_for_path(_context_uid, _repo_id, _commit_id, _path):
890 repo_init = self._factory.repo_libgit2(wire)
890 repo_init = self._factory.repo_libgit2(wire)
891
891
892 with repo_init as repo:
892 with repo_init as repo:
893 commit = repo[commit_id]
893 commit = repo[commit_id]
894 try:
894 try:
895 tree = commit.tree[path]
895 tree = commit.tree[path]
896 except KeyError:
896 except KeyError:
897 return None, None, None
897 return None, None, None
898
898
899 return tree.id.hex, tree.type, tree.filemode
899 return tree.id.hex, tree.type, tree.filemode
900 return _tree_and_type_for_path(context_uid, repo_id, commit_id, path)
900 return _tree_and_type_for_path(context_uid, repo_id, commit_id, path)
901
901
902 @reraise_safe_exceptions
902 @reraise_safe_exceptions
903 def tree_items(self, wire, tree_id):
903 def tree_items(self, wire, tree_id):
904 cache_on, context_uid, repo_id = self._cache_on(wire)
904 cache_on, context_uid, repo_id = self._cache_on(wire)
905 @self.region.conditional_cache_on_arguments(condition=cache_on)
905 @self.region.conditional_cache_on_arguments(condition=cache_on)
906 def _tree_items(_repo_id, _tree_id):
906 def _tree_items(_repo_id, _tree_id):
907
907
908 repo_init = self._factory.repo_libgit2(wire)
908 repo_init = self._factory.repo_libgit2(wire)
909 with repo_init as repo:
909 with repo_init as repo:
910 try:
910 try:
911 tree = repo[tree_id]
911 tree = repo[tree_id]
912 except KeyError:
912 except KeyError:
913 raise ObjectMissing('No tree with id: {}'.format(tree_id))
913 raise ObjectMissing('No tree with id: {}'.format(tree_id))
914
914
915 result = []
915 result = []
916 for item in tree:
916 for item in tree:
917 item_sha = item.hex
917 item_sha = item.hex
918 item_mode = item.filemode
918 item_mode = item.filemode
919 item_type = item.type
919 item_type = item.type
920
920
921 if item_type == 'commit':
921 if item_type == 'commit':
922 # NOTE(marcink): submodules we translate to 'link' for backward compat
922 # NOTE(marcink): submodules we translate to 'link' for backward compat
923 item_type = 'link'
923 item_type = 'link'
924
924
925 result.append((item.name, item_mode, item_sha, item_type))
925 result.append((item.name, item_mode, item_sha, item_type))
926 return result
926 return result
927 return _tree_items(repo_id, tree_id)
927 return _tree_items(repo_id, tree_id)
928
928
929 @reraise_safe_exceptions
929 @reraise_safe_exceptions
930 def diff(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
930 def diff_2(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
931 """
932 Old version that uses subprocess to call diff
933 """
931
934
932 flags = [
935 flags = [
933 '-U%s' % context, '--full-index', '--binary', '--patch',
936 '-U%s' % context, '--patch',
934 '--find-renames', '--abbrev=40']
937 '--binary',
938 '--find-renames',
939 '--no-indent-heuristic',
940 # '--indent-heuristic',
941 #'--full-index',
942 #'--abbrev=40'
943 ]
935
944
936 if opt_ignorews:
945 if opt_ignorews:
937 flags.append('--ignore-all-space')
946 flags.append('--ignore-all-space')
938
947
939 if commit_id_1 == self.EMPTY_COMMIT:
948 if commit_id_1 == self.EMPTY_COMMIT:
940 cmd = ['show'] + flags + [commit_id_2]
949 cmd = ['show'] + flags + [commit_id_2]
941 else:
950 else:
942 cmd = ['diff'] + flags + [commit_id_1, commit_id_2]
951 cmd = ['diff'] + flags + [commit_id_1, commit_id_2]
943
952
944 if file_filter:
953 if file_filter:
945 cmd.extend(['--', file_filter])
954 cmd.extend(['--', file_filter])
946
955
947 diff, __ = self.run_git_command(wire, cmd)
956 diff, __ = self.run_git_command(wire, cmd)
948 # If we used 'show' command, strip first few lines (until actual diff
957 # If we used 'show' command, strip first few lines (until actual diff
949 # starts)
958 # starts)
950 if commit_id_1 == self.EMPTY_COMMIT:
959 if commit_id_1 == self.EMPTY_COMMIT:
951 lines = diff.splitlines()
960 lines = diff.splitlines()
952 x = 0
961 x = 0
953 for line in lines:
962 for line in lines:
954 if line.startswith('diff'):
963 if line.startswith('diff'):
955 break
964 break
956 x += 1
965 x += 1
957 # Append new line just like 'diff' command do
966 # Append new line just like 'diff' command do
958 diff = '\n'.join(lines[x:]) + '\n'
967 diff = '\n'.join(lines[x:]) + '\n'
968 return diff
959
969
960 return diff
970 @reraise_safe_exceptions
971 def diff(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
972 repo_init = self._factory.repo_libgit2(wire)
973 with repo_init as repo:
974 swap = True
975 flags = 0
976 flags |= pygit2.GIT_DIFF_SHOW_BINARY
977
978 if opt_ignorews:
979 flags |= pygit2.GIT_DIFF_IGNORE_WHITESPACE
980
981 if commit_id_1 == self.EMPTY_COMMIT:
982 comm1 = repo[commit_id_2]
983 diff_obj = comm1.tree.diff_to_tree(
984 flags=flags, context_lines=context, swap=swap)
985
986 else:
987 comm1 = repo[commit_id_2]
988 comm2 = repo[commit_id_1]
989 diff_obj = comm1.tree.diff_to_tree(
990 comm2.tree, flags=flags, context_lines=context, swap=swap)
991
992 diff_obj.find_similar(flags=pygit2.GIT_DIFF_FIND_RENAMES)
993
994 if file_filter:
995 for p in diff_obj:
996 if p.delta.old_file.path == file_filter:
997 return p.patch
998 return diff_obj.patch
961
999
962 @reraise_safe_exceptions
1000 @reraise_safe_exceptions
963 def node_history(self, wire, commit_id, path, limit):
1001 def node_history(self, wire, commit_id, path, limit):
964 cache_on, context_uid, repo_id = self._cache_on(wire)
1002 cache_on, context_uid, repo_id = self._cache_on(wire)
965 @self.region.conditional_cache_on_arguments(condition=cache_on)
1003 @self.region.conditional_cache_on_arguments(condition=cache_on)
966 def _node_history(_context_uid, _repo_id, _commit_id, _path, _limit):
1004 def _node_history(_context_uid, _repo_id, _commit_id, _path, _limit):
967 # optimize for n==1, rev-list is much faster for that use-case
1005 # optimize for n==1, rev-list is much faster for that use-case
968 if limit == 1:
1006 if limit == 1:
969 cmd = ['rev-list', '-1', commit_id, '--', path]
1007 cmd = ['rev-list', '-1', commit_id, '--', path]
970 else:
1008 else:
971 cmd = ['log']
1009 cmd = ['log']
972 if limit:
1010 if limit:
973 cmd.extend(['-n', str(safe_int(limit, 0))])
1011 cmd.extend(['-n', str(safe_int(limit, 0))])
974 cmd.extend(['--pretty=format: %H', '-s', commit_id, '--', path])
1012 cmd.extend(['--pretty=format: %H', '-s', commit_id, '--', path])
975
1013
976 output, __ = self.run_git_command(wire, cmd)
1014 output, __ = self.run_git_command(wire, cmd)
977 commit_ids = re.findall(r'[0-9a-fA-F]{40}', output)
1015 commit_ids = re.findall(r'[0-9a-fA-F]{40}', output)
978
1016
979 return [x for x in commit_ids]
1017 return [x for x in commit_ids]
980 return _node_history(context_uid, repo_id, commit_id, path, limit)
1018 return _node_history(context_uid, repo_id, commit_id, path, limit)
981
1019
982 @reraise_safe_exceptions
1020 @reraise_safe_exceptions
983 def node_annotate(self, wire, commit_id, path):
1021 def node_annotate(self, wire, commit_id, path):
984
1022
985 cmd = ['blame', '-l', '--root', '-r', commit_id, '--', path]
1023 cmd = ['blame', '-l', '--root', '-r', commit_id, '--', path]
986 # -l ==> outputs long shas (and we need all 40 characters)
1024 # -l ==> outputs long shas (and we need all 40 characters)
987 # --root ==> doesn't put '^' character for boundaries
1025 # --root ==> doesn't put '^' character for boundaries
988 # -r commit_id ==> blames for the given commit
1026 # -r commit_id ==> blames for the given commit
989 output, __ = self.run_git_command(wire, cmd)
1027 output, __ = self.run_git_command(wire, cmd)
990
1028
991 result = []
1029 result = []
992 for i, blame_line in enumerate(output.split('\n')[:-1]):
1030 for i, blame_line in enumerate(output.split('\n')[:-1]):
993 line_no = i + 1
1031 line_no = i + 1
994 commit_id, line = re.split(r' ', blame_line, 1)
1032 commit_id, line = re.split(r' ', blame_line, 1)
995 result.append((line_no, commit_id, line))
1033 result.append((line_no, commit_id, line))
996 return result
1034 return result
997
1035
998 @reraise_safe_exceptions
1036 @reraise_safe_exceptions
999 def update_server_info(self, wire):
1037 def update_server_info(self, wire):
1000 repo = self._factory.repo(wire)
1038 repo = self._factory.repo(wire)
1001 update_server_info(repo)
1039 update_server_info(repo)
1002
1040
1003 @reraise_safe_exceptions
1041 @reraise_safe_exceptions
1004 def get_all_commit_ids(self, wire):
1042 def get_all_commit_ids(self, wire):
1005
1043
1006 cache_on, context_uid, repo_id = self._cache_on(wire)
1044 cache_on, context_uid, repo_id = self._cache_on(wire)
1007 @self.region.conditional_cache_on_arguments(condition=cache_on)
1045 @self.region.conditional_cache_on_arguments(condition=cache_on)
1008 def _get_all_commit_ids(_context_uid, _repo_id):
1046 def _get_all_commit_ids(_context_uid, _repo_id):
1009
1047
1010 cmd = ['rev-list', '--reverse', '--date-order', '--branches', '--tags']
1048 cmd = ['rev-list', '--reverse', '--date-order', '--branches', '--tags']
1011 try:
1049 try:
1012 output, __ = self.run_git_command(wire, cmd)
1050 output, __ = self.run_git_command(wire, cmd)
1013 return output.splitlines()
1051 return output.splitlines()
1014 except Exception:
1052 except Exception:
1015 # Can be raised for empty repositories
1053 # Can be raised for empty repositories
1016 return []
1054 return []
1017 return _get_all_commit_ids(context_uid, repo_id)
1055 return _get_all_commit_ids(context_uid, repo_id)
1018
1056
1019 @reraise_safe_exceptions
1057 @reraise_safe_exceptions
1020 def run_git_command(self, wire, cmd, **opts):
1058 def run_git_command(self, wire, cmd, **opts):
1021 path = wire.get('path', None)
1059 path = wire.get('path', None)
1022
1060
1023 if path and os.path.isdir(path):
1061 if path and os.path.isdir(path):
1024 opts['cwd'] = path
1062 opts['cwd'] = path
1025
1063
1026 if '_bare' in opts:
1064 if '_bare' in opts:
1027 _copts = []
1065 _copts = []
1028 del opts['_bare']
1066 del opts['_bare']
1029 else:
1067 else:
1030 _copts = ['-c', 'core.quotepath=false', ]
1068 _copts = ['-c', 'core.quotepath=false', ]
1031 safe_call = False
1069 safe_call = False
1032 if '_safe' in opts:
1070 if '_safe' in opts:
1033 # no exc on failure
1071 # no exc on failure
1034 del opts['_safe']
1072 del opts['_safe']
1035 safe_call = True
1073 safe_call = True
1036
1074
1037 if '_copts' in opts:
1075 if '_copts' in opts:
1038 _copts.extend(opts['_copts'] or [])
1076 _copts.extend(opts['_copts'] or [])
1039 del opts['_copts']
1077 del opts['_copts']
1040
1078
1041 gitenv = os.environ.copy()
1079 gitenv = os.environ.copy()
1042 gitenv.update(opts.pop('extra_env', {}))
1080 gitenv.update(opts.pop('extra_env', {}))
1043 # need to clean fix GIT_DIR !
1081 # need to clean fix GIT_DIR !
1044 if 'GIT_DIR' in gitenv:
1082 if 'GIT_DIR' in gitenv:
1045 del gitenv['GIT_DIR']
1083 del gitenv['GIT_DIR']
1046 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
1084 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
1047 gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1'
1085 gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1'
1048
1086
1049 cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
1087 cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
1050 _opts = {'env': gitenv, 'shell': False}
1088 _opts = {'env': gitenv, 'shell': False}
1051
1089
1052 try:
1090 try:
1053 _opts.update(opts)
1091 _opts.update(opts)
1054 p = subprocessio.SubprocessIOChunker(cmd, **_opts)
1092 p = subprocessio.SubprocessIOChunker(cmd, **_opts)
1055
1093
1056 return ''.join(p), ''.join(p.error)
1094 return ''.join(p), ''.join(p.error)
1057 except (EnvironmentError, OSError) as err:
1095 except (EnvironmentError, OSError) as err:
1058 cmd = ' '.join(cmd) # human friendly CMD
1096 cmd = ' '.join(cmd) # human friendly CMD
1059 tb_err = ("Couldn't run git command (%s).\n"
1097 tb_err = ("Couldn't run git command (%s).\n"
1060 "Original error was:%s\n"
1098 "Original error was:%s\n"
1061 "Call options:%s\n"
1099 "Call options:%s\n"
1062 % (cmd, err, _opts))
1100 % (cmd, err, _opts))
1063 log.exception(tb_err)
1101 log.exception(tb_err)
1064 if safe_call:
1102 if safe_call:
1065 return '', err
1103 return '', err
1066 else:
1104 else:
1067 raise exceptions.VcsException()(tb_err)
1105 raise exceptions.VcsException()(tb_err)
1068
1106
1069 @reraise_safe_exceptions
1107 @reraise_safe_exceptions
1070 def install_hooks(self, wire, force=False):
1108 def install_hooks(self, wire, force=False):
1071 from vcsserver.hook_utils import install_git_hooks
1109 from vcsserver.hook_utils import install_git_hooks
1072 bare = self.bare(wire)
1110 bare = self.bare(wire)
1073 path = wire['path']
1111 path = wire['path']
1074 return install_git_hooks(path, bare, force_create=force)
1112 return install_git_hooks(path, bare, force_create=force)
1075
1113
1076 @reraise_safe_exceptions
1114 @reraise_safe_exceptions
1077 def get_hooks_info(self, wire):
1115 def get_hooks_info(self, wire):
1078 from vcsserver.hook_utils import (
1116 from vcsserver.hook_utils import (
1079 get_git_pre_hook_version, get_git_post_hook_version)
1117 get_git_pre_hook_version, get_git_post_hook_version)
1080 bare = self.bare(wire)
1118 bare = self.bare(wire)
1081 path = wire['path']
1119 path = wire['path']
1082 return {
1120 return {
1083 'pre_version': get_git_pre_hook_version(path, bare),
1121 'pre_version': get_git_pre_hook_version(path, bare),
1084 'post_version': get_git_post_hook_version(path, bare),
1122 'post_version': get_git_post_hook_version(path, bare),
1085 }
1123 }
General Comments 0
You need to be logged in to leave comments. Login now