##// END OF EJS Templates
git: fix remote repo compare logic that wrongly used repos to fetch for cross fork review
super-admin -
r1163:c9256ee2 default
parent child Browse files
Show More
@@ -1,1462 +1,1466 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import collections
18 import collections
19 import logging
19 import logging
20 import os
20 import os
21 import re
21 import re
22 import stat
22 import stat
23 import traceback
23 import traceback
24 import urllib.request
24 import urllib.request
25 import urllib.parse
25 import urllib.parse
26 import urllib.error
26 import urllib.error
27 from functools import wraps
27 from functools import wraps
28
28
29 import more_itertools
29 import more_itertools
30 import pygit2
30 import pygit2
31 from pygit2 import Repository as LibGit2Repo
31 from pygit2 import Repository as LibGit2Repo
32 from pygit2 import index as LibGit2Index
32 from pygit2 import index as LibGit2Index
33 from dulwich import index, objects
33 from dulwich import index, objects
34 from dulwich.client import HttpGitClient, LocalGitClient, FetchPackResult
34 from dulwich.client import HttpGitClient, LocalGitClient, FetchPackResult
35 from dulwich.errors import (
35 from dulwich.errors import (
36 NotGitRepository, ChecksumMismatch, WrongObjectException,
36 NotGitRepository, ChecksumMismatch, WrongObjectException,
37 MissingCommitError, ObjectMissing, HangupException,
37 MissingCommitError, ObjectMissing, HangupException,
38 UnexpectedCommandError)
38 UnexpectedCommandError)
39 from dulwich.repo import Repo as DulwichRepo
39 from dulwich.repo import Repo as DulwichRepo
40 from dulwich.server import update_server_info
40 from dulwich.server import update_server_info
41
41
42 from vcsserver import exceptions, settings, subprocessio
42 from vcsserver import exceptions, settings, subprocessio
43 from vcsserver.str_utils import safe_str, safe_int, safe_bytes, ascii_bytes
43 from vcsserver.str_utils import safe_str, safe_int, safe_bytes, ascii_bytes
44 from vcsserver.base import RepoFactory, obfuscate_qs, ArchiveNode, store_archive_in_cache, BytesEnvelope, BinaryEnvelope
44 from vcsserver.base import RepoFactory, obfuscate_qs, ArchiveNode, store_archive_in_cache, BytesEnvelope, BinaryEnvelope
45 from vcsserver.hgcompat import (
45 from vcsserver.hgcompat import (
46 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
46 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
47 from vcsserver.git_lfs.lib import LFSOidStore
47 from vcsserver.git_lfs.lib import LFSOidStore
48 from vcsserver.vcs_base import RemoteBase
48 from vcsserver.vcs_base import RemoteBase
49
49
50 DIR_STAT = stat.S_IFDIR
50 DIR_STAT = stat.S_IFDIR
51 FILE_MODE = stat.S_IFMT
51 FILE_MODE = stat.S_IFMT
52 GIT_LINK = objects.S_IFGITLINK
52 GIT_LINK = objects.S_IFGITLINK
53 PEELED_REF_MARKER = b'^{}'
53 PEELED_REF_MARKER = b'^{}'
54 HEAD_MARKER = b'HEAD'
54 HEAD_MARKER = b'HEAD'
55
55
56 log = logging.getLogger(__name__)
56 log = logging.getLogger(__name__)
57
57
58
58
59 def reraise_safe_exceptions(func):
59 def reraise_safe_exceptions(func):
60 """Converts Dulwich exceptions to something neutral."""
60 """Converts Dulwich exceptions to something neutral."""
61
61
62 @wraps(func)
62 @wraps(func)
63 def wrapper(*args, **kwargs):
63 def wrapper(*args, **kwargs):
64 try:
64 try:
65 return func(*args, **kwargs)
65 return func(*args, **kwargs)
66 except (ChecksumMismatch, WrongObjectException, MissingCommitError, ObjectMissing,) as e:
66 except (ChecksumMismatch, WrongObjectException, MissingCommitError, ObjectMissing,) as e:
67 exc = exceptions.LookupException(org_exc=e)
67 exc = exceptions.LookupException(org_exc=e)
68 raise exc(safe_str(e))
68 raise exc(safe_str(e))
69 except (HangupException, UnexpectedCommandError) as e:
69 except (HangupException, UnexpectedCommandError) as e:
70 exc = exceptions.VcsException(org_exc=e)
70 exc = exceptions.VcsException(org_exc=e)
71 raise exc(safe_str(e))
71 raise exc(safe_str(e))
72 except Exception:
72 except Exception:
73 # NOTE(marcink): because of how dulwich handles some exceptions
73 # NOTE(marcink): because of how dulwich handles some exceptions
74 # (KeyError on empty repos), we cannot track this and catch all
74 # (KeyError on empty repos), we cannot track this and catch all
75 # exceptions, it's an exceptions from other handlers
75 # exceptions, it's an exceptions from other handlers
76 #if not hasattr(e, '_vcs_kind'):
76 #if not hasattr(e, '_vcs_kind'):
77 #log.exception("Unhandled exception in git remote call")
77 #log.exception("Unhandled exception in git remote call")
78 #raise_from_original(exceptions.UnhandledException)
78 #raise_from_original(exceptions.UnhandledException)
79 raise
79 raise
80 return wrapper
80 return wrapper
81
81
82
82
83 class Repo(DulwichRepo):
83 class Repo(DulwichRepo):
84 """
84 """
85 A wrapper for dulwich Repo class.
85 A wrapper for dulwich Repo class.
86
86
87 Since dulwich is sometimes keeping .idx file descriptors open, it leads to
87 Since dulwich is sometimes keeping .idx file descriptors open, it leads to
88 "Too many open files" error. We need to close all opened file descriptors
88 "Too many open files" error. We need to close all opened file descriptors
89 once the repo object is destroyed.
89 once the repo object is destroyed.
90 """
90 """
91 def __del__(self):
91 def __del__(self):
92 if hasattr(self, 'object_store'):
92 if hasattr(self, 'object_store'):
93 self.close()
93 self.close()
94
94
95
95
96 class Repository(LibGit2Repo):
96 class Repository(LibGit2Repo):
97
97
98 def __enter__(self):
98 def __enter__(self):
99 return self
99 return self
100
100
101 def __exit__(self, exc_type, exc_val, exc_tb):
101 def __exit__(self, exc_type, exc_val, exc_tb):
102 self.free()
102 self.free()
103
103
104
104
105 class GitFactory(RepoFactory):
105 class GitFactory(RepoFactory):
106 repo_type = 'git'
106 repo_type = 'git'
107
107
108 def _create_repo(self, wire, create, use_libgit2=False):
108 def _create_repo(self, wire, create, use_libgit2=False):
109 if use_libgit2:
109 if use_libgit2:
110 repo = Repository(safe_bytes(wire['path']))
110 repo = Repository(safe_bytes(wire['path']))
111 else:
111 else:
112 # dulwich mode
112 # dulwich mode
113 repo_path = safe_str(wire['path'], to_encoding=settings.WIRE_ENCODING)
113 repo_path = safe_str(wire['path'], to_encoding=settings.WIRE_ENCODING)
114 repo = Repo(repo_path)
114 repo = Repo(repo_path)
115
115
116 log.debug('repository created: got GIT object: %s', repo)
116 log.debug('repository created: got GIT object: %s', repo)
117 return repo
117 return repo
118
118
119 def repo(self, wire, create=False, use_libgit2=False):
119 def repo(self, wire, create=False, use_libgit2=False):
120 """
120 """
121 Get a repository instance for the given path.
121 Get a repository instance for the given path.
122 """
122 """
123 return self._create_repo(wire, create, use_libgit2)
123 return self._create_repo(wire, create, use_libgit2)
124
124
125 def repo_libgit2(self, wire):
125 def repo_libgit2(self, wire):
126 return self.repo(wire, use_libgit2=True)
126 return self.repo(wire, use_libgit2=True)
127
127
128
128
129 def create_signature_from_string(author_str, **kwargs):
129 def create_signature_from_string(author_str, **kwargs):
130 """
130 """
131 Creates a pygit2.Signature object from a string of the format 'Name <email>'.
131 Creates a pygit2.Signature object from a string of the format 'Name <email>'.
132
132
133 :param author_str: String of the format 'Name <email>'
133 :param author_str: String of the format 'Name <email>'
134 :return: pygit2.Signature object
134 :return: pygit2.Signature object
135 """
135 """
136 match = re.match(r'^(.+) <(.+)>$', author_str)
136 match = re.match(r'^(.+) <(.+)>$', author_str)
137 if match is None:
137 if match is None:
138 raise ValueError(f"Invalid format: {author_str}")
138 raise ValueError(f"Invalid format: {author_str}")
139
139
140 name, email = match.groups()
140 name, email = match.groups()
141 return pygit2.Signature(name, email, **kwargs)
141 return pygit2.Signature(name, email, **kwargs)
142
142
143
143
144 def get_obfuscated_url(url_obj):
144 def get_obfuscated_url(url_obj):
145 url_obj.passwd = b'*****' if url_obj.passwd else url_obj.passwd
145 url_obj.passwd = b'*****' if url_obj.passwd else url_obj.passwd
146 url_obj.query = obfuscate_qs(url_obj.query)
146 url_obj.query = obfuscate_qs(url_obj.query)
147 obfuscated_uri = str(url_obj)
147 obfuscated_uri = str(url_obj)
148 return obfuscated_uri
148 return obfuscated_uri
149
149
150
150
151 class GitRemote(RemoteBase):
151 class GitRemote(RemoteBase):
152
152
153 def __init__(self, factory):
153 def __init__(self, factory):
154 self._factory = factory
154 self._factory = factory
155 self._bulk_methods = {
155 self._bulk_methods = {
156 "date": self.date,
156 "date": self.date,
157 "author": self.author,
157 "author": self.author,
158 "branch": self.branch,
158 "branch": self.branch,
159 "message": self.message,
159 "message": self.message,
160 "parents": self.parents,
160 "parents": self.parents,
161 "_commit": self.revision,
161 "_commit": self.revision,
162 }
162 }
163 self._bulk_file_methods = {
163 self._bulk_file_methods = {
164 "size": self.get_node_size,
164 "size": self.get_node_size,
165 "data": self.get_node_data,
165 "data": self.get_node_data,
166 "flags": self.get_node_flags,
166 "flags": self.get_node_flags,
167 "is_binary": self.get_node_is_binary,
167 "is_binary": self.get_node_is_binary,
168 "md5": self.md5_hash
168 "md5": self.md5_hash
169 }
169 }
170
170
171 def _wire_to_config(self, wire):
171 def _wire_to_config(self, wire):
172 if 'config' in wire:
172 if 'config' in wire:
173 return {x[0] + '_' + x[1]: x[2] for x in wire['config']}
173 return {x[0] + '_' + x[1]: x[2] for x in wire['config']}
174 return {}
174 return {}
175
175
176 def _remote_conf(self, config):
176 def _remote_conf(self, config):
177 params = [
177 params = [
178 '-c', 'core.askpass=""',
178 '-c', 'core.askpass=""',
179 ]
179 ]
180 ssl_cert_dir = config.get('vcs_ssl_dir')
180 ssl_cert_dir = config.get('vcs_ssl_dir')
181 if ssl_cert_dir:
181 if ssl_cert_dir:
182 params.extend(['-c', f'http.sslCAinfo={ssl_cert_dir}'])
182 params.extend(['-c', f'http.sslCAinfo={ssl_cert_dir}'])
183 return params
183 return params
184
184
185 @reraise_safe_exceptions
185 @reraise_safe_exceptions
186 def discover_git_version(self):
186 def discover_git_version(self):
187 stdout, _ = self.run_git_command(
187 stdout, _ = self.run_git_command(
188 {}, ['--version'], _bare=True, _safe=True)
188 {}, ['--version'], _bare=True, _safe=True)
189 prefix = b'git version'
189 prefix = b'git version'
190 if stdout.startswith(prefix):
190 if stdout.startswith(prefix):
191 stdout = stdout[len(prefix):]
191 stdout = stdout[len(prefix):]
192 return safe_str(stdout.strip())
192 return safe_str(stdout.strip())
193
193
194 @reraise_safe_exceptions
194 @reraise_safe_exceptions
195 def is_empty(self, wire):
195 def is_empty(self, wire):
196 repo_init = self._factory.repo_libgit2(wire)
196 repo_init = self._factory.repo_libgit2(wire)
197 with repo_init as repo:
197 with repo_init as repo:
198
198
199 try:
199 try:
200 has_head = repo.head.name
200 has_head = repo.head.name
201 if has_head:
201 if has_head:
202 return False
202 return False
203
203
204 # NOTE(marcink): check again using more expensive method
204 # NOTE(marcink): check again using more expensive method
205 return repo.is_empty
205 return repo.is_empty
206 except Exception:
206 except Exception:
207 pass
207 pass
208
208
209 return True
209 return True
210
210
211 @reraise_safe_exceptions
211 @reraise_safe_exceptions
212 def assert_correct_path(self, wire):
212 def assert_correct_path(self, wire):
213 cache_on, context_uid, repo_id = self._cache_on(wire)
213 cache_on, context_uid, repo_id = self._cache_on(wire)
214 region = self._region(wire)
214 region = self._region(wire)
215
215
216 @region.conditional_cache_on_arguments(condition=cache_on)
216 @region.conditional_cache_on_arguments(condition=cache_on)
217 def _assert_correct_path(_context_uid, _repo_id, fast_check):
217 def _assert_correct_path(_context_uid, _repo_id, fast_check):
218 if fast_check:
218 if fast_check:
219 path = safe_str(wire['path'])
219 path = safe_str(wire['path'])
220 if pygit2.discover_repository(path):
220 if pygit2.discover_repository(path):
221 return True
221 return True
222 return False
222 return False
223 else:
223 else:
224 try:
224 try:
225 repo_init = self._factory.repo_libgit2(wire)
225 repo_init = self._factory.repo_libgit2(wire)
226 with repo_init:
226 with repo_init:
227 pass
227 pass
228 except pygit2.GitError:
228 except pygit2.GitError:
229 path = wire.get('path')
229 path = wire.get('path')
230 tb = traceback.format_exc()
230 tb = traceback.format_exc()
231 log.debug("Invalid Git path `%s`, tb: %s", path, tb)
231 log.debug("Invalid Git path `%s`, tb: %s", path, tb)
232 return False
232 return False
233 return True
233 return True
234
234
235 return _assert_correct_path(context_uid, repo_id, True)
235 return _assert_correct_path(context_uid, repo_id, True)
236
236
237 @reraise_safe_exceptions
237 @reraise_safe_exceptions
238 def bare(self, wire):
238 def bare(self, wire):
239 repo_init = self._factory.repo_libgit2(wire)
239 repo_init = self._factory.repo_libgit2(wire)
240 with repo_init as repo:
240 with repo_init as repo:
241 return repo.is_bare
241 return repo.is_bare
242
242
243 @reraise_safe_exceptions
243 @reraise_safe_exceptions
244 def get_node_data(self, wire, commit_id, path):
244 def get_node_data(self, wire, commit_id, path):
245 repo_init = self._factory.repo_libgit2(wire)
245 repo_init = self._factory.repo_libgit2(wire)
246 with repo_init as repo:
246 with repo_init as repo:
247 commit = repo[commit_id]
247 commit = repo[commit_id]
248 blob_obj = commit.tree[path]
248 blob_obj = commit.tree[path]
249
249
250 if blob_obj.type != pygit2.GIT_OBJ_BLOB:
250 if blob_obj.type != pygit2.GIT_OBJ_BLOB:
251 raise exceptions.LookupException()(
251 raise exceptions.LookupException()(
252 f'Tree for commit_id:{commit_id} is not a blob: {blob_obj.type_str}')
252 f'Tree for commit_id:{commit_id} is not a blob: {blob_obj.type_str}')
253
253
254 return BytesEnvelope(blob_obj.data)
254 return BytesEnvelope(blob_obj.data)
255
255
256 @reraise_safe_exceptions
256 @reraise_safe_exceptions
257 def get_node_size(self, wire, commit_id, path):
257 def get_node_size(self, wire, commit_id, path):
258 repo_init = self._factory.repo_libgit2(wire)
258 repo_init = self._factory.repo_libgit2(wire)
259 with repo_init as repo:
259 with repo_init as repo:
260 commit = repo[commit_id]
260 commit = repo[commit_id]
261 blob_obj = commit.tree[path]
261 blob_obj = commit.tree[path]
262
262
263 if blob_obj.type != pygit2.GIT_OBJ_BLOB:
263 if blob_obj.type != pygit2.GIT_OBJ_BLOB:
264 raise exceptions.LookupException()(
264 raise exceptions.LookupException()(
265 f'Tree for commit_id:{commit_id} is not a blob: {blob_obj.type_str}')
265 f'Tree for commit_id:{commit_id} is not a blob: {blob_obj.type_str}')
266
266
267 return blob_obj.size
267 return blob_obj.size
268
268
269 @reraise_safe_exceptions
269 @reraise_safe_exceptions
270 def get_node_flags(self, wire, commit_id, path):
270 def get_node_flags(self, wire, commit_id, path):
271 repo_init = self._factory.repo_libgit2(wire)
271 repo_init = self._factory.repo_libgit2(wire)
272 with repo_init as repo:
272 with repo_init as repo:
273 commit = repo[commit_id]
273 commit = repo[commit_id]
274 blob_obj = commit.tree[path]
274 blob_obj = commit.tree[path]
275
275
276 if blob_obj.type != pygit2.GIT_OBJ_BLOB:
276 if blob_obj.type != pygit2.GIT_OBJ_BLOB:
277 raise exceptions.LookupException()(
277 raise exceptions.LookupException()(
278 f'Tree for commit_id:{commit_id} is not a blob: {blob_obj.type_str}')
278 f'Tree for commit_id:{commit_id} is not a blob: {blob_obj.type_str}')
279
279
280 return blob_obj.filemode
280 return blob_obj.filemode
281
281
282 @reraise_safe_exceptions
282 @reraise_safe_exceptions
283 def get_node_is_binary(self, wire, commit_id, path):
283 def get_node_is_binary(self, wire, commit_id, path):
284 repo_init = self._factory.repo_libgit2(wire)
284 repo_init = self._factory.repo_libgit2(wire)
285 with repo_init as repo:
285 with repo_init as repo:
286 commit = repo[commit_id]
286 commit = repo[commit_id]
287 blob_obj = commit.tree[path]
287 blob_obj = commit.tree[path]
288
288
289 if blob_obj.type != pygit2.GIT_OBJ_BLOB:
289 if blob_obj.type != pygit2.GIT_OBJ_BLOB:
290 raise exceptions.LookupException()(
290 raise exceptions.LookupException()(
291 f'Tree for commit_id:{commit_id} is not a blob: {blob_obj.type_str}')
291 f'Tree for commit_id:{commit_id} is not a blob: {blob_obj.type_str}')
292
292
293 return blob_obj.is_binary
293 return blob_obj.is_binary
294
294
295 @reraise_safe_exceptions
295 @reraise_safe_exceptions
296 def blob_as_pretty_string(self, wire, sha):
296 def blob_as_pretty_string(self, wire, sha):
297 repo_init = self._factory.repo_libgit2(wire)
297 repo_init = self._factory.repo_libgit2(wire)
298 with repo_init as repo:
298 with repo_init as repo:
299 blob_obj = repo[sha]
299 blob_obj = repo[sha]
300 return BytesEnvelope(blob_obj.data)
300 return BytesEnvelope(blob_obj.data)
301
301
302 @reraise_safe_exceptions
302 @reraise_safe_exceptions
303 def blob_raw_length(self, wire, sha):
303 def blob_raw_length(self, wire, sha):
304 cache_on, context_uid, repo_id = self._cache_on(wire)
304 cache_on, context_uid, repo_id = self._cache_on(wire)
305 region = self._region(wire)
305 region = self._region(wire)
306
306
307 @region.conditional_cache_on_arguments(condition=cache_on)
307 @region.conditional_cache_on_arguments(condition=cache_on)
308 def _blob_raw_length(_repo_id, _sha):
308 def _blob_raw_length(_repo_id, _sha):
309
309
310 repo_init = self._factory.repo_libgit2(wire)
310 repo_init = self._factory.repo_libgit2(wire)
311 with repo_init as repo:
311 with repo_init as repo:
312 blob = repo[sha]
312 blob = repo[sha]
313 return blob.size
313 return blob.size
314
314
315 return _blob_raw_length(repo_id, sha)
315 return _blob_raw_length(repo_id, sha)
316
316
317 def _parse_lfs_pointer(self, raw_content):
317 def _parse_lfs_pointer(self, raw_content):
318 spec_string = b'version https://git-lfs.github.com/spec'
318 spec_string = b'version https://git-lfs.github.com/spec'
319 if raw_content and raw_content.startswith(spec_string):
319 if raw_content and raw_content.startswith(spec_string):
320
320
321 pattern = re.compile(rb"""
321 pattern = re.compile(rb"""
322 (?:\n)?
322 (?:\n)?
323 ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
323 ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
324 ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
324 ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
325 ^size[ ](?P<oid_size>[0-9]+)\n
325 ^size[ ](?P<oid_size>[0-9]+)\n
326 (?:\n)?
326 (?:\n)?
327 """, re.VERBOSE | re.MULTILINE)
327 """, re.VERBOSE | re.MULTILINE)
328 match = pattern.match(raw_content)
328 match = pattern.match(raw_content)
329 if match:
329 if match:
330 return match.groupdict()
330 return match.groupdict()
331
331
332 return {}
332 return {}
333
333
334 @reraise_safe_exceptions
334 @reraise_safe_exceptions
335 def is_large_file(self, wire, commit_id):
335 def is_large_file(self, wire, commit_id):
336 cache_on, context_uid, repo_id = self._cache_on(wire)
336 cache_on, context_uid, repo_id = self._cache_on(wire)
337 region = self._region(wire)
337 region = self._region(wire)
338
338
339 @region.conditional_cache_on_arguments(condition=cache_on)
339 @region.conditional_cache_on_arguments(condition=cache_on)
340 def _is_large_file(_repo_id, _sha):
340 def _is_large_file(_repo_id, _sha):
341 repo_init = self._factory.repo_libgit2(wire)
341 repo_init = self._factory.repo_libgit2(wire)
342 with repo_init as repo:
342 with repo_init as repo:
343 blob = repo[commit_id]
343 blob = repo[commit_id]
344 if blob.is_binary:
344 if blob.is_binary:
345 return {}
345 return {}
346
346
347 return self._parse_lfs_pointer(blob.data)
347 return self._parse_lfs_pointer(blob.data)
348
348
349 return _is_large_file(repo_id, commit_id)
349 return _is_large_file(repo_id, commit_id)
350
350
351 @reraise_safe_exceptions
351 @reraise_safe_exceptions
352 def is_binary(self, wire, tree_id):
352 def is_binary(self, wire, tree_id):
353 cache_on, context_uid, repo_id = self._cache_on(wire)
353 cache_on, context_uid, repo_id = self._cache_on(wire)
354 region = self._region(wire)
354 region = self._region(wire)
355
355
356 @region.conditional_cache_on_arguments(condition=cache_on)
356 @region.conditional_cache_on_arguments(condition=cache_on)
357 def _is_binary(_repo_id, _tree_id):
357 def _is_binary(_repo_id, _tree_id):
358 repo_init = self._factory.repo_libgit2(wire)
358 repo_init = self._factory.repo_libgit2(wire)
359 with repo_init as repo:
359 with repo_init as repo:
360 blob_obj = repo[tree_id]
360 blob_obj = repo[tree_id]
361 return blob_obj.is_binary
361 return blob_obj.is_binary
362
362
363 return _is_binary(repo_id, tree_id)
363 return _is_binary(repo_id, tree_id)
364
364
365 @reraise_safe_exceptions
365 @reraise_safe_exceptions
366 def md5_hash(self, wire, commit_id, path):
366 def md5_hash(self, wire, commit_id, path):
367 cache_on, context_uid, repo_id = self._cache_on(wire)
367 cache_on, context_uid, repo_id = self._cache_on(wire)
368 region = self._region(wire)
368 region = self._region(wire)
369
369
370 @region.conditional_cache_on_arguments(condition=cache_on)
370 @region.conditional_cache_on_arguments(condition=cache_on)
371 def _md5_hash(_repo_id, _commit_id, _path):
371 def _md5_hash(_repo_id, _commit_id, _path):
372 repo_init = self._factory.repo_libgit2(wire)
372 repo_init = self._factory.repo_libgit2(wire)
373 with repo_init as repo:
373 with repo_init as repo:
374 commit = repo[_commit_id]
374 commit = repo[_commit_id]
375 blob_obj = commit.tree[_path]
375 blob_obj = commit.tree[_path]
376
376
377 if blob_obj.type != pygit2.GIT_OBJ_BLOB:
377 if blob_obj.type != pygit2.GIT_OBJ_BLOB:
378 raise exceptions.LookupException()(
378 raise exceptions.LookupException()(
379 f'Tree for commit_id:{_commit_id} is not a blob: {blob_obj.type_str}')
379 f'Tree for commit_id:{_commit_id} is not a blob: {blob_obj.type_str}')
380
380
381 return ''
381 return ''
382
382
383 return _md5_hash(repo_id, commit_id, path)
383 return _md5_hash(repo_id, commit_id, path)
384
384
385 @reraise_safe_exceptions
385 @reraise_safe_exceptions
386 def in_largefiles_store(self, wire, oid):
386 def in_largefiles_store(self, wire, oid):
387 conf = self._wire_to_config(wire)
387 conf = self._wire_to_config(wire)
388 repo_init = self._factory.repo_libgit2(wire)
388 repo_init = self._factory.repo_libgit2(wire)
389 with repo_init as repo:
389 with repo_init as repo:
390 repo_name = repo.path
390 repo_name = repo.path
391
391
392 store_location = conf.get('vcs_git_lfs_store_location')
392 store_location = conf.get('vcs_git_lfs_store_location')
393 if store_location:
393 if store_location:
394
394
395 store = LFSOidStore(
395 store = LFSOidStore(
396 oid=oid, repo=repo_name, store_location=store_location)
396 oid=oid, repo=repo_name, store_location=store_location)
397 return store.has_oid()
397 return store.has_oid()
398
398
399 return False
399 return False
400
400
401 @reraise_safe_exceptions
401 @reraise_safe_exceptions
402 def store_path(self, wire, oid):
402 def store_path(self, wire, oid):
403 conf = self._wire_to_config(wire)
403 conf = self._wire_to_config(wire)
404 repo_init = self._factory.repo_libgit2(wire)
404 repo_init = self._factory.repo_libgit2(wire)
405 with repo_init as repo:
405 with repo_init as repo:
406 repo_name = repo.path
406 repo_name = repo.path
407
407
408 store_location = conf.get('vcs_git_lfs_store_location')
408 store_location = conf.get('vcs_git_lfs_store_location')
409 if store_location:
409 if store_location:
410 store = LFSOidStore(
410 store = LFSOidStore(
411 oid=oid, repo=repo_name, store_location=store_location)
411 oid=oid, repo=repo_name, store_location=store_location)
412 return store.oid_path
412 return store.oid_path
413 raise ValueError(f'Unable to fetch oid with path {oid}')
413 raise ValueError(f'Unable to fetch oid with path {oid}')
414
414
415 @reraise_safe_exceptions
415 @reraise_safe_exceptions
416 def bulk_request(self, wire, rev, pre_load):
416 def bulk_request(self, wire, rev, pre_load):
417 cache_on, context_uid, repo_id = self._cache_on(wire)
417 cache_on, context_uid, repo_id = self._cache_on(wire)
418 region = self._region(wire)
418 region = self._region(wire)
419
419
420 @region.conditional_cache_on_arguments(condition=cache_on)
420 @region.conditional_cache_on_arguments(condition=cache_on)
421 def _bulk_request(_repo_id, _rev, _pre_load):
421 def _bulk_request(_repo_id, _rev, _pre_load):
422 result = {}
422 result = {}
423 for attr in pre_load:
423 for attr in pre_load:
424 try:
424 try:
425 method = self._bulk_methods[attr]
425 method = self._bulk_methods[attr]
426 wire.update({'cache': False}) # disable cache for bulk calls so we don't double cache
426 wire.update({'cache': False}) # disable cache for bulk calls so we don't double cache
427 args = [wire, rev]
427 args = [wire, rev]
428 result[attr] = method(*args)
428 result[attr] = method(*args)
429 except KeyError as e:
429 except KeyError as e:
430 raise exceptions.VcsException(e)(f"Unknown bulk attribute: {attr}")
430 raise exceptions.VcsException(e)(f"Unknown bulk attribute: {attr}")
431 return result
431 return result
432
432
433 return _bulk_request(repo_id, rev, sorted(pre_load))
433 return _bulk_request(repo_id, rev, sorted(pre_load))
434
434
435 @reraise_safe_exceptions
435 @reraise_safe_exceptions
436 def bulk_file_request(self, wire, commit_id, path, pre_load):
436 def bulk_file_request(self, wire, commit_id, path, pre_load):
437 cache_on, context_uid, repo_id = self._cache_on(wire)
437 cache_on, context_uid, repo_id = self._cache_on(wire)
438 region = self._region(wire)
438 region = self._region(wire)
439
439
440 @region.conditional_cache_on_arguments(condition=cache_on)
440 @region.conditional_cache_on_arguments(condition=cache_on)
441 def _bulk_file_request(_repo_id, _commit_id, _path, _pre_load):
441 def _bulk_file_request(_repo_id, _commit_id, _path, _pre_load):
442 result = {}
442 result = {}
443 for attr in pre_load:
443 for attr in pre_load:
444 try:
444 try:
445 method = self._bulk_file_methods[attr]
445 method = self._bulk_file_methods[attr]
446 wire.update({'cache': False}) # disable cache for bulk calls so we don't double cache
446 wire.update({'cache': False}) # disable cache for bulk calls so we don't double cache
447 result[attr] = method(wire, _commit_id, _path)
447 result[attr] = method(wire, _commit_id, _path)
448 except KeyError as e:
448 except KeyError as e:
449 raise exceptions.VcsException(e)(f'Unknown bulk attribute: "{attr}"')
449 raise exceptions.VcsException(e)(f'Unknown bulk attribute: "{attr}"')
450 return result
450 return result
451
451
452 return BinaryEnvelope(_bulk_file_request(repo_id, commit_id, path, sorted(pre_load)))
452 return BinaryEnvelope(_bulk_file_request(repo_id, commit_id, path, sorted(pre_load)))
453
453
454 def _build_opener(self, url: str):
454 def _build_opener(self, url: str):
455 handlers = []
455 handlers = []
456 url_obj = url_parser(safe_bytes(url))
456 url_obj = url_parser(safe_bytes(url))
457 authinfo = url_obj.authinfo()[1]
457 authinfo = url_obj.authinfo()[1]
458
458
459 if authinfo:
459 if authinfo:
460 # create a password manager
460 # create a password manager
461 passmgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
461 passmgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
462 passmgr.add_password(*authinfo)
462 passmgr.add_password(*authinfo)
463
463
464 handlers.extend((httpbasicauthhandler(passmgr),
464 handlers.extend((httpbasicauthhandler(passmgr),
465 httpdigestauthhandler(passmgr)))
465 httpdigestauthhandler(passmgr)))
466
466
467 return urllib.request.build_opener(*handlers)
467 return urllib.request.build_opener(*handlers)
468
468
469 @reraise_safe_exceptions
469 @reraise_safe_exceptions
470 def check_url(self, url, config):
470 def check_url(self, url, config):
471 url_obj = url_parser(safe_bytes(url))
471 url_obj = url_parser(safe_bytes(url))
472
472
473 test_uri = safe_str(url_obj.authinfo()[0])
473 test_uri = safe_str(url_obj.authinfo()[0])
474 obfuscated_uri = get_obfuscated_url(url_obj)
474 obfuscated_uri = get_obfuscated_url(url_obj)
475
475
476 log.info("Checking URL for remote cloning/import: %s", obfuscated_uri)
476 log.info("Checking URL for remote cloning/import: %s", obfuscated_uri)
477
477
478 if not test_uri.endswith('info/refs'):
478 if not test_uri.endswith('info/refs'):
479 test_uri = test_uri.rstrip('/') + '/info/refs'
479 test_uri = test_uri.rstrip('/') + '/info/refs'
480
480
481 o = self._build_opener(test_uri)
481 o = self._build_opener(test_uri)
482 o.addheaders = [('User-Agent', 'git/1.7.8.0')] # fake some git
482 o.addheaders = [('User-Agent', 'git/1.7.8.0')] # fake some git
483
483
484 q = {"service": 'git-upload-pack'}
484 q = {"service": 'git-upload-pack'}
485 qs = f'?{urllib.parse.urlencode(q)}'
485 qs = f'?{urllib.parse.urlencode(q)}'
486 cu = f"{test_uri}{qs}"
486 cu = f"{test_uri}{qs}"
487
487
488 try:
488 try:
489 req = urllib.request.Request(cu, None, {})
489 req = urllib.request.Request(cu, None, {})
490 log.debug("Trying to open URL %s", obfuscated_uri)
490 log.debug("Trying to open URL %s", obfuscated_uri)
491 resp = o.open(req)
491 resp = o.open(req)
492 if resp.code != 200:
492 if resp.code != 200:
493 raise exceptions.URLError()('Return Code is not 200')
493 raise exceptions.URLError()('Return Code is not 200')
494 except Exception as e:
494 except Exception as e:
495 log.warning("URL cannot be opened: %s", obfuscated_uri, exc_info=True)
495 log.warning("URL cannot be opened: %s", obfuscated_uri, exc_info=True)
496 # means it cannot be cloned
496 # means it cannot be cloned
497 raise exceptions.URLError(e)(f"[{obfuscated_uri}] org_exc: {e}")
497 raise exceptions.URLError(e)(f"[{obfuscated_uri}] org_exc: {e}")
498
498
499 # now detect if it's proper git repo
499 # now detect if it's proper git repo
500 gitdata: bytes = resp.read()
500 gitdata: bytes = resp.read()
501
501
502 if b'service=git-upload-pack' in gitdata:
502 if b'service=git-upload-pack' in gitdata:
503 pass
503 pass
504 elif re.findall(br'[0-9a-fA-F]{40}\s+refs', gitdata):
504 elif re.findall(br'[0-9a-fA-F]{40}\s+refs', gitdata):
505 # old style git can return some other format!
505 # old style git can return some other format!
506 pass
506 pass
507 else:
507 else:
508 e = None
508 e = None
509 raise exceptions.URLError(e)(
509 raise exceptions.URLError(e)(
510 f"url [{obfuscated_uri}] does not look like an hg repo org_exc: {e}")
510 f"url [{obfuscated_uri}] does not look like an hg repo org_exc: {e}")
511
511
512 return True
512 return True
513
513
514 @reraise_safe_exceptions
514 @reraise_safe_exceptions
515 def clone(self, wire, url, deferred, valid_refs, update_after_clone):
515 def clone(self, wire, url, deferred, valid_refs, update_after_clone):
516 # TODO(marcink): deprecate this method. Last i checked we don't use it anymore
516 # TODO(marcink): deprecate this method. Last i checked we don't use it anymore
517 remote_refs = self.pull(wire, url, apply_refs=False)
517 remote_refs = self.pull(wire, url, apply_refs=False)
518 repo = self._factory.repo(wire)
518 repo = self._factory.repo(wire)
519 if isinstance(valid_refs, list):
519 if isinstance(valid_refs, list):
520 valid_refs = tuple(valid_refs)
520 valid_refs = tuple(valid_refs)
521
521
522 for k in remote_refs:
522 for k in remote_refs:
523 # only parse heads/tags and skip so called deferred tags
523 # only parse heads/tags and skip so called deferred tags
524 if k.startswith(valid_refs) and not k.endswith(deferred):
524 if k.startswith(valid_refs) and not k.endswith(deferred):
525 repo[k] = remote_refs[k]
525 repo[k] = remote_refs[k]
526
526
527 if update_after_clone:
527 if update_after_clone:
528 # we want to checkout HEAD
528 # we want to checkout HEAD
529 repo["HEAD"] = remote_refs["HEAD"]
529 repo["HEAD"] = remote_refs["HEAD"]
530 index.build_index_from_tree(repo.path, repo.index_path(),
530 index.build_index_from_tree(repo.path, repo.index_path(),
531 repo.object_store, repo["HEAD"].tree)
531 repo.object_store, repo["HEAD"].tree)
532
532
533 @reraise_safe_exceptions
533 @reraise_safe_exceptions
534 def branch(self, wire, commit_id):
534 def branch(self, wire, commit_id):
535 cache_on, context_uid, repo_id = self._cache_on(wire)
535 cache_on, context_uid, repo_id = self._cache_on(wire)
536 region = self._region(wire)
536 region = self._region(wire)
537
537
538 @region.conditional_cache_on_arguments(condition=cache_on)
538 @region.conditional_cache_on_arguments(condition=cache_on)
539 def _branch(_context_uid, _repo_id, _commit_id):
539 def _branch(_context_uid, _repo_id, _commit_id):
540 regex = re.compile('^refs/heads')
540 regex = re.compile('^refs/heads')
541
541
542 def filter_with(ref):
542 def filter_with(ref):
543 return regex.match(ref[0]) and ref[1] == _commit_id
543 return regex.match(ref[0]) and ref[1] == _commit_id
544
544
545 branches = list(filter(filter_with, list(self.get_refs(wire).items())))
545 branches = list(filter(filter_with, list(self.get_refs(wire).items())))
546 return [x[0].split('refs/heads/')[-1] for x in branches]
546 return [x[0].split('refs/heads/')[-1] for x in branches]
547
547
548 return _branch(context_uid, repo_id, commit_id)
548 return _branch(context_uid, repo_id, commit_id)
549
549
550 @reraise_safe_exceptions
550 @reraise_safe_exceptions
551 def commit_branches(self, wire, commit_id):
551 def commit_branches(self, wire, commit_id):
552 cache_on, context_uid, repo_id = self._cache_on(wire)
552 cache_on, context_uid, repo_id = self._cache_on(wire)
553 region = self._region(wire)
553 region = self._region(wire)
554
554
555 @region.conditional_cache_on_arguments(condition=cache_on)
555 @region.conditional_cache_on_arguments(condition=cache_on)
556 def _commit_branches(_context_uid, _repo_id, _commit_id):
556 def _commit_branches(_context_uid, _repo_id, _commit_id):
557 repo_init = self._factory.repo_libgit2(wire)
557 repo_init = self._factory.repo_libgit2(wire)
558 with repo_init as repo:
558 with repo_init as repo:
559 branches = [x for x in repo.branches.with_commit(_commit_id)]
559 branches = [x for x in repo.branches.with_commit(_commit_id)]
560 return branches
560 return branches
561
561
562 return _commit_branches(context_uid, repo_id, commit_id)
562 return _commit_branches(context_uid, repo_id, commit_id)
563
563
564 @reraise_safe_exceptions
564 @reraise_safe_exceptions
565 def add_object(self, wire, content):
565 def add_object(self, wire, content):
566 repo_init = self._factory.repo_libgit2(wire)
566 repo_init = self._factory.repo_libgit2(wire)
567 with repo_init as repo:
567 with repo_init as repo:
568 blob = objects.Blob()
568 blob = objects.Blob()
569 blob.set_raw_string(content)
569 blob.set_raw_string(content)
570 repo.object_store.add_object(blob)
570 repo.object_store.add_object(blob)
571 return blob.id
571 return blob.id
572
572
573 @reraise_safe_exceptions
573 @reraise_safe_exceptions
574 def create_commit(self, wire, author, committer, message, branch, new_tree_id, date_args: list[int, int] = None):
574 def create_commit(self, wire, author, committer, message, branch, new_tree_id, date_args: list[int, int] = None):
575 repo_init = self._factory.repo_libgit2(wire)
575 repo_init = self._factory.repo_libgit2(wire)
576 with repo_init as repo:
576 with repo_init as repo:
577
577
578 if date_args:
578 if date_args:
579 current_time, offset = date_args
579 current_time, offset = date_args
580
580
581 kw = {
581 kw = {
582 'time': current_time,
582 'time': current_time,
583 'offset': offset
583 'offset': offset
584 }
584 }
585 author = create_signature_from_string(author, **kw)
585 author = create_signature_from_string(author, **kw)
586 committer = create_signature_from_string(committer, **kw)
586 committer = create_signature_from_string(committer, **kw)
587
587
588 tree = new_tree_id
588 tree = new_tree_id
589 if isinstance(tree, (bytes, str)):
589 if isinstance(tree, (bytes, str)):
590 # validate this tree is in the repo...
590 # validate this tree is in the repo...
591 tree = repo[safe_str(tree)].id
591 tree = repo[safe_str(tree)].id
592
592
593 parents = []
593 parents = []
594 # ensure we COMMIT on top of given branch head
594 # ensure we COMMIT on top of given branch head
595 # check if this repo has ANY branches, otherwise it's a new branch case we need to make
595 # check if this repo has ANY branches, otherwise it's a new branch case we need to make
596 if branch in repo.branches.local:
596 if branch in repo.branches.local:
597 parents += [repo.branches[branch].target]
597 parents += [repo.branches[branch].target]
598 elif [x for x in repo.branches.local]:
598 elif [x for x in repo.branches.local]:
599 parents += [repo.head.target]
599 parents += [repo.head.target]
600 #else:
600 #else:
601 # in case we want to commit on new branch we create it on top of HEAD
601 # in case we want to commit on new branch we create it on top of HEAD
602 #repo.branches.local.create(branch, repo.revparse_single('HEAD'))
602 #repo.branches.local.create(branch, repo.revparse_single('HEAD'))
603
603
604 # # Create a new commit
604 # # Create a new commit
605 commit_oid = repo.create_commit(
605 commit_oid = repo.create_commit(
606 f'refs/heads/{branch}', # the name of the reference to update
606 f'refs/heads/{branch}', # the name of the reference to update
607 author, # the author of the commit
607 author, # the author of the commit
608 committer, # the committer of the commit
608 committer, # the committer of the commit
609 message, # the commit message
609 message, # the commit message
610 tree, # the tree produced by the index
610 tree, # the tree produced by the index
611 parents # list of parents for the new commit, usually just one,
611 parents # list of parents for the new commit, usually just one,
612 )
612 )
613
613
614 new_commit_id = safe_str(commit_oid)
614 new_commit_id = safe_str(commit_oid)
615
615
616 return new_commit_id
616 return new_commit_id
617
617
618 @reraise_safe_exceptions
618 @reraise_safe_exceptions
619 def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
619 def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
620
620
621 def mode2pygit(mode):
621 def mode2pygit(mode):
622 """
622 """
623 git only supports two filemode 644 and 755
623 git only supports two filemode 644 and 755
624
624
625 0o100755 -> 33261
625 0o100755 -> 33261
626 0o100644 -> 33188
626 0o100644 -> 33188
627 """
627 """
628 return {
628 return {
629 0o100644: pygit2.GIT_FILEMODE_BLOB,
629 0o100644: pygit2.GIT_FILEMODE_BLOB,
630 0o100755: pygit2.GIT_FILEMODE_BLOB_EXECUTABLE,
630 0o100755: pygit2.GIT_FILEMODE_BLOB_EXECUTABLE,
631 0o120000: pygit2.GIT_FILEMODE_LINK
631 0o120000: pygit2.GIT_FILEMODE_LINK
632 }.get(mode) or pygit2.GIT_FILEMODE_BLOB
632 }.get(mode) or pygit2.GIT_FILEMODE_BLOB
633
633
634 repo_init = self._factory.repo_libgit2(wire)
634 repo_init = self._factory.repo_libgit2(wire)
635 with repo_init as repo:
635 with repo_init as repo:
636 repo_index = repo.index
636 repo_index = repo.index
637
637
638 for pathspec in updated:
638 for pathspec in updated:
639 blob_id = repo.create_blob(pathspec['content'])
639 blob_id = repo.create_blob(pathspec['content'])
640 ie = pygit2.IndexEntry(pathspec['path'], blob_id, mode2pygit(pathspec['mode']))
640 ie = pygit2.IndexEntry(pathspec['path'], blob_id, mode2pygit(pathspec['mode']))
641 repo_index.add(ie)
641 repo_index.add(ie)
642
642
643 for pathspec in removed:
643 for pathspec in removed:
644 repo_index.remove(pathspec)
644 repo_index.remove(pathspec)
645
645
646 # Write changes to the index
646 # Write changes to the index
647 repo_index.write()
647 repo_index.write()
648
648
649 # Create a tree from the updated index
649 # Create a tree from the updated index
650 commit_tree = repo_index.write_tree()
650 commit_tree = repo_index.write_tree()
651
651
652 new_tree_id = commit_tree
652 new_tree_id = commit_tree
653
653
654 author = commit_data['author']
654 author = commit_data['author']
655 committer = commit_data['committer']
655 committer = commit_data['committer']
656 message = commit_data['message']
656 message = commit_data['message']
657
657
658 date_args = [int(commit_data['commit_time']), int(commit_data['commit_timezone'])]
658 date_args = [int(commit_data['commit_time']), int(commit_data['commit_timezone'])]
659
659
660 new_commit_id = self.create_commit(wire, author, committer, message, branch,
660 new_commit_id = self.create_commit(wire, author, committer, message, branch,
661 new_tree_id, date_args=date_args)
661 new_tree_id, date_args=date_args)
662
662
663 # libgit2, ensure the branch is there and exists
663 # libgit2, ensure the branch is there and exists
664 self.create_branch(wire, branch, new_commit_id)
664 self.create_branch(wire, branch, new_commit_id)
665
665
666 # libgit2, set new ref to this created commit
666 # libgit2, set new ref to this created commit
667 self.set_refs(wire, f'refs/heads/{branch}', new_commit_id)
667 self.set_refs(wire, f'refs/heads/{branch}', new_commit_id)
668
668
669 return new_commit_id
669 return new_commit_id
670
670
671 @reraise_safe_exceptions
671 @reraise_safe_exceptions
672 def pull(self, wire, url, apply_refs=True, refs=None, update_after=False):
672 def pull(self, wire, url, apply_refs=True, refs=None, update_after=False):
673 if url != 'default' and '://' not in url:
673 if url != 'default' and '://' not in url:
674 client = LocalGitClient(url)
674 client = LocalGitClient(url)
675 else:
675 else:
676 url_obj = url_parser(safe_bytes(url))
676 url_obj = url_parser(safe_bytes(url))
677 o = self._build_opener(url)
677 o = self._build_opener(url)
678 url = url_obj.authinfo()[0]
678 url = url_obj.authinfo()[0]
679 client = HttpGitClient(base_url=url, opener=o)
679 client = HttpGitClient(base_url=url, opener=o)
680 repo = self._factory.repo(wire)
680 repo = self._factory.repo(wire)
681
681
682 determine_wants = repo.object_store.determine_wants_all
682 determine_wants = repo.object_store.determine_wants_all
683 if refs:
683 if refs:
684 refs = [ascii_bytes(x) for x in refs]
684 refs = [ascii_bytes(x) for x in refs]
685
685
686 def determine_wants_requested(remote_refs):
686 def determine_wants_requested(remote_refs):
687 determined = []
687 determined = []
688 for ref_name, ref_hash in remote_refs.items():
688 for ref_name, ref_hash in remote_refs.items():
689 bytes_ref_name = safe_bytes(ref_name)
689 bytes_ref_name = safe_bytes(ref_name)
690
690
691 if bytes_ref_name in refs:
691 if bytes_ref_name in refs:
692 bytes_ref_hash = safe_bytes(ref_hash)
692 bytes_ref_hash = safe_bytes(ref_hash)
693 determined.append(bytes_ref_hash)
693 determined.append(bytes_ref_hash)
694 return determined
694 return determined
695
695
696 # swap with our custom requested wants
696 # swap with our custom requested wants
697 determine_wants = determine_wants_requested
697 determine_wants = determine_wants_requested
698
698
699 try:
699 try:
700 remote_refs = client.fetch(
700 remote_refs = client.fetch(
701 path=url, target=repo, determine_wants=determine_wants)
701 path=url, target=repo, determine_wants=determine_wants)
702
702
703 except NotGitRepository as e:
703 except NotGitRepository as e:
704 log.warning(
704 log.warning(
705 'Trying to fetch from "%s" failed, not a Git repository.', url)
705 'Trying to fetch from "%s" failed, not a Git repository.', url)
706 # Exception can contain unicode which we convert
706 # Exception can contain unicode which we convert
707 raise exceptions.AbortException(e)(repr(e))
707 raise exceptions.AbortException(e)(repr(e))
708
708
709 # mikhail: client.fetch() returns all the remote refs, but fetches only
709 # mikhail: client.fetch() returns all the remote refs, but fetches only
710 # refs filtered by `determine_wants` function. We need to filter result
710 # refs filtered by `determine_wants` function. We need to filter result
711 # as well
711 # as well
712 if refs:
712 if refs:
713 remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
713 remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
714
714
715 if apply_refs:
715 if apply_refs:
716 # TODO: johbo: Needs proper test coverage with a git repository
716 # TODO: johbo: Needs proper test coverage with a git repository
717 # that contains a tag object, so that we would end up with
717 # that contains a tag object, so that we would end up with
718 # a peeled ref at this point.
718 # a peeled ref at this point.
719 for k in remote_refs:
719 for k in remote_refs:
720 if k.endswith(PEELED_REF_MARKER):
720 if k.endswith(PEELED_REF_MARKER):
721 log.debug("Skipping peeled reference %s", k)
721 log.debug("Skipping peeled reference %s", k)
722 continue
722 continue
723 repo[k] = remote_refs[k]
723 repo[k] = remote_refs[k]
724
724
725 if refs and not update_after:
725 if refs and not update_after:
726 # mikhail: explicitly set the head to the last ref.
726 # mikhail: explicitly set the head to the last ref.
727 repo[HEAD_MARKER] = remote_refs[refs[-1]]
727 repo[HEAD_MARKER] = remote_refs[refs[-1]]
728
728
729 if update_after:
729 if update_after:
730 # we want to check out HEAD
730 # we want to check out HEAD
731 repo[HEAD_MARKER] = remote_refs[HEAD_MARKER]
731 repo[HEAD_MARKER] = remote_refs[HEAD_MARKER]
732 index.build_index_from_tree(repo.path, repo.index_path(),
732 index.build_index_from_tree(repo.path, repo.index_path(),
733 repo.object_store, repo[HEAD_MARKER].tree)
733 repo.object_store, repo[HEAD_MARKER].tree)
734
734
735 if isinstance(remote_refs, FetchPackResult):
735 if isinstance(remote_refs, FetchPackResult):
736 return remote_refs.refs
736 return remote_refs.refs
737 return remote_refs
737 return remote_refs
738
738
739 @reraise_safe_exceptions
739 @reraise_safe_exceptions
740 def sync_fetch(self, wire, url, refs=None, all_refs=False):
740 def sync_fetch(self, wire, url, refs=None, all_refs=False):
741 self._factory.repo(wire)
741 self._factory.repo(wire)
742 if refs and not isinstance(refs, (list, tuple)):
742 if refs and not isinstance(refs, (list, tuple)):
743 refs = [refs]
743 refs = [refs]
744
744
745 config = self._wire_to_config(wire)
745 config = self._wire_to_config(wire)
746 # get all remote refs we'll use to fetch later
746 # get all remote refs we'll use to fetch later
747 cmd = ['ls-remote']
747 cmd = ['ls-remote']
748 if not all_refs:
748 if not all_refs:
749 cmd += ['--heads', '--tags']
749 cmd += ['--heads', '--tags']
750 cmd += [url]
750 cmd += [url]
751 output, __ = self.run_git_command(
751 output, __ = self.run_git_command(
752 wire, cmd, fail_on_stderr=False,
752 wire, cmd, fail_on_stderr=False,
753 _copts=self._remote_conf(config),
753 _copts=self._remote_conf(config),
754 extra_env={'GIT_TERMINAL_PROMPT': '0'})
754 extra_env={'GIT_TERMINAL_PROMPT': '0'})
755
755
756 remote_refs = collections.OrderedDict()
756 remote_refs = collections.OrderedDict()
757 fetch_refs = []
757 fetch_refs = []
758
758
759 for ref_line in output.splitlines():
759 for ref_line in output.splitlines():
760 sha, ref = ref_line.split(b'\t')
760 sha, ref = ref_line.split(b'\t')
761 sha = sha.strip()
761 sha = sha.strip()
762 if ref in remote_refs:
762 if ref in remote_refs:
763 # duplicate, skip
763 # duplicate, skip
764 continue
764 continue
765 if ref.endswith(PEELED_REF_MARKER):
765 if ref.endswith(PEELED_REF_MARKER):
766 log.debug("Skipping peeled reference %s", ref)
766 log.debug("Skipping peeled reference %s", ref)
767 continue
767 continue
768 # don't sync HEAD
768 # don't sync HEAD
769 if ref in [HEAD_MARKER]:
769 if ref in [HEAD_MARKER]:
770 continue
770 continue
771
771
772 remote_refs[ref] = sha
772 remote_refs[ref] = sha
773
773
774 if refs and sha in refs:
774 if refs and sha in refs:
775 # we filter fetch using our specified refs
775 # we filter fetch using our specified refs
776 fetch_refs.append(f'{safe_str(ref)}:{safe_str(ref)}')
776 fetch_refs.append(f'{safe_str(ref)}:{safe_str(ref)}')
777 elif not refs:
777 elif not refs:
778 fetch_refs.append(f'{safe_str(ref)}:{safe_str(ref)}')
778 fetch_refs.append(f'{safe_str(ref)}:{safe_str(ref)}')
779 log.debug('Finished obtaining fetch refs, total: %s', len(fetch_refs))
779 log.debug('Finished obtaining fetch refs, total: %s', len(fetch_refs))
780
780
781 if fetch_refs:
781 if fetch_refs:
782 for chunk in more_itertools.chunked(fetch_refs, 1024 * 4):
782 for chunk in more_itertools.chunked(fetch_refs, 1024 * 4):
783 fetch_refs_chunks = list(chunk)
783 fetch_refs_chunks = list(chunk)
784 log.debug('Fetching %s refs from import url', len(fetch_refs_chunks))
784 log.debug('Fetching %s refs from import url', len(fetch_refs_chunks))
785 self.run_git_command(
785 self.run_git_command(
786 wire, ['fetch', url, '--force', '--prune', '--'] + fetch_refs_chunks,
786 wire, ['fetch', url, '--force', '--prune', '--'] + fetch_refs_chunks,
787 fail_on_stderr=False,
787 fail_on_stderr=False,
788 _copts=self._remote_conf(config),
788 _copts=self._remote_conf(config),
789 extra_env={'GIT_TERMINAL_PROMPT': '0'})
789 extra_env={'GIT_TERMINAL_PROMPT': '0'})
790
790
791 return remote_refs
791 return remote_refs
792
792
793 @reraise_safe_exceptions
793 @reraise_safe_exceptions
794 def sync_push(self, wire, url, refs=None):
794 def sync_push(self, wire, url, refs=None):
795 if not self.check_url(url, wire):
795 if not self.check_url(url, wire):
796 return
796 return
797 config = self._wire_to_config(wire)
797 config = self._wire_to_config(wire)
798 self._factory.repo(wire)
798 self._factory.repo(wire)
799 self.run_git_command(
799 self.run_git_command(
800 wire, ['push', url, '--mirror'], fail_on_stderr=False,
800 wire, ['push', url, '--mirror'], fail_on_stderr=False,
801 _copts=self._remote_conf(config),
801 _copts=self._remote_conf(config),
802 extra_env={'GIT_TERMINAL_PROMPT': '0'})
802 extra_env={'GIT_TERMINAL_PROMPT': '0'})
803
803
804 @reraise_safe_exceptions
804 @reraise_safe_exceptions
805 def get_remote_refs(self, wire, url):
805 def get_remote_refs(self, wire, url):
806 repo = Repo(url)
806 repo = Repo(url)
807 return repo.get_refs()
807 return repo.get_refs()
808
808
809 @reraise_safe_exceptions
809 @reraise_safe_exceptions
810 def get_description(self, wire):
810 def get_description(self, wire):
811 repo = self._factory.repo(wire)
811 repo = self._factory.repo(wire)
812 return repo.get_description()
812 return repo.get_description()
813
813
814 @reraise_safe_exceptions
814 @reraise_safe_exceptions
815 def get_missing_revs(self, wire, rev1, rev2, path2):
815 def get_missing_revs(self, wire, rev1, rev2, other_repo_path):
816 origin_repo_path = wire['path']
816 repo = self._factory.repo(wire)
817 repo = self._factory.repo(wire)
817 LocalGitClient(thin_packs=False).fetch(path2, repo)
818 # fetch from other_repo_path to our origin repo
819 LocalGitClient(thin_packs=False).fetch(other_repo_path, repo)
818
820
819 wire_remote = wire.copy()
821 wire_remote = wire.copy()
820 wire_remote['path'] = path2
822 wire_remote['path'] = other_repo_path
821 repo_remote = self._factory.repo(wire_remote)
823 repo_remote = self._factory.repo(wire_remote)
822 LocalGitClient(thin_packs=False).fetch(path2, repo_remote)
824
825 # fetch from origin_repo_path to our remote repo
826 LocalGitClient(thin_packs=False).fetch(origin_repo_path, repo_remote)
823
827
824 revs = [
828 revs = [
825 x.commit.id
829 x.commit.id
826 for x in repo_remote.get_walker(include=[safe_bytes(rev2)], exclude=[safe_bytes(rev1)])]
830 for x in repo_remote.get_walker(include=[safe_bytes(rev2)], exclude=[safe_bytes(rev1)])]
827 return revs
831 return revs
828
832
829 @reraise_safe_exceptions
833 @reraise_safe_exceptions
830 def get_object(self, wire, sha, maybe_unreachable=False):
834 def get_object(self, wire, sha, maybe_unreachable=False):
831 cache_on, context_uid, repo_id = self._cache_on(wire)
835 cache_on, context_uid, repo_id = self._cache_on(wire)
832 region = self._region(wire)
836 region = self._region(wire)
833
837
834 @region.conditional_cache_on_arguments(condition=cache_on)
838 @region.conditional_cache_on_arguments(condition=cache_on)
835 def _get_object(_context_uid, _repo_id, _sha):
839 def _get_object(_context_uid, _repo_id, _sha):
836 repo_init = self._factory.repo_libgit2(wire)
840 repo_init = self._factory.repo_libgit2(wire)
837 with repo_init as repo:
841 with repo_init as repo:
838
842
839 missing_commit_err = 'Commit {} does not exist for `{}`'.format(sha, wire['path'])
843 missing_commit_err = 'Commit {} does not exist for `{}`'.format(sha, wire['path'])
840 try:
844 try:
841 commit = repo.revparse_single(sha)
845 commit = repo.revparse_single(sha)
842 except KeyError:
846 except KeyError:
843 # NOTE(marcink): KeyError doesn't give us any meaningful information
847 # NOTE(marcink): KeyError doesn't give us any meaningful information
844 # here, we instead give something more explicit
848 # here, we instead give something more explicit
845 e = exceptions.RefNotFoundException('SHA: %s not found', sha)
849 e = exceptions.RefNotFoundException('SHA: %s not found', sha)
846 raise exceptions.LookupException(e)(missing_commit_err)
850 raise exceptions.LookupException(e)(missing_commit_err)
847 except ValueError as e:
851 except ValueError as e:
848 raise exceptions.LookupException(e)(missing_commit_err)
852 raise exceptions.LookupException(e)(missing_commit_err)
849
853
850 is_tag = False
854 is_tag = False
851 if isinstance(commit, pygit2.Tag):
855 if isinstance(commit, pygit2.Tag):
852 commit = repo.get(commit.target)
856 commit = repo.get(commit.target)
853 is_tag = True
857 is_tag = True
854
858
855 check_dangling = True
859 check_dangling = True
856 if is_tag:
860 if is_tag:
857 check_dangling = False
861 check_dangling = False
858
862
859 if check_dangling and maybe_unreachable:
863 if check_dangling and maybe_unreachable:
860 check_dangling = False
864 check_dangling = False
861
865
862 # we used a reference and it parsed means we're not having a dangling commit
866 # we used a reference and it parsed means we're not having a dangling commit
863 if sha != commit.hex:
867 if sha != commit.hex:
864 check_dangling = False
868 check_dangling = False
865
869
866 if check_dangling:
870 if check_dangling:
867 # check for dangling commit
871 # check for dangling commit
868 for branch in repo.branches.with_commit(commit.hex):
872 for branch in repo.branches.with_commit(commit.hex):
869 if branch:
873 if branch:
870 break
874 break
871 else:
875 else:
872 # NOTE(marcink): Empty error doesn't give us any meaningful information
876 # NOTE(marcink): Empty error doesn't give us any meaningful information
873 # here, we instead give something more explicit
877 # here, we instead give something more explicit
874 e = exceptions.RefNotFoundException('SHA: %s not found in branches', sha)
878 e = exceptions.RefNotFoundException('SHA: %s not found in branches', sha)
875 raise exceptions.LookupException(e)(missing_commit_err)
879 raise exceptions.LookupException(e)(missing_commit_err)
876
880
877 commit_id = commit.hex
881 commit_id = commit.hex
878 type_str = commit.type_str
882 type_str = commit.type_str
879
883
880 return {
884 return {
881 'id': commit_id,
885 'id': commit_id,
882 'type': type_str,
886 'type': type_str,
883 'commit_id': commit_id,
887 'commit_id': commit_id,
884 'idx': 0
888 'idx': 0
885 }
889 }
886
890
887 return _get_object(context_uid, repo_id, sha)
891 return _get_object(context_uid, repo_id, sha)
888
892
889 @reraise_safe_exceptions
893 @reraise_safe_exceptions
890 def get_refs(self, wire):
894 def get_refs(self, wire):
891 cache_on, context_uid, repo_id = self._cache_on(wire)
895 cache_on, context_uid, repo_id = self._cache_on(wire)
892 region = self._region(wire)
896 region = self._region(wire)
893
897
894 @region.conditional_cache_on_arguments(condition=cache_on)
898 @region.conditional_cache_on_arguments(condition=cache_on)
895 def _get_refs(_context_uid, _repo_id):
899 def _get_refs(_context_uid, _repo_id):
896
900
897 repo_init = self._factory.repo_libgit2(wire)
901 repo_init = self._factory.repo_libgit2(wire)
898 with repo_init as repo:
902 with repo_init as repo:
899 regex = re.compile('^refs/(heads|tags)/')
903 regex = re.compile('^refs/(heads|tags)/')
900 return {x.name: x.target.hex for x in
904 return {x.name: x.target.hex for x in
901 [ref for ref in repo.listall_reference_objects() if regex.match(ref.name)]}
905 [ref for ref in repo.listall_reference_objects() if regex.match(ref.name)]}
902
906
903 return _get_refs(context_uid, repo_id)
907 return _get_refs(context_uid, repo_id)
904
908
905 @reraise_safe_exceptions
909 @reraise_safe_exceptions
906 def get_branch_pointers(self, wire):
910 def get_branch_pointers(self, wire):
907 cache_on, context_uid, repo_id = self._cache_on(wire)
911 cache_on, context_uid, repo_id = self._cache_on(wire)
908 region = self._region(wire)
912 region = self._region(wire)
909
913
910 @region.conditional_cache_on_arguments(condition=cache_on)
914 @region.conditional_cache_on_arguments(condition=cache_on)
911 def _get_branch_pointers(_context_uid, _repo_id):
915 def _get_branch_pointers(_context_uid, _repo_id):
912
916
913 repo_init = self._factory.repo_libgit2(wire)
917 repo_init = self._factory.repo_libgit2(wire)
914 regex = re.compile('^refs/heads')
918 regex = re.compile('^refs/heads')
915 with repo_init as repo:
919 with repo_init as repo:
916 branches = [ref for ref in repo.listall_reference_objects() if regex.match(ref.name)]
920 branches = [ref for ref in repo.listall_reference_objects() if regex.match(ref.name)]
917 return {x.target.hex: x.shorthand for x in branches}
921 return {x.target.hex: x.shorthand for x in branches}
918
922
919 return _get_branch_pointers(context_uid, repo_id)
923 return _get_branch_pointers(context_uid, repo_id)
920
924
921 @reraise_safe_exceptions
925 @reraise_safe_exceptions
922 def head(self, wire, show_exc=True):
926 def head(self, wire, show_exc=True):
923 cache_on, context_uid, repo_id = self._cache_on(wire)
927 cache_on, context_uid, repo_id = self._cache_on(wire)
924 region = self._region(wire)
928 region = self._region(wire)
925
929
926 @region.conditional_cache_on_arguments(condition=cache_on)
930 @region.conditional_cache_on_arguments(condition=cache_on)
927 def _head(_context_uid, _repo_id, _show_exc):
931 def _head(_context_uid, _repo_id, _show_exc):
928 repo_init = self._factory.repo_libgit2(wire)
932 repo_init = self._factory.repo_libgit2(wire)
929 with repo_init as repo:
933 with repo_init as repo:
930 try:
934 try:
931 return repo.head.peel().hex
935 return repo.head.peel().hex
932 except Exception:
936 except Exception:
933 if show_exc:
937 if show_exc:
934 raise
938 raise
935 return _head(context_uid, repo_id, show_exc)
939 return _head(context_uid, repo_id, show_exc)
936
940
937 @reraise_safe_exceptions
941 @reraise_safe_exceptions
938 def init(self, wire):
942 def init(self, wire):
939 repo_path = safe_str(wire['path'])
943 repo_path = safe_str(wire['path'])
940 pygit2.init_repository(repo_path, bare=False)
944 pygit2.init_repository(repo_path, bare=False)
941
945
942 @reraise_safe_exceptions
946 @reraise_safe_exceptions
943 def init_bare(self, wire):
947 def init_bare(self, wire):
944 repo_path = safe_str(wire['path'])
948 repo_path = safe_str(wire['path'])
945 pygit2.init_repository(repo_path, bare=True)
949 pygit2.init_repository(repo_path, bare=True)
946
950
947 @reraise_safe_exceptions
951 @reraise_safe_exceptions
948 def revision(self, wire, rev):
952 def revision(self, wire, rev):
949
953
950 cache_on, context_uid, repo_id = self._cache_on(wire)
954 cache_on, context_uid, repo_id = self._cache_on(wire)
951 region = self._region(wire)
955 region = self._region(wire)
952
956
953 @region.conditional_cache_on_arguments(condition=cache_on)
957 @region.conditional_cache_on_arguments(condition=cache_on)
954 def _revision(_context_uid, _repo_id, _rev):
958 def _revision(_context_uid, _repo_id, _rev):
955 repo_init = self._factory.repo_libgit2(wire)
959 repo_init = self._factory.repo_libgit2(wire)
956 with repo_init as repo:
960 with repo_init as repo:
957 commit = repo[rev]
961 commit = repo[rev]
958 obj_data = {
962 obj_data = {
959 'id': commit.id.hex,
963 'id': commit.id.hex,
960 }
964 }
961 # tree objects itself don't have tree_id attribute
965 # tree objects itself don't have tree_id attribute
962 if hasattr(commit, 'tree_id'):
966 if hasattr(commit, 'tree_id'):
963 obj_data['tree'] = commit.tree_id.hex
967 obj_data['tree'] = commit.tree_id.hex
964
968
965 return obj_data
969 return obj_data
966 return _revision(context_uid, repo_id, rev)
970 return _revision(context_uid, repo_id, rev)
967
971
968 @reraise_safe_exceptions
972 @reraise_safe_exceptions
969 def date(self, wire, commit_id):
973 def date(self, wire, commit_id):
970 cache_on, context_uid, repo_id = self._cache_on(wire)
974 cache_on, context_uid, repo_id = self._cache_on(wire)
971 region = self._region(wire)
975 region = self._region(wire)
972
976
973 @region.conditional_cache_on_arguments(condition=cache_on)
977 @region.conditional_cache_on_arguments(condition=cache_on)
974 def _date(_repo_id, _commit_id):
978 def _date(_repo_id, _commit_id):
975 repo_init = self._factory.repo_libgit2(wire)
979 repo_init = self._factory.repo_libgit2(wire)
976 with repo_init as repo:
980 with repo_init as repo:
977 commit = repo[commit_id]
981 commit = repo[commit_id]
978
982
979 if hasattr(commit, 'commit_time'):
983 if hasattr(commit, 'commit_time'):
980 commit_time, commit_time_offset = commit.commit_time, commit.commit_time_offset
984 commit_time, commit_time_offset = commit.commit_time, commit.commit_time_offset
981 else:
985 else:
982 commit = commit.get_object()
986 commit = commit.get_object()
983 commit_time, commit_time_offset = commit.commit_time, commit.commit_time_offset
987 commit_time, commit_time_offset = commit.commit_time, commit.commit_time_offset
984
988
985 # TODO(marcink): check dulwich difference of offset vs timezone
989 # TODO(marcink): check dulwich difference of offset vs timezone
986 return [commit_time, commit_time_offset]
990 return [commit_time, commit_time_offset]
987 return _date(repo_id, commit_id)
991 return _date(repo_id, commit_id)
988
992
989 @reraise_safe_exceptions
993 @reraise_safe_exceptions
990 def author(self, wire, commit_id):
994 def author(self, wire, commit_id):
991 cache_on, context_uid, repo_id = self._cache_on(wire)
995 cache_on, context_uid, repo_id = self._cache_on(wire)
992 region = self._region(wire)
996 region = self._region(wire)
993
997
994 @region.conditional_cache_on_arguments(condition=cache_on)
998 @region.conditional_cache_on_arguments(condition=cache_on)
995 def _author(_repo_id, _commit_id):
999 def _author(_repo_id, _commit_id):
996 repo_init = self._factory.repo_libgit2(wire)
1000 repo_init = self._factory.repo_libgit2(wire)
997 with repo_init as repo:
1001 with repo_init as repo:
998 commit = repo[commit_id]
1002 commit = repo[commit_id]
999
1003
1000 if hasattr(commit, 'author'):
1004 if hasattr(commit, 'author'):
1001 author = commit.author
1005 author = commit.author
1002 else:
1006 else:
1003 author = commit.get_object().author
1007 author = commit.get_object().author
1004
1008
1005 if author.email:
1009 if author.email:
1006 return f"{author.name} <{author.email}>"
1010 return f"{author.name} <{author.email}>"
1007
1011
1008 try:
1012 try:
1009 return f"{author.name}"
1013 return f"{author.name}"
1010 except Exception:
1014 except Exception:
1011 return f"{safe_str(author.raw_name)}"
1015 return f"{safe_str(author.raw_name)}"
1012
1016
1013 return _author(repo_id, commit_id)
1017 return _author(repo_id, commit_id)
1014
1018
1015 @reraise_safe_exceptions
1019 @reraise_safe_exceptions
1016 def message(self, wire, commit_id):
1020 def message(self, wire, commit_id):
1017 cache_on, context_uid, repo_id = self._cache_on(wire)
1021 cache_on, context_uid, repo_id = self._cache_on(wire)
1018 region = self._region(wire)
1022 region = self._region(wire)
1019
1023
1020 @region.conditional_cache_on_arguments(condition=cache_on)
1024 @region.conditional_cache_on_arguments(condition=cache_on)
1021 def _message(_repo_id, _commit_id):
1025 def _message(_repo_id, _commit_id):
1022 repo_init = self._factory.repo_libgit2(wire)
1026 repo_init = self._factory.repo_libgit2(wire)
1023 with repo_init as repo:
1027 with repo_init as repo:
1024 commit = repo[commit_id]
1028 commit = repo[commit_id]
1025 return commit.message
1029 return commit.message
1026 return _message(repo_id, commit_id)
1030 return _message(repo_id, commit_id)
1027
1031
1028 @reraise_safe_exceptions
1032 @reraise_safe_exceptions
1029 def parents(self, wire, commit_id):
1033 def parents(self, wire, commit_id):
1030 cache_on, context_uid, repo_id = self._cache_on(wire)
1034 cache_on, context_uid, repo_id = self._cache_on(wire)
1031 region = self._region(wire)
1035 region = self._region(wire)
1032
1036
1033 @region.conditional_cache_on_arguments(condition=cache_on)
1037 @region.conditional_cache_on_arguments(condition=cache_on)
1034 def _parents(_repo_id, _commit_id):
1038 def _parents(_repo_id, _commit_id):
1035 repo_init = self._factory.repo_libgit2(wire)
1039 repo_init = self._factory.repo_libgit2(wire)
1036 with repo_init as repo:
1040 with repo_init as repo:
1037 commit = repo[commit_id]
1041 commit = repo[commit_id]
1038 if hasattr(commit, 'parent_ids'):
1042 if hasattr(commit, 'parent_ids'):
1039 parent_ids = commit.parent_ids
1043 parent_ids = commit.parent_ids
1040 else:
1044 else:
1041 parent_ids = commit.get_object().parent_ids
1045 parent_ids = commit.get_object().parent_ids
1042
1046
1043 return [x.hex for x in parent_ids]
1047 return [x.hex for x in parent_ids]
1044 return _parents(repo_id, commit_id)
1048 return _parents(repo_id, commit_id)
1045
1049
1046 @reraise_safe_exceptions
1050 @reraise_safe_exceptions
1047 def children(self, wire, commit_id):
1051 def children(self, wire, commit_id):
1048 cache_on, context_uid, repo_id = self._cache_on(wire)
1052 cache_on, context_uid, repo_id = self._cache_on(wire)
1049 region = self._region(wire)
1053 region = self._region(wire)
1050
1054
1051 head = self.head(wire)
1055 head = self.head(wire)
1052
1056
1053 @region.conditional_cache_on_arguments(condition=cache_on)
1057 @region.conditional_cache_on_arguments(condition=cache_on)
1054 def _children(_repo_id, _commit_id):
1058 def _children(_repo_id, _commit_id):
1055
1059
1056 output, __ = self.run_git_command(
1060 output, __ = self.run_git_command(
1057 wire, ['rev-list', '--all', '--children', f'{commit_id}^..{head}'])
1061 wire, ['rev-list', '--all', '--children', f'{commit_id}^..{head}'])
1058
1062
1059 child_ids = []
1063 child_ids = []
1060 pat = re.compile(fr'^{commit_id}')
1064 pat = re.compile(fr'^{commit_id}')
1061 for line in output.splitlines():
1065 for line in output.splitlines():
1062 line = safe_str(line)
1066 line = safe_str(line)
1063 if pat.match(line):
1067 if pat.match(line):
1064 found_ids = line.split(' ')[1:]
1068 found_ids = line.split(' ')[1:]
1065 child_ids.extend(found_ids)
1069 child_ids.extend(found_ids)
1066 break
1070 break
1067
1071
1068 return child_ids
1072 return child_ids
1069 return _children(repo_id, commit_id)
1073 return _children(repo_id, commit_id)
1070
1074
1071 @reraise_safe_exceptions
1075 @reraise_safe_exceptions
1072 def set_refs(self, wire, key, value):
1076 def set_refs(self, wire, key, value):
1073 repo_init = self._factory.repo_libgit2(wire)
1077 repo_init = self._factory.repo_libgit2(wire)
1074 with repo_init as repo:
1078 with repo_init as repo:
1075 repo.references.create(key, value, force=True)
1079 repo.references.create(key, value, force=True)
1076
1080
1077 @reraise_safe_exceptions
1081 @reraise_safe_exceptions
1078 def create_branch(self, wire, branch_name, commit_id, force=False):
1082 def create_branch(self, wire, branch_name, commit_id, force=False):
1079 repo_init = self._factory.repo_libgit2(wire)
1083 repo_init = self._factory.repo_libgit2(wire)
1080 with repo_init as repo:
1084 with repo_init as repo:
1081 if commit_id:
1085 if commit_id:
1082 commit = repo[commit_id]
1086 commit = repo[commit_id]
1083 else:
1087 else:
1084 # if commit is not given just use the HEAD
1088 # if commit is not given just use the HEAD
1085 commit = repo.head()
1089 commit = repo.head()
1086
1090
1087 if force:
1091 if force:
1088 repo.branches.local.create(branch_name, commit, force=force)
1092 repo.branches.local.create(branch_name, commit, force=force)
1089 elif not repo.branches.get(branch_name):
1093 elif not repo.branches.get(branch_name):
1090 # create only if that branch isn't existing
1094 # create only if that branch isn't existing
1091 repo.branches.local.create(branch_name, commit, force=force)
1095 repo.branches.local.create(branch_name, commit, force=force)
1092
1096
1093 @reraise_safe_exceptions
1097 @reraise_safe_exceptions
1094 def remove_ref(self, wire, key):
1098 def remove_ref(self, wire, key):
1095 repo_init = self._factory.repo_libgit2(wire)
1099 repo_init = self._factory.repo_libgit2(wire)
1096 with repo_init as repo:
1100 with repo_init as repo:
1097 repo.references.delete(key)
1101 repo.references.delete(key)
1098
1102
1099 @reraise_safe_exceptions
1103 @reraise_safe_exceptions
1100 def tag_remove(self, wire, tag_name):
1104 def tag_remove(self, wire, tag_name):
1101 repo_init = self._factory.repo_libgit2(wire)
1105 repo_init = self._factory.repo_libgit2(wire)
1102 with repo_init as repo:
1106 with repo_init as repo:
1103 key = f'refs/tags/{tag_name}'
1107 key = f'refs/tags/{tag_name}'
1104 repo.references.delete(key)
1108 repo.references.delete(key)
1105
1109
1106 @reraise_safe_exceptions
1110 @reraise_safe_exceptions
1107 def tree_changes(self, wire, source_id, target_id):
1111 def tree_changes(self, wire, source_id, target_id):
1108 repo = self._factory.repo(wire)
1112 repo = self._factory.repo(wire)
1109 # source can be empty
1113 # source can be empty
1110 source_id = safe_bytes(source_id if source_id else b'')
1114 source_id = safe_bytes(source_id if source_id else b'')
1111 target_id = safe_bytes(target_id)
1115 target_id = safe_bytes(target_id)
1112
1116
1113 source = repo[source_id].tree if source_id else None
1117 source = repo[source_id].tree if source_id else None
1114 target = repo[target_id].tree
1118 target = repo[target_id].tree
1115 result = repo.object_store.tree_changes(source, target)
1119 result = repo.object_store.tree_changes(source, target)
1116
1120
1117 added = set()
1121 added = set()
1118 modified = set()
1122 modified = set()
1119 deleted = set()
1123 deleted = set()
1120 for (old_path, new_path), (_, _), (_, _) in list(result):
1124 for (old_path, new_path), (_, _), (_, _) in list(result):
1121 if new_path and old_path:
1125 if new_path and old_path:
1122 modified.add(new_path)
1126 modified.add(new_path)
1123 elif new_path and not old_path:
1127 elif new_path and not old_path:
1124 added.add(new_path)
1128 added.add(new_path)
1125 elif not new_path and old_path:
1129 elif not new_path and old_path:
1126 deleted.add(old_path)
1130 deleted.add(old_path)
1127
1131
1128 return list(added), list(modified), list(deleted)
1132 return list(added), list(modified), list(deleted)
1129
1133
1130 @reraise_safe_exceptions
1134 @reraise_safe_exceptions
1131 def tree_and_type_for_path(self, wire, commit_id, path):
1135 def tree_and_type_for_path(self, wire, commit_id, path):
1132
1136
1133 cache_on, context_uid, repo_id = self._cache_on(wire)
1137 cache_on, context_uid, repo_id = self._cache_on(wire)
1134 region = self._region(wire)
1138 region = self._region(wire)
1135
1139
1136 @region.conditional_cache_on_arguments(condition=cache_on)
1140 @region.conditional_cache_on_arguments(condition=cache_on)
1137 def _tree_and_type_for_path(_context_uid, _repo_id, _commit_id, _path):
1141 def _tree_and_type_for_path(_context_uid, _repo_id, _commit_id, _path):
1138 repo_init = self._factory.repo_libgit2(wire)
1142 repo_init = self._factory.repo_libgit2(wire)
1139
1143
1140 with repo_init as repo:
1144 with repo_init as repo:
1141 commit = repo[commit_id]
1145 commit = repo[commit_id]
1142 try:
1146 try:
1143 tree = commit.tree[path]
1147 tree = commit.tree[path]
1144 except KeyError:
1148 except KeyError:
1145 return None, None, None
1149 return None, None, None
1146
1150
1147 return tree.id.hex, tree.type_str, tree.filemode
1151 return tree.id.hex, tree.type_str, tree.filemode
1148 return _tree_and_type_for_path(context_uid, repo_id, commit_id, path)
1152 return _tree_and_type_for_path(context_uid, repo_id, commit_id, path)
1149
1153
1150 @reraise_safe_exceptions
1154 @reraise_safe_exceptions
1151 def tree_items(self, wire, tree_id):
1155 def tree_items(self, wire, tree_id):
1152 cache_on, context_uid, repo_id = self._cache_on(wire)
1156 cache_on, context_uid, repo_id = self._cache_on(wire)
1153 region = self._region(wire)
1157 region = self._region(wire)
1154
1158
1155 @region.conditional_cache_on_arguments(condition=cache_on)
1159 @region.conditional_cache_on_arguments(condition=cache_on)
1156 def _tree_items(_repo_id, _tree_id):
1160 def _tree_items(_repo_id, _tree_id):
1157
1161
1158 repo_init = self._factory.repo_libgit2(wire)
1162 repo_init = self._factory.repo_libgit2(wire)
1159 with repo_init as repo:
1163 with repo_init as repo:
1160 try:
1164 try:
1161 tree = repo[tree_id]
1165 tree = repo[tree_id]
1162 except KeyError:
1166 except KeyError:
1163 raise ObjectMissing(f'No tree with id: {tree_id}')
1167 raise ObjectMissing(f'No tree with id: {tree_id}')
1164
1168
1165 result = []
1169 result = []
1166 for item in tree:
1170 for item in tree:
1167 item_sha = item.hex
1171 item_sha = item.hex
1168 item_mode = item.filemode
1172 item_mode = item.filemode
1169 item_type = item.type_str
1173 item_type = item.type_str
1170
1174
1171 if item_type == 'commit':
1175 if item_type == 'commit':
1172 # NOTE(marcink): submodules we translate to 'link' for backward compat
1176 # NOTE(marcink): submodules we translate to 'link' for backward compat
1173 item_type = 'link'
1177 item_type = 'link'
1174
1178
1175 result.append((item.name, item_mode, item_sha, item_type))
1179 result.append((item.name, item_mode, item_sha, item_type))
1176 return result
1180 return result
1177 return _tree_items(repo_id, tree_id)
1181 return _tree_items(repo_id, tree_id)
1178
1182
1179 @reraise_safe_exceptions
1183 @reraise_safe_exceptions
1180 def diff_2(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
1184 def diff_2(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
1181 """
1185 """
1182 Old version that uses subprocess to call diff
1186 Old version that uses subprocess to call diff
1183 """
1187 """
1184
1188
1185 flags = [
1189 flags = [
1186 f'-U{context}', '--patch',
1190 f'-U{context}', '--patch',
1187 '--binary',
1191 '--binary',
1188 '--find-renames',
1192 '--find-renames',
1189 '--no-indent-heuristic',
1193 '--no-indent-heuristic',
1190 # '--indent-heuristic',
1194 # '--indent-heuristic',
1191 #'--full-index',
1195 #'--full-index',
1192 #'--abbrev=40'
1196 #'--abbrev=40'
1193 ]
1197 ]
1194
1198
1195 if opt_ignorews:
1199 if opt_ignorews:
1196 flags.append('--ignore-all-space')
1200 flags.append('--ignore-all-space')
1197
1201
1198 if commit_id_1 == self.EMPTY_COMMIT:
1202 if commit_id_1 == self.EMPTY_COMMIT:
1199 cmd = ['show'] + flags + [commit_id_2]
1203 cmd = ['show'] + flags + [commit_id_2]
1200 else:
1204 else:
1201 cmd = ['diff'] + flags + [commit_id_1, commit_id_2]
1205 cmd = ['diff'] + flags + [commit_id_1, commit_id_2]
1202
1206
1203 if file_filter:
1207 if file_filter:
1204 cmd.extend(['--', file_filter])
1208 cmd.extend(['--', file_filter])
1205
1209
1206 diff, __ = self.run_git_command(wire, cmd)
1210 diff, __ = self.run_git_command(wire, cmd)
1207 # If we used 'show' command, strip first few lines (until actual diff
1211 # If we used 'show' command, strip first few lines (until actual diff
1208 # starts)
1212 # starts)
1209 if commit_id_1 == self.EMPTY_COMMIT:
1213 if commit_id_1 == self.EMPTY_COMMIT:
1210 lines = diff.splitlines()
1214 lines = diff.splitlines()
1211 x = 0
1215 x = 0
1212 for line in lines:
1216 for line in lines:
1213 if line.startswith(b'diff'):
1217 if line.startswith(b'diff'):
1214 break
1218 break
1215 x += 1
1219 x += 1
1216 # Append new line just like 'diff' command do
1220 # Append new line just like 'diff' command do
1217 diff = '\n'.join(lines[x:]) + '\n'
1221 diff = '\n'.join(lines[x:]) + '\n'
1218 return diff
1222 return diff
1219
1223
1220 @reraise_safe_exceptions
1224 @reraise_safe_exceptions
1221 def diff(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
1225 def diff(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
1222 repo_init = self._factory.repo_libgit2(wire)
1226 repo_init = self._factory.repo_libgit2(wire)
1223
1227
1224 with repo_init as repo:
1228 with repo_init as repo:
1225 swap = True
1229 swap = True
1226 flags = 0
1230 flags = 0
1227 flags |= pygit2.GIT_DIFF_SHOW_BINARY
1231 flags |= pygit2.GIT_DIFF_SHOW_BINARY
1228
1232
1229 if opt_ignorews:
1233 if opt_ignorews:
1230 flags |= pygit2.GIT_DIFF_IGNORE_WHITESPACE
1234 flags |= pygit2.GIT_DIFF_IGNORE_WHITESPACE
1231
1235
1232 if commit_id_1 == self.EMPTY_COMMIT:
1236 if commit_id_1 == self.EMPTY_COMMIT:
1233 comm1 = repo[commit_id_2]
1237 comm1 = repo[commit_id_2]
1234 diff_obj = comm1.tree.diff_to_tree(
1238 diff_obj = comm1.tree.diff_to_tree(
1235 flags=flags, context_lines=context, swap=swap)
1239 flags=flags, context_lines=context, swap=swap)
1236
1240
1237 else:
1241 else:
1238 comm1 = repo[commit_id_2]
1242 comm1 = repo[commit_id_2]
1239 comm2 = repo[commit_id_1]
1243 comm2 = repo[commit_id_1]
1240 diff_obj = comm1.tree.diff_to_tree(
1244 diff_obj = comm1.tree.diff_to_tree(
1241 comm2.tree, flags=flags, context_lines=context, swap=swap)
1245 comm2.tree, flags=flags, context_lines=context, swap=swap)
1242 similar_flags = 0
1246 similar_flags = 0
1243 similar_flags |= pygit2.GIT_DIFF_FIND_RENAMES
1247 similar_flags |= pygit2.GIT_DIFF_FIND_RENAMES
1244 diff_obj.find_similar(flags=similar_flags)
1248 diff_obj.find_similar(flags=similar_flags)
1245
1249
1246 if file_filter:
1250 if file_filter:
1247 for p in diff_obj:
1251 for p in diff_obj:
1248 if p.delta.old_file.path == file_filter:
1252 if p.delta.old_file.path == file_filter:
1249 return BytesEnvelope(p.data) or BytesEnvelope(b'')
1253 return BytesEnvelope(p.data) or BytesEnvelope(b'')
1250 # fo matching path == no diff
1254 # fo matching path == no diff
1251 return BytesEnvelope(b'')
1255 return BytesEnvelope(b'')
1252
1256
1253 return BytesEnvelope(safe_bytes(diff_obj.patch)) or BytesEnvelope(b'')
1257 return BytesEnvelope(safe_bytes(diff_obj.patch)) or BytesEnvelope(b'')
1254
1258
1255 @reraise_safe_exceptions
1259 @reraise_safe_exceptions
1256 def node_history(self, wire, commit_id, path, limit):
1260 def node_history(self, wire, commit_id, path, limit):
1257 cache_on, context_uid, repo_id = self._cache_on(wire)
1261 cache_on, context_uid, repo_id = self._cache_on(wire)
1258 region = self._region(wire)
1262 region = self._region(wire)
1259
1263
1260 @region.conditional_cache_on_arguments(condition=cache_on)
1264 @region.conditional_cache_on_arguments(condition=cache_on)
1261 def _node_history(_context_uid, _repo_id, _commit_id, _path, _limit):
1265 def _node_history(_context_uid, _repo_id, _commit_id, _path, _limit):
1262 # optimize for n==1, rev-list is much faster for that use-case
1266 # optimize for n==1, rev-list is much faster for that use-case
1263 if limit == 1:
1267 if limit == 1:
1264 cmd = ['rev-list', '-1', commit_id, '--', path]
1268 cmd = ['rev-list', '-1', commit_id, '--', path]
1265 else:
1269 else:
1266 cmd = ['log']
1270 cmd = ['log']
1267 if limit:
1271 if limit:
1268 cmd.extend(['-n', str(safe_int(limit, 0))])
1272 cmd.extend(['-n', str(safe_int(limit, 0))])
1269 cmd.extend(['--pretty=format: %H', '-s', commit_id, '--', path])
1273 cmd.extend(['--pretty=format: %H', '-s', commit_id, '--', path])
1270
1274
1271 output, __ = self.run_git_command(wire, cmd)
1275 output, __ = self.run_git_command(wire, cmd)
1272 commit_ids = re.findall(rb'[0-9a-fA-F]{40}', output)
1276 commit_ids = re.findall(rb'[0-9a-fA-F]{40}', output)
1273
1277
1274 return [x for x in commit_ids]
1278 return [x for x in commit_ids]
1275 return _node_history(context_uid, repo_id, commit_id, path, limit)
1279 return _node_history(context_uid, repo_id, commit_id, path, limit)
1276
1280
1277 @reraise_safe_exceptions
1281 @reraise_safe_exceptions
1278 def node_annotate_legacy(self, wire, commit_id, path):
1282 def node_annotate_legacy(self, wire, commit_id, path):
1279 # note: replaced by pygit2 implementation
1283 # note: replaced by pygit2 implementation
1280 cmd = ['blame', '-l', '--root', '-r', commit_id, '--', path]
1284 cmd = ['blame', '-l', '--root', '-r', commit_id, '--', path]
1281 # -l ==> outputs long shas (and we need all 40 characters)
1285 # -l ==> outputs long shas (and we need all 40 characters)
1282 # --root ==> doesn't put '^' character for boundaries
1286 # --root ==> doesn't put '^' character for boundaries
1283 # -r commit_id ==> blames for the given commit
1287 # -r commit_id ==> blames for the given commit
1284 output, __ = self.run_git_command(wire, cmd)
1288 output, __ = self.run_git_command(wire, cmd)
1285
1289
1286 result = []
1290 result = []
1287 for i, blame_line in enumerate(output.splitlines()[:-1]):
1291 for i, blame_line in enumerate(output.splitlines()[:-1]):
1288 line_no = i + 1
1292 line_no = i + 1
1289 blame_commit_id, line = re.split(rb' ', blame_line, 1)
1293 blame_commit_id, line = re.split(rb' ', blame_line, 1)
1290 result.append((line_no, blame_commit_id, line))
1294 result.append((line_no, blame_commit_id, line))
1291
1295
1292 return result
1296 return result
1293
1297
1294 @reraise_safe_exceptions
1298 @reraise_safe_exceptions
1295 def node_annotate(self, wire, commit_id, path):
1299 def node_annotate(self, wire, commit_id, path):
1296
1300
1297 result_libgit = []
1301 result_libgit = []
1298 repo_init = self._factory.repo_libgit2(wire)
1302 repo_init = self._factory.repo_libgit2(wire)
1299 with repo_init as repo:
1303 with repo_init as repo:
1300 commit = repo[commit_id]
1304 commit = repo[commit_id]
1301 blame_obj = repo.blame(path, newest_commit=commit_id)
1305 blame_obj = repo.blame(path, newest_commit=commit_id)
1302 for i, line in enumerate(commit.tree[path].data.splitlines()):
1306 for i, line in enumerate(commit.tree[path].data.splitlines()):
1303 line_no = i + 1
1307 line_no = i + 1
1304 hunk = blame_obj.for_line(line_no)
1308 hunk = blame_obj.for_line(line_no)
1305 blame_commit_id = hunk.final_commit_id.hex
1309 blame_commit_id = hunk.final_commit_id.hex
1306
1310
1307 result_libgit.append((line_no, blame_commit_id, line))
1311 result_libgit.append((line_no, blame_commit_id, line))
1308
1312
1309 return BinaryEnvelope(result_libgit)
1313 return BinaryEnvelope(result_libgit)
1310
1314
1311 @reraise_safe_exceptions
1315 @reraise_safe_exceptions
1312 def update_server_info(self, wire):
1316 def update_server_info(self, wire):
1313 repo = self._factory.repo(wire)
1317 repo = self._factory.repo(wire)
1314 update_server_info(repo)
1318 update_server_info(repo)
1315
1319
1316 @reraise_safe_exceptions
1320 @reraise_safe_exceptions
1317 def get_all_commit_ids(self, wire):
1321 def get_all_commit_ids(self, wire):
1318
1322
1319 cache_on, context_uid, repo_id = self._cache_on(wire)
1323 cache_on, context_uid, repo_id = self._cache_on(wire)
1320 region = self._region(wire)
1324 region = self._region(wire)
1321
1325
1322 @region.conditional_cache_on_arguments(condition=cache_on)
1326 @region.conditional_cache_on_arguments(condition=cache_on)
1323 def _get_all_commit_ids(_context_uid, _repo_id):
1327 def _get_all_commit_ids(_context_uid, _repo_id):
1324
1328
1325 cmd = ['rev-list', '--reverse', '--date-order', '--branches', '--tags']
1329 cmd = ['rev-list', '--reverse', '--date-order', '--branches', '--tags']
1326 try:
1330 try:
1327 output, __ = self.run_git_command(wire, cmd)
1331 output, __ = self.run_git_command(wire, cmd)
1328 return output.splitlines()
1332 return output.splitlines()
1329 except Exception:
1333 except Exception:
1330 # Can be raised for empty repositories
1334 # Can be raised for empty repositories
1331 return []
1335 return []
1332
1336
1333 @region.conditional_cache_on_arguments(condition=cache_on)
1337 @region.conditional_cache_on_arguments(condition=cache_on)
1334 def _get_all_commit_ids_pygit2(_context_uid, _repo_id):
1338 def _get_all_commit_ids_pygit2(_context_uid, _repo_id):
1335 repo_init = self._factory.repo_libgit2(wire)
1339 repo_init = self._factory.repo_libgit2(wire)
1336 from pygit2 import GIT_SORT_REVERSE, GIT_SORT_TIME, GIT_BRANCH_ALL
1340 from pygit2 import GIT_SORT_REVERSE, GIT_SORT_TIME, GIT_BRANCH_ALL
1337 results = []
1341 results = []
1338 with repo_init as repo:
1342 with repo_init as repo:
1339 for commit in repo.walk(repo.head.target, GIT_SORT_TIME | GIT_BRANCH_ALL | GIT_SORT_REVERSE):
1343 for commit in repo.walk(repo.head.target, GIT_SORT_TIME | GIT_BRANCH_ALL | GIT_SORT_REVERSE):
1340 results.append(commit.id.hex)
1344 results.append(commit.id.hex)
1341
1345
1342 return _get_all_commit_ids(context_uid, repo_id)
1346 return _get_all_commit_ids(context_uid, repo_id)
1343
1347
1344 @reraise_safe_exceptions
1348 @reraise_safe_exceptions
1345 def run_git_command(self, wire, cmd, **opts):
1349 def run_git_command(self, wire, cmd, **opts):
1346 path = wire.get('path', None)
1350 path = wire.get('path', None)
1347
1351
1348 if path and os.path.isdir(path):
1352 if path and os.path.isdir(path):
1349 opts['cwd'] = path
1353 opts['cwd'] = path
1350
1354
1351 if '_bare' in opts:
1355 if '_bare' in opts:
1352 _copts = []
1356 _copts = []
1353 del opts['_bare']
1357 del opts['_bare']
1354 else:
1358 else:
1355 _copts = ['-c', 'core.quotepath=false', '-c', 'advice.diverging=false']
1359 _copts = ['-c', 'core.quotepath=false', '-c', 'advice.diverging=false']
1356 safe_call = False
1360 safe_call = False
1357 if '_safe' in opts:
1361 if '_safe' in opts:
1358 # no exc on failure
1362 # no exc on failure
1359 del opts['_safe']
1363 del opts['_safe']
1360 safe_call = True
1364 safe_call = True
1361
1365
1362 if '_copts' in opts:
1366 if '_copts' in opts:
1363 _copts.extend(opts['_copts'] or [])
1367 _copts.extend(opts['_copts'] or [])
1364 del opts['_copts']
1368 del opts['_copts']
1365
1369
1366 gitenv = os.environ.copy()
1370 gitenv = os.environ.copy()
1367 gitenv.update(opts.pop('extra_env', {}))
1371 gitenv.update(opts.pop('extra_env', {}))
1368 # need to clean fix GIT_DIR !
1372 # need to clean fix GIT_DIR !
1369 if 'GIT_DIR' in gitenv:
1373 if 'GIT_DIR' in gitenv:
1370 del gitenv['GIT_DIR']
1374 del gitenv['GIT_DIR']
1371 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
1375 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
1372 gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1'
1376 gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1'
1373
1377
1374 cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
1378 cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
1375 _opts = {'env': gitenv, 'shell': False}
1379 _opts = {'env': gitenv, 'shell': False}
1376
1380
1377 proc = None
1381 proc = None
1378 try:
1382 try:
1379 _opts.update(opts)
1383 _opts.update(opts)
1380 proc = subprocessio.SubprocessIOChunker(cmd, **_opts)
1384 proc = subprocessio.SubprocessIOChunker(cmd, **_opts)
1381
1385
1382 return b''.join(proc), b''.join(proc.stderr)
1386 return b''.join(proc), b''.join(proc.stderr)
1383 except OSError as err:
1387 except OSError as err:
1384 cmd = ' '.join(map(safe_str, cmd)) # human friendly CMD
1388 cmd = ' '.join(map(safe_str, cmd)) # human friendly CMD
1385 tb_err = ("Couldn't run git command (%s).\n"
1389 tb_err = ("Couldn't run git command (%s).\n"
1386 "Original error was:%s\n"
1390 "Original error was:%s\n"
1387 "Call options:%s\n"
1391 "Call options:%s\n"
1388 % (cmd, err, _opts))
1392 % (cmd, err, _opts))
1389 log.exception(tb_err)
1393 log.exception(tb_err)
1390 if safe_call:
1394 if safe_call:
1391 return '', err
1395 return '', err
1392 else:
1396 else:
1393 raise exceptions.VcsException()(tb_err)
1397 raise exceptions.VcsException()(tb_err)
1394 finally:
1398 finally:
1395 if proc:
1399 if proc:
1396 proc.close()
1400 proc.close()
1397
1401
1398 @reraise_safe_exceptions
1402 @reraise_safe_exceptions
1399 def install_hooks(self, wire, force=False):
1403 def install_hooks(self, wire, force=False):
1400 from vcsserver.hook_utils import install_git_hooks
1404 from vcsserver.hook_utils import install_git_hooks
1401 bare = self.bare(wire)
1405 bare = self.bare(wire)
1402 path = wire['path']
1406 path = wire['path']
1403 binary_dir = settings.BINARY_DIR
1407 binary_dir = settings.BINARY_DIR
1404 if binary_dir:
1408 if binary_dir:
1405 os.path.join(binary_dir, 'python3')
1409 os.path.join(binary_dir, 'python3')
1406 return install_git_hooks(path, bare, force_create=force)
1410 return install_git_hooks(path, bare, force_create=force)
1407
1411
1408 @reraise_safe_exceptions
1412 @reraise_safe_exceptions
1409 def get_hooks_info(self, wire):
1413 def get_hooks_info(self, wire):
1410 from vcsserver.hook_utils import (
1414 from vcsserver.hook_utils import (
1411 get_git_pre_hook_version, get_git_post_hook_version)
1415 get_git_pre_hook_version, get_git_post_hook_version)
1412 bare = self.bare(wire)
1416 bare = self.bare(wire)
1413 path = wire['path']
1417 path = wire['path']
1414 return {
1418 return {
1415 'pre_version': get_git_pre_hook_version(path, bare),
1419 'pre_version': get_git_pre_hook_version(path, bare),
1416 'post_version': get_git_post_hook_version(path, bare),
1420 'post_version': get_git_post_hook_version(path, bare),
1417 }
1421 }
1418
1422
1419 @reraise_safe_exceptions
1423 @reraise_safe_exceptions
1420 def set_head_ref(self, wire, head_name):
1424 def set_head_ref(self, wire, head_name):
1421 log.debug('Setting refs/head to `%s`', head_name)
1425 log.debug('Setting refs/head to `%s`', head_name)
1422 repo_init = self._factory.repo_libgit2(wire)
1426 repo_init = self._factory.repo_libgit2(wire)
1423 with repo_init as repo:
1427 with repo_init as repo:
1424 repo.set_head(f'refs/heads/{head_name}')
1428 repo.set_head(f'refs/heads/{head_name}')
1425
1429
1426 return [head_name] + [f'set HEAD to refs/heads/{head_name}']
1430 return [head_name] + [f'set HEAD to refs/heads/{head_name}']
1427
1431
1428 @reraise_safe_exceptions
1432 @reraise_safe_exceptions
1429 def archive_repo(self, wire, archive_name_key, kind, mtime, archive_at_path,
1433 def archive_repo(self, wire, archive_name_key, kind, mtime, archive_at_path,
1430 archive_dir_name, commit_id, cache_config):
1434 archive_dir_name, commit_id, cache_config):
1431
1435
1432 def file_walker(_commit_id, path):
1436 def file_walker(_commit_id, path):
1433 repo_init = self._factory.repo_libgit2(wire)
1437 repo_init = self._factory.repo_libgit2(wire)
1434
1438
1435 with repo_init as repo:
1439 with repo_init as repo:
1436 commit = repo[commit_id]
1440 commit = repo[commit_id]
1437
1441
1438 if path in ['', '/']:
1442 if path in ['', '/']:
1439 tree = commit.tree
1443 tree = commit.tree
1440 else:
1444 else:
1441 tree = commit.tree[path.rstrip('/')]
1445 tree = commit.tree[path.rstrip('/')]
1442 tree_id = tree.id.hex
1446 tree_id = tree.id.hex
1443 try:
1447 try:
1444 tree = repo[tree_id]
1448 tree = repo[tree_id]
1445 except KeyError:
1449 except KeyError:
1446 raise ObjectMissing(f'No tree with id: {tree_id}')
1450 raise ObjectMissing(f'No tree with id: {tree_id}')
1447
1451
1448 index = LibGit2Index.Index()
1452 index = LibGit2Index.Index()
1449 index.read_tree(tree)
1453 index.read_tree(tree)
1450 file_iter = index
1454 file_iter = index
1451
1455
1452 for file_node in file_iter:
1456 for file_node in file_iter:
1453 file_path = file_node.path
1457 file_path = file_node.path
1454 mode = file_node.mode
1458 mode = file_node.mode
1455 is_link = stat.S_ISLNK(mode)
1459 is_link = stat.S_ISLNK(mode)
1456 if mode == pygit2.GIT_FILEMODE_COMMIT:
1460 if mode == pygit2.GIT_FILEMODE_COMMIT:
1457 log.debug('Skipping path %s as a commit node', file_path)
1461 log.debug('Skipping path %s as a commit node', file_path)
1458 continue
1462 continue
1459 yield ArchiveNode(file_path, mode, is_link, repo[file_node.hex].read_raw)
1463 yield ArchiveNode(file_path, mode, is_link, repo[file_node.hex].read_raw)
1460
1464
1461 return store_archive_in_cache(
1465 return store_archive_in_cache(
1462 file_walker, archive_name_key, kind, mtime, archive_at_path, archive_dir_name, commit_id, cache_config=cache_config)
1466 file_walker, archive_name_key, kind, mtime, archive_at_path, archive_dir_name, commit_id, cache_config=cache_config)
General Comments 0
You need to be logged in to leave comments. Login now