##// END OF EJS Templates
git: rename fetch into pull because this is what it actually does.
marcink -
r550:964721d2 default
parent child Browse files
Show More
@@ -1,711 +1,716 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2018 RhodeCode GmbH
2 # Copyright (C) 2014-2018 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 import collections
17 import collections
18 import logging
18 import logging
19 import os
19 import os
20 import posixpath as vcspath
20 import posixpath as vcspath
21 import re
21 import re
22 import stat
22 import stat
23 import traceback
23 import traceback
24 import urllib
24 import urllib
25 import urllib2
25 import urllib2
26 from functools import wraps
26 from functools import wraps
27
27
28 from dulwich import index, objects
28 from dulwich import index, objects
29 from dulwich.client import HttpGitClient, LocalGitClient
29 from dulwich.client import HttpGitClient, LocalGitClient
30 from dulwich.errors import (
30 from dulwich.errors import (
31 NotGitRepository, ChecksumMismatch, WrongObjectException,
31 NotGitRepository, ChecksumMismatch, WrongObjectException,
32 MissingCommitError, ObjectMissing, HangupException,
32 MissingCommitError, ObjectMissing, HangupException,
33 UnexpectedCommandError)
33 UnexpectedCommandError)
34 from dulwich.repo import Repo as DulwichRepo, Tag
34 from dulwich.repo import Repo as DulwichRepo, Tag
35 from dulwich.server import update_server_info
35 from dulwich.server import update_server_info
36
36
37 from vcsserver import exceptions, settings, subprocessio
37 from vcsserver import exceptions, settings, subprocessio
38 from vcsserver.utils import safe_str
38 from vcsserver.utils import safe_str
39 from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original
39 from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original
40 from vcsserver.hgcompat import (
40 from vcsserver.hgcompat import (
41 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
41 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
42 from vcsserver.git_lfs.lib import LFSOidStore
42 from vcsserver.git_lfs.lib import LFSOidStore
43
43
44 DIR_STAT = stat.S_IFDIR
44 DIR_STAT = stat.S_IFDIR
45 FILE_MODE = stat.S_IFMT
45 FILE_MODE = stat.S_IFMT
46 GIT_LINK = objects.S_IFGITLINK
46 GIT_LINK = objects.S_IFGITLINK
47
47
48 log = logging.getLogger(__name__)
48 log = logging.getLogger(__name__)
49
49
50
50
51 def reraise_safe_exceptions(func):
51 def reraise_safe_exceptions(func):
52 """Converts Dulwich exceptions to something neutral."""
52 """Converts Dulwich exceptions to something neutral."""
53 @wraps(func)
53 @wraps(func)
54 def wrapper(*args, **kwargs):
54 def wrapper(*args, **kwargs):
55 try:
55 try:
56 return func(*args, **kwargs)
56 return func(*args, **kwargs)
57 except (ChecksumMismatch, WrongObjectException, MissingCommitError,
57 except (ChecksumMismatch, WrongObjectException, MissingCommitError,
58 ObjectMissing) as e:
58 ObjectMissing) as e:
59 raise exceptions.LookupException(e)(e.message)
59 raise exceptions.LookupException(e)(e.message)
60 except (HangupException, UnexpectedCommandError) as e:
60 except (HangupException, UnexpectedCommandError) as e:
61 raise exceptions.VcsException(e)(e.message)
61 raise exceptions.VcsException(e)(e.message)
62 except Exception as e:
62 except Exception as e:
63 # NOTE(marcink): becuase of how dulwich handles some exceptions
63 # NOTE(marcink): becuase of how dulwich handles some exceptions
64 # (KeyError on empty repos), we cannot track this and catch all
64 # (KeyError on empty repos), we cannot track this and catch all
65 # exceptions, it's an exceptions from other handlers
65 # exceptions, it's an exceptions from other handlers
66 #if not hasattr(e, '_vcs_kind'):
66 #if not hasattr(e, '_vcs_kind'):
67 #log.exception("Unhandled exception in git remote call")
67 #log.exception("Unhandled exception in git remote call")
68 #raise_from_original(exceptions.UnhandledException)
68 #raise_from_original(exceptions.UnhandledException)
69 raise
69 raise
70 return wrapper
70 return wrapper
71
71
72
72
73 class Repo(DulwichRepo):
73 class Repo(DulwichRepo):
74 """
74 """
75 A wrapper for dulwich Repo class.
75 A wrapper for dulwich Repo class.
76
76
77 Since dulwich is sometimes keeping .idx file descriptors open, it leads to
77 Since dulwich is sometimes keeping .idx file descriptors open, it leads to
78 "Too many open files" error. We need to close all opened file descriptors
78 "Too many open files" error. We need to close all opened file descriptors
79 once the repo object is destroyed.
79 once the repo object is destroyed.
80
80
81 TODO: mikhail: please check if we need this wrapper after updating dulwich
81 TODO: mikhail: please check if we need this wrapper after updating dulwich
82 to 0.12.0 +
82 to 0.12.0 +
83 """
83 """
84 def __del__(self):
84 def __del__(self):
85 if hasattr(self, 'object_store'):
85 if hasattr(self, 'object_store'):
86 self.close()
86 self.close()
87
87
88
88
89 class GitFactory(RepoFactory):
89 class GitFactory(RepoFactory):
90 repo_type = 'git'
90 repo_type = 'git'
91
91
92 def _create_repo(self, wire, create):
92 def _create_repo(self, wire, create):
93 repo_path = str_to_dulwich(wire['path'])
93 repo_path = str_to_dulwich(wire['path'])
94 return Repo(repo_path)
94 return Repo(repo_path)
95
95
96
96
97 class GitRemote(object):
97 class GitRemote(object):
98
98
99 def __init__(self, factory):
99 def __init__(self, factory):
100 self._factory = factory
100 self._factory = factory
101
101 self.peeled_ref_marker = '^{}'
102 self._bulk_methods = {
102 self._bulk_methods = {
103 "author": self.commit_attribute,
103 "author": self.commit_attribute,
104 "date": self.get_object_attrs,
104 "date": self.get_object_attrs,
105 "message": self.commit_attribute,
105 "message": self.commit_attribute,
106 "parents": self.commit_attribute,
106 "parents": self.commit_attribute,
107 "_commit": self.revision,
107 "_commit": self.revision,
108 }
108 }
109
109
110 def _wire_to_config(self, wire):
110 def _wire_to_config(self, wire):
111 if 'config' in wire:
111 if 'config' in wire:
112 return dict([(x[0] + '_' + x[1], x[2]) for x in wire['config']])
112 return dict([(x[0] + '_' + x[1], x[2]) for x in wire['config']])
113 return {}
113 return {}
114
114
115 def _assign_ref(self, wire, ref, commit_id):
115 def _assign_ref(self, wire, ref, commit_id):
116 repo = self._factory.repo(wire)
116 repo = self._factory.repo(wire)
117 repo[ref] = commit_id
117 repo[ref] = commit_id
118
118
119 @reraise_safe_exceptions
119 @reraise_safe_exceptions
120 def add_object(self, wire, content):
120 def add_object(self, wire, content):
121 repo = self._factory.repo(wire)
121 repo = self._factory.repo(wire)
122 blob = objects.Blob()
122 blob = objects.Blob()
123 blob.set_raw_string(content)
123 blob.set_raw_string(content)
124 repo.object_store.add_object(blob)
124 repo.object_store.add_object(blob)
125 return blob.id
125 return blob.id
126
126
127 @reraise_safe_exceptions
127 @reraise_safe_exceptions
128 def assert_correct_path(self, wire):
128 def assert_correct_path(self, wire):
129 path = wire.get('path')
129 path = wire.get('path')
130 try:
130 try:
131 self._factory.repo(wire)
131 self._factory.repo(wire)
132 except NotGitRepository as e:
132 except NotGitRepository as e:
133 tb = traceback.format_exc()
133 tb = traceback.format_exc()
134 log.debug("Invalid Git path `%s`, tb: %s", path, tb)
134 log.debug("Invalid Git path `%s`, tb: %s", path, tb)
135 return False
135 return False
136
136
137 return True
137 return True
138
138
139 @reraise_safe_exceptions
139 @reraise_safe_exceptions
140 def bare(self, wire):
140 def bare(self, wire):
141 repo = self._factory.repo(wire)
141 repo = self._factory.repo(wire)
142 return repo.bare
142 return repo.bare
143
143
144 @reraise_safe_exceptions
144 @reraise_safe_exceptions
145 def blob_as_pretty_string(self, wire, sha):
145 def blob_as_pretty_string(self, wire, sha):
146 repo = self._factory.repo(wire)
146 repo = self._factory.repo(wire)
147 return repo[sha].as_pretty_string()
147 return repo[sha].as_pretty_string()
148
148
149 @reraise_safe_exceptions
149 @reraise_safe_exceptions
150 def blob_raw_length(self, wire, sha):
150 def blob_raw_length(self, wire, sha):
151 repo = self._factory.repo(wire)
151 repo = self._factory.repo(wire)
152 blob = repo[sha]
152 blob = repo[sha]
153 return blob.raw_length()
153 return blob.raw_length()
154
154
155 def _parse_lfs_pointer(self, raw_content):
155 def _parse_lfs_pointer(self, raw_content):
156
156
157 spec_string = 'version https://git-lfs.github.com/spec'
157 spec_string = 'version https://git-lfs.github.com/spec'
158 if raw_content and raw_content.startswith(spec_string):
158 if raw_content and raw_content.startswith(spec_string):
159 pattern = re.compile(r"""
159 pattern = re.compile(r"""
160 (?:\n)?
160 (?:\n)?
161 ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
161 ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
162 ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
162 ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
163 ^size[ ](?P<oid_size>[0-9]+)\n
163 ^size[ ](?P<oid_size>[0-9]+)\n
164 (?:\n)?
164 (?:\n)?
165 """, re.VERBOSE | re.MULTILINE)
165 """, re.VERBOSE | re.MULTILINE)
166 match = pattern.match(raw_content)
166 match = pattern.match(raw_content)
167 if match:
167 if match:
168 return match.groupdict()
168 return match.groupdict()
169
169
170 return {}
170 return {}
171
171
172 @reraise_safe_exceptions
172 @reraise_safe_exceptions
173 def is_large_file(self, wire, sha):
173 def is_large_file(self, wire, sha):
174 repo = self._factory.repo(wire)
174 repo = self._factory.repo(wire)
175 blob = repo[sha]
175 blob = repo[sha]
176 return self._parse_lfs_pointer(blob.as_raw_string())
176 return self._parse_lfs_pointer(blob.as_raw_string())
177
177
178 @reraise_safe_exceptions
178 @reraise_safe_exceptions
179 def in_largefiles_store(self, wire, oid):
179 def in_largefiles_store(self, wire, oid):
180 repo = self._factory.repo(wire)
180 repo = self._factory.repo(wire)
181 conf = self._wire_to_config(wire)
181 conf = self._wire_to_config(wire)
182
182
183 store_location = conf.get('vcs_git_lfs_store_location')
183 store_location = conf.get('vcs_git_lfs_store_location')
184 if store_location:
184 if store_location:
185 repo_name = repo.path
185 repo_name = repo.path
186 store = LFSOidStore(
186 store = LFSOidStore(
187 oid=oid, repo=repo_name, store_location=store_location)
187 oid=oid, repo=repo_name, store_location=store_location)
188 return store.has_oid()
188 return store.has_oid()
189
189
190 return False
190 return False
191
191
192 @reraise_safe_exceptions
192 @reraise_safe_exceptions
193 def store_path(self, wire, oid):
193 def store_path(self, wire, oid):
194 repo = self._factory.repo(wire)
194 repo = self._factory.repo(wire)
195 conf = self._wire_to_config(wire)
195 conf = self._wire_to_config(wire)
196
196
197 store_location = conf.get('vcs_git_lfs_store_location')
197 store_location = conf.get('vcs_git_lfs_store_location')
198 if store_location:
198 if store_location:
199 repo_name = repo.path
199 repo_name = repo.path
200 store = LFSOidStore(
200 store = LFSOidStore(
201 oid=oid, repo=repo_name, store_location=store_location)
201 oid=oid, repo=repo_name, store_location=store_location)
202 return store.oid_path
202 return store.oid_path
203 raise ValueError('Unable to fetch oid with path {}'.format(oid))
203 raise ValueError('Unable to fetch oid with path {}'.format(oid))
204
204
205 @reraise_safe_exceptions
205 @reraise_safe_exceptions
206 def bulk_request(self, wire, rev, pre_load):
206 def bulk_request(self, wire, rev, pre_load):
207 result = {}
207 result = {}
208 for attr in pre_load:
208 for attr in pre_load:
209 try:
209 try:
210 method = self._bulk_methods[attr]
210 method = self._bulk_methods[attr]
211 args = [wire, rev]
211 args = [wire, rev]
212 if attr == "date":
212 if attr == "date":
213 args.extend(["commit_time", "commit_timezone"])
213 args.extend(["commit_time", "commit_timezone"])
214 elif attr in ["author", "message", "parents"]:
214 elif attr in ["author", "message", "parents"]:
215 args.append(attr)
215 args.append(attr)
216 result[attr] = method(*args)
216 result[attr] = method(*args)
217 except KeyError as e:
217 except KeyError as e:
218 raise exceptions.VcsException(e)(
218 raise exceptions.VcsException(e)(
219 "Unknown bulk attribute: %s" % attr)
219 "Unknown bulk attribute: %s" % attr)
220 return result
220 return result
221
221
222 def _build_opener(self, url):
222 def _build_opener(self, url):
223 handlers = []
223 handlers = []
224 url_obj = url_parser(url)
224 url_obj = url_parser(url)
225 _, authinfo = url_obj.authinfo()
225 _, authinfo = url_obj.authinfo()
226
226
227 if authinfo:
227 if authinfo:
228 # create a password manager
228 # create a password manager
229 passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
229 passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
230 passmgr.add_password(*authinfo)
230 passmgr.add_password(*authinfo)
231
231
232 handlers.extend((httpbasicauthhandler(passmgr),
232 handlers.extend((httpbasicauthhandler(passmgr),
233 httpdigestauthhandler(passmgr)))
233 httpdigestauthhandler(passmgr)))
234
234
235 return urllib2.build_opener(*handlers)
235 return urllib2.build_opener(*handlers)
236
236
237 @reraise_safe_exceptions
237 @reraise_safe_exceptions
238 def check_url(self, url, config):
238 def check_url(self, url, config):
239 url_obj = url_parser(url)
239 url_obj = url_parser(url)
240 test_uri, _ = url_obj.authinfo()
240 test_uri, _ = url_obj.authinfo()
241 url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
241 url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
242 url_obj.query = obfuscate_qs(url_obj.query)
242 url_obj.query = obfuscate_qs(url_obj.query)
243 cleaned_uri = str(url_obj)
243 cleaned_uri = str(url_obj)
244 log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
244 log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
245
245
246 if not test_uri.endswith('info/refs'):
246 if not test_uri.endswith('info/refs'):
247 test_uri = test_uri.rstrip('/') + '/info/refs'
247 test_uri = test_uri.rstrip('/') + '/info/refs'
248
248
249 o = self._build_opener(url)
249 o = self._build_opener(url)
250 o.addheaders = [('User-Agent', 'git/1.7.8.0')] # fake some git
250 o.addheaders = [('User-Agent', 'git/1.7.8.0')] # fake some git
251
251
252 q = {"service": 'git-upload-pack'}
252 q = {"service": 'git-upload-pack'}
253 qs = '?%s' % urllib.urlencode(q)
253 qs = '?%s' % urllib.urlencode(q)
254 cu = "%s%s" % (test_uri, qs)
254 cu = "%s%s" % (test_uri, qs)
255 req = urllib2.Request(cu, None, {})
255 req = urllib2.Request(cu, None, {})
256
256
257 try:
257 try:
258 log.debug("Trying to open URL %s", cleaned_uri)
258 log.debug("Trying to open URL %s", cleaned_uri)
259 resp = o.open(req)
259 resp = o.open(req)
260 if resp.code != 200:
260 if resp.code != 200:
261 raise exceptions.URLError()('Return Code is not 200')
261 raise exceptions.URLError()('Return Code is not 200')
262 except Exception as e:
262 except Exception as e:
263 log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
263 log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
264 # means it cannot be cloned
264 # means it cannot be cloned
265 raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e))
265 raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e))
266
266
267 # now detect if it's proper git repo
267 # now detect if it's proper git repo
268 gitdata = resp.read()
268 gitdata = resp.read()
269 if 'service=git-upload-pack' in gitdata:
269 if 'service=git-upload-pack' in gitdata:
270 pass
270 pass
271 elif re.findall(r'[0-9a-fA-F]{40}\s+refs', gitdata):
271 elif re.findall(r'[0-9a-fA-F]{40}\s+refs', gitdata):
272 # old style git can return some other format !
272 # old style git can return some other format !
273 pass
273 pass
274 else:
274 else:
275 raise exceptions.URLError()(
275 raise exceptions.URLError()(
276 "url [%s] does not look like an git" % (cleaned_uri,))
276 "url [%s] does not look like an git" % (cleaned_uri,))
277
277
278 return True
278 return True
279
279
280 @reraise_safe_exceptions
280 @reraise_safe_exceptions
281 def clone(self, wire, url, deferred, valid_refs, update_after_clone):
281 def clone(self, wire, url, deferred, valid_refs, update_after_clone):
282 remote_refs = self.fetch(wire, url, apply_refs=False)
282 # TODO(marcink): deprecate this method. Last i checked we don't use it anymore
283 remote_refs = self.pull(wire, url, apply_refs=False)
283 repo = self._factory.repo(wire)
284 repo = self._factory.repo(wire)
284 if isinstance(valid_refs, list):
285 if isinstance(valid_refs, list):
285 valid_refs = tuple(valid_refs)
286 valid_refs = tuple(valid_refs)
286
287
287 for k in remote_refs:
288 for k in remote_refs:
288 # only parse heads/tags and skip so called deferred tags
289 # only parse heads/tags and skip so called deferred tags
289 if k.startswith(valid_refs) and not k.endswith(deferred):
290 if k.startswith(valid_refs) and not k.endswith(deferred):
290 repo[k] = remote_refs[k]
291 repo[k] = remote_refs[k]
291
292
292 if update_after_clone:
293 if update_after_clone:
293 # we want to checkout HEAD
294 # we want to checkout HEAD
294 repo["HEAD"] = remote_refs["HEAD"]
295 repo["HEAD"] = remote_refs["HEAD"]
295 index.build_index_from_tree(repo.path, repo.index_path(),
296 index.build_index_from_tree(repo.path, repo.index_path(),
296 repo.object_store, repo["HEAD"].tree)
297 repo.object_store, repo["HEAD"].tree)
297
298
298 # TODO: this is quite complex, check if that can be simplified
299 # TODO: this is quite complex, check if that can be simplified
299 @reraise_safe_exceptions
300 @reraise_safe_exceptions
300 def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
301 def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
301 repo = self._factory.repo(wire)
302 repo = self._factory.repo(wire)
302 object_store = repo.object_store
303 object_store = repo.object_store
303
304
304 # Create tree and populates it with blobs
305 # Create tree and populates it with blobs
305 commit_tree = commit_tree and repo[commit_tree] or objects.Tree()
306 commit_tree = commit_tree and repo[commit_tree] or objects.Tree()
306
307
307 for node in updated:
308 for node in updated:
308 # Compute subdirs if needed
309 # Compute subdirs if needed
309 dirpath, nodename = vcspath.split(node['path'])
310 dirpath, nodename = vcspath.split(node['path'])
310 dirnames = map(safe_str, dirpath and dirpath.split('/') or [])
311 dirnames = map(safe_str, dirpath and dirpath.split('/') or [])
311 parent = commit_tree
312 parent = commit_tree
312 ancestors = [('', parent)]
313 ancestors = [('', parent)]
313
314
314 # Tries to dig for the deepest existing tree
315 # Tries to dig for the deepest existing tree
315 while dirnames:
316 while dirnames:
316 curdir = dirnames.pop(0)
317 curdir = dirnames.pop(0)
317 try:
318 try:
318 dir_id = parent[curdir][1]
319 dir_id = parent[curdir][1]
319 except KeyError:
320 except KeyError:
320 # put curdir back into dirnames and stops
321 # put curdir back into dirnames and stops
321 dirnames.insert(0, curdir)
322 dirnames.insert(0, curdir)
322 break
323 break
323 else:
324 else:
324 # If found, updates parent
325 # If found, updates parent
325 parent = repo[dir_id]
326 parent = repo[dir_id]
326 ancestors.append((curdir, parent))
327 ancestors.append((curdir, parent))
327 # Now parent is deepest existing tree and we need to create
328 # Now parent is deepest existing tree and we need to create
328 # subtrees for dirnames (in reverse order)
329 # subtrees for dirnames (in reverse order)
329 # [this only applies for nodes from added]
330 # [this only applies for nodes from added]
330 new_trees = []
331 new_trees = []
331
332
332 blob = objects.Blob.from_string(node['content'])
333 blob = objects.Blob.from_string(node['content'])
333
334
334 if dirnames:
335 if dirnames:
335 # If there are trees which should be created we need to build
336 # If there are trees which should be created we need to build
336 # them now (in reverse order)
337 # them now (in reverse order)
337 reversed_dirnames = list(reversed(dirnames))
338 reversed_dirnames = list(reversed(dirnames))
338 curtree = objects.Tree()
339 curtree = objects.Tree()
339 curtree[node['node_path']] = node['mode'], blob.id
340 curtree[node['node_path']] = node['mode'], blob.id
340 new_trees.append(curtree)
341 new_trees.append(curtree)
341 for dirname in reversed_dirnames[:-1]:
342 for dirname in reversed_dirnames[:-1]:
342 newtree = objects.Tree()
343 newtree = objects.Tree()
343 newtree[dirname] = (DIR_STAT, curtree.id)
344 newtree[dirname] = (DIR_STAT, curtree.id)
344 new_trees.append(newtree)
345 new_trees.append(newtree)
345 curtree = newtree
346 curtree = newtree
346 parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
347 parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
347 else:
348 else:
348 parent.add(
349 parent.add(
349 name=node['node_path'], mode=node['mode'], hexsha=blob.id)
350 name=node['node_path'], mode=node['mode'], hexsha=blob.id)
350
351
351 new_trees.append(parent)
352 new_trees.append(parent)
352 # Update ancestors
353 # Update ancestors
353 reversed_ancestors = reversed(
354 reversed_ancestors = reversed(
354 [(a[1], b[1], b[0]) for a, b in zip(ancestors, ancestors[1:])])
355 [(a[1], b[1], b[0]) for a, b in zip(ancestors, ancestors[1:])])
355 for parent, tree, path in reversed_ancestors:
356 for parent, tree, path in reversed_ancestors:
356 parent[path] = (DIR_STAT, tree.id)
357 parent[path] = (DIR_STAT, tree.id)
357 object_store.add_object(tree)
358 object_store.add_object(tree)
358
359
359 object_store.add_object(blob)
360 object_store.add_object(blob)
360 for tree in new_trees:
361 for tree in new_trees:
361 object_store.add_object(tree)
362 object_store.add_object(tree)
362
363
363 for node_path in removed:
364 for node_path in removed:
364 paths = node_path.split('/')
365 paths = node_path.split('/')
365 tree = commit_tree
366 tree = commit_tree
366 trees = [tree]
367 trees = [tree]
367 # Traverse deep into the forest...
368 # Traverse deep into the forest...
368 for path in paths:
369 for path in paths:
369 try:
370 try:
370 obj = repo[tree[path][1]]
371 obj = repo[tree[path][1]]
371 if isinstance(obj, objects.Tree):
372 if isinstance(obj, objects.Tree):
372 trees.append(obj)
373 trees.append(obj)
373 tree = obj
374 tree = obj
374 except KeyError:
375 except KeyError:
375 break
376 break
376 # Cut down the blob and all rotten trees on the way back...
377 # Cut down the blob and all rotten trees on the way back...
377 for path, tree in reversed(zip(paths, trees)):
378 for path, tree in reversed(zip(paths, trees)):
378 del tree[path]
379 del tree[path]
379 if tree:
380 if tree:
380 # This tree still has elements - don't remove it or any
381 # This tree still has elements - don't remove it or any
381 # of it's parents
382 # of it's parents
382 break
383 break
383
384
384 object_store.add_object(commit_tree)
385 object_store.add_object(commit_tree)
385
386
386 # Create commit
387 # Create commit
387 commit = objects.Commit()
388 commit = objects.Commit()
388 commit.tree = commit_tree.id
389 commit.tree = commit_tree.id
389 for k, v in commit_data.iteritems():
390 for k, v in commit_data.iteritems():
390 setattr(commit, k, v)
391 setattr(commit, k, v)
391 object_store.add_object(commit)
392 object_store.add_object(commit)
392
393
393 ref = 'refs/heads/%s' % branch
394 ref = 'refs/heads/%s' % branch
394 repo.refs[ref] = commit.id
395 repo.refs[ref] = commit.id
395
396
396 return commit.id
397 return commit.id
397
398
398 @reraise_safe_exceptions
399 @reraise_safe_exceptions
399 def fetch(self, wire, url, apply_refs=True, refs=None):
400 def pull(self, wire, url, apply_refs=True, refs=None):
400 if url != 'default' and '://' not in url:
401 if url != 'default' and '://' not in url:
401 client = LocalGitClient(url)
402 client = LocalGitClient(url)
402 else:
403 else:
403 url_obj = url_parser(url)
404 url_obj = url_parser(url)
404 o = self._build_opener(url)
405 o = self._build_opener(url)
405 url, _ = url_obj.authinfo()
406 url, _ = url_obj.authinfo()
406 client = HttpGitClient(base_url=url, opener=o)
407 client = HttpGitClient(base_url=url, opener=o)
407 repo = self._factory.repo(wire)
408 repo = self._factory.repo(wire)
408
409
409 determine_wants = repo.object_store.determine_wants_all
410 determine_wants = repo.object_store.determine_wants_all
410 if refs:
411 if refs:
411 def determine_wants_requested(references):
412 def determine_wants_requested(references):
412 return [references[r] for r in references if r in refs]
413 return [references[r] for r in references if r in refs]
413 determine_wants = determine_wants_requested
414 determine_wants = determine_wants_requested
414
415
415 try:
416 try:
416 remote_refs = client.fetch(
417 remote_refs = client.fetch(
417 path=url, target=repo, determine_wants=determine_wants)
418 path=url, target=repo, determine_wants=determine_wants)
418 except NotGitRepository as e:
419 except NotGitRepository as e:
419 log.warning(
420 log.warning(
420 'Trying to fetch from "%s" failed, not a Git repository.', url)
421 'Trying to fetch from "%s" failed, not a Git repository.', url)
421 # Exception can contain unicode which we convert
422 # Exception can contain unicode which we convert
422 raise exceptions.AbortException(e)(repr(e))
423 raise exceptions.AbortException(e)(repr(e))
423
424
424 # mikhail: client.fetch() returns all the remote refs, but fetches only
425 # mikhail: client.fetch() returns all the remote refs, but fetches only
425 # refs filtered by `determine_wants` function. We need to filter result
426 # refs filtered by `determine_wants` function. We need to filter result
426 # as well
427 # as well
427 if refs:
428 if refs:
428 remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
429 remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
429
430
430 if apply_refs:
431 if apply_refs:
431 # TODO: johbo: Needs proper test coverage with a git repository
432 # TODO: johbo: Needs proper test coverage with a git repository
432 # that contains a tag object, so that we would end up with
433 # that contains a tag object, so that we would end up with
433 # a peeled ref at this point.
434 # a peeled ref at this point.
434 PEELED_REF_MARKER = '^{}'
435 for k in remote_refs:
435 for k in remote_refs:
436 if k.endswith(PEELED_REF_MARKER):
436 if k.endswith(self.peeled_ref_marker):
437 log.info("Skipping peeled reference %s", k)
437 log.debug("Skipping peeled reference %s", k)
438 continue
438 continue
439 repo[k] = remote_refs[k]
439 repo[k] = remote_refs[k]
440
440
441 if refs:
441 if refs:
442 # mikhail: explicitly set the head to the last ref.
442 # mikhail: explicitly set the head to the last ref.
443 repo['HEAD'] = remote_refs[refs[-1]]
443 repo['HEAD'] = remote_refs[refs[-1]]
444
444
445 # TODO: mikhail: should we return remote_refs here to be
446 # consistent?
447 else:
445 else:
448 return remote_refs
446 return remote_refs
449
447
450 @reraise_safe_exceptions
448 @reraise_safe_exceptions
451 def sync_fetch(self, wire, url, refs=None):
449 def sync_fetch(self, wire, url, refs=None):
452 repo = self._factory.repo(wire)
450 repo = self._factory.repo(wire)
453 if refs and not isinstance(refs, (list, tuple)):
451 if refs and not isinstance(refs, (list, tuple)):
454 refs = [refs]
452 refs = [refs]
455
453
456 # get remote heads
454 # get all remote refs we'll use to fetch later
457 output, __ = self.run_git_command(
455 output, __ = self.run_git_command(
458 wire, ['ls-remote', url], fail_on_stderr=False,
456 wire, ['ls-remote', url], fail_on_stderr=False,
459 _copts=['-c', 'core.askpass=""'],
457 _copts=['-c', 'core.askpass=""'],
460 extra_env={'GIT_TERMINAL_PROMPT': '0'})
458 extra_env={'GIT_TERMINAL_PROMPT': '0'})
461
459
462 remote_refs = collections.OrderedDict()
460 remote_refs = collections.OrderedDict()
463 fetch_refs = []
461 fetch_refs = []
462
464 for ref_line in output.splitlines():
463 for ref_line in output.splitlines():
465 sha, ref = ref_line.split('\t')
464 sha, ref = ref_line.split('\t')
466 sha = sha.strip()
465 sha = sha.strip()
466 if ref in remote_refs:
467 # duplicate, skip
468 continue
469 if ref.endswith(self.peeled_ref_marker):
470 log.debug("Skipping peeled reference %s", ref)
471 continue
467 remote_refs[ref] = sha
472 remote_refs[ref] = sha
468
473
469 if refs and sha in refs:
474 if refs and sha in refs:
470 # we filter fetch using our specified refs
475 # we filter fetch using our specified refs
471 fetch_refs.append('{}:{}'.format(ref, ref))
476 fetch_refs.append('{}:{}'.format(ref, ref))
472 elif not refs:
477 elif not refs:
473 fetch_refs.append('{}:{}'.format(ref, ref))
478 fetch_refs.append('{}:{}'.format(ref, ref))
474
479
475 fetch_refs.append('{}:{}'.format(ref, ref))
480 if fetch_refs:
476
477 _out, _err = self.run_git_command(
481 _out, _err = self.run_git_command(
478 wire, ['fetch', url, '--'] + fetch_refs, fail_on_stderr=False,
482 wire, ['fetch', url, '--prune', '--'] + fetch_refs,
483 fail_on_stderr=False,
479 _copts=['-c', 'core.askpass=""'],
484 _copts=['-c', 'core.askpass=""'],
480 extra_env={'GIT_TERMINAL_PROMPT': '0'})
485 extra_env={'GIT_TERMINAL_PROMPT': '0'})
481
486
482 return remote_refs
487 return remote_refs
483
488
484 @reraise_safe_exceptions
489 @reraise_safe_exceptions
485 def sync_push(self, wire, url, refs=None):
490 def sync_push(self, wire, url, refs=None):
486 if not self.check_url(url, wire):
491 if not self.check_url(url, wire):
487 return
492 return
488
493
489 repo = self._factory.repo(wire)
494 repo = self._factory.repo(wire)
490 self.run_git_command(
495 self.run_git_command(
491 wire, ['push', url, '--mirror'], fail_on_stderr=False,
496 wire, ['push', url, '--mirror'], fail_on_stderr=False,
492 _copts=['-c', 'core.askpass=""'],
497 _copts=['-c', 'core.askpass=""'],
493 extra_env={'GIT_TERMINAL_PROMPT': '0'})
498 extra_env={'GIT_TERMINAL_PROMPT': '0'})
494
499
495 @reraise_safe_exceptions
500 @reraise_safe_exceptions
496 def get_remote_refs(self, wire, url):
501 def get_remote_refs(self, wire, url):
497 repo = Repo(url)
502 repo = Repo(url)
498 return repo.get_refs()
503 return repo.get_refs()
499
504
500 @reraise_safe_exceptions
505 @reraise_safe_exceptions
501 def get_description(self, wire):
506 def get_description(self, wire):
502 repo = self._factory.repo(wire)
507 repo = self._factory.repo(wire)
503 return repo.get_description()
508 return repo.get_description()
504
509
505 @reraise_safe_exceptions
510 @reraise_safe_exceptions
506 def get_file_history(self, wire, file_path, commit_id, limit):
511 def get_file_history(self, wire, file_path, commit_id, limit):
507 repo = self._factory.repo(wire)
512 repo = self._factory.repo(wire)
508 include = [commit_id]
513 include = [commit_id]
509 paths = [file_path]
514 paths = [file_path]
510
515
511 walker = repo.get_walker(include, paths=paths, max_entries=limit)
516 walker = repo.get_walker(include, paths=paths, max_entries=limit)
512 return [x.commit.id for x in walker]
517 return [x.commit.id for x in walker]
513
518
514 @reraise_safe_exceptions
519 @reraise_safe_exceptions
515 def get_missing_revs(self, wire, rev1, rev2, path2):
520 def get_missing_revs(self, wire, rev1, rev2, path2):
516 repo = self._factory.repo(wire)
521 repo = self._factory.repo(wire)
517 LocalGitClient(thin_packs=False).fetch(path2, repo)
522 LocalGitClient(thin_packs=False).fetch(path2, repo)
518
523
519 wire_remote = wire.copy()
524 wire_remote = wire.copy()
520 wire_remote['path'] = path2
525 wire_remote['path'] = path2
521 repo_remote = self._factory.repo(wire_remote)
526 repo_remote = self._factory.repo(wire_remote)
522 LocalGitClient(thin_packs=False).fetch(wire["path"], repo_remote)
527 LocalGitClient(thin_packs=False).fetch(wire["path"], repo_remote)
523
528
524 revs = [
529 revs = [
525 x.commit.id
530 x.commit.id
526 for x in repo_remote.get_walker(include=[rev2], exclude=[rev1])]
531 for x in repo_remote.get_walker(include=[rev2], exclude=[rev1])]
527 return revs
532 return revs
528
533
529 @reraise_safe_exceptions
534 @reraise_safe_exceptions
530 def get_object(self, wire, sha):
535 def get_object(self, wire, sha):
531 repo = self._factory.repo(wire)
536 repo = self._factory.repo(wire)
532 obj = repo.get_object(sha)
537 obj = repo.get_object(sha)
533 commit_id = obj.id
538 commit_id = obj.id
534
539
535 if isinstance(obj, Tag):
540 if isinstance(obj, Tag):
536 commit_id = obj.object[1]
541 commit_id = obj.object[1]
537
542
538 return {
543 return {
539 'id': obj.id,
544 'id': obj.id,
540 'type': obj.type_name,
545 'type': obj.type_name,
541 'commit_id': commit_id
546 'commit_id': commit_id
542 }
547 }
543
548
544 @reraise_safe_exceptions
549 @reraise_safe_exceptions
545 def get_object_attrs(self, wire, sha, *attrs):
550 def get_object_attrs(self, wire, sha, *attrs):
546 repo = self._factory.repo(wire)
551 repo = self._factory.repo(wire)
547 obj = repo.get_object(sha)
552 obj = repo.get_object(sha)
548 return list(getattr(obj, a) for a in attrs)
553 return list(getattr(obj, a) for a in attrs)
549
554
550 @reraise_safe_exceptions
555 @reraise_safe_exceptions
551 def get_refs(self, wire):
556 def get_refs(self, wire):
552 repo = self._factory.repo(wire)
557 repo = self._factory.repo(wire)
553 result = {}
558 result = {}
554 for ref, sha in repo.refs.as_dict().items():
559 for ref, sha in repo.refs.as_dict().items():
555 peeled_sha = repo.get_peeled(ref)
560 peeled_sha = repo.get_peeled(ref)
556 result[ref] = peeled_sha
561 result[ref] = peeled_sha
557 return result
562 return result
558
563
559 @reraise_safe_exceptions
564 @reraise_safe_exceptions
560 def get_refs_path(self, wire):
565 def get_refs_path(self, wire):
561 repo = self._factory.repo(wire)
566 repo = self._factory.repo(wire)
562 return repo.refs.path
567 return repo.refs.path
563
568
564 @reraise_safe_exceptions
569 @reraise_safe_exceptions
565 def head(self, wire, show_exc=True):
570 def head(self, wire, show_exc=True):
566 repo = self._factory.repo(wire)
571 repo = self._factory.repo(wire)
567 try:
572 try:
568 return repo.head()
573 return repo.head()
569 except Exception:
574 except Exception:
570 if show_exc:
575 if show_exc:
571 raise
576 raise
572
577
573 @reraise_safe_exceptions
578 @reraise_safe_exceptions
574 def init(self, wire):
579 def init(self, wire):
575 repo_path = str_to_dulwich(wire['path'])
580 repo_path = str_to_dulwich(wire['path'])
576 self.repo = Repo.init(repo_path)
581 self.repo = Repo.init(repo_path)
577
582
578 @reraise_safe_exceptions
583 @reraise_safe_exceptions
579 def init_bare(self, wire):
584 def init_bare(self, wire):
580 repo_path = str_to_dulwich(wire['path'])
585 repo_path = str_to_dulwich(wire['path'])
581 self.repo = Repo.init_bare(repo_path)
586 self.repo = Repo.init_bare(repo_path)
582
587
583 @reraise_safe_exceptions
588 @reraise_safe_exceptions
584 def revision(self, wire, rev):
589 def revision(self, wire, rev):
585 repo = self._factory.repo(wire)
590 repo = self._factory.repo(wire)
586 obj = repo[rev]
591 obj = repo[rev]
587 obj_data = {
592 obj_data = {
588 'id': obj.id,
593 'id': obj.id,
589 }
594 }
590 try:
595 try:
591 obj_data['tree'] = obj.tree
596 obj_data['tree'] = obj.tree
592 except AttributeError:
597 except AttributeError:
593 pass
598 pass
594 return obj_data
599 return obj_data
595
600
596 @reraise_safe_exceptions
601 @reraise_safe_exceptions
597 def commit_attribute(self, wire, rev, attr):
602 def commit_attribute(self, wire, rev, attr):
598 repo = self._factory.repo(wire)
603 repo = self._factory.repo(wire)
599 obj = repo[rev]
604 obj = repo[rev]
600 return getattr(obj, attr)
605 return getattr(obj, attr)
601
606
602 @reraise_safe_exceptions
607 @reraise_safe_exceptions
603 def set_refs(self, wire, key, value):
608 def set_refs(self, wire, key, value):
604 repo = self._factory.repo(wire)
609 repo = self._factory.repo(wire)
605 repo.refs[key] = value
610 repo.refs[key] = value
606
611
607 @reraise_safe_exceptions
612 @reraise_safe_exceptions
608 def remove_ref(self, wire, key):
613 def remove_ref(self, wire, key):
609 repo = self._factory.repo(wire)
614 repo = self._factory.repo(wire)
610 del repo.refs[key]
615 del repo.refs[key]
611
616
612 @reraise_safe_exceptions
617 @reraise_safe_exceptions
613 def tree_changes(self, wire, source_id, target_id):
618 def tree_changes(self, wire, source_id, target_id):
614 repo = self._factory.repo(wire)
619 repo = self._factory.repo(wire)
615 source = repo[source_id].tree if source_id else None
620 source = repo[source_id].tree if source_id else None
616 target = repo[target_id].tree
621 target = repo[target_id].tree
617 result = repo.object_store.tree_changes(source, target)
622 result = repo.object_store.tree_changes(source, target)
618 return list(result)
623 return list(result)
619
624
620 @reraise_safe_exceptions
625 @reraise_safe_exceptions
621 def tree_items(self, wire, tree_id):
626 def tree_items(self, wire, tree_id):
622 repo = self._factory.repo(wire)
627 repo = self._factory.repo(wire)
623 tree = repo[tree_id]
628 tree = repo[tree_id]
624
629
625 result = []
630 result = []
626 for item in tree.iteritems():
631 for item in tree.iteritems():
627 item_sha = item.sha
632 item_sha = item.sha
628 item_mode = item.mode
633 item_mode = item.mode
629
634
630 if FILE_MODE(item_mode) == GIT_LINK:
635 if FILE_MODE(item_mode) == GIT_LINK:
631 item_type = "link"
636 item_type = "link"
632 else:
637 else:
633 item_type = repo[item_sha].type_name
638 item_type = repo[item_sha].type_name
634
639
635 result.append((item.path, item_mode, item_sha, item_type))
640 result.append((item.path, item_mode, item_sha, item_type))
636 return result
641 return result
637
642
638 @reraise_safe_exceptions
643 @reraise_safe_exceptions
639 def update_server_info(self, wire):
644 def update_server_info(self, wire):
640 repo = self._factory.repo(wire)
645 repo = self._factory.repo(wire)
641 update_server_info(repo)
646 update_server_info(repo)
642
647
643 @reraise_safe_exceptions
648 @reraise_safe_exceptions
644 def discover_git_version(self):
649 def discover_git_version(self):
645 stdout, _ = self.run_git_command(
650 stdout, _ = self.run_git_command(
646 {}, ['--version'], _bare=True, _safe=True)
651 {}, ['--version'], _bare=True, _safe=True)
647 prefix = 'git version'
652 prefix = 'git version'
648 if stdout.startswith(prefix):
653 if stdout.startswith(prefix):
649 stdout = stdout[len(prefix):]
654 stdout = stdout[len(prefix):]
650 return stdout.strip()
655 return stdout.strip()
651
656
652 @reraise_safe_exceptions
657 @reraise_safe_exceptions
653 def run_git_command(self, wire, cmd, **opts):
658 def run_git_command(self, wire, cmd, **opts):
654 path = wire.get('path', None)
659 path = wire.get('path', None)
655
660
656 if path and os.path.isdir(path):
661 if path and os.path.isdir(path):
657 opts['cwd'] = path
662 opts['cwd'] = path
658
663
659 if '_bare' in opts:
664 if '_bare' in opts:
660 _copts = []
665 _copts = []
661 del opts['_bare']
666 del opts['_bare']
662 else:
667 else:
663 _copts = ['-c', 'core.quotepath=false', ]
668 _copts = ['-c', 'core.quotepath=false', ]
664 safe_call = False
669 safe_call = False
665 if '_safe' in opts:
670 if '_safe' in opts:
666 # no exc on failure
671 # no exc on failure
667 del opts['_safe']
672 del opts['_safe']
668 safe_call = True
673 safe_call = True
669
674
670 if '_copts' in opts:
675 if '_copts' in opts:
671 _copts.extend(opts['_copts'] or [])
676 _copts.extend(opts['_copts'] or [])
672 del opts['_copts']
677 del opts['_copts']
673
678
674 gitenv = os.environ.copy()
679 gitenv = os.environ.copy()
675 gitenv.update(opts.pop('extra_env', {}))
680 gitenv.update(opts.pop('extra_env', {}))
676 # need to clean fix GIT_DIR !
681 # need to clean fix GIT_DIR !
677 if 'GIT_DIR' in gitenv:
682 if 'GIT_DIR' in gitenv:
678 del gitenv['GIT_DIR']
683 del gitenv['GIT_DIR']
679 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
684 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
680 gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1'
685 gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1'
681
686
682 cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
687 cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
683
688
684 try:
689 try:
685 _opts = {'env': gitenv, 'shell': False}
690 _opts = {'env': gitenv, 'shell': False}
686 _opts.update(opts)
691 _opts.update(opts)
687 p = subprocessio.SubprocessIOChunker(cmd, **_opts)
692 p = subprocessio.SubprocessIOChunker(cmd, **_opts)
688
693
689 return ''.join(p), ''.join(p.error)
694 return ''.join(p), ''.join(p.error)
690 except (EnvironmentError, OSError) as err:
695 except (EnvironmentError, OSError) as err:
691 cmd = ' '.join(cmd) # human friendly CMD
696 cmd = ' '.join(cmd) # human friendly CMD
692 tb_err = ("Couldn't run git command (%s).\n"
697 tb_err = ("Couldn't run git command (%s).\n"
693 "Original error was:%s\n" % (cmd, err))
698 "Original error was:%s\n" % (cmd, err))
694 log.exception(tb_err)
699 log.exception(tb_err)
695 if safe_call:
700 if safe_call:
696 return '', err
701 return '', err
697 else:
702 else:
698 raise exceptions.VcsException()(tb_err)
703 raise exceptions.VcsException()(tb_err)
699
704
700 @reraise_safe_exceptions
705 @reraise_safe_exceptions
701 def install_hooks(self, wire, force=False):
706 def install_hooks(self, wire, force=False):
702 from vcsserver.hook_utils import install_git_hooks
707 from vcsserver.hook_utils import install_git_hooks
703 repo = self._factory.repo(wire)
708 repo = self._factory.repo(wire)
704 return install_git_hooks(repo.path, repo.bare, force_create=force)
709 return install_git_hooks(repo.path, repo.bare, force_create=force)
705
710
706
711
707 def str_to_dulwich(value):
712 def str_to_dulwich(value):
708 """
713 """
709 Dulwich 0.10.1a requires `unicode` objects to be passed in.
714 Dulwich 0.10.1a requires `unicode` objects to be passed in.
710 """
715 """
711 return value.decode(settings.WIRE_ENCODING)
716 return value.decode(settings.WIRE_ENCODING)
@@ -1,165 +1,165 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2018 RhodeCode GmbH
2 # Copyright (C) 2014-2018 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import inspect
18 import inspect
19
19
20 import pytest
20 import pytest
21 import dulwich.errors
21 import dulwich.errors
22 from mock import Mock, patch
22 from mock import Mock, patch
23
23
24 from vcsserver import git
24 from vcsserver import git
25
25
26
26
27 SAMPLE_REFS = {
27 SAMPLE_REFS = {
28 'HEAD': 'fd627b9e0dd80b47be81af07c4a98518244ed2f7',
28 'HEAD': 'fd627b9e0dd80b47be81af07c4a98518244ed2f7',
29 'refs/tags/v0.1.9': '341d28f0eec5ddf0b6b77871e13c2bbd6bec685c',
29 'refs/tags/v0.1.9': '341d28f0eec5ddf0b6b77871e13c2bbd6bec685c',
30 'refs/tags/v0.1.8': '74ebce002c088b8a5ecf40073db09375515ecd68',
30 'refs/tags/v0.1.8': '74ebce002c088b8a5ecf40073db09375515ecd68',
31 'refs/tags/v0.1.1': 'e6ea6d16e2f26250124a1f4b4fe37a912f9d86a0',
31 'refs/tags/v0.1.1': 'e6ea6d16e2f26250124a1f4b4fe37a912f9d86a0',
32 'refs/tags/v0.1.3': '5a3a8fb005554692b16e21dee62bf02667d8dc3e',
32 'refs/tags/v0.1.3': '5a3a8fb005554692b16e21dee62bf02667d8dc3e',
33 }
33 }
34
34
35
35
36 @pytest.fixture
36 @pytest.fixture
37 def git_remote():
37 def git_remote():
38 """
38 """
39 A GitRemote instance with a mock factory.
39 A GitRemote instance with a mock factory.
40 """
40 """
41 factory = Mock()
41 factory = Mock()
42 remote = git.GitRemote(factory)
42 remote = git.GitRemote(factory)
43 return remote
43 return remote
44
44
45
45
46 def test_discover_git_version(git_remote):
46 def test_discover_git_version(git_remote):
47 version = git_remote.discover_git_version()
47 version = git_remote.discover_git_version()
48 assert version
48 assert version
49
49
50
50
51 class TestGitFetch(object):
51 class TestGitFetch(object):
52 def setup(self):
52 def setup(self):
53 self.mock_repo = Mock()
53 self.mock_repo = Mock()
54 factory = Mock()
54 factory = Mock()
55 factory.repo = Mock(return_value=self.mock_repo)
55 factory.repo = Mock(return_value=self.mock_repo)
56 self.remote_git = git.GitRemote(factory)
56 self.remote_git = git.GitRemote(factory)
57
57
58 def test_fetches_all_when_no_commit_ids_specified(self):
58 def test_fetches_all_when_no_commit_ids_specified(self):
59 def side_effect(determine_wants, *args, **kwargs):
59 def side_effect(determine_wants, *args, **kwargs):
60 determine_wants(SAMPLE_REFS)
60 determine_wants(SAMPLE_REFS)
61
61
62 with patch('dulwich.client.LocalGitClient.fetch') as mock_fetch:
62 with patch('dulwich.client.LocalGitClient.fetch') as mock_fetch:
63 mock_fetch.side_effect = side_effect
63 mock_fetch.side_effect = side_effect
64 self.remote_git.fetch(wire=None, url='/tmp/', apply_refs=False)
64 self.remote_git.pull(wire=None, url='/tmp/', apply_refs=False)
65 determine_wants = self.mock_repo.object_store.determine_wants_all
65 determine_wants = self.mock_repo.object_store.determine_wants_all
66 determine_wants.assert_called_once_with(SAMPLE_REFS)
66 determine_wants.assert_called_once_with(SAMPLE_REFS)
67
67
68 def test_fetches_specified_commits(self):
68 def test_fetches_specified_commits(self):
69 selected_refs = {
69 selected_refs = {
70 'refs/tags/v0.1.8': '74ebce002c088b8a5ecf40073db09375515ecd68',
70 'refs/tags/v0.1.8': '74ebce002c088b8a5ecf40073db09375515ecd68',
71 'refs/tags/v0.1.3': '5a3a8fb005554692b16e21dee62bf02667d8dc3e',
71 'refs/tags/v0.1.3': '5a3a8fb005554692b16e21dee62bf02667d8dc3e',
72 }
72 }
73
73
74 def side_effect(determine_wants, *args, **kwargs):
74 def side_effect(determine_wants, *args, **kwargs):
75 result = determine_wants(SAMPLE_REFS)
75 result = determine_wants(SAMPLE_REFS)
76 assert sorted(result) == sorted(selected_refs.values())
76 assert sorted(result) == sorted(selected_refs.values())
77 return result
77 return result
78
78
79 with patch('dulwich.client.LocalGitClient.fetch') as mock_fetch:
79 with patch('dulwich.client.LocalGitClient.fetch') as mock_fetch:
80 mock_fetch.side_effect = side_effect
80 mock_fetch.side_effect = side_effect
81 self.remote_git.fetch(
81 self.remote_git.pull(
82 wire=None, url='/tmp/', apply_refs=False,
82 wire=None, url='/tmp/', apply_refs=False,
83 refs=selected_refs.keys())
83 refs=selected_refs.keys())
84 determine_wants = self.mock_repo.object_store.determine_wants_all
84 determine_wants = self.mock_repo.object_store.determine_wants_all
85 assert determine_wants.call_count == 0
85 assert determine_wants.call_count == 0
86
86
87 def test_get_remote_refs(self):
87 def test_get_remote_refs(self):
88 factory = Mock()
88 factory = Mock()
89 remote_git = git.GitRemote(factory)
89 remote_git = git.GitRemote(factory)
90 url = 'http://example.com/test/test.git'
90 url = 'http://example.com/test/test.git'
91 sample_refs = {
91 sample_refs = {
92 'refs/tags/v0.1.8': '74ebce002c088b8a5ecf40073db09375515ecd68',
92 'refs/tags/v0.1.8': '74ebce002c088b8a5ecf40073db09375515ecd68',
93 'refs/tags/v0.1.3': '5a3a8fb005554692b16e21dee62bf02667d8dc3e',
93 'refs/tags/v0.1.3': '5a3a8fb005554692b16e21dee62bf02667d8dc3e',
94 }
94 }
95
95
96 with patch('vcsserver.git.Repo', create=False) as mock_repo:
96 with patch('vcsserver.git.Repo', create=False) as mock_repo:
97 mock_repo().get_refs.return_value = sample_refs
97 mock_repo().get_refs.return_value = sample_refs
98 remote_refs = remote_git.get_remote_refs(wire=None, url=url)
98 remote_refs = remote_git.get_remote_refs(wire=None, url=url)
99 mock_repo().get_refs.assert_called_once_with()
99 mock_repo().get_refs.assert_called_once_with()
100 assert remote_refs == sample_refs
100 assert remote_refs == sample_refs
101
101
102 def test_remove_ref(self):
102 def test_remove_ref(self):
103 ref_to_remove = 'refs/tags/v0.1.9'
103 ref_to_remove = 'refs/tags/v0.1.9'
104 self.mock_repo.refs = SAMPLE_REFS.copy()
104 self.mock_repo.refs = SAMPLE_REFS.copy()
105 self.remote_git.remove_ref(None, ref_to_remove)
105 self.remote_git.remove_ref(None, ref_to_remove)
106 assert ref_to_remove not in self.mock_repo.refs
106 assert ref_to_remove not in self.mock_repo.refs
107
107
108
108
109 class TestReraiseSafeExceptions(object):
109 class TestReraiseSafeExceptions(object):
110 def test_method_decorated_with_reraise_safe_exceptions(self):
110 def test_method_decorated_with_reraise_safe_exceptions(self):
111 factory = Mock()
111 factory = Mock()
112 git_remote = git.GitRemote(factory)
112 git_remote = git.GitRemote(factory)
113
113
114 def fake_function():
114 def fake_function():
115 return None
115 return None
116
116
117 decorator = git.reraise_safe_exceptions(fake_function)
117 decorator = git.reraise_safe_exceptions(fake_function)
118
118
119 methods = inspect.getmembers(git_remote, predicate=inspect.ismethod)
119 methods = inspect.getmembers(git_remote, predicate=inspect.ismethod)
120 for method_name, method in methods:
120 for method_name, method in methods:
121 if not method_name.startswith('_'):
121 if not method_name.startswith('_'):
122 assert method.im_func.__code__ == decorator.__code__
122 assert method.im_func.__code__ == decorator.__code__
123
123
124 @pytest.mark.parametrize('side_effect, expected_type', [
124 @pytest.mark.parametrize('side_effect, expected_type', [
125 (dulwich.errors.ChecksumMismatch('0000000', 'deadbeef'), 'lookup'),
125 (dulwich.errors.ChecksumMismatch('0000000', 'deadbeef'), 'lookup'),
126 (dulwich.errors.NotCommitError('deadbeef'), 'lookup'),
126 (dulwich.errors.NotCommitError('deadbeef'), 'lookup'),
127 (dulwich.errors.MissingCommitError('deadbeef'), 'lookup'),
127 (dulwich.errors.MissingCommitError('deadbeef'), 'lookup'),
128 (dulwich.errors.ObjectMissing('deadbeef'), 'lookup'),
128 (dulwich.errors.ObjectMissing('deadbeef'), 'lookup'),
129 (dulwich.errors.HangupException(), 'error'),
129 (dulwich.errors.HangupException(), 'error'),
130 (dulwich.errors.UnexpectedCommandError('test-cmd'), 'error'),
130 (dulwich.errors.UnexpectedCommandError('test-cmd'), 'error'),
131 ])
131 ])
132 def test_safe_exceptions_reraised(self, side_effect, expected_type):
132 def test_safe_exceptions_reraised(self, side_effect, expected_type):
133 @git.reraise_safe_exceptions
133 @git.reraise_safe_exceptions
134 def fake_method():
134 def fake_method():
135 raise side_effect
135 raise side_effect
136
136
137 with pytest.raises(Exception) as exc_info:
137 with pytest.raises(Exception) as exc_info:
138 fake_method()
138 fake_method()
139 assert type(exc_info.value) == Exception
139 assert type(exc_info.value) == Exception
140 assert exc_info.value._vcs_kind == expected_type
140 assert exc_info.value._vcs_kind == expected_type
141
141
142
142
143 class TestDulwichRepoWrapper(object):
143 class TestDulwichRepoWrapper(object):
144 def test_calls_close_on_delete(self):
144 def test_calls_close_on_delete(self):
145 isdir_patcher = patch('dulwich.repo.os.path.isdir', return_value=True)
145 isdir_patcher = patch('dulwich.repo.os.path.isdir', return_value=True)
146 with isdir_patcher:
146 with isdir_patcher:
147 repo = git.Repo('/tmp/abcde')
147 repo = git.Repo('/tmp/abcde')
148 with patch.object(git.DulwichRepo, 'close') as close_mock:
148 with patch.object(git.DulwichRepo, 'close') as close_mock:
149 del repo
149 del repo
150 close_mock.assert_called_once_with()
150 close_mock.assert_called_once_with()
151
151
152
152
153 class TestGitFactory(object):
153 class TestGitFactory(object):
154 def test_create_repo_returns_dulwich_wrapper(self):
154 def test_create_repo_returns_dulwich_wrapper(self):
155
155
156 with patch('vcsserver.lib.rc_cache.region_meta.dogpile_cache_regions') as mock:
156 with patch('vcsserver.lib.rc_cache.region_meta.dogpile_cache_regions') as mock:
157 mock.side_effect = {'repo_objects': ''}
157 mock.side_effect = {'repo_objects': ''}
158 factory = git.GitFactory()
158 factory = git.GitFactory()
159 wire = {
159 wire = {
160 'path': '/tmp/abcde'
160 'path': '/tmp/abcde'
161 }
161 }
162 isdir_patcher = patch('dulwich.repo.os.path.isdir', return_value=True)
162 isdir_patcher = patch('dulwich.repo.os.path.isdir', return_value=True)
163 with isdir_patcher:
163 with isdir_patcher:
164 result = factory._create_repo(wire, True)
164 result = factory._create_repo(wire, True)
165 assert isinstance(result, git.Repo)
165 assert isinstance(result, git.Repo)
General Comments 0
You need to be logged in to leave comments. Login now