##// END OF EJS Templates
git: use GIT_DISCOVERY_ACROSS_FILESYSTEM for better compatability on NFS servers and shadow repo.
marcink -
r460:2ba2d6b7 default
parent child Browse files
Show More
@@ -1,669 +1,670 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2018 RhodeCode GmbH
2 # Copyright (C) 2014-2018 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import logging
18 import logging
19 import os
19 import os
20 import posixpath as vcspath
20 import posixpath as vcspath
21 import re
21 import re
22 import stat
22 import stat
23 import traceback
23 import traceback
24 import urllib
24 import urllib
25 import urllib2
25 import urllib2
26 from functools import wraps
26 from functools import wraps
27
27
28 from dulwich import index, objects
28 from dulwich import index, objects
29 from dulwich.client import HttpGitClient, LocalGitClient
29 from dulwich.client import HttpGitClient, LocalGitClient
30 from dulwich.errors import (
30 from dulwich.errors import (
31 NotGitRepository, ChecksumMismatch, WrongObjectException,
31 NotGitRepository, ChecksumMismatch, WrongObjectException,
32 MissingCommitError, ObjectMissing, HangupException,
32 MissingCommitError, ObjectMissing, HangupException,
33 UnexpectedCommandError)
33 UnexpectedCommandError)
34 from dulwich.repo import Repo as DulwichRepo, Tag
34 from dulwich.repo import Repo as DulwichRepo, Tag
35 from dulwich.server import update_server_info
35 from dulwich.server import update_server_info
36
36
37 from vcsserver import exceptions, settings, subprocessio
37 from vcsserver import exceptions, settings, subprocessio
38 from vcsserver.utils import safe_str
38 from vcsserver.utils import safe_str
39 from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original
39 from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original
40 from vcsserver.hgcompat import (
40 from vcsserver.hgcompat import (
41 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
41 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
42 from vcsserver.git_lfs.lib import LFSOidStore
42 from vcsserver.git_lfs.lib import LFSOidStore
43
43
44 DIR_STAT = stat.S_IFDIR
44 DIR_STAT = stat.S_IFDIR
45 FILE_MODE = stat.S_IFMT
45 FILE_MODE = stat.S_IFMT
46 GIT_LINK = objects.S_IFGITLINK
46 GIT_LINK = objects.S_IFGITLINK
47
47
48 log = logging.getLogger(__name__)
48 log = logging.getLogger(__name__)
49
49
50
50
51 def reraise_safe_exceptions(func):
51 def reraise_safe_exceptions(func):
52 """Converts Dulwich exceptions to something neutral."""
52 """Converts Dulwich exceptions to something neutral."""
53 @wraps(func)
53 @wraps(func)
54 def wrapper(*args, **kwargs):
54 def wrapper(*args, **kwargs):
55 try:
55 try:
56 return func(*args, **kwargs)
56 return func(*args, **kwargs)
57 except (ChecksumMismatch, WrongObjectException, MissingCommitError,
57 except (ChecksumMismatch, WrongObjectException, MissingCommitError,
58 ObjectMissing) as e:
58 ObjectMissing) as e:
59 raise exceptions.LookupException(e.message)
59 raise exceptions.LookupException(e.message)
60 except (HangupException, UnexpectedCommandError) as e:
60 except (HangupException, UnexpectedCommandError) as e:
61 raise exceptions.VcsException(e.message)
61 raise exceptions.VcsException(e.message)
62 except Exception as e:
62 except Exception as e:
63 # NOTE(marcink): becuase of how dulwich handles some exceptions
63 # NOTE(marcink): becuase of how dulwich handles some exceptions
64 # (KeyError on empty repos), we cannot track this and catch all
64 # (KeyError on empty repos), we cannot track this and catch all
65 # exceptions, it's an exceptions from other handlers
65 # exceptions, it's an exceptions from other handlers
66 #if not hasattr(e, '_vcs_kind'):
66 #if not hasattr(e, '_vcs_kind'):
67 #log.exception("Unhandled exception in git remote call")
67 #log.exception("Unhandled exception in git remote call")
68 #raise_from_original(exceptions.UnhandledException)
68 #raise_from_original(exceptions.UnhandledException)
69 raise
69 raise
70 return wrapper
70 return wrapper
71
71
72
72
73 class Repo(DulwichRepo):
73 class Repo(DulwichRepo):
74 """
74 """
75 A wrapper for dulwich Repo class.
75 A wrapper for dulwich Repo class.
76
76
77 Since dulwich is sometimes keeping .idx file descriptors open, it leads to
77 Since dulwich is sometimes keeping .idx file descriptors open, it leads to
78 "Too many open files" error. We need to close all opened file descriptors
78 "Too many open files" error. We need to close all opened file descriptors
79 once the repo object is destroyed.
79 once the repo object is destroyed.
80
80
81 TODO: mikhail: please check if we need this wrapper after updating dulwich
81 TODO: mikhail: please check if we need this wrapper after updating dulwich
82 to 0.12.0 +
82 to 0.12.0 +
83 """
83 """
84 def __del__(self):
84 def __del__(self):
85 if hasattr(self, 'object_store'):
85 if hasattr(self, 'object_store'):
86 self.close()
86 self.close()
87
87
88
88
89 class GitFactory(RepoFactory):
89 class GitFactory(RepoFactory):
90
90
91 def _create_repo(self, wire, create):
91 def _create_repo(self, wire, create):
92 repo_path = str_to_dulwich(wire['path'])
92 repo_path = str_to_dulwich(wire['path'])
93 return Repo(repo_path)
93 return Repo(repo_path)
94
94
95
95
96 class GitRemote(object):
96 class GitRemote(object):
97
97
98 def __init__(self, factory):
98 def __init__(self, factory):
99 self._factory = factory
99 self._factory = factory
100
100
101 self._bulk_methods = {
101 self._bulk_methods = {
102 "author": self.commit_attribute,
102 "author": self.commit_attribute,
103 "date": self.get_object_attrs,
103 "date": self.get_object_attrs,
104 "message": self.commit_attribute,
104 "message": self.commit_attribute,
105 "parents": self.commit_attribute,
105 "parents": self.commit_attribute,
106 "_commit": self.revision,
106 "_commit": self.revision,
107 }
107 }
108
108
109 def _wire_to_config(self, wire):
109 def _wire_to_config(self, wire):
110 if 'config' in wire:
110 if 'config' in wire:
111 return dict([(x[0] + '_' + x[1], x[2]) for x in wire['config']])
111 return dict([(x[0] + '_' + x[1], x[2]) for x in wire['config']])
112 return {}
112 return {}
113
113
114 def _assign_ref(self, wire, ref, commit_id):
114 def _assign_ref(self, wire, ref, commit_id):
115 repo = self._factory.repo(wire)
115 repo = self._factory.repo(wire)
116 repo[ref] = commit_id
116 repo[ref] = commit_id
117
117
118 @reraise_safe_exceptions
118 @reraise_safe_exceptions
119 def add_object(self, wire, content):
119 def add_object(self, wire, content):
120 repo = self._factory.repo(wire)
120 repo = self._factory.repo(wire)
121 blob = objects.Blob()
121 blob = objects.Blob()
122 blob.set_raw_string(content)
122 blob.set_raw_string(content)
123 repo.object_store.add_object(blob)
123 repo.object_store.add_object(blob)
124 return blob.id
124 return blob.id
125
125
126 @reraise_safe_exceptions
126 @reraise_safe_exceptions
127 def assert_correct_path(self, wire):
127 def assert_correct_path(self, wire):
128 path = wire.get('path')
128 path = wire.get('path')
129 try:
129 try:
130 self._factory.repo(wire)
130 self._factory.repo(wire)
131 except NotGitRepository as e:
131 except NotGitRepository as e:
132 tb = traceback.format_exc()
132 tb = traceback.format_exc()
133 log.debug("Invalid Git path `%s`, tb: %s", path, tb)
133 log.debug("Invalid Git path `%s`, tb: %s", path, tb)
134 return False
134 return False
135
135
136 return True
136 return True
137
137
138 @reraise_safe_exceptions
138 @reraise_safe_exceptions
139 def bare(self, wire):
139 def bare(self, wire):
140 repo = self._factory.repo(wire)
140 repo = self._factory.repo(wire)
141 return repo.bare
141 return repo.bare
142
142
143 @reraise_safe_exceptions
143 @reraise_safe_exceptions
144 def blob_as_pretty_string(self, wire, sha):
144 def blob_as_pretty_string(self, wire, sha):
145 repo = self._factory.repo(wire)
145 repo = self._factory.repo(wire)
146 return repo[sha].as_pretty_string()
146 return repo[sha].as_pretty_string()
147
147
148 @reraise_safe_exceptions
148 @reraise_safe_exceptions
149 def blob_raw_length(self, wire, sha):
149 def blob_raw_length(self, wire, sha):
150 repo = self._factory.repo(wire)
150 repo = self._factory.repo(wire)
151 blob = repo[sha]
151 blob = repo[sha]
152 return blob.raw_length()
152 return blob.raw_length()
153
153
154 def _parse_lfs_pointer(self, raw_content):
154 def _parse_lfs_pointer(self, raw_content):
155
155
156 spec_string = 'version https://git-lfs.github.com/spec'
156 spec_string = 'version https://git-lfs.github.com/spec'
157 if raw_content and raw_content.startswith(spec_string):
157 if raw_content and raw_content.startswith(spec_string):
158 pattern = re.compile(r"""
158 pattern = re.compile(r"""
159 (?:\n)?
159 (?:\n)?
160 ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
160 ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
161 ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
161 ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
162 ^size[ ](?P<oid_size>[0-9]+)\n
162 ^size[ ](?P<oid_size>[0-9]+)\n
163 (?:\n)?
163 (?:\n)?
164 """, re.VERBOSE | re.MULTILINE)
164 """, re.VERBOSE | re.MULTILINE)
165 match = pattern.match(raw_content)
165 match = pattern.match(raw_content)
166 if match:
166 if match:
167 return match.groupdict()
167 return match.groupdict()
168
168
169 return {}
169 return {}
170
170
171 @reraise_safe_exceptions
171 @reraise_safe_exceptions
172 def is_large_file(self, wire, sha):
172 def is_large_file(self, wire, sha):
173 repo = self._factory.repo(wire)
173 repo = self._factory.repo(wire)
174 blob = repo[sha]
174 blob = repo[sha]
175 return self._parse_lfs_pointer(blob.as_raw_string())
175 return self._parse_lfs_pointer(blob.as_raw_string())
176
176
177 @reraise_safe_exceptions
177 @reraise_safe_exceptions
178 def in_largefiles_store(self, wire, oid):
178 def in_largefiles_store(self, wire, oid):
179 repo = self._factory.repo(wire)
179 repo = self._factory.repo(wire)
180 conf = self._wire_to_config(wire)
180 conf = self._wire_to_config(wire)
181
181
182 store_location = conf.get('vcs_git_lfs_store_location')
182 store_location = conf.get('vcs_git_lfs_store_location')
183 if store_location:
183 if store_location:
184 repo_name = repo.path
184 repo_name = repo.path
185 store = LFSOidStore(
185 store = LFSOidStore(
186 oid=oid, repo=repo_name, store_location=store_location)
186 oid=oid, repo=repo_name, store_location=store_location)
187 return store.has_oid()
187 return store.has_oid()
188
188
189 return False
189 return False
190
190
191 @reraise_safe_exceptions
191 @reraise_safe_exceptions
192 def store_path(self, wire, oid):
192 def store_path(self, wire, oid):
193 repo = self._factory.repo(wire)
193 repo = self._factory.repo(wire)
194 conf = self._wire_to_config(wire)
194 conf = self._wire_to_config(wire)
195
195
196 store_location = conf.get('vcs_git_lfs_store_location')
196 store_location = conf.get('vcs_git_lfs_store_location')
197 if store_location:
197 if store_location:
198 repo_name = repo.path
198 repo_name = repo.path
199 store = LFSOidStore(
199 store = LFSOidStore(
200 oid=oid, repo=repo_name, store_location=store_location)
200 oid=oid, repo=repo_name, store_location=store_location)
201 return store.oid_path
201 return store.oid_path
202 raise ValueError('Unable to fetch oid with path {}'.format(oid))
202 raise ValueError('Unable to fetch oid with path {}'.format(oid))
203
203
204 @reraise_safe_exceptions
204 @reraise_safe_exceptions
205 def bulk_request(self, wire, rev, pre_load):
205 def bulk_request(self, wire, rev, pre_load):
206 result = {}
206 result = {}
207 for attr in pre_load:
207 for attr in pre_load:
208 try:
208 try:
209 method = self._bulk_methods[attr]
209 method = self._bulk_methods[attr]
210 args = [wire, rev]
210 args = [wire, rev]
211 if attr == "date":
211 if attr == "date":
212 args.extend(["commit_time", "commit_timezone"])
212 args.extend(["commit_time", "commit_timezone"])
213 elif attr in ["author", "message", "parents"]:
213 elif attr in ["author", "message", "parents"]:
214 args.append(attr)
214 args.append(attr)
215 result[attr] = method(*args)
215 result[attr] = method(*args)
216 except KeyError:
216 except KeyError:
217 raise exceptions.VcsException(
217 raise exceptions.VcsException(
218 "Unknown bulk attribute: %s" % attr)
218 "Unknown bulk attribute: %s" % attr)
219 return result
219 return result
220
220
221 def _build_opener(self, url):
221 def _build_opener(self, url):
222 handlers = []
222 handlers = []
223 url_obj = url_parser(url)
223 url_obj = url_parser(url)
224 _, authinfo = url_obj.authinfo()
224 _, authinfo = url_obj.authinfo()
225
225
226 if authinfo:
226 if authinfo:
227 # create a password manager
227 # create a password manager
228 passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
228 passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
229 passmgr.add_password(*authinfo)
229 passmgr.add_password(*authinfo)
230
230
231 handlers.extend((httpbasicauthhandler(passmgr),
231 handlers.extend((httpbasicauthhandler(passmgr),
232 httpdigestauthhandler(passmgr)))
232 httpdigestauthhandler(passmgr)))
233
233
234 return urllib2.build_opener(*handlers)
234 return urllib2.build_opener(*handlers)
235
235
236 @reraise_safe_exceptions
236 @reraise_safe_exceptions
237 def check_url(self, url, config):
237 def check_url(self, url, config):
238 url_obj = url_parser(url)
238 url_obj = url_parser(url)
239 test_uri, _ = url_obj.authinfo()
239 test_uri, _ = url_obj.authinfo()
240 url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
240 url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
241 url_obj.query = obfuscate_qs(url_obj.query)
241 url_obj.query = obfuscate_qs(url_obj.query)
242 cleaned_uri = str(url_obj)
242 cleaned_uri = str(url_obj)
243 log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
243 log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
244
244
245 if not test_uri.endswith('info/refs'):
245 if not test_uri.endswith('info/refs'):
246 test_uri = test_uri.rstrip('/') + '/info/refs'
246 test_uri = test_uri.rstrip('/') + '/info/refs'
247
247
248 o = self._build_opener(url)
248 o = self._build_opener(url)
249 o.addheaders = [('User-Agent', 'git/1.7.8.0')] # fake some git
249 o.addheaders = [('User-Agent', 'git/1.7.8.0')] # fake some git
250
250
251 q = {"service": 'git-upload-pack'}
251 q = {"service": 'git-upload-pack'}
252 qs = '?%s' % urllib.urlencode(q)
252 qs = '?%s' % urllib.urlencode(q)
253 cu = "%s%s" % (test_uri, qs)
253 cu = "%s%s" % (test_uri, qs)
254 req = urllib2.Request(cu, None, {})
254 req = urllib2.Request(cu, None, {})
255
255
256 try:
256 try:
257 log.debug("Trying to open URL %s", cleaned_uri)
257 log.debug("Trying to open URL %s", cleaned_uri)
258 resp = o.open(req)
258 resp = o.open(req)
259 if resp.code != 200:
259 if resp.code != 200:
260 raise exceptions.URLError('Return Code is not 200')
260 raise exceptions.URLError('Return Code is not 200')
261 except Exception as e:
261 except Exception as e:
262 log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
262 log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
263 # means it cannot be cloned
263 # means it cannot be cloned
264 raise exceptions.URLError("[%s] org_exc: %s" % (cleaned_uri, e))
264 raise exceptions.URLError("[%s] org_exc: %s" % (cleaned_uri, e))
265
265
266 # now detect if it's proper git repo
266 # now detect if it's proper git repo
267 gitdata = resp.read()
267 gitdata = resp.read()
268 if 'service=git-upload-pack' in gitdata:
268 if 'service=git-upload-pack' in gitdata:
269 pass
269 pass
270 elif re.findall(r'[0-9a-fA-F]{40}\s+refs', gitdata):
270 elif re.findall(r'[0-9a-fA-F]{40}\s+refs', gitdata):
271 # old style git can return some other format !
271 # old style git can return some other format !
272 pass
272 pass
273 else:
273 else:
274 raise exceptions.URLError(
274 raise exceptions.URLError(
275 "url [%s] does not look like an git" % (cleaned_uri,))
275 "url [%s] does not look like an git" % (cleaned_uri,))
276
276
277 return True
277 return True
278
278
279 @reraise_safe_exceptions
279 @reraise_safe_exceptions
280 def clone(self, wire, url, deferred, valid_refs, update_after_clone):
280 def clone(self, wire, url, deferred, valid_refs, update_after_clone):
281 remote_refs = self.fetch(wire, url, apply_refs=False)
281 remote_refs = self.fetch(wire, url, apply_refs=False)
282 repo = self._factory.repo(wire)
282 repo = self._factory.repo(wire)
283 if isinstance(valid_refs, list):
283 if isinstance(valid_refs, list):
284 valid_refs = tuple(valid_refs)
284 valid_refs = tuple(valid_refs)
285
285
286 for k in remote_refs:
286 for k in remote_refs:
287 # only parse heads/tags and skip so called deferred tags
287 # only parse heads/tags and skip so called deferred tags
288 if k.startswith(valid_refs) and not k.endswith(deferred):
288 if k.startswith(valid_refs) and not k.endswith(deferred):
289 repo[k] = remote_refs[k]
289 repo[k] = remote_refs[k]
290
290
291 if update_after_clone:
291 if update_after_clone:
292 # we want to checkout HEAD
292 # we want to checkout HEAD
293 repo["HEAD"] = remote_refs["HEAD"]
293 repo["HEAD"] = remote_refs["HEAD"]
294 index.build_index_from_tree(repo.path, repo.index_path(),
294 index.build_index_from_tree(repo.path, repo.index_path(),
295 repo.object_store, repo["HEAD"].tree)
295 repo.object_store, repo["HEAD"].tree)
296
296
297 # TODO: this is quite complex, check if that can be simplified
297 # TODO: this is quite complex, check if that can be simplified
298 @reraise_safe_exceptions
298 @reraise_safe_exceptions
299 def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
299 def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
300 repo = self._factory.repo(wire)
300 repo = self._factory.repo(wire)
301 object_store = repo.object_store
301 object_store = repo.object_store
302
302
303 # Create tree and populates it with blobs
303 # Create tree and populates it with blobs
304 commit_tree = commit_tree and repo[commit_tree] or objects.Tree()
304 commit_tree = commit_tree and repo[commit_tree] or objects.Tree()
305
305
306 for node in updated:
306 for node in updated:
307 # Compute subdirs if needed
307 # Compute subdirs if needed
308 dirpath, nodename = vcspath.split(node['path'])
308 dirpath, nodename = vcspath.split(node['path'])
309 dirnames = map(safe_str, dirpath and dirpath.split('/') or [])
309 dirnames = map(safe_str, dirpath and dirpath.split('/') or [])
310 parent = commit_tree
310 parent = commit_tree
311 ancestors = [('', parent)]
311 ancestors = [('', parent)]
312
312
313 # Tries to dig for the deepest existing tree
313 # Tries to dig for the deepest existing tree
314 while dirnames:
314 while dirnames:
315 curdir = dirnames.pop(0)
315 curdir = dirnames.pop(0)
316 try:
316 try:
317 dir_id = parent[curdir][1]
317 dir_id = parent[curdir][1]
318 except KeyError:
318 except KeyError:
319 # put curdir back into dirnames and stops
319 # put curdir back into dirnames and stops
320 dirnames.insert(0, curdir)
320 dirnames.insert(0, curdir)
321 break
321 break
322 else:
322 else:
323 # If found, updates parent
323 # If found, updates parent
324 parent = repo[dir_id]
324 parent = repo[dir_id]
325 ancestors.append((curdir, parent))
325 ancestors.append((curdir, parent))
326 # Now parent is deepest existing tree and we need to create
326 # Now parent is deepest existing tree and we need to create
327 # subtrees for dirnames (in reverse order)
327 # subtrees for dirnames (in reverse order)
328 # [this only applies for nodes from added]
328 # [this only applies for nodes from added]
329 new_trees = []
329 new_trees = []
330
330
331 blob = objects.Blob.from_string(node['content'])
331 blob = objects.Blob.from_string(node['content'])
332
332
333 if dirnames:
333 if dirnames:
334 # If there are trees which should be created we need to build
334 # If there are trees which should be created we need to build
335 # them now (in reverse order)
335 # them now (in reverse order)
336 reversed_dirnames = list(reversed(dirnames))
336 reversed_dirnames = list(reversed(dirnames))
337 curtree = objects.Tree()
337 curtree = objects.Tree()
338 curtree[node['node_path']] = node['mode'], blob.id
338 curtree[node['node_path']] = node['mode'], blob.id
339 new_trees.append(curtree)
339 new_trees.append(curtree)
340 for dirname in reversed_dirnames[:-1]:
340 for dirname in reversed_dirnames[:-1]:
341 newtree = objects.Tree()
341 newtree = objects.Tree()
342 newtree[dirname] = (DIR_STAT, curtree.id)
342 newtree[dirname] = (DIR_STAT, curtree.id)
343 new_trees.append(newtree)
343 new_trees.append(newtree)
344 curtree = newtree
344 curtree = newtree
345 parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
345 parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
346 else:
346 else:
347 parent.add(
347 parent.add(
348 name=node['node_path'], mode=node['mode'], hexsha=blob.id)
348 name=node['node_path'], mode=node['mode'], hexsha=blob.id)
349
349
350 new_trees.append(parent)
350 new_trees.append(parent)
351 # Update ancestors
351 # Update ancestors
352 reversed_ancestors = reversed(
352 reversed_ancestors = reversed(
353 [(a[1], b[1], b[0]) for a, b in zip(ancestors, ancestors[1:])])
353 [(a[1], b[1], b[0]) for a, b in zip(ancestors, ancestors[1:])])
354 for parent, tree, path in reversed_ancestors:
354 for parent, tree, path in reversed_ancestors:
355 parent[path] = (DIR_STAT, tree.id)
355 parent[path] = (DIR_STAT, tree.id)
356 object_store.add_object(tree)
356 object_store.add_object(tree)
357
357
358 object_store.add_object(blob)
358 object_store.add_object(blob)
359 for tree in new_trees:
359 for tree in new_trees:
360 object_store.add_object(tree)
360 object_store.add_object(tree)
361
361
362 for node_path in removed:
362 for node_path in removed:
363 paths = node_path.split('/')
363 paths = node_path.split('/')
364 tree = commit_tree
364 tree = commit_tree
365 trees = [tree]
365 trees = [tree]
366 # Traverse deep into the forest...
366 # Traverse deep into the forest...
367 for path in paths:
367 for path in paths:
368 try:
368 try:
369 obj = repo[tree[path][1]]
369 obj = repo[tree[path][1]]
370 if isinstance(obj, objects.Tree):
370 if isinstance(obj, objects.Tree):
371 trees.append(obj)
371 trees.append(obj)
372 tree = obj
372 tree = obj
373 except KeyError:
373 except KeyError:
374 break
374 break
375 # Cut down the blob and all rotten trees on the way back...
375 # Cut down the blob and all rotten trees on the way back...
376 for path, tree in reversed(zip(paths, trees)):
376 for path, tree in reversed(zip(paths, trees)):
377 del tree[path]
377 del tree[path]
378 if tree:
378 if tree:
379 # This tree still has elements - don't remove it or any
379 # This tree still has elements - don't remove it or any
380 # of it's parents
380 # of it's parents
381 break
381 break
382
382
383 object_store.add_object(commit_tree)
383 object_store.add_object(commit_tree)
384
384
385 # Create commit
385 # Create commit
386 commit = objects.Commit()
386 commit = objects.Commit()
387 commit.tree = commit_tree.id
387 commit.tree = commit_tree.id
388 for k, v in commit_data.iteritems():
388 for k, v in commit_data.iteritems():
389 setattr(commit, k, v)
389 setattr(commit, k, v)
390 object_store.add_object(commit)
390 object_store.add_object(commit)
391
391
392 ref = 'refs/heads/%s' % branch
392 ref = 'refs/heads/%s' % branch
393 repo.refs[ref] = commit.id
393 repo.refs[ref] = commit.id
394
394
395 return commit.id
395 return commit.id
396
396
397 @reraise_safe_exceptions
397 @reraise_safe_exceptions
398 def fetch(self, wire, url, apply_refs=True, refs=None):
398 def fetch(self, wire, url, apply_refs=True, refs=None):
399 if url != 'default' and '://' not in url:
399 if url != 'default' and '://' not in url:
400 client = LocalGitClient(url)
400 client = LocalGitClient(url)
401 else:
401 else:
402 url_obj = url_parser(url)
402 url_obj = url_parser(url)
403 o = self._build_opener(url)
403 o = self._build_opener(url)
404 url, _ = url_obj.authinfo()
404 url, _ = url_obj.authinfo()
405 client = HttpGitClient(base_url=url, opener=o)
405 client = HttpGitClient(base_url=url, opener=o)
406 repo = self._factory.repo(wire)
406 repo = self._factory.repo(wire)
407
407
408 determine_wants = repo.object_store.determine_wants_all
408 determine_wants = repo.object_store.determine_wants_all
409 if refs:
409 if refs:
410 def determine_wants_requested(references):
410 def determine_wants_requested(references):
411 return [references[r] for r in references if r in refs]
411 return [references[r] for r in references if r in refs]
412 determine_wants = determine_wants_requested
412 determine_wants = determine_wants_requested
413
413
414 try:
414 try:
415 remote_refs = client.fetch(
415 remote_refs = client.fetch(
416 path=url, target=repo, determine_wants=determine_wants)
416 path=url, target=repo, determine_wants=determine_wants)
417 except NotGitRepository as e:
417 except NotGitRepository as e:
418 log.warning(
418 log.warning(
419 'Trying to fetch from "%s" failed, not a Git repository.', url)
419 'Trying to fetch from "%s" failed, not a Git repository.', url)
420 # Exception can contain unicode which we convert
420 # Exception can contain unicode which we convert
421 raise exceptions.AbortException(repr(e))
421 raise exceptions.AbortException(repr(e))
422
422
423 # mikhail: client.fetch() returns all the remote refs, but fetches only
423 # mikhail: client.fetch() returns all the remote refs, but fetches only
424 # refs filtered by `determine_wants` function. We need to filter result
424 # refs filtered by `determine_wants` function. We need to filter result
425 # as well
425 # as well
426 if refs:
426 if refs:
427 remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
427 remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
428
428
429 if apply_refs:
429 if apply_refs:
430 # TODO: johbo: Needs proper test coverage with a git repository
430 # TODO: johbo: Needs proper test coverage with a git repository
431 # that contains a tag object, so that we would end up with
431 # that contains a tag object, so that we would end up with
432 # a peeled ref at this point.
432 # a peeled ref at this point.
433 PEELED_REF_MARKER = '^{}'
433 PEELED_REF_MARKER = '^{}'
434 for k in remote_refs:
434 for k in remote_refs:
435 if k.endswith(PEELED_REF_MARKER):
435 if k.endswith(PEELED_REF_MARKER):
436 log.info("Skipping peeled reference %s", k)
436 log.info("Skipping peeled reference %s", k)
437 continue
437 continue
438 repo[k] = remote_refs[k]
438 repo[k] = remote_refs[k]
439
439
440 if refs:
440 if refs:
441 # mikhail: explicitly set the head to the last ref.
441 # mikhail: explicitly set the head to the last ref.
442 repo['HEAD'] = remote_refs[refs[-1]]
442 repo['HEAD'] = remote_refs[refs[-1]]
443
443
444 # TODO: mikhail: should we return remote_refs here to be
444 # TODO: mikhail: should we return remote_refs here to be
445 # consistent?
445 # consistent?
446 else:
446 else:
447 return remote_refs
447 return remote_refs
448
448
449 @reraise_safe_exceptions
449 @reraise_safe_exceptions
450 def sync_push(self, wire, url, refs=None):
450 def sync_push(self, wire, url, refs=None):
451 if self.check_url(url, wire):
451 if self.check_url(url, wire):
452 repo = self._factory.repo(wire)
452 repo = self._factory.repo(wire)
453 self.run_git_command(
453 self.run_git_command(
454 wire, ['push', url, '--mirror'], fail_on_stderr=False,
454 wire, ['push', url, '--mirror'], fail_on_stderr=False,
455 _copts=['-c', 'core.askpass=""'],
455 _copts=['-c', 'core.askpass=""'],
456 extra_env={'GIT_TERMINAL_PROMPT': '0'})
456 extra_env={'GIT_TERMINAL_PROMPT': '0'})
457
457
458 @reraise_safe_exceptions
458 @reraise_safe_exceptions
459 def get_remote_refs(self, wire, url):
459 def get_remote_refs(self, wire, url):
460 repo = Repo(url)
460 repo = Repo(url)
461 return repo.get_refs()
461 return repo.get_refs()
462
462
463 @reraise_safe_exceptions
463 @reraise_safe_exceptions
464 def get_description(self, wire):
464 def get_description(self, wire):
465 repo = self._factory.repo(wire)
465 repo = self._factory.repo(wire)
466 return repo.get_description()
466 return repo.get_description()
467
467
468 @reraise_safe_exceptions
468 @reraise_safe_exceptions
469 def get_file_history(self, wire, file_path, commit_id, limit):
469 def get_file_history(self, wire, file_path, commit_id, limit):
470 repo = self._factory.repo(wire)
470 repo = self._factory.repo(wire)
471 include = [commit_id]
471 include = [commit_id]
472 paths = [file_path]
472 paths = [file_path]
473
473
474 walker = repo.get_walker(include, paths=paths, max_entries=limit)
474 walker = repo.get_walker(include, paths=paths, max_entries=limit)
475 return [x.commit.id for x in walker]
475 return [x.commit.id for x in walker]
476
476
477 @reraise_safe_exceptions
477 @reraise_safe_exceptions
478 def get_missing_revs(self, wire, rev1, rev2, path2):
478 def get_missing_revs(self, wire, rev1, rev2, path2):
479 repo = self._factory.repo(wire)
479 repo = self._factory.repo(wire)
480 LocalGitClient(thin_packs=False).fetch(path2, repo)
480 LocalGitClient(thin_packs=False).fetch(path2, repo)
481
481
482 wire_remote = wire.copy()
482 wire_remote = wire.copy()
483 wire_remote['path'] = path2
483 wire_remote['path'] = path2
484 repo_remote = self._factory.repo(wire_remote)
484 repo_remote = self._factory.repo(wire_remote)
485 LocalGitClient(thin_packs=False).fetch(wire["path"], repo_remote)
485 LocalGitClient(thin_packs=False).fetch(wire["path"], repo_remote)
486
486
487 revs = [
487 revs = [
488 x.commit.id
488 x.commit.id
489 for x in repo_remote.get_walker(include=[rev2], exclude=[rev1])]
489 for x in repo_remote.get_walker(include=[rev2], exclude=[rev1])]
490 return revs
490 return revs
491
491
492 @reraise_safe_exceptions
492 @reraise_safe_exceptions
493 def get_object(self, wire, sha):
493 def get_object(self, wire, sha):
494 repo = self._factory.repo(wire)
494 repo = self._factory.repo(wire)
495 obj = repo.get_object(sha)
495 obj = repo.get_object(sha)
496 commit_id = obj.id
496 commit_id = obj.id
497
497
498 if isinstance(obj, Tag):
498 if isinstance(obj, Tag):
499 commit_id = obj.object[1]
499 commit_id = obj.object[1]
500
500
501 return {
501 return {
502 'id': obj.id,
502 'id': obj.id,
503 'type': obj.type_name,
503 'type': obj.type_name,
504 'commit_id': commit_id
504 'commit_id': commit_id
505 }
505 }
506
506
507 @reraise_safe_exceptions
507 @reraise_safe_exceptions
508 def get_object_attrs(self, wire, sha, *attrs):
508 def get_object_attrs(self, wire, sha, *attrs):
509 repo = self._factory.repo(wire)
509 repo = self._factory.repo(wire)
510 obj = repo.get_object(sha)
510 obj = repo.get_object(sha)
511 return list(getattr(obj, a) for a in attrs)
511 return list(getattr(obj, a) for a in attrs)
512
512
513 @reraise_safe_exceptions
513 @reraise_safe_exceptions
514 def get_refs(self, wire):
514 def get_refs(self, wire):
515 repo = self._factory.repo(wire)
515 repo = self._factory.repo(wire)
516 result = {}
516 result = {}
517 for ref, sha in repo.refs.as_dict().items():
517 for ref, sha in repo.refs.as_dict().items():
518 peeled_sha = repo.get_peeled(ref)
518 peeled_sha = repo.get_peeled(ref)
519 result[ref] = peeled_sha
519 result[ref] = peeled_sha
520 return result
520 return result
521
521
522 @reraise_safe_exceptions
522 @reraise_safe_exceptions
523 def get_refs_path(self, wire):
523 def get_refs_path(self, wire):
524 repo = self._factory.repo(wire)
524 repo = self._factory.repo(wire)
525 return repo.refs.path
525 return repo.refs.path
526
526
527 @reraise_safe_exceptions
527 @reraise_safe_exceptions
528 def head(self, wire):
528 def head(self, wire):
529 repo = self._factory.repo(wire)
529 repo = self._factory.repo(wire)
530 return repo.head()
530 return repo.head()
531
531
532 @reraise_safe_exceptions
532 @reraise_safe_exceptions
533 def init(self, wire):
533 def init(self, wire):
534 repo_path = str_to_dulwich(wire['path'])
534 repo_path = str_to_dulwich(wire['path'])
535 self.repo = Repo.init(repo_path)
535 self.repo = Repo.init(repo_path)
536
536
537 @reraise_safe_exceptions
537 @reraise_safe_exceptions
538 def init_bare(self, wire):
538 def init_bare(self, wire):
539 repo_path = str_to_dulwich(wire['path'])
539 repo_path = str_to_dulwich(wire['path'])
540 self.repo = Repo.init_bare(repo_path)
540 self.repo = Repo.init_bare(repo_path)
541
541
542 @reraise_safe_exceptions
542 @reraise_safe_exceptions
543 def revision(self, wire, rev):
543 def revision(self, wire, rev):
544 repo = self._factory.repo(wire)
544 repo = self._factory.repo(wire)
545 obj = repo[rev]
545 obj = repo[rev]
546 obj_data = {
546 obj_data = {
547 'id': obj.id,
547 'id': obj.id,
548 }
548 }
549 try:
549 try:
550 obj_data['tree'] = obj.tree
550 obj_data['tree'] = obj.tree
551 except AttributeError:
551 except AttributeError:
552 pass
552 pass
553 return obj_data
553 return obj_data
554
554
555 @reraise_safe_exceptions
555 @reraise_safe_exceptions
556 def commit_attribute(self, wire, rev, attr):
556 def commit_attribute(self, wire, rev, attr):
557 repo = self._factory.repo(wire)
557 repo = self._factory.repo(wire)
558 obj = repo[rev]
558 obj = repo[rev]
559 return getattr(obj, attr)
559 return getattr(obj, attr)
560
560
561 @reraise_safe_exceptions
561 @reraise_safe_exceptions
562 def set_refs(self, wire, key, value):
562 def set_refs(self, wire, key, value):
563 repo = self._factory.repo(wire)
563 repo = self._factory.repo(wire)
564 repo.refs[key] = value
564 repo.refs[key] = value
565
565
566 @reraise_safe_exceptions
566 @reraise_safe_exceptions
567 def remove_ref(self, wire, key):
567 def remove_ref(self, wire, key):
568 repo = self._factory.repo(wire)
568 repo = self._factory.repo(wire)
569 del repo.refs[key]
569 del repo.refs[key]
570
570
571 @reraise_safe_exceptions
571 @reraise_safe_exceptions
572 def tree_changes(self, wire, source_id, target_id):
572 def tree_changes(self, wire, source_id, target_id):
573 repo = self._factory.repo(wire)
573 repo = self._factory.repo(wire)
574 source = repo[source_id].tree if source_id else None
574 source = repo[source_id].tree if source_id else None
575 target = repo[target_id].tree
575 target = repo[target_id].tree
576 result = repo.object_store.tree_changes(source, target)
576 result = repo.object_store.tree_changes(source, target)
577 return list(result)
577 return list(result)
578
578
579 @reraise_safe_exceptions
579 @reraise_safe_exceptions
580 def tree_items(self, wire, tree_id):
580 def tree_items(self, wire, tree_id):
581 repo = self._factory.repo(wire)
581 repo = self._factory.repo(wire)
582 tree = repo[tree_id]
582 tree = repo[tree_id]
583
583
584 result = []
584 result = []
585 for item in tree.iteritems():
585 for item in tree.iteritems():
586 item_sha = item.sha
586 item_sha = item.sha
587 item_mode = item.mode
587 item_mode = item.mode
588
588
589 if FILE_MODE(item_mode) == GIT_LINK:
589 if FILE_MODE(item_mode) == GIT_LINK:
590 item_type = "link"
590 item_type = "link"
591 else:
591 else:
592 item_type = repo[item_sha].type_name
592 item_type = repo[item_sha].type_name
593
593
594 result.append((item.path, item_mode, item_sha, item_type))
594 result.append((item.path, item_mode, item_sha, item_type))
595 return result
595 return result
596
596
597 @reraise_safe_exceptions
597 @reraise_safe_exceptions
598 def update_server_info(self, wire):
598 def update_server_info(self, wire):
599 repo = self._factory.repo(wire)
599 repo = self._factory.repo(wire)
600 update_server_info(repo)
600 update_server_info(repo)
601
601
602 @reraise_safe_exceptions
602 @reraise_safe_exceptions
603 def discover_git_version(self):
603 def discover_git_version(self):
604 stdout, _ = self.run_git_command(
604 stdout, _ = self.run_git_command(
605 {}, ['--version'], _bare=True, _safe=True)
605 {}, ['--version'], _bare=True, _safe=True)
606 prefix = 'git version'
606 prefix = 'git version'
607 if stdout.startswith(prefix):
607 if stdout.startswith(prefix):
608 stdout = stdout[len(prefix):]
608 stdout = stdout[len(prefix):]
609 return stdout.strip()
609 return stdout.strip()
610
610
611 @reraise_safe_exceptions
611 @reraise_safe_exceptions
612 def run_git_command(self, wire, cmd, **opts):
612 def run_git_command(self, wire, cmd, **opts):
613 path = wire.get('path', None)
613 path = wire.get('path', None)
614
614
615 if path and os.path.isdir(path):
615 if path and os.path.isdir(path):
616 opts['cwd'] = path
616 opts['cwd'] = path
617
617
618 if '_bare' in opts:
618 if '_bare' in opts:
619 _copts = []
619 _copts = []
620 del opts['_bare']
620 del opts['_bare']
621 else:
621 else:
622 _copts = ['-c', 'core.quotepath=false', ]
622 _copts = ['-c', 'core.quotepath=false', ]
623 safe_call = False
623 safe_call = False
624 if '_safe' in opts:
624 if '_safe' in opts:
625 # no exc on failure
625 # no exc on failure
626 del opts['_safe']
626 del opts['_safe']
627 safe_call = True
627 safe_call = True
628
628
629 if '_copts' in opts:
629 if '_copts' in opts:
630 _copts.extend(opts['_copts'] or [])
630 _copts.extend(opts['_copts'] or [])
631 del opts['_copts']
631 del opts['_copts']
632
632
633 gitenv = os.environ.copy()
633 gitenv = os.environ.copy()
634 gitenv.update(opts.pop('extra_env', {}))
634 gitenv.update(opts.pop('extra_env', {}))
635 # need to clean fix GIT_DIR !
635 # need to clean fix GIT_DIR !
636 if 'GIT_DIR' in gitenv:
636 if 'GIT_DIR' in gitenv:
637 del gitenv['GIT_DIR']
637 del gitenv['GIT_DIR']
638 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
638 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
639 gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1'
639
640
640 cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
641 cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
641
642
642 try:
643 try:
643 _opts = {'env': gitenv, 'shell': False}
644 _opts = {'env': gitenv, 'shell': False}
644 _opts.update(opts)
645 _opts.update(opts)
645 p = subprocessio.SubprocessIOChunker(cmd, **_opts)
646 p = subprocessio.SubprocessIOChunker(cmd, **_opts)
646
647
647 return ''.join(p), ''.join(p.error)
648 return ''.join(p), ''.join(p.error)
648 except (EnvironmentError, OSError) as err:
649 except (EnvironmentError, OSError) as err:
649 cmd = ' '.join(cmd) # human friendly CMD
650 cmd = ' '.join(cmd) # human friendly CMD
650 tb_err = ("Couldn't run git command (%s).\n"
651 tb_err = ("Couldn't run git command (%s).\n"
651 "Original error was:%s\n" % (cmd, err))
652 "Original error was:%s\n" % (cmd, err))
652 log.exception(tb_err)
653 log.exception(tb_err)
653 if safe_call:
654 if safe_call:
654 return '', err
655 return '', err
655 else:
656 else:
656 raise exceptions.VcsException(tb_err)
657 raise exceptions.VcsException(tb_err)
657
658
658 @reraise_safe_exceptions
659 @reraise_safe_exceptions
659 def install_hooks(self, wire, force=False):
660 def install_hooks(self, wire, force=False):
660 from vcsserver.hook_utils import install_git_hooks
661 from vcsserver.hook_utils import install_git_hooks
661 repo = self._factory.repo(wire)
662 repo = self._factory.repo(wire)
662 return install_git_hooks(repo.path, repo.bare, force_create=force)
663 return install_git_hooks(repo.path, repo.bare, force_create=force)
663
664
664
665
665 def str_to_dulwich(value):
666 def str_to_dulwich(value):
666 """
667 """
667 Dulwich 0.10.1a requires `unicode` objects to be passed in.
668 Dulwich 0.10.1a requires `unicode` objects to be passed in.
668 """
669 """
669 return value.decode(settings.WIRE_ENCODING)
670 return value.decode(settings.WIRE_ENCODING)
General Comments 0
You need to be logged in to leave comments. Login now