##// END OF EJS Templates
caches: new cache implementation for remote functions
marcink -
r739:6b84a339 default
parent child Browse files
Show More
@@ -1,891 +1,994 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2019 RhodeCode GmbH
2 # Copyright (C) 2014-2019 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17 import collections
18 import collections
18 import logging
19 import logging
19 import os
20 import os
20 import posixpath as vcspath
21 import posixpath as vcspath
21 import re
22 import re
22 import stat
23 import stat
23 import traceback
24 import traceback
24 import urllib
25 import urllib
25 import urllib2
26 import urllib2
26 from functools import wraps
27 from functools import wraps
27
28
28 import more_itertools
29 import more_itertools
29 import pygit2
30 import pygit2
30 from pygit2 import Repository as LibGit2Repo
31 from pygit2 import Repository as LibGit2Repo
31 from dulwich import index, objects
32 from dulwich import index, objects
32 from dulwich.client import HttpGitClient, LocalGitClient
33 from dulwich.client import HttpGitClient, LocalGitClient
33 from dulwich.errors import (
34 from dulwich.errors import (
34 NotGitRepository, ChecksumMismatch, WrongObjectException,
35 NotGitRepository, ChecksumMismatch, WrongObjectException,
35 MissingCommitError, ObjectMissing, HangupException,
36 MissingCommitError, ObjectMissing, HangupException,
36 UnexpectedCommandError)
37 UnexpectedCommandError)
37 from dulwich.repo import Repo as DulwichRepo
38 from dulwich.repo import Repo as DulwichRepo
38 from dulwich.server import update_server_info
39 from dulwich.server import update_server_info
39
40
40 from vcsserver import exceptions, settings, subprocessio
41 from vcsserver import exceptions, settings, subprocessio
41 from vcsserver.utils import safe_str
42 from vcsserver.utils import safe_str
42 from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original
43 from vcsserver.base import RepoFactory, obfuscate_qs
43 from vcsserver.hgcompat import (
44 from vcsserver.hgcompat import (
44 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
45 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
45 from vcsserver.git_lfs.lib import LFSOidStore
46 from vcsserver.git_lfs.lib import LFSOidStore
46
47
47 DIR_STAT = stat.S_IFDIR
48 DIR_STAT = stat.S_IFDIR
48 FILE_MODE = stat.S_IFMT
49 FILE_MODE = stat.S_IFMT
49 GIT_LINK = objects.S_IFGITLINK
50 GIT_LINK = objects.S_IFGITLINK
51 PEELED_REF_MARKER = '^{}'
52
50
53
51 log = logging.getLogger(__name__)
54 log = logging.getLogger(__name__)
52
55
53
56
57 def str_to_dulwich(value):
58 """
59 Dulwich 0.10.1a requires `unicode` objects to be passed in.
60 """
61 return value.decode(settings.WIRE_ENCODING)
62
63
54 def reraise_safe_exceptions(func):
64 def reraise_safe_exceptions(func):
55 """Converts Dulwich exceptions to something neutral."""
65 """Converts Dulwich exceptions to something neutral."""
56
66
57 @wraps(func)
67 @wraps(func)
58 def wrapper(*args, **kwargs):
68 def wrapper(*args, **kwargs):
59 try:
69 try:
60 return func(*args, **kwargs)
70 return func(*args, **kwargs)
61 except (ChecksumMismatch, WrongObjectException, MissingCommitError, ObjectMissing,) as e:
71 except (ChecksumMismatch, WrongObjectException, MissingCommitError, ObjectMissing,) as e:
62 exc = exceptions.LookupException(org_exc=e)
72 exc = exceptions.LookupException(org_exc=e)
63 raise exc(safe_str(e))
73 raise exc(safe_str(e))
64 except (HangupException, UnexpectedCommandError) as e:
74 except (HangupException, UnexpectedCommandError) as e:
65 exc = exceptions.VcsException(org_exc=e)
75 exc = exceptions.VcsException(org_exc=e)
66 raise exc(safe_str(e))
76 raise exc(safe_str(e))
67 except Exception as e:
77 except Exception as e:
68 # NOTE(marcink): becuase of how dulwich handles some exceptions
78 # NOTE(marcink): becuase of how dulwich handles some exceptions
69 # (KeyError on empty repos), we cannot track this and catch all
79 # (KeyError on empty repos), we cannot track this and catch all
70 # exceptions, it's an exceptions from other handlers
80 # exceptions, it's an exceptions from other handlers
71 #if not hasattr(e, '_vcs_kind'):
81 #if not hasattr(e, '_vcs_kind'):
72 #log.exception("Unhandled exception in git remote call")
82 #log.exception("Unhandled exception in git remote call")
73 #raise_from_original(exceptions.UnhandledException)
83 #raise_from_original(exceptions.UnhandledException)
74 raise
84 raise
75 return wrapper
85 return wrapper
76
86
77
87
78 class Repo(DulwichRepo):
88 class Repo(DulwichRepo):
79 """
89 """
80 A wrapper for dulwich Repo class.
90 A wrapper for dulwich Repo class.
81
91
82 Since dulwich is sometimes keeping .idx file descriptors open, it leads to
92 Since dulwich is sometimes keeping .idx file descriptors open, it leads to
83 "Too many open files" error. We need to close all opened file descriptors
93 "Too many open files" error. We need to close all opened file descriptors
84 once the repo object is destroyed.
94 once the repo object is destroyed.
85 """
95 """
86 def __del__(self):
96 def __del__(self):
87 if hasattr(self, 'object_store'):
97 if hasattr(self, 'object_store'):
88 self.close()
98 self.close()
89
99
90
100
91 class Repository(LibGit2Repo):
101 class Repository(LibGit2Repo):
92
102
93 def __enter__(self):
103 def __enter__(self):
94 return self
104 return self
95
105
96 def __exit__(self, exc_type, exc_val, exc_tb):
106 def __exit__(self, exc_type, exc_val, exc_tb):
97 self.free()
107 self.free()
98
108
99
109
100 class GitFactory(RepoFactory):
110 class GitFactory(RepoFactory):
101 repo_type = 'git'
111 repo_type = 'git'
102
112
103 def _create_repo(self, wire, create, use_libgit2=False):
113 def _create_repo(self, wire, create, use_libgit2=False):
104 if use_libgit2:
114 if use_libgit2:
105 return Repository(wire['path'])
115 return Repository(wire['path'])
106 else:
116 else:
107 repo_path = str_to_dulwich(wire['path'])
117 repo_path = str_to_dulwich(wire['path'])
108 return Repo(repo_path)
118 return Repo(repo_path)
109
119
110 def repo(self, wire, create=False, use_libgit2=False):
120 def repo(self, wire, create=False, use_libgit2=False):
111 """
121 """
112 Get a repository instance for the given path.
122 Get a repository instance for the given path.
113 """
123 """
114 region = self._cache_region
115 context = wire.get('context', None)
116 repo_path = wire.get('path', '')
117 context_uid = '{}'.format(context)
118 cache = wire.get('cache', True)
119 cache_on = context and cache
120
121 @region.conditional_cache_on_arguments(condition=cache_on)
122 def create_new_repo(_repo_type, _repo_path, _context_uid, _use_libgit2):
123 return self._create_repo(wire, create, use_libgit2)
124 return self._create_repo(wire, create, use_libgit2)
124
125
125 repo = create_new_repo(self.repo_type, repo_path, context_uid, use_libgit2)
126 return repo
127
128 def repo_libgit2(self, wire):
126 def repo_libgit2(self, wire):
129 return self.repo(wire, use_libgit2=True)
127 return self.repo(wire, use_libgit2=True)
130
128
131
129
132 class GitRemote(object):
130 class GitRemote(object):
133
131
134 def __init__(self, factory):
132 def __init__(self, factory):
135 self._factory = factory
133 self._factory = factory
136 self.peeled_ref_marker = '^{}'
137 self._bulk_methods = {
134 self._bulk_methods = {
138 "date": self.date,
135 "date": self.date,
139 "author": self.author,
136 "author": self.author,
137 "branch": self.branch,
140 "message": self.message,
138 "message": self.message,
141 "parents": self.parents,
139 "parents": self.parents,
142 "_commit": self.revision,
140 "_commit": self.revision,
143 }
141 }
142 self.region = self._factory._cache_region
144
143
145 def _wire_to_config(self, wire):
144 def _wire_to_config(self, wire):
146 if 'config' in wire:
145 if 'config' in wire:
147 return dict([(x[0] + '_' + x[1], x[2]) for x in wire['config']])
146 return dict([(x[0] + '_' + x[1], x[2]) for x in wire['config']])
148 return {}
147 return {}
149
148
150 def _remote_conf(self, config):
149 def _remote_conf(self, config):
151 params = [
150 params = [
152 '-c', 'core.askpass=""',
151 '-c', 'core.askpass=""',
153 ]
152 ]
154 ssl_cert_dir = config.get('vcs_ssl_dir')
153 ssl_cert_dir = config.get('vcs_ssl_dir')
155 if ssl_cert_dir:
154 if ssl_cert_dir:
156 params.extend(['-c', 'http.sslCAinfo={}'.format(ssl_cert_dir)])
155 params.extend(['-c', 'http.sslCAinfo={}'.format(ssl_cert_dir)])
157 return params
156 return params
158
157
158 def _cache_on(self, wire):
159 context = wire.get('context', '')
160 context_uid = '{}'.format(context)
161 repo_id = wire.get('repo_id', '')
162 cache = wire.get('cache', True)
163 cache_on = context and cache
164 return cache_on, context_uid, repo_id
165
166 @reraise_safe_exceptions
167 def discover_git_version(self):
168 stdout, _ = self.run_git_command(
169 {}, ['--version'], _bare=True, _safe=True)
170 prefix = 'git version'
171 if stdout.startswith(prefix):
172 stdout = stdout[len(prefix):]
173 return stdout.strip()
174
159 @reraise_safe_exceptions
175 @reraise_safe_exceptions
160 def is_empty(self, wire):
176 def is_empty(self, wire):
161 repo_init = self._factory.repo_libgit2(wire)
177 repo_init = self._factory.repo_libgit2(wire)
162 with repo_init as repo:
178 with repo_init as repo:
163
179
164 try:
180 try:
165 has_head = repo.head.name
181 has_head = repo.head.name
166 if has_head:
182 if has_head:
167 return False
183 return False
168
184
169 # NOTE(marcink): check again using more expensive method
185 # NOTE(marcink): check again using more expensive method
170 return repo.is_empty
186 return repo.is_empty
171 except Exception:
187 except Exception:
172 pass
188 pass
173
189
174 return True
190 return True
175
191
176 @reraise_safe_exceptions
192 @reraise_safe_exceptions
177 def add_object(self, wire, content):
193 def add_object(self, wire, content):
178 repo_init = self._factory.repo_libgit2(wire)
194 repo_init = self._factory.repo_libgit2(wire)
179 with repo_init as repo:
195 with repo_init as repo:
180 blob = objects.Blob()
196 blob = objects.Blob()
181 blob.set_raw_string(content)
197 blob.set_raw_string(content)
182 repo.object_store.add_object(blob)
198 repo.object_store.add_object(blob)
183 return blob.id
199 return blob.id
184
200
185 @reraise_safe_exceptions
201 @reraise_safe_exceptions
186 def assert_correct_path(self, wire):
202 def assert_correct_path(self, wire):
203 cache_on, context_uid, repo_id = self._cache_on(wire)
204 @self.region.conditional_cache_on_arguments(condition=cache_on)
205 def _assert_correct_path(_context_uid, _repo_id):
187 try:
206 try:
188 repo_init = self._factory.repo_libgit2(wire)
207 repo_init = self._factory.repo_libgit2(wire)
189 with repo_init as repo:
208 with repo_init as repo:
190 pass
209 pass
191 except pygit2.GitError:
210 except pygit2.GitError:
192 path = wire.get('path')
211 path = wire.get('path')
193 tb = traceback.format_exc()
212 tb = traceback.format_exc()
194 log.debug("Invalid Git path `%s`, tb: %s", path, tb)
213 log.debug("Invalid Git path `%s`, tb: %s", path, tb)
195 return False
214 return False
196
215
197 return True
216 return True
217 return _assert_correct_path(context_uid, repo_id)
198
218
199 @reraise_safe_exceptions
219 @reraise_safe_exceptions
200 def bare(self, wire):
220 def bare(self, wire):
201 repo_init = self._factory.repo_libgit2(wire)
221 repo_init = self._factory.repo_libgit2(wire)
202 with repo_init as repo:
222 with repo_init as repo:
203 return repo.is_bare
223 return repo.is_bare
204
224
205 @reraise_safe_exceptions
225 @reraise_safe_exceptions
206 def blob_as_pretty_string(self, wire, sha):
226 def blob_as_pretty_string(self, wire, sha):
207 repo_init = self._factory.repo_libgit2(wire)
227 repo_init = self._factory.repo_libgit2(wire)
208 with repo_init as repo:
228 with repo_init as repo:
209 blob_obj = repo[sha]
229 blob_obj = repo[sha]
210 blob = blob_obj.data
230 blob = blob_obj.data
211 return blob
231 return blob
212
232
213 @reraise_safe_exceptions
233 @reraise_safe_exceptions
214 def blob_raw_length(self, wire, sha):
234 def blob_raw_length(self, wire, sha):
235 cache_on, context_uid, repo_id = self._cache_on(wire)
236 @self.region.conditional_cache_on_arguments(condition=cache_on)
237 def _blob_raw_length(_context_uid, _repo_id, _sha):
238
215 repo_init = self._factory.repo_libgit2(wire)
239 repo_init = self._factory.repo_libgit2(wire)
216 with repo_init as repo:
240 with repo_init as repo:
217 blob = repo[sha]
241 blob = repo[sha]
218 return blob.size
242 return blob.size
219
243
244 return _blob_raw_length(context_uid, repo_id, sha)
245
220 def _parse_lfs_pointer(self, raw_content):
246 def _parse_lfs_pointer(self, raw_content):
221
247
222 spec_string = 'version https://git-lfs.github.com/spec'
248 spec_string = 'version https://git-lfs.github.com/spec'
223 if raw_content and raw_content.startswith(spec_string):
249 if raw_content and raw_content.startswith(spec_string):
224 pattern = re.compile(r"""
250 pattern = re.compile(r"""
225 (?:\n)?
251 (?:\n)?
226 ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
252 ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
227 ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
253 ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
228 ^size[ ](?P<oid_size>[0-9]+)\n
254 ^size[ ](?P<oid_size>[0-9]+)\n
229 (?:\n)?
255 (?:\n)?
230 """, re.VERBOSE | re.MULTILINE)
256 """, re.VERBOSE | re.MULTILINE)
231 match = pattern.match(raw_content)
257 match = pattern.match(raw_content)
232 if match:
258 if match:
233 return match.groupdict()
259 return match.groupdict()
234
260
235 return {}
261 return {}
236
262
237 @reraise_safe_exceptions
263 @reraise_safe_exceptions
238 def is_large_file(self, wire, sha):
264 def is_large_file(self, wire, sha):
265
266 cache_on, context_uid, repo_id = self._cache_on(wire)
267 @self.region.conditional_cache_on_arguments(condition=cache_on)
268 def _is_large_file(_context_uid, _repo_id, _sha):
239 repo_init = self._factory.repo_libgit2(wire)
269 repo_init = self._factory.repo_libgit2(wire)
240 with repo_init as repo:
270 with repo_init as repo:
241 blob = repo[sha]
271 blob = repo[sha]
242 if blob.is_binary:
272 if blob.is_binary:
243 return {}
273 return {}
244
274
245 return self._parse_lfs_pointer(blob.data)
275 return self._parse_lfs_pointer(blob.data)
246
276
277 return _is_large_file(context_uid, repo_id, sha)
278
247 @reraise_safe_exceptions
279 @reraise_safe_exceptions
248 def in_largefiles_store(self, wire, oid):
280 def in_largefiles_store(self, wire, oid):
249 conf = self._wire_to_config(wire)
281 conf = self._wire_to_config(wire)
250 repo_init = self._factory.repo_libgit2(wire)
282 repo_init = self._factory.repo_libgit2(wire)
251 with repo_init as repo:
283 with repo_init as repo:
252 repo_name = repo.path
284 repo_name = repo.path
253
285
254 store_location = conf.get('vcs_git_lfs_store_location')
286 store_location = conf.get('vcs_git_lfs_store_location')
255 if store_location:
287 if store_location:
256
288
257 store = LFSOidStore(
289 store = LFSOidStore(
258 oid=oid, repo=repo_name, store_location=store_location)
290 oid=oid, repo=repo_name, store_location=store_location)
259 return store.has_oid()
291 return store.has_oid()
260
292
261 return False
293 return False
262
294
263 @reraise_safe_exceptions
295 @reraise_safe_exceptions
264 def store_path(self, wire, oid):
296 def store_path(self, wire, oid):
265 conf = self._wire_to_config(wire)
297 conf = self._wire_to_config(wire)
266 repo_init = self._factory.repo_libgit2(wire)
298 repo_init = self._factory.repo_libgit2(wire)
267 with repo_init as repo:
299 with repo_init as repo:
268 repo_name = repo.path
300 repo_name = repo.path
269
301
270 store_location = conf.get('vcs_git_lfs_store_location')
302 store_location = conf.get('vcs_git_lfs_store_location')
271 if store_location:
303 if store_location:
272 store = LFSOidStore(
304 store = LFSOidStore(
273 oid=oid, repo=repo_name, store_location=store_location)
305 oid=oid, repo=repo_name, store_location=store_location)
274 return store.oid_path
306 return store.oid_path
275 raise ValueError('Unable to fetch oid with path {}'.format(oid))
307 raise ValueError('Unable to fetch oid with path {}'.format(oid))
276
308
277 @reraise_safe_exceptions
309 @reraise_safe_exceptions
278 def bulk_request(self, wire, rev, pre_load):
310 def bulk_request(self, wire, rev, pre_load):
311 cache_on, context_uid, repo_id = self._cache_on(wire)
312 @self.region.conditional_cache_on_arguments(condition=cache_on)
313 def _bulk_request(_context_uid, _repo_id, _rev, _pre_load):
279 result = {}
314 result = {}
280 for attr in pre_load:
315 for attr in pre_load:
281 try:
316 try:
282 method = self._bulk_methods[attr]
317 method = self._bulk_methods[attr]
283 args = [wire, rev]
318 args = [wire, rev]
284 result[attr] = method(*args)
319 result[attr] = method(*args)
285 except KeyError as e:
320 except KeyError as e:
286 raise exceptions.VcsException(e)("Unknown bulk attribute: %s" % attr)
321 raise exceptions.VcsException(e)(
322 "Unknown bulk attribute: %s" % attr)
287 return result
323 return result
288
324
325 return _bulk_request(context_uid, repo_id, rev, sorted(pre_load))
326
289 def _build_opener(self, url):
327 def _build_opener(self, url):
290 handlers = []
328 handlers = []
291 url_obj = url_parser(url)
329 url_obj = url_parser(url)
292 _, authinfo = url_obj.authinfo()
330 _, authinfo = url_obj.authinfo()
293
331
294 if authinfo:
332 if authinfo:
295 # create a password manager
333 # create a password manager
296 passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
334 passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
297 passmgr.add_password(*authinfo)
335 passmgr.add_password(*authinfo)
298
336
299 handlers.extend((httpbasicauthhandler(passmgr),
337 handlers.extend((httpbasicauthhandler(passmgr),
300 httpdigestauthhandler(passmgr)))
338 httpdigestauthhandler(passmgr)))
301
339
302 return urllib2.build_opener(*handlers)
340 return urllib2.build_opener(*handlers)
303
341
304 def _type_id_to_name(self, type_id):
342 def _type_id_to_name(self, type_id):
305 return {
343 return {
306 1: b'commit',
344 1: b'commit',
307 2: b'tree',
345 2: b'tree',
308 3: b'blob',
346 3: b'blob',
309 4: b'tag'
347 4: b'tag'
310 }[type_id]
348 }[type_id]
311
349
312 @reraise_safe_exceptions
350 @reraise_safe_exceptions
313 def check_url(self, url, config):
351 def check_url(self, url, config):
314 url_obj = url_parser(url)
352 url_obj = url_parser(url)
315 test_uri, _ = url_obj.authinfo()
353 test_uri, _ = url_obj.authinfo()
316 url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
354 url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
317 url_obj.query = obfuscate_qs(url_obj.query)
355 url_obj.query = obfuscate_qs(url_obj.query)
318 cleaned_uri = str(url_obj)
356 cleaned_uri = str(url_obj)
319 log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
357 log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
320
358
321 if not test_uri.endswith('info/refs'):
359 if not test_uri.endswith('info/refs'):
322 test_uri = test_uri.rstrip('/') + '/info/refs'
360 test_uri = test_uri.rstrip('/') + '/info/refs'
323
361
324 o = self._build_opener(url)
362 o = self._build_opener(url)
325 o.addheaders = [('User-Agent', 'git/1.7.8.0')] # fake some git
363 o.addheaders = [('User-Agent', 'git/1.7.8.0')] # fake some git
326
364
327 q = {"service": 'git-upload-pack'}
365 q = {"service": 'git-upload-pack'}
328 qs = '?%s' % urllib.urlencode(q)
366 qs = '?%s' % urllib.urlencode(q)
329 cu = "%s%s" % (test_uri, qs)
367 cu = "%s%s" % (test_uri, qs)
330 req = urllib2.Request(cu, None, {})
368 req = urllib2.Request(cu, None, {})
331
369
332 try:
370 try:
333 log.debug("Trying to open URL %s", cleaned_uri)
371 log.debug("Trying to open URL %s", cleaned_uri)
334 resp = o.open(req)
372 resp = o.open(req)
335 if resp.code != 200:
373 if resp.code != 200:
336 raise exceptions.URLError()('Return Code is not 200')
374 raise exceptions.URLError()('Return Code is not 200')
337 except Exception as e:
375 except Exception as e:
338 log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
376 log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
339 # means it cannot be cloned
377 # means it cannot be cloned
340 raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e))
378 raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e))
341
379
342 # now detect if it's proper git repo
380 # now detect if it's proper git repo
343 gitdata = resp.read()
381 gitdata = resp.read()
344 if 'service=git-upload-pack' in gitdata:
382 if 'service=git-upload-pack' in gitdata:
345 pass
383 pass
346 elif re.findall(r'[0-9a-fA-F]{40}\s+refs', gitdata):
384 elif re.findall(r'[0-9a-fA-F]{40}\s+refs', gitdata):
347 # old style git can return some other format !
385 # old style git can return some other format !
348 pass
386 pass
349 else:
387 else:
350 raise exceptions.URLError()(
388 raise exceptions.URLError()(
351 "url [%s] does not look like an git" % (cleaned_uri,))
389 "url [%s] does not look like an git" % (cleaned_uri,))
352
390
353 return True
391 return True
354
392
355 @reraise_safe_exceptions
393 @reraise_safe_exceptions
356 def clone(self, wire, url, deferred, valid_refs, update_after_clone):
394 def clone(self, wire, url, deferred, valid_refs, update_after_clone):
357 # TODO(marcink): deprecate this method. Last i checked we don't use it anymore
395 # TODO(marcink): deprecate this method. Last i checked we don't use it anymore
358 remote_refs = self.pull(wire, url, apply_refs=False)
396 remote_refs = self.pull(wire, url, apply_refs=False)
359 repo = self._factory.repo(wire)
397 repo = self._factory.repo(wire)
360 if isinstance(valid_refs, list):
398 if isinstance(valid_refs, list):
361 valid_refs = tuple(valid_refs)
399 valid_refs = tuple(valid_refs)
362
400
363 for k in remote_refs:
401 for k in remote_refs:
364 # only parse heads/tags and skip so called deferred tags
402 # only parse heads/tags and skip so called deferred tags
365 if k.startswith(valid_refs) and not k.endswith(deferred):
403 if k.startswith(valid_refs) and not k.endswith(deferred):
366 repo[k] = remote_refs[k]
404 repo[k] = remote_refs[k]
367
405
368 if update_after_clone:
406 if update_after_clone:
369 # we want to checkout HEAD
407 # we want to checkout HEAD
370 repo["HEAD"] = remote_refs["HEAD"]
408 repo["HEAD"] = remote_refs["HEAD"]
371 index.build_index_from_tree(repo.path, repo.index_path(),
409 index.build_index_from_tree(repo.path, repo.index_path(),
372 repo.object_store, repo["HEAD"].tree)
410 repo.object_store, repo["HEAD"].tree)
373
411
412 @reraise_safe_exceptions
413 def branch(self, wire, commit_id):
414 cache_on, context_uid, repo_id = self._cache_on(wire)
415 cache_on = False
416 @self.region.conditional_cache_on_arguments(condition=cache_on)
417 def _branch(_context_uid, _repo_id, _commit_id):
418 regex = re.compile('^refs/heads')
419
420 def filter_with(ref):
421 return regex.match(ref[0]) and ref[1] == _commit_id
422
423 branches = filter(filter_with, self.get_refs(wire).items())
424 return [x[0].split('refs/heads/')[-1] for x in branches]
425
426 return _branch(context_uid, repo_id, commit_id)
427
428 @reraise_safe_exceptions
429 def commit_branches(self, wire, commit_id):
430 cache_on, context_uid, repo_id = self._cache_on(wire)
431 @self.region.conditional_cache_on_arguments(condition=cache_on)
432 def _commit_branches(_context_uid, _repo_id, _commit_id):
433 repo_init = self._factory.repo_libgit2(wire)
434 with repo_init as repo:
435 branches = [x for x in repo.branches.with_commit(_commit_id)]
436 return branches
437
438 return _commit_branches(context_uid, repo_id, commit_id)
439
374 # TODO: this is quite complex, check if that can be simplified
440 # TODO: this is quite complex, check if that can be simplified
375 @reraise_safe_exceptions
441 @reraise_safe_exceptions
376 def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
442 def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
377 repo = self._factory.repo(wire)
443 repo = self._factory.repo(wire)
378 object_store = repo.object_store
444 object_store = repo.object_store
379
445
380 # Create tree and populates it with blobs
446 # Create tree and populates it with blobs
381 commit_tree = commit_tree and repo[commit_tree] or objects.Tree()
447 commit_tree = commit_tree and repo[commit_tree] or objects.Tree()
382
448
383 for node in updated:
449 for node in updated:
384 # Compute subdirs if needed
450 # Compute subdirs if needed
385 dirpath, nodename = vcspath.split(node['path'])
451 dirpath, nodename = vcspath.split(node['path'])
386 dirnames = map(safe_str, dirpath and dirpath.split('/') or [])
452 dirnames = map(safe_str, dirpath and dirpath.split('/') or [])
387 parent = commit_tree
453 parent = commit_tree
388 ancestors = [('', parent)]
454 ancestors = [('', parent)]
389
455
390 # Tries to dig for the deepest existing tree
456 # Tries to dig for the deepest existing tree
391 while dirnames:
457 while dirnames:
392 curdir = dirnames.pop(0)
458 curdir = dirnames.pop(0)
393 try:
459 try:
394 dir_id = parent[curdir][1]
460 dir_id = parent[curdir][1]
395 except KeyError:
461 except KeyError:
396 # put curdir back into dirnames and stops
462 # put curdir back into dirnames and stops
397 dirnames.insert(0, curdir)
463 dirnames.insert(0, curdir)
398 break
464 break
399 else:
465 else:
400 # If found, updates parent
466 # If found, updates parent
401 parent = repo[dir_id]
467 parent = repo[dir_id]
402 ancestors.append((curdir, parent))
468 ancestors.append((curdir, parent))
403 # Now parent is deepest existing tree and we need to create
469 # Now parent is deepest existing tree and we need to create
404 # subtrees for dirnames (in reverse order)
470 # subtrees for dirnames (in reverse order)
405 # [this only applies for nodes from added]
471 # [this only applies for nodes from added]
406 new_trees = []
472 new_trees = []
407
473
408 blob = objects.Blob.from_string(node['content'])
474 blob = objects.Blob.from_string(node['content'])
409
475
410 if dirnames:
476 if dirnames:
411 # If there are trees which should be created we need to build
477 # If there are trees which should be created we need to build
412 # them now (in reverse order)
478 # them now (in reverse order)
413 reversed_dirnames = list(reversed(dirnames))
479 reversed_dirnames = list(reversed(dirnames))
414 curtree = objects.Tree()
480 curtree = objects.Tree()
415 curtree[node['node_path']] = node['mode'], blob.id
481 curtree[node['node_path']] = node['mode'], blob.id
416 new_trees.append(curtree)
482 new_trees.append(curtree)
417 for dirname in reversed_dirnames[:-1]:
483 for dirname in reversed_dirnames[:-1]:
418 newtree = objects.Tree()
484 newtree = objects.Tree()
419 newtree[dirname] = (DIR_STAT, curtree.id)
485 newtree[dirname] = (DIR_STAT, curtree.id)
420 new_trees.append(newtree)
486 new_trees.append(newtree)
421 curtree = newtree
487 curtree = newtree
422 parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
488 parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
423 else:
489 else:
424 parent.add(name=node['node_path'], mode=node['mode'], hexsha=blob.id)
490 parent.add(name=node['node_path'], mode=node['mode'], hexsha=blob.id)
425
491
426 new_trees.append(parent)
492 new_trees.append(parent)
427 # Update ancestors
493 # Update ancestors
428 reversed_ancestors = reversed(
494 reversed_ancestors = reversed(
429 [(a[1], b[1], b[0]) for a, b in zip(ancestors, ancestors[1:])])
495 [(a[1], b[1], b[0]) for a, b in zip(ancestors, ancestors[1:])])
430 for parent, tree, path in reversed_ancestors:
496 for parent, tree, path in reversed_ancestors:
431 parent[path] = (DIR_STAT, tree.id)
497 parent[path] = (DIR_STAT, tree.id)
432 object_store.add_object(tree)
498 object_store.add_object(tree)
433
499
434 object_store.add_object(blob)
500 object_store.add_object(blob)
435 for tree in new_trees:
501 for tree in new_trees:
436 object_store.add_object(tree)
502 object_store.add_object(tree)
437
503
438 for node_path in removed:
504 for node_path in removed:
439 paths = node_path.split('/')
505 paths = node_path.split('/')
440 tree = commit_tree
506 tree = commit_tree
441 trees = [tree]
507 trees = [tree]
442 # Traverse deep into the forest...
508 # Traverse deep into the forest...
443 for path in paths:
509 for path in paths:
444 try:
510 try:
445 obj = repo[tree[path][1]]
511 obj = repo[tree[path][1]]
446 if isinstance(obj, objects.Tree):
512 if isinstance(obj, objects.Tree):
447 trees.append(obj)
513 trees.append(obj)
448 tree = obj
514 tree = obj
449 except KeyError:
515 except KeyError:
450 break
516 break
451 # Cut down the blob and all rotten trees on the way back...
517 # Cut down the blob and all rotten trees on the way back...
452 for path, tree in reversed(zip(paths, trees)):
518 for path, tree in reversed(zip(paths, trees)):
453 del tree[path]
519 del tree[path]
454 if tree:
520 if tree:
455 # This tree still has elements - don't remove it or any
521 # This tree still has elements - don't remove it or any
456 # of it's parents
522 # of it's parents
457 break
523 break
458
524
459 object_store.add_object(commit_tree)
525 object_store.add_object(commit_tree)
460
526
461 # Create commit
527 # Create commit
462 commit = objects.Commit()
528 commit = objects.Commit()
463 commit.tree = commit_tree.id
529 commit.tree = commit_tree.id
464 for k, v in commit_data.iteritems():
530 for k, v in commit_data.iteritems():
465 setattr(commit, k, v)
531 setattr(commit, k, v)
466 object_store.add_object(commit)
532 object_store.add_object(commit)
467
533
468 self.create_branch(wire, branch, commit.id)
534 self.create_branch(wire, branch, commit.id)
469
535
470 # dulwich set-ref
536 # dulwich set-ref
471 ref = 'refs/heads/%s' % branch
537 ref = 'refs/heads/%s' % branch
472 repo.refs[ref] = commit.id
538 repo.refs[ref] = commit.id
473
539
474 return commit.id
540 return commit.id
475
541
476 @reraise_safe_exceptions
542 @reraise_safe_exceptions
477 def pull(self, wire, url, apply_refs=True, refs=None, update_after=False):
543 def pull(self, wire, url, apply_refs=True, refs=None, update_after=False):
478 if url != 'default' and '://' not in url:
544 if url != 'default' and '://' not in url:
479 client = LocalGitClient(url)
545 client = LocalGitClient(url)
480 else:
546 else:
481 url_obj = url_parser(url)
547 url_obj = url_parser(url)
482 o = self._build_opener(url)
548 o = self._build_opener(url)
483 url, _ = url_obj.authinfo()
549 url, _ = url_obj.authinfo()
484 client = HttpGitClient(base_url=url, opener=o)
550 client = HttpGitClient(base_url=url, opener=o)
485 repo = self._factory.repo(wire)
551 repo = self._factory.repo(wire)
486
552
487 determine_wants = repo.object_store.determine_wants_all
553 determine_wants = repo.object_store.determine_wants_all
488 if refs:
554 if refs:
489 def determine_wants_requested(references):
555 def determine_wants_requested(references):
490 return [references[r] for r in references if r in refs]
556 return [references[r] for r in references if r in refs]
491 determine_wants = determine_wants_requested
557 determine_wants = determine_wants_requested
492
558
493 try:
559 try:
494 remote_refs = client.fetch(
560 remote_refs = client.fetch(
495 path=url, target=repo, determine_wants=determine_wants)
561 path=url, target=repo, determine_wants=determine_wants)
496 except NotGitRepository as e:
562 except NotGitRepository as e:
497 log.warning(
563 log.warning(
498 'Trying to fetch from "%s" failed, not a Git repository.', url)
564 'Trying to fetch from "%s" failed, not a Git repository.', url)
499 # Exception can contain unicode which we convert
565 # Exception can contain unicode which we convert
500 raise exceptions.AbortException(e)(repr(e))
566 raise exceptions.AbortException(e)(repr(e))
501
567
502 # mikhail: client.fetch() returns all the remote refs, but fetches only
568 # mikhail: client.fetch() returns all the remote refs, but fetches only
503 # refs filtered by `determine_wants` function. We need to filter result
569 # refs filtered by `determine_wants` function. We need to filter result
504 # as well
570 # as well
505 if refs:
571 if refs:
506 remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
572 remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
507
573
508 if apply_refs:
574 if apply_refs:
509 # TODO: johbo: Needs proper test coverage with a git repository
575 # TODO: johbo: Needs proper test coverage with a git repository
510 # that contains a tag object, so that we would end up with
576 # that contains a tag object, so that we would end up with
511 # a peeled ref at this point.
577 # a peeled ref at this point.
512 for k in remote_refs:
578 for k in remote_refs:
513 if k.endswith(self.peeled_ref_marker):
579 if k.endswith(PEELED_REF_MARKER):
514 log.debug("Skipping peeled reference %s", k)
580 log.debug("Skipping peeled reference %s", k)
515 continue
581 continue
516 repo[k] = remote_refs[k]
582 repo[k] = remote_refs[k]
517
583
518 if refs and not update_after:
584 if refs and not update_after:
519 # mikhail: explicitly set the head to the last ref.
585 # mikhail: explicitly set the head to the last ref.
520 repo['HEAD'] = remote_refs[refs[-1]]
586 repo['HEAD'] = remote_refs[refs[-1]]
521
587
522 if update_after:
588 if update_after:
523 # we want to checkout HEAD
589 # we want to checkout HEAD
524 repo["HEAD"] = remote_refs["HEAD"]
590 repo["HEAD"] = remote_refs["HEAD"]
525 index.build_index_from_tree(repo.path, repo.index_path(),
591 index.build_index_from_tree(repo.path, repo.index_path(),
526 repo.object_store, repo["HEAD"].tree)
592 repo.object_store, repo["HEAD"].tree)
527 return remote_refs
593 return remote_refs
528
594
529 @reraise_safe_exceptions
595 @reraise_safe_exceptions
530 def sync_fetch(self, wire, url, refs=None):
596 def sync_fetch(self, wire, url, refs=None):
531 repo = self._factory.repo(wire)
597 repo = self._factory.repo(wire)
532 if refs and not isinstance(refs, (list, tuple)):
598 if refs and not isinstance(refs, (list, tuple)):
533 refs = [refs]
599 refs = [refs]
534 config = self._wire_to_config(wire)
600 config = self._wire_to_config(wire)
535 # get all remote refs we'll use to fetch later
601 # get all remote refs we'll use to fetch later
536 output, __ = self.run_git_command(
602 output, __ = self.run_git_command(
537 wire, ['ls-remote', url], fail_on_stderr=False,
603 wire, ['ls-remote', url], fail_on_stderr=False,
538 _copts=self._remote_conf(config),
604 _copts=self._remote_conf(config),
539 extra_env={'GIT_TERMINAL_PROMPT': '0'})
605 extra_env={'GIT_TERMINAL_PROMPT': '0'})
540
606
541 remote_refs = collections.OrderedDict()
607 remote_refs = collections.OrderedDict()
542 fetch_refs = []
608 fetch_refs = []
543
609
544 for ref_line in output.splitlines():
610 for ref_line in output.splitlines():
545 sha, ref = ref_line.split('\t')
611 sha, ref = ref_line.split('\t')
546 sha = sha.strip()
612 sha = sha.strip()
547 if ref in remote_refs:
613 if ref in remote_refs:
548 # duplicate, skip
614 # duplicate, skip
549 continue
615 continue
550 if ref.endswith(self.peeled_ref_marker):
616 if ref.endswith(PEELED_REF_MARKER):
551 log.debug("Skipping peeled reference %s", ref)
617 log.debug("Skipping peeled reference %s", ref)
552 continue
618 continue
553 # don't sync HEAD
619 # don't sync HEAD
554 if ref in ['HEAD']:
620 if ref in ['HEAD']:
555 continue
621 continue
556
622
557 remote_refs[ref] = sha
623 remote_refs[ref] = sha
558
624
559 if refs and sha in refs:
625 if refs and sha in refs:
560 # we filter fetch using our specified refs
626 # we filter fetch using our specified refs
561 fetch_refs.append('{}:{}'.format(ref, ref))
627 fetch_refs.append('{}:{}'.format(ref, ref))
562 elif not refs:
628 elif not refs:
563 fetch_refs.append('{}:{}'.format(ref, ref))
629 fetch_refs.append('{}:{}'.format(ref, ref))
564 log.debug('Finished obtaining fetch refs, total: %s', len(fetch_refs))
630 log.debug('Finished obtaining fetch refs, total: %s', len(fetch_refs))
565 if fetch_refs:
631 if fetch_refs:
566 for chunk in more_itertools.chunked(fetch_refs, 1024 * 4):
632 for chunk in more_itertools.chunked(fetch_refs, 1024 * 4):
567 fetch_refs_chunks = list(chunk)
633 fetch_refs_chunks = list(chunk)
568 log.debug('Fetching %s refs from import url', len(fetch_refs_chunks))
634 log.debug('Fetching %s refs from import url', len(fetch_refs_chunks))
569 _out, _err = self.run_git_command(
635 _out, _err = self.run_git_command(
570 wire, ['fetch', url, '--force', '--prune', '--'] + fetch_refs_chunks,
636 wire, ['fetch', url, '--force', '--prune', '--'] + fetch_refs_chunks,
571 fail_on_stderr=False,
637 fail_on_stderr=False,
572 _copts=self._remote_conf(config),
638 _copts=self._remote_conf(config),
573 extra_env={'GIT_TERMINAL_PROMPT': '0'})
639 extra_env={'GIT_TERMINAL_PROMPT': '0'})
574
640
575 return remote_refs
641 return remote_refs
576
642
577 @reraise_safe_exceptions
643 @reraise_safe_exceptions
578 def sync_push(self, wire, url, refs=None):
644 def sync_push(self, wire, url, refs=None):
579 if not self.check_url(url, wire):
645 if not self.check_url(url, wire):
580 return
646 return
581 config = self._wire_to_config(wire)
647 config = self._wire_to_config(wire)
582 repo = self._factory.repo(wire)
648 self._factory.repo(wire)
583 self.run_git_command(
649 self.run_git_command(
584 wire, ['push', url, '--mirror'], fail_on_stderr=False,
650 wire, ['push', url, '--mirror'], fail_on_stderr=False,
585 _copts=self._remote_conf(config),
651 _copts=self._remote_conf(config),
586 extra_env={'GIT_TERMINAL_PROMPT': '0'})
652 extra_env={'GIT_TERMINAL_PROMPT': '0'})
587
653
588 @reraise_safe_exceptions
654 @reraise_safe_exceptions
589 def get_remote_refs(self, wire, url):
655 def get_remote_refs(self, wire, url):
590 repo = Repo(url)
656 repo = Repo(url)
591 return repo.get_refs()
657 return repo.get_refs()
592
658
593 @reraise_safe_exceptions
659 @reraise_safe_exceptions
594 def get_description(self, wire):
660 def get_description(self, wire):
595 repo = self._factory.repo(wire)
661 repo = self._factory.repo(wire)
596 return repo.get_description()
662 return repo.get_description()
597
663
598 @reraise_safe_exceptions
664 @reraise_safe_exceptions
599 def get_missing_revs(self, wire, rev1, rev2, path2):
665 def get_missing_revs(self, wire, rev1, rev2, path2):
600 repo = self._factory.repo(wire)
666 repo = self._factory.repo(wire)
601 LocalGitClient(thin_packs=False).fetch(path2, repo)
667 LocalGitClient(thin_packs=False).fetch(path2, repo)
602
668
603 wire_remote = wire.copy()
669 wire_remote = wire.copy()
604 wire_remote['path'] = path2
670 wire_remote['path'] = path2
605 repo_remote = self._factory.repo(wire_remote)
671 repo_remote = self._factory.repo(wire_remote)
606 LocalGitClient(thin_packs=False).fetch(wire["path"], repo_remote)
672 LocalGitClient(thin_packs=False).fetch(wire["path"], repo_remote)
607
673
608 revs = [
674 revs = [
609 x.commit.id
675 x.commit.id
610 for x in repo_remote.get_walker(include=[rev2], exclude=[rev1])]
676 for x in repo_remote.get_walker(include=[rev2], exclude=[rev1])]
611 return revs
677 return revs
612
678
613 @reraise_safe_exceptions
679 @reraise_safe_exceptions
614 def get_object(self, wire, sha):
680 def get_object(self, wire, sha):
681
682 cache_on, context_uid, repo_id = self._cache_on(wire)
683 @self.region.conditional_cache_on_arguments(condition=cache_on)
684 def _get_object(_context_uid, _repo_id, _sha):
615 repo_init = self._factory.repo_libgit2(wire)
685 repo_init = self._factory.repo_libgit2(wire)
616 with repo_init as repo:
686 with repo_init as repo:
617
687
618 missing_commit_err = 'Commit {} does not exist for `{}`'.format(sha, wire['path'])
688 missing_commit_err = 'Commit {} does not exist for `{}`'.format(sha, wire['path'])
619 try:
689 try:
620 commit = repo.revparse_single(sha)
690 commit = repo.revparse_single(sha)
621 except (KeyError, ValueError) as e:
691 except (KeyError, ValueError) as e:
622 raise exceptions.LookupException(e)(missing_commit_err)
692 raise exceptions.LookupException(e)(missing_commit_err)
623
693
624 if isinstance(commit, pygit2.Tag):
694 if isinstance(commit, pygit2.Tag):
625 commit = repo.get(commit.target)
695 commit = repo.get(commit.target)
626
696
627 # check for dangling commit
697 # check for dangling commit
628 branches = [x for x in repo.branches.with_commit(commit.hex)]
698 branches = [x for x in repo.branches.with_commit(commit.hex)]
629 if not branches:
699 if not branches:
630 raise exceptions.LookupException(None)(missing_commit_err)
700 raise exceptions.LookupException(None)(missing_commit_err)
631
701
632 commit_id = commit.hex
702 commit_id = commit.hex
633 type_id = commit.type
703 type_id = commit.type
634
704
635 return {
705 return {
636 'id': commit_id,
706 'id': commit_id,
637 'type': self._type_id_to_name(type_id),
707 'type': self._type_id_to_name(type_id),
638 'commit_id': commit_id,
708 'commit_id': commit_id,
639 'idx': 0
709 'idx': 0
640 }
710 }
641
711
712 return _get_object(context_uid, repo_id, sha)
713
642 @reraise_safe_exceptions
714 @reraise_safe_exceptions
643 def get_refs(self, wire):
715 def get_refs(self, wire):
716 cache_on, context_uid, repo_id = self._cache_on(wire)
717 @self.region.conditional_cache_on_arguments(condition=cache_on)
718 def _get_refs(_context_uid, _repo_id):
719
644 repo_init = self._factory.repo_libgit2(wire)
720 repo_init = self._factory.repo_libgit2(wire)
645 with repo_init as repo:
721 with repo_init as repo:
646 result = {}
722 regex = re.compile('^refs/(heads|tags)/')
647 for ref in repo.references:
723 return {x.name: x.target.hex for x in
648 peeled_sha = repo.lookup_reference(ref).peel()
724 filter(lambda ref: regex.match(ref.name) ,repo.listall_reference_objects())}
649 result[ref] = peeled_sha.hex
725
726 return _get_refs(context_uid, repo_id)
650
727
651 return result
728 @reraise_safe_exceptions
729 def get_branch_pointers(self, wire):
730 cache_on, context_uid, repo_id = self._cache_on(wire)
731 @self.region.conditional_cache_on_arguments(condition=cache_on)
732 def _get_branch_pointers(_context_uid, _repo_id):
733
734 repo_init = self._factory.repo_libgit2(wire)
735 regex = re.compile('^refs/heads')
736 with repo_init as repo:
737 branches = filter(lambda ref: regex.match(ref.name), repo.listall_reference_objects())
738 return {x.target.hex: x.shorthand for x in branches}
739
740 return _get_branch_pointers(context_uid, repo_id)
652
741
653 @reraise_safe_exceptions
742 @reraise_safe_exceptions
654 def head(self, wire, show_exc=True):
743 def head(self, wire, show_exc=True):
744 cache_on, context_uid, repo_id = self._cache_on(wire)
745 @self.region.conditional_cache_on_arguments(condition=cache_on)
746 def _head(_context_uid, _repo_id, _show_exc):
655 repo_init = self._factory.repo_libgit2(wire)
747 repo_init = self._factory.repo_libgit2(wire)
656 with repo_init as repo:
748 with repo_init as repo:
657 try:
749 try:
658 return repo.head.peel().hex
750 return repo.head.peel().hex
659 except Exception:
751 except Exception:
660 if show_exc:
752 if show_exc:
661 raise
753 raise
754 return _head(context_uid, repo_id, show_exc)
662
755
663 @reraise_safe_exceptions
756 @reraise_safe_exceptions
664 def init(self, wire):
757 def init(self, wire):
665 repo_path = str_to_dulwich(wire['path'])
758 repo_path = str_to_dulwich(wire['path'])
666 self.repo = Repo.init(repo_path)
759 self.repo = Repo.init(repo_path)
667
760
668 @reraise_safe_exceptions
761 @reraise_safe_exceptions
669 def init_bare(self, wire):
762 def init_bare(self, wire):
670 repo_path = str_to_dulwich(wire['path'])
763 repo_path = str_to_dulwich(wire['path'])
671 self.repo = Repo.init_bare(repo_path)
764 self.repo = Repo.init_bare(repo_path)
672
765
673 @reraise_safe_exceptions
766 @reraise_safe_exceptions
674 def revision(self, wire, rev):
767 def revision(self, wire, rev):
768
769 cache_on, context_uid, repo_id = self._cache_on(wire)
770 @self.region.conditional_cache_on_arguments(condition=cache_on)
771 def _revision(_context_uid, _repo_id, _rev):
675 repo_init = self._factory.repo_libgit2(wire)
772 repo_init = self._factory.repo_libgit2(wire)
676 with repo_init as repo:
773 with repo_init as repo:
677 commit = repo[rev]
774 commit = repo[rev]
678 obj_data = {
775 obj_data = {
679 'id': commit.id.hex,
776 'id': commit.id.hex,
680 }
777 }
681 # tree objects itself don't have tree_id attribute
778 # tree objects itself don't have tree_id attribute
682 if hasattr(commit, 'tree_id'):
779 if hasattr(commit, 'tree_id'):
683 obj_data['tree'] = commit.tree_id.hex
780 obj_data['tree'] = commit.tree_id.hex
684
781
685 return obj_data
782 return obj_data
783 return _revision(context_uid, repo_id, rev)
686
784
687 @reraise_safe_exceptions
785 @reraise_safe_exceptions
688 def date(self, wire, rev):
786 def date(self, wire, rev):
689 repo_init = self._factory.repo_libgit2(wire)
787 repo_init = self._factory.repo_libgit2(wire)
690 with repo_init as repo:
788 with repo_init as repo:
691 commit = repo[rev]
789 commit = repo[rev]
692 # TODO(marcink): check dulwich difference of offset vs timezone
790 # TODO(marcink): check dulwich difference of offset vs timezone
693 return [commit.commit_time, commit.commit_time_offset]
791 return [commit.commit_time, commit.commit_time_offset]
694
792
695 @reraise_safe_exceptions
793 @reraise_safe_exceptions
696 def author(self, wire, rev):
794 def author(self, wire, rev):
697 repo_init = self._factory.repo_libgit2(wire)
795 repo_init = self._factory.repo_libgit2(wire)
698 with repo_init as repo:
796 with repo_init as repo:
699 commit = repo[rev]
797 commit = repo[rev]
700 if commit.author.email:
798 if commit.author.email:
701 return u"{} <{}>".format(commit.author.name, commit.author.email)
799 return u"{} <{}>".format(commit.author.name, commit.author.email)
702
800
703 return u"{}".format(commit.author.raw_name)
801 return u"{}".format(commit.author.raw_name)
704
802
705 @reraise_safe_exceptions
803 @reraise_safe_exceptions
706 def message(self, wire, rev):
804 def message(self, wire, rev):
707 repo_init = self._factory.repo_libgit2(wire)
805 repo_init = self._factory.repo_libgit2(wire)
708 with repo_init as repo:
806 with repo_init as repo:
709 commit = repo[rev]
807 commit = repo[rev]
710 return commit.message
808 return commit.message
711
809
712 @reraise_safe_exceptions
810 @reraise_safe_exceptions
713 def parents(self, wire, rev):
811 def parents(self, wire, rev):
812 cache_on, context_uid, repo_id = self._cache_on(wire)
813 @self.region.conditional_cache_on_arguments(condition=cache_on)
814 def _parents(_context_uid, _repo_id, _rev):
714 repo_init = self._factory.repo_libgit2(wire)
815 repo_init = self._factory.repo_libgit2(wire)
715 with repo_init as repo:
816 with repo_init as repo:
716 commit = repo[rev]
817 commit = repo[rev]
717 return [x.hex for x in commit.parent_ids]
818 return [x.hex for x in commit.parent_ids]
819 return _parents(context_uid, repo_id, rev)
718
820
719 @reraise_safe_exceptions
821 @reraise_safe_exceptions
720 def set_refs(self, wire, key, value):
822 def set_refs(self, wire, key, value):
721 repo_init = self._factory.repo_libgit2(wire)
823 repo_init = self._factory.repo_libgit2(wire)
722 with repo_init as repo:
824 with repo_init as repo:
723 repo.references.create(key, value, force=True)
825 repo.references.create(key, value, force=True)
724
826
725 @reraise_safe_exceptions
827 @reraise_safe_exceptions
726 def create_branch(self, wire, branch_name, commit_id, force=False):
828 def create_branch(self, wire, branch_name, commit_id, force=False):
727 repo_init = self._factory.repo_libgit2(wire)
829 repo_init = self._factory.repo_libgit2(wire)
728 with repo_init as repo:
830 with repo_init as repo:
729 commit = repo[commit_id]
831 commit = repo[commit_id]
730
832
731 if force:
833 if force:
732 repo.branches.local.create(branch_name, commit, force=force)
834 repo.branches.local.create(branch_name, commit, force=force)
733 elif not repo.branches.get(branch_name):
835 elif not repo.branches.get(branch_name):
734 # create only if that branch isn't existing
836 # create only if that branch isn't existing
735 repo.branches.local.create(branch_name, commit, force=force)
837 repo.branches.local.create(branch_name, commit, force=force)
736
838
737 @reraise_safe_exceptions
839 @reraise_safe_exceptions
738 def remove_ref(self, wire, key):
840 def remove_ref(self, wire, key):
739 repo_init = self._factory.repo_libgit2(wire)
841 repo_init = self._factory.repo_libgit2(wire)
740 with repo_init as repo:
842 with repo_init as repo:
741 repo.references.delete(key)
843 repo.references.delete(key)
742
844
743 @reraise_safe_exceptions
845 @reraise_safe_exceptions
744 def tag_remove(self, wire, tag_name):
846 def tag_remove(self, wire, tag_name):
745 repo_init = self._factory.repo_libgit2(wire)
847 repo_init = self._factory.repo_libgit2(wire)
746 with repo_init as repo:
848 with repo_init as repo:
747 key = 'refs/tags/{}'.format(tag_name)
849 key = 'refs/tags/{}'.format(tag_name)
748 repo.references.delete(key)
850 repo.references.delete(key)
749
851
750 @reraise_safe_exceptions
852 @reraise_safe_exceptions
751 def tree_changes(self, wire, source_id, target_id):
853 def tree_changes(self, wire, source_id, target_id):
752 # TODO(marcink): remove this seems it's only used by tests
854 # TODO(marcink): remove this seems it's only used by tests
753 repo = self._factory.repo(wire)
855 repo = self._factory.repo(wire)
754 source = repo[source_id].tree if source_id else None
856 source = repo[source_id].tree if source_id else None
755 target = repo[target_id].tree
857 target = repo[target_id].tree
756 result = repo.object_store.tree_changes(source, target)
858 result = repo.object_store.tree_changes(source, target)
757 return list(result)
859 return list(result)
758
860
759 @reraise_safe_exceptions
861 @reraise_safe_exceptions
760 def tree_and_type_for_path(self, wire, commit_id, path):
862 def tree_and_type_for_path(self, wire, commit_id, path):
863
864 cache_on, context_uid, repo_id = self._cache_on(wire)
865 @self.region.conditional_cache_on_arguments(condition=cache_on)
866 def _tree_and_type_for_path(_context_uid, _repo_id, _commit_id, _path):
761 repo_init = self._factory.repo_libgit2(wire)
867 repo_init = self._factory.repo_libgit2(wire)
762
868
763 with repo_init as repo:
869 with repo_init as repo:
764 commit = repo[commit_id]
870 commit = repo[commit_id]
765 try:
871 try:
766 tree = commit.tree[path]
872 tree = commit.tree[path]
767 except KeyError:
873 except KeyError:
768 return None, None, None
874 return None, None, None
769
875
770 return tree.id.hex, tree.type, tree.filemode
876 return tree.id.hex, tree.type, tree.filemode
877 return _tree_and_type_for_path(context_uid, repo_id, commit_id, path)
771
878
772 @reraise_safe_exceptions
879 @reraise_safe_exceptions
773 def tree_items(self, wire, tree_id):
880 def tree_items(self, wire, tree_id):
881
882 cache_on, context_uid, repo_id = self._cache_on(wire)
883 @self.region.conditional_cache_on_arguments(condition=cache_on)
884 def _tree_items(_context_uid, _repo_id, _tree_id):
885
774 repo_init = self._factory.repo_libgit2(wire)
886 repo_init = self._factory.repo_libgit2(wire)
775
776 with repo_init as repo:
887 with repo_init as repo:
777 try:
888 try:
778 tree = repo[tree_id]
889 tree = repo[tree_id]
779 except KeyError:
890 except KeyError:
780 raise ObjectMissing('No tree with id: {}'.format(tree_id))
891 raise ObjectMissing('No tree with id: {}'.format(tree_id))
781
892
782 result = []
893 result = []
783 for item in tree:
894 for item in tree:
784 item_sha = item.hex
895 item_sha = item.hex
785 item_mode = item.filemode
896 item_mode = item.filemode
786 item_type = item.type
897 item_type = item.type
787
898
788 if item_type == 'commit':
899 if item_type == 'commit':
789 # NOTE(marcink): submodules we translate to 'link' for backward compat
900 # NOTE(marcink): submodules we translate to 'link' for backward compat
790 item_type = 'link'
901 item_type = 'link'
791
902
792 result.append((item.name, item_mode, item_sha, item_type))
903 result.append((item.name, item_mode, item_sha, item_type))
793 return result
904 return result
905 return _tree_items(context_uid, repo_id, tree_id)
794
906
795 @reraise_safe_exceptions
907 @reraise_safe_exceptions
796 def update_server_info(self, wire):
908 def update_server_info(self, wire):
797 repo = self._factory.repo(wire)
909 repo = self._factory.repo(wire)
798 update_server_info(repo)
910 update_server_info(repo)
799
911
800 @reraise_safe_exceptions
912 @reraise_safe_exceptions
801 def discover_git_version(self):
913 def get_all_commit_ids(self, wire):
802 stdout, _ = self.run_git_command(
803 {}, ['--version'], _bare=True, _safe=True)
804 prefix = 'git version'
805 if stdout.startswith(prefix):
806 stdout = stdout[len(prefix):]
807 return stdout.strip()
808
914
809 @reraise_safe_exceptions
915 cache_on, context_uid, repo_id = self._cache_on(wire)
810 def get_all_commit_ids(self, wire):
916 @self.region.conditional_cache_on_arguments(condition=cache_on)
917 def _get_all_commit_ids(_context_uid, _repo_id):
811
918
812 cmd = ['rev-list', '--reverse', '--date-order', '--branches', '--tags']
919 cmd = ['rev-list', '--reverse', '--date-order', '--branches', '--tags']
813 try:
920 try:
814 output, __ = self.run_git_command(wire, cmd)
921 output, __ = self.run_git_command(wire, cmd)
815 return output.splitlines()
922 return output.splitlines()
816 except Exception:
923 except Exception:
817 # Can be raised for empty repositories
924 # Can be raised for empty repositories
818 return []
925 return []
926 return _get_all_commit_ids(context_uid, repo_id)
819
927
820 @reraise_safe_exceptions
928 @reraise_safe_exceptions
821 def run_git_command(self, wire, cmd, **opts):
929 def run_git_command(self, wire, cmd, **opts):
822 path = wire.get('path', None)
930 path = wire.get('path', None)
823
931
824 if path and os.path.isdir(path):
932 if path and os.path.isdir(path):
825 opts['cwd'] = path
933 opts['cwd'] = path
826
934
827 if '_bare' in opts:
935 if '_bare' in opts:
828 _copts = []
936 _copts = []
829 del opts['_bare']
937 del opts['_bare']
830 else:
938 else:
831 _copts = ['-c', 'core.quotepath=false', ]
939 _copts = ['-c', 'core.quotepath=false', ]
832 safe_call = False
940 safe_call = False
833 if '_safe' in opts:
941 if '_safe' in opts:
834 # no exc on failure
942 # no exc on failure
835 del opts['_safe']
943 del opts['_safe']
836 safe_call = True
944 safe_call = True
837
945
838 if '_copts' in opts:
946 if '_copts' in opts:
839 _copts.extend(opts['_copts'] or [])
947 _copts.extend(opts['_copts'] or [])
840 del opts['_copts']
948 del opts['_copts']
841
949
842 gitenv = os.environ.copy()
950 gitenv = os.environ.copy()
843 gitenv.update(opts.pop('extra_env', {}))
951 gitenv.update(opts.pop('extra_env', {}))
844 # need to clean fix GIT_DIR !
952 # need to clean fix GIT_DIR !
845 if 'GIT_DIR' in gitenv:
953 if 'GIT_DIR' in gitenv:
846 del gitenv['GIT_DIR']
954 del gitenv['GIT_DIR']
847 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
955 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
848 gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1'
956 gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1'
849
957
850 cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
958 cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
851 _opts = {'env': gitenv, 'shell': False}
959 _opts = {'env': gitenv, 'shell': False}
852
960
853 try:
961 try:
854 _opts.update(opts)
962 _opts.update(opts)
855 p = subprocessio.SubprocessIOChunker(cmd, **_opts)
963 p = subprocessio.SubprocessIOChunker(cmd, **_opts)
856
964
857 return ''.join(p), ''.join(p.error)
965 return ''.join(p), ''.join(p.error)
858 except (EnvironmentError, OSError) as err:
966 except (EnvironmentError, OSError) as err:
859 cmd = ' '.join(cmd) # human friendly CMD
967 cmd = ' '.join(cmd) # human friendly CMD
860 tb_err = ("Couldn't run git command (%s).\n"
968 tb_err = ("Couldn't run git command (%s).\n"
861 "Original error was:%s\n"
969 "Original error was:%s\n"
862 "Call options:%s\n"
970 "Call options:%s\n"
863 % (cmd, err, _opts))
971 % (cmd, err, _opts))
864 log.exception(tb_err)
972 log.exception(tb_err)
865 if safe_call:
973 if safe_call:
866 return '', err
974 return '', err
867 else:
975 else:
868 raise exceptions.VcsException()(tb_err)
976 raise exceptions.VcsException()(tb_err)
869
977
870 @reraise_safe_exceptions
978 @reraise_safe_exceptions
871 def install_hooks(self, wire, force=False):
979 def install_hooks(self, wire, force=False):
872 from vcsserver.hook_utils import install_git_hooks
980 from vcsserver.hook_utils import install_git_hooks
873 repo = self._factory.repo(wire)
981 bare = self.bare(wire)
874 return install_git_hooks(repo.path, repo.bare, force_create=force)
982 path = wire['path']
983 return install_git_hooks(path, bare, force_create=force)
875
984
876 @reraise_safe_exceptions
985 @reraise_safe_exceptions
877 def get_hooks_info(self, wire):
986 def get_hooks_info(self, wire):
878 from vcsserver.hook_utils import (
987 from vcsserver.hook_utils import (
879 get_git_pre_hook_version, get_git_post_hook_version)
988 get_git_pre_hook_version, get_git_post_hook_version)
880 repo = self._factory.repo(wire)
989 bare = self.bare(wire)
990 path = wire['path']
881 return {
991 return {
882 'pre_version': get_git_pre_hook_version(repo.path, repo.bare),
992 'pre_version': get_git_pre_hook_version(path, bare),
883 'post_version': get_git_post_hook_version(repo.path, repo.bare),
993 'post_version': get_git_post_hook_version(path, bare),
884 }
994 }
885
886
887 def str_to_dulwich(value):
888 """
889 Dulwich 0.10.1a requires `unicode` objects to be passed in.
890 """
891 return value.decode(settings.WIRE_ENCODING)
@@ -1,874 +1,926 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2019 RhodeCode GmbH
2 # Copyright (C) 2014-2019 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import io
18 import io
19 import logging
19 import logging
20 import stat
20 import stat
21 import urllib
21 import urllib
22 import urllib2
22 import urllib2
23 import traceback
23 import traceback
24
24
25 from hgext import largefiles, rebase
25 from hgext import largefiles, rebase
26 from hgext.strip import strip as hgext_strip
26 from hgext.strip import strip as hgext_strip
27 from mercurial import commands
27 from mercurial import commands
28 from mercurial import unionrepo
28 from mercurial import unionrepo
29 from mercurial import verify
29 from mercurial import verify
30
30
31 import vcsserver
31 import vcsserver
32 from vcsserver import exceptions
32 from vcsserver import exceptions
33 from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original
33 from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original
34 from vcsserver.hgcompat import (
34 from vcsserver.hgcompat import (
35 archival, bin, clone, config as hgconfig, diffopts, hex, get_ctx,
35 archival, bin, clone, config as hgconfig, diffopts, hex, get_ctx,
36 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler,
36 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler,
37 makepeer, instance, match, memctx, exchange, memfilectx, nullrev, hg_merge,
37 makepeer, instance, match, memctx, exchange, memfilectx, nullrev, hg_merge,
38 patch, peer, revrange, ui, hg_tag, Abort, LookupError, RepoError,
38 patch, peer, revrange, ui, hg_tag, Abort, LookupError, RepoError,
39 RepoLookupError, InterventionRequired, RequirementError)
39 RepoLookupError, InterventionRequired, RequirementError)
40
40
41 log = logging.getLogger(__name__)
41 log = logging.getLogger(__name__)
42
42
43
43
44 def make_ui_from_config(repo_config):
44 def make_ui_from_config(repo_config):
45
45
46 class LoggingUI(ui.ui):
46 class LoggingUI(ui.ui):
47 def status(self, *msg, **opts):
47 def status(self, *msg, **opts):
48 log.info(' '.join(msg).rstrip('\n'))
48 log.info(' '.join(msg).rstrip('\n'))
49 super(LoggingUI, self).status(*msg, **opts)
49 super(LoggingUI, self).status(*msg, **opts)
50
50
51 def warn(self, *msg, **opts):
51 def warn(self, *msg, **opts):
52 log.warn(' '.join(msg).rstrip('\n'))
52 log.warn(' '.join(msg).rstrip('\n'))
53 super(LoggingUI, self).warn(*msg, **opts)
53 super(LoggingUI, self).warn(*msg, **opts)
54
54
55 def error(self, *msg, **opts):
55 def error(self, *msg, **opts):
56 log.error(' '.join(msg).rstrip('\n'))
56 log.error(' '.join(msg).rstrip('\n'))
57 super(LoggingUI, self).error(*msg, **opts)
57 super(LoggingUI, self).error(*msg, **opts)
58
58
59 def note(self, *msg, **opts):
59 def note(self, *msg, **opts):
60 log.info(' '.join(msg).rstrip('\n'))
60 log.info(' '.join(msg).rstrip('\n'))
61 super(LoggingUI, self).note(*msg, **opts)
61 super(LoggingUI, self).note(*msg, **opts)
62
62
63 def debug(self, *msg, **opts):
63 def debug(self, *msg, **opts):
64 log.debug(' '.join(msg).rstrip('\n'))
64 log.debug(' '.join(msg).rstrip('\n'))
65 super(LoggingUI, self).debug(*msg, **opts)
65 super(LoggingUI, self).debug(*msg, **opts)
66
66
67 baseui = LoggingUI()
67 baseui = LoggingUI()
68
68
69 # clean the baseui object
69 # clean the baseui object
70 baseui._ocfg = hgconfig.config()
70 baseui._ocfg = hgconfig.config()
71 baseui._ucfg = hgconfig.config()
71 baseui._ucfg = hgconfig.config()
72 baseui._tcfg = hgconfig.config()
72 baseui._tcfg = hgconfig.config()
73
73
74 for section, option, value in repo_config:
74 for section, option, value in repo_config:
75 baseui.setconfig(section, option, value)
75 baseui.setconfig(section, option, value)
76
76
77 # make our hgweb quiet so it doesn't print output
77 # make our hgweb quiet so it doesn't print output
78 baseui.setconfig('ui', 'quiet', 'true')
78 baseui.setconfig('ui', 'quiet', 'true')
79
79
80 baseui.setconfig('ui', 'paginate', 'never')
80 baseui.setconfig('ui', 'paginate', 'never')
81 # for better Error reporting of Mercurial
81 # for better Error reporting of Mercurial
82 baseui.setconfig('ui', 'message-output', 'stderr')
82 baseui.setconfig('ui', 'message-output', 'stderr')
83
83
84 # force mercurial to only use 1 thread, otherwise it may try to set a
84 # force mercurial to only use 1 thread, otherwise it may try to set a
85 # signal in a non-main thread, thus generating a ValueError.
85 # signal in a non-main thread, thus generating a ValueError.
86 baseui.setconfig('worker', 'numcpus', 1)
86 baseui.setconfig('worker', 'numcpus', 1)
87
87
88 # If there is no config for the largefiles extension, we explicitly disable
88 # If there is no config for the largefiles extension, we explicitly disable
89 # it here. This overrides settings from repositories hgrc file. Recent
89 # it here. This overrides settings from repositories hgrc file. Recent
90 # mercurial versions enable largefiles in hgrc on clone from largefile
90 # mercurial versions enable largefiles in hgrc on clone from largefile
91 # repo.
91 # repo.
92 if not baseui.hasconfig('extensions', 'largefiles'):
92 if not baseui.hasconfig('extensions', 'largefiles'):
93 log.debug('Explicitly disable largefiles extension for repo.')
93 log.debug('Explicitly disable largefiles extension for repo.')
94 baseui.setconfig('extensions', 'largefiles', '!')
94 baseui.setconfig('extensions', 'largefiles', '!')
95
95
96 return baseui
96 return baseui
97
97
98
98
99 def reraise_safe_exceptions(func):
99 def reraise_safe_exceptions(func):
100 """Decorator for converting mercurial exceptions to something neutral."""
100 """Decorator for converting mercurial exceptions to something neutral."""
101
101
102 def wrapper(*args, **kwargs):
102 def wrapper(*args, **kwargs):
103 try:
103 try:
104 return func(*args, **kwargs)
104 return func(*args, **kwargs)
105 except (Abort, InterventionRequired) as e:
105 except (Abort, InterventionRequired) as e:
106 raise_from_original(exceptions.AbortException(e))
106 raise_from_original(exceptions.AbortException(e))
107 except RepoLookupError as e:
107 except RepoLookupError as e:
108 raise_from_original(exceptions.LookupException(e))
108 raise_from_original(exceptions.LookupException(e))
109 except RequirementError as e:
109 except RequirementError as e:
110 raise_from_original(exceptions.RequirementException(e))
110 raise_from_original(exceptions.RequirementException(e))
111 except RepoError as e:
111 except RepoError as e:
112 raise_from_original(exceptions.VcsException(e))
112 raise_from_original(exceptions.VcsException(e))
113 except LookupError as e:
113 except LookupError as e:
114 raise_from_original(exceptions.LookupException(e))
114 raise_from_original(exceptions.LookupException(e))
115 except Exception as e:
115 except Exception as e:
116 if not hasattr(e, '_vcs_kind'):
116 if not hasattr(e, '_vcs_kind'):
117 log.exception("Unhandled exception in hg remote call")
117 log.exception("Unhandled exception in hg remote call")
118 raise_from_original(exceptions.UnhandledException(e))
118 raise_from_original(exceptions.UnhandledException(e))
119
119
120 raise
120 raise
121 return wrapper
121 return wrapper
122
122
123
123
124 class MercurialFactory(RepoFactory):
124 class MercurialFactory(RepoFactory):
125 repo_type = 'hg'
125 repo_type = 'hg'
126
126
127 def _create_config(self, config, hooks=True):
127 def _create_config(self, config, hooks=True):
128 if not hooks:
128 if not hooks:
129 hooks_to_clean = frozenset((
129 hooks_to_clean = frozenset((
130 'changegroup.repo_size', 'preoutgoing.pre_pull',
130 'changegroup.repo_size', 'preoutgoing.pre_pull',
131 'outgoing.pull_logger', 'prechangegroup.pre_push'))
131 'outgoing.pull_logger', 'prechangegroup.pre_push'))
132 new_config = []
132 new_config = []
133 for section, option, value in config:
133 for section, option, value in config:
134 if section == 'hooks' and option in hooks_to_clean:
134 if section == 'hooks' and option in hooks_to_clean:
135 continue
135 continue
136 new_config.append((section, option, value))
136 new_config.append((section, option, value))
137 config = new_config
137 config = new_config
138
138
139 baseui = make_ui_from_config(config)
139 baseui = make_ui_from_config(config)
140 return baseui
140 return baseui
141
141
142 def _create_repo(self, wire, create):
142 def _create_repo(self, wire, create):
143 baseui = self._create_config(wire["config"])
143 baseui = self._create_config(wire["config"])
144 return instance(baseui, wire["path"], create)
144 return instance(baseui, wire["path"], create)
145
145
146 def repo(self, wire, create=False):
146 def repo(self, wire, create=False):
147 """
147 """
148 Get a repository instance for the given path.
148 Get a repository instance for the given path.
149 """
149 """
150 region = self._cache_region
151 context = wire.get('context', None)
152 repo_path = wire.get('path', '')
153 context_uid = '{}'.format(context)
154 cache = wire.get('cache', True)
155 cache_on = context and cache
156
157 @region.conditional_cache_on_arguments(condition=cache_on)
158 def create_new_repo(_repo_type, _repo_path, _context_uid):
159 return self._create_repo(wire, create)
150 return self._create_repo(wire, create)
160
151
161 return create_new_repo(self.repo_type, repo_path, context_uid)
162
163
152
164 class HgRemote(object):
153 class HgRemote(object):
165
154
166 def __init__(self, factory):
155 def __init__(self, factory):
167 self._factory = factory
156 self._factory = factory
168
169 self._bulk_methods = {
157 self._bulk_methods = {
170 "affected_files": self.ctx_files,
158 "affected_files": self.ctx_files,
171 "author": self.ctx_user,
159 "author": self.ctx_user,
172 "branch": self.ctx_branch,
160 "branch": self.ctx_branch,
173 "children": self.ctx_children,
161 "children": self.ctx_children,
174 "date": self.ctx_date,
162 "date": self.ctx_date,
175 "message": self.ctx_description,
163 "message": self.ctx_description,
176 "parents": self.ctx_parents,
164 "parents": self.ctx_parents,
177 "status": self.ctx_status,
165 "status": self.ctx_status,
178 "obsolete": self.ctx_obsolete,
166 "obsolete": self.ctx_obsolete,
179 "phase": self.ctx_phase,
167 "phase": self.ctx_phase,
180 "hidden": self.ctx_hidden,
168 "hidden": self.ctx_hidden,
181 "_file_paths": self.ctx_list,
169 "_file_paths": self.ctx_list,
182 }
170 }
171 self.region = self._factory._cache_region
183
172
184 def _get_ctx(self, repo, ref):
173 def _get_ctx(self, repo, ref):
185 return get_ctx(repo, ref)
174 return get_ctx(repo, ref)
186
175
176 def _cache_on(self, wire):
177 context = wire.get('context', '')
178 context_uid = '{}'.format(context)
179 repo_id = wire.get('repo_id', '')
180 cache = wire.get('cache', True)
181 cache_on = context and cache
182 return cache_on, context_uid, repo_id
183
187 @reraise_safe_exceptions
184 @reraise_safe_exceptions
188 def discover_hg_version(self):
185 def discover_hg_version(self):
189 from mercurial import util
186 from mercurial import util
190 return util.version()
187 return util.version()
191
188
192 @reraise_safe_exceptions
189 @reraise_safe_exceptions
193 def is_empty(self, wire):
190 def is_empty(self, wire):
194 repo = self._factory.repo(wire)
191 repo = self._factory.repo(wire)
195
192
196 try:
193 try:
197 return len(repo) == 0
194 return len(repo) == 0
198 except Exception:
195 except Exception:
199 log.exception("failed to read object_store")
196 log.exception("failed to read object_store")
200 return False
197 return False
201
198
202 @reraise_safe_exceptions
199 @reraise_safe_exceptions
203 def archive_repo(self, archive_path, mtime, file_info, kind):
200 def archive_repo(self, archive_path, mtime, file_info, kind):
204 if kind == "tgz":
201 if kind == "tgz":
205 archiver = archival.tarit(archive_path, mtime, "gz")
202 archiver = archival.tarit(archive_path, mtime, "gz")
206 elif kind == "tbz2":
203 elif kind == "tbz2":
207 archiver = archival.tarit(archive_path, mtime, "bz2")
204 archiver = archival.tarit(archive_path, mtime, "bz2")
208 elif kind == 'zip':
205 elif kind == 'zip':
209 archiver = archival.zipit(archive_path, mtime)
206 archiver = archival.zipit(archive_path, mtime)
210 else:
207 else:
211 raise exceptions.ArchiveException()(
208 raise exceptions.ArchiveException()(
212 'Remote does not support: "%s".' % kind)
209 'Remote does not support: "%s".' % kind)
213
210
214 for f_path, f_mode, f_is_link, f_content in file_info:
211 for f_path, f_mode, f_is_link, f_content in file_info:
215 archiver.addfile(f_path, f_mode, f_is_link, f_content)
212 archiver.addfile(f_path, f_mode, f_is_link, f_content)
216 archiver.done()
213 archiver.done()
217
214
218 @reraise_safe_exceptions
215 @reraise_safe_exceptions
219 def bookmarks(self, wire):
216 def bookmarks(self, wire):
217 cache_on, context_uid, repo_id = self._cache_on(wire)
218 @self.region.conditional_cache_on_arguments(condition=cache_on)
219 def _bookmarks(_context_uid, _repo_id):
220 repo = self._factory.repo(wire)
220 repo = self._factory.repo(wire)
221 return dict(repo._bookmarks)
221 return dict(repo._bookmarks)
222
222
223 return _bookmarks(context_uid, repo_id)
224
223 @reraise_safe_exceptions
225 @reraise_safe_exceptions
224 def branches(self, wire, normal, closed):
226 def branches(self, wire, normal, closed):
227 cache_on, context_uid, repo_id = self._cache_on(wire)
228 @self.region.conditional_cache_on_arguments(condition=cache_on)
229 def _branches(_context_uid, _repo_id, _normal, _closed):
225 repo = self._factory.repo(wire)
230 repo = self._factory.repo(wire)
226 iter_branches = repo.branchmap().iterbranches()
231 iter_branches = repo.branchmap().iterbranches()
227 bt = {}
232 bt = {}
228 for branch_name, _heads, tip, is_closed in iter_branches:
233 for branch_name, _heads, tip, is_closed in iter_branches:
229 if normal and not is_closed:
234 if normal and not is_closed:
230 bt[branch_name] = tip
235 bt[branch_name] = tip
231 if closed and is_closed:
236 if closed and is_closed:
232 bt[branch_name] = tip
237 bt[branch_name] = tip
233
238
234 return bt
239 return bt
235
240
241 return _branches(context_uid, repo_id, normal, closed)
242
236 @reraise_safe_exceptions
243 @reraise_safe_exceptions
237 def bulk_request(self, wire, rev, pre_load):
244 def bulk_request(self, wire, rev, pre_load):
245 cache_on, context_uid, repo_id = self._cache_on(wire)
246 @self.region.conditional_cache_on_arguments(condition=cache_on)
247 def _bulk_request(_context_uid, _repo_id, _rev, _pre_load):
238 result = {}
248 result = {}
239 for attr in pre_load:
249 for attr in pre_load:
240 try:
250 try:
241 method = self._bulk_methods[attr]
251 method = self._bulk_methods[attr]
242 result[attr] = method(wire, rev)
252 result[attr] = method(wire, rev)
243 except KeyError as e:
253 except KeyError as e:
244 raise exceptions.VcsException(e)(
254 raise exceptions.VcsException(e)(
245 'Unknown bulk attribute: "%s"' % attr)
255 'Unknown bulk attribute: "%s"' % attr)
246 return result
256 return result
247
257
258 return _bulk_request(context_uid, repo_id, rev, sorted(pre_load))
259
248 @reraise_safe_exceptions
260 @reraise_safe_exceptions
249 def clone(self, wire, source, dest, update_after_clone=False, hooks=True):
261 def clone(self, wire, source, dest, update_after_clone=False, hooks=True):
250 baseui = self._factory._create_config(wire["config"], hooks=hooks)
262 baseui = self._factory._create_config(wire["config"], hooks=hooks)
251 clone(baseui, source, dest, noupdate=not update_after_clone)
263 clone(baseui, source, dest, noupdate=not update_after_clone)
252
264
253 @reraise_safe_exceptions
265 @reraise_safe_exceptions
254 def commitctx(
266 def commitctx(self, wire, message, parents, commit_time, commit_timezone,
255 self, wire, message, parents, commit_time, commit_timezone,
256 user, files, extra, removed, updated):
267 user, files, extra, removed, updated):
257
268
258 repo = self._factory.repo(wire)
269 repo = self._factory.repo(wire)
259 baseui = self._factory._create_config(wire['config'])
270 baseui = self._factory._create_config(wire['config'])
260 publishing = baseui.configbool('phases', 'publish')
271 publishing = baseui.configbool('phases', 'publish')
261 if publishing:
272 if publishing:
262 new_commit = 'public'
273 new_commit = 'public'
263 else:
274 else:
264 new_commit = 'draft'
275 new_commit = 'draft'
265
276
266 def _filectxfn(_repo, ctx, path):
277 def _filectxfn(_repo, ctx, path):
267 """
278 """
268 Marks given path as added/changed/removed in a given _repo. This is
279 Marks given path as added/changed/removed in a given _repo. This is
269 for internal mercurial commit function.
280 for internal mercurial commit function.
270 """
281 """
271
282
272 # check if this path is removed
283 # check if this path is removed
273 if path in removed:
284 if path in removed:
274 # returning None is a way to mark node for removal
285 # returning None is a way to mark node for removal
275 return None
286 return None
276
287
277 # check if this path is added
288 # check if this path is added
278 for node in updated:
289 for node in updated:
279 if node['path'] == path:
290 if node['path'] == path:
280 return memfilectx(
291 return memfilectx(
281 _repo,
292 _repo,
282 changectx=ctx,
293 changectx=ctx,
283 path=node['path'],
294 path=node['path'],
284 data=node['content'],
295 data=node['content'],
285 islink=False,
296 islink=False,
286 isexec=bool(node['mode'] & stat.S_IXUSR),
297 isexec=bool(node['mode'] & stat.S_IXUSR),
287 copied=False)
298 copysource=False)
288
299
289 raise exceptions.AbortException()(
300 raise exceptions.AbortException()(
290 "Given path haven't been marked as added, "
301 "Given path haven't been marked as added, "
291 "changed or removed (%s)" % path)
302 "changed or removed (%s)" % path)
292
303
293 with repo.ui.configoverride({('phases', 'new-commit'): new_commit}):
304 with repo.ui.configoverride({('phases', 'new-commit'): new_commit}):
294
305
295 commit_ctx = memctx(
306 commit_ctx = memctx(
296 repo=repo,
307 repo=repo,
297 parents=parents,
308 parents=parents,
298 text=message,
309 text=message,
299 files=files,
310 files=files,
300 filectxfn=_filectxfn,
311 filectxfn=_filectxfn,
301 user=user,
312 user=user,
302 date=(commit_time, commit_timezone),
313 date=(commit_time, commit_timezone),
303 extra=extra)
314 extra=extra)
304
315
305 n = repo.commitctx(commit_ctx)
316 n = repo.commitctx(commit_ctx)
306 new_id = hex(n)
317 new_id = hex(n)
307
318
308 return new_id
319 return new_id
309
320
310 @reraise_safe_exceptions
321 @reraise_safe_exceptions
311 def ctx_branch(self, wire, revision):
322 def ctx_branch(self, wire, revision):
323
324 cache_on, context_uid, repo_id = self._cache_on(wire)
325 @self.region.conditional_cache_on_arguments(condition=cache_on)
326 def _ctx_branch(_context_uid, _repo_id, _revision):
312 repo = self._factory.repo(wire)
327 repo = self._factory.repo(wire)
313 ctx = self._get_ctx(repo, revision)
328 ctx = self._get_ctx(repo, revision)
314 return ctx.branch()
329 return ctx.branch()
315
330 return _ctx_branch(context_uid, repo_id, revision)
316 @reraise_safe_exceptions
317 def ctx_children(self, wire, revision):
318 repo = self._factory.repo(wire)
319 ctx = self._get_ctx(repo, revision)
320 return [child.rev() for child in ctx.children()]
321
331
322 @reraise_safe_exceptions
332 @reraise_safe_exceptions
323 def ctx_date(self, wire, revision):
333 def ctx_date(self, wire, revision):
324 repo = self._factory.repo(wire)
334 repo = self._factory.repo(wire)
325 ctx = self._get_ctx(repo, revision)
335 ctx = self._get_ctx(repo, revision)
326 return ctx.date()
336 return ctx.date()
327
337
328 @reraise_safe_exceptions
338 @reraise_safe_exceptions
329 def ctx_description(self, wire, revision):
339 def ctx_description(self, wire, revision):
330 repo = self._factory.repo(wire)
340 repo = self._factory.repo(wire)
331 ctx = self._get_ctx(repo, revision)
341 ctx = self._get_ctx(repo, revision)
332 return ctx.description()
342 return ctx.description()
333
343
334 @reraise_safe_exceptions
344 @reraise_safe_exceptions
335 def ctx_files(self, wire, revision):
345 def ctx_files(self, wire, revision):
346
347 cache_on, context_uid, repo_id = self._cache_on(wire)
348 @self.region.conditional_cache_on_arguments(condition=cache_on)
349 def _ctx_files(_context_uid, _repo_id, _revision):
336 repo = self._factory.repo(wire)
350 repo = self._factory.repo(wire)
337 ctx = self._get_ctx(repo, revision)
351 ctx = self._get_ctx(repo, revision)
338 return ctx.files()
352 return ctx.files()
339
353
354 return _ctx_files(context_uid, repo_id, revision)
355
340 @reraise_safe_exceptions
356 @reraise_safe_exceptions
341 def ctx_list(self, path, revision):
357 def ctx_list(self, path, revision):
342 repo = self._factory.repo(path)
358 repo = self._factory.repo(path)
343 ctx = self._get_ctx(repo, revision)
359 ctx = self._get_ctx(repo, revision)
344 return list(ctx)
360 return list(ctx)
345
361
346 @reraise_safe_exceptions
362 @reraise_safe_exceptions
347 def ctx_parents(self, wire, revision):
363 def ctx_parents(self, wire, revision):
364 cache_on, context_uid, repo_id = self._cache_on(wire)
365 @self.region.conditional_cache_on_arguments(condition=cache_on)
366 def _ctx_parents(_context_uid, _repo_id, _revision):
348 repo = self._factory.repo(wire)
367 repo = self._factory.repo(wire)
349 ctx = self._get_ctx(repo, revision)
368 ctx = self._get_ctx(repo, revision)
350 return [parent.rev() for parent in ctx.parents()]
369 return [parent.rev() for parent in ctx.parents()
370 if not (parent.hidden() or parent.obsolete())]
371
372 return _ctx_parents(context_uid, repo_id, revision)
373
374 @reraise_safe_exceptions
375 def ctx_children(self, wire, revision):
376 cache_on, context_uid, repo_id = self._cache_on(wire)
377 @self.region.conditional_cache_on_arguments(condition=cache_on)
378 def _ctx_children(_context_uid, _repo_id, _revision):
379 repo = self._factory.repo(wire)
380 ctx = self._get_ctx(repo, revision)
381 return [child.rev() for child in ctx.children()
382 if not (child.hidden() or child.obsolete())]
383
384 return _ctx_children(context_uid, repo_id, revision)
351
385
352 @reraise_safe_exceptions
386 @reraise_safe_exceptions
353 def ctx_phase(self, wire, revision):
387 def ctx_phase(self, wire, revision):
354 repo = self._factory.repo(wire)
388 repo = self._factory.repo(wire)
355 ctx = self._get_ctx(repo, revision)
389 ctx = self._get_ctx(repo, revision)
356 # public=0, draft=1, secret=3
390 # public=0, draft=1, secret=3
357 return ctx.phase()
391 return ctx.phase()
358
392
359 @reraise_safe_exceptions
393 @reraise_safe_exceptions
360 def ctx_obsolete(self, wire, revision):
394 def ctx_obsolete(self, wire, revision):
361 repo = self._factory.repo(wire)
395 repo = self._factory.repo(wire)
362 ctx = self._get_ctx(repo, revision)
396 ctx = self._get_ctx(repo, revision)
363 return ctx.obsolete()
397 return ctx.obsolete()
364
398
365 @reraise_safe_exceptions
399 @reraise_safe_exceptions
366 def ctx_hidden(self, wire, revision):
400 def ctx_hidden(self, wire, revision):
367 repo = self._factory.repo(wire)
401 repo = self._factory.repo(wire)
368 ctx = self._get_ctx(repo, revision)
402 ctx = self._get_ctx(repo, revision)
369 return ctx.hidden()
403 return ctx.hidden()
370
404
371 @reraise_safe_exceptions
405 @reraise_safe_exceptions
372 def ctx_substate(self, wire, revision):
406 def ctx_substate(self, wire, revision):
373 repo = self._factory.repo(wire)
407 repo = self._factory.repo(wire)
374 ctx = self._get_ctx(repo, revision)
408 ctx = self._get_ctx(repo, revision)
375 return ctx.substate
409 return ctx.substate
376
410
377 @reraise_safe_exceptions
411 @reraise_safe_exceptions
378 def ctx_status(self, wire, revision):
412 def ctx_status(self, wire, revision):
379 repo = self._factory.repo(wire)
413 repo = self._factory.repo(wire)
380 ctx = self._get_ctx(repo, revision)
414 ctx = self._get_ctx(repo, revision)
381 status = repo[ctx.p1().node()].status(other=ctx.node())
415 status = repo[ctx.p1().node()].status(other=ctx.node())
382 # object of status (odd, custom named tuple in mercurial) is not
416 # object of status (odd, custom named tuple in mercurial) is not
383 # correctly serializable, we make it a list, as the underling
417 # correctly serializable, we make it a list, as the underling
384 # API expects this to be a list
418 # API expects this to be a list
385 return list(status)
419 return list(status)
386
420
387 @reraise_safe_exceptions
421 @reraise_safe_exceptions
388 def ctx_user(self, wire, revision):
422 def ctx_user(self, wire, revision):
389 repo = self._factory.repo(wire)
423 repo = self._factory.repo(wire)
390 ctx = self._get_ctx(repo, revision)
424 ctx = self._get_ctx(repo, revision)
391 return ctx.user()
425 return ctx.user()
392
426
393 @reraise_safe_exceptions
427 @reraise_safe_exceptions
394 def check_url(self, url, config):
428 def check_url(self, url, config):
395 _proto = None
429 _proto = None
396 if '+' in url[:url.find('://')]:
430 if '+' in url[:url.find('://')]:
397 _proto = url[0:url.find('+')]
431 _proto = url[0:url.find('+')]
398 url = url[url.find('+') + 1:]
432 url = url[url.find('+') + 1:]
399 handlers = []
433 handlers = []
400 url_obj = url_parser(url)
434 url_obj = url_parser(url)
401 test_uri, authinfo = url_obj.authinfo()
435 test_uri, authinfo = url_obj.authinfo()
402 url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
436 url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
403 url_obj.query = obfuscate_qs(url_obj.query)
437 url_obj.query = obfuscate_qs(url_obj.query)
404
438
405 cleaned_uri = str(url_obj)
439 cleaned_uri = str(url_obj)
406 log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
440 log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
407
441
408 if authinfo:
442 if authinfo:
409 # create a password manager
443 # create a password manager
410 passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
444 passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
411 passmgr.add_password(*authinfo)
445 passmgr.add_password(*authinfo)
412
446
413 handlers.extend((httpbasicauthhandler(passmgr),
447 handlers.extend((httpbasicauthhandler(passmgr),
414 httpdigestauthhandler(passmgr)))
448 httpdigestauthhandler(passmgr)))
415
449
416 o = urllib2.build_opener(*handlers)
450 o = urllib2.build_opener(*handlers)
417 o.addheaders = [('Content-Type', 'application/mercurial-0.1'),
451 o.addheaders = [('Content-Type', 'application/mercurial-0.1'),
418 ('Accept', 'application/mercurial-0.1')]
452 ('Accept', 'application/mercurial-0.1')]
419
453
420 q = {"cmd": 'between'}
454 q = {"cmd": 'between'}
421 q.update({'pairs': "%s-%s" % ('0' * 40, '0' * 40)})
455 q.update({'pairs': "%s-%s" % ('0' * 40, '0' * 40)})
422 qs = '?%s' % urllib.urlencode(q)
456 qs = '?%s' % urllib.urlencode(q)
423 cu = "%s%s" % (test_uri, qs)
457 cu = "%s%s" % (test_uri, qs)
424 req = urllib2.Request(cu, None, {})
458 req = urllib2.Request(cu, None, {})
425
459
426 try:
460 try:
427 log.debug("Trying to open URL %s", cleaned_uri)
461 log.debug("Trying to open URL %s", cleaned_uri)
428 resp = o.open(req)
462 resp = o.open(req)
429 if resp.code != 200:
463 if resp.code != 200:
430 raise exceptions.URLError()('Return Code is not 200')
464 raise exceptions.URLError()('Return Code is not 200')
431 except Exception as e:
465 except Exception as e:
432 log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
466 log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
433 # means it cannot be cloned
467 # means it cannot be cloned
434 raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e))
468 raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e))
435
469
436 # now check if it's a proper hg repo, but don't do it for svn
470 # now check if it's a proper hg repo, but don't do it for svn
437 try:
471 try:
438 if _proto == 'svn':
472 if _proto == 'svn':
439 pass
473 pass
440 else:
474 else:
441 # check for pure hg repos
475 # check for pure hg repos
442 log.debug(
476 log.debug(
443 "Verifying if URL is a Mercurial repository: %s",
477 "Verifying if URL is a Mercurial repository: %s",
444 cleaned_uri)
478 cleaned_uri)
445 ui = make_ui_from_config(config)
479 ui = make_ui_from_config(config)
446 peer_checker = makepeer(ui, url)
480 peer_checker = makepeer(ui, url)
447 peer_checker.lookup('tip')
481 peer_checker.lookup('tip')
448 except Exception as e:
482 except Exception as e:
449 log.warning("URL is not a valid Mercurial repository: %s",
483 log.warning("URL is not a valid Mercurial repository: %s",
450 cleaned_uri)
484 cleaned_uri)
451 raise exceptions.URLError(e)(
485 raise exceptions.URLError(e)(
452 "url [%s] does not look like an hg repo org_exc: %s"
486 "url [%s] does not look like an hg repo org_exc: %s"
453 % (cleaned_uri, e))
487 % (cleaned_uri, e))
454
488
455 log.info("URL is a valid Mercurial repository: %s", cleaned_uri)
489 log.info("URL is a valid Mercurial repository: %s", cleaned_uri)
456 return True
490 return True
457
491
458 @reraise_safe_exceptions
492 @reraise_safe_exceptions
459 def diff(
493 def diff(self, wire, rev1, rev2, file_filter, opt_git, opt_ignorews, context):
460 self, wire, rev1, rev2, file_filter, opt_git, opt_ignorews,
461 context):
462 repo = self._factory.repo(wire)
494 repo = self._factory.repo(wire)
463
495
464 if file_filter:
496 if file_filter:
465 match_filter = match(file_filter[0], '', [file_filter[1]])
497 match_filter = match(file_filter[0], '', [file_filter[1]])
466 else:
498 else:
467 match_filter = file_filter
499 match_filter = file_filter
468 opts = diffopts(git=opt_git, ignorews=opt_ignorews, context=context)
500 opts = diffopts(git=opt_git, ignorews=opt_ignorews, context=context)
469
501
470 try:
502 try:
471 return "".join(patch.diff(
503 return "".join(patch.diff(
472 repo, node1=rev1, node2=rev2, match=match_filter, opts=opts))
504 repo, node1=rev1, node2=rev2, match=match_filter, opts=opts))
473 except RepoLookupError as e:
505 except RepoLookupError as e:
474 raise exceptions.LookupException(e)()
506 raise exceptions.LookupException(e)()
475
507
476 @reraise_safe_exceptions
508 @reraise_safe_exceptions
477 def node_history(self, wire, revision, path, limit):
509 def node_history(self, wire, revision, path, limit):
478 repo = self._factory.repo(wire)
510 repo = self._factory.repo(wire)
479
511
480 ctx = self._get_ctx(repo, revision)
512 ctx = self._get_ctx(repo, revision)
481 fctx = ctx.filectx(path)
513 fctx = ctx.filectx(path)
482
514
483 def history_iter():
515 def history_iter():
484 limit_rev = fctx.rev()
516 limit_rev = fctx.rev()
485 for obj in reversed(list(fctx.filelog())):
517 for obj in reversed(list(fctx.filelog())):
486 obj = fctx.filectx(obj)
518 obj = fctx.filectx(obj)
487 ctx = obj.changectx()
519 ctx = obj.changectx()
488 if ctx.hidden() or ctx.obsolete():
520 if ctx.hidden() or ctx.obsolete():
489 continue
521 continue
490
522
491 if limit_rev >= obj.rev():
523 if limit_rev >= obj.rev():
492 yield obj
524 yield obj
493
525
494 history = []
526 history = []
495 for cnt, obj in enumerate(history_iter()):
527 for cnt, obj in enumerate(history_iter()):
496 if limit and cnt >= limit:
528 if limit and cnt >= limit:
497 break
529 break
498 history.append(hex(obj.node()))
530 history.append(hex(obj.node()))
499
531
500 return [x for x in history]
532 return [x for x in history]
501
533
502 @reraise_safe_exceptions
534 @reraise_safe_exceptions
503 def node_history_untill(self, wire, revision, path, limit):
535 def node_history_untill(self, wire, revision, path, limit):
504 repo = self._factory.repo(wire)
536 repo = self._factory.repo(wire)
505 ctx = self._get_ctx(repo, revision)
537 ctx = self._get_ctx(repo, revision)
506 fctx = ctx.filectx(path)
538 fctx = ctx.filectx(path)
507
539
508 file_log = list(fctx.filelog())
540 file_log = list(fctx.filelog())
509 if limit:
541 if limit:
510 # Limit to the last n items
542 # Limit to the last n items
511 file_log = file_log[-limit:]
543 file_log = file_log[-limit:]
512
544
513 return [hex(fctx.filectx(cs).node()) for cs in reversed(file_log)]
545 return [hex(fctx.filectx(cs).node()) for cs in reversed(file_log)]
514
546
515 @reraise_safe_exceptions
547 @reraise_safe_exceptions
516 def fctx_annotate(self, wire, revision, path):
548 def fctx_annotate(self, wire, revision, path):
517 repo = self._factory.repo(wire)
549 repo = self._factory.repo(wire)
518 ctx = self._get_ctx(repo, revision)
550 ctx = self._get_ctx(repo, revision)
519 fctx = ctx.filectx(path)
551 fctx = ctx.filectx(path)
520
552
521 result = []
553 result = []
522 for i, annotate_obj in enumerate(fctx.annotate(), 1):
554 for i, annotate_obj in enumerate(fctx.annotate(), 1):
523 ln_no = i
555 ln_no = i
524 sha = hex(annotate_obj.fctx.node())
556 sha = hex(annotate_obj.fctx.node())
525 content = annotate_obj.text
557 content = annotate_obj.text
526 result.append((ln_no, sha, content))
558 result.append((ln_no, sha, content))
527 return result
559 return result
528
560
529 @reraise_safe_exceptions
561 @reraise_safe_exceptions
530 def fctx_data(self, wire, revision, path):
562 def fctx_node_data(self, wire, revision, path):
531 repo = self._factory.repo(wire)
563 repo = self._factory.repo(wire)
532 ctx = self._get_ctx(repo, revision)
564 ctx = self._get_ctx(repo, revision)
533 fctx = ctx.filectx(path)
565 fctx = ctx.filectx(path)
534 return fctx.data()
566 return fctx.data()
535
567
536 @reraise_safe_exceptions
568 @reraise_safe_exceptions
537 def fctx_flags(self, wire, revision, path):
569 def fctx_flags(self, wire, revision, path):
538 repo = self._factory.repo(wire)
570 repo = self._factory.repo(wire)
539 ctx = self._get_ctx(repo, revision)
571 ctx = self._get_ctx(repo, revision)
540 fctx = ctx.filectx(path)
572 fctx = ctx.filectx(path)
541 return fctx.flags()
573 return fctx.flags()
542
574
543 @reraise_safe_exceptions
575 @reraise_safe_exceptions
544 def fctx_size(self, wire, revision, path):
576 def fctx_size(self, wire, revision, path):
545 repo = self._factory.repo(wire)
577 repo = self._factory.repo(wire)
546 ctx = self._get_ctx(repo, revision)
578 ctx = self._get_ctx(repo, revision)
547 fctx = ctx.filectx(path)
579 fctx = ctx.filectx(path)
548 return fctx.size()
580 return fctx.size()
549
581
550 @reraise_safe_exceptions
582 @reraise_safe_exceptions
551 def get_all_commit_ids(self, wire, name):
583 def get_all_commit_ids(self, wire, name):
584 cache_on, context_uid, repo_id = self._cache_on(wire)
585 @self.region.conditional_cache_on_arguments(condition=cache_on)
586 def _get_all_commit_ids(_context_uid, _repo_id, _name):
552 repo = self._factory.repo(wire)
587 repo = self._factory.repo(wire)
553 repo = repo.filtered(name)
588 repo = repo.filtered(name)
554 revs = map(lambda x: hex(x[7]), repo.changelog.index)
589 revs = map(lambda x: hex(x[7]), repo.changelog.index)
555 return revs
590 return revs
591 return _get_all_commit_ids(context_uid, repo_id, name)
556
592
557 @reraise_safe_exceptions
593 @reraise_safe_exceptions
558 def get_config_value(self, wire, section, name, untrusted=False):
594 def get_config_value(self, wire, section, name, untrusted=False):
559 repo = self._factory.repo(wire)
595 repo = self._factory.repo(wire)
560 return repo.ui.config(section, name, untrusted=untrusted)
596 return repo.ui.config(section, name, untrusted=untrusted)
561
597
562 @reraise_safe_exceptions
598 @reraise_safe_exceptions
563 def get_config_bool(self, wire, section, name, untrusted=False):
599 def get_config_bool(self, wire, section, name, untrusted=False):
564 repo = self._factory.repo(wire)
600 repo = self._factory.repo(wire)
565 return repo.ui.configbool(section, name, untrusted=untrusted)
601 return repo.ui.configbool(section, name, untrusted=untrusted)
566
602
567 @reraise_safe_exceptions
603 @reraise_safe_exceptions
568 def get_config_list(self, wire, section, name, untrusted=False):
604 def get_config_list(self, wire, section, name, untrusted=False):
569 repo = self._factory.repo(wire)
605 repo = self._factory.repo(wire)
570 return repo.ui.configlist(section, name, untrusted=untrusted)
606 return repo.ui.configlist(section, name, untrusted=untrusted)
571
607
572 @reraise_safe_exceptions
608 @reraise_safe_exceptions
573 def is_large_file(self, wire, path):
609 def is_large_file(self, wire, path):
610 cache_on, context_uid, repo_id = self._cache_on(wire)
611 @self.region.conditional_cache_on_arguments(condition=cache_on)
612 def _is_large_file(_context_uid, _repo_id, _path):
574 return largefiles.lfutil.isstandin(path)
613 return largefiles.lfutil.isstandin(path)
575
614
615 return _is_large_file(context_uid, repo_id, path)
616
576 @reraise_safe_exceptions
617 @reraise_safe_exceptions
577 def in_largefiles_store(self, wire, sha):
618 def in_largefiles_store(self, wire, sha):
578 repo = self._factory.repo(wire)
619 repo = self._factory.repo(wire)
579 return largefiles.lfutil.instore(repo, sha)
620 return largefiles.lfutil.instore(repo, sha)
580
621
581 @reraise_safe_exceptions
622 @reraise_safe_exceptions
582 def in_user_cache(self, wire, sha):
623 def in_user_cache(self, wire, sha):
583 repo = self._factory.repo(wire)
624 repo = self._factory.repo(wire)
584 return largefiles.lfutil.inusercache(repo.ui, sha)
625 return largefiles.lfutil.inusercache(repo.ui, sha)
585
626
586 @reraise_safe_exceptions
627 @reraise_safe_exceptions
587 def store_path(self, wire, sha):
628 def store_path(self, wire, sha):
588 repo = self._factory.repo(wire)
629 repo = self._factory.repo(wire)
589 return largefiles.lfutil.storepath(repo, sha)
630 return largefiles.lfutil.storepath(repo, sha)
590
631
591 @reraise_safe_exceptions
632 @reraise_safe_exceptions
592 def link(self, wire, sha, path):
633 def link(self, wire, sha, path):
593 repo = self._factory.repo(wire)
634 repo = self._factory.repo(wire)
594 largefiles.lfutil.link(
635 largefiles.lfutil.link(
595 largefiles.lfutil.usercachepath(repo.ui, sha), path)
636 largefiles.lfutil.usercachepath(repo.ui, sha), path)
596
637
597 @reraise_safe_exceptions
638 @reraise_safe_exceptions
598 def localrepository(self, wire, create=False):
639 def localrepository(self, wire, create=False):
599 self._factory.repo(wire, create=create)
640 self._factory.repo(wire, create=create)
600
641
601 @reraise_safe_exceptions
642 @reraise_safe_exceptions
602 def lookup(self, wire, revision, both):
643 def lookup(self, wire, revision, both):
644 cache_on, context_uid, repo_id = self._cache_on(wire)
645 @self.region.conditional_cache_on_arguments(condition=cache_on)
646 def _lookup(_context_uid, _repo_id, _revision, _both):
603
647
604 repo = self._factory.repo(wire)
648 repo = self._factory.repo(wire)
605
649 rev = _revision
606 if isinstance(revision, int):
650 if isinstance(rev, int):
607 # NOTE(marcink):
651 # NOTE(marcink):
608 # since Mercurial doesn't support negative indexes properly
652 # since Mercurial doesn't support negative indexes properly
609 # we need to shift accordingly by one to get proper index, e.g
653 # we need to shift accordingly by one to get proper index, e.g
610 # repo[-1] => repo[-2]
654 # repo[-1] => repo[-2]
611 # repo[0] => repo[-1]
655 # repo[0] => repo[-1]
612 if revision <= 0:
656 if rev <= 0:
613 revision = revision + -1
657 rev = rev + -1
614 try:
658 try:
615 ctx = self._get_ctx(repo, revision)
659 ctx = self._get_ctx(repo, rev)
616 except (TypeError, RepoLookupError) as e:
660 except (TypeError, RepoLookupError) as e:
617 e._org_exc_tb = traceback.format_exc()
661 e._org_exc_tb = traceback.format_exc()
618 raise exceptions.LookupException(e)(revision)
662 raise exceptions.LookupException(e)(rev)
619 except LookupError as e:
663 except LookupError as e:
620 e._org_exc_tb = traceback.format_exc()
664 e._org_exc_tb = traceback.format_exc()
621 raise exceptions.LookupException(e)(e.name)
665 raise exceptions.LookupException(e)(e.name)
622
666
623 if not both:
667 if not both:
624 return ctx.hex()
668 return ctx.hex()
625
669
626 ctx = repo[ctx.hex()]
670 ctx = repo[ctx.hex()]
627 return ctx.hex(), ctx.rev()
671 return ctx.hex(), ctx.rev()
628
672
673 return _lookup(context_uid, repo_id, revision, both)
674
629 @reraise_safe_exceptions
675 @reraise_safe_exceptions
630 def pull(self, wire, url, commit_ids=None):
676 def pull(self, wire, url, commit_ids=None):
631 repo = self._factory.repo(wire)
677 repo = self._factory.repo(wire)
632 # Disable any prompts for this repo
678 # Disable any prompts for this repo
633 repo.ui.setconfig('ui', 'interactive', 'off', '-y')
679 repo.ui.setconfig('ui', 'interactive', 'off', '-y')
634
680
635 remote = peer(repo, {}, url)
681 remote = peer(repo, {}, url)
636 # Disable any prompts for this remote
682 # Disable any prompts for this remote
637 remote.ui.setconfig('ui', 'interactive', 'off', '-y')
683 remote.ui.setconfig('ui', 'interactive', 'off', '-y')
638
684
639 if commit_ids:
685 if commit_ids:
640 commit_ids = [bin(commit_id) for commit_id in commit_ids]
686 commit_ids = [bin(commit_id) for commit_id in commit_ids]
641
687
642 return exchange.pull(
688 return exchange.pull(
643 repo, remote, heads=commit_ids, force=None).cgresult
689 repo, remote, heads=commit_ids, force=None).cgresult
644
690
645 @reraise_safe_exceptions
691 @reraise_safe_exceptions
646 def sync_push(self, wire, url):
692 def sync_push(self, wire, url):
647 if not self.check_url(url, wire['config']):
693 if not self.check_url(url, wire['config']):
648 return
694 return
649
695
650 repo = self._factory.repo(wire)
696 repo = self._factory.repo(wire)
651
697
652 # Disable any prompts for this repo
698 # Disable any prompts for this repo
653 repo.ui.setconfig('ui', 'interactive', 'off', '-y')
699 repo.ui.setconfig('ui', 'interactive', 'off', '-y')
654
700
655 bookmarks = dict(repo._bookmarks).keys()
701 bookmarks = dict(repo._bookmarks).keys()
656 remote = peer(repo, {}, url)
702 remote = peer(repo, {}, url)
657 # Disable any prompts for this remote
703 # Disable any prompts for this remote
658 remote.ui.setconfig('ui', 'interactive', 'off', '-y')
704 remote.ui.setconfig('ui', 'interactive', 'off', '-y')
659
705
660 return exchange.push(
706 return exchange.push(
661 repo, remote, newbranch=True, bookmarks=bookmarks).cgresult
707 repo, remote, newbranch=True, bookmarks=bookmarks).cgresult
662
708
663 @reraise_safe_exceptions
709 @reraise_safe_exceptions
664 def revision(self, wire, rev):
710 def revision(self, wire, rev):
665 repo = self._factory.repo(wire)
711 repo = self._factory.repo(wire)
666 ctx = self._get_ctx(repo, rev)
712 ctx = self._get_ctx(repo, rev)
667 return ctx.rev()
713 return ctx.rev()
668
714
669 @reraise_safe_exceptions
715 @reraise_safe_exceptions
670 def rev_range(self, wire, filter):
716 def rev_range(self, wire, commit_filter):
717 cache_on, context_uid, repo_id = self._cache_on(wire)
718 @self.region.conditional_cache_on_arguments(condition=cache_on)
719 def _rev_range(_context_uid, _repo_id, _filter):
671 repo = self._factory.repo(wire)
720 repo = self._factory.repo(wire)
672 revisions = [rev for rev in revrange(repo, filter)]
721 revisions = [rev for rev in revrange(repo, commit_filter)]
673 return revisions
722 return revisions
674
723
724 return _rev_range(context_uid, repo_id, sorted(commit_filter))
725
675 @reraise_safe_exceptions
726 @reraise_safe_exceptions
676 def rev_range_hash(self, wire, node):
727 def rev_range_hash(self, wire, node):
677 repo = self._factory.repo(wire)
728 repo = self._factory.repo(wire)
678
729
679 def get_revs(repo, rev_opt):
730 def get_revs(repo, rev_opt):
680 if rev_opt:
731 if rev_opt:
681 revs = revrange(repo, rev_opt)
732 revs = revrange(repo, rev_opt)
682 if len(revs) == 0:
733 if len(revs) == 0:
683 return (nullrev, nullrev)
734 return (nullrev, nullrev)
684 return max(revs), min(revs)
735 return max(revs), min(revs)
685 else:
736 else:
686 return len(repo) - 1, 0
737 return len(repo) - 1, 0
687
738
688 stop, start = get_revs(repo, [node + ':'])
739 stop, start = get_revs(repo, [node + ':'])
689 revs = [hex(repo[r].node()) for r in xrange(start, stop + 1)]
740 revs = [hex(repo[r].node()) for r in xrange(start, stop + 1)]
690 return revs
741 return revs
691
742
692 @reraise_safe_exceptions
743 @reraise_safe_exceptions
693 def revs_from_revspec(self, wire, rev_spec, *args, **kwargs):
744 def revs_from_revspec(self, wire, rev_spec, *args, **kwargs):
694 other_path = kwargs.pop('other_path', None)
745 other_path = kwargs.pop('other_path', None)
695
746
696 # case when we want to compare two independent repositories
747 # case when we want to compare two independent repositories
697 if other_path and other_path != wire["path"]:
748 if other_path and other_path != wire["path"]:
698 baseui = self._factory._create_config(wire["config"])
749 baseui = self._factory._create_config(wire["config"])
699 repo = unionrepo.makeunionrepository(baseui, other_path, wire["path"])
750 repo = unionrepo.makeunionrepository(baseui, other_path, wire["path"])
700 else:
751 else:
701 repo = self._factory.repo(wire)
752 repo = self._factory.repo(wire)
702 return list(repo.revs(rev_spec, *args))
753 return list(repo.revs(rev_spec, *args))
703
754
704 @reraise_safe_exceptions
755 @reraise_safe_exceptions
705 def strip(self, wire, revision, update, backup):
756 def strip(self, wire, revision, update, backup):
706 repo = self._factory.repo(wire)
757 repo = self._factory.repo(wire)
707 ctx = self._get_ctx(repo, revision)
758 ctx = self._get_ctx(repo, revision)
708 hgext_strip(
759 hgext_strip(
709 repo.baseui, repo, ctx.node(), update=update, backup=backup)
760 repo.baseui, repo, ctx.node(), update=update, backup=backup)
710
761
711 @reraise_safe_exceptions
762 @reraise_safe_exceptions
712 def verify(self, wire,):
763 def verify(self, wire,):
713 repo = self._factory.repo(wire)
764 repo = self._factory.repo(wire)
714 baseui = self._factory._create_config(wire['config'])
765 baseui = self._factory._create_config(wire['config'])
715 baseui.setconfig('ui', 'quiet', 'false')
766 baseui.setconfig('ui', 'quiet', 'false')
716 output = io.BytesIO()
767 output = io.BytesIO()
717
768
718 def write(data, **unused_kwargs):
769 def write(data, **unused_kwargs):
719 output.write(data)
770 output.write(data)
720 baseui.write = write
771 baseui.write = write
721
772
722 repo.ui = baseui
773 repo.ui = baseui
723 verify.verify(repo)
774 verify.verify(repo)
724 return output.getvalue()
775 return output.getvalue()
725
776
726 @reraise_safe_exceptions
777 @reraise_safe_exceptions
727 def tag(self, wire, name, revision, message, local, user,
778 def tag(self, wire, name, revision, message, local, user, tag_time, tag_timezone):
728 tag_time, tag_timezone):
729 repo = self._factory.repo(wire)
779 repo = self._factory.repo(wire)
730 ctx = self._get_ctx(repo, revision)
780 ctx = self._get_ctx(repo, revision)
731 node = ctx.node()
781 node = ctx.node()
732
782
733 date = (tag_time, tag_timezone)
783 date = (tag_time, tag_timezone)
734 try:
784 try:
735 hg_tag.tag(repo, name, node, message, local, user, date)
785 hg_tag.tag(repo, name, node, message, local, user, date)
736 except Abort as e:
786 except Abort as e:
737 log.exception("Tag operation aborted")
787 log.exception("Tag operation aborted")
738 # Exception can contain unicode which we convert
788 # Exception can contain unicode which we convert
739 raise exceptions.AbortException(e)(repr(e))
789 raise exceptions.AbortException(e)(repr(e))
740
790
741 @reraise_safe_exceptions
791 @reraise_safe_exceptions
742 def tags(self, wire):
792 def tags(self, wire):
793 cache_on, context_uid, repo_id = self._cache_on(wire)
794 @self.region.conditional_cache_on_arguments(condition=cache_on)
795 def _tags(_context_uid, _repo_id):
743 repo = self._factory.repo(wire)
796 repo = self._factory.repo(wire)
744 return repo.tags()
797 return repo.tags()
745
798
799 return _tags(context_uid, repo_id)
800
746 @reraise_safe_exceptions
801 @reraise_safe_exceptions
747 def update(self, wire, node=None, clean=False):
802 def update(self, wire, node=None, clean=False):
748 repo = self._factory.repo(wire)
803 repo = self._factory.repo(wire)
749 baseui = self._factory._create_config(wire['config'])
804 baseui = self._factory._create_config(wire['config'])
750 commands.update(baseui, repo, node=node, clean=clean)
805 commands.update(baseui, repo, node=node, clean=clean)
751
806
752 @reraise_safe_exceptions
807 @reraise_safe_exceptions
753 def identify(self, wire):
808 def identify(self, wire):
754 repo = self._factory.repo(wire)
809 repo = self._factory.repo(wire)
755 baseui = self._factory._create_config(wire['config'])
810 baseui = self._factory._create_config(wire['config'])
756 output = io.BytesIO()
811 output = io.BytesIO()
757 baseui.write = output.write
812 baseui.write = output.write
758 # This is required to get a full node id
813 # This is required to get a full node id
759 baseui.debugflag = True
814 baseui.debugflag = True
760 commands.identify(baseui, repo, id=True)
815 commands.identify(baseui, repo, id=True)
761
816
762 return output.getvalue()
817 return output.getvalue()
763
818
764 @reraise_safe_exceptions
819 @reraise_safe_exceptions
765 def pull_cmd(self, wire, source, bookmark=None, branch=None, revision=None,
820 def pull_cmd(self, wire, source, bookmark=None, branch=None, revision=None, hooks=True):
766 hooks=True):
767 repo = self._factory.repo(wire)
821 repo = self._factory.repo(wire)
768 baseui = self._factory._create_config(wire['config'], hooks=hooks)
822 baseui = self._factory._create_config(wire['config'], hooks=hooks)
769
823
770 # Mercurial internally has a lot of logic that checks ONLY if
824 # Mercurial internally has a lot of logic that checks ONLY if
771 # option is defined, we just pass those if they are defined then
825 # option is defined, we just pass those if they are defined then
772 opts = {}
826 opts = {}
773 if bookmark:
827 if bookmark:
774 opts['bookmark'] = bookmark
828 opts['bookmark'] = bookmark
775 if branch:
829 if branch:
776 opts['branch'] = branch
830 opts['branch'] = branch
777 if revision:
831 if revision:
778 opts['rev'] = revision
832 opts['rev'] = revision
779
833
780 commands.pull(baseui, repo, source, **opts)
834 commands.pull(baseui, repo, source, **opts)
781
835
782 @reraise_safe_exceptions
836 @reraise_safe_exceptions
783 def heads(self, wire, branch=None):
837 def heads(self, wire, branch=None):
784 repo = self._factory.repo(wire)
838 repo = self._factory.repo(wire)
785 baseui = self._factory._create_config(wire['config'])
839 baseui = self._factory._create_config(wire['config'])
786 output = io.BytesIO()
840 output = io.BytesIO()
787
841
788 def write(data, **unused_kwargs):
842 def write(data, **unused_kwargs):
789 output.write(data)
843 output.write(data)
790
844
791 baseui.write = write
845 baseui.write = write
792 if branch:
846 if branch:
793 args = [branch]
847 args = [branch]
794 else:
848 else:
795 args = []
849 args = []
796 commands.heads(baseui, repo, template='{node} ', *args)
850 commands.heads(baseui, repo, template='{node} ', *args)
797
851
798 return output.getvalue()
852 return output.getvalue()
799
853
800 @reraise_safe_exceptions
854 @reraise_safe_exceptions
801 def ancestor(self, wire, revision1, revision2):
855 def ancestor(self, wire, revision1, revision2):
802 repo = self._factory.repo(wire)
856 repo = self._factory.repo(wire)
803 changelog = repo.changelog
857 changelog = repo.changelog
804 lookup = repo.lookup
858 lookup = repo.lookup
805 a = changelog.ancestor(lookup(revision1), lookup(revision2))
859 a = changelog.ancestor(lookup(revision1), lookup(revision2))
806 return hex(a)
860 return hex(a)
807
861
808 @reraise_safe_exceptions
862 @reraise_safe_exceptions
809 def push(self, wire, revisions, dest_path, hooks=True,
863 def push(self, wire, revisions, dest_path, hooks=True, push_branches=False):
810 push_branches=False):
811 repo = self._factory.repo(wire)
864 repo = self._factory.repo(wire)
812 baseui = self._factory._create_config(wire['config'], hooks=hooks)
865 baseui = self._factory._create_config(wire['config'], hooks=hooks)
813 commands.push(baseui, repo, dest=dest_path, rev=revisions,
866 commands.push(baseui, repo, dest=dest_path, rev=revisions,
814 new_branch=push_branches)
867 new_branch=push_branches)
815
868
816 @reraise_safe_exceptions
869 @reraise_safe_exceptions
817 def merge(self, wire, revision):
870 def merge(self, wire, revision):
818 repo = self._factory.repo(wire)
871 repo = self._factory.repo(wire)
819 baseui = self._factory._create_config(wire['config'])
872 baseui = self._factory._create_config(wire['config'])
820 repo.ui.setconfig('ui', 'merge', 'internal:dump')
873 repo.ui.setconfig('ui', 'merge', 'internal:dump')
821
874
822 # In case of sub repositories are used mercurial prompts the user in
875 # In case of sub repositories are used mercurial prompts the user in
823 # case of merge conflicts or different sub repository sources. By
876 # case of merge conflicts or different sub repository sources. By
824 # setting the interactive flag to `False` mercurial doesn't prompt the
877 # setting the interactive flag to `False` mercurial doesn't prompt the
825 # used but instead uses a default value.
878 # used but instead uses a default value.
826 repo.ui.setconfig('ui', 'interactive', False)
879 repo.ui.setconfig('ui', 'interactive', False)
827 commands.merge(baseui, repo, rev=revision)
880 commands.merge(baseui, repo, rev=revision)
828
881
829 @reraise_safe_exceptions
882 @reraise_safe_exceptions
830 def merge_state(self, wire):
883 def merge_state(self, wire):
831 repo = self._factory.repo(wire)
884 repo = self._factory.repo(wire)
832 repo.ui.setconfig('ui', 'merge', 'internal:dump')
885 repo.ui.setconfig('ui', 'merge', 'internal:dump')
833
886
834 # In case of sub repositories are used mercurial prompts the user in
887 # In case of sub repositories are used mercurial prompts the user in
835 # case of merge conflicts or different sub repository sources. By
888 # case of merge conflicts or different sub repository sources. By
836 # setting the interactive flag to `False` mercurial doesn't prompt the
889 # setting the interactive flag to `False` mercurial doesn't prompt the
837 # used but instead uses a default value.
890 # used but instead uses a default value.
838 repo.ui.setconfig('ui', 'interactive', False)
891 repo.ui.setconfig('ui', 'interactive', False)
839 ms = hg_merge.mergestate(repo)
892 ms = hg_merge.mergestate(repo)
840 return [x for x in ms.unresolved()]
893 return [x for x in ms.unresolved()]
841
894
842 @reraise_safe_exceptions
895 @reraise_safe_exceptions
843 def commit(self, wire, message, username, close_branch=False):
896 def commit(self, wire, message, username, close_branch=False):
844 repo = self._factory.repo(wire)
897 repo = self._factory.repo(wire)
845 baseui = self._factory._create_config(wire['config'])
898 baseui = self._factory._create_config(wire['config'])
846 repo.ui.setconfig('ui', 'username', username)
899 repo.ui.setconfig('ui', 'username', username)
847 commands.commit(baseui, repo, message=message, close_branch=close_branch)
900 commands.commit(baseui, repo, message=message, close_branch=close_branch)
848
901
849
850 @reraise_safe_exceptions
902 @reraise_safe_exceptions
851 def rebase(self, wire, source=None, dest=None, abort=False):
903 def rebase(self, wire, source=None, dest=None, abort=False):
852 repo = self._factory.repo(wire)
904 repo = self._factory.repo(wire)
853 baseui = self._factory._create_config(wire['config'])
905 baseui = self._factory._create_config(wire['config'])
854 repo.ui.setconfig('ui', 'merge', 'internal:dump')
906 repo.ui.setconfig('ui', 'merge', 'internal:dump')
855 rebase.rebase(
907 rebase.rebase(
856 baseui, repo, base=source, dest=dest, abort=abort, keep=not abort)
908 baseui, repo, base=source, dest=dest, abort=abort, keep=not abort)
857
909
858 @reraise_safe_exceptions
910 @reraise_safe_exceptions
859 def bookmark(self, wire, bookmark, revision=None):
911 def bookmark(self, wire, bookmark, revision=None):
860 repo = self._factory.repo(wire)
912 repo = self._factory.repo(wire)
861 baseui = self._factory._create_config(wire['config'])
913 baseui = self._factory._create_config(wire['config'])
862 commands.bookmark(baseui, repo, bookmark, rev=revision, force=True)
914 commands.bookmark(baseui, repo, bookmark, rev=revision, force=True)
863
915
864 @reraise_safe_exceptions
916 @reraise_safe_exceptions
865 def install_hooks(self, wire, force=False):
917 def install_hooks(self, wire, force=False):
866 # we don't need any special hooks for Mercurial
918 # we don't need any special hooks for Mercurial
867 pass
919 pass
868
920
869 @reraise_safe_exceptions
921 @reraise_safe_exceptions
870 def get_hooks_info(self, wire):
922 def get_hooks_info(self, wire):
871 return {
923 return {
872 'pre_version': vcsserver.__version__,
924 'pre_version': vcsserver.__version__,
873 'post_version': vcsserver.__version__,
925 'post_version': vcsserver.__version__,
874 }
926 }
@@ -1,772 +1,793 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2019 RhodeCode GmbH
2 # Copyright (C) 2014-2019 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 from __future__ import absolute_import
18 from __future__ import absolute_import
19
19
20 import os
20 import os
21 import subprocess
21 import subprocess
22 from urllib2 import URLError
22 from urllib2 import URLError
23 import urlparse
23 import urlparse
24 import logging
24 import logging
25 import posixpath as vcspath
25 import posixpath as vcspath
26 import StringIO
26 import StringIO
27 import urllib
27 import urllib
28 import traceback
28 import traceback
29
29
30 import svn.client
30 import svn.client
31 import svn.core
31 import svn.core
32 import svn.delta
32 import svn.delta
33 import svn.diff
33 import svn.diff
34 import svn.fs
34 import svn.fs
35 import svn.repos
35 import svn.repos
36
36
37 from vcsserver import svn_diff, exceptions, subprocessio, settings
37 from vcsserver import svn_diff, exceptions, subprocessio, settings
38 from vcsserver.base import RepoFactory, raise_from_original
38 from vcsserver.base import RepoFactory, raise_from_original
39
39
40 log = logging.getLogger(__name__)
40 log = logging.getLogger(__name__)
41
41
42
42
43 # Set of svn compatible version flags.
43 # Set of svn compatible version flags.
44 # Compare with subversion/svnadmin/svnadmin.c
44 # Compare with subversion/svnadmin/svnadmin.c
45 svn_compatible_versions = {
45 svn_compatible_versions = {
46 'pre-1.4-compatible',
46 'pre-1.4-compatible',
47 'pre-1.5-compatible',
47 'pre-1.5-compatible',
48 'pre-1.6-compatible',
48 'pre-1.6-compatible',
49 'pre-1.8-compatible',
49 'pre-1.8-compatible',
50 'pre-1.9-compatible'
50 'pre-1.9-compatible'
51 }
51 }
52
52
53 svn_compatible_versions_map = {
53 svn_compatible_versions_map = {
54 'pre-1.4-compatible': '1.3',
54 'pre-1.4-compatible': '1.3',
55 'pre-1.5-compatible': '1.4',
55 'pre-1.5-compatible': '1.4',
56 'pre-1.6-compatible': '1.5',
56 'pre-1.6-compatible': '1.5',
57 'pre-1.8-compatible': '1.7',
57 'pre-1.8-compatible': '1.7',
58 'pre-1.9-compatible': '1.8',
58 'pre-1.9-compatible': '1.8',
59 }
59 }
60
60
61
61
62 def reraise_safe_exceptions(func):
62 def reraise_safe_exceptions(func):
63 """Decorator for converting svn exceptions to something neutral."""
63 """Decorator for converting svn exceptions to something neutral."""
64 def wrapper(*args, **kwargs):
64 def wrapper(*args, **kwargs):
65 try:
65 try:
66 return func(*args, **kwargs)
66 return func(*args, **kwargs)
67 except Exception as e:
67 except Exception as e:
68 if not hasattr(e, '_vcs_kind'):
68 if not hasattr(e, '_vcs_kind'):
69 log.exception("Unhandled exception in svn remote call")
69 log.exception("Unhandled exception in svn remote call")
70 raise_from_original(exceptions.UnhandledException(e))
70 raise_from_original(exceptions.UnhandledException(e))
71 raise
71 raise
72 return wrapper
72 return wrapper
73
73
74
74
75 class SubversionFactory(RepoFactory):
75 class SubversionFactory(RepoFactory):
76 repo_type = 'svn'
76 repo_type = 'svn'
77
77
78 def _create_repo(self, wire, create, compatible_version):
78 def _create_repo(self, wire, create, compatible_version):
79 path = svn.core.svn_path_canonicalize(wire['path'])
79 path = svn.core.svn_path_canonicalize(wire['path'])
80 if create:
80 if create:
81 fs_config = {'compatible-version': '1.9'}
81 fs_config = {'compatible-version': '1.9'}
82 if compatible_version:
82 if compatible_version:
83 if compatible_version not in svn_compatible_versions:
83 if compatible_version not in svn_compatible_versions:
84 raise Exception('Unknown SVN compatible version "{}"'
84 raise Exception('Unknown SVN compatible version "{}"'
85 .format(compatible_version))
85 .format(compatible_version))
86 fs_config['compatible-version'] = \
86 fs_config['compatible-version'] = \
87 svn_compatible_versions_map[compatible_version]
87 svn_compatible_versions_map[compatible_version]
88
88
89 log.debug('Create SVN repo with config "%s"', fs_config)
89 log.debug('Create SVN repo with config "%s"', fs_config)
90 repo = svn.repos.create(path, "", "", None, fs_config)
90 repo = svn.repos.create(path, "", "", None, fs_config)
91 else:
91 else:
92 repo = svn.repos.open(path)
92 repo = svn.repos.open(path)
93
93
94 log.debug('Got SVN object: %s', repo)
94 log.debug('Got SVN object: %s', repo)
95 return repo
95 return repo
96
96
97 def repo(self, wire, create=False, compatible_version=None):
97 def repo(self, wire, create=False, compatible_version=None):
98 """
98 """
99 Get a repository instance for the given path.
99 Get a repository instance for the given path.
100 """
100 """
101 region = self._cache_region
102 context = wire.get('context', None)
103 repo_path = wire.get('path', '')
104 context_uid = '{}'.format(context)
105 cache = wire.get('cache', True)
106 cache_on = context and cache
107
108 @region.conditional_cache_on_arguments(condition=cache_on)
109 def create_new_repo(_repo_type, _repo_path, _context_uid, compatible_version_id):
110 return self._create_repo(wire, create, compatible_version)
101 return self._create_repo(wire, create, compatible_version)
111
102
112 return create_new_repo(self.repo_type, repo_path, context_uid,
113 compatible_version)
114
115
103
116 NODE_TYPE_MAPPING = {
104 NODE_TYPE_MAPPING = {
117 svn.core.svn_node_file: 'file',
105 svn.core.svn_node_file: 'file',
118 svn.core.svn_node_dir: 'dir',
106 svn.core.svn_node_dir: 'dir',
119 }
107 }
120
108
121
109
122 class SvnRemote(object):
110 class SvnRemote(object):
123
111
124 def __init__(self, factory, hg_factory=None):
112 def __init__(self, factory, hg_factory=None):
125 self._factory = factory
113 self._factory = factory
126 # TODO: Remove once we do not use internal Mercurial objects anymore
114 # TODO: Remove once we do not use internal Mercurial objects anymore
127 # for subversion
115 # for subversion
128 self._hg_factory = hg_factory
116 self._hg_factory = hg_factory
117 self.region = self._factory._cache_region
118
119 def _cache_on(self, wire):
120 context = wire.get('context', '')
121 context_uid = '{}'.format(context)
122 repo_id = wire.get('repo_id', '')
123 cache = wire.get('cache', True)
124 cache_on = context and cache
125 return cache_on, context_uid, repo_id
129
126
130 @reraise_safe_exceptions
127 @reraise_safe_exceptions
131 def discover_svn_version(self):
128 def discover_svn_version(self):
132 try:
129 try:
133 import svn.core
130 import svn.core
134 svn_ver = svn.core.SVN_VERSION
131 svn_ver = svn.core.SVN_VERSION
135 except ImportError:
132 except ImportError:
136 svn_ver = None
133 svn_ver = None
137 return svn_ver
134 return svn_ver
138
135
139 @reraise_safe_exceptions
136 @reraise_safe_exceptions
140 def is_empty(self, wire):
137 def is_empty(self, wire):
141 repo = self._factory.repo(wire)
142
138
143 try:
139 try:
144 return self.lookup(wire, -1) == 0
140 return self.lookup(wire, -1) == 0
145 except Exception:
141 except Exception:
146 log.exception("failed to read object_store")
142 log.exception("failed to read object_store")
147 return False
143 return False
148
144
149 def check_url(self, url, config_items):
145 def check_url(self, url, config_items):
150 # this can throw exception if not installed, but we detect this
146 # this can throw exception if not installed, but we detect this
151 from hgsubversion import svnrepo
147 from hgsubversion import svnrepo
152
148
153 baseui = self._hg_factory._create_config(config_items)
149 baseui = self._hg_factory._create_config(config_items)
154 # uuid function get's only valid UUID from proper repo, else
150 # uuid function get's only valid UUID from proper repo, else
155 # throws exception
151 # throws exception
156 try:
152 try:
157 svnrepo.svnremoterepo(baseui, url).svn.uuid
153 svnrepo.svnremoterepo(baseui, url).svn.uuid
158 except Exception:
154 except Exception:
159 tb = traceback.format_exc()
155 tb = traceback.format_exc()
160 log.debug("Invalid Subversion url: `%s`, tb: %s", url, tb)
156 log.debug("Invalid Subversion url: `%s`, tb: %s", url, tb)
161 raise URLError(
157 raise URLError(
162 '"%s" is not a valid Subversion source url.' % (url, ))
158 '"%s" is not a valid Subversion source url.' % (url, ))
163 return True
159 return True
164
160
165 def is_path_valid_repository(self, wire, path):
161 def is_path_valid_repository(self, wire, path):
166
162
167 # NOTE(marcink): short circuit the check for SVN repo
163 # NOTE(marcink): short circuit the check for SVN repo
168 # the repos.open might be expensive to check, but we have one cheap
164 # the repos.open might be expensive to check, but we have one cheap
169 # pre condition that we can use, to check for 'format' file
165 # pre condition that we can use, to check for 'format' file
170
166
171 if not os.path.isfile(os.path.join(path, 'format')):
167 if not os.path.isfile(os.path.join(path, 'format')):
172 return False
168 return False
173
169
174 try:
170 try:
175 svn.repos.open(path)
171 svn.repos.open(path)
176 except svn.core.SubversionException:
172 except svn.core.SubversionException:
177 tb = traceback.format_exc()
173 tb = traceback.format_exc()
178 log.debug("Invalid Subversion path `%s`, tb: %s", path, tb)
174 log.debug("Invalid Subversion path `%s`, tb: %s", path, tb)
179 return False
175 return False
180 return True
176 return True
181
177
182 @reraise_safe_exceptions
178 @reraise_safe_exceptions
183 def verify(self, wire,):
179 def verify(self, wire,):
184 repo_path = wire['path']
180 repo_path = wire['path']
185 if not self.is_path_valid_repository(wire, repo_path):
181 if not self.is_path_valid_repository(wire, repo_path):
186 raise Exception(
182 raise Exception(
187 "Path %s is not a valid Subversion repository." % repo_path)
183 "Path %s is not a valid Subversion repository." % repo_path)
188
184
189 cmd = ['svnadmin', 'info', repo_path]
185 cmd = ['svnadmin', 'info', repo_path]
190 stdout, stderr = subprocessio.run_command(cmd)
186 stdout, stderr = subprocessio.run_command(cmd)
191 return stdout
187 return stdout
192
188
193 def lookup(self, wire, revision):
189 def lookup(self, wire, revision):
194 if revision not in [-1, None, 'HEAD']:
190 if revision not in [-1, None, 'HEAD']:
195 raise NotImplementedError
191 raise NotImplementedError
196 repo = self._factory.repo(wire)
192 repo = self._factory.repo(wire)
197 fs_ptr = svn.repos.fs(repo)
193 fs_ptr = svn.repos.fs(repo)
198 head = svn.fs.youngest_rev(fs_ptr)
194 head = svn.fs.youngest_rev(fs_ptr)
199 return head
195 return head
200
196
201 def lookup_interval(self, wire, start_ts, end_ts):
197 def lookup_interval(self, wire, start_ts, end_ts):
202 repo = self._factory.repo(wire)
198 repo = self._factory.repo(wire)
203 fsobj = svn.repos.fs(repo)
199 fsobj = svn.repos.fs(repo)
204 start_rev = None
200 start_rev = None
205 end_rev = None
201 end_rev = None
206 if start_ts:
202 if start_ts:
207 start_ts_svn = apr_time_t(start_ts)
203 start_ts_svn = apr_time_t(start_ts)
208 start_rev = svn.repos.dated_revision(repo, start_ts_svn) + 1
204 start_rev = svn.repos.dated_revision(repo, start_ts_svn) + 1
209 else:
205 else:
210 start_rev = 1
206 start_rev = 1
211 if end_ts:
207 if end_ts:
212 end_ts_svn = apr_time_t(end_ts)
208 end_ts_svn = apr_time_t(end_ts)
213 end_rev = svn.repos.dated_revision(repo, end_ts_svn)
209 end_rev = svn.repos.dated_revision(repo, end_ts_svn)
214 else:
210 else:
215 end_rev = svn.fs.youngest_rev(fsobj)
211 end_rev = svn.fs.youngest_rev(fsobj)
216 return start_rev, end_rev
212 return start_rev, end_rev
217
213
218 def revision_properties(self, wire, revision):
214 def revision_properties(self, wire, revision):
215
216 cache_on, context_uid, repo_id = self._cache_on(wire)
217 @self.region.conditional_cache_on_arguments(condition=cache_on)
218 def _revision_properties(_context_uid, _repo_id, _revision):
219 repo = self._factory.repo(wire)
219 repo = self._factory.repo(wire)
220 fs_ptr = svn.repos.fs(repo)
220 fs_ptr = svn.repos.fs(repo)
221 return svn.fs.revision_proplist(fs_ptr, revision)
221 return svn.fs.revision_proplist(fs_ptr, revision)
222 return _revision_properties(context_uid, repo_id, revision)
222
223
223 def revision_changes(self, wire, revision):
224 def revision_changes(self, wire, revision):
224
225
225 repo = self._factory.repo(wire)
226 repo = self._factory.repo(wire)
226 fsobj = svn.repos.fs(repo)
227 fsobj = svn.repos.fs(repo)
227 rev_root = svn.fs.revision_root(fsobj, revision)
228 rev_root = svn.fs.revision_root(fsobj, revision)
228
229
229 editor = svn.repos.ChangeCollector(fsobj, rev_root)
230 editor = svn.repos.ChangeCollector(fsobj, rev_root)
230 editor_ptr, editor_baton = svn.delta.make_editor(editor)
231 editor_ptr, editor_baton = svn.delta.make_editor(editor)
231 base_dir = ""
232 base_dir = ""
232 send_deltas = False
233 send_deltas = False
233 svn.repos.replay2(
234 svn.repos.replay2(
234 rev_root, base_dir, svn.core.SVN_INVALID_REVNUM, send_deltas,
235 rev_root, base_dir, svn.core.SVN_INVALID_REVNUM, send_deltas,
235 editor_ptr, editor_baton, None)
236 editor_ptr, editor_baton, None)
236
237
237 added = []
238 added = []
238 changed = []
239 changed = []
239 removed = []
240 removed = []
240
241
241 # TODO: CHANGE_ACTION_REPLACE: Figure out where it belongs
242 # TODO: CHANGE_ACTION_REPLACE: Figure out where it belongs
242 for path, change in editor.changes.iteritems():
243 for path, change in editor.changes.iteritems():
243 # TODO: Decide what to do with directory nodes. Subversion can add
244 # TODO: Decide what to do with directory nodes. Subversion can add
244 # empty directories.
245 # empty directories.
245
246
246 if change.item_kind == svn.core.svn_node_dir:
247 if change.item_kind == svn.core.svn_node_dir:
247 continue
248 continue
248 if change.action in [svn.repos.CHANGE_ACTION_ADD]:
249 if change.action in [svn.repos.CHANGE_ACTION_ADD]:
249 added.append(path)
250 added.append(path)
250 elif change.action in [svn.repos.CHANGE_ACTION_MODIFY,
251 elif change.action in [svn.repos.CHANGE_ACTION_MODIFY,
251 svn.repos.CHANGE_ACTION_REPLACE]:
252 svn.repos.CHANGE_ACTION_REPLACE]:
252 changed.append(path)
253 changed.append(path)
253 elif change.action in [svn.repos.CHANGE_ACTION_DELETE]:
254 elif change.action in [svn.repos.CHANGE_ACTION_DELETE]:
254 removed.append(path)
255 removed.append(path)
255 else:
256 else:
256 raise NotImplementedError(
257 raise NotImplementedError(
257 "Action %s not supported on path %s" % (
258 "Action %s not supported on path %s" % (
258 change.action, path))
259 change.action, path))
259
260
260 changes = {
261 changes = {
261 'added': added,
262 'added': added,
262 'changed': changed,
263 'changed': changed,
263 'removed': removed,
264 'removed': removed,
264 }
265 }
265 return changes
266 return changes
266
267
268 @reraise_safe_exceptions
267 def node_history(self, wire, path, revision, limit):
269 def node_history(self, wire, path, revision, limit):
270 cache_on, context_uid, repo_id = self._cache_on(wire)
271 @self.region.conditional_cache_on_arguments(condition=cache_on)
272 def _assert_correct_path(_context_uid, _repo_id, _path, _revision, _limit):
268 cross_copies = False
273 cross_copies = False
269 repo = self._factory.repo(wire)
274 repo = self._factory.repo(wire)
270 fsobj = svn.repos.fs(repo)
275 fsobj = svn.repos.fs(repo)
271 rev_root = svn.fs.revision_root(fsobj, revision)
276 rev_root = svn.fs.revision_root(fsobj, revision)
272
277
273 history_revisions = []
278 history_revisions = []
274 history = svn.fs.node_history(rev_root, path)
279 history = svn.fs.node_history(rev_root, path)
275 history = svn.fs.history_prev(history, cross_copies)
280 history = svn.fs.history_prev(history, cross_copies)
276 while history:
281 while history:
277 __, node_revision = svn.fs.history_location(history)
282 __, node_revision = svn.fs.history_location(history)
278 history_revisions.append(node_revision)
283 history_revisions.append(node_revision)
279 if limit and len(history_revisions) >= limit:
284 if limit and len(history_revisions) >= limit:
280 break
285 break
281 history = svn.fs.history_prev(history, cross_copies)
286 history = svn.fs.history_prev(history, cross_copies)
282 return history_revisions
287 return history_revisions
288 return _assert_correct_path(context_uid, repo_id, path, revision, limit)
283
289
284 def node_properties(self, wire, path, revision):
290 def node_properties(self, wire, path, revision):
285 repo = self._factory.repo(wire)
291 repo = self._factory.repo(wire)
286 fsobj = svn.repos.fs(repo)
292 fsobj = svn.repos.fs(repo)
287 rev_root = svn.fs.revision_root(fsobj, revision)
293 rev_root = svn.fs.revision_root(fsobj, revision)
288 return svn.fs.node_proplist(rev_root, path)
294 return svn.fs.node_proplist(rev_root, path)
289
295
290 def file_annotate(self, wire, path, revision):
296 def file_annotate(self, wire, path, revision):
291 abs_path = 'file://' + urllib.pathname2url(
297 abs_path = 'file://' + urllib.pathname2url(
292 vcspath.join(wire['path'], path))
298 vcspath.join(wire['path'], path))
293 file_uri = svn.core.svn_path_canonicalize(abs_path)
299 file_uri = svn.core.svn_path_canonicalize(abs_path)
294
300
295 start_rev = svn_opt_revision_value_t(0)
301 start_rev = svn_opt_revision_value_t(0)
296 peg_rev = svn_opt_revision_value_t(revision)
302 peg_rev = svn_opt_revision_value_t(revision)
297 end_rev = peg_rev
303 end_rev = peg_rev
298
304
299 annotations = []
305 annotations = []
300
306
301 def receiver(line_no, revision, author, date, line, pool):
307 def receiver(line_no, revision, author, date, line, pool):
302 annotations.append((line_no, revision, line))
308 annotations.append((line_no, revision, line))
303
309
304 # TODO: Cannot use blame5, missing typemap function in the swig code
310 # TODO: Cannot use blame5, missing typemap function in the swig code
305 try:
311 try:
306 svn.client.blame2(
312 svn.client.blame2(
307 file_uri, peg_rev, start_rev, end_rev,
313 file_uri, peg_rev, start_rev, end_rev,
308 receiver, svn.client.create_context())
314 receiver, svn.client.create_context())
309 except svn.core.SubversionException as exc:
315 except svn.core.SubversionException as exc:
310 log.exception("Error during blame operation.")
316 log.exception("Error during blame operation.")
311 raise Exception(
317 raise Exception(
312 "Blame not supported or file does not exist at path %s. "
318 "Blame not supported or file does not exist at path %s. "
313 "Error %s." % (path, exc))
319 "Error %s." % (path, exc))
314
320
315 return annotations
321 return annotations
316
322
317 def get_node_type(self, wire, path, rev=None):
323 def get_node_type(self, wire, path, revision=None):
324
325 cache_on, context_uid, repo_id = self._cache_on(wire)
326 @self.region.conditional_cache_on_arguments(condition=cache_on)
327 def _get_node_type(_context_uid, _repo_id, _path, _revision):
318 repo = self._factory.repo(wire)
328 repo = self._factory.repo(wire)
319 fs_ptr = svn.repos.fs(repo)
329 fs_ptr = svn.repos.fs(repo)
320 if rev is None:
330 if _revision is None:
321 rev = svn.fs.youngest_rev(fs_ptr)
331 _revision = svn.fs.youngest_rev(fs_ptr)
322 root = svn.fs.revision_root(fs_ptr, rev)
332 root = svn.fs.revision_root(fs_ptr, _revision)
323 node = svn.fs.check_path(root, path)
333 node = svn.fs.check_path(root, path)
324 return NODE_TYPE_MAPPING.get(node, None)
334 return NODE_TYPE_MAPPING.get(node, None)
335 return _get_node_type(context_uid, repo_id, path, revision)
325
336
326 def get_nodes(self, wire, path, revision=None):
337 def get_nodes(self, wire, path, revision=None):
338
339 cache_on, context_uid, repo_id = self._cache_on(wire)
340 @self.region.conditional_cache_on_arguments(condition=cache_on)
341 def _get_nodes(_context_uid, _repo_id, _path, _revision):
327 repo = self._factory.repo(wire)
342 repo = self._factory.repo(wire)
328 fsobj = svn.repos.fs(repo)
343 fsobj = svn.repos.fs(repo)
329 if revision is None:
344 if _revision is None:
330 revision = svn.fs.youngest_rev(fsobj)
345 _revision = svn.fs.youngest_rev(fsobj)
331 root = svn.fs.revision_root(fsobj, revision)
346 root = svn.fs.revision_root(fsobj, _revision)
332 entries = svn.fs.dir_entries(root, path)
347 entries = svn.fs.dir_entries(root, path)
333 result = []
348 result = []
334 for entry_path, entry_info in entries.iteritems():
349 for entry_path, entry_info in entries.iteritems():
335 result.append(
350 result.append(
336 (entry_path, NODE_TYPE_MAPPING.get(entry_info.kind, None)))
351 (entry_path, NODE_TYPE_MAPPING.get(entry_info.kind, None)))
337 return result
352 return result
353 return _get_nodes(context_uid, repo_id, path, revision)
338
354
339 def get_file_content(self, wire, path, rev=None):
355 def get_file_content(self, wire, path, rev=None):
340 repo = self._factory.repo(wire)
356 repo = self._factory.repo(wire)
341 fsobj = svn.repos.fs(repo)
357 fsobj = svn.repos.fs(repo)
342 if rev is None:
358 if rev is None:
343 rev = svn.fs.youngest_revision(fsobj)
359 rev = svn.fs.youngest_revision(fsobj)
344 root = svn.fs.revision_root(fsobj, rev)
360 root = svn.fs.revision_root(fsobj, rev)
345 content = svn.core.Stream(svn.fs.file_contents(root, path))
361 content = svn.core.Stream(svn.fs.file_contents(root, path))
346 return content.read()
362 return content.read()
347
363
348 def get_file_size(self, wire, path, revision=None):
364 def get_file_size(self, wire, path, revision=None):
365
366 cache_on, context_uid, repo_id = self._cache_on(wire)
367 @self.region.conditional_cache_on_arguments(condition=cache_on)
368 def _get_file_size(_context_uid, _repo_id, _path, _revision):
349 repo = self._factory.repo(wire)
369 repo = self._factory.repo(wire)
350 fsobj = svn.repos.fs(repo)
370 fsobj = svn.repos.fs(repo)
351 if revision is None:
371 if _revision is None:
352 revision = svn.fs.youngest_revision(fsobj)
372 _revision = svn.fs.youngest_revision(fsobj)
353 root = svn.fs.revision_root(fsobj, revision)
373 root = svn.fs.revision_root(fsobj, _revision)
354 size = svn.fs.file_length(root, path)
374 size = svn.fs.file_length(root, path)
355 return size
375 return size
376 return _get_file_size(context_uid, repo_id, path, revision)
356
377
357 def create_repository(self, wire, compatible_version=None):
378 def create_repository(self, wire, compatible_version=None):
358 log.info('Creating Subversion repository in path "%s"', wire['path'])
379 log.info('Creating Subversion repository in path "%s"', wire['path'])
359 self._factory.repo(wire, create=True,
380 self._factory.repo(wire, create=True,
360 compatible_version=compatible_version)
381 compatible_version=compatible_version)
361
382
362 def get_url_and_credentials(self, src_url):
383 def get_url_and_credentials(self, src_url):
363 obj = urlparse.urlparse(src_url)
384 obj = urlparse.urlparse(src_url)
364 username = obj.username or None
385 username = obj.username or None
365 password = obj.password or None
386 password = obj.password or None
366 return username, password, src_url
387 return username, password, src_url
367
388
368 def import_remote_repository(self, wire, src_url):
389 def import_remote_repository(self, wire, src_url):
369 repo_path = wire['path']
390 repo_path = wire['path']
370 if not self.is_path_valid_repository(wire, repo_path):
391 if not self.is_path_valid_repository(wire, repo_path):
371 raise Exception(
392 raise Exception(
372 "Path %s is not a valid Subversion repository." % repo_path)
393 "Path %s is not a valid Subversion repository." % repo_path)
373
394
374 username, password, src_url = self.get_url_and_credentials(src_url)
395 username, password, src_url = self.get_url_and_credentials(src_url)
375 rdump_cmd = ['svnrdump', 'dump', '--non-interactive',
396 rdump_cmd = ['svnrdump', 'dump', '--non-interactive',
376 '--trust-server-cert-failures=unknown-ca']
397 '--trust-server-cert-failures=unknown-ca']
377 if username and password:
398 if username and password:
378 rdump_cmd += ['--username', username, '--password', password]
399 rdump_cmd += ['--username', username, '--password', password]
379 rdump_cmd += [src_url]
400 rdump_cmd += [src_url]
380
401
381 rdump = subprocess.Popen(
402 rdump = subprocess.Popen(
382 rdump_cmd,
403 rdump_cmd,
383 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
404 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
384 load = subprocess.Popen(
405 load = subprocess.Popen(
385 ['svnadmin', 'load', repo_path], stdin=rdump.stdout)
406 ['svnadmin', 'load', repo_path], stdin=rdump.stdout)
386
407
387 # TODO: johbo: This can be a very long operation, might be better
408 # TODO: johbo: This can be a very long operation, might be better
388 # to track some kind of status and provide an api to check if the
409 # to track some kind of status and provide an api to check if the
389 # import is done.
410 # import is done.
390 rdump.wait()
411 rdump.wait()
391 load.wait()
412 load.wait()
392
413
393 log.debug('Return process ended with code: %s', rdump.returncode)
414 log.debug('Return process ended with code: %s', rdump.returncode)
394 if rdump.returncode != 0:
415 if rdump.returncode != 0:
395 errors = rdump.stderr.read()
416 errors = rdump.stderr.read()
396 log.error('svnrdump dump failed: statuscode %s: message: %s',
417 log.error('svnrdump dump failed: statuscode %s: message: %s',
397 rdump.returncode, errors)
418 rdump.returncode, errors)
398 reason = 'UNKNOWN'
419 reason = 'UNKNOWN'
399 if 'svnrdump: E230001:' in errors:
420 if 'svnrdump: E230001:' in errors:
400 reason = 'INVALID_CERTIFICATE'
421 reason = 'INVALID_CERTIFICATE'
401
422
402 if reason == 'UNKNOWN':
423 if reason == 'UNKNOWN':
403 reason = 'UNKNOWN:{}'.format(errors)
424 reason = 'UNKNOWN:{}'.format(errors)
404 raise Exception(
425 raise Exception(
405 'Failed to dump the remote repository from %s. Reason:%s' % (
426 'Failed to dump the remote repository from %s. Reason:%s' % (
406 src_url, reason))
427 src_url, reason))
407 if load.returncode != 0:
428 if load.returncode != 0:
408 raise Exception(
429 raise Exception(
409 'Failed to load the dump of remote repository from %s.' %
430 'Failed to load the dump of remote repository from %s.' %
410 (src_url, ))
431 (src_url, ))
411
432
412 def commit(self, wire, message, author, timestamp, updated, removed):
433 def commit(self, wire, message, author, timestamp, updated, removed):
413 assert isinstance(message, str)
434 assert isinstance(message, str)
414 assert isinstance(author, str)
435 assert isinstance(author, str)
415
436
416 repo = self._factory.repo(wire)
437 repo = self._factory.repo(wire)
417 fsobj = svn.repos.fs(repo)
438 fsobj = svn.repos.fs(repo)
418
439
419 rev = svn.fs.youngest_rev(fsobj)
440 rev = svn.fs.youngest_rev(fsobj)
420 txn = svn.repos.fs_begin_txn_for_commit(repo, rev, author, message)
441 txn = svn.repos.fs_begin_txn_for_commit(repo, rev, author, message)
421 txn_root = svn.fs.txn_root(txn)
442 txn_root = svn.fs.txn_root(txn)
422
443
423 for node in updated:
444 for node in updated:
424 TxnNodeProcessor(node, txn_root).update()
445 TxnNodeProcessor(node, txn_root).update()
425 for node in removed:
446 for node in removed:
426 TxnNodeProcessor(node, txn_root).remove()
447 TxnNodeProcessor(node, txn_root).remove()
427
448
428 commit_id = svn.repos.fs_commit_txn(repo, txn)
449 commit_id = svn.repos.fs_commit_txn(repo, txn)
429
450
430 if timestamp:
451 if timestamp:
431 apr_time = apr_time_t(timestamp)
452 apr_time = apr_time_t(timestamp)
432 ts_formatted = svn.core.svn_time_to_cstring(apr_time)
453 ts_formatted = svn.core.svn_time_to_cstring(apr_time)
433 svn.fs.change_rev_prop(fsobj, commit_id, 'svn:date', ts_formatted)
454 svn.fs.change_rev_prop(fsobj, commit_id, 'svn:date', ts_formatted)
434
455
435 log.debug('Committed revision "%s" to "%s".', commit_id, wire['path'])
456 log.debug('Committed revision "%s" to "%s".', commit_id, wire['path'])
436 return commit_id
457 return commit_id
437
458
438 def diff(self, wire, rev1, rev2, path1=None, path2=None,
459 def diff(self, wire, rev1, rev2, path1=None, path2=None,
439 ignore_whitespace=False, context=3):
460 ignore_whitespace=False, context=3):
440
461
441 wire.update(cache=False)
462 wire.update(cache=False)
442 repo = self._factory.repo(wire)
463 repo = self._factory.repo(wire)
443 diff_creator = SvnDiffer(
464 diff_creator = SvnDiffer(
444 repo, rev1, path1, rev2, path2, ignore_whitespace, context)
465 repo, rev1, path1, rev2, path2, ignore_whitespace, context)
445 try:
466 try:
446 return diff_creator.generate_diff()
467 return diff_creator.generate_diff()
447 except svn.core.SubversionException as e:
468 except svn.core.SubversionException as e:
448 log.exception(
469 log.exception(
449 "Error during diff operation operation. "
470 "Error during diff operation operation. "
450 "Path might not exist %s, %s" % (path1, path2))
471 "Path might not exist %s, %s" % (path1, path2))
451 return ""
472 return ""
452
473
453 @reraise_safe_exceptions
474 @reraise_safe_exceptions
454 def is_large_file(self, wire, path):
475 def is_large_file(self, wire, path):
455 return False
476 return False
456
477
457 @reraise_safe_exceptions
478 @reraise_safe_exceptions
458 def run_svn_command(self, wire, cmd, **opts):
479 def run_svn_command(self, wire, cmd, **opts):
459 path = wire.get('path', None)
480 path = wire.get('path', None)
460
481
461 if path and os.path.isdir(path):
482 if path and os.path.isdir(path):
462 opts['cwd'] = path
483 opts['cwd'] = path
463
484
464 safe_call = False
485 safe_call = False
465 if '_safe' in opts:
486 if '_safe' in opts:
466 safe_call = True
487 safe_call = True
467
488
468 svnenv = os.environ.copy()
489 svnenv = os.environ.copy()
469 svnenv.update(opts.pop('extra_env', {}))
490 svnenv.update(opts.pop('extra_env', {}))
470
491
471 _opts = {'env': svnenv, 'shell': False}
492 _opts = {'env': svnenv, 'shell': False}
472
493
473 try:
494 try:
474 _opts.update(opts)
495 _opts.update(opts)
475 p = subprocessio.SubprocessIOChunker(cmd, **_opts)
496 p = subprocessio.SubprocessIOChunker(cmd, **_opts)
476
497
477 return ''.join(p), ''.join(p.error)
498 return ''.join(p), ''.join(p.error)
478 except (EnvironmentError, OSError) as err:
499 except (EnvironmentError, OSError) as err:
479 cmd = ' '.join(cmd) # human friendly CMD
500 cmd = ' '.join(cmd) # human friendly CMD
480 tb_err = ("Couldn't run svn command (%s).\n"
501 tb_err = ("Couldn't run svn command (%s).\n"
481 "Original error was:%s\n"
502 "Original error was:%s\n"
482 "Call options:%s\n"
503 "Call options:%s\n"
483 % (cmd, err, _opts))
504 % (cmd, err, _opts))
484 log.exception(tb_err)
505 log.exception(tb_err)
485 if safe_call:
506 if safe_call:
486 return '', err
507 return '', err
487 else:
508 else:
488 raise exceptions.VcsException()(tb_err)
509 raise exceptions.VcsException()(tb_err)
489
510
490 @reraise_safe_exceptions
511 @reraise_safe_exceptions
491 def install_hooks(self, wire, force=False):
512 def install_hooks(self, wire, force=False):
492 from vcsserver.hook_utils import install_svn_hooks
513 from vcsserver.hook_utils import install_svn_hooks
493 repo_path = wire['path']
514 repo_path = wire['path']
494 binary_dir = settings.BINARY_DIR
515 binary_dir = settings.BINARY_DIR
495 executable = None
516 executable = None
496 if binary_dir:
517 if binary_dir:
497 executable = os.path.join(binary_dir, 'python')
518 executable = os.path.join(binary_dir, 'python')
498 return install_svn_hooks(
519 return install_svn_hooks(
499 repo_path, executable=executable, force_create=force)
520 repo_path, executable=executable, force_create=force)
500
521
501 @reraise_safe_exceptions
522 @reraise_safe_exceptions
502 def get_hooks_info(self, wire):
523 def get_hooks_info(self, wire):
503 from vcsserver.hook_utils import (
524 from vcsserver.hook_utils import (
504 get_svn_pre_hook_version, get_svn_post_hook_version)
525 get_svn_pre_hook_version, get_svn_post_hook_version)
505 repo_path = wire['path']
526 repo_path = wire['path']
506 return {
527 return {
507 'pre_version': get_svn_pre_hook_version(repo_path),
528 'pre_version': get_svn_pre_hook_version(repo_path),
508 'post_version': get_svn_post_hook_version(repo_path),
529 'post_version': get_svn_post_hook_version(repo_path),
509 }
530 }
510
531
511
532
512 class SvnDiffer(object):
533 class SvnDiffer(object):
513 """
534 """
514 Utility to create diffs based on difflib and the Subversion api
535 Utility to create diffs based on difflib and the Subversion api
515 """
536 """
516
537
517 binary_content = False
538 binary_content = False
518
539
519 def __init__(
540 def __init__(
520 self, repo, src_rev, src_path, tgt_rev, tgt_path,
541 self, repo, src_rev, src_path, tgt_rev, tgt_path,
521 ignore_whitespace, context):
542 ignore_whitespace, context):
522 self.repo = repo
543 self.repo = repo
523 self.ignore_whitespace = ignore_whitespace
544 self.ignore_whitespace = ignore_whitespace
524 self.context = context
545 self.context = context
525
546
526 fsobj = svn.repos.fs(repo)
547 fsobj = svn.repos.fs(repo)
527
548
528 self.tgt_rev = tgt_rev
549 self.tgt_rev = tgt_rev
529 self.tgt_path = tgt_path or ''
550 self.tgt_path = tgt_path or ''
530 self.tgt_root = svn.fs.revision_root(fsobj, tgt_rev)
551 self.tgt_root = svn.fs.revision_root(fsobj, tgt_rev)
531 self.tgt_kind = svn.fs.check_path(self.tgt_root, self.tgt_path)
552 self.tgt_kind = svn.fs.check_path(self.tgt_root, self.tgt_path)
532
553
533 self.src_rev = src_rev
554 self.src_rev = src_rev
534 self.src_path = src_path or self.tgt_path
555 self.src_path = src_path or self.tgt_path
535 self.src_root = svn.fs.revision_root(fsobj, src_rev)
556 self.src_root = svn.fs.revision_root(fsobj, src_rev)
536 self.src_kind = svn.fs.check_path(self.src_root, self.src_path)
557 self.src_kind = svn.fs.check_path(self.src_root, self.src_path)
537
558
538 self._validate()
559 self._validate()
539
560
540 def _validate(self):
561 def _validate(self):
541 if (self.tgt_kind != svn.core.svn_node_none and
562 if (self.tgt_kind != svn.core.svn_node_none and
542 self.src_kind != svn.core.svn_node_none and
563 self.src_kind != svn.core.svn_node_none and
543 self.src_kind != self.tgt_kind):
564 self.src_kind != self.tgt_kind):
544 # TODO: johbo: proper error handling
565 # TODO: johbo: proper error handling
545 raise Exception(
566 raise Exception(
546 "Source and target are not compatible for diff generation. "
567 "Source and target are not compatible for diff generation. "
547 "Source type: %s, target type: %s" %
568 "Source type: %s, target type: %s" %
548 (self.src_kind, self.tgt_kind))
569 (self.src_kind, self.tgt_kind))
549
570
550 def generate_diff(self):
571 def generate_diff(self):
551 buf = StringIO.StringIO()
572 buf = StringIO.StringIO()
552 if self.tgt_kind == svn.core.svn_node_dir:
573 if self.tgt_kind == svn.core.svn_node_dir:
553 self._generate_dir_diff(buf)
574 self._generate_dir_diff(buf)
554 else:
575 else:
555 self._generate_file_diff(buf)
576 self._generate_file_diff(buf)
556 return buf.getvalue()
577 return buf.getvalue()
557
578
558 def _generate_dir_diff(self, buf):
579 def _generate_dir_diff(self, buf):
559 editor = DiffChangeEditor()
580 editor = DiffChangeEditor()
560 editor_ptr, editor_baton = svn.delta.make_editor(editor)
581 editor_ptr, editor_baton = svn.delta.make_editor(editor)
561 svn.repos.dir_delta2(
582 svn.repos.dir_delta2(
562 self.src_root,
583 self.src_root,
563 self.src_path,
584 self.src_path,
564 '', # src_entry
585 '', # src_entry
565 self.tgt_root,
586 self.tgt_root,
566 self.tgt_path,
587 self.tgt_path,
567 editor_ptr, editor_baton,
588 editor_ptr, editor_baton,
568 authorization_callback_allow_all,
589 authorization_callback_allow_all,
569 False, # text_deltas
590 False, # text_deltas
570 svn.core.svn_depth_infinity, # depth
591 svn.core.svn_depth_infinity, # depth
571 False, # entry_props
592 False, # entry_props
572 False, # ignore_ancestry
593 False, # ignore_ancestry
573 )
594 )
574
595
575 for path, __, change in sorted(editor.changes):
596 for path, __, change in sorted(editor.changes):
576 self._generate_node_diff(
597 self._generate_node_diff(
577 buf, change, path, self.tgt_path, path, self.src_path)
598 buf, change, path, self.tgt_path, path, self.src_path)
578
599
579 def _generate_file_diff(self, buf):
600 def _generate_file_diff(self, buf):
580 change = None
601 change = None
581 if self.src_kind == svn.core.svn_node_none:
602 if self.src_kind == svn.core.svn_node_none:
582 change = "add"
603 change = "add"
583 elif self.tgt_kind == svn.core.svn_node_none:
604 elif self.tgt_kind == svn.core.svn_node_none:
584 change = "delete"
605 change = "delete"
585 tgt_base, tgt_path = vcspath.split(self.tgt_path)
606 tgt_base, tgt_path = vcspath.split(self.tgt_path)
586 src_base, src_path = vcspath.split(self.src_path)
607 src_base, src_path = vcspath.split(self.src_path)
587 self._generate_node_diff(
608 self._generate_node_diff(
588 buf, change, tgt_path, tgt_base, src_path, src_base)
609 buf, change, tgt_path, tgt_base, src_path, src_base)
589
610
590 def _generate_node_diff(
611 def _generate_node_diff(
591 self, buf, change, tgt_path, tgt_base, src_path, src_base):
612 self, buf, change, tgt_path, tgt_base, src_path, src_base):
592
613
593 if self.src_rev == self.tgt_rev and tgt_base == src_base:
614 if self.src_rev == self.tgt_rev and tgt_base == src_base:
594 # makes consistent behaviour with git/hg to return empty diff if
615 # makes consistent behaviour with git/hg to return empty diff if
595 # we compare same revisions
616 # we compare same revisions
596 return
617 return
597
618
598 tgt_full_path = vcspath.join(tgt_base, tgt_path)
619 tgt_full_path = vcspath.join(tgt_base, tgt_path)
599 src_full_path = vcspath.join(src_base, src_path)
620 src_full_path = vcspath.join(src_base, src_path)
600
621
601 self.binary_content = False
622 self.binary_content = False
602 mime_type = self._get_mime_type(tgt_full_path)
623 mime_type = self._get_mime_type(tgt_full_path)
603
624
604 if mime_type and not mime_type.startswith('text'):
625 if mime_type and not mime_type.startswith('text'):
605 self.binary_content = True
626 self.binary_content = True
606 buf.write("=" * 67 + '\n')
627 buf.write("=" * 67 + '\n')
607 buf.write("Cannot display: file marked as a binary type.\n")
628 buf.write("Cannot display: file marked as a binary type.\n")
608 buf.write("svn:mime-type = %s\n" % mime_type)
629 buf.write("svn:mime-type = %s\n" % mime_type)
609 buf.write("Index: %s\n" % (tgt_path, ))
630 buf.write("Index: %s\n" % (tgt_path, ))
610 buf.write("=" * 67 + '\n')
631 buf.write("=" * 67 + '\n')
611 buf.write("diff --git a/%(tgt_path)s b/%(tgt_path)s\n" % {
632 buf.write("diff --git a/%(tgt_path)s b/%(tgt_path)s\n" % {
612 'tgt_path': tgt_path})
633 'tgt_path': tgt_path})
613
634
614 if change == 'add':
635 if change == 'add':
615 # TODO: johbo: SVN is missing a zero here compared to git
636 # TODO: johbo: SVN is missing a zero here compared to git
616 buf.write("new file mode 10644\n")
637 buf.write("new file mode 10644\n")
617
638
618 #TODO(marcink): intro to binary detection of svn patches
639 #TODO(marcink): intro to binary detection of svn patches
619 # if self.binary_content:
640 # if self.binary_content:
620 # buf.write('GIT binary patch\n')
641 # buf.write('GIT binary patch\n')
621
642
622 buf.write("--- /dev/null\t(revision 0)\n")
643 buf.write("--- /dev/null\t(revision 0)\n")
623 src_lines = []
644 src_lines = []
624 else:
645 else:
625 if change == 'delete':
646 if change == 'delete':
626 buf.write("deleted file mode 10644\n")
647 buf.write("deleted file mode 10644\n")
627
648
628 #TODO(marcink): intro to binary detection of svn patches
649 #TODO(marcink): intro to binary detection of svn patches
629 # if self.binary_content:
650 # if self.binary_content:
630 # buf.write('GIT binary patch\n')
651 # buf.write('GIT binary patch\n')
631
652
632 buf.write("--- a/%s\t(revision %s)\n" % (
653 buf.write("--- a/%s\t(revision %s)\n" % (
633 src_path, self.src_rev))
654 src_path, self.src_rev))
634 src_lines = self._svn_readlines(self.src_root, src_full_path)
655 src_lines = self._svn_readlines(self.src_root, src_full_path)
635
656
636 if change == 'delete':
657 if change == 'delete':
637 buf.write("+++ /dev/null\t(revision %s)\n" % (self.tgt_rev, ))
658 buf.write("+++ /dev/null\t(revision %s)\n" % (self.tgt_rev, ))
638 tgt_lines = []
659 tgt_lines = []
639 else:
660 else:
640 buf.write("+++ b/%s\t(revision %s)\n" % (
661 buf.write("+++ b/%s\t(revision %s)\n" % (
641 tgt_path, self.tgt_rev))
662 tgt_path, self.tgt_rev))
642 tgt_lines = self._svn_readlines(self.tgt_root, tgt_full_path)
663 tgt_lines = self._svn_readlines(self.tgt_root, tgt_full_path)
643
664
644 if not self.binary_content:
665 if not self.binary_content:
645 udiff = svn_diff.unified_diff(
666 udiff = svn_diff.unified_diff(
646 src_lines, tgt_lines, context=self.context,
667 src_lines, tgt_lines, context=self.context,
647 ignore_blank_lines=self.ignore_whitespace,
668 ignore_blank_lines=self.ignore_whitespace,
648 ignore_case=False,
669 ignore_case=False,
649 ignore_space_changes=self.ignore_whitespace)
670 ignore_space_changes=self.ignore_whitespace)
650 buf.writelines(udiff)
671 buf.writelines(udiff)
651
672
652 def _get_mime_type(self, path):
673 def _get_mime_type(self, path):
653 try:
674 try:
654 mime_type = svn.fs.node_prop(
675 mime_type = svn.fs.node_prop(
655 self.tgt_root, path, svn.core.SVN_PROP_MIME_TYPE)
676 self.tgt_root, path, svn.core.SVN_PROP_MIME_TYPE)
656 except svn.core.SubversionException:
677 except svn.core.SubversionException:
657 mime_type = svn.fs.node_prop(
678 mime_type = svn.fs.node_prop(
658 self.src_root, path, svn.core.SVN_PROP_MIME_TYPE)
679 self.src_root, path, svn.core.SVN_PROP_MIME_TYPE)
659 return mime_type
680 return mime_type
660
681
661 def _svn_readlines(self, fs_root, node_path):
682 def _svn_readlines(self, fs_root, node_path):
662 if self.binary_content:
683 if self.binary_content:
663 return []
684 return []
664 node_kind = svn.fs.check_path(fs_root, node_path)
685 node_kind = svn.fs.check_path(fs_root, node_path)
665 if node_kind not in (
686 if node_kind not in (
666 svn.core.svn_node_file, svn.core.svn_node_symlink):
687 svn.core.svn_node_file, svn.core.svn_node_symlink):
667 return []
688 return []
668 content = svn.core.Stream(
689 content = svn.core.Stream(
669 svn.fs.file_contents(fs_root, node_path)).read()
690 svn.fs.file_contents(fs_root, node_path)).read()
670 return content.splitlines(True)
691 return content.splitlines(True)
671
692
672
693
673
694
674 class DiffChangeEditor(svn.delta.Editor):
695 class DiffChangeEditor(svn.delta.Editor):
675 """
696 """
676 Records changes between two given revisions
697 Records changes between two given revisions
677 """
698 """
678
699
679 def __init__(self):
700 def __init__(self):
680 self.changes = []
701 self.changes = []
681
702
682 def delete_entry(self, path, revision, parent_baton, pool=None):
703 def delete_entry(self, path, revision, parent_baton, pool=None):
683 self.changes.append((path, None, 'delete'))
704 self.changes.append((path, None, 'delete'))
684
705
685 def add_file(
706 def add_file(
686 self, path, parent_baton, copyfrom_path, copyfrom_revision,
707 self, path, parent_baton, copyfrom_path, copyfrom_revision,
687 file_pool=None):
708 file_pool=None):
688 self.changes.append((path, 'file', 'add'))
709 self.changes.append((path, 'file', 'add'))
689
710
690 def open_file(self, path, parent_baton, base_revision, file_pool=None):
711 def open_file(self, path, parent_baton, base_revision, file_pool=None):
691 self.changes.append((path, 'file', 'change'))
712 self.changes.append((path, 'file', 'change'))
692
713
693
714
694 def authorization_callback_allow_all(root, path, pool):
715 def authorization_callback_allow_all(root, path, pool):
695 return True
716 return True
696
717
697
718
698 class TxnNodeProcessor(object):
719 class TxnNodeProcessor(object):
699 """
720 """
700 Utility to process the change of one node within a transaction root.
721 Utility to process the change of one node within a transaction root.
701
722
702 It encapsulates the knowledge of how to add, update or remove
723 It encapsulates the knowledge of how to add, update or remove
703 a node for a given transaction root. The purpose is to support the method
724 a node for a given transaction root. The purpose is to support the method
704 `SvnRemote.commit`.
725 `SvnRemote.commit`.
705 """
726 """
706
727
707 def __init__(self, node, txn_root):
728 def __init__(self, node, txn_root):
708 assert isinstance(node['path'], str)
729 assert isinstance(node['path'], str)
709
730
710 self.node = node
731 self.node = node
711 self.txn_root = txn_root
732 self.txn_root = txn_root
712
733
713 def update(self):
734 def update(self):
714 self._ensure_parent_dirs()
735 self._ensure_parent_dirs()
715 self._add_file_if_node_does_not_exist()
736 self._add_file_if_node_does_not_exist()
716 self._update_file_content()
737 self._update_file_content()
717 self._update_file_properties()
738 self._update_file_properties()
718
739
719 def remove(self):
740 def remove(self):
720 svn.fs.delete(self.txn_root, self.node['path'])
741 svn.fs.delete(self.txn_root, self.node['path'])
721 # TODO: Clean up directory if empty
742 # TODO: Clean up directory if empty
722
743
723 def _ensure_parent_dirs(self):
744 def _ensure_parent_dirs(self):
724 curdir = vcspath.dirname(self.node['path'])
745 curdir = vcspath.dirname(self.node['path'])
725 dirs_to_create = []
746 dirs_to_create = []
726 while not self._svn_path_exists(curdir):
747 while not self._svn_path_exists(curdir):
727 dirs_to_create.append(curdir)
748 dirs_to_create.append(curdir)
728 curdir = vcspath.dirname(curdir)
749 curdir = vcspath.dirname(curdir)
729
750
730 for curdir in reversed(dirs_to_create):
751 for curdir in reversed(dirs_to_create):
731 log.debug('Creating missing directory "%s"', curdir)
752 log.debug('Creating missing directory "%s"', curdir)
732 svn.fs.make_dir(self.txn_root, curdir)
753 svn.fs.make_dir(self.txn_root, curdir)
733
754
734 def _svn_path_exists(self, path):
755 def _svn_path_exists(self, path):
735 path_status = svn.fs.check_path(self.txn_root, path)
756 path_status = svn.fs.check_path(self.txn_root, path)
736 return path_status != svn.core.svn_node_none
757 return path_status != svn.core.svn_node_none
737
758
738 def _add_file_if_node_does_not_exist(self):
759 def _add_file_if_node_does_not_exist(self):
739 kind = svn.fs.check_path(self.txn_root, self.node['path'])
760 kind = svn.fs.check_path(self.txn_root, self.node['path'])
740 if kind == svn.core.svn_node_none:
761 if kind == svn.core.svn_node_none:
741 svn.fs.make_file(self.txn_root, self.node['path'])
762 svn.fs.make_file(self.txn_root, self.node['path'])
742
763
743 def _update_file_content(self):
764 def _update_file_content(self):
744 assert isinstance(self.node['content'], str)
765 assert isinstance(self.node['content'], str)
745 handler, baton = svn.fs.apply_textdelta(
766 handler, baton = svn.fs.apply_textdelta(
746 self.txn_root, self.node['path'], None, None)
767 self.txn_root, self.node['path'], None, None)
747 svn.delta.svn_txdelta_send_string(self.node['content'], handler, baton)
768 svn.delta.svn_txdelta_send_string(self.node['content'], handler, baton)
748
769
749 def _update_file_properties(self):
770 def _update_file_properties(self):
750 properties = self.node.get('properties', {})
771 properties = self.node.get('properties', {})
751 for key, value in properties.iteritems():
772 for key, value in properties.iteritems():
752 svn.fs.change_node_prop(
773 svn.fs.change_node_prop(
753 self.txn_root, self.node['path'], key, value)
774 self.txn_root, self.node['path'], key, value)
754
775
755
776
756 def apr_time_t(timestamp):
777 def apr_time_t(timestamp):
757 """
778 """
758 Convert a Python timestamp into APR timestamp type apr_time_t
779 Convert a Python timestamp into APR timestamp type apr_time_t
759 """
780 """
760 return timestamp * 1E6
781 return timestamp * 1E6
761
782
762
783
763 def svn_opt_revision_value_t(num):
784 def svn_opt_revision_value_t(num):
764 """
785 """
765 Put `num` into a `svn_opt_revision_value_t` structure.
786 Put `num` into a `svn_opt_revision_value_t` structure.
766 """
787 """
767 value = svn.core.svn_opt_revision_value_t()
788 value = svn.core.svn_opt_revision_value_t()
768 value.number = num
789 value.number = num
769 revision = svn.core.svn_opt_revision_t()
790 revision = svn.core.svn_opt_revision_t()
770 revision.kind = svn.core.svn_opt_revision_number
791 revision.kind = svn.core.svn_opt_revision_number
771 revision.value = value
792 revision.value = value
772 return revision
793 return revision
@@ -1,160 +1,160 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2019 RhodeCode GmbH
2 # Copyright (C) 2014-2019 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import inspect
18 import inspect
19
19
20 import pytest
20 import pytest
21 import dulwich.errors
21 import dulwich.errors
22 from mock import Mock, patch
22 from mock import Mock, patch
23
23
24 from vcsserver import git
24 from vcsserver import git
25
25
26
26
27 SAMPLE_REFS = {
27 SAMPLE_REFS = {
28 'HEAD': 'fd627b9e0dd80b47be81af07c4a98518244ed2f7',
28 'HEAD': 'fd627b9e0dd80b47be81af07c4a98518244ed2f7',
29 'refs/tags/v0.1.9': '341d28f0eec5ddf0b6b77871e13c2bbd6bec685c',
29 'refs/tags/v0.1.9': '341d28f0eec5ddf0b6b77871e13c2bbd6bec685c',
30 'refs/tags/v0.1.8': '74ebce002c088b8a5ecf40073db09375515ecd68',
30 'refs/tags/v0.1.8': '74ebce002c088b8a5ecf40073db09375515ecd68',
31 'refs/tags/v0.1.1': 'e6ea6d16e2f26250124a1f4b4fe37a912f9d86a0',
31 'refs/tags/v0.1.1': 'e6ea6d16e2f26250124a1f4b4fe37a912f9d86a0',
32 'refs/tags/v0.1.3': '5a3a8fb005554692b16e21dee62bf02667d8dc3e',
32 'refs/tags/v0.1.3': '5a3a8fb005554692b16e21dee62bf02667d8dc3e',
33 }
33 }
34
34
35
35
36 @pytest.fixture
36 @pytest.fixture
37 def git_remote():
37 def git_remote():
38 """
38 """
39 A GitRemote instance with a mock factory.
39 A GitRemote instance with a mock factory.
40 """
40 """
41 factory = Mock()
41 factory = Mock()
42 remote = git.GitRemote(factory)
42 remote = git.GitRemote(factory)
43 return remote
43 return remote
44
44
45
45
46 def test_discover_git_version(git_remote):
46 def test_discover_git_version(git_remote):
47 version = git_remote.discover_git_version()
47 version = git_remote.discover_git_version()
48 assert version
48 assert version
49
49
50
50
51 class TestGitFetch(object):
51 class TestGitFetch(object):
52 def setup(self):
52 def setup(self):
53 self.mock_repo = Mock()
53 self.mock_repo = Mock()
54 factory = Mock()
54 factory = Mock()
55 factory.repo = Mock(return_value=self.mock_repo)
55 factory.repo = Mock(return_value=self.mock_repo)
56 self.remote_git = git.GitRemote(factory)
56 self.remote_git = git.GitRemote(factory)
57
57
58 def test_fetches_all_when_no_commit_ids_specified(self):
58 def test_fetches_all_when_no_commit_ids_specified(self):
59 def side_effect(determine_wants, *args, **kwargs):
59 def side_effect(determine_wants, *args, **kwargs):
60 determine_wants(SAMPLE_REFS)
60 determine_wants(SAMPLE_REFS)
61
61
62 with patch('dulwich.client.LocalGitClient.fetch') as mock_fetch:
62 with patch('dulwich.client.LocalGitClient.fetch') as mock_fetch:
63 mock_fetch.side_effect = side_effect
63 mock_fetch.side_effect = side_effect
64 self.remote_git.pull(wire=None, url='/tmp/', apply_refs=False)
64 self.remote_git.pull(wire={}, url='/tmp/', apply_refs=False)
65 determine_wants = self.mock_repo.object_store.determine_wants_all
65 determine_wants = self.mock_repo.object_store.determine_wants_all
66 determine_wants.assert_called_once_with(SAMPLE_REFS)
66 determine_wants.assert_called_once_with(SAMPLE_REFS)
67
67
68 def test_fetches_specified_commits(self):
68 def test_fetches_specified_commits(self):
69 selected_refs = {
69 selected_refs = {
70 'refs/tags/v0.1.8': '74ebce002c088b8a5ecf40073db09375515ecd68',
70 'refs/tags/v0.1.8': '74ebce002c088b8a5ecf40073db09375515ecd68',
71 'refs/tags/v0.1.3': '5a3a8fb005554692b16e21dee62bf02667d8dc3e',
71 'refs/tags/v0.1.3': '5a3a8fb005554692b16e21dee62bf02667d8dc3e',
72 }
72 }
73
73
74 def side_effect(determine_wants, *args, **kwargs):
74 def side_effect(determine_wants, *args, **kwargs):
75 result = determine_wants(SAMPLE_REFS)
75 result = determine_wants(SAMPLE_REFS)
76 assert sorted(result) == sorted(selected_refs.values())
76 assert sorted(result) == sorted(selected_refs.values())
77 return result
77 return result
78
78
79 with patch('dulwich.client.LocalGitClient.fetch') as mock_fetch:
79 with patch('dulwich.client.LocalGitClient.fetch') as mock_fetch:
80 mock_fetch.side_effect = side_effect
80 mock_fetch.side_effect = side_effect
81 self.remote_git.pull(
81 self.remote_git.pull(
82 wire=None, url='/tmp/', apply_refs=False,
82 wire={}, url='/tmp/', apply_refs=False,
83 refs=selected_refs.keys())
83 refs=selected_refs.keys())
84 determine_wants = self.mock_repo.object_store.determine_wants_all
84 determine_wants = self.mock_repo.object_store.determine_wants_all
85 assert determine_wants.call_count == 0
85 assert determine_wants.call_count == 0
86
86
87 def test_get_remote_refs(self):
87 def test_get_remote_refs(self):
88 factory = Mock()
88 factory = Mock()
89 remote_git = git.GitRemote(factory)
89 remote_git = git.GitRemote(factory)
90 url = 'http://example.com/test/test.git'
90 url = 'http://example.com/test/test.git'
91 sample_refs = {
91 sample_refs = {
92 'refs/tags/v0.1.8': '74ebce002c088b8a5ecf40073db09375515ecd68',
92 'refs/tags/v0.1.8': '74ebce002c088b8a5ecf40073db09375515ecd68',
93 'refs/tags/v0.1.3': '5a3a8fb005554692b16e21dee62bf02667d8dc3e',
93 'refs/tags/v0.1.3': '5a3a8fb005554692b16e21dee62bf02667d8dc3e',
94 }
94 }
95
95
96 with patch('vcsserver.git.Repo', create=False) as mock_repo:
96 with patch('vcsserver.git.Repo', create=False) as mock_repo:
97 mock_repo().get_refs.return_value = sample_refs
97 mock_repo().get_refs.return_value = sample_refs
98 remote_refs = remote_git.get_remote_refs(wire=None, url=url)
98 remote_refs = remote_git.get_remote_refs(wire={}, url=url)
99 mock_repo().get_refs.assert_called_once_with()
99 mock_repo().get_refs.assert_called_once_with()
100 assert remote_refs == sample_refs
100 assert remote_refs == sample_refs
101
101
102
102
103 class TestReraiseSafeExceptions(object):
103 class TestReraiseSafeExceptions(object):
104
104
105 def test_method_decorated_with_reraise_safe_exceptions(self):
105 def test_method_decorated_with_reraise_safe_exceptions(self):
106 factory = Mock()
106 factory = Mock()
107 git_remote = git.GitRemote(factory)
107 git_remote = git.GitRemote(factory)
108
108
109 def fake_function():
109 def fake_function():
110 return None
110 return None
111
111
112 decorator = git.reraise_safe_exceptions(fake_function)
112 decorator = git.reraise_safe_exceptions(fake_function)
113
113
114 methods = inspect.getmembers(git_remote, predicate=inspect.ismethod)
114 methods = inspect.getmembers(git_remote, predicate=inspect.ismethod)
115 for method_name, method in methods:
115 for method_name, method in methods:
116 if not method_name.startswith('_'):
116 if not method_name.startswith('_'):
117 assert method.im_func.__code__ == decorator.__code__
117 assert method.im_func.__code__ == decorator.__code__
118
118
119 @pytest.mark.parametrize('side_effect, expected_type', [
119 @pytest.mark.parametrize('side_effect, expected_type', [
120 (dulwich.errors.ChecksumMismatch('0000000', 'deadbeef'), 'lookup'),
120 (dulwich.errors.ChecksumMismatch('0000000', 'deadbeef'), 'lookup'),
121 (dulwich.errors.NotCommitError('deadbeef'), 'lookup'),
121 (dulwich.errors.NotCommitError('deadbeef'), 'lookup'),
122 (dulwich.errors.MissingCommitError('deadbeef'), 'lookup'),
122 (dulwich.errors.MissingCommitError('deadbeef'), 'lookup'),
123 (dulwich.errors.ObjectMissing('deadbeef'), 'lookup'),
123 (dulwich.errors.ObjectMissing('deadbeef'), 'lookup'),
124 (dulwich.errors.HangupException(), 'error'),
124 (dulwich.errors.HangupException(), 'error'),
125 (dulwich.errors.UnexpectedCommandError('test-cmd'), 'error'),
125 (dulwich.errors.UnexpectedCommandError('test-cmd'), 'error'),
126 ])
126 ])
127 def test_safe_exceptions_reraised(self, side_effect, expected_type):
127 def test_safe_exceptions_reraised(self, side_effect, expected_type):
128 @git.reraise_safe_exceptions
128 @git.reraise_safe_exceptions
129 def fake_method():
129 def fake_method():
130 raise side_effect
130 raise side_effect
131
131
132 with pytest.raises(Exception) as exc_info:
132 with pytest.raises(Exception) as exc_info:
133 fake_method()
133 fake_method()
134 assert type(exc_info.value) == Exception
134 assert type(exc_info.value) == Exception
135 assert exc_info.value._vcs_kind == expected_type
135 assert exc_info.value._vcs_kind == expected_type
136
136
137
137
138 class TestDulwichRepoWrapper(object):
138 class TestDulwichRepoWrapper(object):
139 def test_calls_close_on_delete(self):
139 def test_calls_close_on_delete(self):
140 isdir_patcher = patch('dulwich.repo.os.path.isdir', return_value=True)
140 isdir_patcher = patch('dulwich.repo.os.path.isdir', return_value=True)
141 with isdir_patcher:
141 with isdir_patcher:
142 repo = git.Repo('/tmp/abcde')
142 repo = git.Repo('/tmp/abcde')
143 with patch.object(git.DulwichRepo, 'close') as close_mock:
143 with patch.object(git.DulwichRepo, 'close') as close_mock:
144 del repo
144 del repo
145 close_mock.assert_called_once_with()
145 close_mock.assert_called_once_with()
146
146
147
147
148 class TestGitFactory(object):
148 class TestGitFactory(object):
149 def test_create_repo_returns_dulwich_wrapper(self):
149 def test_create_repo_returns_dulwich_wrapper(self):
150
150
151 with patch('vcsserver.lib.rc_cache.region_meta.dogpile_cache_regions') as mock:
151 with patch('vcsserver.lib.rc_cache.region_meta.dogpile_cache_regions') as mock:
152 mock.side_effect = {'repo_objects': ''}
152 mock.side_effect = {'repo_objects': ''}
153 factory = git.GitFactory()
153 factory = git.GitFactory()
154 wire = {
154 wire = {
155 'path': '/tmp/abcde'
155 'path': '/tmp/abcde'
156 }
156 }
157 isdir_patcher = patch('dulwich.repo.os.path.isdir', return_value=True)
157 isdir_patcher = patch('dulwich.repo.os.path.isdir', return_value=True)
158 with isdir_patcher:
158 with isdir_patcher:
159 result = factory._create_repo(wire, True)
159 result = factory._create_repo(wire, True)
160 assert isinstance(result, git.Repo)
160 assert isinstance(result, git.Repo)
@@ -1,127 +1,108 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2019 RhodeCode GmbH
2 # Copyright (C) 2014-2019 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import inspect
18 import inspect
19 import sys
19 import sys
20 import traceback
20 import traceback
21
21
22 import pytest
22 import pytest
23 from mercurial.error import LookupError
23 from mercurial.error import LookupError
24 from mock import Mock, MagicMock, patch
24 from mock import Mock, MagicMock, patch
25
25
26 from vcsserver import exceptions, hg, hgcompat
26 from vcsserver import exceptions, hg, hgcompat
27
27
28
28
29 class TestHGLookup(object):
30 def setup(self):
31 self.mock_repo = MagicMock()
32 self.mock_repo.__getitem__.side_effect = LookupError(
33 'revision_or_commit_id', 'index', 'message')
34 factory = Mock()
35 factory.repo = Mock(return_value=self.mock_repo)
36 self.remote_hg = hg.HgRemote(factory)
37
38 def test_fail_lookup_hg(self):
39 with pytest.raises(Exception) as exc_info:
40 self.remote_hg.lookup(
41 wire=None, revision='revision_or_commit_id', both=True)
42
43 assert exc_info.value._vcs_kind == 'lookup'
44 assert 'revision_or_commit_id' in exc_info.value.args
45
46
47 class TestDiff(object):
29 class TestDiff(object):
48 def test_raising_safe_exception_when_lookup_failed(self):
30 def test_raising_safe_exception_when_lookup_failed(self):
49 repo = Mock()
31
50 factory = Mock()
32 factory = Mock()
51 factory.repo = Mock(return_value=repo)
52 hg_remote = hg.HgRemote(factory)
33 hg_remote = hg.HgRemote(factory)
53 with patch('mercurial.patch.diff') as diff_mock:
34 with patch('mercurial.patch.diff') as diff_mock:
54 diff_mock.side_effect = LookupError(
35 diff_mock.side_effect = LookupError(
55 'deadbeef', 'index', 'message')
36 'deadbeef', 'index', 'message')
56 with pytest.raises(Exception) as exc_info:
37 with pytest.raises(Exception) as exc_info:
57 hg_remote.diff(
38 hg_remote.diff(
58 wire=None, rev1='deadbeef', rev2='deadbee1',
39 wire={}, rev1='deadbeef', rev2='deadbee1',
59 file_filter=None, opt_git=True, opt_ignorews=True,
40 file_filter=None, opt_git=True, opt_ignorews=True,
60 context=3)
41 context=3)
61 assert type(exc_info.value) == Exception
42 assert type(exc_info.value) == Exception
62 assert exc_info.value._vcs_kind == 'lookup'
43 assert exc_info.value._vcs_kind == 'lookup'
63
44
64
45
65 class TestReraiseSafeExceptions(object):
46 class TestReraiseSafeExceptions(object):
66 def test_method_decorated_with_reraise_safe_exceptions(self):
47 def test_method_decorated_with_reraise_safe_exceptions(self):
67 factory = Mock()
48 factory = Mock()
68 hg_remote = hg.HgRemote(factory)
49 hg_remote = hg.HgRemote(factory)
69 methods = inspect.getmembers(hg_remote, predicate=inspect.ismethod)
50 methods = inspect.getmembers(hg_remote, predicate=inspect.ismethod)
70 decorator = hg.reraise_safe_exceptions(None)
51 decorator = hg.reraise_safe_exceptions(None)
71 for method_name, method in methods:
52 for method_name, method in methods:
72 if not method_name.startswith('_'):
53 if not method_name.startswith('_'):
73 assert method.im_func.__code__ == decorator.__code__
54 assert method.im_func.__code__ == decorator.__code__
74
55
75 @pytest.mark.parametrize('side_effect, expected_type', [
56 @pytest.mark.parametrize('side_effect, expected_type', [
76 (hgcompat.Abort(), 'abort'),
57 (hgcompat.Abort(), 'abort'),
77 (hgcompat.InterventionRequired(), 'abort'),
58 (hgcompat.InterventionRequired(), 'abort'),
78 (hgcompat.RepoLookupError(), 'lookup'),
59 (hgcompat.RepoLookupError(), 'lookup'),
79 (hgcompat.LookupError('deadbeef', 'index', 'message'), 'lookup'),
60 (hgcompat.LookupError('deadbeef', 'index', 'message'), 'lookup'),
80 (hgcompat.RepoError(), 'error'),
61 (hgcompat.RepoError(), 'error'),
81 (hgcompat.RequirementError(), 'requirement'),
62 (hgcompat.RequirementError(), 'requirement'),
82 ])
63 ])
83 def test_safe_exceptions_reraised(self, side_effect, expected_type):
64 def test_safe_exceptions_reraised(self, side_effect, expected_type):
84 @hg.reraise_safe_exceptions
65 @hg.reraise_safe_exceptions
85 def fake_method():
66 def fake_method():
86 raise side_effect
67 raise side_effect
87
68
88 with pytest.raises(Exception) as exc_info:
69 with pytest.raises(Exception) as exc_info:
89 fake_method()
70 fake_method()
90 assert type(exc_info.value) == Exception
71 assert type(exc_info.value) == Exception
91 assert exc_info.value._vcs_kind == expected_type
72 assert exc_info.value._vcs_kind == expected_type
92
73
93 def test_keeps_original_traceback(self):
74 def test_keeps_original_traceback(self):
94 @hg.reraise_safe_exceptions
75 @hg.reraise_safe_exceptions
95 def fake_method():
76 def fake_method():
96 try:
77 try:
97 raise hgcompat.Abort()
78 raise hgcompat.Abort()
98 except:
79 except:
99 self.original_traceback = traceback.format_tb(
80 self.original_traceback = traceback.format_tb(
100 sys.exc_info()[2])
81 sys.exc_info()[2])
101 raise
82 raise
102
83
103 try:
84 try:
104 fake_method()
85 fake_method()
105 except Exception:
86 except Exception:
106 new_traceback = traceback.format_tb(sys.exc_info()[2])
87 new_traceback = traceback.format_tb(sys.exc_info()[2])
107
88
108 new_traceback_tail = new_traceback[-len(self.original_traceback):]
89 new_traceback_tail = new_traceback[-len(self.original_traceback):]
109 assert new_traceback_tail == self.original_traceback
90 assert new_traceback_tail == self.original_traceback
110
91
111 def test_maps_unknow_exceptions_to_unhandled(self):
92 def test_maps_unknow_exceptions_to_unhandled(self):
112 @hg.reraise_safe_exceptions
93 @hg.reraise_safe_exceptions
113 def stub_method():
94 def stub_method():
114 raise ValueError('stub')
95 raise ValueError('stub')
115
96
116 with pytest.raises(Exception) as exc_info:
97 with pytest.raises(Exception) as exc_info:
117 stub_method()
98 stub_method()
118 assert exc_info.value._vcs_kind == 'unhandled'
99 assert exc_info.value._vcs_kind == 'unhandled'
119
100
120 def test_does_not_map_known_exceptions(self):
101 def test_does_not_map_known_exceptions(self):
121 @hg.reraise_safe_exceptions
102 @hg.reraise_safe_exceptions
122 def stub_method():
103 def stub_method():
123 raise exceptions.LookupException()('stub')
104 raise exceptions.LookupException()('stub')
124
105
125 with pytest.raises(Exception) as exc_info:
106 with pytest.raises(Exception) as exc_info:
126 stub_method()
107 stub_method()
127 assert exc_info.value._vcs_kind == 'lookup'
108 assert exc_info.value._vcs_kind == 'lookup'
@@ -1,82 +1,87 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2019 RhodeCode GmbH
2 # Copyright (C) 2014-2019 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import io
18 import io
19 import mock
19 import mock
20 import pytest
20 import pytest
21 import sys
21 import sys
22
22
23
23
24 class MockPopen(object):
24 class MockPopen(object):
25 def __init__(self, stderr):
25 def __init__(self, stderr):
26 self.stdout = io.BytesIO('')
26 self.stdout = io.BytesIO('')
27 self.stderr = io.BytesIO(stderr)
27 self.stderr = io.BytesIO(stderr)
28 self.returncode = 1
28 self.returncode = 1
29
29
30 def wait(self):
30 def wait(self):
31 pass
31 pass
32
32
33
33
34 INVALID_CERTIFICATE_STDERR = '\n'.join([
34 INVALID_CERTIFICATE_STDERR = '\n'.join([
35 'svnrdump: E230001: Unable to connect to a repository at URL url',
35 'svnrdump: E230001: Unable to connect to a repository at URL url',
36 'svnrdump: E230001: Server SSL certificate verification failed: issuer is not trusted',
36 'svnrdump: E230001: Server SSL certificate verification failed: issuer is not trusted',
37 ])
37 ])
38
38
39
39
40 @pytest.mark.parametrize('stderr,expected_reason', [
40 @pytest.mark.parametrize('stderr,expected_reason', [
41 (INVALID_CERTIFICATE_STDERR, 'INVALID_CERTIFICATE'),
41 (INVALID_CERTIFICATE_STDERR, 'INVALID_CERTIFICATE'),
42 ('svnrdump: E123456', 'UNKNOWN:svnrdump: E123456'),
42 ('svnrdump: E123456', 'UNKNOWN:svnrdump: E123456'),
43 ], ids=['invalid-cert-stderr', 'svnrdump-err-123456'])
43 ], ids=['invalid-cert-stderr', 'svnrdump-err-123456'])
44 @pytest.mark.xfail(sys.platform == "cygwin",
44 @pytest.mark.xfail(sys.platform == "cygwin",
45 reason="SVN not packaged for Cygwin")
45 reason="SVN not packaged for Cygwin")
46 def test_import_remote_repository_certificate_error(stderr, expected_reason):
46 def test_import_remote_repository_certificate_error(stderr, expected_reason):
47 from vcsserver import svn
47 from vcsserver import svn
48 factory = mock.Mock()
49 factory.repo = mock.Mock(return_value=mock.Mock())
48
50
49 remote = svn.SvnRemote(None)
51 remote = svn.SvnRemote(factory)
50 remote.is_path_valid_repository = lambda wire, path: True
52 remote.is_path_valid_repository = lambda wire, path: True
51
53
52 with mock.patch('subprocess.Popen',
54 with mock.patch('subprocess.Popen',
53 return_value=MockPopen(stderr)):
55 return_value=MockPopen(stderr)):
54 with pytest.raises(Exception) as excinfo:
56 with pytest.raises(Exception) as excinfo:
55 remote.import_remote_repository({'path': 'path'}, 'url')
57 remote.import_remote_repository({'path': 'path'}, 'url')
56
58
57 expected_error_args = (
59 expected_error_args = (
58 'Failed to dump the remote repository from url. Reason:{}'.format(expected_reason),)
60 'Failed to dump the remote repository from url. Reason:{}'.format(expected_reason),)
59
61
60 assert excinfo.value.args == expected_error_args
62 assert excinfo.value.args == expected_error_args
61
63
62
64
63 def test_svn_libraries_can_be_imported():
65 def test_svn_libraries_can_be_imported():
64 import svn
66 import svn
65 import svn.client
67 import svn.client
66 assert svn.client is not None
68 assert svn.client is not None
67
69
68
70
69 @pytest.mark.parametrize('example_url, parts', [
71 @pytest.mark.parametrize('example_url, parts', [
70 ('http://server.com', (None, None, 'http://server.com')),
72 ('http://server.com', (None, None, 'http://server.com')),
71 ('http://user@server.com', ('user', None, 'http://user@server.com')),
73 ('http://user@server.com', ('user', None, 'http://user@server.com')),
72 ('http://user:pass@server.com', ('user', 'pass', 'http://user:pass@server.com')),
74 ('http://user:pass@server.com', ('user', 'pass', 'http://user:pass@server.com')),
73 ('<script>', (None, None, '<script>')),
75 ('<script>', (None, None, '<script>')),
74 ('http://', (None, None, 'http://')),
76 ('http://', (None, None, 'http://')),
75 ])
77 ])
76 def test_username_password_extraction_from_url(example_url, parts):
78 def test_username_password_extraction_from_url(example_url, parts):
77 from vcsserver import svn
79 from vcsserver import svn
78
80
79 remote = svn.SvnRemote(None)
81 factory = mock.Mock()
82 factory.repo = mock.Mock(return_value=mock.Mock())
83
84 remote = svn.SvnRemote(factory)
80 remote.is_path_valid_repository = lambda wire, path: True
85 remote.is_path_valid_repository = lambda wire, path: True
81
86
82 assert remote.get_url_and_credentials(example_url) == parts
87 assert remote.get_url_and_credentials(example_url) == parts
General Comments 0
You need to be logged in to leave comments. Login now