##// END OF EJS Templates
repositories: implemented faster dedicated checks for empty repositories
marcink -
r698:65b1b84c default
parent child
Show More
@@ -1,742 +1,751
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2019 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17 import collections
18 18 import logging
19 19 import os
20 20 import posixpath as vcspath
21 21 import re
22 22 import stat
23 23 import traceback
24 24 import urllib
25 25 import urllib2
26 26 from functools import wraps
27 27
28 28 import more_itertools
29 29 from dulwich import index, objects
30 30 from dulwich.client import HttpGitClient, LocalGitClient
31 31 from dulwich.errors import (
32 32 NotGitRepository, ChecksumMismatch, WrongObjectException,
33 33 MissingCommitError, ObjectMissing, HangupException,
34 34 UnexpectedCommandError)
35 35 from dulwich.repo import Repo as DulwichRepo, Tag
36 36 from dulwich.server import update_server_info
37 37
38 38 from vcsserver import exceptions, settings, subprocessio
39 39 from vcsserver.utils import safe_str
40 40 from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original
41 41 from vcsserver.hgcompat import (
42 42 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
43 43 from vcsserver.git_lfs.lib import LFSOidStore
44 44
45 45 DIR_STAT = stat.S_IFDIR
46 46 FILE_MODE = stat.S_IFMT
47 47 GIT_LINK = objects.S_IFGITLINK
48 48
49 49 log = logging.getLogger(__name__)
50 50
51 51
52 52 def reraise_safe_exceptions(func):
53 53 """Converts Dulwich exceptions to something neutral."""
54 54 @wraps(func)
55 55 def wrapper(*args, **kwargs):
56 56 try:
57 57 return func(*args, **kwargs)
58 58 except (ChecksumMismatch, WrongObjectException, MissingCommitError,
59 59 ObjectMissing) as e:
60 60 exc = exceptions.LookupException(e)
61 61 raise exc(e)
62 62 except (HangupException, UnexpectedCommandError) as e:
63 63 exc = exceptions.VcsException(e)
64 64 raise exc(e)
65 65 except Exception as e:
66 66 # NOTE(marcink): becuase of how dulwich handles some exceptions
67 67 # (KeyError on empty repos), we cannot track this and catch all
68 68 # exceptions, it's an exceptions from other handlers
69 69 #if not hasattr(e, '_vcs_kind'):
70 70 #log.exception("Unhandled exception in git remote call")
71 71 #raise_from_original(exceptions.UnhandledException)
72 72 raise
73 73 return wrapper
74 74
75 75
76 76 class Repo(DulwichRepo):
77 77 """
78 78 A wrapper for dulwich Repo class.
79 79
80 80 Since dulwich is sometimes keeping .idx file descriptors open, it leads to
81 81 "Too many open files" error. We need to close all opened file descriptors
82 82 once the repo object is destroyed.
83 83
84 84 TODO: mikhail: please check if we need this wrapper after updating dulwich
85 85 to 0.12.0 +
86 86 """
87 87 def __del__(self):
88 88 if hasattr(self, 'object_store'):
89 89 self.close()
90 90
91 91
92 92 class GitFactory(RepoFactory):
93 93 repo_type = 'git'
94 94
95 95 def _create_repo(self, wire, create):
96 96 repo_path = str_to_dulwich(wire['path'])
97 97 return Repo(repo_path)
98 98
99 99
100 100 class GitRemote(object):
101 101
102 102 def __init__(self, factory):
103 103 self._factory = factory
104 104 self.peeled_ref_marker = '^{}'
105 105 self._bulk_methods = {
106 106 "author": self.commit_attribute,
107 107 "date": self.get_object_attrs,
108 108 "message": self.commit_attribute,
109 109 "parents": self.commit_attribute,
110 110 "_commit": self.revision,
111 111 }
112 112
113 113 def _wire_to_config(self, wire):
114 114 if 'config' in wire:
115 115 return dict([(x[0] + '_' + x[1], x[2]) for x in wire['config']])
116 116 return {}
117 117
118 118 def _assign_ref(self, wire, ref, commit_id):
119 119 repo = self._factory.repo(wire)
120 120 repo[ref] = commit_id
121 121
122 122 def _remote_conf(self, config):
123 123 params = [
124 124 '-c', 'core.askpass=""',
125 125 ]
126 126 ssl_cert_dir = config.get('vcs_ssl_dir')
127 127 if ssl_cert_dir:
128 128 params.extend(['-c', 'http.sslCAinfo={}'.format(ssl_cert_dir)])
129 129 return params
130 130
131 131 @reraise_safe_exceptions
132 def is_empty(self, wire):
133 repo = self._factory.repo(wire)
134 try:
135 return not repo.head()
136 except Exception:
137 log.exception("failed to read object_store")
138 return True
139
140 @reraise_safe_exceptions
132 141 def add_object(self, wire, content):
133 142 repo = self._factory.repo(wire)
134 143 blob = objects.Blob()
135 144 blob.set_raw_string(content)
136 145 repo.object_store.add_object(blob)
137 146 return blob.id
138 147
139 148 @reraise_safe_exceptions
140 149 def assert_correct_path(self, wire):
141 150 path = wire.get('path')
142 151 try:
143 152 self._factory.repo(wire)
144 153 except NotGitRepository as e:
145 154 tb = traceback.format_exc()
146 155 log.debug("Invalid Git path `%s`, tb: %s", path, tb)
147 156 return False
148 157
149 158 return True
150 159
151 160 @reraise_safe_exceptions
152 161 def bare(self, wire):
153 162 repo = self._factory.repo(wire)
154 163 return repo.bare
155 164
156 165 @reraise_safe_exceptions
157 166 def blob_as_pretty_string(self, wire, sha):
158 167 repo = self._factory.repo(wire)
159 168 return repo[sha].as_pretty_string()
160 169
161 170 @reraise_safe_exceptions
162 171 def blob_raw_length(self, wire, sha):
163 172 repo = self._factory.repo(wire)
164 173 blob = repo[sha]
165 174 return blob.raw_length()
166 175
167 176 def _parse_lfs_pointer(self, raw_content):
168 177
169 178 spec_string = 'version https://git-lfs.github.com/spec'
170 179 if raw_content and raw_content.startswith(spec_string):
171 180 pattern = re.compile(r"""
172 181 (?:\n)?
173 182 ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
174 183 ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
175 184 ^size[ ](?P<oid_size>[0-9]+)\n
176 185 (?:\n)?
177 186 """, re.VERBOSE | re.MULTILINE)
178 187 match = pattern.match(raw_content)
179 188 if match:
180 189 return match.groupdict()
181 190
182 191 return {}
183 192
184 193 @reraise_safe_exceptions
185 194 def is_large_file(self, wire, sha):
186 195 repo = self._factory.repo(wire)
187 196 blob = repo[sha]
188 197 return self._parse_lfs_pointer(blob.as_raw_string())
189 198
190 199 @reraise_safe_exceptions
191 200 def in_largefiles_store(self, wire, oid):
192 201 repo = self._factory.repo(wire)
193 202 conf = self._wire_to_config(wire)
194 203
195 204 store_location = conf.get('vcs_git_lfs_store_location')
196 205 if store_location:
197 206 repo_name = repo.path
198 207 store = LFSOidStore(
199 208 oid=oid, repo=repo_name, store_location=store_location)
200 209 return store.has_oid()
201 210
202 211 return False
203 212
204 213 @reraise_safe_exceptions
205 214 def store_path(self, wire, oid):
206 215 repo = self._factory.repo(wire)
207 216 conf = self._wire_to_config(wire)
208 217
209 218 store_location = conf.get('vcs_git_lfs_store_location')
210 219 if store_location:
211 220 repo_name = repo.path
212 221 store = LFSOidStore(
213 222 oid=oid, repo=repo_name, store_location=store_location)
214 223 return store.oid_path
215 224 raise ValueError('Unable to fetch oid with path {}'.format(oid))
216 225
217 226 @reraise_safe_exceptions
218 227 def bulk_request(self, wire, rev, pre_load):
219 228 result = {}
220 229 for attr in pre_load:
221 230 try:
222 231 method = self._bulk_methods[attr]
223 232 args = [wire, rev]
224 233 if attr == "date":
225 234 args.extend(["commit_time", "commit_timezone"])
226 235 elif attr in ["author", "message", "parents"]:
227 236 args.append(attr)
228 237 result[attr] = method(*args)
229 238 except KeyError as e:
230 239 raise exceptions.VcsException(e)(
231 240 "Unknown bulk attribute: %s" % attr)
232 241 return result
233 242
234 243 def _build_opener(self, url):
235 244 handlers = []
236 245 url_obj = url_parser(url)
237 246 _, authinfo = url_obj.authinfo()
238 247
239 248 if authinfo:
240 249 # create a password manager
241 250 passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
242 251 passmgr.add_password(*authinfo)
243 252
244 253 handlers.extend((httpbasicauthhandler(passmgr),
245 254 httpdigestauthhandler(passmgr)))
246 255
247 256 return urllib2.build_opener(*handlers)
248 257
249 258 @reraise_safe_exceptions
250 259 def check_url(self, url, config):
251 260 url_obj = url_parser(url)
252 261 test_uri, _ = url_obj.authinfo()
253 262 url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
254 263 url_obj.query = obfuscate_qs(url_obj.query)
255 264 cleaned_uri = str(url_obj)
256 265 log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
257 266
258 267 if not test_uri.endswith('info/refs'):
259 268 test_uri = test_uri.rstrip('/') + '/info/refs'
260 269
261 270 o = self._build_opener(url)
262 271 o.addheaders = [('User-Agent', 'git/1.7.8.0')] # fake some git
263 272
264 273 q = {"service": 'git-upload-pack'}
265 274 qs = '?%s' % urllib.urlencode(q)
266 275 cu = "%s%s" % (test_uri, qs)
267 276 req = urllib2.Request(cu, None, {})
268 277
269 278 try:
270 279 log.debug("Trying to open URL %s", cleaned_uri)
271 280 resp = o.open(req)
272 281 if resp.code != 200:
273 282 raise exceptions.URLError()('Return Code is not 200')
274 283 except Exception as e:
275 284 log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
276 285 # means it cannot be cloned
277 286 raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e))
278 287
279 288 # now detect if it's proper git repo
280 289 gitdata = resp.read()
281 290 if 'service=git-upload-pack' in gitdata:
282 291 pass
283 292 elif re.findall(r'[0-9a-fA-F]{40}\s+refs', gitdata):
284 293 # old style git can return some other format !
285 294 pass
286 295 else:
287 296 raise exceptions.URLError()(
288 297 "url [%s] does not look like an git" % (cleaned_uri,))
289 298
290 299 return True
291 300
292 301 @reraise_safe_exceptions
293 302 def clone(self, wire, url, deferred, valid_refs, update_after_clone):
294 303 # TODO(marcink): deprecate this method. Last i checked we don't use it anymore
295 304 remote_refs = self.pull(wire, url, apply_refs=False)
296 305 repo = self._factory.repo(wire)
297 306 if isinstance(valid_refs, list):
298 307 valid_refs = tuple(valid_refs)
299 308
300 309 for k in remote_refs:
301 310 # only parse heads/tags and skip so called deferred tags
302 311 if k.startswith(valid_refs) and not k.endswith(deferred):
303 312 repo[k] = remote_refs[k]
304 313
305 314 if update_after_clone:
306 315 # we want to checkout HEAD
307 316 repo["HEAD"] = remote_refs["HEAD"]
308 317 index.build_index_from_tree(repo.path, repo.index_path(),
309 318 repo.object_store, repo["HEAD"].tree)
310 319
311 320 # TODO: this is quite complex, check if that can be simplified
312 321 @reraise_safe_exceptions
313 322 def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
314 323 repo = self._factory.repo(wire)
315 324 object_store = repo.object_store
316 325
317 326 # Create tree and populates it with blobs
318 327 commit_tree = commit_tree and repo[commit_tree] or objects.Tree()
319 328
320 329 for node in updated:
321 330 # Compute subdirs if needed
322 331 dirpath, nodename = vcspath.split(node['path'])
323 332 dirnames = map(safe_str, dirpath and dirpath.split('/') or [])
324 333 parent = commit_tree
325 334 ancestors = [('', parent)]
326 335
327 336 # Tries to dig for the deepest existing tree
328 337 while dirnames:
329 338 curdir = dirnames.pop(0)
330 339 try:
331 340 dir_id = parent[curdir][1]
332 341 except KeyError:
333 342 # put curdir back into dirnames and stops
334 343 dirnames.insert(0, curdir)
335 344 break
336 345 else:
337 346 # If found, updates parent
338 347 parent = repo[dir_id]
339 348 ancestors.append((curdir, parent))
340 349 # Now parent is deepest existing tree and we need to create
341 350 # subtrees for dirnames (in reverse order)
342 351 # [this only applies for nodes from added]
343 352 new_trees = []
344 353
345 354 blob = objects.Blob.from_string(node['content'])
346 355
347 356 if dirnames:
348 357 # If there are trees which should be created we need to build
349 358 # them now (in reverse order)
350 359 reversed_dirnames = list(reversed(dirnames))
351 360 curtree = objects.Tree()
352 361 curtree[node['node_path']] = node['mode'], blob.id
353 362 new_trees.append(curtree)
354 363 for dirname in reversed_dirnames[:-1]:
355 364 newtree = objects.Tree()
356 365 newtree[dirname] = (DIR_STAT, curtree.id)
357 366 new_trees.append(newtree)
358 367 curtree = newtree
359 368 parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
360 369 else:
361 370 parent.add(
362 371 name=node['node_path'], mode=node['mode'], hexsha=blob.id)
363 372
364 373 new_trees.append(parent)
365 374 # Update ancestors
366 375 reversed_ancestors = reversed(
367 376 [(a[1], b[1], b[0]) for a, b in zip(ancestors, ancestors[1:])])
368 377 for parent, tree, path in reversed_ancestors:
369 378 parent[path] = (DIR_STAT, tree.id)
370 379 object_store.add_object(tree)
371 380
372 381 object_store.add_object(blob)
373 382 for tree in new_trees:
374 383 object_store.add_object(tree)
375 384
376 385 for node_path in removed:
377 386 paths = node_path.split('/')
378 387 tree = commit_tree
379 388 trees = [tree]
380 389 # Traverse deep into the forest...
381 390 for path in paths:
382 391 try:
383 392 obj = repo[tree[path][1]]
384 393 if isinstance(obj, objects.Tree):
385 394 trees.append(obj)
386 395 tree = obj
387 396 except KeyError:
388 397 break
389 398 # Cut down the blob and all rotten trees on the way back...
390 399 for path, tree in reversed(zip(paths, trees)):
391 400 del tree[path]
392 401 if tree:
393 402 # This tree still has elements - don't remove it or any
394 403 # of it's parents
395 404 break
396 405
397 406 object_store.add_object(commit_tree)
398 407
399 408 # Create commit
400 409 commit = objects.Commit()
401 410 commit.tree = commit_tree.id
402 411 for k, v in commit_data.iteritems():
403 412 setattr(commit, k, v)
404 413 object_store.add_object(commit)
405 414
406 415 ref = 'refs/heads/%s' % branch
407 416 repo.refs[ref] = commit.id
408 417
409 418 return commit.id
410 419
411 420 @reraise_safe_exceptions
412 421 def pull(self, wire, url, apply_refs=True, refs=None, update_after=False):
413 422 if url != 'default' and '://' not in url:
414 423 client = LocalGitClient(url)
415 424 else:
416 425 url_obj = url_parser(url)
417 426 o = self._build_opener(url)
418 427 url, _ = url_obj.authinfo()
419 428 client = HttpGitClient(base_url=url, opener=o)
420 429 repo = self._factory.repo(wire)
421 430
422 431 determine_wants = repo.object_store.determine_wants_all
423 432 if refs:
424 433 def determine_wants_requested(references):
425 434 return [references[r] for r in references if r in refs]
426 435 determine_wants = determine_wants_requested
427 436
428 437 try:
429 438 remote_refs = client.fetch(
430 439 path=url, target=repo, determine_wants=determine_wants)
431 440 except NotGitRepository as e:
432 441 log.warning(
433 442 'Trying to fetch from "%s" failed, not a Git repository.', url)
434 443 # Exception can contain unicode which we convert
435 444 raise exceptions.AbortException(e)(repr(e))
436 445
437 446 # mikhail: client.fetch() returns all the remote refs, but fetches only
438 447 # refs filtered by `determine_wants` function. We need to filter result
439 448 # as well
440 449 if refs:
441 450 remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
442 451
443 452 if apply_refs:
444 453 # TODO: johbo: Needs proper test coverage with a git repository
445 454 # that contains a tag object, so that we would end up with
446 455 # a peeled ref at this point.
447 456 for k in remote_refs:
448 457 if k.endswith(self.peeled_ref_marker):
449 458 log.debug("Skipping peeled reference %s", k)
450 459 continue
451 460 repo[k] = remote_refs[k]
452 461
453 462 if refs and not update_after:
454 463 # mikhail: explicitly set the head to the last ref.
455 464 repo['HEAD'] = remote_refs[refs[-1]]
456 465
457 466 if update_after:
458 467 # we want to checkout HEAD
459 468 repo["HEAD"] = remote_refs["HEAD"]
460 469 index.build_index_from_tree(repo.path, repo.index_path(),
461 470 repo.object_store, repo["HEAD"].tree)
462 471 return remote_refs
463 472
464 473 @reraise_safe_exceptions
465 474 def sync_fetch(self, wire, url, refs=None):
466 475 repo = self._factory.repo(wire)
467 476 if refs and not isinstance(refs, (list, tuple)):
468 477 refs = [refs]
469 478 config = self._wire_to_config(wire)
470 479 # get all remote refs we'll use to fetch later
471 480 output, __ = self.run_git_command(
472 481 wire, ['ls-remote', url], fail_on_stderr=False,
473 482 _copts=self._remote_conf(config),
474 483 extra_env={'GIT_TERMINAL_PROMPT': '0'})
475 484
476 485 remote_refs = collections.OrderedDict()
477 486 fetch_refs = []
478 487
479 488 for ref_line in output.splitlines():
480 489 sha, ref = ref_line.split('\t')
481 490 sha = sha.strip()
482 491 if ref in remote_refs:
483 492 # duplicate, skip
484 493 continue
485 494 if ref.endswith(self.peeled_ref_marker):
486 495 log.debug("Skipping peeled reference %s", ref)
487 496 continue
488 497 # don't sync HEAD
489 498 if ref in ['HEAD']:
490 499 continue
491 500
492 501 remote_refs[ref] = sha
493 502
494 503 if refs and sha in refs:
495 504 # we filter fetch using our specified refs
496 505 fetch_refs.append('{}:{}'.format(ref, ref))
497 506 elif not refs:
498 507 fetch_refs.append('{}:{}'.format(ref, ref))
499 508 log.debug('Finished obtaining fetch refs, total: %s', len(fetch_refs))
500 509 if fetch_refs:
501 510 for chunk in more_itertools.chunked(fetch_refs, 1024 * 4):
502 511 fetch_refs_chunks = list(chunk)
503 512 log.debug('Fetching %s refs from import url', len(fetch_refs_chunks))
504 513 _out, _err = self.run_git_command(
505 514 wire, ['fetch', url, '--force', '--prune', '--'] + fetch_refs_chunks,
506 515 fail_on_stderr=False,
507 516 _copts=self._remote_conf(config),
508 517 extra_env={'GIT_TERMINAL_PROMPT': '0'})
509 518
510 519 return remote_refs
511 520
512 521 @reraise_safe_exceptions
513 522 def sync_push(self, wire, url, refs=None):
514 523 if not self.check_url(url, wire):
515 524 return
516 525 config = self._wire_to_config(wire)
517 526 repo = self._factory.repo(wire)
518 527 self.run_git_command(
519 528 wire, ['push', url, '--mirror'], fail_on_stderr=False,
520 529 _copts=self._remote_conf(config),
521 530 extra_env={'GIT_TERMINAL_PROMPT': '0'})
522 531
523 532 @reraise_safe_exceptions
524 533 def get_remote_refs(self, wire, url):
525 534 repo = Repo(url)
526 535 return repo.get_refs()
527 536
528 537 @reraise_safe_exceptions
529 538 def get_description(self, wire):
530 539 repo = self._factory.repo(wire)
531 540 return repo.get_description()
532 541
533 542 @reraise_safe_exceptions
534 543 def get_missing_revs(self, wire, rev1, rev2, path2):
535 544 repo = self._factory.repo(wire)
536 545 LocalGitClient(thin_packs=False).fetch(path2, repo)
537 546
538 547 wire_remote = wire.copy()
539 548 wire_remote['path'] = path2
540 549 repo_remote = self._factory.repo(wire_remote)
541 550 LocalGitClient(thin_packs=False).fetch(wire["path"], repo_remote)
542 551
543 552 revs = [
544 553 x.commit.id
545 554 for x in repo_remote.get_walker(include=[rev2], exclude=[rev1])]
546 555 return revs
547 556
548 557 @reraise_safe_exceptions
549 558 def get_object(self, wire, sha):
550 559 repo = self._factory.repo(wire)
551 560 obj = repo.get_object(sha)
552 561 commit_id = obj.id
553 562
554 563 if isinstance(obj, Tag):
555 564 commit_id = obj.object[1]
556 565
557 566 return {
558 567 'id': obj.id,
559 568 'type': obj.type_name,
560 569 'commit_id': commit_id
561 570 }
562 571
563 572 @reraise_safe_exceptions
564 573 def get_object_attrs(self, wire, sha, *attrs):
565 574 repo = self._factory.repo(wire)
566 575 obj = repo.get_object(sha)
567 576 return list(getattr(obj, a) for a in attrs)
568 577
569 578 @reraise_safe_exceptions
570 579 def get_refs(self, wire):
571 580 repo = self._factory.repo(wire)
572 581 result = {}
573 582 for ref, sha in repo.refs.as_dict().items():
574 583 peeled_sha = repo.get_peeled(ref)
575 584 result[ref] = peeled_sha
576 585 return result
577 586
578 587 @reraise_safe_exceptions
579 588 def get_refs_path(self, wire):
580 589 repo = self._factory.repo(wire)
581 590 return repo.refs.path
582 591
583 592 @reraise_safe_exceptions
584 593 def head(self, wire, show_exc=True):
585 594 repo = self._factory.repo(wire)
586 595 try:
587 596 return repo.head()
588 597 except Exception:
589 598 if show_exc:
590 599 raise
591 600
592 601 @reraise_safe_exceptions
593 602 def init(self, wire):
594 603 repo_path = str_to_dulwich(wire['path'])
595 604 self.repo = Repo.init(repo_path)
596 605
597 606 @reraise_safe_exceptions
598 607 def init_bare(self, wire):
599 608 repo_path = str_to_dulwich(wire['path'])
600 609 self.repo = Repo.init_bare(repo_path)
601 610
602 611 @reraise_safe_exceptions
603 612 def revision(self, wire, rev):
604 613 repo = self._factory.repo(wire)
605 614 obj = repo[rev]
606 615 obj_data = {
607 616 'id': obj.id,
608 617 }
609 618 try:
610 619 obj_data['tree'] = obj.tree
611 620 except AttributeError:
612 621 pass
613 622 return obj_data
614 623
615 624 @reraise_safe_exceptions
616 625 def commit_attribute(self, wire, rev, attr):
617 626 repo = self._factory.repo(wire)
618 627 obj = repo[rev]
619 628 return getattr(obj, attr)
620 629
621 630 @reraise_safe_exceptions
622 631 def set_refs(self, wire, key, value):
623 632 repo = self._factory.repo(wire)
624 633 repo.refs[key] = value
625 634
626 635 @reraise_safe_exceptions
627 636 def remove_ref(self, wire, key):
628 637 repo = self._factory.repo(wire)
629 638 del repo.refs[key]
630 639
631 640 @reraise_safe_exceptions
632 641 def tree_changes(self, wire, source_id, target_id):
633 642 repo = self._factory.repo(wire)
634 643 source = repo[source_id].tree if source_id else None
635 644 target = repo[target_id].tree
636 645 result = repo.object_store.tree_changes(source, target)
637 646 return list(result)
638 647
639 648 @reraise_safe_exceptions
640 649 def tree_items(self, wire, tree_id):
641 650 repo = self._factory.repo(wire)
642 651 tree = repo[tree_id]
643 652
644 653 result = []
645 654 for item in tree.iteritems():
646 655 item_sha = item.sha
647 656 item_mode = item.mode
648 657
649 658 if FILE_MODE(item_mode) == GIT_LINK:
650 659 item_type = "link"
651 660 else:
652 661 item_type = repo[item_sha].type_name
653 662
654 663 result.append((item.path, item_mode, item_sha, item_type))
655 664 return result
656 665
657 666 @reraise_safe_exceptions
658 667 def update_server_info(self, wire):
659 668 repo = self._factory.repo(wire)
660 669 update_server_info(repo)
661 670
662 671 @reraise_safe_exceptions
663 672 def discover_git_version(self):
664 673 stdout, _ = self.run_git_command(
665 674 {}, ['--version'], _bare=True, _safe=True)
666 675 prefix = 'git version'
667 676 if stdout.startswith(prefix):
668 677 stdout = stdout[len(prefix):]
669 678 return stdout.strip()
670 679
671 680 @reraise_safe_exceptions
672 681 def run_git_command(self, wire, cmd, **opts):
673 682 path = wire.get('path', None)
674 683
675 684 if path and os.path.isdir(path):
676 685 opts['cwd'] = path
677 686
678 687 if '_bare' in opts:
679 688 _copts = []
680 689 del opts['_bare']
681 690 else:
682 691 _copts = ['-c', 'core.quotepath=false', ]
683 692 safe_call = False
684 693 if '_safe' in opts:
685 694 # no exc on failure
686 695 del opts['_safe']
687 696 safe_call = True
688 697
689 698 if '_copts' in opts:
690 699 _copts.extend(opts['_copts'] or [])
691 700 del opts['_copts']
692 701
693 702 gitenv = os.environ.copy()
694 703 gitenv.update(opts.pop('extra_env', {}))
695 704 # need to clean fix GIT_DIR !
696 705 if 'GIT_DIR' in gitenv:
697 706 del gitenv['GIT_DIR']
698 707 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
699 708 gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1'
700 709
701 710 cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
702 711 _opts = {'env': gitenv, 'shell': False}
703 712
704 713 try:
705 714 _opts.update(opts)
706 715 p = subprocessio.SubprocessIOChunker(cmd, **_opts)
707 716
708 717 return ''.join(p), ''.join(p.error)
709 718 except (EnvironmentError, OSError) as err:
710 719 cmd = ' '.join(cmd) # human friendly CMD
711 720 tb_err = ("Couldn't run git command (%s).\n"
712 721 "Original error was:%s\n"
713 722 "Call options:%s\n"
714 723 % (cmd, err, _opts))
715 724 log.exception(tb_err)
716 725 if safe_call:
717 726 return '', err
718 727 else:
719 728 raise exceptions.VcsException()(tb_err)
720 729
721 730 @reraise_safe_exceptions
722 731 def install_hooks(self, wire, force=False):
723 732 from vcsserver.hook_utils import install_git_hooks
724 733 repo = self._factory.repo(wire)
725 734 return install_git_hooks(repo.path, repo.bare, force_create=force)
726 735
727 736 @reraise_safe_exceptions
728 737 def get_hooks_info(self, wire):
729 738 from vcsserver.hook_utils import (
730 739 get_git_pre_hook_version, get_git_post_hook_version)
731 740 repo = self._factory.repo(wire)
732 741 return {
733 742 'pre_version': get_git_pre_hook_version(repo.path, repo.bare),
734 743 'post_version': get_git_post_hook_version(repo.path, repo.bare),
735 744 }
736 745
737 746
738 747 def str_to_dulwich(value):
739 748 """
740 749 Dulwich 0.10.1a requires `unicode` objects to be passed in.
741 750 """
742 751 return value.decode(settings.WIRE_ENCODING)
@@ -1,846 +1,856
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2019 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import io
19 19 import logging
20 20 import stat
21 21 import urllib
22 22 import urllib2
23 23 import traceback
24 24
25 25 from hgext import largefiles, rebase
26 26 from hgext.strip import strip as hgext_strip
27 27 from mercurial import commands
28 28 from mercurial import unionrepo
29 29 from mercurial import verify
30 30
31 31 import vcsserver
32 32 from vcsserver import exceptions
33 33 from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original
34 34 from vcsserver.hgcompat import (
35 35 archival, bin, clone, config as hgconfig, diffopts, hex, get_ctx,
36 36 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler,
37 37 makepeer, instance, match, memctx, exchange, memfilectx, nullrev, hg_merge,
38 38 patch, peer, revrange, ui, hg_tag, Abort, LookupError, RepoError,
39 39 RepoLookupError, InterventionRequired, RequirementError)
40 40
41 41 log = logging.getLogger(__name__)
42 42
43 43
44 44 def make_ui_from_config(repo_config):
45 45
46 46 class LoggingUI(ui.ui):
47 47 def status(self, *msg, **opts):
48 48 log.info(' '.join(msg).rstrip('\n'))
49 49 super(LoggingUI, self).status(*msg, **opts)
50 50
51 51 def warn(self, *msg, **opts):
52 52 log.warn(' '.join(msg).rstrip('\n'))
53 53 super(LoggingUI, self).warn(*msg, **opts)
54 54
55 55 def error(self, *msg, **opts):
56 56 log.error(' '.join(msg).rstrip('\n'))
57 57 super(LoggingUI, self).error(*msg, **opts)
58 58
59 59 def note(self, *msg, **opts):
60 60 log.info(' '.join(msg).rstrip('\n'))
61 61 super(LoggingUI, self).note(*msg, **opts)
62 62
63 63 def debug(self, *msg, **opts):
64 64 log.debug(' '.join(msg).rstrip('\n'))
65 65 super(LoggingUI, self).debug(*msg, **opts)
66 66
67 67 baseui = LoggingUI()
68 68
69 69 # clean the baseui object
70 70 baseui._ocfg = hgconfig.config()
71 71 baseui._ucfg = hgconfig.config()
72 72 baseui._tcfg = hgconfig.config()
73 73
74 74 for section, option, value in repo_config:
75 75 baseui.setconfig(section, option, value)
76 76
77 77 # make our hgweb quiet so it doesn't print output
78 78 baseui.setconfig('ui', 'quiet', 'true')
79 79
80 80 baseui.setconfig('ui', 'paginate', 'never')
81 81 # for better Error reporting of Mercurial
82 82 baseui.setconfig('ui', 'message-output', 'stderr')
83 83
84 84 # force mercurial to only use 1 thread, otherwise it may try to set a
85 85 # signal in a non-main thread, thus generating a ValueError.
86 86 baseui.setconfig('worker', 'numcpus', 1)
87 87
88 88 # If there is no config for the largefiles extension, we explicitly disable
89 89 # it here. This overrides settings from repositories hgrc file. Recent
90 90 # mercurial versions enable largefiles in hgrc on clone from largefile
91 91 # repo.
92 92 if not baseui.hasconfig('extensions', 'largefiles'):
93 93 log.debug('Explicitly disable largefiles extension for repo.')
94 94 baseui.setconfig('extensions', 'largefiles', '!')
95 95
96 96 return baseui
97 97
98 98
99 99 def reraise_safe_exceptions(func):
100 100 """Decorator for converting mercurial exceptions to something neutral."""
101 101 def wrapper(*args, **kwargs):
102 102 try:
103 103 return func(*args, **kwargs)
104 104 except (Abort, InterventionRequired) as e:
105 105 raise_from_original(exceptions.AbortException(e))
106 106 except RepoLookupError as e:
107 107 raise_from_original(exceptions.LookupException(e))
108 108 except RequirementError as e:
109 109 raise_from_original(exceptions.RequirementException(e))
110 110 except RepoError as e:
111 111 raise_from_original(exceptions.VcsException(e))
112 112 except LookupError as e:
113 113 raise_from_original(exceptions.LookupException(e))
114 114 except Exception as e:
115 115 if not hasattr(e, '_vcs_kind'):
116 116 log.exception("Unhandled exception in hg remote call")
117 117 raise_from_original(exceptions.UnhandledException(e))
118 118
119 119 raise
120 120 return wrapper
121 121
122 122
123 123 class MercurialFactory(RepoFactory):
124 124 repo_type = 'hg'
125 125
126 126 def _create_config(self, config, hooks=True):
127 127 if not hooks:
128 128 hooks_to_clean = frozenset((
129 129 'changegroup.repo_size', 'preoutgoing.pre_pull',
130 130 'outgoing.pull_logger', 'prechangegroup.pre_push'))
131 131 new_config = []
132 132 for section, option, value in config:
133 133 if section == 'hooks' and option in hooks_to_clean:
134 134 continue
135 135 new_config.append((section, option, value))
136