##// END OF EJS Templates
exceptions: don't report lookup errors as exceptions stored in the exception store....
marcink -
r843:d6e71ccf default
parent child Browse files
Show More
@@ -1,117 +1,121 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2019 RhodeCode GmbH
2 # Copyright (C) 2014-2019 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 """
18 """
19 Special exception handling over the wire.
19 Special exception handling over the wire.
20
20
21 Since we cannot assume that our client is able to import our exception classes,
21 Since we cannot assume that our client is able to import our exception classes,
22 this module provides a "wrapping" mechanism to raise plain exceptions
22 this module provides a "wrapping" mechanism to raise plain exceptions
23 which contain an extra attribute `_vcs_kind` to allow a client to distinguish
23 which contain an extra attribute `_vcs_kind` to allow a client to distinguish
24 different error conditions.
24 different error conditions.
25 """
25 """
26
26
27 from pyramid.httpexceptions import HTTPLocked, HTTPForbidden
27 from pyramid.httpexceptions import HTTPLocked, HTTPForbidden
28
28
29
29
30 def _make_exception(kind, org_exc, *args):
30 def _make_exception(kind, org_exc, *args):
31 """
31 """
32 Prepares a base `Exception` instance to be sent over the wire.
32 Prepares a base `Exception` instance to be sent over the wire.
33
33
34 To give our caller a hint what this is about, it will attach an attribute
34 To give our caller a hint what this is about, it will attach an attribute
35 `_vcs_kind` to the exception.
35 `_vcs_kind` to the exception.
36 """
36 """
37 exc = Exception(*args)
37 exc = Exception(*args)
38 exc._vcs_kind = kind
38 exc._vcs_kind = kind
39 exc._org_exc = org_exc
39 exc._org_exc = org_exc
40 exc._org_exc_tb = getattr(org_exc, '_org_exc_tb', '')
40 exc._org_exc_tb = getattr(org_exc, '_org_exc_tb', '')
41 return exc
41 return exc
42
42
43
43
44 def AbortException(org_exc=None):
44 def AbortException(org_exc=None):
45 def _make_exception_wrapper(*args):
45 def _make_exception_wrapper(*args):
46 return _make_exception('abort', org_exc, *args)
46 return _make_exception('abort', org_exc, *args)
47 return _make_exception_wrapper
47 return _make_exception_wrapper
48
48
49
49
50 def ArchiveException(org_exc=None):
50 def ArchiveException(org_exc=None):
51 def _make_exception_wrapper(*args):
51 def _make_exception_wrapper(*args):
52 return _make_exception('archive', org_exc, *args)
52 return _make_exception('archive', org_exc, *args)
53 return _make_exception_wrapper
53 return _make_exception_wrapper
54
54
55
55
56 def LookupException(org_exc=None):
56 def LookupException(org_exc=None):
57 def _make_exception_wrapper(*args):
57 def _make_exception_wrapper(*args):
58 return _make_exception('lookup', org_exc, *args)
58 return _make_exception('lookup', org_exc, *args)
59 return _make_exception_wrapper
59 return _make_exception_wrapper
60
60
61
61
62 def VcsException(org_exc=None):
62 def VcsException(org_exc=None):
63 def _make_exception_wrapper(*args):
63 def _make_exception_wrapper(*args):
64 return _make_exception('error', org_exc, *args)
64 return _make_exception('error', org_exc, *args)
65 return _make_exception_wrapper
65 return _make_exception_wrapper
66
66
67
67
68 def RepositoryLockedException(org_exc=None):
68 def RepositoryLockedException(org_exc=None):
69 def _make_exception_wrapper(*args):
69 def _make_exception_wrapper(*args):
70 return _make_exception('repo_locked', org_exc, *args)
70 return _make_exception('repo_locked', org_exc, *args)
71 return _make_exception_wrapper
71 return _make_exception_wrapper
72
72
73
73
74 def RepositoryBranchProtectedException(org_exc=None):
74 def RepositoryBranchProtectedException(org_exc=None):
75 def _make_exception_wrapper(*args):
75 def _make_exception_wrapper(*args):
76 return _make_exception('repo_branch_protected', org_exc, *args)
76 return _make_exception('repo_branch_protected', org_exc, *args)
77 return _make_exception_wrapper
77 return _make_exception_wrapper
78
78
79
79
80 def RequirementException(org_exc=None):
80 def RequirementException(org_exc=None):
81 def _make_exception_wrapper(*args):
81 def _make_exception_wrapper(*args):
82 return _make_exception('requirement', org_exc, *args)
82 return _make_exception('requirement', org_exc, *args)
83 return _make_exception_wrapper
83 return _make_exception_wrapper
84
84
85
85
86 def UnhandledException(org_exc=None):
86 def UnhandledException(org_exc=None):
87 def _make_exception_wrapper(*args):
87 def _make_exception_wrapper(*args):
88 return _make_exception('unhandled', org_exc, *args)
88 return _make_exception('unhandled', org_exc, *args)
89 return _make_exception_wrapper
89 return _make_exception_wrapper
90
90
91
91
92 def URLError(org_exc=None):
92 def URLError(org_exc=None):
93 def _make_exception_wrapper(*args):
93 def _make_exception_wrapper(*args):
94 return _make_exception('url_error', org_exc, *args)
94 return _make_exception('url_error', org_exc, *args)
95 return _make_exception_wrapper
95 return _make_exception_wrapper
96
96
97
97
98 def SubrepoMergeException(org_exc=None):
98 def SubrepoMergeException(org_exc=None):
99 def _make_exception_wrapper(*args):
99 def _make_exception_wrapper(*args):
100 return _make_exception('subrepo_merge_error', org_exc, *args)
100 return _make_exception('subrepo_merge_error', org_exc, *args)
101 return _make_exception_wrapper
101 return _make_exception_wrapper
102
102
103
103
104 class HTTPRepoLocked(HTTPLocked):
104 class HTTPRepoLocked(HTTPLocked):
105 """
105 """
106 Subclass of HTTPLocked response that allows to set the title and status
106 Subclass of HTTPLocked response that allows to set the title and status
107 code via constructor arguments.
107 code via constructor arguments.
108 """
108 """
109 def __init__(self, title, status_code=None, **kwargs):
109 def __init__(self, title, status_code=None, **kwargs):
110 self.code = status_code or HTTPLocked.code
110 self.code = status_code or HTTPLocked.code
111 self.title = title
111 self.title = title
112 super(HTTPRepoLocked, self).__init__(**kwargs)
112 super(HTTPRepoLocked, self).__init__(**kwargs)
113
113
114
114
115 class HTTPRepoBranchProtected(HTTPForbidden):
115 class HTTPRepoBranchProtected(HTTPForbidden):
116 def __init__(self, *args, **kwargs):
116 def __init__(self, *args, **kwargs):
117 super(HTTPForbidden, self).__init__(*args, **kwargs)
117 super(HTTPForbidden, self).__init__(*args, **kwargs)
118
119
120 class RefNotFoundException(KeyError):
121 pass
@@ -1,1181 +1,1189 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2019 RhodeCode GmbH
2 # Copyright (C) 2014-2019 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import collections
18 import collections
19 import logging
19 import logging
20 import os
20 import os
21 import posixpath as vcspath
21 import posixpath as vcspath
22 import re
22 import re
23 import stat
23 import stat
24 import traceback
24 import traceback
25 import urllib
25 import urllib
26 import urllib2
26 import urllib2
27 from functools import wraps
27 from functools import wraps
28
28
29 import more_itertools
29 import more_itertools
30 import pygit2
30 import pygit2
31 from pygit2 import Repository as LibGit2Repo
31 from pygit2 import Repository as LibGit2Repo
32 from dulwich import index, objects
32 from dulwich import index, objects
33 from dulwich.client import HttpGitClient, LocalGitClient
33 from dulwich.client import HttpGitClient, LocalGitClient
34 from dulwich.errors import (
34 from dulwich.errors import (
35 NotGitRepository, ChecksumMismatch, WrongObjectException,
35 NotGitRepository, ChecksumMismatch, WrongObjectException,
36 MissingCommitError, ObjectMissing, HangupException,
36 MissingCommitError, ObjectMissing, HangupException,
37 UnexpectedCommandError)
37 UnexpectedCommandError)
38 from dulwich.repo import Repo as DulwichRepo
38 from dulwich.repo import Repo as DulwichRepo
39 from dulwich.server import update_server_info
39 from dulwich.server import update_server_info
40
40
41 from vcsserver import exceptions, settings, subprocessio
41 from vcsserver import exceptions, settings, subprocessio
42 from vcsserver.utils import safe_str, safe_int, safe_unicode
42 from vcsserver.utils import safe_str, safe_int, safe_unicode
43 from vcsserver.base import RepoFactory, obfuscate_qs
43 from vcsserver.base import RepoFactory, obfuscate_qs
44 from vcsserver.hgcompat import (
44 from vcsserver.hgcompat import (
45 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
45 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
46 from vcsserver.git_lfs.lib import LFSOidStore
46 from vcsserver.git_lfs.lib import LFSOidStore
47 from vcsserver.vcs_base import RemoteBase
47 from vcsserver.vcs_base import RemoteBase
48
48
49 DIR_STAT = stat.S_IFDIR
49 DIR_STAT = stat.S_IFDIR
50 FILE_MODE = stat.S_IFMT
50 FILE_MODE = stat.S_IFMT
51 GIT_LINK = objects.S_IFGITLINK
51 GIT_LINK = objects.S_IFGITLINK
52 PEELED_REF_MARKER = '^{}'
52 PEELED_REF_MARKER = '^{}'
53
53
54
54
55 log = logging.getLogger(__name__)
55 log = logging.getLogger(__name__)
56
56
57
57
58 def str_to_dulwich(value):
58 def str_to_dulwich(value):
59 """
59 """
60 Dulwich 0.10.1a requires `unicode` objects to be passed in.
60 Dulwich 0.10.1a requires `unicode` objects to be passed in.
61 """
61 """
62 return value.decode(settings.WIRE_ENCODING)
62 return value.decode(settings.WIRE_ENCODING)
63
63
64
64
65 def reraise_safe_exceptions(func):
65 def reraise_safe_exceptions(func):
66 """Converts Dulwich exceptions to something neutral."""
66 """Converts Dulwich exceptions to something neutral."""
67
67
68 @wraps(func)
68 @wraps(func)
69 def wrapper(*args, **kwargs):
69 def wrapper(*args, **kwargs):
70 try:
70 try:
71 return func(*args, **kwargs)
71 return func(*args, **kwargs)
72 except (ChecksumMismatch, WrongObjectException, MissingCommitError, ObjectMissing,) as e:
72 except (ChecksumMismatch, WrongObjectException, MissingCommitError, ObjectMissing,) as e:
73 exc = exceptions.LookupException(org_exc=e)
73 exc = exceptions.LookupException(org_exc=e)
74 raise exc(safe_str(e))
74 raise exc(safe_str(e))
75 except (HangupException, UnexpectedCommandError) as e:
75 except (HangupException, UnexpectedCommandError) as e:
76 exc = exceptions.VcsException(org_exc=e)
76 exc = exceptions.VcsException(org_exc=e)
77 raise exc(safe_str(e))
77 raise exc(safe_str(e))
78 except Exception as e:
78 except Exception as e:
79 # NOTE(marcink): becuase of how dulwich handles some exceptions
79 # NOTE(marcink): becuase of how dulwich handles some exceptions
80 # (KeyError on empty repos), we cannot track this and catch all
80 # (KeyError on empty repos), we cannot track this and catch all
81 # exceptions, it's an exceptions from other handlers
81 # exceptions, it's an exceptions from other handlers
82 #if not hasattr(e, '_vcs_kind'):
82 #if not hasattr(e, '_vcs_kind'):
83 #log.exception("Unhandled exception in git remote call")
83 #log.exception("Unhandled exception in git remote call")
84 #raise_from_original(exceptions.UnhandledException)
84 #raise_from_original(exceptions.UnhandledException)
85 raise
85 raise
86 return wrapper
86 return wrapper
87
87
88
88
89 class Repo(DulwichRepo):
89 class Repo(DulwichRepo):
90 """
90 """
91 A wrapper for dulwich Repo class.
91 A wrapper for dulwich Repo class.
92
92
93 Since dulwich is sometimes keeping .idx file descriptors open, it leads to
93 Since dulwich is sometimes keeping .idx file descriptors open, it leads to
94 "Too many open files" error. We need to close all opened file descriptors
94 "Too many open files" error. We need to close all opened file descriptors
95 once the repo object is destroyed.
95 once the repo object is destroyed.
96 """
96 """
97 def __del__(self):
97 def __del__(self):
98 if hasattr(self, 'object_store'):
98 if hasattr(self, 'object_store'):
99 self.close()
99 self.close()
100
100
101
101
102 class Repository(LibGit2Repo):
102 class Repository(LibGit2Repo):
103
103
104 def __enter__(self):
104 def __enter__(self):
105 return self
105 return self
106
106
107 def __exit__(self, exc_type, exc_val, exc_tb):
107 def __exit__(self, exc_type, exc_val, exc_tb):
108 self.free()
108 self.free()
109
109
110
110
111 class GitFactory(RepoFactory):
111 class GitFactory(RepoFactory):
112 repo_type = 'git'
112 repo_type = 'git'
113
113
114 def _create_repo(self, wire, create, use_libgit2=False):
114 def _create_repo(self, wire, create, use_libgit2=False):
115 if use_libgit2:
115 if use_libgit2:
116 return Repository(wire['path'])
116 return Repository(wire['path'])
117 else:
117 else:
118 repo_path = str_to_dulwich(wire['path'])
118 repo_path = str_to_dulwich(wire['path'])
119 return Repo(repo_path)
119 return Repo(repo_path)
120
120
121 def repo(self, wire, create=False, use_libgit2=False):
121 def repo(self, wire, create=False, use_libgit2=False):
122 """
122 """
123 Get a repository instance for the given path.
123 Get a repository instance for the given path.
124 """
124 """
125 return self._create_repo(wire, create, use_libgit2)
125 return self._create_repo(wire, create, use_libgit2)
126
126
127 def repo_libgit2(self, wire):
127 def repo_libgit2(self, wire):
128 return self.repo(wire, use_libgit2=True)
128 return self.repo(wire, use_libgit2=True)
129
129
130
130
131 class GitRemote(RemoteBase):
131 class GitRemote(RemoteBase):
132
132
133 def __init__(self, factory):
133 def __init__(self, factory):
134 self._factory = factory
134 self._factory = factory
135 self._bulk_methods = {
135 self._bulk_methods = {
136 "date": self.date,
136 "date": self.date,
137 "author": self.author,
137 "author": self.author,
138 "branch": self.branch,
138 "branch": self.branch,
139 "message": self.message,
139 "message": self.message,
140 "parents": self.parents,
140 "parents": self.parents,
141 "_commit": self.revision,
141 "_commit": self.revision,
142 }
142 }
143
143
144 def _wire_to_config(self, wire):
144 def _wire_to_config(self, wire):
145 if 'config' in wire:
145 if 'config' in wire:
146 return dict([(x[0] + '_' + x[1], x[2]) for x in wire['config']])
146 return dict([(x[0] + '_' + x[1], x[2]) for x in wire['config']])
147 return {}
147 return {}
148
148
149 def _remote_conf(self, config):
149 def _remote_conf(self, config):
150 params = [
150 params = [
151 '-c', 'core.askpass=""',
151 '-c', 'core.askpass=""',
152 ]
152 ]
153 ssl_cert_dir = config.get('vcs_ssl_dir')
153 ssl_cert_dir = config.get('vcs_ssl_dir')
154 if ssl_cert_dir:
154 if ssl_cert_dir:
155 params.extend(['-c', 'http.sslCAinfo={}'.format(ssl_cert_dir)])
155 params.extend(['-c', 'http.sslCAinfo={}'.format(ssl_cert_dir)])
156 return params
156 return params
157
157
158 @reraise_safe_exceptions
158 @reraise_safe_exceptions
159 def discover_git_version(self):
159 def discover_git_version(self):
160 stdout, _ = self.run_git_command(
160 stdout, _ = self.run_git_command(
161 {}, ['--version'], _bare=True, _safe=True)
161 {}, ['--version'], _bare=True, _safe=True)
162 prefix = 'git version'
162 prefix = 'git version'
163 if stdout.startswith(prefix):
163 if stdout.startswith(prefix):
164 stdout = stdout[len(prefix):]
164 stdout = stdout[len(prefix):]
165 return stdout.strip()
165 return stdout.strip()
166
166
167 @reraise_safe_exceptions
167 @reraise_safe_exceptions
168 def is_empty(self, wire):
168 def is_empty(self, wire):
169 repo_init = self._factory.repo_libgit2(wire)
169 repo_init = self._factory.repo_libgit2(wire)
170 with repo_init as repo:
170 with repo_init as repo:
171
171
172 try:
172 try:
173 has_head = repo.head.name
173 has_head = repo.head.name
174 if has_head:
174 if has_head:
175 return False
175 return False
176
176
177 # NOTE(marcink): check again using more expensive method
177 # NOTE(marcink): check again using more expensive method
178 return repo.is_empty
178 return repo.is_empty
179 except Exception:
179 except Exception:
180 pass
180 pass
181
181
182 return True
182 return True
183
183
184 @reraise_safe_exceptions
184 @reraise_safe_exceptions
185 def assert_correct_path(self, wire):
185 def assert_correct_path(self, wire):
186 cache_on, context_uid, repo_id = self._cache_on(wire)
186 cache_on, context_uid, repo_id = self._cache_on(wire)
187 @self.region.conditional_cache_on_arguments(condition=cache_on)
187 @self.region.conditional_cache_on_arguments(condition=cache_on)
188 def _assert_correct_path(_context_uid, _repo_id):
188 def _assert_correct_path(_context_uid, _repo_id):
189 try:
189 try:
190 repo_init = self._factory.repo_libgit2(wire)
190 repo_init = self._factory.repo_libgit2(wire)
191 with repo_init as repo:
191 with repo_init as repo:
192 pass
192 pass
193 except pygit2.GitError:
193 except pygit2.GitError:
194 path = wire.get('path')
194 path = wire.get('path')
195 tb = traceback.format_exc()
195 tb = traceback.format_exc()
196 log.debug("Invalid Git path `%s`, tb: %s", path, tb)
196 log.debug("Invalid Git path `%s`, tb: %s", path, tb)
197 return False
197 return False
198
198
199 return True
199 return True
200 return _assert_correct_path(context_uid, repo_id)
200 return _assert_correct_path(context_uid, repo_id)
201
201
202 @reraise_safe_exceptions
202 @reraise_safe_exceptions
203 def bare(self, wire):
203 def bare(self, wire):
204 repo_init = self._factory.repo_libgit2(wire)
204 repo_init = self._factory.repo_libgit2(wire)
205 with repo_init as repo:
205 with repo_init as repo:
206 return repo.is_bare
206 return repo.is_bare
207
207
208 @reraise_safe_exceptions
208 @reraise_safe_exceptions
209 def blob_as_pretty_string(self, wire, sha):
209 def blob_as_pretty_string(self, wire, sha):
210 repo_init = self._factory.repo_libgit2(wire)
210 repo_init = self._factory.repo_libgit2(wire)
211 with repo_init as repo:
211 with repo_init as repo:
212 blob_obj = repo[sha]
212 blob_obj = repo[sha]
213 blob = blob_obj.data
213 blob = blob_obj.data
214 return blob
214 return blob
215
215
216 @reraise_safe_exceptions
216 @reraise_safe_exceptions
217 def blob_raw_length(self, wire, sha):
217 def blob_raw_length(self, wire, sha):
218 cache_on, context_uid, repo_id = self._cache_on(wire)
218 cache_on, context_uid, repo_id = self._cache_on(wire)
219 @self.region.conditional_cache_on_arguments(condition=cache_on)
219 @self.region.conditional_cache_on_arguments(condition=cache_on)
220 def _blob_raw_length(_repo_id, _sha):
220 def _blob_raw_length(_repo_id, _sha):
221
221
222 repo_init = self._factory.repo_libgit2(wire)
222 repo_init = self._factory.repo_libgit2(wire)
223 with repo_init as repo:
223 with repo_init as repo:
224 blob = repo[sha]
224 blob = repo[sha]
225 return blob.size
225 return blob.size
226
226
227 return _blob_raw_length(repo_id, sha)
227 return _blob_raw_length(repo_id, sha)
228
228
229 def _parse_lfs_pointer(self, raw_content):
229 def _parse_lfs_pointer(self, raw_content):
230
230
231 spec_string = 'version https://git-lfs.github.com/spec'
231 spec_string = 'version https://git-lfs.github.com/spec'
232 if raw_content and raw_content.startswith(spec_string):
232 if raw_content and raw_content.startswith(spec_string):
233 pattern = re.compile(r"""
233 pattern = re.compile(r"""
234 (?:\n)?
234 (?:\n)?
235 ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
235 ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
236 ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
236 ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
237 ^size[ ](?P<oid_size>[0-9]+)\n
237 ^size[ ](?P<oid_size>[0-9]+)\n
238 (?:\n)?
238 (?:\n)?
239 """, re.VERBOSE | re.MULTILINE)
239 """, re.VERBOSE | re.MULTILINE)
240 match = pattern.match(raw_content)
240 match = pattern.match(raw_content)
241 if match:
241 if match:
242 return match.groupdict()
242 return match.groupdict()
243
243
244 return {}
244 return {}
245
245
246 @reraise_safe_exceptions
246 @reraise_safe_exceptions
247 def is_large_file(self, wire, commit_id):
247 def is_large_file(self, wire, commit_id):
248 cache_on, context_uid, repo_id = self._cache_on(wire)
248 cache_on, context_uid, repo_id = self._cache_on(wire)
249
249
250 @self.region.conditional_cache_on_arguments(condition=cache_on)
250 @self.region.conditional_cache_on_arguments(condition=cache_on)
251 def _is_large_file(_repo_id, _sha):
251 def _is_large_file(_repo_id, _sha):
252 repo_init = self._factory.repo_libgit2(wire)
252 repo_init = self._factory.repo_libgit2(wire)
253 with repo_init as repo:
253 with repo_init as repo:
254 blob = repo[commit_id]
254 blob = repo[commit_id]
255 if blob.is_binary:
255 if blob.is_binary:
256 return {}
256 return {}
257
257
258 return self._parse_lfs_pointer(blob.data)
258 return self._parse_lfs_pointer(blob.data)
259
259
260 return _is_large_file(repo_id, commit_id)
260 return _is_large_file(repo_id, commit_id)
261
261
262 @reraise_safe_exceptions
262 @reraise_safe_exceptions
263 def is_binary(self, wire, tree_id):
263 def is_binary(self, wire, tree_id):
264 cache_on, context_uid, repo_id = self._cache_on(wire)
264 cache_on, context_uid, repo_id = self._cache_on(wire)
265
265
266 @self.region.conditional_cache_on_arguments(condition=cache_on)
266 @self.region.conditional_cache_on_arguments(condition=cache_on)
267 def _is_binary(_repo_id, _tree_id):
267 def _is_binary(_repo_id, _tree_id):
268 repo_init = self._factory.repo_libgit2(wire)
268 repo_init = self._factory.repo_libgit2(wire)
269 with repo_init as repo:
269 with repo_init as repo:
270 blob_obj = repo[tree_id]
270 blob_obj = repo[tree_id]
271 return blob_obj.is_binary
271 return blob_obj.is_binary
272
272
273 return _is_binary(repo_id, tree_id)
273 return _is_binary(repo_id, tree_id)
274
274
275 @reraise_safe_exceptions
275 @reraise_safe_exceptions
276 def in_largefiles_store(self, wire, oid):
276 def in_largefiles_store(self, wire, oid):
277 conf = self._wire_to_config(wire)
277 conf = self._wire_to_config(wire)
278 repo_init = self._factory.repo_libgit2(wire)
278 repo_init = self._factory.repo_libgit2(wire)
279 with repo_init as repo:
279 with repo_init as repo:
280 repo_name = repo.path
280 repo_name = repo.path
281
281
282 store_location = conf.get('vcs_git_lfs_store_location')
282 store_location = conf.get('vcs_git_lfs_store_location')
283 if store_location:
283 if store_location:
284
284
285 store = LFSOidStore(
285 store = LFSOidStore(
286 oid=oid, repo=repo_name, store_location=store_location)
286 oid=oid, repo=repo_name, store_location=store_location)
287 return store.has_oid()
287 return store.has_oid()
288
288
289 return False
289 return False
290
290
291 @reraise_safe_exceptions
291 @reraise_safe_exceptions
292 def store_path(self, wire, oid):
292 def store_path(self, wire, oid):
293 conf = self._wire_to_config(wire)
293 conf = self._wire_to_config(wire)
294 repo_init = self._factory.repo_libgit2(wire)
294 repo_init = self._factory.repo_libgit2(wire)
295 with repo_init as repo:
295 with repo_init as repo:
296 repo_name = repo.path
296 repo_name = repo.path
297
297
298 store_location = conf.get('vcs_git_lfs_store_location')
298 store_location = conf.get('vcs_git_lfs_store_location')
299 if store_location:
299 if store_location:
300 store = LFSOidStore(
300 store = LFSOidStore(
301 oid=oid, repo=repo_name, store_location=store_location)
301 oid=oid, repo=repo_name, store_location=store_location)
302 return store.oid_path
302 return store.oid_path
303 raise ValueError('Unable to fetch oid with path {}'.format(oid))
303 raise ValueError('Unable to fetch oid with path {}'.format(oid))
304
304
305 @reraise_safe_exceptions
305 @reraise_safe_exceptions
306 def bulk_request(self, wire, rev, pre_load):
306 def bulk_request(self, wire, rev, pre_load):
307 cache_on, context_uid, repo_id = self._cache_on(wire)
307 cache_on, context_uid, repo_id = self._cache_on(wire)
308 @self.region.conditional_cache_on_arguments(condition=cache_on)
308 @self.region.conditional_cache_on_arguments(condition=cache_on)
309 def _bulk_request(_repo_id, _rev, _pre_load):
309 def _bulk_request(_repo_id, _rev, _pre_load):
310 result = {}
310 result = {}
311 for attr in pre_load:
311 for attr in pre_load:
312 try:
312 try:
313 method = self._bulk_methods[attr]
313 method = self._bulk_methods[attr]
314 args = [wire, rev]
314 args = [wire, rev]
315 result[attr] = method(*args)
315 result[attr] = method(*args)
316 except KeyError as e:
316 except KeyError as e:
317 raise exceptions.VcsException(e)(
317 raise exceptions.VcsException(e)(
318 "Unknown bulk attribute: %s" % attr)
318 "Unknown bulk attribute: %s" % attr)
319 return result
319 return result
320
320
321 return _bulk_request(repo_id, rev, sorted(pre_load))
321 return _bulk_request(repo_id, rev, sorted(pre_load))
322
322
323 def _build_opener(self, url):
323 def _build_opener(self, url):
324 handlers = []
324 handlers = []
325 url_obj = url_parser(url)
325 url_obj = url_parser(url)
326 _, authinfo = url_obj.authinfo()
326 _, authinfo = url_obj.authinfo()
327
327
328 if authinfo:
328 if authinfo:
329 # create a password manager
329 # create a password manager
330 passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
330 passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
331 passmgr.add_password(*authinfo)
331 passmgr.add_password(*authinfo)
332
332
333 handlers.extend((httpbasicauthhandler(passmgr),
333 handlers.extend((httpbasicauthhandler(passmgr),
334 httpdigestauthhandler(passmgr)))
334 httpdigestauthhandler(passmgr)))
335
335
336 return urllib2.build_opener(*handlers)
336 return urllib2.build_opener(*handlers)
337
337
338 def _type_id_to_name(self, type_id):
338 def _type_id_to_name(self, type_id):
339 return {
339 return {
340 1: b'commit',
340 1: b'commit',
341 2: b'tree',
341 2: b'tree',
342 3: b'blob',
342 3: b'blob',
343 4: b'tag'
343 4: b'tag'
344 }[type_id]
344 }[type_id]
345
345
346 @reraise_safe_exceptions
346 @reraise_safe_exceptions
347 def check_url(self, url, config):
347 def check_url(self, url, config):
348 url_obj = url_parser(url)
348 url_obj = url_parser(url)
349 test_uri, _ = url_obj.authinfo()
349 test_uri, _ = url_obj.authinfo()
350 url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
350 url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
351 url_obj.query = obfuscate_qs(url_obj.query)
351 url_obj.query = obfuscate_qs(url_obj.query)
352 cleaned_uri = str(url_obj)
352 cleaned_uri = str(url_obj)
353 log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
353 log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
354
354
355 if not test_uri.endswith('info/refs'):
355 if not test_uri.endswith('info/refs'):
356 test_uri = test_uri.rstrip('/') + '/info/refs'
356 test_uri = test_uri.rstrip('/') + '/info/refs'
357
357
358 o = self._build_opener(url)
358 o = self._build_opener(url)
359 o.addheaders = [('User-Agent', 'git/1.7.8.0')] # fake some git
359 o.addheaders = [('User-Agent', 'git/1.7.8.0')] # fake some git
360
360
361 q = {"service": 'git-upload-pack'}
361 q = {"service": 'git-upload-pack'}
362 qs = '?%s' % urllib.urlencode(q)
362 qs = '?%s' % urllib.urlencode(q)
363 cu = "%s%s" % (test_uri, qs)
363 cu = "%s%s" % (test_uri, qs)
364 req = urllib2.Request(cu, None, {})
364 req = urllib2.Request(cu, None, {})
365
365
366 try:
366 try:
367 log.debug("Trying to open URL %s", cleaned_uri)
367 log.debug("Trying to open URL %s", cleaned_uri)
368 resp = o.open(req)
368 resp = o.open(req)
369 if resp.code != 200:
369 if resp.code != 200:
370 raise exceptions.URLError()('Return Code is not 200')
370 raise exceptions.URLError()('Return Code is not 200')
371 except Exception as e:
371 except Exception as e:
372 log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
372 log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
373 # means it cannot be cloned
373 # means it cannot be cloned
374 raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e))
374 raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e))
375
375
376 # now detect if it's proper git repo
376 # now detect if it's proper git repo
377 gitdata = resp.read()
377 gitdata = resp.read()
378 if 'service=git-upload-pack' in gitdata:
378 if 'service=git-upload-pack' in gitdata:
379 pass
379 pass
380 elif re.findall(r'[0-9a-fA-F]{40}\s+refs', gitdata):
380 elif re.findall(r'[0-9a-fA-F]{40}\s+refs', gitdata):
381 # old style git can return some other format !
381 # old style git can return some other format !
382 pass
382 pass
383 else:
383 else:
384 raise exceptions.URLError()(
384 raise exceptions.URLError()(
385 "url [%s] does not look like an git" % (cleaned_uri,))
385 "url [%s] does not look like an git" % (cleaned_uri,))
386
386
387 return True
387 return True
388
388
389 @reraise_safe_exceptions
389 @reraise_safe_exceptions
390 def clone(self, wire, url, deferred, valid_refs, update_after_clone):
390 def clone(self, wire, url, deferred, valid_refs, update_after_clone):
391 # TODO(marcink): deprecate this method. Last i checked we don't use it anymore
391 # TODO(marcink): deprecate this method. Last i checked we don't use it anymore
392 remote_refs = self.pull(wire, url, apply_refs=False)
392 remote_refs = self.pull(wire, url, apply_refs=False)
393 repo = self._factory.repo(wire)
393 repo = self._factory.repo(wire)
394 if isinstance(valid_refs, list):
394 if isinstance(valid_refs, list):
395 valid_refs = tuple(valid_refs)
395 valid_refs = tuple(valid_refs)
396
396
397 for k in remote_refs:
397 for k in remote_refs:
398 # only parse heads/tags and skip so called deferred tags
398 # only parse heads/tags and skip so called deferred tags
399 if k.startswith(valid_refs) and not k.endswith(deferred):
399 if k.startswith(valid_refs) and not k.endswith(deferred):
400 repo[k] = remote_refs[k]
400 repo[k] = remote_refs[k]
401
401
402 if update_after_clone:
402 if update_after_clone:
403 # we want to checkout HEAD
403 # we want to checkout HEAD
404 repo["HEAD"] = remote_refs["HEAD"]
404 repo["HEAD"] = remote_refs["HEAD"]
405 index.build_index_from_tree(repo.path, repo.index_path(),
405 index.build_index_from_tree(repo.path, repo.index_path(),
406 repo.object_store, repo["HEAD"].tree)
406 repo.object_store, repo["HEAD"].tree)
407
407
408 @reraise_safe_exceptions
408 @reraise_safe_exceptions
409 def branch(self, wire, commit_id):
409 def branch(self, wire, commit_id):
410 cache_on, context_uid, repo_id = self._cache_on(wire)
410 cache_on, context_uid, repo_id = self._cache_on(wire)
411 @self.region.conditional_cache_on_arguments(condition=cache_on)
411 @self.region.conditional_cache_on_arguments(condition=cache_on)
412 def _branch(_context_uid, _repo_id, _commit_id):
412 def _branch(_context_uid, _repo_id, _commit_id):
413 regex = re.compile('^refs/heads')
413 regex = re.compile('^refs/heads')
414
414
415 def filter_with(ref):
415 def filter_with(ref):
416 return regex.match(ref[0]) and ref[1] == _commit_id
416 return regex.match(ref[0]) and ref[1] == _commit_id
417
417
418 branches = filter(filter_with, self.get_refs(wire).items())
418 branches = filter(filter_with, self.get_refs(wire).items())
419 return [x[0].split('refs/heads/')[-1] for x in branches]
419 return [x[0].split('refs/heads/')[-1] for x in branches]
420
420
421 return _branch(context_uid, repo_id, commit_id)
421 return _branch(context_uid, repo_id, commit_id)
422
422
423 @reraise_safe_exceptions
423 @reraise_safe_exceptions
424 def commit_branches(self, wire, commit_id):
424 def commit_branches(self, wire, commit_id):
425 cache_on, context_uid, repo_id = self._cache_on(wire)
425 cache_on, context_uid, repo_id = self._cache_on(wire)
426 @self.region.conditional_cache_on_arguments(condition=cache_on)
426 @self.region.conditional_cache_on_arguments(condition=cache_on)
427 def _commit_branches(_context_uid, _repo_id, _commit_id):
427 def _commit_branches(_context_uid, _repo_id, _commit_id):
428 repo_init = self._factory.repo_libgit2(wire)
428 repo_init = self._factory.repo_libgit2(wire)
429 with repo_init as repo:
429 with repo_init as repo:
430 branches = [x for x in repo.branches.with_commit(_commit_id)]
430 branches = [x for x in repo.branches.with_commit(_commit_id)]
431 return branches
431 return branches
432
432
433 return _commit_branches(context_uid, repo_id, commit_id)
433 return _commit_branches(context_uid, repo_id, commit_id)
434
434
435 @reraise_safe_exceptions
435 @reraise_safe_exceptions
436 def add_object(self, wire, content):
436 def add_object(self, wire, content):
437 repo_init = self._factory.repo_libgit2(wire)
437 repo_init = self._factory.repo_libgit2(wire)
438 with repo_init as repo:
438 with repo_init as repo:
439 blob = objects.Blob()
439 blob = objects.Blob()
440 blob.set_raw_string(content)
440 blob.set_raw_string(content)
441 repo.object_store.add_object(blob)
441 repo.object_store.add_object(blob)
442 return blob.id
442 return blob.id
443
443
444 # TODO: this is quite complex, check if that can be simplified
444 # TODO: this is quite complex, check if that can be simplified
445 @reraise_safe_exceptions
445 @reraise_safe_exceptions
446 def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
446 def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
447 repo = self._factory.repo(wire)
447 repo = self._factory.repo(wire)
448 object_store = repo.object_store
448 object_store = repo.object_store
449
449
450 # Create tree and populates it with blobs
450 # Create tree and populates it with blobs
451 commit_tree = commit_tree and repo[commit_tree] or objects.Tree()
451 commit_tree = commit_tree and repo[commit_tree] or objects.Tree()
452
452
453 for node in updated:
453 for node in updated:
454 # Compute subdirs if needed
454 # Compute subdirs if needed
455 dirpath, nodename = vcspath.split(node['path'])
455 dirpath, nodename = vcspath.split(node['path'])
456 dirnames = map(safe_str, dirpath and dirpath.split('/') or [])
456 dirnames = map(safe_str, dirpath and dirpath.split('/') or [])
457 parent = commit_tree
457 parent = commit_tree
458 ancestors = [('', parent)]
458 ancestors = [('', parent)]
459
459
460 # Tries to dig for the deepest existing tree
460 # Tries to dig for the deepest existing tree
461 while dirnames:
461 while dirnames:
462 curdir = dirnames.pop(0)
462 curdir = dirnames.pop(0)
463 try:
463 try:
464 dir_id = parent[curdir][1]
464 dir_id = parent[curdir][1]
465 except KeyError:
465 except KeyError:
466 # put curdir back into dirnames and stops
466 # put curdir back into dirnames and stops
467 dirnames.insert(0, curdir)
467 dirnames.insert(0, curdir)
468 break
468 break
469 else:
469 else:
470 # If found, updates parent
470 # If found, updates parent
471 parent = repo[dir_id]
471 parent = repo[dir_id]
472 ancestors.append((curdir, parent))
472 ancestors.append((curdir, parent))
473 # Now parent is deepest existing tree and we need to create
473 # Now parent is deepest existing tree and we need to create
474 # subtrees for dirnames (in reverse order)
474 # subtrees for dirnames (in reverse order)
475 # [this only applies for nodes from added]
475 # [this only applies for nodes from added]
476 new_trees = []
476 new_trees = []
477
477
478 blob = objects.Blob.from_string(node['content'])
478 blob = objects.Blob.from_string(node['content'])
479
479
480 if dirnames:
480 if dirnames:
481 # If there are trees which should be created we need to build
481 # If there are trees which should be created we need to build
482 # them now (in reverse order)
482 # them now (in reverse order)
483 reversed_dirnames = list(reversed(dirnames))
483 reversed_dirnames = list(reversed(dirnames))
484 curtree = objects.Tree()
484 curtree = objects.Tree()
485 curtree[node['node_path']] = node['mode'], blob.id
485 curtree[node['node_path']] = node['mode'], blob.id
486 new_trees.append(curtree)
486 new_trees.append(curtree)
487 for dirname in reversed_dirnames[:-1]:
487 for dirname in reversed_dirnames[:-1]:
488 newtree = objects.Tree()
488 newtree = objects.Tree()
489 newtree[dirname] = (DIR_STAT, curtree.id)
489 newtree[dirname] = (DIR_STAT, curtree.id)
490 new_trees.append(newtree)
490 new_trees.append(newtree)
491 curtree = newtree
491 curtree = newtree
492 parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
492 parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
493 else:
493 else:
494 parent.add(name=node['node_path'], mode=node['mode'], hexsha=blob.id)
494 parent.add(name=node['node_path'], mode=node['mode'], hexsha=blob.id)
495
495
496 new_trees.append(parent)
496 new_trees.append(parent)
497 # Update ancestors
497 # Update ancestors
498 reversed_ancestors = reversed(
498 reversed_ancestors = reversed(
499 [(a[1], b[1], b[0]) for a, b in zip(ancestors, ancestors[1:])])
499 [(a[1], b[1], b[0]) for a, b in zip(ancestors, ancestors[1:])])
500 for parent, tree, path in reversed_ancestors:
500 for parent, tree, path in reversed_ancestors:
501 parent[path] = (DIR_STAT, tree.id)
501 parent[path] = (DIR_STAT, tree.id)
502 object_store.add_object(tree)
502 object_store.add_object(tree)
503
503
504 object_store.add_object(blob)
504 object_store.add_object(blob)
505 for tree in new_trees:
505 for tree in new_trees:
506 object_store.add_object(tree)
506 object_store.add_object(tree)
507
507
508 for node_path in removed:
508 for node_path in removed:
509 paths = node_path.split('/')
509 paths = node_path.split('/')
510 tree = commit_tree
510 tree = commit_tree
511 trees = [tree]
511 trees = [tree]
512 # Traverse deep into the forest...
512 # Traverse deep into the forest...
513 for path in paths:
513 for path in paths:
514 try:
514 try:
515 obj = repo[tree[path][1]]
515 obj = repo[tree[path][1]]
516 if isinstance(obj, objects.Tree):
516 if isinstance(obj, objects.Tree):
517 trees.append(obj)
517 trees.append(obj)
518 tree = obj
518 tree = obj
519 except KeyError:
519 except KeyError:
520 break
520 break
521 # Cut down the blob and all rotten trees on the way back...
521 # Cut down the blob and all rotten trees on the way back...
522 for path, tree in reversed(zip(paths, trees)):
522 for path, tree in reversed(zip(paths, trees)):
523 del tree[path]
523 del tree[path]
524 if tree:
524 if tree:
525 # This tree still has elements - don't remove it or any
525 # This tree still has elements - don't remove it or any
526 # of it's parents
526 # of it's parents
527 break
527 break
528
528
529 object_store.add_object(commit_tree)
529 object_store.add_object(commit_tree)
530
530
531 # Create commit
531 # Create commit
532 commit = objects.Commit()
532 commit = objects.Commit()
533 commit.tree = commit_tree.id
533 commit.tree = commit_tree.id
534 for k, v in commit_data.iteritems():
534 for k, v in commit_data.iteritems():
535 setattr(commit, k, v)
535 setattr(commit, k, v)
536 object_store.add_object(commit)
536 object_store.add_object(commit)
537
537
538 self.create_branch(wire, branch, commit.id)
538 self.create_branch(wire, branch, commit.id)
539
539
540 # dulwich set-ref
540 # dulwich set-ref
541 ref = 'refs/heads/%s' % branch
541 ref = 'refs/heads/%s' % branch
542 repo.refs[ref] = commit.id
542 repo.refs[ref] = commit.id
543
543
544 return commit.id
544 return commit.id
545
545
546 @reraise_safe_exceptions
546 @reraise_safe_exceptions
547 def pull(self, wire, url, apply_refs=True, refs=None, update_after=False):
547 def pull(self, wire, url, apply_refs=True, refs=None, update_after=False):
548 if url != 'default' and '://' not in url:
548 if url != 'default' and '://' not in url:
549 client = LocalGitClient(url)
549 client = LocalGitClient(url)
550 else:
550 else:
551 url_obj = url_parser(url)
551 url_obj = url_parser(url)
552 o = self._build_opener(url)
552 o = self._build_opener(url)
553 url, _ = url_obj.authinfo()
553 url, _ = url_obj.authinfo()
554 client = HttpGitClient(base_url=url, opener=o)
554 client = HttpGitClient(base_url=url, opener=o)
555 repo = self._factory.repo(wire)
555 repo = self._factory.repo(wire)
556
556
557 determine_wants = repo.object_store.determine_wants_all
557 determine_wants = repo.object_store.determine_wants_all
558 if refs:
558 if refs:
559 def determine_wants_requested(references):
559 def determine_wants_requested(references):
560 return [references[r] for r in references if r in refs]
560 return [references[r] for r in references if r in refs]
561 determine_wants = determine_wants_requested
561 determine_wants = determine_wants_requested
562
562
563 try:
563 try:
564 remote_refs = client.fetch(
564 remote_refs = client.fetch(
565 path=url, target=repo, determine_wants=determine_wants)
565 path=url, target=repo, determine_wants=determine_wants)
566 except NotGitRepository as e:
566 except NotGitRepository as e:
567 log.warning(
567 log.warning(
568 'Trying to fetch from "%s" failed, not a Git repository.', url)
568 'Trying to fetch from "%s" failed, not a Git repository.', url)
569 # Exception can contain unicode which we convert
569 # Exception can contain unicode which we convert
570 raise exceptions.AbortException(e)(repr(e))
570 raise exceptions.AbortException(e)(repr(e))
571
571
572 # mikhail: client.fetch() returns all the remote refs, but fetches only
572 # mikhail: client.fetch() returns all the remote refs, but fetches only
573 # refs filtered by `determine_wants` function. We need to filter result
573 # refs filtered by `determine_wants` function. We need to filter result
574 # as well
574 # as well
575 if refs:
575 if refs:
576 remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
576 remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
577
577
578 if apply_refs:
578 if apply_refs:
579 # TODO: johbo: Needs proper test coverage with a git repository
579 # TODO: johbo: Needs proper test coverage with a git repository
580 # that contains a tag object, so that we would end up with
580 # that contains a tag object, so that we would end up with
581 # a peeled ref at this point.
581 # a peeled ref at this point.
582 for k in remote_refs:
582 for k in remote_refs:
583 if k.endswith(PEELED_REF_MARKER):
583 if k.endswith(PEELED_REF_MARKER):
584 log.debug("Skipping peeled reference %s", k)
584 log.debug("Skipping peeled reference %s", k)
585 continue
585 continue
586 repo[k] = remote_refs[k]
586 repo[k] = remote_refs[k]
587
587
588 if refs and not update_after:
588 if refs and not update_after:
589 # mikhail: explicitly set the head to the last ref.
589 # mikhail: explicitly set the head to the last ref.
590 repo['HEAD'] = remote_refs[refs[-1]]
590 repo['HEAD'] = remote_refs[refs[-1]]
591
591
592 if update_after:
592 if update_after:
593 # we want to checkout HEAD
593 # we want to checkout HEAD
594 repo["HEAD"] = remote_refs["HEAD"]
594 repo["HEAD"] = remote_refs["HEAD"]
595 index.build_index_from_tree(repo.path, repo.index_path(),
595 index.build_index_from_tree(repo.path, repo.index_path(),
596 repo.object_store, repo["HEAD"].tree)
596 repo.object_store, repo["HEAD"].tree)
597 return remote_refs
597 return remote_refs
598
598
599 @reraise_safe_exceptions
599 @reraise_safe_exceptions
600 def sync_fetch(self, wire, url, refs=None, all_refs=False):
600 def sync_fetch(self, wire, url, refs=None, all_refs=False):
601 repo = self._factory.repo(wire)
601 repo = self._factory.repo(wire)
602 if refs and not isinstance(refs, (list, tuple)):
602 if refs and not isinstance(refs, (list, tuple)):
603 refs = [refs]
603 refs = [refs]
604
604
605 config = self._wire_to_config(wire)
605 config = self._wire_to_config(wire)
606 # get all remote refs we'll use to fetch later
606 # get all remote refs we'll use to fetch later
607 cmd = ['ls-remote']
607 cmd = ['ls-remote']
608 if not all_refs:
608 if not all_refs:
609 cmd += ['--heads', '--tags']
609 cmd += ['--heads', '--tags']
610 cmd += [url]
610 cmd += [url]
611 output, __ = self.run_git_command(
611 output, __ = self.run_git_command(
612 wire, cmd, fail_on_stderr=False,
612 wire, cmd, fail_on_stderr=False,
613 _copts=self._remote_conf(config),
613 _copts=self._remote_conf(config),
614 extra_env={'GIT_TERMINAL_PROMPT': '0'})
614 extra_env={'GIT_TERMINAL_PROMPT': '0'})
615
615
616 remote_refs = collections.OrderedDict()
616 remote_refs = collections.OrderedDict()
617 fetch_refs = []
617 fetch_refs = []
618
618
619 for ref_line in output.splitlines():
619 for ref_line in output.splitlines():
620 sha, ref = ref_line.split('\t')
620 sha, ref = ref_line.split('\t')
621 sha = sha.strip()
621 sha = sha.strip()
622 if ref in remote_refs:
622 if ref in remote_refs:
623 # duplicate, skip
623 # duplicate, skip
624 continue
624 continue
625 if ref.endswith(PEELED_REF_MARKER):
625 if ref.endswith(PEELED_REF_MARKER):
626 log.debug("Skipping peeled reference %s", ref)
626 log.debug("Skipping peeled reference %s", ref)
627 continue
627 continue
628 # don't sync HEAD
628 # don't sync HEAD
629 if ref in ['HEAD']:
629 if ref in ['HEAD']:
630 continue
630 continue
631
631
632 remote_refs[ref] = sha
632 remote_refs[ref] = sha
633
633
634 if refs and sha in refs:
634 if refs and sha in refs:
635 # we filter fetch using our specified refs
635 # we filter fetch using our specified refs
636 fetch_refs.append('{}:{}'.format(ref, ref))
636 fetch_refs.append('{}:{}'.format(ref, ref))
637 elif not refs:
637 elif not refs:
638 fetch_refs.append('{}:{}'.format(ref, ref))
638 fetch_refs.append('{}:{}'.format(ref, ref))
639 log.debug('Finished obtaining fetch refs, total: %s', len(fetch_refs))
639 log.debug('Finished obtaining fetch refs, total: %s', len(fetch_refs))
640
640
641 if fetch_refs:
641 if fetch_refs:
642 for chunk in more_itertools.chunked(fetch_refs, 1024 * 4):
642 for chunk in more_itertools.chunked(fetch_refs, 1024 * 4):
643 fetch_refs_chunks = list(chunk)
643 fetch_refs_chunks = list(chunk)
644 log.debug('Fetching %s refs from import url', len(fetch_refs_chunks))
644 log.debug('Fetching %s refs from import url', len(fetch_refs_chunks))
645 _out, _err = self.run_git_command(
645 _out, _err = self.run_git_command(
646 wire, ['fetch', url, '--force', '--prune', '--'] + fetch_refs_chunks,
646 wire, ['fetch', url, '--force', '--prune', '--'] + fetch_refs_chunks,
647 fail_on_stderr=False,
647 fail_on_stderr=False,
648 _copts=self._remote_conf(config),
648 _copts=self._remote_conf(config),
649 extra_env={'GIT_TERMINAL_PROMPT': '0'})
649 extra_env={'GIT_TERMINAL_PROMPT': '0'})
650
650
651 return remote_refs
651 return remote_refs
652
652
653 @reraise_safe_exceptions
653 @reraise_safe_exceptions
654 def sync_push(self, wire, url, refs=None):
654 def sync_push(self, wire, url, refs=None):
655 if not self.check_url(url, wire):
655 if not self.check_url(url, wire):
656 return
656 return
657 config = self._wire_to_config(wire)
657 config = self._wire_to_config(wire)
658 self._factory.repo(wire)
658 self._factory.repo(wire)
659 self.run_git_command(
659 self.run_git_command(
660 wire, ['push', url, '--mirror'], fail_on_stderr=False,
660 wire, ['push', url, '--mirror'], fail_on_stderr=False,
661 _copts=self._remote_conf(config),
661 _copts=self._remote_conf(config),
662 extra_env={'GIT_TERMINAL_PROMPT': '0'})
662 extra_env={'GIT_TERMINAL_PROMPT': '0'})
663
663
664 @reraise_safe_exceptions
664 @reraise_safe_exceptions
665 def get_remote_refs(self, wire, url):
665 def get_remote_refs(self, wire, url):
666 repo = Repo(url)
666 repo = Repo(url)
667 return repo.get_refs()
667 return repo.get_refs()
668
668
669 @reraise_safe_exceptions
669 @reraise_safe_exceptions
670 def get_description(self, wire):
670 def get_description(self, wire):
671 repo = self._factory.repo(wire)
671 repo = self._factory.repo(wire)
672 return repo.get_description()
672 return repo.get_description()
673
673
674 @reraise_safe_exceptions
674 @reraise_safe_exceptions
675 def get_missing_revs(self, wire, rev1, rev2, path2):
675 def get_missing_revs(self, wire, rev1, rev2, path2):
676 repo = self._factory.repo(wire)
676 repo = self._factory.repo(wire)
677 LocalGitClient(thin_packs=False).fetch(path2, repo)
677 LocalGitClient(thin_packs=False).fetch(path2, repo)
678
678
679 wire_remote = wire.copy()
679 wire_remote = wire.copy()
680 wire_remote['path'] = path2
680 wire_remote['path'] = path2
681 repo_remote = self._factory.repo(wire_remote)
681 repo_remote = self._factory.repo(wire_remote)
682 LocalGitClient(thin_packs=False).fetch(wire["path"], repo_remote)
682 LocalGitClient(thin_packs=False).fetch(wire["path"], repo_remote)
683
683
684 revs = [
684 revs = [
685 x.commit.id
685 x.commit.id
686 for x in repo_remote.get_walker(include=[rev2], exclude=[rev1])]
686 for x in repo_remote.get_walker(include=[rev2], exclude=[rev1])]
687 return revs
687 return revs
688
688
689 @reraise_safe_exceptions
689 @reraise_safe_exceptions
690 def get_object(self, wire, sha):
690 def get_object(self, wire, sha):
691 cache_on, context_uid, repo_id = self._cache_on(wire)
691 cache_on, context_uid, repo_id = self._cache_on(wire)
692 @self.region.conditional_cache_on_arguments(condition=cache_on)
692 @self.region.conditional_cache_on_arguments(condition=cache_on)
693 def _get_object(_context_uid, _repo_id, _sha):
693 def _get_object(_context_uid, _repo_id, _sha):
694 repo_init = self._factory.repo_libgit2(wire)
694 repo_init = self._factory.repo_libgit2(wire)
695 with repo_init as repo:
695 with repo_init as repo:
696
696
697 missing_commit_err = 'Commit {} does not exist for `{}`'.format(sha, wire['path'])
697 missing_commit_err = 'Commit {} does not exist for `{}`'.format(sha, wire['path'])
698 try:
698 try:
699 commit = repo.revparse_single(sha)
699 commit = repo.revparse_single(sha)
700 except (KeyError, ValueError) as e:
700 except KeyError:
701 # NOTE(marcink): KeyError doesn't give us any meaningful information
702 # here, we instead give something more explicit
703 e = exceptions.RefNotFoundException('SHA: %s not found', sha)
704 raise exceptions.LookupException(e)(missing_commit_err)
705 except ValueError as e:
701 raise exceptions.LookupException(e)(missing_commit_err)
706 raise exceptions.LookupException(e)(missing_commit_err)
702
707
703 is_tag = False
708 is_tag = False
704 if isinstance(commit, pygit2.Tag):
709 if isinstance(commit, pygit2.Tag):
705 commit = repo.get(commit.target)
710 commit = repo.get(commit.target)
706 is_tag = True
711 is_tag = True
707
712
708 check_dangling = True
713 check_dangling = True
709 if is_tag:
714 if is_tag:
710 check_dangling = False
715 check_dangling = False
711
716
712 # we used a reference and it parsed means we're not having a dangling commit
717 # we used a reference and it parsed means we're not having a dangling commit
713 if sha != commit.hex:
718 if sha != commit.hex:
714 check_dangling = False
719 check_dangling = False
715
720
716 if check_dangling:
721 if check_dangling:
717 # check for dangling commit
722 # check for dangling commit
718 for branch in repo.branches.with_commit(commit.hex):
723 for branch in repo.branches.with_commit(commit.hex):
719 if branch:
724 if branch:
720 break
725 break
721 else:
726 else:
722 raise exceptions.LookupException(None)(missing_commit_err)
727 # NOTE(marcink): Empty error doesn't give us any meaningful information
728 # here, we instead give something more explicit
729 e = exceptions.RefNotFoundException('SHA: %s not found in branches', sha)
730 raise exceptions.LookupException(e)(missing_commit_err)
723
731
724 commit_id = commit.hex
732 commit_id = commit.hex
725 type_id = commit.type
733 type_id = commit.type
726
734
727 return {
735 return {
728 'id': commit_id,
736 'id': commit_id,
729 'type': self._type_id_to_name(type_id),
737 'type': self._type_id_to_name(type_id),
730 'commit_id': commit_id,
738 'commit_id': commit_id,
731 'idx': 0
739 'idx': 0
732 }
740 }
733
741
734 return _get_object(context_uid, repo_id, sha)
742 return _get_object(context_uid, repo_id, sha)
735
743
736 @reraise_safe_exceptions
744 @reraise_safe_exceptions
737 def get_refs(self, wire):
745 def get_refs(self, wire):
738 cache_on, context_uid, repo_id = self._cache_on(wire)
746 cache_on, context_uid, repo_id = self._cache_on(wire)
739 @self.region.conditional_cache_on_arguments(condition=cache_on)
747 @self.region.conditional_cache_on_arguments(condition=cache_on)
740 def _get_refs(_context_uid, _repo_id):
748 def _get_refs(_context_uid, _repo_id):
741
749
742 repo_init = self._factory.repo_libgit2(wire)
750 repo_init = self._factory.repo_libgit2(wire)
743 with repo_init as repo:
751 with repo_init as repo:
744 regex = re.compile('^refs/(heads|tags)/')
752 regex = re.compile('^refs/(heads|tags)/')
745 return {x.name: x.target.hex for x in
753 return {x.name: x.target.hex for x in
746 filter(lambda ref: regex.match(ref.name) ,repo.listall_reference_objects())}
754 filter(lambda ref: regex.match(ref.name) ,repo.listall_reference_objects())}
747
755
748 return _get_refs(context_uid, repo_id)
756 return _get_refs(context_uid, repo_id)
749
757
750 @reraise_safe_exceptions
758 @reraise_safe_exceptions
751 def get_branch_pointers(self, wire):
759 def get_branch_pointers(self, wire):
752 cache_on, context_uid, repo_id = self._cache_on(wire)
760 cache_on, context_uid, repo_id = self._cache_on(wire)
753 @self.region.conditional_cache_on_arguments(condition=cache_on)
761 @self.region.conditional_cache_on_arguments(condition=cache_on)
754 def _get_branch_pointers(_context_uid, _repo_id):
762 def _get_branch_pointers(_context_uid, _repo_id):
755
763
756 repo_init = self._factory.repo_libgit2(wire)
764 repo_init = self._factory.repo_libgit2(wire)
757 regex = re.compile('^refs/heads')
765 regex = re.compile('^refs/heads')
758 with repo_init as repo:
766 with repo_init as repo:
759 branches = filter(lambda ref: regex.match(ref.name), repo.listall_reference_objects())
767 branches = filter(lambda ref: regex.match(ref.name), repo.listall_reference_objects())
760 return {x.target.hex: x.shorthand for x in branches}
768 return {x.target.hex: x.shorthand for x in branches}
761
769
762 return _get_branch_pointers(context_uid, repo_id)
770 return _get_branch_pointers(context_uid, repo_id)
763
771
764 @reraise_safe_exceptions
772 @reraise_safe_exceptions
765 def head(self, wire, show_exc=True):
773 def head(self, wire, show_exc=True):
766 cache_on, context_uid, repo_id = self._cache_on(wire)
774 cache_on, context_uid, repo_id = self._cache_on(wire)
767 @self.region.conditional_cache_on_arguments(condition=cache_on)
775 @self.region.conditional_cache_on_arguments(condition=cache_on)
768 def _head(_context_uid, _repo_id, _show_exc):
776 def _head(_context_uid, _repo_id, _show_exc):
769 repo_init = self._factory.repo_libgit2(wire)
777 repo_init = self._factory.repo_libgit2(wire)
770 with repo_init as repo:
778 with repo_init as repo:
771 try:
779 try:
772 return repo.head.peel().hex
780 return repo.head.peel().hex
773 except Exception:
781 except Exception:
774 if show_exc:
782 if show_exc:
775 raise
783 raise
776 return _head(context_uid, repo_id, show_exc)
784 return _head(context_uid, repo_id, show_exc)
777
785
778 @reraise_safe_exceptions
786 @reraise_safe_exceptions
779 def init(self, wire):
787 def init(self, wire):
780 repo_path = str_to_dulwich(wire['path'])
788 repo_path = str_to_dulwich(wire['path'])
781 self.repo = Repo.init(repo_path)
789 self.repo = Repo.init(repo_path)
782
790
783 @reraise_safe_exceptions
791 @reraise_safe_exceptions
784 def init_bare(self, wire):
792 def init_bare(self, wire):
785 repo_path = str_to_dulwich(wire['path'])
793 repo_path = str_to_dulwich(wire['path'])
786 self.repo = Repo.init_bare(repo_path)
794 self.repo = Repo.init_bare(repo_path)
787
795
788 @reraise_safe_exceptions
796 @reraise_safe_exceptions
789 def revision(self, wire, rev):
797 def revision(self, wire, rev):
790
798
791 cache_on, context_uid, repo_id = self._cache_on(wire)
799 cache_on, context_uid, repo_id = self._cache_on(wire)
792 @self.region.conditional_cache_on_arguments(condition=cache_on)
800 @self.region.conditional_cache_on_arguments(condition=cache_on)
793 def _revision(_context_uid, _repo_id, _rev):
801 def _revision(_context_uid, _repo_id, _rev):
794 repo_init = self._factory.repo_libgit2(wire)
802 repo_init = self._factory.repo_libgit2(wire)
795 with repo_init as repo:
803 with repo_init as repo:
796 commit = repo[rev]
804 commit = repo[rev]
797 obj_data = {
805 obj_data = {
798 'id': commit.id.hex,
806 'id': commit.id.hex,
799 }
807 }
800 # tree objects itself don't have tree_id attribute
808 # tree objects itself don't have tree_id attribute
801 if hasattr(commit, 'tree_id'):
809 if hasattr(commit, 'tree_id'):
802 obj_data['tree'] = commit.tree_id.hex
810 obj_data['tree'] = commit.tree_id.hex
803
811
804 return obj_data
812 return obj_data
805 return _revision(context_uid, repo_id, rev)
813 return _revision(context_uid, repo_id, rev)
806
814
807 @reraise_safe_exceptions
815 @reraise_safe_exceptions
808 def date(self, wire, commit_id):
816 def date(self, wire, commit_id):
809 cache_on, context_uid, repo_id = self._cache_on(wire)
817 cache_on, context_uid, repo_id = self._cache_on(wire)
810 @self.region.conditional_cache_on_arguments(condition=cache_on)
818 @self.region.conditional_cache_on_arguments(condition=cache_on)
811 def _date(_repo_id, _commit_id):
819 def _date(_repo_id, _commit_id):
812 repo_init = self._factory.repo_libgit2(wire)
820 repo_init = self._factory.repo_libgit2(wire)
813 with repo_init as repo:
821 with repo_init as repo:
814 commit = repo[commit_id]
822 commit = repo[commit_id]
815
823
816 if hasattr(commit, 'commit_time'):
824 if hasattr(commit, 'commit_time'):
817 commit_time, commit_time_offset = commit.commit_time, commit.commit_time_offset
825 commit_time, commit_time_offset = commit.commit_time, commit.commit_time_offset
818 else:
826 else:
819 commit = commit.get_object()
827 commit = commit.get_object()
820 commit_time, commit_time_offset = commit.commit_time, commit.commit_time_offset
828 commit_time, commit_time_offset = commit.commit_time, commit.commit_time_offset
821
829
822 # TODO(marcink): check dulwich difference of offset vs timezone
830 # TODO(marcink): check dulwich difference of offset vs timezone
823 return [commit_time, commit_time_offset]
831 return [commit_time, commit_time_offset]
824 return _date(repo_id, commit_id)
832 return _date(repo_id, commit_id)
825
833
826 @reraise_safe_exceptions
834 @reraise_safe_exceptions
827 def author(self, wire, commit_id):
835 def author(self, wire, commit_id):
828 cache_on, context_uid, repo_id = self._cache_on(wire)
836 cache_on, context_uid, repo_id = self._cache_on(wire)
829 @self.region.conditional_cache_on_arguments(condition=cache_on)
837 @self.region.conditional_cache_on_arguments(condition=cache_on)
830 def _author(_repo_id, _commit_id):
838 def _author(_repo_id, _commit_id):
831 repo_init = self._factory.repo_libgit2(wire)
839 repo_init = self._factory.repo_libgit2(wire)
832 with repo_init as repo:
840 with repo_init as repo:
833 commit = repo[commit_id]
841 commit = repo[commit_id]
834
842
835 if hasattr(commit, 'author'):
843 if hasattr(commit, 'author'):
836 author = commit.author
844 author = commit.author
837 else:
845 else:
838 author = commit.get_object().author
846 author = commit.get_object().author
839
847
840 if author.email:
848 if author.email:
841 return u"{} <{}>".format(author.name, author.email)
849 return u"{} <{}>".format(author.name, author.email)
842
850
843 try:
851 try:
844 return u"{}".format(author.name)
852 return u"{}".format(author.name)
845 except Exception:
853 except Exception:
846 return u"{}".format(safe_unicode(author.raw_name))
854 return u"{}".format(safe_unicode(author.raw_name))
847
855
848 return _author(repo_id, commit_id)
856 return _author(repo_id, commit_id)
849
857
850 @reraise_safe_exceptions
858 @reraise_safe_exceptions
851 def message(self, wire, commit_id):
859 def message(self, wire, commit_id):
852 cache_on, context_uid, repo_id = self._cache_on(wire)
860 cache_on, context_uid, repo_id = self._cache_on(wire)
853 @self.region.conditional_cache_on_arguments(condition=cache_on)
861 @self.region.conditional_cache_on_arguments(condition=cache_on)
854 def _message(_repo_id, _commit_id):
862 def _message(_repo_id, _commit_id):
855 repo_init = self._factory.repo_libgit2(wire)
863 repo_init = self._factory.repo_libgit2(wire)
856 with repo_init as repo:
864 with repo_init as repo:
857 commit = repo[commit_id]
865 commit = repo[commit_id]
858 return commit.message
866 return commit.message
859 return _message(repo_id, commit_id)
867 return _message(repo_id, commit_id)
860
868
861 @reraise_safe_exceptions
869 @reraise_safe_exceptions
862 def parents(self, wire, commit_id):
870 def parents(self, wire, commit_id):
863 cache_on, context_uid, repo_id = self._cache_on(wire)
871 cache_on, context_uid, repo_id = self._cache_on(wire)
864 @self.region.conditional_cache_on_arguments(condition=cache_on)
872 @self.region.conditional_cache_on_arguments(condition=cache_on)
865 def _parents(_repo_id, _commit_id):
873 def _parents(_repo_id, _commit_id):
866 repo_init = self._factory.repo_libgit2(wire)
874 repo_init = self._factory.repo_libgit2(wire)
867 with repo_init as repo:
875 with repo_init as repo:
868 commit = repo[commit_id]
876 commit = repo[commit_id]
869 if hasattr(commit, 'parent_ids'):
877 if hasattr(commit, 'parent_ids'):
870 parent_ids = commit.parent_ids
878 parent_ids = commit.parent_ids
871 else:
879 else:
872 parent_ids = commit.get_object().parent_ids
880 parent_ids = commit.get_object().parent_ids
873
881
874 return [x.hex for x in parent_ids]
882 return [x.hex for x in parent_ids]
875 return _parents(repo_id, commit_id)
883 return _parents(repo_id, commit_id)
876
884
877 @reraise_safe_exceptions
885 @reraise_safe_exceptions
878 def children(self, wire, commit_id):
886 def children(self, wire, commit_id):
879 cache_on, context_uid, repo_id = self._cache_on(wire)
887 cache_on, context_uid, repo_id = self._cache_on(wire)
880 @self.region.conditional_cache_on_arguments(condition=cache_on)
888 @self.region.conditional_cache_on_arguments(condition=cache_on)
881 def _children(_repo_id, _commit_id):
889 def _children(_repo_id, _commit_id):
882 output, __ = self.run_git_command(
890 output, __ = self.run_git_command(
883 wire, ['rev-list', '--all', '--children'])
891 wire, ['rev-list', '--all', '--children'])
884
892
885 child_ids = []
893 child_ids = []
886 pat = re.compile(r'^%s' % commit_id)
894 pat = re.compile(r'^%s' % commit_id)
887 for l in output.splitlines():
895 for l in output.splitlines():
888 if pat.match(l):
896 if pat.match(l):
889 found_ids = l.split(' ')[1:]
897 found_ids = l.split(' ')[1:]
890 child_ids.extend(found_ids)
898 child_ids.extend(found_ids)
891
899
892 return child_ids
900 return child_ids
893 return _children(repo_id, commit_id)
901 return _children(repo_id, commit_id)
894
902
895 @reraise_safe_exceptions
903 @reraise_safe_exceptions
896 def set_refs(self, wire, key, value):
904 def set_refs(self, wire, key, value):
897 repo_init = self._factory.repo_libgit2(wire)
905 repo_init = self._factory.repo_libgit2(wire)
898 with repo_init as repo:
906 with repo_init as repo:
899 repo.references.create(key, value, force=True)
907 repo.references.create(key, value, force=True)
900
908
901 @reraise_safe_exceptions
909 @reraise_safe_exceptions
902 def create_branch(self, wire, branch_name, commit_id, force=False):
910 def create_branch(self, wire, branch_name, commit_id, force=False):
903 repo_init = self._factory.repo_libgit2(wire)
911 repo_init = self._factory.repo_libgit2(wire)
904 with repo_init as repo:
912 with repo_init as repo:
905 commit = repo[commit_id]
913 commit = repo[commit_id]
906
914
907 if force:
915 if force:
908 repo.branches.local.create(branch_name, commit, force=force)
916 repo.branches.local.create(branch_name, commit, force=force)
909 elif not repo.branches.get(branch_name):
917 elif not repo.branches.get(branch_name):
910 # create only if that branch isn't existing
918 # create only if that branch isn't existing
911 repo.branches.local.create(branch_name, commit, force=force)
919 repo.branches.local.create(branch_name, commit, force=force)
912
920
913 @reraise_safe_exceptions
921 @reraise_safe_exceptions
914 def remove_ref(self, wire, key):
922 def remove_ref(self, wire, key):
915 repo_init = self._factory.repo_libgit2(wire)
923 repo_init = self._factory.repo_libgit2(wire)
916 with repo_init as repo:
924 with repo_init as repo:
917 repo.references.delete(key)
925 repo.references.delete(key)
918
926
919 @reraise_safe_exceptions
927 @reraise_safe_exceptions
920 def tag_remove(self, wire, tag_name):
928 def tag_remove(self, wire, tag_name):
921 repo_init = self._factory.repo_libgit2(wire)
929 repo_init = self._factory.repo_libgit2(wire)
922 with repo_init as repo:
930 with repo_init as repo:
923 key = 'refs/tags/{}'.format(tag_name)
931 key = 'refs/tags/{}'.format(tag_name)
924 repo.references.delete(key)
932 repo.references.delete(key)
925
933
926 @reraise_safe_exceptions
934 @reraise_safe_exceptions
927 def tree_changes(self, wire, source_id, target_id):
935 def tree_changes(self, wire, source_id, target_id):
928 # TODO(marcink): remove this seems it's only used by tests
936 # TODO(marcink): remove this seems it's only used by tests
929 repo = self._factory.repo(wire)
937 repo = self._factory.repo(wire)
930 source = repo[source_id].tree if source_id else None
938 source = repo[source_id].tree if source_id else None
931 target = repo[target_id].tree
939 target = repo[target_id].tree
932 result = repo.object_store.tree_changes(source, target)
940 result = repo.object_store.tree_changes(source, target)
933 return list(result)
941 return list(result)
934
942
935 @reraise_safe_exceptions
943 @reraise_safe_exceptions
936 def tree_and_type_for_path(self, wire, commit_id, path):
944 def tree_and_type_for_path(self, wire, commit_id, path):
937
945
938 cache_on, context_uid, repo_id = self._cache_on(wire)
946 cache_on, context_uid, repo_id = self._cache_on(wire)
939 @self.region.conditional_cache_on_arguments(condition=cache_on)
947 @self.region.conditional_cache_on_arguments(condition=cache_on)
940 def _tree_and_type_for_path(_context_uid, _repo_id, _commit_id, _path):
948 def _tree_and_type_for_path(_context_uid, _repo_id, _commit_id, _path):
941 repo_init = self._factory.repo_libgit2(wire)
949 repo_init = self._factory.repo_libgit2(wire)
942
950
943 with repo_init as repo:
951 with repo_init as repo:
944 commit = repo[commit_id]
952 commit = repo[commit_id]
945 try:
953 try:
946 tree = commit.tree[path]
954 tree = commit.tree[path]
947 except KeyError:
955 except KeyError:
948 return None, None, None
956 return None, None, None
949
957
950 return tree.id.hex, tree.type, tree.filemode
958 return tree.id.hex, tree.type, tree.filemode
951 return _tree_and_type_for_path(context_uid, repo_id, commit_id, path)
959 return _tree_and_type_for_path(context_uid, repo_id, commit_id, path)
952
960
953 @reraise_safe_exceptions
961 @reraise_safe_exceptions
954 def tree_items(self, wire, tree_id):
962 def tree_items(self, wire, tree_id):
955 cache_on, context_uid, repo_id = self._cache_on(wire)
963 cache_on, context_uid, repo_id = self._cache_on(wire)
956 @self.region.conditional_cache_on_arguments(condition=cache_on)
964 @self.region.conditional_cache_on_arguments(condition=cache_on)
957 def _tree_items(_repo_id, _tree_id):
965 def _tree_items(_repo_id, _tree_id):
958
966
959 repo_init = self._factory.repo_libgit2(wire)
967 repo_init = self._factory.repo_libgit2(wire)
960 with repo_init as repo:
968 with repo_init as repo:
961 try:
969 try:
962 tree = repo[tree_id]
970 tree = repo[tree_id]
963 except KeyError:
971 except KeyError:
964 raise ObjectMissing('No tree with id: {}'.format(tree_id))
972 raise ObjectMissing('No tree with id: {}'.format(tree_id))
965
973
966 result = []
974 result = []
967 for item in tree:
975 for item in tree:
968 item_sha = item.hex
976 item_sha = item.hex
969 item_mode = item.filemode
977 item_mode = item.filemode
970 item_type = item.type
978 item_type = item.type
971
979
972 if item_type == 'commit':
980 if item_type == 'commit':
973 # NOTE(marcink): submodules we translate to 'link' for backward compat
981 # NOTE(marcink): submodules we translate to 'link' for backward compat
974 item_type = 'link'
982 item_type = 'link'
975
983
976 result.append((item.name, item_mode, item_sha, item_type))
984 result.append((item.name, item_mode, item_sha, item_type))
977 return result
985 return result
978 return _tree_items(repo_id, tree_id)
986 return _tree_items(repo_id, tree_id)
979
987
980 @reraise_safe_exceptions
988 @reraise_safe_exceptions
981 def diff_2(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
989 def diff_2(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
982 """
990 """
983 Old version that uses subprocess to call diff
991 Old version that uses subprocess to call diff
984 """
992 """
985
993
986 flags = [
994 flags = [
987 '-U%s' % context, '--patch',
995 '-U%s' % context, '--patch',
988 '--binary',
996 '--binary',
989 '--find-renames',
997 '--find-renames',
990 '--no-indent-heuristic',
998 '--no-indent-heuristic',
991 # '--indent-heuristic',
999 # '--indent-heuristic',
992 #'--full-index',
1000 #'--full-index',
993 #'--abbrev=40'
1001 #'--abbrev=40'
994 ]
1002 ]
995
1003
996 if opt_ignorews:
1004 if opt_ignorews:
997 flags.append('--ignore-all-space')
1005 flags.append('--ignore-all-space')
998
1006
999 if commit_id_1 == self.EMPTY_COMMIT:
1007 if commit_id_1 == self.EMPTY_COMMIT:
1000 cmd = ['show'] + flags + [commit_id_2]
1008 cmd = ['show'] + flags + [commit_id_2]
1001 else:
1009 else:
1002 cmd = ['diff'] + flags + [commit_id_1, commit_id_2]
1010 cmd = ['diff'] + flags + [commit_id_1, commit_id_2]
1003
1011
1004 if file_filter:
1012 if file_filter:
1005 cmd.extend(['--', file_filter])
1013 cmd.extend(['--', file_filter])
1006
1014
1007 diff, __ = self.run_git_command(wire, cmd)
1015 diff, __ = self.run_git_command(wire, cmd)
1008 # If we used 'show' command, strip first few lines (until actual diff
1016 # If we used 'show' command, strip first few lines (until actual diff
1009 # starts)
1017 # starts)
1010 if commit_id_1 == self.EMPTY_COMMIT:
1018 if commit_id_1 == self.EMPTY_COMMIT:
1011 lines = diff.splitlines()
1019 lines = diff.splitlines()
1012 x = 0
1020 x = 0
1013 for line in lines:
1021 for line in lines:
1014 if line.startswith('diff'):
1022 if line.startswith('diff'):
1015 break
1023 break
1016 x += 1
1024 x += 1
1017 # Append new line just like 'diff' command do
1025 # Append new line just like 'diff' command do
1018 diff = '\n'.join(lines[x:]) + '\n'
1026 diff = '\n'.join(lines[x:]) + '\n'
1019 return diff
1027 return diff
1020
1028
1021 @reraise_safe_exceptions
1029 @reraise_safe_exceptions
1022 def diff(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
1030 def diff(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
1023 repo_init = self._factory.repo_libgit2(wire)
1031 repo_init = self._factory.repo_libgit2(wire)
1024 with repo_init as repo:
1032 with repo_init as repo:
1025 swap = True
1033 swap = True
1026 flags = 0
1034 flags = 0
1027 flags |= pygit2.GIT_DIFF_SHOW_BINARY
1035 flags |= pygit2.GIT_DIFF_SHOW_BINARY
1028
1036
1029 if opt_ignorews:
1037 if opt_ignorews:
1030 flags |= pygit2.GIT_DIFF_IGNORE_WHITESPACE
1038 flags |= pygit2.GIT_DIFF_IGNORE_WHITESPACE
1031
1039
1032 if commit_id_1 == self.EMPTY_COMMIT:
1040 if commit_id_1 == self.EMPTY_COMMIT:
1033 comm1 = repo[commit_id_2]
1041 comm1 = repo[commit_id_2]
1034 diff_obj = comm1.tree.diff_to_tree(
1042 diff_obj = comm1.tree.diff_to_tree(
1035 flags=flags, context_lines=context, swap=swap)
1043 flags=flags, context_lines=context, swap=swap)
1036
1044
1037 else:
1045 else:
1038 comm1 = repo[commit_id_2]
1046 comm1 = repo[commit_id_2]
1039 comm2 = repo[commit_id_1]
1047 comm2 = repo[commit_id_1]
1040 diff_obj = comm1.tree.diff_to_tree(
1048 diff_obj = comm1.tree.diff_to_tree(
1041 comm2.tree, flags=flags, context_lines=context, swap=swap)
1049 comm2.tree, flags=flags, context_lines=context, swap=swap)
1042 similar_flags = 0
1050 similar_flags = 0
1043 similar_flags |= pygit2.GIT_DIFF_FIND_RENAMES
1051 similar_flags |= pygit2.GIT_DIFF_FIND_RENAMES
1044 diff_obj.find_similar(flags=similar_flags)
1052 diff_obj.find_similar(flags=similar_flags)
1045
1053
1046 if file_filter:
1054 if file_filter:
1047 for p in diff_obj:
1055 for p in diff_obj:
1048 if p.delta.old_file.path == file_filter:
1056 if p.delta.old_file.path == file_filter:
1049 return p.patch or ''
1057 return p.patch or ''
1050 # fo matching path == no diff
1058 # fo matching path == no diff
1051 return ''
1059 return ''
1052 return diff_obj.patch or ''
1060 return diff_obj.patch or ''
1053
1061
1054 @reraise_safe_exceptions
1062 @reraise_safe_exceptions
1055 def node_history(self, wire, commit_id, path, limit):
1063 def node_history(self, wire, commit_id, path, limit):
1056 cache_on, context_uid, repo_id = self._cache_on(wire)
1064 cache_on, context_uid, repo_id = self._cache_on(wire)
1057 @self.region.conditional_cache_on_arguments(condition=cache_on)
1065 @self.region.conditional_cache_on_arguments(condition=cache_on)
1058 def _node_history(_context_uid, _repo_id, _commit_id, _path, _limit):
1066 def _node_history(_context_uid, _repo_id, _commit_id, _path, _limit):
1059 # optimize for n==1, rev-list is much faster for that use-case
1067 # optimize for n==1, rev-list is much faster for that use-case
1060 if limit == 1:
1068 if limit == 1:
1061 cmd = ['rev-list', '-1', commit_id, '--', path]
1069 cmd = ['rev-list', '-1', commit_id, '--', path]
1062 else:
1070 else:
1063 cmd = ['log']
1071 cmd = ['log']
1064 if limit:
1072 if limit:
1065 cmd.extend(['-n', str(safe_int(limit, 0))])
1073 cmd.extend(['-n', str(safe_int(limit, 0))])
1066 cmd.extend(['--pretty=format: %H', '-s', commit_id, '--', path])
1074 cmd.extend(['--pretty=format: %H', '-s', commit_id, '--', path])
1067
1075
1068 output, __ = self.run_git_command(wire, cmd)
1076 output, __ = self.run_git_command(wire, cmd)
1069 commit_ids = re.findall(r'[0-9a-fA-F]{40}', output)
1077 commit_ids = re.findall(r'[0-9a-fA-F]{40}', output)
1070
1078
1071 return [x for x in commit_ids]
1079 return [x for x in commit_ids]
1072 return _node_history(context_uid, repo_id, commit_id, path, limit)
1080 return _node_history(context_uid, repo_id, commit_id, path, limit)
1073
1081
1074 @reraise_safe_exceptions
1082 @reraise_safe_exceptions
1075 def node_annotate(self, wire, commit_id, path):
1083 def node_annotate(self, wire, commit_id, path):
1076
1084
1077 cmd = ['blame', '-l', '--root', '-r', commit_id, '--', path]
1085 cmd = ['blame', '-l', '--root', '-r', commit_id, '--', path]
1078 # -l ==> outputs long shas (and we need all 40 characters)
1086 # -l ==> outputs long shas (and we need all 40 characters)
1079 # --root ==> doesn't put '^' character for boundaries
1087 # --root ==> doesn't put '^' character for boundaries
1080 # -r commit_id ==> blames for the given commit
1088 # -r commit_id ==> blames for the given commit
1081 output, __ = self.run_git_command(wire, cmd)
1089 output, __ = self.run_git_command(wire, cmd)
1082
1090
1083 result = []
1091 result = []
1084 for i, blame_line in enumerate(output.split('\n')[:-1]):
1092 for i, blame_line in enumerate(output.split('\n')[:-1]):
1085 line_no = i + 1
1093 line_no = i + 1
1086 commit_id, line = re.split(r' ', blame_line, 1)
1094 commit_id, line = re.split(r' ', blame_line, 1)
1087 result.append((line_no, commit_id, line))
1095 result.append((line_no, commit_id, line))
1088 return result
1096 return result
1089
1097
1090 @reraise_safe_exceptions
1098 @reraise_safe_exceptions
1091 def update_server_info(self, wire):
1099 def update_server_info(self, wire):
1092 repo = self._factory.repo(wire)
1100 repo = self._factory.repo(wire)
1093 update_server_info(repo)
1101 update_server_info(repo)
1094
1102
1095 @reraise_safe_exceptions
1103 @reraise_safe_exceptions
1096 def get_all_commit_ids(self, wire):
1104 def get_all_commit_ids(self, wire):
1097
1105
1098 cache_on, context_uid, repo_id = self._cache_on(wire)
1106 cache_on, context_uid, repo_id = self._cache_on(wire)
1099 @self.region.conditional_cache_on_arguments(condition=cache_on)
1107 @self.region.conditional_cache_on_arguments(condition=cache_on)
1100 def _get_all_commit_ids(_context_uid, _repo_id):
1108 def _get_all_commit_ids(_context_uid, _repo_id):
1101
1109
1102 cmd = ['rev-list', '--reverse', '--date-order', '--branches', '--tags']
1110 cmd = ['rev-list', '--reverse', '--date-order', '--branches', '--tags']
1103 try:
1111 try:
1104 output, __ = self.run_git_command(wire, cmd)
1112 output, __ = self.run_git_command(wire, cmd)
1105 return output.splitlines()
1113 return output.splitlines()
1106 except Exception:
1114 except Exception:
1107 # Can be raised for empty repositories
1115 # Can be raised for empty repositories
1108 return []
1116 return []
1109 return _get_all_commit_ids(context_uid, repo_id)
1117 return _get_all_commit_ids(context_uid, repo_id)
1110
1118
1111 @reraise_safe_exceptions
1119 @reraise_safe_exceptions
1112 def run_git_command(self, wire, cmd, **opts):
1120 def run_git_command(self, wire, cmd, **opts):
1113 path = wire.get('path', None)
1121 path = wire.get('path', None)
1114
1122
1115 if path and os.path.isdir(path):
1123 if path and os.path.isdir(path):
1116 opts['cwd'] = path
1124 opts['cwd'] = path
1117
1125
1118 if '_bare' in opts:
1126 if '_bare' in opts:
1119 _copts = []
1127 _copts = []
1120 del opts['_bare']
1128 del opts['_bare']
1121 else:
1129 else:
1122 _copts = ['-c', 'core.quotepath=false', ]
1130 _copts = ['-c', 'core.quotepath=false', ]
1123 safe_call = False
1131 safe_call = False
1124 if '_safe' in opts:
1132 if '_safe' in opts:
1125 # no exc on failure
1133 # no exc on failure
1126 del opts['_safe']
1134 del opts['_safe']
1127 safe_call = True
1135 safe_call = True
1128
1136
1129 if '_copts' in opts:
1137 if '_copts' in opts:
1130 _copts.extend(opts['_copts'] or [])
1138 _copts.extend(opts['_copts'] or [])
1131 del opts['_copts']
1139 del opts['_copts']
1132
1140
1133 gitenv = os.environ.copy()
1141 gitenv = os.environ.copy()
1134 gitenv.update(opts.pop('extra_env', {}))
1142 gitenv.update(opts.pop('extra_env', {}))
1135 # need to clean fix GIT_DIR !
1143 # need to clean fix GIT_DIR !
1136 if 'GIT_DIR' in gitenv:
1144 if 'GIT_DIR' in gitenv:
1137 del gitenv['GIT_DIR']
1145 del gitenv['GIT_DIR']
1138 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
1146 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
1139 gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1'
1147 gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1'
1140
1148
1141 cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
1149 cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
1142 _opts = {'env': gitenv, 'shell': False}
1150 _opts = {'env': gitenv, 'shell': False}
1143
1151
1144 proc = None
1152 proc = None
1145 try:
1153 try:
1146 _opts.update(opts)
1154 _opts.update(opts)
1147 proc = subprocessio.SubprocessIOChunker(cmd, **_opts)
1155 proc = subprocessio.SubprocessIOChunker(cmd, **_opts)
1148
1156
1149 return ''.join(proc), ''.join(proc.error)
1157 return ''.join(proc), ''.join(proc.error)
1150 except (EnvironmentError, OSError) as err:
1158 except (EnvironmentError, OSError) as err:
1151 cmd = ' '.join(cmd) # human friendly CMD
1159 cmd = ' '.join(cmd) # human friendly CMD
1152 tb_err = ("Couldn't run git command (%s).\n"
1160 tb_err = ("Couldn't run git command (%s).\n"
1153 "Original error was:%s\n"
1161 "Original error was:%s\n"
1154 "Call options:%s\n"
1162 "Call options:%s\n"
1155 % (cmd, err, _opts))
1163 % (cmd, err, _opts))
1156 log.exception(tb_err)
1164 log.exception(tb_err)
1157 if safe_call:
1165 if safe_call:
1158 return '', err
1166 return '', err
1159 else:
1167 else:
1160 raise exceptions.VcsException()(tb_err)
1168 raise exceptions.VcsException()(tb_err)
1161 finally:
1169 finally:
1162 if proc:
1170 if proc:
1163 proc.close()
1171 proc.close()
1164
1172
1165 @reraise_safe_exceptions
1173 @reraise_safe_exceptions
1166 def install_hooks(self, wire, force=False):
1174 def install_hooks(self, wire, force=False):
1167 from vcsserver.hook_utils import install_git_hooks
1175 from vcsserver.hook_utils import install_git_hooks
1168 bare = self.bare(wire)
1176 bare = self.bare(wire)
1169 path = wire['path']
1177 path = wire['path']
1170 return install_git_hooks(path, bare, force_create=force)
1178 return install_git_hooks(path, bare, force_create=force)
1171
1179
1172 @reraise_safe_exceptions
1180 @reraise_safe_exceptions
1173 def get_hooks_info(self, wire):
1181 def get_hooks_info(self, wire):
1174 from vcsserver.hook_utils import (
1182 from vcsserver.hook_utils import (
1175 get_git_pre_hook_version, get_git_post_hook_version)
1183 get_git_pre_hook_version, get_git_post_hook_version)
1176 bare = self.bare(wire)
1184 bare = self.bare(wire)
1177 path = wire['path']
1185 path = wire['path']
1178 return {
1186 return {
1179 'pre_version': get_git_pre_hook_version(path, bare),
1187 'pre_version': get_git_pre_hook_version(path, bare),
1180 'post_version': get_git_post_hook_version(path, bare),
1188 'post_version': get_git_post_hook_version(path, bare),
1181 }
1189 }
@@ -1,675 +1,688 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2019 RhodeCode GmbH
2 # Copyright (C) 2014-2019 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import os
18 import os
19 import sys
19 import sys
20 import base64
20 import base64
21 import locale
21 import locale
22 import logging
22 import logging
23 import uuid
23 import uuid
24 import wsgiref.util
24 import wsgiref.util
25 import traceback
25 import traceback
26 import tempfile
26 import tempfile
27 from itertools import chain
27 from itertools import chain
28 from cStringIO import StringIO
28 from cStringIO import StringIO
29
29
30 import simplejson as json
30 import simplejson as json
31 import msgpack
31 import msgpack
32 from pyramid.config import Configurator
32 from pyramid.config import Configurator
33 from pyramid.settings import asbool, aslist
33 from pyramid.settings import asbool, aslist
34 from pyramid.wsgi import wsgiapp
34 from pyramid.wsgi import wsgiapp
35 from pyramid.compat import configparser
35 from pyramid.compat import configparser
36 from pyramid.response import Response
36 from pyramid.response import Response
37
37
38 from vcsserver.utils import safe_int
38 from vcsserver.utils import safe_int
39
39
40 log = logging.getLogger(__name__)
40 log = logging.getLogger(__name__)
41
41
42 # due to Mercurial/glibc2.27 problems we need to detect if locale settings are
42 # due to Mercurial/glibc2.27 problems we need to detect if locale settings are
43 # causing problems and "fix" it in case they do and fallback to LC_ALL = C
43 # causing problems and "fix" it in case they do and fallback to LC_ALL = C
44
44
45 try:
45 try:
46 locale.setlocale(locale.LC_ALL, '')
46 locale.setlocale(locale.LC_ALL, '')
47 except locale.Error as e:
47 except locale.Error as e:
48 log.error(
48 log.error(
49 'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
49 'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
50 os.environ['LC_ALL'] = 'C'
50 os.environ['LC_ALL'] = 'C'
51
51
52 import vcsserver
52 import vcsserver
53 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
53 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
54 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
54 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
55 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
55 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
56 from vcsserver.echo_stub.echo_app import EchoApp
56 from vcsserver.echo_stub.echo_app import EchoApp
57 from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected
57 from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected
58 from vcsserver.lib.exc_tracking import store_exception
58 from vcsserver.lib.exc_tracking import store_exception
59 from vcsserver.server import VcsServer
59 from vcsserver.server import VcsServer
60
60
61 try:
61 try:
62 from vcsserver.git import GitFactory, GitRemote
62 from vcsserver.git import GitFactory, GitRemote
63 except ImportError:
63 except ImportError:
64 GitFactory = None
64 GitFactory = None
65 GitRemote = None
65 GitRemote = None
66
66
67 try:
67 try:
68 from vcsserver.hg import MercurialFactory, HgRemote
68 from vcsserver.hg import MercurialFactory, HgRemote
69 except ImportError:
69 except ImportError:
70 MercurialFactory = None
70 MercurialFactory = None
71 HgRemote = None
71 HgRemote = None
72
72
73 try:
73 try:
74 from vcsserver.svn import SubversionFactory, SvnRemote
74 from vcsserver.svn import SubversionFactory, SvnRemote
75 except ImportError:
75 except ImportError:
76 SubversionFactory = None
76 SubversionFactory = None
77 SvnRemote = None
77 SvnRemote = None
78
78
79
79
80 def _is_request_chunked(environ):
80 def _is_request_chunked(environ):
81 stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
81 stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
82 return stream
82 return stream
83
83
84
84
85 def _int_setting(settings, name, default):
85 def _int_setting(settings, name, default):
86 settings[name] = int(settings.get(name, default))
86 settings[name] = int(settings.get(name, default))
87 return settings[name]
87 return settings[name]
88
88
89
89
90 def _bool_setting(settings, name, default):
90 def _bool_setting(settings, name, default):
91 input_val = settings.get(name, default)
91 input_val = settings.get(name, default)
92 if isinstance(input_val, unicode):
92 if isinstance(input_val, unicode):
93 input_val = input_val.encode('utf8')
93 input_val = input_val.encode('utf8')
94 settings[name] = asbool(input_val)
94 settings[name] = asbool(input_val)
95 return settings[name]
95 return settings[name]
96
96
97
97
98 def _list_setting(settings, name, default):
98 def _list_setting(settings, name, default):
99 raw_value = settings.get(name, default)
99 raw_value = settings.get(name, default)
100
100
101 # Otherwise we assume it uses pyramids space/newline separation.
101 # Otherwise we assume it uses pyramids space/newline separation.
102 settings[name] = aslist(raw_value)
102 settings[name] = aslist(raw_value)
103 return settings[name]
103 return settings[name]
104
104
105
105
106 def _string_setting(settings, name, default, lower=True, default_when_empty=False):
106 def _string_setting(settings, name, default, lower=True, default_when_empty=False):
107 value = settings.get(name, default)
107 value = settings.get(name, default)
108
108
109 if default_when_empty and not value:
109 if default_when_empty and not value:
110 # use default value when value is empty
110 # use default value when value is empty
111 value = default
111 value = default
112
112
113 if lower:
113 if lower:
114 value = value.lower()
114 value = value.lower()
115 settings[name] = value
115 settings[name] = value
116 return settings[name]
116 return settings[name]
117
117
118
118
119 class VCS(object):
119 class VCS(object):
120 def __init__(self, locale_conf=None, cache_config=None):
120 def __init__(self, locale_conf=None, cache_config=None):
121 self.locale = locale_conf
121 self.locale = locale_conf
122 self.cache_config = cache_config
122 self.cache_config = cache_config
123 self._configure_locale()
123 self._configure_locale()
124
124
125 if GitFactory and GitRemote:
125 if GitFactory and GitRemote:
126 git_factory = GitFactory()
126 git_factory = GitFactory()
127 self._git_remote = GitRemote(git_factory)
127 self._git_remote = GitRemote(git_factory)
128 else:
128 else:
129 log.info("Git client import failed")
129 log.info("Git client import failed")
130
130
131 if MercurialFactory and HgRemote:
131 if MercurialFactory and HgRemote:
132 hg_factory = MercurialFactory()
132 hg_factory = MercurialFactory()
133 self._hg_remote = HgRemote(hg_factory)
133 self._hg_remote = HgRemote(hg_factory)
134 else:
134 else:
135 log.info("Mercurial client import failed")
135 log.info("Mercurial client import failed")
136
136
137 if SubversionFactory and SvnRemote:
137 if SubversionFactory and SvnRemote:
138 svn_factory = SubversionFactory()
138 svn_factory = SubversionFactory()
139
139
140 # hg factory is used for svn url validation
140 # hg factory is used for svn url validation
141 hg_factory = MercurialFactory()
141 hg_factory = MercurialFactory()
142 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
142 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
143 else:
143 else:
144 log.info("Subversion client import failed")
144 log.info("Subversion client import failed")
145
145
146 self._vcsserver = VcsServer()
146 self._vcsserver = VcsServer()
147
147
148 def _configure_locale(self):
148 def _configure_locale(self):
149 if self.locale:
149 if self.locale:
150 log.info('Settings locale: `LC_ALL` to %s', self.locale)
150 log.info('Settings locale: `LC_ALL` to %s', self.locale)
151 else:
151 else:
152 log.info(
152 log.info(
153 'Configuring locale subsystem based on environment variables')
153 'Configuring locale subsystem based on environment variables')
154 try:
154 try:
155 # If self.locale is the empty string, then the locale
155 # If self.locale is the empty string, then the locale
156 # module will use the environment variables. See the
156 # module will use the environment variables. See the
157 # documentation of the package `locale`.
157 # documentation of the package `locale`.
158 locale.setlocale(locale.LC_ALL, self.locale)
158 locale.setlocale(locale.LC_ALL, self.locale)
159
159
160 language_code, encoding = locale.getlocale()
160 language_code, encoding = locale.getlocale()
161 log.info(
161 log.info(
162 'Locale set to language code "%s" with encoding "%s".',
162 'Locale set to language code "%s" with encoding "%s".',
163 language_code, encoding)
163 language_code, encoding)
164 except locale.Error:
164 except locale.Error:
165 log.exception(
165 log.exception(
166 'Cannot set locale, not configuring the locale system')
166 'Cannot set locale, not configuring the locale system')
167
167
168
168
169 class WsgiProxy(object):
169 class WsgiProxy(object):
170 def __init__(self, wsgi):
170 def __init__(self, wsgi):
171 self.wsgi = wsgi
171 self.wsgi = wsgi
172
172
173 def __call__(self, environ, start_response):
173 def __call__(self, environ, start_response):
174 input_data = environ['wsgi.input'].read()
174 input_data = environ['wsgi.input'].read()
175 input_data = msgpack.unpackb(input_data)
175 input_data = msgpack.unpackb(input_data)
176
176
177 error = None
177 error = None
178 try:
178 try:
179 data, status, headers = self.wsgi.handle(
179 data, status, headers = self.wsgi.handle(
180 input_data['environment'], input_data['input_data'],
180 input_data['environment'], input_data['input_data'],
181 *input_data['args'], **input_data['kwargs'])
181 *input_data['args'], **input_data['kwargs'])
182 except Exception as e:
182 except Exception as e:
183 data, status, headers = [], None, None
183 data, status, headers = [], None, None
184 error = {
184 error = {
185 'message': str(e),
185 'message': str(e),
186 '_vcs_kind': getattr(e, '_vcs_kind', None)
186 '_vcs_kind': getattr(e, '_vcs_kind', None)
187 }
187 }
188
188
189 start_response(200, {})
189 start_response(200, {})
190 return self._iterator(error, status, headers, data)
190 return self._iterator(error, status, headers, data)
191
191
192 def _iterator(self, error, status, headers, data):
192 def _iterator(self, error, status, headers, data):
193 initial_data = [
193 initial_data = [
194 error,
194 error,
195 status,
195 status,
196 headers,
196 headers,
197 ]
197 ]
198
198
199 for d in chain(initial_data, data):
199 for d in chain(initial_data, data):
200 yield msgpack.packb(d)
200 yield msgpack.packb(d)
201
201
202
202
203 def not_found(request):
203 def not_found(request):
204 return {'status': '404 NOT FOUND'}
204 return {'status': '404 NOT FOUND'}
205
205
206
206
207 class VCSViewPredicate(object):
207 class VCSViewPredicate(object):
208 def __init__(self, val, config):
208 def __init__(self, val, config):
209 self.remotes = val
209 self.remotes = val
210
210
211 def text(self):
211 def text(self):
212 return 'vcs view method = %s' % (self.remotes.keys(),)
212 return 'vcs view method = %s' % (self.remotes.keys(),)
213
213
214 phash = text
214 phash = text
215
215
216 def __call__(self, context, request):
216 def __call__(self, context, request):
217 """
217 """
218 View predicate that returns true if given backend is supported by
218 View predicate that returns true if given backend is supported by
219 defined remotes.
219 defined remotes.
220 """
220 """
221 backend = request.matchdict.get('backend')
221 backend = request.matchdict.get('backend')
222 return backend in self.remotes
222 return backend in self.remotes
223
223
224
224
225 class HTTPApplication(object):
225 class HTTPApplication(object):
226 ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
226 ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
227
227
228 remote_wsgi = remote_wsgi
228 remote_wsgi = remote_wsgi
229 _use_echo_app = False
229 _use_echo_app = False
230
230
231 def __init__(self, settings=None, global_config=None):
231 def __init__(self, settings=None, global_config=None):
232 self._sanitize_settings_and_apply_defaults(settings)
232 self._sanitize_settings_and_apply_defaults(settings)
233
233
234 self.config = Configurator(settings=settings)
234 self.config = Configurator(settings=settings)
235 self.global_config = global_config
235 self.global_config = global_config
236 self.config.include('vcsserver.lib.rc_cache')
236 self.config.include('vcsserver.lib.rc_cache')
237
237
238 settings_locale = settings.get('locale', '') or 'en_US.UTF-8'
238 settings_locale = settings.get('locale', '') or 'en_US.UTF-8'
239 vcs = VCS(locale_conf=settings_locale, cache_config=settings)
239 vcs = VCS(locale_conf=settings_locale, cache_config=settings)
240 self._remotes = {
240 self._remotes = {
241 'hg': vcs._hg_remote,
241 'hg': vcs._hg_remote,
242 'git': vcs._git_remote,
242 'git': vcs._git_remote,
243 'svn': vcs._svn_remote,
243 'svn': vcs._svn_remote,
244 'server': vcs._vcsserver,
244 'server': vcs._vcsserver,
245 }
245 }
246 if settings.get('dev.use_echo_app', 'false').lower() == 'true':
246 if settings.get('dev.use_echo_app', 'false').lower() == 'true':
247 self._use_echo_app = True
247 self._use_echo_app = True
248 log.warning("Using EchoApp for VCS operations.")
248 log.warning("Using EchoApp for VCS operations.")
249 self.remote_wsgi = remote_wsgi_stub
249 self.remote_wsgi = remote_wsgi_stub
250
250
251 self._configure_settings(global_config, settings)
251 self._configure_settings(global_config, settings)
252 self._configure()
252 self._configure()
253
253
254 def _configure_settings(self, global_config, app_settings):
254 def _configure_settings(self, global_config, app_settings):
255 """
255 """
256 Configure the settings module.
256 Configure the settings module.
257 """
257 """
258 settings_merged = global_config.copy()
258 settings_merged = global_config.copy()
259 settings_merged.update(app_settings)
259 settings_merged.update(app_settings)
260
260
261 git_path = app_settings.get('git_path', None)
261 git_path = app_settings.get('git_path', None)
262 if git_path:
262 if git_path:
263 settings.GIT_EXECUTABLE = git_path
263 settings.GIT_EXECUTABLE = git_path
264 binary_dir = app_settings.get('core.binary_dir', None)
264 binary_dir = app_settings.get('core.binary_dir', None)
265 if binary_dir:
265 if binary_dir:
266 settings.BINARY_DIR = binary_dir
266 settings.BINARY_DIR = binary_dir
267
267
268 # Store the settings to make them available to other modules.
268 # Store the settings to make them available to other modules.
269 vcsserver.PYRAMID_SETTINGS = settings_merged
269 vcsserver.PYRAMID_SETTINGS = settings_merged
270 vcsserver.CONFIG = settings_merged
270 vcsserver.CONFIG = settings_merged
271
271
272 def _sanitize_settings_and_apply_defaults(self, settings):
272 def _sanitize_settings_and_apply_defaults(self, settings):
273 temp_store = tempfile.gettempdir()
273 temp_store = tempfile.gettempdir()
274 default_cache_dir = os.path.join(temp_store, 'rc_cache')
274 default_cache_dir = os.path.join(temp_store, 'rc_cache')
275
275
276 # save default, cache dir, and use it for all backends later.
276 # save default, cache dir, and use it for all backends later.
277 default_cache_dir = _string_setting(
277 default_cache_dir = _string_setting(
278 settings,
278 settings,
279 'cache_dir',
279 'cache_dir',
280 default_cache_dir, lower=False, default_when_empty=True)
280 default_cache_dir, lower=False, default_when_empty=True)
281
281
282 # ensure we have our dir created
282 # ensure we have our dir created
283 if not os.path.isdir(default_cache_dir):
283 if not os.path.isdir(default_cache_dir):
284 os.makedirs(default_cache_dir, mode=0o755)
284 os.makedirs(default_cache_dir, mode=0o755)
285
285
286 # exception store cache
286 # exception store cache
287 _string_setting(
287 _string_setting(
288 settings,
288 settings,
289 'exception_tracker.store_path',
289 'exception_tracker.store_path',
290 temp_store, lower=False, default_when_empty=True)
290 temp_store, lower=False, default_when_empty=True)
291
291
292 # repo_object cache
292 # repo_object cache
293 _string_setting(
293 _string_setting(
294 settings,
294 settings,
295 'rc_cache.repo_object.backend',
295 'rc_cache.repo_object.backend',
296 'dogpile.cache.rc.file_namespace', lower=False)
296 'dogpile.cache.rc.file_namespace', lower=False)
297 _int_setting(
297 _int_setting(
298 settings,
298 settings,
299 'rc_cache.repo_object.expiration_time',
299 'rc_cache.repo_object.expiration_time',
300 30 * 24 * 60 * 60)
300 30 * 24 * 60 * 60)
301 _string_setting(
301 _string_setting(
302 settings,
302 settings,
303 'rc_cache.repo_object.arguments.filename',
303 'rc_cache.repo_object.arguments.filename',
304 os.path.join(default_cache_dir, 'vcsserver_cache_1'), lower=False)
304 os.path.join(default_cache_dir, 'vcsserver_cache_1'), lower=False)
305
305
306 def _configure(self):
306 def _configure(self):
307 self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory)
307 self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory)
308
308
309 self.config.add_route('service', '/_service')
309 self.config.add_route('service', '/_service')
310 self.config.add_route('status', '/status')
310 self.config.add_route('status', '/status')
311 self.config.add_route('hg_proxy', '/proxy/hg')
311 self.config.add_route('hg_proxy', '/proxy/hg')
312 self.config.add_route('git_proxy', '/proxy/git')
312 self.config.add_route('git_proxy', '/proxy/git')
313
313
314 # rpc methods
314 # rpc methods
315 self.config.add_route('vcs', '/{backend}')
315 self.config.add_route('vcs', '/{backend}')
316
316
317 # streaming rpc remote methods
317 # streaming rpc remote methods
318 self.config.add_route('vcs_stream', '/{backend}/stream')
318 self.config.add_route('vcs_stream', '/{backend}/stream')
319
319
320 # vcs operations clone/push as streaming
320 # vcs operations clone/push as streaming
321 self.config.add_route('stream_git', '/stream/git/*repo_name')
321 self.config.add_route('stream_git', '/stream/git/*repo_name')
322 self.config.add_route('stream_hg', '/stream/hg/*repo_name')
322 self.config.add_route('stream_hg', '/stream/hg/*repo_name')
323
323
324 self.config.add_view(self.status_view, route_name='status', renderer='json')
324 self.config.add_view(self.status_view, route_name='status', renderer='json')
325 self.config.add_view(self.service_view, route_name='service', renderer='msgpack')
325 self.config.add_view(self.service_view, route_name='service', renderer='msgpack')
326
326
327 self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
327 self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
328 self.config.add_view(self.git_proxy(), route_name='git_proxy')
328 self.config.add_view(self.git_proxy(), route_name='git_proxy')
329 self.config.add_view(self.vcs_view, route_name='vcs', renderer='msgpack',
329 self.config.add_view(self.vcs_view, route_name='vcs', renderer='msgpack',
330 vcs_view=self._remotes)
330 vcs_view=self._remotes)
331 self.config.add_view(self.vcs_stream_view, route_name='vcs_stream',
331 self.config.add_view(self.vcs_stream_view, route_name='vcs_stream',
332 vcs_view=self._remotes)
332 vcs_view=self._remotes)
333
333
334 self.config.add_view(self.hg_stream(), route_name='stream_hg')
334 self.config.add_view(self.hg_stream(), route_name='stream_hg')
335 self.config.add_view(self.git_stream(), route_name='stream_git')
335 self.config.add_view(self.git_stream(), route_name='stream_git')
336
336
337 self.config.add_view_predicate('vcs_view', VCSViewPredicate)
337 self.config.add_view_predicate('vcs_view', VCSViewPredicate)
338
338
339 self.config.add_notfound_view(not_found, renderer='json')
339 self.config.add_notfound_view(not_found, renderer='json')
340
340
341 self.config.add_view(self.handle_vcs_exception, context=Exception)
341 self.config.add_view(self.handle_vcs_exception, context=Exception)
342
342
343 self.config.add_tween(
343 self.config.add_tween(
344 'vcsserver.tweens.request_wrapper.RequestWrapperTween',
344 'vcsserver.tweens.request_wrapper.RequestWrapperTween',
345 )
345 )
346 self.config.add_request_method(
346 self.config.add_request_method(
347 'vcsserver.lib.request_counter.get_request_counter',
347 'vcsserver.lib.request_counter.get_request_counter',
348 'request_count')
348 'request_count')
349
349
350 def wsgi_app(self):
350 def wsgi_app(self):
351 return self.config.make_wsgi_app()
351 return self.config.make_wsgi_app()
352
352
353 def _vcs_view_params(self, request):
353 def _vcs_view_params(self, request):
354 remote = self._remotes[request.matchdict['backend']]
354 remote = self._remotes[request.matchdict['backend']]
355 payload = msgpack.unpackb(request.body, use_list=True)
355 payload = msgpack.unpackb(request.body, use_list=True)
356 method = payload.get('method')
356 method = payload.get('method')
357 params = payload['params']
357 params = payload['params']
358 wire = params.get('wire')
358 wire = params.get('wire')
359 args = params.get('args')
359 args = params.get('args')
360 kwargs = params.get('kwargs')
360 kwargs = params.get('kwargs')
361 context_uid = None
361 context_uid = None
362
362
363 if wire:
363 if wire:
364 try:
364 try:
365 wire['context'] = context_uid = uuid.UUID(wire['context'])
365 wire['context'] = context_uid = uuid.UUID(wire['context'])
366 except KeyError:
366 except KeyError:
367 pass
367 pass
368 args.insert(0, wire)
368 args.insert(0, wire)
369 repo_state_uid = wire.get('repo_state_uid') if wire else None
369 repo_state_uid = wire.get('repo_state_uid') if wire else None
370
370
371 # NOTE(marcink): trading complexity for slight performance
371 # NOTE(marcink): trading complexity for slight performance
372 if log.isEnabledFor(logging.DEBUG):
372 if log.isEnabledFor(logging.DEBUG):
373 no_args_methods = [
373 no_args_methods = [
374 'archive_repo'
374 'archive_repo'
375 ]
375 ]
376 if method in no_args_methods:
376 if method in no_args_methods:
377 call_args = ''
377 call_args = ''
378 else:
378 else:
379 call_args = args[1:]
379 call_args = args[1:]
380
380
381 log.debug('method requested:%s with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s',
381 log.debug('method requested:%s with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s',
382 method, call_args, kwargs, context_uid, repo_state_uid)
382 method, call_args, kwargs, context_uid, repo_state_uid)
383
383
384 return payload, remote, method, args, kwargs
384 return payload, remote, method, args, kwargs
385
385
386 def vcs_view(self, request):
386 def vcs_view(self, request):
387
387
388 payload, remote, method, args, kwargs = self._vcs_view_params(request)
388 payload, remote, method, args, kwargs = self._vcs_view_params(request)
389 payload_id = payload.get('id')
389 payload_id = payload.get('id')
390
390
391 try:
391 try:
392 resp = getattr(remote, method)(*args, **kwargs)
392 resp = getattr(remote, method)(*args, **kwargs)
393 except Exception as e:
393 except Exception as e:
394 exc_info = list(sys.exc_info())
394 exc_info = list(sys.exc_info())
395 exc_type, exc_value, exc_traceback = exc_info
395 exc_type, exc_value, exc_traceback = exc_info
396
396
397 org_exc = getattr(e, '_org_exc', None)
397 org_exc = getattr(e, '_org_exc', None)
398 org_exc_name = None
398 org_exc_name = None
399 org_exc_tb = ''
399 org_exc_tb = ''
400 if org_exc:
400 if org_exc:
401 org_exc_name = org_exc.__class__.__name__
401 org_exc_name = org_exc.__class__.__name__
402 org_exc_tb = getattr(e, '_org_exc_tb', '')
402 org_exc_tb = getattr(e, '_org_exc_tb', '')
403 # replace our "faked" exception with our org
403 # replace our "faked" exception with our org
404 exc_info[0] = org_exc.__class__
404 exc_info[0] = org_exc.__class__
405 exc_info[1] = org_exc
405 exc_info[1] = org_exc
406
406
407 should_store_exc = True
408 if org_exc:
409 def get_exc_fqn(_exc_obj):
410 module_name = getattr(org_exc.__class__, '__module__', 'UNKNOWN')
411 return module_name + '.' + org_exc_name
412
413 exc_fqn = get_exc_fqn(org_exc)
414
415 if exc_fqn in ['mercurial.error.RepoLookupError',
416 'vcsserver.exceptions.RefNotFoundException']:
417 should_store_exc = False
418
419 if should_store_exc:
407 store_exception(id(exc_info), exc_info)
420 store_exception(id(exc_info), exc_info)
408
421
409 tb_info = ''.join(
422 tb_info = ''.join(
410 traceback.format_exception(exc_type, exc_value, exc_traceback))
423 traceback.format_exception(exc_type, exc_value, exc_traceback))
411
424
412 type_ = e.__class__.__name__
425 type_ = e.__class__.__name__
413 if type_ not in self.ALLOWED_EXCEPTIONS:
426 if type_ not in self.ALLOWED_EXCEPTIONS:
414 type_ = None
427 type_ = None
415
428
416 resp = {
429 resp = {
417 'id': payload_id,
430 'id': payload_id,
418 'error': {
431 'error': {
419 'message': e.message,
432 'message': e.message,
420 'traceback': tb_info,
433 'traceback': tb_info,
421 'org_exc': org_exc_name,
434 'org_exc': org_exc_name,
422 'org_exc_tb': org_exc_tb,
435 'org_exc_tb': org_exc_tb,
423 'type': type_
436 'type': type_
424 }
437 }
425 }
438 }
426 try:
439 try:
427 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
440 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
428 except AttributeError:
441 except AttributeError:
429 pass
442 pass
430 else:
443 else:
431 resp = {
444 resp = {
432 'id': payload_id,
445 'id': payload_id,
433 'result': resp
446 'result': resp
434 }
447 }
435
448
436 return resp
449 return resp
437
450
438 def vcs_stream_view(self, request):
451 def vcs_stream_view(self, request):
439 payload, remote, method, args, kwargs = self._vcs_view_params(request)
452 payload, remote, method, args, kwargs = self._vcs_view_params(request)
440 # this method has a stream: marker we remove it here
453 # this method has a stream: marker we remove it here
441 method = method.split('stream:')[-1]
454 method = method.split('stream:')[-1]
442 chunk_size = safe_int(payload.get('chunk_size')) or 4096
455 chunk_size = safe_int(payload.get('chunk_size')) or 4096
443
456
444 try:
457 try:
445 resp = getattr(remote, method)(*args, **kwargs)
458 resp = getattr(remote, method)(*args, **kwargs)
446 except Exception as e:
459 except Exception as e:
447 raise
460 raise
448
461
449 def get_chunked_data(method_resp):
462 def get_chunked_data(method_resp):
450 stream = StringIO(method_resp)
463 stream = StringIO(method_resp)
451 while 1:
464 while 1:
452 chunk = stream.read(chunk_size)
465 chunk = stream.read(chunk_size)
453 if not chunk:
466 if not chunk:
454 break
467 break
455 yield chunk
468 yield chunk
456
469
457 response = Response(app_iter=get_chunked_data(resp))
470 response = Response(app_iter=get_chunked_data(resp))
458 response.content_type = 'application/octet-stream'
471 response.content_type = 'application/octet-stream'
459
472
460 return response
473 return response
461
474
462 def status_view(self, request):
475 def status_view(self, request):
463 import vcsserver
476 import vcsserver
464 return {'status': 'OK', 'vcsserver_version': vcsserver.__version__,
477 return {'status': 'OK', 'vcsserver_version': vcsserver.__version__,
465 'pid': os.getpid()}
478 'pid': os.getpid()}
466
479
467 def service_view(self, request):
480 def service_view(self, request):
468 import vcsserver
481 import vcsserver
469
482
470 payload = msgpack.unpackb(request.body, use_list=True)
483 payload = msgpack.unpackb(request.body, use_list=True)
471 server_config, app_config = {}, {}
484 server_config, app_config = {}, {}
472
485
473 try:
486 try:
474 path = self.global_config['__file__']
487 path = self.global_config['__file__']
475 config = configparser.RawConfigParser()
488 config = configparser.RawConfigParser()
476
489
477 config.read(path)
490 config.read(path)
478
491
479 if config.has_section('server:main'):
492 if config.has_section('server:main'):
480 server_config = dict(config.items('server:main'))
493 server_config = dict(config.items('server:main'))
481 if config.has_section('app:main'):
494 if config.has_section('app:main'):
482 app_config = dict(config.items('app:main'))
495 app_config = dict(config.items('app:main'))
483
496
484 except Exception:
497 except Exception:
485 log.exception('Failed to read .ini file for display')
498 log.exception('Failed to read .ini file for display')
486
499
487 environ = os.environ.items()
500 environ = os.environ.items()
488
501
489 resp = {
502 resp = {
490 'id': payload.get('id'),
503 'id': payload.get('id'),
491 'result': dict(
504 'result': dict(
492 version=vcsserver.__version__,
505 version=vcsserver.__version__,
493 config=server_config,
506 config=server_config,
494 app_config=app_config,
507 app_config=app_config,
495 environ=environ,
508 environ=environ,
496 payload=payload,
509 payload=payload,
497 )
510 )
498 }
511 }
499 return resp
512 return resp
500
513
501 def _msgpack_renderer_factory(self, info):
514 def _msgpack_renderer_factory(self, info):
502 def _render(value, system):
515 def _render(value, system):
503 request = system.get('request')
516 request = system.get('request')
504 if request is not None:
517 if request is not None:
505 response = request.response
518 response = request.response
506 ct = response.content_type
519 ct = response.content_type
507 if ct == response.default_content_type:
520 if ct == response.default_content_type:
508 response.content_type = 'application/x-msgpack'
521 response.content_type = 'application/x-msgpack'
509 return msgpack.packb(value)
522 return msgpack.packb(value)
510 return _render
523 return _render
511
524
512 def set_env_from_config(self, environ, config):
525 def set_env_from_config(self, environ, config):
513 dict_conf = {}
526 dict_conf = {}
514 try:
527 try:
515 for elem in config:
528 for elem in config:
516 if elem[0] == 'rhodecode':
529 if elem[0] == 'rhodecode':
517 dict_conf = json.loads(elem[2])
530 dict_conf = json.loads(elem[2])
518 break
531 break
519 except Exception:
532 except Exception:
520 log.exception('Failed to fetch SCM CONFIG')
533 log.exception('Failed to fetch SCM CONFIG')
521 return
534 return
522
535
523 username = dict_conf.get('username')
536 username = dict_conf.get('username')
524 if username:
537 if username:
525 environ['REMOTE_USER'] = username
538 environ['REMOTE_USER'] = username
526 # mercurial specific, some extension api rely on this
539 # mercurial specific, some extension api rely on this
527 environ['HGUSER'] = username
540 environ['HGUSER'] = username
528
541
529 ip = dict_conf.get('ip')
542 ip = dict_conf.get('ip')
530 if ip:
543 if ip:
531 environ['REMOTE_HOST'] = ip
544 environ['REMOTE_HOST'] = ip
532
545
533 if _is_request_chunked(environ):
546 if _is_request_chunked(environ):
534 # set the compatibility flag for webob
547 # set the compatibility flag for webob
535 environ['wsgi.input_terminated'] = True
548 environ['wsgi.input_terminated'] = True
536
549
537 def hg_proxy(self):
550 def hg_proxy(self):
538 @wsgiapp
551 @wsgiapp
539 def _hg_proxy(environ, start_response):
552 def _hg_proxy(environ, start_response):
540 app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
553 app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
541 return app(environ, start_response)
554 return app(environ, start_response)
542 return _hg_proxy
555 return _hg_proxy
543
556
544 def git_proxy(self):
557 def git_proxy(self):
545 @wsgiapp
558 @wsgiapp
546 def _git_proxy(environ, start_response):
559 def _git_proxy(environ, start_response):
547 app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
560 app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
548 return app(environ, start_response)
561 return app(environ, start_response)
549 return _git_proxy
562 return _git_proxy
550
563
551 def hg_stream(self):
564 def hg_stream(self):
552 if self._use_echo_app:
565 if self._use_echo_app:
553 @wsgiapp
566 @wsgiapp
554 def _hg_stream(environ, start_response):
567 def _hg_stream(environ, start_response):
555 app = EchoApp('fake_path', 'fake_name', None)
568 app = EchoApp('fake_path', 'fake_name', None)
556 return app(environ, start_response)
569 return app(environ, start_response)
557 return _hg_stream
570 return _hg_stream
558 else:
571 else:
559 @wsgiapp
572 @wsgiapp
560 def _hg_stream(environ, start_response):
573 def _hg_stream(environ, start_response):
561 log.debug('http-app: handling hg stream')
574 log.debug('http-app: handling hg stream')
562 repo_path = environ['HTTP_X_RC_REPO_PATH']
575 repo_path = environ['HTTP_X_RC_REPO_PATH']
563 repo_name = environ['HTTP_X_RC_REPO_NAME']
576 repo_name = environ['HTTP_X_RC_REPO_NAME']
564 packed_config = base64.b64decode(
577 packed_config = base64.b64decode(
565 environ['HTTP_X_RC_REPO_CONFIG'])
578 environ['HTTP_X_RC_REPO_CONFIG'])
566 config = msgpack.unpackb(packed_config)
579 config = msgpack.unpackb(packed_config)
567 app = scm_app.create_hg_wsgi_app(
580 app = scm_app.create_hg_wsgi_app(
568 repo_path, repo_name, config)
581 repo_path, repo_name, config)
569
582
570 # Consistent path information for hgweb
583 # Consistent path information for hgweb
571 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
584 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
572 environ['REPO_NAME'] = repo_name
585 environ['REPO_NAME'] = repo_name
573 self.set_env_from_config(environ, config)
586 self.set_env_from_config(environ, config)
574
587
575 log.debug('http-app: starting app handler '
588 log.debug('http-app: starting app handler '
576 'with %s and process request', app)
589 'with %s and process request', app)
577 return app(environ, ResponseFilter(start_response))
590 return app(environ, ResponseFilter(start_response))
578 return _hg_stream
591 return _hg_stream
579
592
580 def git_stream(self):
593 def git_stream(self):
581 if self._use_echo_app:
594 if self._use_echo_app:
582 @wsgiapp
595 @wsgiapp
583 def _git_stream(environ, start_response):
596 def _git_stream(environ, start_response):
584 app = EchoApp('fake_path', 'fake_name', None)
597 app = EchoApp('fake_path', 'fake_name', None)
585 return app(environ, start_response)
598 return app(environ, start_response)
586 return _git_stream
599 return _git_stream
587 else:
600 else:
588 @wsgiapp
601 @wsgiapp
589 def _git_stream(environ, start_response):
602 def _git_stream(environ, start_response):
590 log.debug('http-app: handling git stream')
603 log.debug('http-app: handling git stream')
591 repo_path = environ['HTTP_X_RC_REPO_PATH']
604 repo_path = environ['HTTP_X_RC_REPO_PATH']
592 repo_name = environ['HTTP_X_RC_REPO_NAME']
605 repo_name = environ['HTTP_X_RC_REPO_NAME']
593 packed_config = base64.b64decode(
606 packed_config = base64.b64decode(
594 environ['HTTP_X_RC_REPO_CONFIG'])
607 environ['HTTP_X_RC_REPO_CONFIG'])
595 config = msgpack.unpackb(packed_config)
608 config = msgpack.unpackb(packed_config)
596
609
597 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
610 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
598 self.set_env_from_config(environ, config)
611 self.set_env_from_config(environ, config)
599
612
600 content_type = environ.get('CONTENT_TYPE', '')
613 content_type = environ.get('CONTENT_TYPE', '')
601
614
602 path = environ['PATH_INFO']
615 path = environ['PATH_INFO']
603 is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
616 is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
604 log.debug(
617 log.debug(
605 'LFS: Detecting if request `%s` is LFS server path based '
618 'LFS: Detecting if request `%s` is LFS server path based '
606 'on content type:`%s`, is_lfs:%s',
619 'on content type:`%s`, is_lfs:%s',
607 path, content_type, is_lfs_request)
620 path, content_type, is_lfs_request)
608
621
609 if not is_lfs_request:
622 if not is_lfs_request:
610 # fallback detection by path
623 # fallback detection by path
611 if GIT_LFS_PROTO_PAT.match(path):
624 if GIT_LFS_PROTO_PAT.match(path):
612 is_lfs_request = True
625 is_lfs_request = True
613 log.debug(
626 log.debug(
614 'LFS: fallback detection by path of: `%s`, is_lfs:%s',
627 'LFS: fallback detection by path of: `%s`, is_lfs:%s',
615 path, is_lfs_request)
628 path, is_lfs_request)
616
629
617 if is_lfs_request:
630 if is_lfs_request:
618 app = scm_app.create_git_lfs_wsgi_app(
631 app = scm_app.create_git_lfs_wsgi_app(
619 repo_path, repo_name, config)
632 repo_path, repo_name, config)
620 else:
633 else:
621 app = scm_app.create_git_wsgi_app(
634 app = scm_app.create_git_wsgi_app(
622 repo_path, repo_name, config)
635 repo_path, repo_name, config)
623
636
624 log.debug('http-app: starting app handler '
637 log.debug('http-app: starting app handler '
625 'with %s and process request', app)
638 'with %s and process request', app)
626
639
627 return app(environ, start_response)
640 return app(environ, start_response)
628
641
629 return _git_stream
642 return _git_stream
630
643
631 def handle_vcs_exception(self, exception, request):
644 def handle_vcs_exception(self, exception, request):
632 _vcs_kind = getattr(exception, '_vcs_kind', '')
645 _vcs_kind = getattr(exception, '_vcs_kind', '')
633 if _vcs_kind == 'repo_locked':
646 if _vcs_kind == 'repo_locked':
634 # Get custom repo-locked status code if present.
647 # Get custom repo-locked status code if present.
635 status_code = request.headers.get('X-RC-Locked-Status-Code')
648 status_code = request.headers.get('X-RC-Locked-Status-Code')
636 return HTTPRepoLocked(
649 return HTTPRepoLocked(
637 title=exception.message, status_code=status_code)
650 title=exception.message, status_code=status_code)
638
651
639 elif _vcs_kind == 'repo_branch_protected':
652 elif _vcs_kind == 'repo_branch_protected':
640 # Get custom repo-branch-protected status code if present.
653 # Get custom repo-branch-protected status code if present.
641 return HTTPRepoBranchProtected(title=exception.message)
654 return HTTPRepoBranchProtected(title=exception.message)
642
655
643 exc_info = request.exc_info
656 exc_info = request.exc_info
644 store_exception(id(exc_info), exc_info)
657 store_exception(id(exc_info), exc_info)
645
658
646 traceback_info = 'unavailable'
659 traceback_info = 'unavailable'
647 if request.exc_info:
660 if request.exc_info:
648 exc_type, exc_value, exc_tb = request.exc_info
661 exc_type, exc_value, exc_tb = request.exc_info
649 traceback_info = ''.join(traceback.format_exception(exc_type, exc_value, exc_tb))
662 traceback_info = ''.join(traceback.format_exception(exc_type, exc_value, exc_tb))
650
663
651 log.error(
664 log.error(
652 'error occurred handling this request for path: %s, \n tb: %s',
665 'error occurred handling this request for path: %s, \n tb: %s',
653 request.path, traceback_info)
666 request.path, traceback_info)
654 raise exception
667 raise exception
655
668
656
669
657 class ResponseFilter(object):
670 class ResponseFilter(object):
658
671
659 def __init__(self, start_response):
672 def __init__(self, start_response):
660 self._start_response = start_response
673 self._start_response = start_response
661
674
662 def __call__(self, status, response_headers, exc_info=None):
675 def __call__(self, status, response_headers, exc_info=None):
663 headers = tuple(
676 headers = tuple(
664 (h, v) for h, v in response_headers
677 (h, v) for h, v in response_headers
665 if not wsgiref.util.is_hop_by_hop(h))
678 if not wsgiref.util.is_hop_by_hop(h))
666 return self._start_response(status, headers, exc_info)
679 return self._start_response(status, headers, exc_info)
667
680
668
681
669 def main(global_config, **settings):
682 def main(global_config, **settings):
670 if MercurialFactory:
683 if MercurialFactory:
671 hgpatches.patch_largefiles_capabilities()
684 hgpatches.patch_largefiles_capabilities()
672 hgpatches.patch_subrepo_type_mapping()
685 hgpatches.patch_subrepo_type_mapping()
673
686
674 app = HTTPApplication(settings=settings, global_config=global_config)
687 app = HTTPApplication(settings=settings, global_config=global_config)
675 return app.wsgi_app()
688 return app.wsgi_app()
General Comments 0
You need to be logged in to leave comments. Login now