##// END OF EJS Templates
vcsserver: use safer maxfd reporting, some linux systems get a problem with this
milka -
r913:279dbe3d default
parent child Browse files
Show More
@@ -1,692 +1,700 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2020 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import os
18 import os
19 import sys
19 import sys
20 import base64
20 import base64
21 import locale
21 import locale
22 import logging
22 import logging
23 import uuid
23 import uuid
24 import wsgiref.util
24 import wsgiref.util
25 import traceback
25 import traceback
26 import tempfile
26 import tempfile
27 import resource
27 import resource
28 import psutil
28 from itertools import chain
29 from itertools import chain
29 from cStringIO import StringIO
30 from cStringIO import StringIO
30
31
31 import simplejson as json
32 import simplejson as json
32 import msgpack
33 import msgpack
33 from pyramid.config import Configurator
34 from pyramid.config import Configurator
34 from pyramid.settings import asbool, aslist
35 from pyramid.settings import asbool, aslist
35 from pyramid.wsgi import wsgiapp
36 from pyramid.wsgi import wsgiapp
36 from pyramid.compat import configparser
37 from pyramid.compat import configparser
37 from pyramid.response import Response
38 from pyramid.response import Response
38
39
39 from vcsserver.utils import safe_int
40 from vcsserver.utils import safe_int
40
41
41 log = logging.getLogger(__name__)
42 log = logging.getLogger(__name__)
42
43
43 # due to Mercurial/glibc2.27 problems we need to detect if locale settings are
44 # due to Mercurial/glibc2.27 problems we need to detect if locale settings are
44 # causing problems and "fix" it in case they do and fallback to LC_ALL = C
45 # causing problems and "fix" it in case they do and fallback to LC_ALL = C
45
46
46 try:
47 try:
47 locale.setlocale(locale.LC_ALL, '')
48 locale.setlocale(locale.LC_ALL, '')
48 except locale.Error as e:
49 except locale.Error as e:
49 log.error(
50 log.error(
50 'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
51 'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
51 os.environ['LC_ALL'] = 'C'
52 os.environ['LC_ALL'] = 'C'
52
53
53 import vcsserver
54 import vcsserver
54 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
55 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
55 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
56 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
56 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
57 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
57 from vcsserver.echo_stub.echo_app import EchoApp
58 from vcsserver.echo_stub.echo_app import EchoApp
58 from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected
59 from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected
59 from vcsserver.lib.exc_tracking import store_exception
60 from vcsserver.lib.exc_tracking import store_exception
60 from vcsserver.server import VcsServer
61 from vcsserver.server import VcsServer
61
62
62 try:
63 try:
63 from vcsserver.git import GitFactory, GitRemote
64 from vcsserver.git import GitFactory, GitRemote
64 except ImportError:
65 except ImportError:
65 GitFactory = None
66 GitFactory = None
66 GitRemote = None
67 GitRemote = None
67
68
68 try:
69 try:
69 from vcsserver.hg import MercurialFactory, HgRemote
70 from vcsserver.hg import MercurialFactory, HgRemote
70 except ImportError:
71 except ImportError:
71 MercurialFactory = None
72 MercurialFactory = None
72 HgRemote = None
73 HgRemote = None
73
74
74 try:
75 try:
75 from vcsserver.svn import SubversionFactory, SvnRemote
76 from vcsserver.svn import SubversionFactory, SvnRemote
76 except ImportError:
77 except ImportError:
77 SubversionFactory = None
78 SubversionFactory = None
78 SvnRemote = None
79 SvnRemote = None
79
80
80
81
81 def _is_request_chunked(environ):
82 def _is_request_chunked(environ):
82 stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
83 stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
83 return stream
84 return stream
84
85
85
86
86 def _int_setting(settings, name, default):
87 def _int_setting(settings, name, default):
87 settings[name] = int(settings.get(name, default))
88 settings[name] = int(settings.get(name, default))
88 return settings[name]
89 return settings[name]
89
90
90
91
91 def _bool_setting(settings, name, default):
92 def _bool_setting(settings, name, default):
92 input_val = settings.get(name, default)
93 input_val = settings.get(name, default)
93 if isinstance(input_val, unicode):
94 if isinstance(input_val, unicode):
94 input_val = input_val.encode('utf8')
95 input_val = input_val.encode('utf8')
95 settings[name] = asbool(input_val)
96 settings[name] = asbool(input_val)
96 return settings[name]
97 return settings[name]
97
98
98
99
99 def _list_setting(settings, name, default):
100 def _list_setting(settings, name, default):
100 raw_value = settings.get(name, default)
101 raw_value = settings.get(name, default)
101
102
102 # Otherwise we assume it uses pyramids space/newline separation.
103 # Otherwise we assume it uses pyramids space/newline separation.
103 settings[name] = aslist(raw_value)
104 settings[name] = aslist(raw_value)
104 return settings[name]
105 return settings[name]
105
106
106
107
107 def _string_setting(settings, name, default, lower=True, default_when_empty=False):
108 def _string_setting(settings, name, default, lower=True, default_when_empty=False):
108 value = settings.get(name, default)
109 value = settings.get(name, default)
109
110
110 if default_when_empty and not value:
111 if default_when_empty and not value:
111 # use default value when value is empty
112 # use default value when value is empty
112 value = default
113 value = default
113
114
114 if lower:
115 if lower:
115 value = value.lower()
116 value = value.lower()
116 settings[name] = value
117 settings[name] = value
117 return settings[name]
118 return settings[name]
118
119
119
120
121 def log_max_fd():
122 try:
123 maxfd = psutil.Process().rlimit(psutil.RLIMIT_NOFILE)[1]
124 log.info('Max file descriptors value: %s', maxfd)
125 except Exception:
126 pass
127
128
120 class VCS(object):
129 class VCS(object):
121 def __init__(self, locale_conf=None, cache_config=None):
130 def __init__(self, locale_conf=None, cache_config=None):
122 self.locale = locale_conf
131 self.locale = locale_conf
123 self.cache_config = cache_config
132 self.cache_config = cache_config
124 self._configure_locale()
133 self._configure_locale()
125
134
126 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
135 log_max_fd()
127 log.info('Max file descriptors value: %s', maxfd)
128
136
129 if GitFactory and GitRemote:
137 if GitFactory and GitRemote:
130 git_factory = GitFactory()
138 git_factory = GitFactory()
131 self._git_remote = GitRemote(git_factory)
139 self._git_remote = GitRemote(git_factory)
132 else:
140 else:
133 log.info("Git client import failed")
141 log.info("Git client import failed")
134
142
135 if MercurialFactory and HgRemote:
143 if MercurialFactory and HgRemote:
136 hg_factory = MercurialFactory()
144 hg_factory = MercurialFactory()
137 self._hg_remote = HgRemote(hg_factory)
145 self._hg_remote = HgRemote(hg_factory)
138 else:
146 else:
139 log.info("Mercurial client import failed")
147 log.info("Mercurial client import failed")
140
148
141 if SubversionFactory and SvnRemote:
149 if SubversionFactory and SvnRemote:
142 svn_factory = SubversionFactory()
150 svn_factory = SubversionFactory()
143
151
144 # hg factory is used for svn url validation
152 # hg factory is used for svn url validation
145 hg_factory = MercurialFactory()
153 hg_factory = MercurialFactory()
146 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
154 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
147 else:
155 else:
148 log.info("Subversion client import failed")
156 log.info("Subversion client import failed")
149
157
150 self._vcsserver = VcsServer()
158 self._vcsserver = VcsServer()
151
159
152 def _configure_locale(self):
160 def _configure_locale(self):
153 if self.locale:
161 if self.locale:
154 log.info('Settings locale: `LC_ALL` to %s', self.locale)
162 log.info('Settings locale: `LC_ALL` to %s', self.locale)
155 else:
163 else:
156 log.info(
164 log.info(
157 'Configuring locale subsystem based on environment variables')
165 'Configuring locale subsystem based on environment variables')
158 try:
166 try:
159 # If self.locale is the empty string, then the locale
167 # If self.locale is the empty string, then the locale
160 # module will use the environment variables. See the
168 # module will use the environment variables. See the
161 # documentation of the package `locale`.
169 # documentation of the package `locale`.
162 locale.setlocale(locale.LC_ALL, self.locale)
170 locale.setlocale(locale.LC_ALL, self.locale)
163
171
164 language_code, encoding = locale.getlocale()
172 language_code, encoding = locale.getlocale()
165 log.info(
173 log.info(
166 'Locale set to language code "%s" with encoding "%s".',
174 'Locale set to language code "%s" with encoding "%s".',
167 language_code, encoding)
175 language_code, encoding)
168 except locale.Error:
176 except locale.Error:
169 log.exception(
177 log.exception(
170 'Cannot set locale, not configuring the locale system')
178 'Cannot set locale, not configuring the locale system')
171
179
172
180
173 class WsgiProxy(object):
181 class WsgiProxy(object):
174 def __init__(self, wsgi):
182 def __init__(self, wsgi):
175 self.wsgi = wsgi
183 self.wsgi = wsgi
176
184
177 def __call__(self, environ, start_response):
185 def __call__(self, environ, start_response):
178 input_data = environ['wsgi.input'].read()
186 input_data = environ['wsgi.input'].read()
179 input_data = msgpack.unpackb(input_data)
187 input_data = msgpack.unpackb(input_data)
180
188
181 error = None
189 error = None
182 try:
190 try:
183 data, status, headers = self.wsgi.handle(
191 data, status, headers = self.wsgi.handle(
184 input_data['environment'], input_data['input_data'],
192 input_data['environment'], input_data['input_data'],
185 *input_data['args'], **input_data['kwargs'])
193 *input_data['args'], **input_data['kwargs'])
186 except Exception as e:
194 except Exception as e:
187 data, status, headers = [], None, None
195 data, status, headers = [], None, None
188 error = {
196 error = {
189 'message': str(e),
197 'message': str(e),
190 '_vcs_kind': getattr(e, '_vcs_kind', None)
198 '_vcs_kind': getattr(e, '_vcs_kind', None)
191 }
199 }
192
200
193 start_response(200, {})
201 start_response(200, {})
194 return self._iterator(error, status, headers, data)
202 return self._iterator(error, status, headers, data)
195
203
196 def _iterator(self, error, status, headers, data):
204 def _iterator(self, error, status, headers, data):
197 initial_data = [
205 initial_data = [
198 error,
206 error,
199 status,
207 status,
200 headers,
208 headers,
201 ]
209 ]
202
210
203 for d in chain(initial_data, data):
211 for d in chain(initial_data, data):
204 yield msgpack.packb(d)
212 yield msgpack.packb(d)
205
213
206
214
207 def not_found(request):
215 def not_found(request):
208 return {'status': '404 NOT FOUND'}
216 return {'status': '404 NOT FOUND'}
209
217
210
218
211 class VCSViewPredicate(object):
219 class VCSViewPredicate(object):
212 def __init__(self, val, config):
220 def __init__(self, val, config):
213 self.remotes = val
221 self.remotes = val
214
222
215 def text(self):
223 def text(self):
216 return 'vcs view method = %s' % (self.remotes.keys(),)
224 return 'vcs view method = %s' % (self.remotes.keys(),)
217
225
218 phash = text
226 phash = text
219
227
220 def __call__(self, context, request):
228 def __call__(self, context, request):
221 """
229 """
222 View predicate that returns true if given backend is supported by
230 View predicate that returns true if given backend is supported by
223 defined remotes.
231 defined remotes.
224 """
232 """
225 backend = request.matchdict.get('backend')
233 backend = request.matchdict.get('backend')
226 return backend in self.remotes
234 return backend in self.remotes
227
235
228
236
229 class HTTPApplication(object):
237 class HTTPApplication(object):
230 ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
238 ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
231
239
232 remote_wsgi = remote_wsgi
240 remote_wsgi = remote_wsgi
233 _use_echo_app = False
241 _use_echo_app = False
234
242
235 def __init__(self, settings=None, global_config=None):
243 def __init__(self, settings=None, global_config=None):
236 self._sanitize_settings_and_apply_defaults(settings)
244 self._sanitize_settings_and_apply_defaults(settings)
237
245
238 self.config = Configurator(settings=settings)
246 self.config = Configurator(settings=settings)
239 self.global_config = global_config
247 self.global_config = global_config
240 self.config.include('vcsserver.lib.rc_cache')
248 self.config.include('vcsserver.lib.rc_cache')
241
249
242 settings_locale = settings.get('locale', '') or 'en_US.UTF-8'
250 settings_locale = settings.get('locale', '') or 'en_US.UTF-8'
243 vcs = VCS(locale_conf=settings_locale, cache_config=settings)
251 vcs = VCS(locale_conf=settings_locale, cache_config=settings)
244 self._remotes = {
252 self._remotes = {
245 'hg': vcs._hg_remote,
253 'hg': vcs._hg_remote,
246 'git': vcs._git_remote,
254 'git': vcs._git_remote,
247 'svn': vcs._svn_remote,
255 'svn': vcs._svn_remote,
248 'server': vcs._vcsserver,
256 'server': vcs._vcsserver,
249 }
257 }
250 if settings.get('dev.use_echo_app', 'false').lower() == 'true':
258 if settings.get('dev.use_echo_app', 'false').lower() == 'true':
251 self._use_echo_app = True
259 self._use_echo_app = True
252 log.warning("Using EchoApp for VCS operations.")
260 log.warning("Using EchoApp for VCS operations.")
253 self.remote_wsgi = remote_wsgi_stub
261 self.remote_wsgi = remote_wsgi_stub
254
262
255 self._configure_settings(global_config, settings)
263 self._configure_settings(global_config, settings)
256 self._configure()
264 self._configure()
257
265
258 def _configure_settings(self, global_config, app_settings):
266 def _configure_settings(self, global_config, app_settings):
259 """
267 """
260 Configure the settings module.
268 Configure the settings module.
261 """
269 """
262 settings_merged = global_config.copy()
270 settings_merged = global_config.copy()
263 settings_merged.update(app_settings)
271 settings_merged.update(app_settings)
264
272
265 git_path = app_settings.get('git_path', None)
273 git_path = app_settings.get('git_path', None)
266 if git_path:
274 if git_path:
267 settings.GIT_EXECUTABLE = git_path
275 settings.GIT_EXECUTABLE = git_path
268 binary_dir = app_settings.get('core.binary_dir', None)
276 binary_dir = app_settings.get('core.binary_dir', None)
269 if binary_dir:
277 if binary_dir:
270 settings.BINARY_DIR = binary_dir
278 settings.BINARY_DIR = binary_dir
271
279
272 # Store the settings to make them available to other modules.
280 # Store the settings to make them available to other modules.
273 vcsserver.PYRAMID_SETTINGS = settings_merged
281 vcsserver.PYRAMID_SETTINGS = settings_merged
274 vcsserver.CONFIG = settings_merged
282 vcsserver.CONFIG = settings_merged
275
283
276 def _sanitize_settings_and_apply_defaults(self, settings):
284 def _sanitize_settings_and_apply_defaults(self, settings):
277 temp_store = tempfile.gettempdir()
285 temp_store = tempfile.gettempdir()
278 default_cache_dir = os.path.join(temp_store, 'rc_cache')
286 default_cache_dir = os.path.join(temp_store, 'rc_cache')
279
287
280 # save default, cache dir, and use it for all backends later.
288 # save default, cache dir, and use it for all backends later.
281 default_cache_dir = _string_setting(
289 default_cache_dir = _string_setting(
282 settings,
290 settings,
283 'cache_dir',
291 'cache_dir',
284 default_cache_dir, lower=False, default_when_empty=True)
292 default_cache_dir, lower=False, default_when_empty=True)
285
293
286 # ensure we have our dir created
294 # ensure we have our dir created
287 if not os.path.isdir(default_cache_dir):
295 if not os.path.isdir(default_cache_dir):
288 os.makedirs(default_cache_dir, mode=0o755)
296 os.makedirs(default_cache_dir, mode=0o755)
289
297
290 # exception store cache
298 # exception store cache
291 _string_setting(
299 _string_setting(
292 settings,
300 settings,
293 'exception_tracker.store_path',
301 'exception_tracker.store_path',
294 temp_store, lower=False, default_when_empty=True)
302 temp_store, lower=False, default_when_empty=True)
295
303
296 # repo_object cache
304 # repo_object cache
297 _string_setting(
305 _string_setting(
298 settings,
306 settings,
299 'rc_cache.repo_object.backend',
307 'rc_cache.repo_object.backend',
300 'dogpile.cache.rc.file_namespace', lower=False)
308 'dogpile.cache.rc.file_namespace', lower=False)
301 _int_setting(
309 _int_setting(
302 settings,
310 settings,
303 'rc_cache.repo_object.expiration_time',
311 'rc_cache.repo_object.expiration_time',
304 30 * 24 * 60 * 60)
312 30 * 24 * 60 * 60)
305 _string_setting(
313 _string_setting(
306 settings,
314 settings,
307 'rc_cache.repo_object.arguments.filename',
315 'rc_cache.repo_object.arguments.filename',
308 os.path.join(default_cache_dir, 'vcsserver_cache_1'), lower=False)
316 os.path.join(default_cache_dir, 'vcsserver_cache_1'), lower=False)
309
317
310 def _configure(self):
318 def _configure(self):
311 self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory)
319 self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory)
312
320
313 self.config.add_route('service', '/_service')
321 self.config.add_route('service', '/_service')
314 self.config.add_route('status', '/status')
322 self.config.add_route('status', '/status')
315 self.config.add_route('hg_proxy', '/proxy/hg')
323 self.config.add_route('hg_proxy', '/proxy/hg')
316 self.config.add_route('git_proxy', '/proxy/git')
324 self.config.add_route('git_proxy', '/proxy/git')
317
325
318 # rpc methods
326 # rpc methods
319 self.config.add_route('vcs', '/{backend}')
327 self.config.add_route('vcs', '/{backend}')
320
328
321 # streaming rpc remote methods
329 # streaming rpc remote methods
322 self.config.add_route('vcs_stream', '/{backend}/stream')
330 self.config.add_route('vcs_stream', '/{backend}/stream')
323
331
324 # vcs operations clone/push as streaming
332 # vcs operations clone/push as streaming
325 self.config.add_route('stream_git', '/stream/git/*repo_name')
333 self.config.add_route('stream_git', '/stream/git/*repo_name')
326 self.config.add_route('stream_hg', '/stream/hg/*repo_name')
334 self.config.add_route('stream_hg', '/stream/hg/*repo_name')
327
335
328 self.config.add_view(self.status_view, route_name='status', renderer='json')
336 self.config.add_view(self.status_view, route_name='status', renderer='json')
329 self.config.add_view(self.service_view, route_name='service', renderer='msgpack')
337 self.config.add_view(self.service_view, route_name='service', renderer='msgpack')
330
338
331 self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
339 self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
332 self.config.add_view(self.git_proxy(), route_name='git_proxy')
340 self.config.add_view(self.git_proxy(), route_name='git_proxy')
333 self.config.add_view(self.vcs_view, route_name='vcs', renderer='msgpack',
341 self.config.add_view(self.vcs_view, route_name='vcs', renderer='msgpack',
334 vcs_view=self._remotes)
342 vcs_view=self._remotes)
335 self.config.add_view(self.vcs_stream_view, route_name='vcs_stream',
343 self.config.add_view(self.vcs_stream_view, route_name='vcs_stream',
336 vcs_view=self._remotes)
344 vcs_view=self._remotes)
337
345
338 self.config.add_view(self.hg_stream(), route_name='stream_hg')
346 self.config.add_view(self.hg_stream(), route_name='stream_hg')
339 self.config.add_view(self.git_stream(), route_name='stream_git')
347 self.config.add_view(self.git_stream(), route_name='stream_git')
340
348
341 self.config.add_view_predicate('vcs_view', VCSViewPredicate)
349 self.config.add_view_predicate('vcs_view', VCSViewPredicate)
342
350
343 self.config.add_notfound_view(not_found, renderer='json')
351 self.config.add_notfound_view(not_found, renderer='json')
344
352
345 self.config.add_view(self.handle_vcs_exception, context=Exception)
353 self.config.add_view(self.handle_vcs_exception, context=Exception)
346
354
347 self.config.add_tween(
355 self.config.add_tween(
348 'vcsserver.tweens.request_wrapper.RequestWrapperTween',
356 'vcsserver.tweens.request_wrapper.RequestWrapperTween',
349 )
357 )
350 self.config.add_request_method(
358 self.config.add_request_method(
351 'vcsserver.lib.request_counter.get_request_counter',
359 'vcsserver.lib.request_counter.get_request_counter',
352 'request_count')
360 'request_count')
353
361
354 def wsgi_app(self):
362 def wsgi_app(self):
355 return self.config.make_wsgi_app()
363 return self.config.make_wsgi_app()
356
364
357 def _vcs_view_params(self, request):
365 def _vcs_view_params(self, request):
358 remote = self._remotes[request.matchdict['backend']]
366 remote = self._remotes[request.matchdict['backend']]
359 payload = msgpack.unpackb(request.body, use_list=True)
367 payload = msgpack.unpackb(request.body, use_list=True)
360 method = payload.get('method')
368 method = payload.get('method')
361 params = payload['params']
369 params = payload['params']
362 wire = params.get('wire')
370 wire = params.get('wire')
363 args = params.get('args')
371 args = params.get('args')
364 kwargs = params.get('kwargs')
372 kwargs = params.get('kwargs')
365 context_uid = None
373 context_uid = None
366
374
367 if wire:
375 if wire:
368 try:
376 try:
369 wire['context'] = context_uid = uuid.UUID(wire['context'])
377 wire['context'] = context_uid = uuid.UUID(wire['context'])
370 except KeyError:
378 except KeyError:
371 pass
379 pass
372 args.insert(0, wire)
380 args.insert(0, wire)
373 repo_state_uid = wire.get('repo_state_uid') if wire else None
381 repo_state_uid = wire.get('repo_state_uid') if wire else None
374
382
375 # NOTE(marcink): trading complexity for slight performance
383 # NOTE(marcink): trading complexity for slight performance
376 if log.isEnabledFor(logging.DEBUG):
384 if log.isEnabledFor(logging.DEBUG):
377 no_args_methods = [
385 no_args_methods = [
378
386
379 ]
387 ]
380 if method in no_args_methods:
388 if method in no_args_methods:
381 call_args = ''
389 call_args = ''
382 else:
390 else:
383 call_args = args[1:]
391 call_args = args[1:]
384
392
385 log.debug('method requested:%s with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s',
393 log.debug('method requested:%s with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s',
386 method, call_args, kwargs, context_uid, repo_state_uid)
394 method, call_args, kwargs, context_uid, repo_state_uid)
387
395
388 return payload, remote, method, args, kwargs
396 return payload, remote, method, args, kwargs
389
397
390 def vcs_view(self, request):
398 def vcs_view(self, request):
391
399
392 payload, remote, method, args, kwargs = self._vcs_view_params(request)
400 payload, remote, method, args, kwargs = self._vcs_view_params(request)
393 payload_id = payload.get('id')
401 payload_id = payload.get('id')
394
402
395 try:
403 try:
396 resp = getattr(remote, method)(*args, **kwargs)
404 resp = getattr(remote, method)(*args, **kwargs)
397 except Exception as e:
405 except Exception as e:
398 exc_info = list(sys.exc_info())
406 exc_info = list(sys.exc_info())
399 exc_type, exc_value, exc_traceback = exc_info
407 exc_type, exc_value, exc_traceback = exc_info
400
408
401 org_exc = getattr(e, '_org_exc', None)
409 org_exc = getattr(e, '_org_exc', None)
402 org_exc_name = None
410 org_exc_name = None
403 org_exc_tb = ''
411 org_exc_tb = ''
404 if org_exc:
412 if org_exc:
405 org_exc_name = org_exc.__class__.__name__
413 org_exc_name = org_exc.__class__.__name__
406 org_exc_tb = getattr(e, '_org_exc_tb', '')
414 org_exc_tb = getattr(e, '_org_exc_tb', '')
407 # replace our "faked" exception with our org
415 # replace our "faked" exception with our org
408 exc_info[0] = org_exc.__class__
416 exc_info[0] = org_exc.__class__
409 exc_info[1] = org_exc
417 exc_info[1] = org_exc
410
418
411 should_store_exc = True
419 should_store_exc = True
412 if org_exc:
420 if org_exc:
413 def get_exc_fqn(_exc_obj):
421 def get_exc_fqn(_exc_obj):
414 module_name = getattr(org_exc.__class__, '__module__', 'UNKNOWN')
422 module_name = getattr(org_exc.__class__, '__module__', 'UNKNOWN')
415 return module_name + '.' + org_exc_name
423 return module_name + '.' + org_exc_name
416
424
417 exc_fqn = get_exc_fqn(org_exc)
425 exc_fqn = get_exc_fqn(org_exc)
418
426
419 if exc_fqn in ['mercurial.error.RepoLookupError',
427 if exc_fqn in ['mercurial.error.RepoLookupError',
420 'vcsserver.exceptions.RefNotFoundException']:
428 'vcsserver.exceptions.RefNotFoundException']:
421 should_store_exc = False
429 should_store_exc = False
422
430
423 if should_store_exc:
431 if should_store_exc:
424 store_exception(id(exc_info), exc_info)
432 store_exception(id(exc_info), exc_info)
425
433
426 tb_info = ''.join(
434 tb_info = ''.join(
427 traceback.format_exception(exc_type, exc_value, exc_traceback))
435 traceback.format_exception(exc_type, exc_value, exc_traceback))
428
436
429 type_ = e.__class__.__name__
437 type_ = e.__class__.__name__
430 if type_ not in self.ALLOWED_EXCEPTIONS:
438 if type_ not in self.ALLOWED_EXCEPTIONS:
431 type_ = None
439 type_ = None
432
440
433 resp = {
441 resp = {
434 'id': payload_id,
442 'id': payload_id,
435 'error': {
443 'error': {
436 'message': e.message,
444 'message': e.message,
437 'traceback': tb_info,
445 'traceback': tb_info,
438 'org_exc': org_exc_name,
446 'org_exc': org_exc_name,
439 'org_exc_tb': org_exc_tb,
447 'org_exc_tb': org_exc_tb,
440 'type': type_
448 'type': type_
441 }
449 }
442 }
450 }
443 try:
451 try:
444 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
452 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
445 except AttributeError:
453 except AttributeError:
446 pass
454 pass
447 else:
455 else:
448 resp = {
456 resp = {
449 'id': payload_id,
457 'id': payload_id,
450 'result': resp
458 'result': resp
451 }
459 }
452
460
453 return resp
461 return resp
454
462
455 def vcs_stream_view(self, request):
463 def vcs_stream_view(self, request):
456 payload, remote, method, args, kwargs = self._vcs_view_params(request)
464 payload, remote, method, args, kwargs = self._vcs_view_params(request)
457 # this method has a stream: marker we remove it here
465 # this method has a stream: marker we remove it here
458 method = method.split('stream:')[-1]
466 method = method.split('stream:')[-1]
459 chunk_size = safe_int(payload.get('chunk_size')) or 4096
467 chunk_size = safe_int(payload.get('chunk_size')) or 4096
460
468
461 try:
469 try:
462 resp = getattr(remote, method)(*args, **kwargs)
470 resp = getattr(remote, method)(*args, **kwargs)
463 except Exception as e:
471 except Exception as e:
464 raise
472 raise
465
473
466 def get_chunked_data(method_resp):
474 def get_chunked_data(method_resp):
467 stream = StringIO(method_resp)
475 stream = StringIO(method_resp)
468 while 1:
476 while 1:
469 chunk = stream.read(chunk_size)
477 chunk = stream.read(chunk_size)
470 if not chunk:
478 if not chunk:
471 break
479 break
472 yield chunk
480 yield chunk
473
481
474 response = Response(app_iter=get_chunked_data(resp))
482 response = Response(app_iter=get_chunked_data(resp))
475 response.content_type = 'application/octet-stream'
483 response.content_type = 'application/octet-stream'
476
484
477 return response
485 return response
478
486
479 def status_view(self, request):
487 def status_view(self, request):
480 import vcsserver
488 import vcsserver
481 return {'status': 'OK', 'vcsserver_version': vcsserver.__version__,
489 return {'status': 'OK', 'vcsserver_version': vcsserver.__version__,
482 'pid': os.getpid()}
490 'pid': os.getpid()}
483
491
484 def service_view(self, request):
492 def service_view(self, request):
485 import vcsserver
493 import vcsserver
486
494
487 payload = msgpack.unpackb(request.body, use_list=True)
495 payload = msgpack.unpackb(request.body, use_list=True)
488 server_config, app_config = {}, {}
496 server_config, app_config = {}, {}
489
497
490 try:
498 try:
491 path = self.global_config['__file__']
499 path = self.global_config['__file__']
492 config = configparser.RawConfigParser()
500 config = configparser.RawConfigParser()
493
501
494 config.read(path)
502 config.read(path)
495
503
496 if config.has_section('server:main'):
504 if config.has_section('server:main'):
497 server_config = dict(config.items('server:main'))
505 server_config = dict(config.items('server:main'))
498 if config.has_section('app:main'):
506 if config.has_section('app:main'):
499 app_config = dict(config.items('app:main'))
507 app_config = dict(config.items('app:main'))
500
508
501 except Exception:
509 except Exception:
502 log.exception('Failed to read .ini file for display')
510 log.exception('Failed to read .ini file for display')
503
511
504 environ = os.environ.items()
512 environ = os.environ.items()
505
513
506 resp = {
514 resp = {
507 'id': payload.get('id'),
515 'id': payload.get('id'),
508 'result': dict(
516 'result': dict(
509 version=vcsserver.__version__,
517 version=vcsserver.__version__,
510 config=server_config,
518 config=server_config,
511 app_config=app_config,
519 app_config=app_config,
512 environ=environ,
520 environ=environ,
513 payload=payload,
521 payload=payload,
514 )
522 )
515 }
523 }
516 return resp
524 return resp
517
525
518 def _msgpack_renderer_factory(self, info):
526 def _msgpack_renderer_factory(self, info):
519 def _render(value, system):
527 def _render(value, system):
520 request = system.get('request')
528 request = system.get('request')
521 if request is not None:
529 if request is not None:
522 response = request.response
530 response = request.response
523 ct = response.content_type
531 ct = response.content_type
524 if ct == response.default_content_type:
532 if ct == response.default_content_type:
525 response.content_type = 'application/x-msgpack'
533 response.content_type = 'application/x-msgpack'
526 return msgpack.packb(value)
534 return msgpack.packb(value)
527 return _render
535 return _render
528
536
529 def set_env_from_config(self, environ, config):
537 def set_env_from_config(self, environ, config):
530 dict_conf = {}
538 dict_conf = {}
531 try:
539 try:
532 for elem in config:
540 for elem in config:
533 if elem[0] == 'rhodecode':
541 if elem[0] == 'rhodecode':
534 dict_conf = json.loads(elem[2])
542 dict_conf = json.loads(elem[2])
535 break
543 break
536 except Exception:
544 except Exception:
537 log.exception('Failed to fetch SCM CONFIG')
545 log.exception('Failed to fetch SCM CONFIG')
538 return
546 return
539
547
540 username = dict_conf.get('username')
548 username = dict_conf.get('username')
541 if username:
549 if username:
542 environ['REMOTE_USER'] = username
550 environ['REMOTE_USER'] = username
543 # mercurial specific, some extension api rely on this
551 # mercurial specific, some extension api rely on this
544 environ['HGUSER'] = username
552 environ['HGUSER'] = username
545
553
546 ip = dict_conf.get('ip')
554 ip = dict_conf.get('ip')
547 if ip:
555 if ip:
548 environ['REMOTE_HOST'] = ip
556 environ['REMOTE_HOST'] = ip
549
557
550 if _is_request_chunked(environ):
558 if _is_request_chunked(environ):
551 # set the compatibility flag for webob
559 # set the compatibility flag for webob
552 environ['wsgi.input_terminated'] = True
560 environ['wsgi.input_terminated'] = True
553
561
554 def hg_proxy(self):
562 def hg_proxy(self):
555 @wsgiapp
563 @wsgiapp
556 def _hg_proxy(environ, start_response):
564 def _hg_proxy(environ, start_response):
557 app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
565 app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
558 return app(environ, start_response)
566 return app(environ, start_response)
559 return _hg_proxy
567 return _hg_proxy
560
568
561 def git_proxy(self):
569 def git_proxy(self):
562 @wsgiapp
570 @wsgiapp
563 def _git_proxy(environ, start_response):
571 def _git_proxy(environ, start_response):
564 app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
572 app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
565 return app(environ, start_response)
573 return app(environ, start_response)
566 return _git_proxy
574 return _git_proxy
567
575
568 def hg_stream(self):
576 def hg_stream(self):
569 if self._use_echo_app:
577 if self._use_echo_app:
570 @wsgiapp
578 @wsgiapp
571 def _hg_stream(environ, start_response):
579 def _hg_stream(environ, start_response):
572 app = EchoApp('fake_path', 'fake_name', None)
580 app = EchoApp('fake_path', 'fake_name', None)
573 return app(environ, start_response)
581 return app(environ, start_response)
574 return _hg_stream
582 return _hg_stream
575 else:
583 else:
576 @wsgiapp
584 @wsgiapp
577 def _hg_stream(environ, start_response):
585 def _hg_stream(environ, start_response):
578 log.debug('http-app: handling hg stream')
586 log.debug('http-app: handling hg stream')
579 repo_path = environ['HTTP_X_RC_REPO_PATH']
587 repo_path = environ['HTTP_X_RC_REPO_PATH']
580 repo_name = environ['HTTP_X_RC_REPO_NAME']
588 repo_name = environ['HTTP_X_RC_REPO_NAME']
581 packed_config = base64.b64decode(
589 packed_config = base64.b64decode(
582 environ['HTTP_X_RC_REPO_CONFIG'])
590 environ['HTTP_X_RC_REPO_CONFIG'])
583 config = msgpack.unpackb(packed_config)
591 config = msgpack.unpackb(packed_config)
584 app = scm_app.create_hg_wsgi_app(
592 app = scm_app.create_hg_wsgi_app(
585 repo_path, repo_name, config)
593 repo_path, repo_name, config)
586
594
587 # Consistent path information for hgweb
595 # Consistent path information for hgweb
588 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
596 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
589 environ['REPO_NAME'] = repo_name
597 environ['REPO_NAME'] = repo_name
590 self.set_env_from_config(environ, config)
598 self.set_env_from_config(environ, config)
591
599
592 log.debug('http-app: starting app handler '
600 log.debug('http-app: starting app handler '
593 'with %s and process request', app)
601 'with %s and process request', app)
594 return app(environ, ResponseFilter(start_response))
602 return app(environ, ResponseFilter(start_response))
595 return _hg_stream
603 return _hg_stream
596
604
597 def git_stream(self):
605 def git_stream(self):
598 if self._use_echo_app:
606 if self._use_echo_app:
599 @wsgiapp
607 @wsgiapp
600 def _git_stream(environ, start_response):
608 def _git_stream(environ, start_response):
601 app = EchoApp('fake_path', 'fake_name', None)
609 app = EchoApp('fake_path', 'fake_name', None)
602 return app(environ, start_response)
610 return app(environ, start_response)
603 return _git_stream
611 return _git_stream
604 else:
612 else:
605 @wsgiapp
613 @wsgiapp
606 def _git_stream(environ, start_response):
614 def _git_stream(environ, start_response):
607 log.debug('http-app: handling git stream')
615 log.debug('http-app: handling git stream')
608 repo_path = environ['HTTP_X_RC_REPO_PATH']
616 repo_path = environ['HTTP_X_RC_REPO_PATH']
609 repo_name = environ['HTTP_X_RC_REPO_NAME']
617 repo_name = environ['HTTP_X_RC_REPO_NAME']
610 packed_config = base64.b64decode(
618 packed_config = base64.b64decode(
611 environ['HTTP_X_RC_REPO_CONFIG'])
619 environ['HTTP_X_RC_REPO_CONFIG'])
612 config = msgpack.unpackb(packed_config)
620 config = msgpack.unpackb(packed_config)
613
621
614 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
622 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
615 self.set_env_from_config(environ, config)
623 self.set_env_from_config(environ, config)
616
624
617 content_type = environ.get('CONTENT_TYPE', '')
625 content_type = environ.get('CONTENT_TYPE', '')
618
626
619 path = environ['PATH_INFO']
627 path = environ['PATH_INFO']
620 is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
628 is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
621 log.debug(
629 log.debug(
622 'LFS: Detecting if request `%s` is LFS server path based '
630 'LFS: Detecting if request `%s` is LFS server path based '
623 'on content type:`%s`, is_lfs:%s',
631 'on content type:`%s`, is_lfs:%s',
624 path, content_type, is_lfs_request)
632 path, content_type, is_lfs_request)
625
633
626 if not is_lfs_request:
634 if not is_lfs_request:
627 # fallback detection by path
635 # fallback detection by path
628 if GIT_LFS_PROTO_PAT.match(path):
636 if GIT_LFS_PROTO_PAT.match(path):
629 is_lfs_request = True
637 is_lfs_request = True
630 log.debug(
638 log.debug(
631 'LFS: fallback detection by path of: `%s`, is_lfs:%s',
639 'LFS: fallback detection by path of: `%s`, is_lfs:%s',
632 path, is_lfs_request)
640 path, is_lfs_request)
633
641
634 if is_lfs_request:
642 if is_lfs_request:
635 app = scm_app.create_git_lfs_wsgi_app(
643 app = scm_app.create_git_lfs_wsgi_app(
636 repo_path, repo_name, config)
644 repo_path, repo_name, config)
637 else:
645 else:
638 app = scm_app.create_git_wsgi_app(
646 app = scm_app.create_git_wsgi_app(
639 repo_path, repo_name, config)
647 repo_path, repo_name, config)
640
648
641 log.debug('http-app: starting app handler '
649 log.debug('http-app: starting app handler '
642 'with %s and process request', app)
650 'with %s and process request', app)
643
651
644 return app(environ, start_response)
652 return app(environ, start_response)
645
653
646 return _git_stream
654 return _git_stream
647
655
648 def handle_vcs_exception(self, exception, request):
656 def handle_vcs_exception(self, exception, request):
649 _vcs_kind = getattr(exception, '_vcs_kind', '')
657 _vcs_kind = getattr(exception, '_vcs_kind', '')
650 if _vcs_kind == 'repo_locked':
658 if _vcs_kind == 'repo_locked':
651 # Get custom repo-locked status code if present.
659 # Get custom repo-locked status code if present.
652 status_code = request.headers.get('X-RC-Locked-Status-Code')
660 status_code = request.headers.get('X-RC-Locked-Status-Code')
653 return HTTPRepoLocked(
661 return HTTPRepoLocked(
654 title=exception.message, status_code=status_code)
662 title=exception.message, status_code=status_code)
655
663
656 elif _vcs_kind == 'repo_branch_protected':
664 elif _vcs_kind == 'repo_branch_protected':
657 # Get custom repo-branch-protected status code if present.
665 # Get custom repo-branch-protected status code if present.
658 return HTTPRepoBranchProtected(title=exception.message)
666 return HTTPRepoBranchProtected(title=exception.message)
659
667
660 exc_info = request.exc_info
668 exc_info = request.exc_info
661 store_exception(id(exc_info), exc_info)
669 store_exception(id(exc_info), exc_info)
662
670
663 traceback_info = 'unavailable'
671 traceback_info = 'unavailable'
664 if request.exc_info:
672 if request.exc_info:
665 exc_type, exc_value, exc_tb = request.exc_info
673 exc_type, exc_value, exc_tb = request.exc_info
666 traceback_info = ''.join(traceback.format_exception(exc_type, exc_value, exc_tb))
674 traceback_info = ''.join(traceback.format_exception(exc_type, exc_value, exc_tb))
667
675
668 log.error(
676 log.error(
669 'error occurred handling this request for path: %s, \n tb: %s',
677 'error occurred handling this request for path: %s, \n tb: %s',
670 request.path, traceback_info)
678 request.path, traceback_info)
671 raise exception
679 raise exception
672
680
673
681
674 class ResponseFilter(object):
682 class ResponseFilter(object):
675
683
676 def __init__(self, start_response):
684 def __init__(self, start_response):
677 self._start_response = start_response
685 self._start_response = start_response
678
686
679 def __call__(self, status, response_headers, exc_info=None):
687 def __call__(self, status, response_headers, exc_info=None):
680 headers = tuple(
688 headers = tuple(
681 (h, v) for h, v in response_headers
689 (h, v) for h, v in response_headers
682 if not wsgiref.util.is_hop_by_hop(h))
690 if not wsgiref.util.is_hop_by_hop(h))
683 return self._start_response(status, headers, exc_info)
691 return self._start_response(status, headers, exc_info)
684
692
685
693
686 def main(global_config, **settings):
694 def main(global_config, **settings):
687 if MercurialFactory:
695 if MercurialFactory:
688 hgpatches.patch_largefiles_capabilities()
696 hgpatches.patch_largefiles_capabilities()
689 hgpatches.patch_subrepo_type_mapping()
697 hgpatches.patch_subrepo_type_mapping()
690
698
691 app = HTTPApplication(settings=settings, global_config=global_config)
699 app = HTTPApplication(settings=settings, global_config=global_config)
692 return app.wsgi_app()
700 return app.wsgi_app()
General Comments 0
You need to be logged in to leave comments. Login now