##// END OF EJS Templates
vcs-error: replace disable_vcs middleware with vcs and http exceptions...
dan -
r682:1b4e984a default
parent child Browse files
Show More
@@ -0,0 +1,42 b''
1 # -*- coding: utf-8 -*-
2
3 # Copyright (C) 2010-2016 RhodeCode GmbH
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
21 import mock
22 import pytest
23 import rhodecode.lib.vcs.client as client
24
25 @pytest.mark.usefixtures('autologin_user', 'app')
26 def test_vcs_available_returns_summary_page(app, backend):
27 url = '/{repo_name}'.format(repo_name=backend.repo.repo_name)
28 response = app.get(url)
29 assert response.status_code == 200
30 assert 'Summary' in response.body
31
32
33 @pytest.mark.usefixtures('autologin_user', 'app')
34 def test_vcs_unavailable_returns_vcs_error_page(app, backend):
35 url = '/{repo_name}'.format(repo_name=backend.repo.repo_name)
36
37 with mock.patch.object(client, '_get_proxy_method') as p:
38 p.side_effect = client.exceptions.PyroVCSCommunicationError()
39 response = app.get(url, expect_errors=True)
40
41 assert response.status_code == 502
42 assert 'Could not connect to VCS Server' in response.body
@@ -1,474 +1,478 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2010-2016 RhodeCode GmbH
3 # Copyright (C) 2010-2016 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 """
21 """
22 Pylons middleware initialization
22 Pylons middleware initialization
23 """
23 """
24 import logging
24 import logging
25 from collections import OrderedDict
25 from collections import OrderedDict
26
26
27 from paste.registry import RegistryManager
27 from paste.registry import RegistryManager
28 from paste.gzipper import make_gzip_middleware
28 from paste.gzipper import make_gzip_middleware
29 from pylons.wsgiapp import PylonsApp
29 from pylons.wsgiapp import PylonsApp
30 from pyramid.authorization import ACLAuthorizationPolicy
30 from pyramid.authorization import ACLAuthorizationPolicy
31 from pyramid.config import Configurator
31 from pyramid.config import Configurator
32 from pyramid.settings import asbool, aslist
32 from pyramid.settings import asbool, aslist
33 from pyramid.wsgi import wsgiapp
33 from pyramid.wsgi import wsgiapp
34 from pyramid.httpexceptions import HTTPError, HTTPInternalServerError, HTTPFound
34 from pyramid.httpexceptions import HTTPError, HTTPInternalServerError, HTTPFound
35 from pyramid.events import ApplicationCreated
35 from pyramid.events import ApplicationCreated
36 import pyramid.httpexceptions as httpexceptions
36 import pyramid.httpexceptions as httpexceptions
37 from pyramid.renderers import render_to_response
37 from pyramid.renderers import render_to_response
38 from routes.middleware import RoutesMiddleware
38 from routes.middleware import RoutesMiddleware
39 import routes.util
39 import routes.util
40
40
41 import rhodecode
41 import rhodecode
42 from rhodecode.config import patches
42 from rhodecode.config import patches
43 from rhodecode.config.routing import STATIC_FILE_PREFIX
43 from rhodecode.config.routing import STATIC_FILE_PREFIX
44 from rhodecode.config.environment import (
44 from rhodecode.config.environment import (
45 load_environment, load_pyramid_environment)
45 load_environment, load_pyramid_environment)
46 from rhodecode.lib.exceptions import VCSServerUnavailable
47 from rhodecode.lib.vcs.exceptions import VCSCommunicationError
46 from rhodecode.lib.middleware import csrf
48 from rhodecode.lib.middleware import csrf
47 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
49 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
48 from rhodecode.lib.middleware.disable_vcs import DisableVCSPagesWrapper
49 from rhodecode.lib.middleware.https_fixup import HttpsFixup
50 from rhodecode.lib.middleware.https_fixup import HttpsFixup
50 from rhodecode.lib.middleware.vcs import VCSMiddleware
51 from rhodecode.lib.middleware.vcs import VCSMiddleware
51 from rhodecode.lib.plugins.utils import register_rhodecode_plugin
52 from rhodecode.lib.plugins.utils import register_rhodecode_plugin
52 from rhodecode.lib.utils2 import aslist as rhodecode_aslist
53 from rhodecode.lib.utils2 import aslist as rhodecode_aslist
53 from rhodecode.subscribers import scan_repositories_if_enabled
54 from rhodecode.subscribers import scan_repositories_if_enabled
54
55
55
56
56 log = logging.getLogger(__name__)
57 log = logging.getLogger(__name__)
57
58
58
59
59 # this is used to avoid avoid the route lookup overhead in routesmiddleware
60 # this is used to avoid avoid the route lookup overhead in routesmiddleware
60 # for certain routes which won't go to pylons to - eg. static files, debugger
61 # for certain routes which won't go to pylons to - eg. static files, debugger
61 # it is only needed for the pylons migration and can be removed once complete
62 # it is only needed for the pylons migration and can be removed once complete
62 class SkippableRoutesMiddleware(RoutesMiddleware):
63 class SkippableRoutesMiddleware(RoutesMiddleware):
63 """ Routes middleware that allows you to skip prefixes """
64 """ Routes middleware that allows you to skip prefixes """
64
65
65 def __init__(self, *args, **kw):
66 def __init__(self, *args, **kw):
66 self.skip_prefixes = kw.pop('skip_prefixes', [])
67 self.skip_prefixes = kw.pop('skip_prefixes', [])
67 super(SkippableRoutesMiddleware, self).__init__(*args, **kw)
68 super(SkippableRoutesMiddleware, self).__init__(*args, **kw)
68
69
69 def __call__(self, environ, start_response):
70 def __call__(self, environ, start_response):
70 for prefix in self.skip_prefixes:
71 for prefix in self.skip_prefixes:
71 if environ['PATH_INFO'].startswith(prefix):
72 if environ['PATH_INFO'].startswith(prefix):
72 # added to avoid the case when a missing /_static route falls
73 # added to avoid the case when a missing /_static route falls
73 # through to pylons and causes an exception as pylons is
74 # through to pylons and causes an exception as pylons is
74 # expecting wsgiorg.routingargs to be set in the environ
75 # expecting wsgiorg.routingargs to be set in the environ
75 # by RoutesMiddleware.
76 # by RoutesMiddleware.
76 if 'wsgiorg.routing_args' not in environ:
77 if 'wsgiorg.routing_args' not in environ:
77 environ['wsgiorg.routing_args'] = (None, {})
78 environ['wsgiorg.routing_args'] = (None, {})
78 return self.app(environ, start_response)
79 return self.app(environ, start_response)
79
80
80 return super(SkippableRoutesMiddleware, self).__call__(
81 return super(SkippableRoutesMiddleware, self).__call__(
81 environ, start_response)
82 environ, start_response)
82
83
83
84
84 def make_app(global_conf, static_files=True, **app_conf):
85 def make_app(global_conf, static_files=True, **app_conf):
85 """Create a Pylons WSGI application and return it
86 """Create a Pylons WSGI application and return it
86
87
87 ``global_conf``
88 ``global_conf``
88 The inherited configuration for this application. Normally from
89 The inherited configuration for this application. Normally from
89 the [DEFAULT] section of the Paste ini file.
90 the [DEFAULT] section of the Paste ini file.
90
91
91 ``app_conf``
92 ``app_conf``
92 The application's local configuration. Normally specified in
93 The application's local configuration. Normally specified in
93 the [app:<name>] section of the Paste ini file (where <name>
94 the [app:<name>] section of the Paste ini file (where <name>
94 defaults to main).
95 defaults to main).
95
96
96 """
97 """
97 # Apply compatibility patches
98 # Apply compatibility patches
98 patches.kombu_1_5_1_python_2_7_11()
99 patches.kombu_1_5_1_python_2_7_11()
99 patches.inspect_getargspec()
100 patches.inspect_getargspec()
100
101
101 # Configure the Pylons environment
102 # Configure the Pylons environment
102 config = load_environment(global_conf, app_conf)
103 config = load_environment(global_conf, app_conf)
103
104
104 # The Pylons WSGI app
105 # The Pylons WSGI app
105 app = PylonsApp(config=config)
106 app = PylonsApp(config=config)
106 if rhodecode.is_test:
107 if rhodecode.is_test:
107 app = csrf.CSRFDetector(app)
108 app = csrf.CSRFDetector(app)
108
109
109 expected_origin = config.get('expected_origin')
110 expected_origin = config.get('expected_origin')
110 if expected_origin:
111 if expected_origin:
111 # The API can be accessed from other Origins.
112 # The API can be accessed from other Origins.
112 app = csrf.OriginChecker(app, expected_origin,
113 app = csrf.OriginChecker(app, expected_origin,
113 skip_urls=[routes.util.url_for('api')])
114 skip_urls=[routes.util.url_for('api')])
114
115
115 # Establish the Registry for this application
116 # Establish the Registry for this application
116 app = RegistryManager(app)
117 app = RegistryManager(app)
117
118
118 app.config = config
119 app.config = config
119
120
120 return app
121 return app
121
122
122
123
123 def make_pyramid_app(global_config, **settings):
124 def make_pyramid_app(global_config, **settings):
124 """
125 """
125 Constructs the WSGI application based on Pyramid and wraps the Pylons based
126 Constructs the WSGI application based on Pyramid and wraps the Pylons based
126 application.
127 application.
127
128
128 Specials:
129 Specials:
129
130
130 * We migrate from Pylons to Pyramid. While doing this, we keep both
131 * We migrate from Pylons to Pyramid. While doing this, we keep both
131 frameworks functional. This involves moving some WSGI middlewares around
132 frameworks functional. This involves moving some WSGI middlewares around
132 and providing access to some data internals, so that the old code is
133 and providing access to some data internals, so that the old code is
133 still functional.
134 still functional.
134
135
135 * The application can also be integrated like a plugin via the call to
136 * The application can also be integrated like a plugin via the call to
136 `includeme`. This is accompanied with the other utility functions which
137 `includeme`. This is accompanied with the other utility functions which
137 are called. Changing this should be done with great care to not break
138 are called. Changing this should be done with great care to not break
138 cases when these fragments are assembled from another place.
139 cases when these fragments are assembled from another place.
139
140
140 """
141 """
141 # The edition string should be available in pylons too, so we add it here
142 # The edition string should be available in pylons too, so we add it here
142 # before copying the settings.
143 # before copying the settings.
143 settings.setdefault('rhodecode.edition', 'Community Edition')
144 settings.setdefault('rhodecode.edition', 'Community Edition')
144
145
145 # As long as our Pylons application does expect "unprepared" settings, make
146 # As long as our Pylons application does expect "unprepared" settings, make
146 # sure that we keep an unmodified copy. This avoids unintentional change of
147 # sure that we keep an unmodified copy. This avoids unintentional change of
147 # behavior in the old application.
148 # behavior in the old application.
148 settings_pylons = settings.copy()
149 settings_pylons = settings.copy()
149
150
150 sanitize_settings_and_apply_defaults(settings)
151 sanitize_settings_and_apply_defaults(settings)
151 config = Configurator(settings=settings)
152 config = Configurator(settings=settings)
152 add_pylons_compat_data(config.registry, global_config, settings_pylons)
153 add_pylons_compat_data(config.registry, global_config, settings_pylons)
153
154
154 load_pyramid_environment(global_config, settings)
155 load_pyramid_environment(global_config, settings)
155
156
156 includeme_first(config)
157 includeme_first(config)
157 includeme(config)
158 includeme(config)
158 pyramid_app = config.make_wsgi_app()
159 pyramid_app = config.make_wsgi_app()
159 pyramid_app = wrap_app_in_wsgi_middlewares(pyramid_app, config)
160 pyramid_app = wrap_app_in_wsgi_middlewares(pyramid_app, config)
160 pyramid_app.config = config
161 pyramid_app.config = config
161 return pyramid_app
162 return pyramid_app
162
163
163
164
164 def make_not_found_view(config):
165 def make_not_found_view(config):
165 """
166 """
166 This creates the view which should be registered as not-found-view to
167 This creates the view which should be registered as not-found-view to
167 pyramid. Basically it contains of the old pylons app, converted to a view.
168 pyramid. Basically it contains of the old pylons app, converted to a view.
168 Additionally it is wrapped by some other middlewares.
169 Additionally it is wrapped by some other middlewares.
169 """
170 """
170 settings = config.registry.settings
171 settings = config.registry.settings
171 vcs_server_enabled = settings['vcs.server.enable']
172 vcs_server_enabled = settings['vcs.server.enable']
172
173
173 # Make pylons app from unprepared settings.
174 # Make pylons app from unprepared settings.
174 pylons_app = make_app(
175 pylons_app = make_app(
175 config.registry._pylons_compat_global_config,
176 config.registry._pylons_compat_global_config,
176 **config.registry._pylons_compat_settings)
177 **config.registry._pylons_compat_settings)
177 config.registry._pylons_compat_config = pylons_app.config
178 config.registry._pylons_compat_config = pylons_app.config
178
179
179 # Appenlight monitoring.
180 # Appenlight monitoring.
180 pylons_app, appenlight_client = wrap_in_appenlight_if_enabled(
181 pylons_app, appenlight_client = wrap_in_appenlight_if_enabled(
181 pylons_app, settings)
182 pylons_app, settings)
182
183
183 # The VCSMiddleware shall operate like a fallback if pyramid doesn't find
184 # The VCSMiddleware shall operate like a fallback if pyramid doesn't find
184 # a view to handle the request. Therefore we wrap it around the pylons app.
185 # a view to handle the request. Therefore we wrap it around the pylons app.
185 if vcs_server_enabled:
186 if vcs_server_enabled:
186 pylons_app = VCSMiddleware(
187 pylons_app = VCSMiddleware(
187 pylons_app, settings, appenlight_client, registry=config.registry)
188 pylons_app, settings, appenlight_client, registry=config.registry)
188
189
189 pylons_app_as_view = wsgiapp(pylons_app)
190 pylons_app_as_view = wsgiapp(pylons_app)
190
191
191 # Protect from VCS Server error related pages when server is not available
192 if not vcs_server_enabled:
193 pylons_app_as_view = DisableVCSPagesWrapper(pylons_app_as_view)
194
195 def pylons_app_with_error_handler(context, request):
192 def pylons_app_with_error_handler(context, request):
196 """
193 """
197 Handle exceptions from rc pylons app:
194 Handle exceptions from rc pylons app:
198
195
199 - old webob type exceptions get converted to pyramid exceptions
196 - old webob type exceptions get converted to pyramid exceptions
200 - pyramid exceptions are passed to the error handler view
197 - pyramid exceptions are passed to the error handler view
201 """
198 """
202 def is_vcs_response(response):
199 def is_vcs_response(response):
203 return 'X-RhodeCode-Backend' in response.headers
200 return 'X-RhodeCode-Backend' in response.headers
204
201
205 def is_http_error(response):
202 def is_http_error(response):
206 # webob type error responses
203 # webob type error responses
207 return (400 <= response.status_int <= 599)
204 return (400 <= response.status_int <= 599)
208
205
209 def is_error_handling_needed(response):
206 def is_error_handling_needed(response):
210 return is_http_error(response) and not is_vcs_response(response)
207 return is_http_error(response) and not is_vcs_response(response)
211
208
212 try:
209 try:
213 response = pylons_app_as_view(context, request)
210 response = pylons_app_as_view(context, request)
214 if is_error_handling_needed(response):
211 if is_error_handling_needed(response):
215 response = webob_to_pyramid_http_response(response)
212 response = webob_to_pyramid_http_response(response)
216 return error_handler(response, request)
213 return error_handler(response, request)
217 except HTTPError as e: # pyramid type exceptions
214 except HTTPError as e: # pyramid type exceptions
218 return error_handler(e, request)
215 return error_handler(e, request)
219 except Exception:
216 except Exception as e:
217 log.exception(e)
218
220 if settings.get('debugtoolbar.enabled', False):
219 if settings.get('debugtoolbar.enabled', False):
221 raise
220 raise
221
222 if isinstance(e, VCSCommunicationError):
223 return error_handler(VCSServerUnavailable(), request)
224
222 return error_handler(HTTPInternalServerError(), request)
225 return error_handler(HTTPInternalServerError(), request)
226
223 return response
227 return response
224
228
225 return pylons_app_with_error_handler
229 return pylons_app_with_error_handler
226
230
227
231
228 def add_pylons_compat_data(registry, global_config, settings):
232 def add_pylons_compat_data(registry, global_config, settings):
229 """
233 """
230 Attach data to the registry to support the Pylons integration.
234 Attach data to the registry to support the Pylons integration.
231 """
235 """
232 registry._pylons_compat_global_config = global_config
236 registry._pylons_compat_global_config = global_config
233 registry._pylons_compat_settings = settings
237 registry._pylons_compat_settings = settings
234
238
235
239
236 def webob_to_pyramid_http_response(webob_response):
240 def webob_to_pyramid_http_response(webob_response):
237 ResponseClass = httpexceptions.status_map[webob_response.status_int]
241 ResponseClass = httpexceptions.status_map[webob_response.status_int]
238 pyramid_response = ResponseClass(webob_response.status)
242 pyramid_response = ResponseClass(webob_response.status)
239 pyramid_response.status = webob_response.status
243 pyramid_response.status = webob_response.status
240 pyramid_response.headers.update(webob_response.headers)
244 pyramid_response.headers.update(webob_response.headers)
241 if pyramid_response.headers['content-type'] == 'text/html':
245 if pyramid_response.headers['content-type'] == 'text/html':
242 pyramid_response.headers['content-type'] = 'text/html; charset=UTF-8'
246 pyramid_response.headers['content-type'] = 'text/html; charset=UTF-8'
243 return pyramid_response
247 return pyramid_response
244
248
245
249
246 def error_handler(exception, request):
250 def error_handler(exception, request):
247 # TODO: dan: replace the old pylons error controller with this
251 # TODO: dan: replace the old pylons error controller with this
248 from rhodecode.model.settings import SettingsModel
252 from rhodecode.model.settings import SettingsModel
249 from rhodecode.lib.utils2 import AttributeDict
253 from rhodecode.lib.utils2 import AttributeDict
250
254
251 try:
255 try:
252 rc_config = SettingsModel().get_all_settings()
256 rc_config = SettingsModel().get_all_settings()
253 except Exception:
257 except Exception:
254 log.exception('failed to fetch settings')
258 log.exception('failed to fetch settings')
255 rc_config = {}
259 rc_config = {}
256
260
257 base_response = HTTPInternalServerError()
261 base_response = HTTPInternalServerError()
258 # prefer original exception for the response since it may have headers set
262 # prefer original exception for the response since it may have headers set
259 if isinstance(exception, HTTPError):
263 if isinstance(exception, HTTPError):
260 base_response = exception
264 base_response = exception
261
265
262 c = AttributeDict()
266 c = AttributeDict()
263 c.error_message = base_response.status
267 c.error_message = base_response.status
264 c.error_explanation = base_response.explanation or str(base_response)
268 c.error_explanation = base_response.explanation or str(base_response)
265 c.visual = AttributeDict()
269 c.visual = AttributeDict()
266
270
267 c.visual.rhodecode_support_url = (
271 c.visual.rhodecode_support_url = (
268 request.registry.settings.get('rhodecode_support_url') or
272 request.registry.settings.get('rhodecode_support_url') or
269 request.route_url('rhodecode_support')
273 request.route_url('rhodecode_support')
270 )
274 )
271 c.redirect_time = 0
275 c.redirect_time = 0
272 c.rhodecode_name = rc_config.get('rhodecode_title', '')
276 c.rhodecode_name = rc_config.get('rhodecode_title', '')
273 if not c.rhodecode_name:
277 if not c.rhodecode_name:
274 c.rhodecode_name = 'Rhodecode'
278 c.rhodecode_name = 'Rhodecode'
275
279
276 response = render_to_response(
280 response = render_to_response(
277 '/errors/error_document.html', {'c': c}, request=request,
281 '/errors/error_document.html', {'c': c}, request=request,
278 response=base_response)
282 response=base_response)
279
283
280 return response
284 return response
281
285
282
286
283 def includeme(config):
287 def includeme(config):
284 settings = config.registry.settings
288 settings = config.registry.settings
285
289
286 # plugin information
290 # plugin information
287 config.registry.rhodecode_plugins = OrderedDict()
291 config.registry.rhodecode_plugins = OrderedDict()
288
292
289 config.add_directive(
293 config.add_directive(
290 'register_rhodecode_plugin', register_rhodecode_plugin)
294 'register_rhodecode_plugin', register_rhodecode_plugin)
291
295
292 if asbool(settings.get('appenlight', 'false')):
296 if asbool(settings.get('appenlight', 'false')):
293 config.include('appenlight_client.ext.pyramid_tween')
297 config.include('appenlight_client.ext.pyramid_tween')
294
298
295 # Includes which are required. The application would fail without them.
299 # Includes which are required. The application would fail without them.
296 config.include('pyramid_mako')
300 config.include('pyramid_mako')
297 config.include('pyramid_beaker')
301 config.include('pyramid_beaker')
298 config.include('rhodecode.channelstream')
302 config.include('rhodecode.channelstream')
299 config.include('rhodecode.admin')
303 config.include('rhodecode.admin')
300 config.include('rhodecode.authentication')
304 config.include('rhodecode.authentication')
301 config.include('rhodecode.integrations')
305 config.include('rhodecode.integrations')
302 config.include('rhodecode.login')
306 config.include('rhodecode.login')
303 config.include('rhodecode.tweens')
307 config.include('rhodecode.tweens')
304 config.include('rhodecode.api')
308 config.include('rhodecode.api')
305 config.include('rhodecode.svn_support')
309 config.include('rhodecode.svn_support')
306 config.add_route(
310 config.add_route(
307 'rhodecode_support', 'https://rhodecode.com/help/', static=True)
311 'rhodecode_support', 'https://rhodecode.com/help/', static=True)
308
312
309 # Add subscribers.
313 # Add subscribers.
310 config.add_subscriber(scan_repositories_if_enabled, ApplicationCreated)
314 config.add_subscriber(scan_repositories_if_enabled, ApplicationCreated)
311
315
312 # Set the authorization policy.
316 # Set the authorization policy.
313 authz_policy = ACLAuthorizationPolicy()
317 authz_policy = ACLAuthorizationPolicy()
314 config.set_authorization_policy(authz_policy)
318 config.set_authorization_policy(authz_policy)
315
319
316 # Set the default renderer for HTML templates to mako.
320 # Set the default renderer for HTML templates to mako.
317 config.add_mako_renderer('.html')
321 config.add_mako_renderer('.html')
318
322
319 # include RhodeCode plugins
323 # include RhodeCode plugins
320 includes = aslist(settings.get('rhodecode.includes', []))
324 includes = aslist(settings.get('rhodecode.includes', []))
321 for inc in includes:
325 for inc in includes:
322 config.include(inc)
326 config.include(inc)
323
327
324 # This is the glue which allows us to migrate in chunks. By registering the
328 # This is the glue which allows us to migrate in chunks. By registering the
325 # pylons based application as the "Not Found" view in Pyramid, we will
329 # pylons based application as the "Not Found" view in Pyramid, we will
326 # fallback to the old application each time the new one does not yet know
330 # fallback to the old application each time the new one does not yet know
327 # how to handle a request.
331 # how to handle a request.
328 config.add_notfound_view(make_not_found_view(config))
332 config.add_notfound_view(make_not_found_view(config))
329
333
330 if not settings.get('debugtoolbar.enabled', False):
334 if not settings.get('debugtoolbar.enabled', False):
331 # if no toolbar, then any exception gets caught and rendered
335 # if no toolbar, then any exception gets caught and rendered
332 config.add_view(error_handler, context=Exception)
336 config.add_view(error_handler, context=Exception)
333
337
334 config.add_view(error_handler, context=HTTPError)
338 config.add_view(error_handler, context=HTTPError)
335
339
336
340
337 def includeme_first(config):
341 def includeme_first(config):
338 # redirect automatic browser favicon.ico requests to correct place
342 # redirect automatic browser favicon.ico requests to correct place
339 def favicon_redirect(context, request):
343 def favicon_redirect(context, request):
340 return HTTPFound(
344 return HTTPFound(
341 request.static_path('rhodecode:public/images/favicon.ico'))
345 request.static_path('rhodecode:public/images/favicon.ico'))
342
346
343 config.add_view(favicon_redirect, route_name='favicon')
347 config.add_view(favicon_redirect, route_name='favicon')
344 config.add_route('favicon', '/favicon.ico')
348 config.add_route('favicon', '/favicon.ico')
345
349
346 config.add_static_view(
350 config.add_static_view(
347 '_static/deform', 'deform:static')
351 '_static/deform', 'deform:static')
348 config.add_static_view(
352 config.add_static_view(
349 '_static/rhodecode', path='rhodecode:public', cache_max_age=3600 * 24)
353 '_static/rhodecode', path='rhodecode:public', cache_max_age=3600 * 24)
350
354
351
355
352 def wrap_app_in_wsgi_middlewares(pyramid_app, config):
356 def wrap_app_in_wsgi_middlewares(pyramid_app, config):
353 """
357 """
354 Apply outer WSGI middlewares around the application.
358 Apply outer WSGI middlewares around the application.
355
359
356 Part of this has been moved up from the Pylons layer, so that the
360 Part of this has been moved up from the Pylons layer, so that the
357 data is also available if old Pylons code is hit through an already ported
361 data is also available if old Pylons code is hit through an already ported
358 view.
362 view.
359 """
363 """
360 settings = config.registry.settings
364 settings = config.registry.settings
361
365
362 # enable https redirects based on HTTP_X_URL_SCHEME set by proxy
366 # enable https redirects based on HTTP_X_URL_SCHEME set by proxy
363 pyramid_app = HttpsFixup(pyramid_app, settings)
367 pyramid_app = HttpsFixup(pyramid_app, settings)
364
368
365 # Add RoutesMiddleware to support the pylons compatibility tween during
369 # Add RoutesMiddleware to support the pylons compatibility tween during
366 # migration to pyramid.
370 # migration to pyramid.
367 pyramid_app = SkippableRoutesMiddleware(
371 pyramid_app = SkippableRoutesMiddleware(
368 pyramid_app, config.registry._pylons_compat_config['routes.map'],
372 pyramid_app, config.registry._pylons_compat_config['routes.map'],
369 skip_prefixes=(STATIC_FILE_PREFIX, '/_debug_toolbar'))
373 skip_prefixes=(STATIC_FILE_PREFIX, '/_debug_toolbar'))
370
374
371 pyramid_app, _ = wrap_in_appenlight_if_enabled(pyramid_app, settings)
375 pyramid_app, _ = wrap_in_appenlight_if_enabled(pyramid_app, settings)
372
376
373 if settings['gzip_responses']:
377 if settings['gzip_responses']:
374 pyramid_app = make_gzip_middleware(
378 pyramid_app = make_gzip_middleware(
375 pyramid_app, settings, compress_level=1)
379 pyramid_app, settings, compress_level=1)
376
380
377 return pyramid_app
381 return pyramid_app
378
382
379
383
380 def sanitize_settings_and_apply_defaults(settings):
384 def sanitize_settings_and_apply_defaults(settings):
381 """
385 """
382 Applies settings defaults and does all type conversion.
386 Applies settings defaults and does all type conversion.
383
387
384 We would move all settings parsing and preparation into this place, so that
388 We would move all settings parsing and preparation into this place, so that
385 we have only one place left which deals with this part. The remaining parts
389 we have only one place left which deals with this part. The remaining parts
386 of the application would start to rely fully on well prepared settings.
390 of the application would start to rely fully on well prepared settings.
387
391
388 This piece would later be split up per topic to avoid a big fat monster
392 This piece would later be split up per topic to avoid a big fat monster
389 function.
393 function.
390 """
394 """
391
395
392 # Pyramid's mako renderer has to search in the templates folder so that the
396 # Pyramid's mako renderer has to search in the templates folder so that the
393 # old templates still work. Ported and new templates are expected to use
397 # old templates still work. Ported and new templates are expected to use
394 # real asset specifications for the includes.
398 # real asset specifications for the includes.
395 mako_directories = settings.setdefault('mako.directories', [
399 mako_directories = settings.setdefault('mako.directories', [
396 # Base templates of the original Pylons application
400 # Base templates of the original Pylons application
397 'rhodecode:templates',
401 'rhodecode:templates',
398 ])
402 ])
399 log.debug(
403 log.debug(
400 "Using the following Mako template directories: %s",
404 "Using the following Mako template directories: %s",
401 mako_directories)
405 mako_directories)
402
406
403 # Default includes, possible to change as a user
407 # Default includes, possible to change as a user
404 pyramid_includes = settings.setdefault('pyramid.includes', [
408 pyramid_includes = settings.setdefault('pyramid.includes', [
405 'rhodecode.lib.middleware.request_wrapper',
409 'rhodecode.lib.middleware.request_wrapper',
406 ])
410 ])
407 log.debug(
411 log.debug(
408 "Using the following pyramid.includes: %s",
412 "Using the following pyramid.includes: %s",
409 pyramid_includes)
413 pyramid_includes)
410
414
411 # TODO: johbo: Re-think this, usually the call to config.include
415 # TODO: johbo: Re-think this, usually the call to config.include
412 # should allow to pass in a prefix.
416 # should allow to pass in a prefix.
413 settings.setdefault('rhodecode.api.url', '/_admin/api')
417 settings.setdefault('rhodecode.api.url', '/_admin/api')
414
418
415 # Sanitize generic settings.
419 # Sanitize generic settings.
416 _list_setting(settings, 'default_encoding', 'UTF-8')
420 _list_setting(settings, 'default_encoding', 'UTF-8')
417 _bool_setting(settings, 'is_test', 'false')
421 _bool_setting(settings, 'is_test', 'false')
418 _bool_setting(settings, 'gzip_responses', 'false')
422 _bool_setting(settings, 'gzip_responses', 'false')
419
423
420 # Call split out functions that sanitize settings for each topic.
424 # Call split out functions that sanitize settings for each topic.
421 _sanitize_appenlight_settings(settings)
425 _sanitize_appenlight_settings(settings)
422 _sanitize_vcs_settings(settings)
426 _sanitize_vcs_settings(settings)
423
427
424 return settings
428 return settings
425
429
426
430
427 def _sanitize_appenlight_settings(settings):
431 def _sanitize_appenlight_settings(settings):
428 _bool_setting(settings, 'appenlight', 'false')
432 _bool_setting(settings, 'appenlight', 'false')
429
433
430
434
431 def _sanitize_vcs_settings(settings):
435 def _sanitize_vcs_settings(settings):
432 """
436 """
433 Applies settings defaults and does type conversion for all VCS related
437 Applies settings defaults and does type conversion for all VCS related
434 settings.
438 settings.
435 """
439 """
436 _string_setting(settings, 'vcs.svn.compatible_version', '')
440 _string_setting(settings, 'vcs.svn.compatible_version', '')
437 _string_setting(settings, 'git_rev_filter', '--all')
441 _string_setting(settings, 'git_rev_filter', '--all')
438 _string_setting(settings, 'vcs.hooks.protocol', 'pyro4')
442 _string_setting(settings, 'vcs.hooks.protocol', 'pyro4')
439 _string_setting(settings, 'vcs.server', '')
443 _string_setting(settings, 'vcs.server', '')
440 _string_setting(settings, 'vcs.server.log_level', 'debug')
444 _string_setting(settings, 'vcs.server.log_level', 'debug')
441 _string_setting(settings, 'vcs.server.protocol', 'pyro4')
445 _string_setting(settings, 'vcs.server.protocol', 'pyro4')
442 _bool_setting(settings, 'startup.import_repos', 'false')
446 _bool_setting(settings, 'startup.import_repos', 'false')
443 _bool_setting(settings, 'vcs.hooks.direct_calls', 'false')
447 _bool_setting(settings, 'vcs.hooks.direct_calls', 'false')
444 _bool_setting(settings, 'vcs.server.enable', 'true')
448 _bool_setting(settings, 'vcs.server.enable', 'true')
445 _bool_setting(settings, 'vcs.start_server', 'false')
449 _bool_setting(settings, 'vcs.start_server', 'false')
446 _list_setting(settings, 'vcs.backends', 'hg, git, svn')
450 _list_setting(settings, 'vcs.backends', 'hg, git, svn')
447 _int_setting(settings, 'vcs.connection_timeout', 3600)
451 _int_setting(settings, 'vcs.connection_timeout', 3600)
448
452
449
453
450 def _int_setting(settings, name, default):
454 def _int_setting(settings, name, default):
451 settings[name] = int(settings.get(name, default))
455 settings[name] = int(settings.get(name, default))
452
456
453
457
454 def _bool_setting(settings, name, default):
458 def _bool_setting(settings, name, default):
455 input = settings.get(name, default)
459 input = settings.get(name, default)
456 if isinstance(input, unicode):
460 if isinstance(input, unicode):
457 input = input.encode('utf8')
461 input = input.encode('utf8')
458 settings[name] = asbool(input)
462 settings[name] = asbool(input)
459
463
460
464
461 def _list_setting(settings, name, default):
465 def _list_setting(settings, name, default):
462 raw_value = settings.get(name, default)
466 raw_value = settings.get(name, default)
463
467
464 old_separator = ','
468 old_separator = ','
465 if old_separator in raw_value:
469 if old_separator in raw_value:
466 # If we get a comma separated list, pass it to our own function.
470 # If we get a comma separated list, pass it to our own function.
467 settings[name] = rhodecode_aslist(raw_value, sep=old_separator)
471 settings[name] = rhodecode_aslist(raw_value, sep=old_separator)
468 else:
472 else:
469 # Otherwise we assume it uses pyramids space/newline separation.
473 # Otherwise we assume it uses pyramids space/newline separation.
470 settings[name] = aslist(raw_value)
474 settings[name] = aslist(raw_value)
471
475
472
476
473 def _string_setting(settings, name, default):
477 def _string_setting(settings, name, default):
474 settings[name] = settings.get(name, default).lower()
478 settings[name] = settings.get(name, default).lower()
@@ -1,122 +1,134 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2010-2016 RhodeCode GmbH
3 # Copyright (C) 2010-2016 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 """
21 """
22 Set of custom exceptions used in RhodeCode
22 Set of custom exceptions used in RhodeCode
23 """
23 """
24
24
25 from webob.exc import HTTPClientError
25 from webob.exc import HTTPClientError
26 from pyramid.httpexceptions import HTTPBadGateway
26
27
27
28
28 class LdapUsernameError(Exception):
29 class LdapUsernameError(Exception):
29 pass
30 pass
30
31
31
32
32 class LdapPasswordError(Exception):
33 class LdapPasswordError(Exception):
33 pass
34 pass
34
35
35
36
36 class LdapConnectionError(Exception):
37 class LdapConnectionError(Exception):
37 pass
38 pass
38
39
39
40
40 class LdapImportError(Exception):
41 class LdapImportError(Exception):
41 pass
42 pass
42
43
43
44
44 class DefaultUserException(Exception):
45 class DefaultUserException(Exception):
45 pass
46 pass
46
47
47
48
48 class UserOwnsReposException(Exception):
49 class UserOwnsReposException(Exception):
49 pass
50 pass
50
51
51
52
52 class UserOwnsRepoGroupsException(Exception):
53 class UserOwnsRepoGroupsException(Exception):
53 pass
54 pass
54
55
55
56
56 class UserOwnsUserGroupsException(Exception):
57 class UserOwnsUserGroupsException(Exception):
57 pass
58 pass
58
59
59
60
60 class UserGroupAssignedException(Exception):
61 class UserGroupAssignedException(Exception):
61 pass
62 pass
62
63
63
64
64 class StatusChangeOnClosedPullRequestError(Exception):
65 class StatusChangeOnClosedPullRequestError(Exception):
65 pass
66 pass
66
67
67
68
68 class AttachedForksError(Exception):
69 class AttachedForksError(Exception):
69 pass
70 pass
70
71
71
72
72 class RepoGroupAssignmentError(Exception):
73 class RepoGroupAssignmentError(Exception):
73 pass
74 pass
74
75
75
76
76 class NonRelativePathError(Exception):
77 class NonRelativePathError(Exception):
77 pass
78 pass
78
79
79
80
80 class HTTPRequirementError(HTTPClientError):
81 class HTTPRequirementError(HTTPClientError):
81 title = explanation = 'Repository Requirement Missing'
82 title = explanation = 'Repository Requirement Missing'
82 reason = None
83 reason = None
83
84
84 def __init__(self, message, *args, **kwargs):
85 def __init__(self, message, *args, **kwargs):
85 self.title = self.explanation = message
86 self.title = self.explanation = message
86 super(HTTPRequirementError, self).__init__(*args, **kwargs)
87 super(HTTPRequirementError, self).__init__(*args, **kwargs)
87 self.args = (message, )
88 self.args = (message, )
88
89
89
90
90 class HTTPLockedRC(HTTPClientError):
91 class HTTPLockedRC(HTTPClientError):
91 """
92 """
92 Special Exception For locked Repos in RhodeCode, the return code can
93 Special Exception For locked Repos in RhodeCode, the return code can
93 be overwritten by _code keyword argument passed into constructors
94 be overwritten by _code keyword argument passed into constructors
94 """
95 """
95 code = 423
96 code = 423
96 title = explanation = 'Repository Locked'
97 title = explanation = 'Repository Locked'
97 reason = None
98 reason = None
98
99
99 def __init__(self, message, *args, **kwargs):
100 def __init__(self, message, *args, **kwargs):
100 from rhodecode import CONFIG
101 from rhodecode import CONFIG
101 from rhodecode.lib.utils2 import safe_int
102 from rhodecode.lib.utils2 import safe_int
102 _code = CONFIG.get('lock_ret_code')
103 _code = CONFIG.get('lock_ret_code')
103 self.code = safe_int(_code, self.code)
104 self.code = safe_int(_code, self.code)
104 self.title = self.explanation = message
105 self.title = self.explanation = message
105 super(HTTPLockedRC, self).__init__(*args, **kwargs)
106 super(HTTPLockedRC, self).__init__(*args, **kwargs)
106 self.args = (message, )
107 self.args = (message, )
107
108
108
109
109 class IMCCommitError(Exception):
110 class IMCCommitError(Exception):
110 pass
111 pass
111
112
112
113
113 class UserCreationError(Exception):
114 class UserCreationError(Exception):
114 pass
115 pass
115
116
116
117
117 class NotAllowedToCreateUserError(Exception):
118 class NotAllowedToCreateUserError(Exception):
118 pass
119 pass
119
120
120
121
121 class RepositoryCreationError(Exception):
122 class RepositoryCreationError(Exception):
122 pass
123 pass
124
125
126 class VCSServerUnavailable(HTTPBadGateway):
127 """ HTTP Exception class for VCS Server errors """
128 code = 502
129 title = 'VCS Server Error'
130 def __init__(self, message=''):
131 self.explanation = 'Could not connect to VCS Server'
132 if message:
133 self.explanation += ': ' + message
134 super(VCSServerUnavailable, self).__init__()
@@ -1,346 +1,346 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2014-2016 RhodeCode GmbH
3 # Copyright (C) 2014-2016 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 """
21 """
22 Provides the implementation of various client utilities to reach the vcsserver.
22 Provides the implementation of various client utilities to reach the vcsserver.
23 """
23 """
24
24
25
25
26 import copy
26 import copy
27 import logging
27 import logging
28 import threading
28 import threading
29 import urlparse
29 import urlparse
30 import uuid
30 import uuid
31 import weakref
31 import weakref
32 from urllib2 import URLError
32 from urllib2 import URLError
33
33
34 import msgpack
34 import msgpack
35 import Pyro4
35 import Pyro4
36 import requests
36 import requests
37 from pyramid.threadlocal import get_current_request
37 from pyramid.threadlocal import get_current_request
38 from Pyro4.errors import CommunicationError, ConnectionClosedError, DaemonError
38 from Pyro4.errors import CommunicationError, ConnectionClosedError, DaemonError
39
39
40 from rhodecode.lib.vcs import exceptions
40 from rhodecode.lib.vcs import exceptions
41 from rhodecode.lib.vcs.conf import settings
41 from rhodecode.lib.vcs.conf import settings
42
42
43 log = logging.getLogger(__name__)
43 log = logging.getLogger(__name__)
44
44
45
45
46 # TODO: mikhail: Keep it in sync with vcsserver's
46 # TODO: mikhail: Keep it in sync with vcsserver's
47 # HTTPApplication.ALLOWED_EXCEPTIONS
47 # HTTPApplication.ALLOWED_EXCEPTIONS
48 EXCEPTIONS_MAP = {
48 EXCEPTIONS_MAP = {
49 'KeyError': KeyError,
49 'KeyError': KeyError,
50 'URLError': URLError,
50 'URLError': URLError,
51 }
51 }
52
52
53
53
54 class HTTPRepoMaker(object):
54 class HTTPRepoMaker(object):
55 def __init__(self, server_and_port, backend_endpoint):
55 def __init__(self, server_and_port, backend_endpoint):
56 self.url = urlparse.urljoin(
56 self.url = urlparse.urljoin(
57 'http://%s' % server_and_port, backend_endpoint)
57 'http://%s' % server_and_port, backend_endpoint)
58
58
59 def __call__(self, path, config, with_wire=None):
59 def __call__(self, path, config, with_wire=None):
60 log.debug('HTTPRepoMaker call on %s', path)
60 log.debug('HTTPRepoMaker call on %s', path)
61 return HTTPRemoteRepo(path, config, self.url, with_wire=with_wire)
61 return HTTPRemoteRepo(path, config, self.url, with_wire=with_wire)
62
62
63 def __getattr__(self, name):
63 def __getattr__(self, name):
64 def f(*args, **kwargs):
64 def f(*args, **kwargs):
65 return self._call(name, *args, **kwargs)
65 return self._call(name, *args, **kwargs)
66 return f
66 return f
67
67
68 @exceptions.map_vcs_exceptions
68 @exceptions.map_vcs_exceptions
69 def _call(self, name, *args, **kwargs):
69 def _call(self, name, *args, **kwargs):
70 payload = {
70 payload = {
71 'id': str(uuid.uuid4()),
71 'id': str(uuid.uuid4()),
72 'method': name,
72 'method': name,
73 'params': {'args': args, 'kwargs': kwargs}
73 'params': {'args': args, 'kwargs': kwargs}
74 }
74 }
75 return _remote_call(self.url, payload, EXCEPTIONS_MAP)
75 return _remote_call(self.url, payload, EXCEPTIONS_MAP)
76
76
77
77
78 class VcsHttpProxy(object):
78 class VcsHttpProxy(object):
79
79
80 CHUNK_SIZE = 16384
80 CHUNK_SIZE = 16384
81
81
82 def __init__(self, server_and_port, backend_endpoint):
82 def __init__(self, server_and_port, backend_endpoint):
83 adapter = requests.adapters.HTTPAdapter(max_retries=5)
83 adapter = requests.adapters.HTTPAdapter(max_retries=5)
84 self.base_url = urlparse.urljoin(
84 self.base_url = urlparse.urljoin(
85 'http://%s' % server_and_port, backend_endpoint)
85 'http://%s' % server_and_port, backend_endpoint)
86 self.session = requests.Session()
86 self.session = requests.Session()
87 self.session.mount('http://', adapter)
87 self.session.mount('http://', adapter)
88
88
89 def handle(self, environment, input_data, *args, **kwargs):
89 def handle(self, environment, input_data, *args, **kwargs):
90 data = {
90 data = {
91 'environment': environment,
91 'environment': environment,
92 'input_data': input_data,
92 'input_data': input_data,
93 'args': args,
93 'args': args,
94 'kwargs': kwargs
94 'kwargs': kwargs
95 }
95 }
96 result = self.session.post(
96 result = self.session.post(
97 self.base_url, msgpack.packb(data), stream=True)
97 self.base_url, msgpack.packb(data), stream=True)
98 return self._get_result(result)
98 return self._get_result(result)
99
99
100 def _deserialize_and_raise(self, error):
100 def _deserialize_and_raise(self, error):
101 exception = Exception(error['message'])
101 exception = Exception(error['message'])
102 try:
102 try:
103 exception._vcs_kind = error['_vcs_kind']
103 exception._vcs_kind = error['_vcs_kind']
104 except KeyError:
104 except KeyError:
105 pass
105 pass
106 raise exception
106 raise exception
107
107
108 def _iterate(self, result):
108 def _iterate(self, result):
109 unpacker = msgpack.Unpacker()
109 unpacker = msgpack.Unpacker()
110 for line in result.iter_content(chunk_size=self.CHUNK_SIZE):
110 for line in result.iter_content(chunk_size=self.CHUNK_SIZE):
111 unpacker.feed(line)
111 unpacker.feed(line)
112 for chunk in unpacker:
112 for chunk in unpacker:
113 yield chunk
113 yield chunk
114
114
115 def _get_result(self, result):
115 def _get_result(self, result):
116 iterator = self._iterate(result)
116 iterator = self._iterate(result)
117 error = iterator.next()
117 error = iterator.next()
118 if error:
118 if error:
119 self._deserialize_and_raise(error)
119 self._deserialize_and_raise(error)
120
120
121 status = iterator.next()
121 status = iterator.next()
122 headers = iterator.next()
122 headers = iterator.next()
123
123
124 return iterator, status, headers
124 return iterator, status, headers
125
125
126
126
127 class HTTPRemoteRepo(object):
127 class HTTPRemoteRepo(object):
128 def __init__(self, path, config, url, with_wire=None):
128 def __init__(self, path, config, url, with_wire=None):
129 self.url = url
129 self.url = url
130 self._wire = {
130 self._wire = {
131 "path": path,
131 "path": path,
132 "config": config,
132 "config": config,
133 "context": str(uuid.uuid4()),
133 "context": str(uuid.uuid4()),
134 }
134 }
135 if with_wire:
135 if with_wire:
136 self._wire.update(with_wire)
136 self._wire.update(with_wire)
137
137
138 def __getattr__(self, name):
138 def __getattr__(self, name):
139 def f(*args, **kwargs):
139 def f(*args, **kwargs):
140 return self._call(name, *args, **kwargs)
140 return self._call(name, *args, **kwargs)
141 return f
141 return f
142
142
143 @exceptions.map_vcs_exceptions
143 @exceptions.map_vcs_exceptions
144 def _call(self, name, *args, **kwargs):
144 def _call(self, name, *args, **kwargs):
145 log.debug('Calling %s@%s', self.url, name)
145 log.debug('Calling %s@%s', self.url, name)
146 # TODO: oliver: This is currently necessary pre-call since the
146 # TODO: oliver: This is currently necessary pre-call since the
147 # config object is being changed for hooking scenarios
147 # config object is being changed for hooking scenarios
148 wire = copy.deepcopy(self._wire)
148 wire = copy.deepcopy(self._wire)
149 wire["config"] = wire["config"].serialize()
149 wire["config"] = wire["config"].serialize()
150 payload = {
150 payload = {
151 'id': str(uuid.uuid4()),
151 'id': str(uuid.uuid4()),
152 'method': name,
152 'method': name,
153 'params': {'wire': wire, 'args': args, 'kwargs': kwargs}
153 'params': {'wire': wire, 'args': args, 'kwargs': kwargs}
154 }
154 }
155 return _remote_call(self.url, payload, EXCEPTIONS_MAP)
155 return _remote_call(self.url, payload, EXCEPTIONS_MAP)
156
156
157 def __getitem__(self, key):
157 def __getitem__(self, key):
158 return self.revision(key)
158 return self.revision(key)
159
159
160
160
161 def _remote_call(url, payload, exceptions_map):
161 def _remote_call(url, payload, exceptions_map):
162 response = requests.post(url, data=msgpack.packb(payload))
162 response = requests.post(url, data=msgpack.packb(payload))
163 response = msgpack.unpackb(response.content)
163 response = msgpack.unpackb(response.content)
164 error = response.get('error')
164 error = response.get('error')
165 if error:
165 if error:
166 type_ = error.get('type', 'Exception')
166 type_ = error.get('type', 'Exception')
167 exc = exceptions_map.get(type_, Exception)
167 exc = exceptions_map.get(type_, Exception)
168 exc = exc(error.get('message'))
168 exc = exc(error.get('message'))
169 try:
169 try:
170 exc._vcs_kind = error['_vcs_kind']
170 exc._vcs_kind = error['_vcs_kind']
171 except KeyError:
171 except KeyError:
172 pass
172 pass
173 raise exc
173 raise exc
174 return response.get('result')
174 return response.get('result')
175
175
176
176
177 class RepoMaker(object):
177 class RepoMaker(object):
178
178
179 def __init__(self, proxy_factory):
179 def __init__(self, proxy_factory):
180 self._proxy_factory = proxy_factory
180 self._proxy_factory = proxy_factory
181
181
182 def __call__(self, path, config, with_wire=None):
182 def __call__(self, path, config, with_wire=None):
183 log.debug('RepoMaker call on %s', path)
183 log.debug('RepoMaker call on %s', path)
184 return RemoteRepo(
184 return RemoteRepo(
185 path, config, remote_proxy=self._proxy_factory(),
185 path, config, remote_proxy=self._proxy_factory(),
186 with_wire=with_wire)
186 with_wire=with_wire)
187
187
188 def __getattr__(self, name):
188 def __getattr__(self, name):
189 remote_proxy = self._proxy_factory()
189 remote_proxy = self._proxy_factory()
190 func = _get_proxy_method(remote_proxy, name)
190 func = _get_proxy_method(remote_proxy, name)
191 return _wrap_remote_call(remote_proxy, func)
191 return _wrap_remote_call(remote_proxy, func)
192
192
193
193
194 class RequestScopeProxyFactory(object):
194 class RequestScopeProxyFactory(object):
195 """
195 """
196 This factory returns pyro proxy instances based on a per request scope.
196 This factory returns pyro proxy instances based on a per request scope.
197 It returns the same instance if called from within the same request and
197 It returns the same instance if called from within the same request and
198 different instances if called from different requests.
198 different instances if called from different requests.
199 """
199 """
200
200
201 def __init__(self, remote_uri):
201 def __init__(self, remote_uri):
202 self._remote_uri = remote_uri
202 self._remote_uri = remote_uri
203 self._proxy_pool = []
203 self._proxy_pool = []
204 self._borrowed_proxies = {}
204 self._borrowed_proxies = {}
205
205
206 def __call__(self, request=None):
206 def __call__(self, request=None):
207 """
207 """
208 Wrapper around `getProxy`.
208 Wrapper around `getProxy`.
209 """
209 """
210 request = request or get_current_request()
210 request = request or get_current_request()
211 return self.getProxy(request)
211 return self.getProxy(request)
212
212
213 def getProxy(self, request):
213 def getProxy(self, request):
214 """
214 """
215 Call this to get the pyro proxy instance for the request.
215 Call this to get the pyro proxy instance for the request.
216 """
216 """
217
217
218 # If called without a request context we return new proxy instances
218 # If called without a request context we return new proxy instances
219 # on every call. This allows to run e.g. invoke tasks.
219 # on every call. This allows to run e.g. invoke tasks.
220 if request is None:
220 if request is None:
221 log.info('Creating pyro proxy without request context for '
221 log.info('Creating pyro proxy without request context for '
222 'remote_uri=%s', self._remote_uri)
222 'remote_uri=%s', self._remote_uri)
223 return Pyro4.Proxy(self._remote_uri)
223 return Pyro4.Proxy(self._remote_uri)
224
224
225 # If there is an already borrowed proxy for the request context we
225 # If there is an already borrowed proxy for the request context we
226 # return that instance instead of creating a new one.
226 # return that instance instead of creating a new one.
227 if request in self._borrowed_proxies:
227 if request in self._borrowed_proxies:
228 return self._borrowed_proxies[request]
228 return self._borrowed_proxies[request]
229
229
230 # Get proxy from pool or create new instance.
230 # Get proxy from pool or create new instance.
231 try:
231 try:
232 proxy = self._proxy_pool.pop()
232 proxy = self._proxy_pool.pop()
233 except IndexError:
233 except IndexError:
234 log.info('Creating pyro proxy for remote_uri=%s', self._remote_uri)
234 log.info('Creating pyro proxy for remote_uri=%s', self._remote_uri)
235 proxy = Pyro4.Proxy(self._remote_uri)
235 proxy = Pyro4.Proxy(self._remote_uri)
236
236
237 # Mark proxy as borrowed for the request context and add a callback
237 # Mark proxy as borrowed for the request context and add a callback
238 # that returns it when the request processing is finished.
238 # that returns it when the request processing is finished.
239 self._borrowed_proxies[request] = proxy
239 self._borrowed_proxies[request] = proxy
240 request.add_finished_callback(self._returnProxy)
240 request.add_finished_callback(self._returnProxy)
241
241
242 return proxy
242 return proxy
243
243
244 def _returnProxy(self, request):
244 def _returnProxy(self, request):
245 """
245 """
246 Callback that gets called by pyramid when the request is finished.
246 Callback that gets called by pyramid when the request is finished.
247 It puts the proxy back into the pool.
247 It puts the proxy back into the pool.
248 """
248 """
249 if request in self._borrowed_proxies:
249 if request in self._borrowed_proxies:
250 proxy = self._borrowed_proxies.pop(request)
250 proxy = self._borrowed_proxies.pop(request)
251 self._proxy_pool.append(proxy)
251 self._proxy_pool.append(proxy)
252 else:
252 else:
253 log.warn('Return proxy for remote_uri=%s but no proxy borrowed '
253 log.warn('Return proxy for remote_uri=%s but no proxy borrowed '
254 'for this request.', self._remote_uri)
254 'for this request.', self._remote_uri)
255
255
256
256
257 class RemoteRepo(object):
257 class RemoteRepo(object):
258
258
259 def __init__(self, path, config, remote_proxy, with_wire=None):
259 def __init__(self, path, config, remote_proxy, with_wire=None):
260 self._wire = {
260 self._wire = {
261 "path": path,
261 "path": path,
262 "config": config,
262 "config": config,
263 "context": self._create_vcs_cache_context(),
263 "context": self._create_vcs_cache_context(),
264 }
264 }
265 if with_wire:
265 if with_wire:
266 self._wire.update(with_wire)
266 self._wire.update(with_wire)
267 self._remote_proxy = remote_proxy
267 self._remote_proxy = remote_proxy
268 self.refs = RefsWrapper(self)
268 self.refs = RefsWrapper(self)
269
269
270 def __getattr__(self, name):
270 def __getattr__(self, name):
271 log.debug('Calling %s@%s', self._remote_proxy, name)
271 log.debug('Calling %s@%s', self._remote_proxy, name)
272 # TODO: oliver: This is currently necessary pre-call since the
272 # TODO: oliver: This is currently necessary pre-call since the
273 # config object is being changed for hooking scenarios
273 # config object is being changed for hooking scenarios
274 wire = copy.deepcopy(self._wire)
274 wire = copy.deepcopy(self._wire)
275 wire["config"] = wire["config"].serialize()
275 wire["config"] = wire["config"].serialize()
276
276
277 try:
277 try:
278 func = _get_proxy_method(self._remote_proxy, name)
278 func = _get_proxy_method(self._remote_proxy, name)
279 except DaemonError as e:
279 except DaemonError as e:
280 if e.message == 'unknown object':
280 if e.message == 'unknown object':
281 raise exceptions.VCSBackendNotSupportedError
281 raise exceptions.VCSBackendNotSupportedError
282 else:
282 else:
283 raise
283 raise
284
284
285 return _wrap_remote_call(self._remote_proxy, func, wire)
285 return _wrap_remote_call(self._remote_proxy, func, wire)
286
286
287 def __getitem__(self, key):
287 def __getitem__(self, key):
288 return self.revision(key)
288 return self.revision(key)
289
289
290 def _create_vcs_cache_context(self):
290 def _create_vcs_cache_context(self):
291 """
291 """
292 Creates a unique string which is passed to the VCSServer on every
292 Creates a unique string which is passed to the VCSServer on every
293 remote call. It is used as cache key in the VCSServer.
293 remote call. It is used as cache key in the VCSServer.
294 """
294 """
295 return str(uuid.uuid4())
295 return str(uuid.uuid4())
296
296
297 def invalidate_vcs_cache(self):
297 def invalidate_vcs_cache(self):
298 """
298 """
299 This is a no-op method for the pyro4 backend but we want to have the
299 This is a no-op method for the pyro4 backend but we want to have the
300 same API for client.RemoteRepo and client_http.RemoteRepo classes.
300 same API for client.RemoteRepo and client_http.RemoteRepo classes.
301 """
301 """
302
302
303
303
304 def _get_proxy_method(proxy, name):
304 def _get_proxy_method(proxy, name):
305 try:
305 try:
306 return getattr(proxy, name)
306 return getattr(proxy, name)
307 except CommunicationError:
307 except CommunicationError:
308 raise CommunicationError(
308 raise exceptions.PyroVCSCommunicationError(
309 'Unable to connect to remote pyro server %s' % proxy)
309 'Unable to connect to remote pyro server %s' % proxy)
310
310
311
311
312 def _wrap_remote_call(proxy, func, *args):
312 def _wrap_remote_call(proxy, func, *args):
313 all_args = list(args)
313 all_args = list(args)
314
314
315 @exceptions.map_vcs_exceptions
315 @exceptions.map_vcs_exceptions
316 def caller(*args, **kwargs):
316 def caller(*args, **kwargs):
317 all_args.extend(args)
317 all_args.extend(args)
318 try:
318 try:
319 return func(*all_args, **kwargs)
319 return func(*all_args, **kwargs)
320 except ConnectionClosedError:
320 except ConnectionClosedError:
321 log.debug('Connection to VCSServer closed, trying to reconnect.')
321 log.debug('Connection to VCSServer closed, trying to reconnect.')
322 proxy._pyroReconnect(tries=settings.PYRO_RECONNECT_TRIES)
322 proxy._pyroReconnect(tries=settings.PYRO_RECONNECT_TRIES)
323
323
324 return func(*all_args, **kwargs)
324 return func(*all_args, **kwargs)
325
325
326 return caller
326 return caller
327
327
328
328
329 class RefsWrapper(object):
329 class RefsWrapper(object):
330
330
331 def __init__(self, repo):
331 def __init__(self, repo):
332 self._repo = weakref.proxy(repo)
332 self._repo = weakref.proxy(repo)
333
333
334 def __setitem__(self, key, value):
334 def __setitem__(self, key, value):
335 self._repo._assign_ref(key, value)
335 self._repo._assign_ref(key, value)
336
336
337
337
338 class FunctionWrapper(object):
338 class FunctionWrapper(object):
339
339
340 def __init__(self, func, wire):
340 def __init__(self, func, wire):
341 self._func = func
341 self._func = func
342 self._wire = wire
342 self._wire = wire
343
343
344 @exceptions.map_vcs_exceptions
344 @exceptions.map_vcs_exceptions
345 def __call__(self, *args, **kwargs):
345 def __call__(self, *args, **kwargs):
346 return self._func(self._wire, *args, **kwargs)
346 return self._func(self._wire, *args, **kwargs)
@@ -1,250 +1,255 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2016-2016 RhodeCode GmbH
3 # Copyright (C) 2016-2016 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 """
21 """
22 Client for the VCSServer implemented based on HTTP.
22 Client for the VCSServer implemented based on HTTP.
23
23
24
24
25 Status
25 Status
26 ------
26 ------
27
27
28 This client implementation shall eventually replace the Pyro4 based
28 This client implementation shall eventually replace the Pyro4 based
29 implementation.
29 implementation.
30 """
30 """
31
31
32 import copy
32 import copy
33 import logging
33 import logging
34 import threading
34 import threading
35 import urllib2
35 import urllib2
36 import urlparse
36 import urlparse
37 import uuid
37 import uuid
38
38
39 import pycurl
39 import msgpack
40 import msgpack
40 import requests
41 import requests
41
42
42 from . import exceptions, CurlSession
43 from . import exceptions, CurlSession
43
44
44
45
45 log = logging.getLogger(__name__)
46 log = logging.getLogger(__name__)
46
47
47
48
48 # TODO: mikhail: Keep it in sync with vcsserver's
49 # TODO: mikhail: Keep it in sync with vcsserver's
49 # HTTPApplication.ALLOWED_EXCEPTIONS
50 # HTTPApplication.ALLOWED_EXCEPTIONS
50 EXCEPTIONS_MAP = {
51 EXCEPTIONS_MAP = {
51 'KeyError': KeyError,
52 'KeyError': KeyError,
52 'URLError': urllib2.URLError,
53 'URLError': urllib2.URLError,
53 }
54 }
54
55
55
56
56 class RepoMaker(object):
57 class RepoMaker(object):
57
58
58 def __init__(self, server_and_port, backend_endpoint, session_factory):
59 def __init__(self, server_and_port, backend_endpoint, session_factory):
59 self.url = urlparse.urljoin(
60 self.url = urlparse.urljoin(
60 'http://%s' % server_and_port, backend_endpoint)
61 'http://%s' % server_and_port, backend_endpoint)
61 self._session_factory = session_factory
62 self._session_factory = session_factory
62
63
63 def __call__(self, path, config, with_wire=None):
64 def __call__(self, path, config, with_wire=None):
64 log.debug('RepoMaker call on %s', path)
65 log.debug('RepoMaker call on %s', path)
65 return RemoteRepo(
66 return RemoteRepo(
66 path, config, self.url, self._session_factory(),
67 path, config, self.url, self._session_factory(),
67 with_wire=with_wire)
68 with_wire=with_wire)
68
69
69 def __getattr__(self, name):
70 def __getattr__(self, name):
70 def f(*args, **kwargs):
71 def f(*args, **kwargs):
71 return self._call(name, *args, **kwargs)
72 return self._call(name, *args, **kwargs)
72 return f
73 return f
73
74
74 @exceptions.map_vcs_exceptions
75 @exceptions.map_vcs_exceptions
75 def _call(self, name, *args, **kwargs):
76 def _call(self, name, *args, **kwargs):
76 payload = {
77 payload = {
77 'id': str(uuid.uuid4()),
78 'id': str(uuid.uuid4()),
78 'method': name,
79 'method': name,
79 'params': {'args': args, 'kwargs': kwargs}
80 'params': {'args': args, 'kwargs': kwargs}
80 }
81 }
81 return _remote_call(
82 return _remote_call(
82 self.url, payload, EXCEPTIONS_MAP, self._session_factory())
83 self.url, payload, EXCEPTIONS_MAP, self._session_factory())
83
84
84
85
85 class RemoteRepo(object):
86 class RemoteRepo(object):
86
87
87 def __init__(self, path, config, url, session, with_wire=None):
88 def __init__(self, path, config, url, session, with_wire=None):
88 self.url = url
89 self.url = url
89 self._session = session
90 self._session = session
90 self._wire = {
91 self._wire = {
91 "path": path,
92 "path": path,
92 "config": config,
93 "config": config,
93 "context": self._create_vcs_cache_context(),
94 "context": self._create_vcs_cache_context(),
94 }
95 }
95 if with_wire:
96 if with_wire:
96 self._wire.update(with_wire)
97 self._wire.update(with_wire)
97
98
98 # johbo: Trading complexity for performance. Avoiding the call to
99 # johbo: Trading complexity for performance. Avoiding the call to
99 # log.debug brings a few percent gain even if is is not active.
100 # log.debug brings a few percent gain even if is is not active.
100 if log.isEnabledFor(logging.DEBUG):
101 if log.isEnabledFor(logging.DEBUG):
101 self._call = self._call_with_logging
102 self._call = self._call_with_logging
102
103
103 def __getattr__(self, name):
104 def __getattr__(self, name):
104 def f(*args, **kwargs):
105 def f(*args, **kwargs):
105 return self._call(name, *args, **kwargs)
106 return self._call(name, *args, **kwargs)
106 return f
107 return f
107
108
108 @exceptions.map_vcs_exceptions
109 @exceptions.map_vcs_exceptions
109 def _call(self, name, *args, **kwargs):
110 def _call(self, name, *args, **kwargs):
110 # TODO: oliver: This is currently necessary pre-call since the
111 # TODO: oliver: This is currently necessary pre-call since the
111 # config object is being changed for hooking scenarios
112 # config object is being changed for hooking scenarios
112 wire = copy.deepcopy(self._wire)
113 wire = copy.deepcopy(self._wire)
113 wire["config"] = wire["config"].serialize()
114 wire["config"] = wire["config"].serialize()
114 payload = {
115 payload = {
115 'id': str(uuid.uuid4()),
116 'id': str(uuid.uuid4()),
116 'method': name,
117 'method': name,
117 'params': {'wire': wire, 'args': args, 'kwargs': kwargs}
118 'params': {'wire': wire, 'args': args, 'kwargs': kwargs}
118 }
119 }
119 return _remote_call(self.url, payload, EXCEPTIONS_MAP, self._session)
120 return _remote_call(self.url, payload, EXCEPTIONS_MAP, self._session)
120
121
121 def _call_with_logging(self, name, *args, **kwargs):
122 def _call_with_logging(self, name, *args, **kwargs):
122 log.debug('Calling %s@%s', self.url, name)
123 log.debug('Calling %s@%s', self.url, name)
123 return RemoteRepo._call(self, name, *args, **kwargs)
124 return RemoteRepo._call(self, name, *args, **kwargs)
124
125
125 def __getitem__(self, key):
126 def __getitem__(self, key):
126 return self.revision(key)
127 return self.revision(key)
127
128
128 def _create_vcs_cache_context(self):
129 def _create_vcs_cache_context(self):
129 """
130 """
130 Creates a unique string which is passed to the VCSServer on every
131 Creates a unique string which is passed to the VCSServer on every
131 remote call. It is used as cache key in the VCSServer.
132 remote call. It is used as cache key in the VCSServer.
132 """
133 """
133 return str(uuid.uuid4())
134 return str(uuid.uuid4())
134
135
135 def invalidate_vcs_cache(self):
136 def invalidate_vcs_cache(self):
136 """
137 """
137 This invalidates the context which is sent to the VCSServer on every
138 This invalidates the context which is sent to the VCSServer on every
138 call to a remote method. It forces the VCSServer to create a fresh
139 call to a remote method. It forces the VCSServer to create a fresh
139 repository instance on the next call to a remote method.
140 repository instance on the next call to a remote method.
140 """
141 """
141 self._wire['context'] = self._create_vcs_cache_context()
142 self._wire['context'] = self._create_vcs_cache_context()
142
143
143
144
144 class RemoteObject(object):
145 class RemoteObject(object):
145
146
146 def __init__(self, url, session):
147 def __init__(self, url, session):
147 self._url = url
148 self._url = url
148 self._session = session
149 self._session = session
149
150
150 # johbo: Trading complexity for performance. Avoiding the call to
151 # johbo: Trading complexity for performance. Avoiding the call to
151 # log.debug brings a few percent gain even if is is not active.
152 # log.debug brings a few percent gain even if is is not active.
152 if log.isEnabledFor(logging.DEBUG):
153 if log.isEnabledFor(logging.DEBUG):
153 self._call = self._call_with_logging
154 self._call = self._call_with_logging
154
155
155 def __getattr__(self, name):
156 def __getattr__(self, name):
156 def f(*args, **kwargs):
157 def f(*args, **kwargs):
157 return self._call(name, *args, **kwargs)
158 return self._call(name, *args, **kwargs)
158 return f
159 return f
159
160
160 @exceptions.map_vcs_exceptions
161 @exceptions.map_vcs_exceptions
161 def _call(self, name, *args, **kwargs):
162 def _call(self, name, *args, **kwargs):
162 payload = {
163 payload = {
163 'id': str(uuid.uuid4()),
164 'id': str(uuid.uuid4()),
164 'method': name,
165 'method': name,
165 'params': {'args': args, 'kwargs': kwargs}
166 'params': {'args': args, 'kwargs': kwargs}
166 }
167 }
167 return _remote_call(self._url, payload, EXCEPTIONS_MAP, self._session)
168 return _remote_call(self._url, payload, EXCEPTIONS_MAP, self._session)
168
169
169 def _call_with_logging(self, name, *args, **kwargs):
170 def _call_with_logging(self, name, *args, **kwargs):
170 log.debug('Calling %s@%s', self._url, name)
171 log.debug('Calling %s@%s', self._url, name)
171 return RemoteObject._call(self, name, *args, **kwargs)
172 return RemoteObject._call(self, name, *args, **kwargs)
172
173
173
174
174 def _remote_call(url, payload, exceptions_map, session):
175 def _remote_call(url, payload, exceptions_map, session):
176 try:
175 response = session.post(url, data=msgpack.packb(payload))
177 response = session.post(url, data=msgpack.packb(payload))
178 except pycurl.error as e:
179 raise exceptions.HttpVCSCommunicationError(e)
180
176 response = msgpack.unpackb(response.content)
181 response = msgpack.unpackb(response.content)
177 error = response.get('error')
182 error = response.get('error')
178 if error:
183 if error:
179 type_ = error.get('type', 'Exception')
184 type_ = error.get('type', 'Exception')
180 exc = exceptions_map.get(type_, Exception)
185 exc = exceptions_map.get(type_, Exception)
181 exc = exc(error.get('message'))
186 exc = exc(error.get('message'))
182 try:
187 try:
183 exc._vcs_kind = error['_vcs_kind']
188 exc._vcs_kind = error['_vcs_kind']
184 except KeyError:
189 except KeyError:
185 pass
190 pass
186 raise exc
191 raise exc
187 return response.get('result')
192 return response.get('result')
188
193
189
194
190 class VcsHttpProxy(object):
195 class VcsHttpProxy(object):
191
196
192 CHUNK_SIZE = 16384
197 CHUNK_SIZE = 16384
193
198
194 def __init__(self, server_and_port, backend_endpoint):
199 def __init__(self, server_and_port, backend_endpoint):
195 adapter = requests.adapters.HTTPAdapter(max_retries=5)
200 adapter = requests.adapters.HTTPAdapter(max_retries=5)
196 self.base_url = urlparse.urljoin(
201 self.base_url = urlparse.urljoin(
197 'http://%s' % server_and_port, backend_endpoint)
202 'http://%s' % server_and_port, backend_endpoint)
198 self.session = requests.Session()
203 self.session = requests.Session()
199 self.session.mount('http://', adapter)
204 self.session.mount('http://', adapter)
200
205
201 def handle(self, environment, input_data, *args, **kwargs):
206 def handle(self, environment, input_data, *args, **kwargs):
202 data = {
207 data = {
203 'environment': environment,
208 'environment': environment,
204 'input_data': input_data,
209 'input_data': input_data,
205 'args': args,
210 'args': args,
206 'kwargs': kwargs
211 'kwargs': kwargs
207 }
212 }
208 result = self.session.post(
213 result = self.session.post(
209 self.base_url, msgpack.packb(data), stream=True)
214 self.base_url, msgpack.packb(data), stream=True)
210 return self._get_result(result)
215 return self._get_result(result)
211
216
212 def _deserialize_and_raise(self, error):
217 def _deserialize_and_raise(self, error):
213 exception = Exception(error['message'])
218 exception = Exception(error['message'])
214 try:
219 try:
215 exception._vcs_kind = error['_vcs_kind']
220 exception._vcs_kind = error['_vcs_kind']
216 except KeyError:
221 except KeyError:
217 pass
222 pass
218 raise exception
223 raise exception
219
224
220 def _iterate(self, result):
225 def _iterate(self, result):
221 unpacker = msgpack.Unpacker()
226 unpacker = msgpack.Unpacker()
222 for line in result.iter_content(chunk_size=self.CHUNK_SIZE):
227 for line in result.iter_content(chunk_size=self.CHUNK_SIZE):
223 unpacker.feed(line)
228 unpacker.feed(line)
224 for chunk in unpacker:
229 for chunk in unpacker:
225 yield chunk
230 yield chunk
226
231
227 def _get_result(self, result):
232 def _get_result(self, result):
228 iterator = self._iterate(result)
233 iterator = self._iterate(result)
229 error = iterator.next()
234 error = iterator.next()
230 if error:
235 if error:
231 self._deserialize_and_raise(error)
236 self._deserialize_and_raise(error)
232
237
233 status = iterator.next()
238 status = iterator.next()
234 headers = iterator.next()
239 headers = iterator.next()
235
240
236 return iterator, status, headers
241 return iterator, status, headers
237
242
238
243
239 class ThreadlocalSessionFactory(object):
244 class ThreadlocalSessionFactory(object):
240 """
245 """
241 Creates one CurlSession per thread on demand.
246 Creates one CurlSession per thread on demand.
242 """
247 """
243
248
244 def __init__(self):
249 def __init__(self):
245 self._thread_local = threading.local()
250 self._thread_local = threading.local()
246
251
247 def __call__(self):
252 def __call__(self):
248 if not hasattr(self._thread_local, 'curl_session'):
253 if not hasattr(self._thread_local, 'curl_session'):
249 self._thread_local.curl_session = CurlSession()
254 self._thread_local.curl_session = CurlSession()
250 return self._thread_local.curl_session
255 return self._thread_local.curl_session
@@ -1,186 +1,197 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2014-2016 RhodeCode GmbH
3 # Copyright (C) 2014-2016 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 """
21 """
22 Custom vcs exceptions module.
22 Custom vcs exceptions module.
23 """
23 """
24
24
25 import functools
25 import functools
26 import urllib2
26 import urllib2
27 import pycurl
28 from Pyro4.errors import CommunicationError
29
30 class VCSCommunicationError(Exception):
31 pass
32
33
34 class PyroVCSCommunicationError(VCSCommunicationError):
35 pass
36
37
38 class HttpVCSCommunicationError(VCSCommunicationError):
39 pass
27
40
28
41
29 class VCSError(Exception):
42 class VCSError(Exception):
30 pass
43 pass
31
44
32
45
33 class RepositoryError(VCSError):
46 class RepositoryError(VCSError):
34 pass
47 pass
35
48
36
49
37 class RepositoryRequirementError(RepositoryError):
50 class RepositoryRequirementError(RepositoryError):
38 pass
51 pass
39
52
40
53
41 class VCSBackendNotSupportedError(VCSError):
54 class VCSBackendNotSupportedError(VCSError):
42 """
55 """
43 Exception raised when VCSServer does not support requested backend
56 Exception raised when VCSServer does not support requested backend
44 """
57 """
45
58
46
59
47 class EmptyRepositoryError(RepositoryError):
60 class EmptyRepositoryError(RepositoryError):
48 pass
61 pass
49
62
50
63
51 class TagAlreadyExistError(RepositoryError):
64 class TagAlreadyExistError(RepositoryError):
52 pass
65 pass
53
66
54
67
55 class TagDoesNotExistError(RepositoryError):
68 class TagDoesNotExistError(RepositoryError):
56 pass
69 pass
57
70
58
71
59 class BranchAlreadyExistError(RepositoryError):
72 class BranchAlreadyExistError(RepositoryError):
60 pass
73 pass
61
74
62
75
63 class BranchDoesNotExistError(RepositoryError):
76 class BranchDoesNotExistError(RepositoryError):
64 pass
77 pass
65
78
66
79
67 class CommitError(RepositoryError):
80 class CommitError(RepositoryError):
68 """
81 """
69 Exceptions related to an existing commit
82 Exceptions related to an existing commit
70 """
83 """
71
84
72
85
73 class CommitDoesNotExistError(CommitError):
86 class CommitDoesNotExistError(CommitError):
74 pass
87 pass
75
88
76
89
77 class CommittingError(RepositoryError):
90 class CommittingError(RepositoryError):
78 """
91 """
79 Exceptions happening while creating a new commit
92 Exceptions happening while creating a new commit
80 """
93 """
81
94
82
95
83 class NothingChangedError(CommittingError):
96 class NothingChangedError(CommittingError):
84 pass
97 pass
85
98
86
99
87 class NodeError(VCSError):
100 class NodeError(VCSError):
88 pass
101 pass
89
102
90
103
91 class RemovedFileNodeError(NodeError):
104 class RemovedFileNodeError(NodeError):
92 pass
105 pass
93
106
94
107
95 class NodeAlreadyExistsError(CommittingError):
108 class NodeAlreadyExistsError(CommittingError):
96 pass
109 pass
97
110
98
111
99 class NodeAlreadyChangedError(CommittingError):
112 class NodeAlreadyChangedError(CommittingError):
100 pass
113 pass
101
114
102
115
103 class NodeDoesNotExistError(CommittingError):
116 class NodeDoesNotExistError(CommittingError):
104 pass
117 pass
105
118
106
119
107 class NodeNotChangedError(CommittingError):
120 class NodeNotChangedError(CommittingError):
108 pass
121 pass
109
122
110
123
111 class NodeAlreadyAddedError(CommittingError):
124 class NodeAlreadyAddedError(CommittingError):
112 pass
125 pass
113
126
114
127
115 class NodeAlreadyRemovedError(CommittingError):
128 class NodeAlreadyRemovedError(CommittingError):
116 pass
129 pass
117
130
118
131
119 class ImproperArchiveTypeError(VCSError):
132 class ImproperArchiveTypeError(VCSError):
120 pass
133 pass
121
134
122
135
123 class CommandError(VCSError):
136 class CommandError(VCSError):
124 pass
137 pass
125
138
126
139
127 class UnhandledException(VCSError):
140 class UnhandledException(VCSError):
128 """
141 """
129 Signals that something unexpected went wrong.
142 Signals that something unexpected went wrong.
130
143
131 This usually means we have a programming error on the side of the VCSServer
144 This usually means we have a programming error on the side of the VCSServer
132 and should inspect the logfile of the VCSServer to find more details.
145 and should inspect the logfile of the VCSServer to find more details.
133 """
146 """
134
147
135
148
136 _EXCEPTION_MAP = {
149 _EXCEPTION_MAP = {
137 'abort': RepositoryError,
150 'abort': RepositoryError,
138 'archive': ImproperArchiveTypeError,
151 'archive': ImproperArchiveTypeError,
139 'error': RepositoryError,
152 'error': RepositoryError,
140 'lookup': CommitDoesNotExistError,
153 'lookup': CommitDoesNotExistError,
141 'repo_locked': RepositoryError,
154 'repo_locked': RepositoryError,
142 'requirement': RepositoryRequirementError,
155 'requirement': RepositoryRequirementError,
143 'unhandled': UnhandledException,
156 'unhandled': UnhandledException,
144 # TODO: johbo: Define our own exception for this and stop abusing
157 # TODO: johbo: Define our own exception for this and stop abusing
145 # urllib's exception class.
158 # urllib's exception class.
146 'url_error': urllib2.URLError,
159 'url_error': urllib2.URLError,
147 }
160 }
148
161
149
162
150 def map_vcs_exceptions(func):
163 def map_vcs_exceptions(func):
151 """
164 """
152 Utility to decorate functions so that plain exceptions are translated.
165 Utility to decorate functions so that plain exceptions are translated.
153
166
154 The translation is based on `exc_map` which maps a `str` indicating
167 The translation is based on `exc_map` which maps a `str` indicating
155 the error type into an exception class representing this error inside
168 the error type into an exception class representing this error inside
156 of the vcs layer.
169 of the vcs layer.
157 """
170 """
158
171
159 @functools.wraps(func)
172 @functools.wraps(func)
160 def wrapper(*args, **kwargs):
173 def wrapper(*args, **kwargs):
161 try:
174 try:
162 return func(*args, **kwargs)
175 return func(*args, **kwargs)
163 except Exception as e:
176 except Exception as e:
164
165 # The error middleware adds information if it finds
177 # The error middleware adds information if it finds
166 # __traceback_info__ in a frame object. This way the remote
178 # __traceback_info__ in a frame object. This way the remote
167 # traceback information is made available in error reports.
179 # traceback information is made available in error reports.
168 remote_tb = getattr(e, '_pyroTraceback', None)
180 remote_tb = getattr(e, '_pyroTraceback', None)
169 if remote_tb:
181 if remote_tb:
170 __traceback_info__ = (
182 __traceback_info__ = (
171 'Found Pyro4 remote traceback information:\n\n' +
183 'Found Pyro4 remote traceback information:\n\n' +
172 '\n'.join(remote_tb))
184 '\n'.join(remote_tb))
173
185
174 # Avoid that remote_tb also appears in the frame
186 # Avoid that remote_tb also appears in the frame
175 del remote_tb
187 del remote_tb
176
188
177 # Special vcs errors had an attribute "_vcs_kind" which is used
189 # Special vcs errors had an attribute "_vcs_kind" which is used
178 # to translate them to the proper exception class in the vcs
190 # to translate them to the proper exception class in the vcs
179 # client layer.
191 # client layer.
180 kind = getattr(e, '_vcs_kind', None)
192 kind = getattr(e, '_vcs_kind', None)
181 if kind:
193 if kind:
182 raise _EXCEPTION_MAP[kind](*e.args)
194 raise _EXCEPTION_MAP[kind](*e.args)
183 else:
195 else:
184 raise
196 raise
185
186 return wrapper
197 return wrapper
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now