Show More
@@ -1,576 +1,576 b'' | |||
|
1 | 1 | # -*- coding: utf-8 -*- |
|
2 | 2 | |
|
3 | 3 | # Copyright (C) 2011-2020 RhodeCode GmbH |
|
4 | 4 | # |
|
5 | 5 | # This program is free software: you can redistribute it and/or modify |
|
6 | 6 | # it under the terms of the GNU Affero General Public License, version 3 |
|
7 | 7 | # (only), as published by the Free Software Foundation. |
|
8 | 8 | # |
|
9 | 9 | # This program is distributed in the hope that it will be useful, |
|
10 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
11 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
12 | 12 | # GNU General Public License for more details. |
|
13 | 13 | # |
|
14 | 14 | # You should have received a copy of the GNU Affero General Public License |
|
15 | 15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
16 | 16 | # |
|
17 | 17 | # This program is dual-licensed. If you wish to learn more about the |
|
18 | 18 | # RhodeCode Enterprise Edition, including its added features, Support services, |
|
19 | 19 | # and proprietary license terms, please see https://rhodecode.com/licenses/ |
|
20 | 20 | |
|
21 | 21 | import itertools |
|
22 | 22 | import logging |
|
23 | 23 | import sys |
|
24 | 24 | import types |
|
25 | 25 | import fnmatch |
|
26 | 26 | |
|
27 | 27 | import decorator |
|
28 | 28 | import venusian |
|
29 | 29 | from collections import OrderedDict |
|
30 | 30 | |
|
31 | 31 | from pyramid.exceptions import ConfigurationError |
|
32 | 32 | from pyramid.renderers import render |
|
33 | 33 | from pyramid.response import Response |
|
34 | 34 | from pyramid.httpexceptions import HTTPNotFound |
|
35 | 35 | |
|
36 | 36 | from rhodecode.api.exc import ( |
|
37 | 37 | JSONRPCBaseError, JSONRPCError, JSONRPCForbidden, JSONRPCValidationError) |
|
38 | 38 | from rhodecode.apps._base import TemplateArgs |
|
39 | 39 | from rhodecode.lib.auth import AuthUser |
|
40 | 40 | from rhodecode.lib.base import get_ip_addr, attach_context_attributes |
|
41 | 41 | from rhodecode.lib.exc_tracking import store_exception |
|
42 | 42 | from rhodecode.lib.ext_json import json |
|
43 | 43 | from rhodecode.lib.utils2 import safe_str |
|
44 | 44 | from rhodecode.lib.plugins.utils import get_plugin_settings |
|
45 | 45 | from rhodecode.model.db import User, UserApiKeys |
|
46 | 46 | |
|
47 | 47 | log = logging.getLogger(__name__) |
|
48 | 48 | |
|
49 | 49 | DEFAULT_RENDERER = 'jsonrpc_renderer' |
|
50 | 50 | DEFAULT_URL = '/_admin/apiv2' |
|
51 | 51 | |
|
52 | 52 | |
|
53 | 53 | def find_methods(jsonrpc_methods, pattern): |
|
54 | 54 | matches = OrderedDict() |
|
55 | 55 | if not isinstance(pattern, (list, tuple)): |
|
56 | 56 | pattern = [pattern] |
|
57 | 57 | |
|
58 | 58 | for single_pattern in pattern: |
|
59 | 59 | for method_name, method in jsonrpc_methods.items(): |
|
60 | 60 | if fnmatch.fnmatch(method_name, single_pattern): |
|
61 | 61 | matches[method_name] = method |
|
62 | 62 | return matches |
|
63 | 63 | |
|
64 | 64 | |
|
65 | 65 | class ExtJsonRenderer(object): |
|
66 | 66 | """ |
|
67 | 67 | Custom renderer that mkaes use of our ext_json lib |
|
68 | 68 | |
|
69 | 69 | """ |
|
70 | 70 | |
|
71 | 71 | def __init__(self, serializer=json.dumps, **kw): |
|
72 | 72 | """ Any keyword arguments will be passed to the ``serializer`` |
|
73 | 73 | function.""" |
|
74 | 74 | self.serializer = serializer |
|
75 | 75 | self.kw = kw |
|
76 | 76 | |
|
77 | 77 | def __call__(self, info): |
|
78 | 78 | """ Returns a plain JSON-encoded string with content-type |
|
79 | 79 | ``application/json``. The content-type may be overridden by |
|
80 | 80 | setting ``request.response.content_type``.""" |
|
81 | 81 | |
|
82 | 82 | def _render(value, system): |
|
83 | 83 | request = system.get('request') |
|
84 | 84 | if request is not None: |
|
85 | 85 | response = request.response |
|
86 | 86 | ct = response.content_type |
|
87 | 87 | if ct == response.default_content_type: |
|
88 | 88 | response.content_type = 'application/json' |
|
89 | 89 | |
|
90 | 90 | return self.serializer(value, **self.kw) |
|
91 | 91 | |
|
92 | 92 | return _render |
|
93 | 93 | |
|
94 | 94 | |
|
95 | 95 | def jsonrpc_response(request, result): |
|
96 | 96 | rpc_id = getattr(request, 'rpc_id', None) |
|
97 | 97 | response = request.response |
|
98 | 98 | |
|
99 | 99 | # store content_type before render is called |
|
100 | 100 | ct = response.content_type |
|
101 | 101 | |
|
102 | 102 | ret_value = '' |
|
103 | 103 | if rpc_id: |
|
104 | 104 | ret_value = { |
|
105 | 105 | 'id': rpc_id, |
|
106 | 106 | 'result': result, |
|
107 | 107 | 'error': None, |
|
108 | 108 | } |
|
109 | 109 | |
|
110 | 110 | # fetch deprecation warnings, and store it inside results |
|
111 | 111 | deprecation = getattr(request, 'rpc_deprecation', None) |
|
112 | 112 | if deprecation: |
|
113 | 113 | ret_value['DEPRECATION_WARNING'] = deprecation |
|
114 | 114 | |
|
115 | 115 | raw_body = render(DEFAULT_RENDERER, ret_value, request=request) |
|
116 | 116 | response.body = safe_str(raw_body, response.charset) |
|
117 | 117 | |
|
118 | 118 | if ct == response.default_content_type: |
|
119 | 119 | response.content_type = 'application/json' |
|
120 | 120 | |
|
121 | 121 | return response |
|
122 | 122 | |
|
123 | 123 | |
|
124 | 124 | def jsonrpc_error(request, message, retid=None, code=None, headers=None): |
|
125 | 125 | """ |
|
126 | 126 | Generate a Response object with a JSON-RPC error body |
|
127 | 127 | |
|
128 | 128 | :param code: |
|
129 | 129 | :param retid: |
|
130 | 130 | :param message: |
|
131 | 131 | """ |
|
132 | 132 | err_dict = {'id': retid, 'result': None, 'error': message} |
|
133 | 133 | body = render(DEFAULT_RENDERER, err_dict, request=request).encode('utf-8') |
|
134 | 134 | |
|
135 | 135 | return Response( |
|
136 | 136 | body=body, |
|
137 | 137 | status=code, |
|
138 | 138 | content_type='application/json', |
|
139 | 139 | headerlist=headers |
|
140 | 140 | ) |
|
141 | 141 | |
|
142 | 142 | |
|
143 | 143 | def exception_view(exc, request): |
|
144 | 144 | rpc_id = getattr(request, 'rpc_id', None) |
|
145 | 145 | |
|
146 | 146 | if isinstance(exc, JSONRPCError): |
|
147 | 147 | fault_message = safe_str(exc.message) |
|
148 | 148 | log.debug('json-rpc error rpc_id:%s "%s"', rpc_id, fault_message) |
|
149 | 149 | elif isinstance(exc, JSONRPCValidationError): |
|
150 | 150 | colander_exc = exc.colander_exception |
|
151 | 151 | # TODO(marcink): think maybe of nicer way to serialize errors ? |
|
152 | 152 | fault_message = colander_exc.asdict() |
|
153 | 153 | log.debug('json-rpc colander error rpc_id:%s "%s"', rpc_id, fault_message) |
|
154 | 154 | elif isinstance(exc, JSONRPCForbidden): |
|
155 | 155 | fault_message = 'Access was denied to this resource.' |
|
156 | 156 | log.warning('json-rpc forbidden call rpc_id:%s "%s"', rpc_id, fault_message) |
|
157 | 157 | elif isinstance(exc, HTTPNotFound): |
|
158 | 158 | method = request.rpc_method |
|
159 | 159 | log.debug('json-rpc method `%s` not found in list of ' |
|
160 | 160 | 'api calls: %s, rpc_id:%s', |
|
161 | 161 | method, request.registry.jsonrpc_methods.keys(), rpc_id) |
|
162 | 162 | |
|
163 | 163 | similar = 'none' |
|
164 | 164 | try: |
|
165 | 165 | similar_paterns = ['*{}*'.format(x) for x in method.split('_')] |
|
166 | 166 | similar_found = find_methods( |
|
167 | 167 | request.registry.jsonrpc_methods, similar_paterns) |
|
168 | 168 | similar = ', '.join(similar_found.keys()) or similar |
|
169 | 169 | except Exception: |
|
170 | 170 | # make the whole above block safe |
|
171 | 171 | pass |
|
172 | 172 | |
|
173 | 173 | fault_message = "No such method: {}. Similar methods: {}".format( |
|
174 | 174 | method, similar) |
|
175 | 175 | else: |
|
176 | 176 | fault_message = 'undefined error' |
|
177 | 177 | exc_info = exc.exc_info() |
|
178 | 178 | store_exception(id(exc_info), exc_info, prefix='rhodecode-api') |
|
179 | 179 | |
|
180 | 180 | statsd = request.registry.statsd |
|
181 | 181 | if statsd: |
|
182 | statsd.incr('rhodecode_exception_total', tags=["exc_source:api"]) | |
|
182 | statsd.incr('rhodecode_exception_total', tags=["exc_source:api", "type:{}".format(exc_info.type)]) | |
|
183 | 183 | |
|
184 | 184 | return jsonrpc_error(request, fault_message, rpc_id) |
|
185 | 185 | |
|
186 | 186 | |
|
187 | 187 | def request_view(request): |
|
188 | 188 | """ |
|
189 | 189 | Main request handling method. It handles all logic to call a specific |
|
190 | 190 | exposed method |
|
191 | 191 | """ |
|
192 | 192 | # cython compatible inspect |
|
193 | 193 | from rhodecode.config.patches import inspect_getargspec |
|
194 | 194 | inspect = inspect_getargspec() |
|
195 | 195 | |
|
196 | 196 | # check if we can find this session using api_key, get_by_auth_token |
|
197 | 197 | # search not expired tokens only |
|
198 | 198 | try: |
|
199 | 199 | api_user = User.get_by_auth_token(request.rpc_api_key) |
|
200 | 200 | |
|
201 | 201 | if api_user is None: |
|
202 | 202 | return jsonrpc_error( |
|
203 | 203 | request, retid=request.rpc_id, message='Invalid API KEY') |
|
204 | 204 | |
|
205 | 205 | if not api_user.active: |
|
206 | 206 | return jsonrpc_error( |
|
207 | 207 | request, retid=request.rpc_id, |
|
208 | 208 | message='Request from this user not allowed') |
|
209 | 209 | |
|
210 | 210 | # check if we are allowed to use this IP |
|
211 | 211 | auth_u = AuthUser( |
|
212 | 212 | api_user.user_id, request.rpc_api_key, ip_addr=request.rpc_ip_addr) |
|
213 | 213 | if not auth_u.ip_allowed: |
|
214 | 214 | return jsonrpc_error( |
|
215 | 215 | request, retid=request.rpc_id, |
|
216 | 216 | message='Request from IP:%s not allowed' % ( |
|
217 | 217 | request.rpc_ip_addr,)) |
|
218 | 218 | else: |
|
219 | 219 | log.info('Access for IP:%s allowed', request.rpc_ip_addr) |
|
220 | 220 | |
|
221 | 221 | # register our auth-user |
|
222 | 222 | request.rpc_user = auth_u |
|
223 | 223 | request.environ['rc_auth_user_id'] = auth_u.user_id |
|
224 | 224 | |
|
225 | 225 | # now check if token is valid for API |
|
226 | 226 | auth_token = request.rpc_api_key |
|
227 | 227 | token_match = api_user.authenticate_by_token( |
|
228 | 228 | auth_token, roles=[UserApiKeys.ROLE_API]) |
|
229 | 229 | invalid_token = not token_match |
|
230 | 230 | |
|
231 | 231 | log.debug('Checking if API KEY is valid with proper role') |
|
232 | 232 | if invalid_token: |
|
233 | 233 | return jsonrpc_error( |
|
234 | 234 | request, retid=request.rpc_id, |
|
235 | 235 | message='API KEY invalid or, has bad role for an API call') |
|
236 | 236 | |
|
237 | 237 | except Exception: |
|
238 | 238 | log.exception('Error on API AUTH') |
|
239 | 239 | return jsonrpc_error( |
|
240 | 240 | request, retid=request.rpc_id, message='Invalid API KEY') |
|
241 | 241 | |
|
242 | 242 | method = request.rpc_method |
|
243 | 243 | func = request.registry.jsonrpc_methods[method] |
|
244 | 244 | |
|
245 | 245 | # now that we have a method, add request._req_params to |
|
246 | 246 | # self.kargs and dispatch control to WGIController |
|
247 | 247 | argspec = inspect.getargspec(func) |
|
248 | 248 | arglist = argspec[0] |
|
249 | 249 | defaults = map(type, argspec[3] or []) |
|
250 | 250 | default_empty = types.NotImplementedType |
|
251 | 251 | |
|
252 | 252 | # kw arguments required by this method |
|
253 | 253 | func_kwargs = dict(itertools.izip_longest( |
|
254 | 254 | reversed(arglist), reversed(defaults), fillvalue=default_empty)) |
|
255 | 255 | |
|
256 | 256 | # This attribute will need to be first param of a method that uses |
|
257 | 257 | # api_key, which is translated to instance of user at that name |
|
258 | 258 | user_var = 'apiuser' |
|
259 | 259 | request_var = 'request' |
|
260 | 260 | |
|
261 | 261 | for arg in [user_var, request_var]: |
|
262 | 262 | if arg not in arglist: |
|
263 | 263 | return jsonrpc_error( |
|
264 | 264 | request, |
|
265 | 265 | retid=request.rpc_id, |
|
266 | 266 | message='This method [%s] does not support ' |
|
267 | 267 | 'required parameter `%s`' % (func.__name__, arg)) |
|
268 | 268 | |
|
269 | 269 | # get our arglist and check if we provided them as args |
|
270 | 270 | for arg, default in func_kwargs.items(): |
|
271 | 271 | if arg in [user_var, request_var]: |
|
272 | 272 | # user_var and request_var are pre-hardcoded parameters and we |
|
273 | 273 | # don't need to do any translation |
|
274 | 274 | continue |
|
275 | 275 | |
|
276 | 276 | # skip the required param check if it's default value is |
|
277 | 277 | # NotImplementedType (default_empty) |
|
278 | 278 | if default == default_empty and arg not in request.rpc_params: |
|
279 | 279 | return jsonrpc_error( |
|
280 | 280 | request, |
|
281 | 281 | retid=request.rpc_id, |
|
282 | 282 | message=('Missing non optional `%s` arg in JSON DATA' % arg) |
|
283 | 283 | ) |
|
284 | 284 | |
|
285 | 285 | # sanitize extra passed arguments |
|
286 | 286 | for k in request.rpc_params.keys()[:]: |
|
287 | 287 | if k not in func_kwargs: |
|
288 | 288 | del request.rpc_params[k] |
|
289 | 289 | |
|
290 | 290 | call_params = request.rpc_params |
|
291 | 291 | call_params.update({ |
|
292 | 292 | 'request': request, |
|
293 | 293 | 'apiuser': auth_u |
|
294 | 294 | }) |
|
295 | 295 | |
|
296 | 296 | # register some common functions for usage |
|
297 | 297 | attach_context_attributes(TemplateArgs(), request, request.rpc_user.user_id) |
|
298 | 298 | |
|
299 | 299 | statsd = request.registry.statsd |
|
300 | 300 | |
|
301 | 301 | try: |
|
302 | 302 | ret_value = func(**call_params) |
|
303 | 303 | resp = jsonrpc_response(request, ret_value) |
|
304 | 304 | if statsd: |
|
305 | 305 | statsd.incr('rhodecode_api_call_success_total') |
|
306 | 306 | return resp |
|
307 | 307 | except JSONRPCBaseError: |
|
308 | 308 | raise |
|
309 | 309 | except Exception: |
|
310 | 310 | log.exception('Unhandled exception occurred on api call: %s', func) |
|
311 | 311 | exc_info = sys.exc_info() |
|
312 | 312 | exc_id, exc_type_name = store_exception( |
|
313 | 313 | id(exc_info), exc_info, prefix='rhodecode-api') |
|
314 | 314 | error_headers = [('RhodeCode-Exception-Id', str(exc_id)), |
|
315 | 315 | ('RhodeCode-Exception-Type', str(exc_type_name))] |
|
316 | 316 | err_resp = jsonrpc_error( |
|
317 | 317 | request, retid=request.rpc_id, message='Internal server error', |
|
318 | 318 | headers=error_headers) |
|
319 | 319 | if statsd: |
|
320 | 320 | statsd.incr('rhodecode_api_call_fail_total') |
|
321 | 321 | return err_resp |
|
322 | 322 | |
|
323 | 323 | |
|
324 | 324 | def setup_request(request): |
|
325 | 325 | """ |
|
326 | 326 | Parse a JSON-RPC request body. It's used inside the predicates method |
|
327 | 327 | to validate and bootstrap requests for usage in rpc calls. |
|
328 | 328 | |
|
329 | 329 | We need to raise JSONRPCError here if we want to return some errors back to |
|
330 | 330 | user. |
|
331 | 331 | """ |
|
332 | 332 | |
|
333 | 333 | log.debug('Executing setup request: %r', request) |
|
334 | 334 | request.rpc_ip_addr = get_ip_addr(request.environ) |
|
335 | 335 | # TODO(marcink): deprecate GET at some point |
|
336 | 336 | if request.method not in ['POST', 'GET']: |
|
337 | 337 | log.debug('unsupported request method "%s"', request.method) |
|
338 | 338 | raise JSONRPCError( |
|
339 | 339 | 'unsupported request method "%s". Please use POST' % request.method) |
|
340 | 340 | |
|
341 | 341 | if 'CONTENT_LENGTH' not in request.environ: |
|
342 | 342 | log.debug("No Content-Length") |
|
343 | 343 | raise JSONRPCError("Empty body, No Content-Length in request") |
|
344 | 344 | |
|
345 | 345 | else: |
|
346 | 346 | length = request.environ['CONTENT_LENGTH'] |
|
347 | 347 | log.debug('Content-Length: %s', length) |
|
348 | 348 | |
|
349 | 349 | if length == 0: |
|
350 | 350 | log.debug("Content-Length is 0") |
|
351 | 351 | raise JSONRPCError("Content-Length is 0") |
|
352 | 352 | |
|
353 | 353 | raw_body = request.body |
|
354 | 354 | log.debug("Loading JSON body now") |
|
355 | 355 | try: |
|
356 | 356 | json_body = json.loads(raw_body) |
|
357 | 357 | except ValueError as e: |
|
358 | 358 | # catch JSON errors Here |
|
359 | 359 | raise JSONRPCError("JSON parse error ERR:%s RAW:%r" % (e, raw_body)) |
|
360 | 360 | |
|
361 | 361 | request.rpc_id = json_body.get('id') |
|
362 | 362 | request.rpc_method = json_body.get('method') |
|
363 | 363 | |
|
364 | 364 | # check required base parameters |
|
365 | 365 | try: |
|
366 | 366 | api_key = json_body.get('api_key') |
|
367 | 367 | if not api_key: |
|
368 | 368 | api_key = json_body.get('auth_token') |
|
369 | 369 | |
|
370 | 370 | if not api_key: |
|
371 | 371 | raise KeyError('api_key or auth_token') |
|
372 | 372 | |
|
373 | 373 | # TODO(marcink): support passing in token in request header |
|
374 | 374 | |
|
375 | 375 | request.rpc_api_key = api_key |
|
376 | 376 | request.rpc_id = json_body['id'] |
|
377 | 377 | request.rpc_method = json_body['method'] |
|
378 | 378 | request.rpc_params = json_body['args'] \ |
|
379 | 379 | if isinstance(json_body['args'], dict) else {} |
|
380 | 380 | |
|
381 | 381 | log.debug('method: %s, params: %.10240r', request.rpc_method, request.rpc_params) |
|
382 | 382 | except KeyError as e: |
|
383 | 383 | raise JSONRPCError('Incorrect JSON data. Missing %s' % e) |
|
384 | 384 | |
|
385 | 385 | log.debug('setup complete, now handling method:%s rpcid:%s', |
|
386 | 386 | request.rpc_method, request.rpc_id, ) |
|
387 | 387 | |
|
388 | 388 | |
|
389 | 389 | class RoutePredicate(object): |
|
390 | 390 | def __init__(self, val, config): |
|
391 | 391 | self.val = val |
|
392 | 392 | |
|
393 | 393 | def text(self): |
|
394 | 394 | return 'jsonrpc route = %s' % self.val |
|
395 | 395 | |
|
396 | 396 | phash = text |
|
397 | 397 | |
|
398 | 398 | def __call__(self, info, request): |
|
399 | 399 | if self.val: |
|
400 | 400 | # potentially setup and bootstrap our call |
|
401 | 401 | setup_request(request) |
|
402 | 402 | |
|
403 | 403 | # Always return True so that even if it isn't a valid RPC it |
|
404 | 404 | # will fall through to the underlaying handlers like notfound_view |
|
405 | 405 | return True |
|
406 | 406 | |
|
407 | 407 | |
|
408 | 408 | class NotFoundPredicate(object): |
|
409 | 409 | def __init__(self, val, config): |
|
410 | 410 | self.val = val |
|
411 | 411 | self.methods = config.registry.jsonrpc_methods |
|
412 | 412 | |
|
413 | 413 | def text(self): |
|
414 | 414 | return 'jsonrpc method not found = {}.'.format(self.val) |
|
415 | 415 | |
|
416 | 416 | phash = text |
|
417 | 417 | |
|
418 | 418 | def __call__(self, info, request): |
|
419 | 419 | return hasattr(request, 'rpc_method') |
|
420 | 420 | |
|
421 | 421 | |
|
422 | 422 | class MethodPredicate(object): |
|
423 | 423 | def __init__(self, val, config): |
|
424 | 424 | self.method = val |
|
425 | 425 | |
|
426 | 426 | def text(self): |
|
427 | 427 | return 'jsonrpc method = %s' % self.method |
|
428 | 428 | |
|
429 | 429 | phash = text |
|
430 | 430 | |
|
431 | 431 | def __call__(self, context, request): |
|
432 | 432 | # we need to explicitly return False here, so pyramid doesn't try to |
|
433 | 433 | # execute our view directly. We need our main handler to execute things |
|
434 | 434 | return getattr(request, 'rpc_method') == self.method |
|
435 | 435 | |
|
436 | 436 | |
|
437 | 437 | def add_jsonrpc_method(config, view, **kwargs): |
|
438 | 438 | # pop the method name |
|
439 | 439 | method = kwargs.pop('method', None) |
|
440 | 440 | |
|
441 | 441 | if method is None: |
|
442 | 442 | raise ConfigurationError( |
|
443 | 443 | 'Cannot register a JSON-RPC method without specifying the "method"') |
|
444 | 444 | |
|
445 | 445 | # we define custom predicate, to enable to detect conflicting methods, |
|
446 | 446 | # those predicates are kind of "translation" from the decorator variables |
|
447 | 447 | # to internal predicates names |
|
448 | 448 | |
|
449 | 449 | kwargs['jsonrpc_method'] = method |
|
450 | 450 | |
|
451 | 451 | # register our view into global view store for validation |
|
452 | 452 | config.registry.jsonrpc_methods[method] = view |
|
453 | 453 | |
|
454 | 454 | # we're using our main request_view handler, here, so each method |
|
455 | 455 | # has a unified handler for itself |
|
456 | 456 | config.add_view(request_view, route_name='apiv2', **kwargs) |
|
457 | 457 | |
|
458 | 458 | |
|
459 | 459 | class jsonrpc_method(object): |
|
460 | 460 | """ |
|
461 | 461 | decorator that works similar to @add_view_config decorator, |
|
462 | 462 | but tailored for our JSON RPC |
|
463 | 463 | """ |
|
464 | 464 | |
|
465 | 465 | venusian = venusian # for testing injection |
|
466 | 466 | |
|
467 | 467 | def __init__(self, method=None, **kwargs): |
|
468 | 468 | self.method = method |
|
469 | 469 | self.kwargs = kwargs |
|
470 | 470 | |
|
471 | 471 | def __call__(self, wrapped): |
|
472 | 472 | kwargs = self.kwargs.copy() |
|
473 | 473 | kwargs['method'] = self.method or wrapped.__name__ |
|
474 | 474 | depth = kwargs.pop('_depth', 0) |
|
475 | 475 | |
|
476 | 476 | def callback(context, name, ob): |
|
477 | 477 | config = context.config.with_package(info.module) |
|
478 | 478 | config.add_jsonrpc_method(view=ob, **kwargs) |
|
479 | 479 | |
|
480 | 480 | info = venusian.attach(wrapped, callback, category='pyramid', |
|
481 | 481 | depth=depth + 1) |
|
482 | 482 | if info.scope == 'class': |
|
483 | 483 | # ensure that attr is set if decorating a class method |
|
484 | 484 | kwargs.setdefault('attr', wrapped.__name__) |
|
485 | 485 | |
|
486 | 486 | kwargs['_info'] = info.codeinfo # fbo action_method |
|
487 | 487 | return wrapped |
|
488 | 488 | |
|
489 | 489 | |
|
490 | 490 | class jsonrpc_deprecated_method(object): |
|
491 | 491 | """ |
|
492 | 492 | Marks method as deprecated, adds log.warning, and inject special key to |
|
493 | 493 | the request variable to mark method as deprecated. |
|
494 | 494 | Also injects special docstring that extract_docs will catch to mark |
|
495 | 495 | method as deprecated. |
|
496 | 496 | |
|
497 | 497 | :param use_method: specify which method should be used instead of |
|
498 | 498 | the decorated one |
|
499 | 499 | |
|
500 | 500 | Use like:: |
|
501 | 501 | |
|
502 | 502 | @jsonrpc_method() |
|
503 | 503 | @jsonrpc_deprecated_method(use_method='new_func', deprecated_at_version='3.0.0') |
|
504 | 504 | def old_func(request, apiuser, arg1, arg2): |
|
505 | 505 | ... |
|
506 | 506 | """ |
|
507 | 507 | |
|
508 | 508 | def __init__(self, use_method, deprecated_at_version): |
|
509 | 509 | self.use_method = use_method |
|
510 | 510 | self.deprecated_at_version = deprecated_at_version |
|
511 | 511 | self.deprecated_msg = '' |
|
512 | 512 | |
|
513 | 513 | def __call__(self, func): |
|
514 | 514 | self.deprecated_msg = 'Please use method `{method}` instead.'.format( |
|
515 | 515 | method=self.use_method) |
|
516 | 516 | |
|
517 | 517 | docstring = """\n |
|
518 | 518 | .. deprecated:: {version} |
|
519 | 519 | |
|
520 | 520 | {deprecation_message} |
|
521 | 521 | |
|
522 | 522 | {original_docstring} |
|
523 | 523 | """ |
|
524 | 524 | func.__doc__ = docstring.format( |
|
525 | 525 | version=self.deprecated_at_version, |
|
526 | 526 | deprecation_message=self.deprecated_msg, |
|
527 | 527 | original_docstring=func.__doc__) |
|
528 | 528 | return decorator.decorator(self.__wrapper, func) |
|
529 | 529 | |
|
530 | 530 | def __wrapper(self, func, *fargs, **fkwargs): |
|
531 | 531 | log.warning('DEPRECATED API CALL on function %s, please ' |
|
532 | 532 | 'use `%s` instead', func, self.use_method) |
|
533 | 533 | # alter function docstring to mark as deprecated, this is picked up |
|
534 | 534 | # via fabric file that generates API DOC. |
|
535 | 535 | result = func(*fargs, **fkwargs) |
|
536 | 536 | |
|
537 | 537 | request = fargs[0] |
|
538 | 538 | request.rpc_deprecation = 'DEPRECATED METHOD ' + self.deprecated_msg |
|
539 | 539 | return result |
|
540 | 540 | |
|
541 | 541 | |
|
542 | 542 | def add_api_methods(config): |
|
543 | 543 | from rhodecode.api.views import ( |
|
544 | 544 | deprecated_api, gist_api, pull_request_api, repo_api, repo_group_api, |
|
545 | 545 | server_api, search_api, testing_api, user_api, user_group_api) |
|
546 | 546 | |
|
547 | 547 | config.scan('rhodecode.api.views') |
|
548 | 548 | |
|
549 | 549 | |
|
550 | 550 | def includeme(config): |
|
551 | 551 | plugin_module = 'rhodecode.api' |
|
552 | 552 | plugin_settings = get_plugin_settings( |
|
553 | 553 | plugin_module, config.registry.settings) |
|
554 | 554 | |
|
555 | 555 | if not hasattr(config.registry, 'jsonrpc_methods'): |
|
556 | 556 | config.registry.jsonrpc_methods = OrderedDict() |
|
557 | 557 | |
|
558 | 558 | # match filter by given method only |
|
559 | 559 | config.add_view_predicate('jsonrpc_method', MethodPredicate) |
|
560 | 560 | config.add_view_predicate('jsonrpc_method_not_found', NotFoundPredicate) |
|
561 | 561 | |
|
562 | 562 | config.add_renderer(DEFAULT_RENDERER, ExtJsonRenderer( |
|
563 | 563 | serializer=json.dumps, indent=4)) |
|
564 | 564 | config.add_directive('add_jsonrpc_method', add_jsonrpc_method) |
|
565 | 565 | |
|
566 | 566 | config.add_route_predicate( |
|
567 | 567 | 'jsonrpc_call', RoutePredicate) |
|
568 | 568 | |
|
569 | 569 | config.add_route( |
|
570 | 570 | 'apiv2', plugin_settings.get('url', DEFAULT_URL), jsonrpc_call=True) |
|
571 | 571 | |
|
572 | 572 | # register some exception handling view |
|
573 | 573 | config.add_view(exception_view, context=JSONRPCBaseError) |
|
574 | 574 | config.add_notfound_view(exception_view, jsonrpc_method_not_found=True) |
|
575 | 575 | |
|
576 | 576 | add_api_methods(config) |
@@ -1,796 +1,797 b'' | |||
|
1 | 1 | # -*- coding: utf-8 -*- |
|
2 | 2 | |
|
3 | 3 | # Copyright (C) 2010-2020 RhodeCode GmbH |
|
4 | 4 | # |
|
5 | 5 | # This program is free software: you can redistribute it and/or modify |
|
6 | 6 | # it under the terms of the GNU Affero General Public License, version 3 |
|
7 | 7 | # (only), as published by the Free Software Foundation. |
|
8 | 8 | # |
|
9 | 9 | # This program is distributed in the hope that it will be useful, |
|
10 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
11 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
12 | 12 | # GNU General Public License for more details. |
|
13 | 13 | # |
|
14 | 14 | # You should have received a copy of the GNU Affero General Public License |
|
15 | 15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
16 | 16 | # |
|
17 | 17 | # This program is dual-licensed. If you wish to learn more about the |
|
18 | 18 | # RhodeCode Enterprise Edition, including its added features, Support services, |
|
19 | 19 | # and proprietary license terms, please see https://rhodecode.com/licenses/ |
|
20 | 20 | |
|
21 | 21 | import os |
|
22 | 22 | import sys |
|
23 | 23 | import logging |
|
24 | 24 | import collections |
|
25 | 25 | import tempfile |
|
26 | 26 | import time |
|
27 | 27 | |
|
28 | 28 | from paste.gzipper import make_gzip_middleware |
|
29 | 29 | import pyramid.events |
|
30 | 30 | from pyramid.wsgi import wsgiapp |
|
31 | 31 | from pyramid.authorization import ACLAuthorizationPolicy |
|
32 | 32 | from pyramid.config import Configurator |
|
33 | 33 | from pyramid.settings import asbool, aslist |
|
34 | 34 | from pyramid.httpexceptions import ( |
|
35 | 35 | HTTPException, HTTPError, HTTPInternalServerError, HTTPFound, HTTPNotFound) |
|
36 | 36 | from pyramid.renderers import render_to_response |
|
37 | 37 | |
|
38 | 38 | from rhodecode.model import meta |
|
39 | 39 | from rhodecode.config import patches |
|
40 | 40 | from rhodecode.config import utils as config_utils |
|
41 | 41 | from rhodecode.config.environment import load_pyramid_environment |
|
42 | 42 | |
|
43 | 43 | import rhodecode.events |
|
44 | 44 | from rhodecode.lib.middleware.vcs import VCSMiddleware |
|
45 | 45 | from rhodecode.lib.request import Request |
|
46 | 46 | from rhodecode.lib.vcs import VCSCommunicationError |
|
47 | 47 | from rhodecode.lib.exceptions import VCSServerUnavailable |
|
48 | 48 | from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled |
|
49 | 49 | from rhodecode.lib.middleware.https_fixup import HttpsFixup |
|
50 | 50 | from rhodecode.lib.plugins.utils import register_rhodecode_plugin |
|
51 | 51 | from rhodecode.lib.utils2 import aslist as rhodecode_aslist, AttributeDict |
|
52 | 52 | from rhodecode.lib.exc_tracking import store_exception |
|
53 | 53 | from rhodecode.subscribers import ( |
|
54 | 54 | scan_repositories_if_enabled, write_js_routes_if_enabled, |
|
55 | 55 | write_metadata_if_needed, write_usage_data) |
|
56 | 56 | from rhodecode.lib.statsd_client import StatsdClient |
|
57 | 57 | |
|
58 | 58 | log = logging.getLogger(__name__) |
|
59 | 59 | |
|
60 | 60 | |
|
61 | 61 | def is_http_error(response): |
|
62 | 62 | # error which should have traceback |
|
63 | 63 | return response.status_code > 499 |
|
64 | 64 | |
|
65 | 65 | |
|
66 | 66 | def should_load_all(): |
|
67 | 67 | """ |
|
68 | 68 | Returns if all application components should be loaded. In some cases it's |
|
69 | 69 | desired to skip apps loading for faster shell script execution |
|
70 | 70 | """ |
|
71 | 71 | ssh_cmd = os.environ.get('RC_CMD_SSH_WRAPPER') |
|
72 | 72 | if ssh_cmd: |
|
73 | 73 | return False |
|
74 | 74 | |
|
75 | 75 | return True |
|
76 | 76 | |
|
77 | 77 | |
|
78 | 78 | def make_pyramid_app(global_config, **settings): |
|
79 | 79 | """ |
|
80 | 80 | Constructs the WSGI application based on Pyramid. |
|
81 | 81 | |
|
82 | 82 | Specials: |
|
83 | 83 | |
|
84 | 84 | * The application can also be integrated like a plugin via the call to |
|
85 | 85 | `includeme`. This is accompanied with the other utility functions which |
|
86 | 86 | are called. Changing this should be done with great care to not break |
|
87 | 87 | cases when these fragments are assembled from another place. |
|
88 | 88 | |
|
89 | 89 | """ |
|
90 | 90 | |
|
91 | 91 | # Allows to use format style "{ENV_NAME}" placeholders in the configuration. It |
|
92 | 92 | # will be replaced by the value of the environment variable "NAME" in this case. |
|
93 | 93 | start_time = time.time() |
|
94 | 94 | log.info('Pyramid app config starting') |
|
95 | 95 | |
|
96 | 96 | # init and bootstrap StatsdClient |
|
97 | 97 | StatsdClient.setup(settings) |
|
98 | 98 | |
|
99 | 99 | debug = asbool(global_config.get('debug')) |
|
100 | 100 | if debug: |
|
101 | 101 | enable_debug() |
|
102 | 102 | |
|
103 | 103 | environ = {'ENV_{}'.format(key): value for key, value in os.environ.items()} |
|
104 | 104 | |
|
105 | 105 | global_config = _substitute_values(global_config, environ) |
|
106 | 106 | settings = _substitute_values(settings, environ) |
|
107 | 107 | |
|
108 | 108 | sanitize_settings_and_apply_defaults(global_config, settings) |
|
109 | 109 | |
|
110 | 110 | config = Configurator(settings=settings) |
|
111 | 111 | # Init our statsd at very start |
|
112 | 112 | config.registry.statsd = StatsdClient.statsd |
|
113 | 113 | |
|
114 | 114 | # Apply compatibility patches |
|
115 | 115 | patches.inspect_getargspec() |
|
116 | 116 | |
|
117 | 117 | load_pyramid_environment(global_config, settings) |
|
118 | 118 | |
|
119 | 119 | # Static file view comes first |
|
120 | 120 | includeme_first(config) |
|
121 | 121 | |
|
122 | 122 | includeme(config) |
|
123 | 123 | |
|
124 | 124 | pyramid_app = config.make_wsgi_app() |
|
125 | 125 | pyramid_app = wrap_app_in_wsgi_middlewares(pyramid_app, config) |
|
126 | 126 | pyramid_app.config = config |
|
127 | 127 | |
|
128 | 128 | config.configure_celery(global_config['__file__']) |
|
129 | 129 | |
|
130 | 130 | # creating the app uses a connection - return it after we are done |
|
131 | 131 | meta.Session.remove() |
|
132 | 132 | statsd = StatsdClient.statsd |
|
133 | 133 | |
|
134 | 134 | total_time = time.time() - start_time |
|
135 | 135 | log.info('Pyramid app `%s` created and configured in %.2fs', |
|
136 | 136 | pyramid_app.func_name, total_time) |
|
137 | 137 | if statsd: |
|
138 | 138 | elapsed_time_ms = 1000.0 * total_time |
|
139 | 139 | statsd.timing('rhodecode_app_bootstrap_timing', elapsed_time_ms, tags=[ |
|
140 | 140 | "pyramid_app:{}".format(pyramid_app.func_name) |
|
141 | 141 | ]) |
|
142 | 142 | return pyramid_app |
|
143 | 143 | |
|
144 | 144 | |
|
145 | 145 | def not_found_view(request): |
|
146 | 146 | """ |
|
147 | 147 | This creates the view which should be registered as not-found-view to |
|
148 | 148 | pyramid. |
|
149 | 149 | """ |
|
150 | 150 | |
|
151 | 151 | if not getattr(request, 'vcs_call', None): |
|
152 | 152 | # handle like regular case with our error_handler |
|
153 | 153 | return error_handler(HTTPNotFound(), request) |
|
154 | 154 | |
|
155 | 155 | # handle not found view as a vcs call |
|
156 | 156 | settings = request.registry.settings |
|
157 | 157 | ae_client = getattr(request, 'ae_client', None) |
|
158 | 158 | vcs_app = VCSMiddleware( |
|
159 | 159 | HTTPNotFound(), request.registry, settings, |
|
160 | 160 | appenlight_client=ae_client) |
|
161 | 161 | |
|
162 | 162 | return wsgiapp(vcs_app)(None, request) |
|
163 | 163 | |
|
164 | 164 | |
|
165 | 165 | def error_handler(exception, request): |
|
166 | 166 | import rhodecode |
|
167 | 167 | from rhodecode.lib import helpers |
|
168 | 168 | from rhodecode.lib.utils2 import str2bool |
|
169 | 169 | |
|
170 | 170 | rhodecode_title = rhodecode.CONFIG.get('rhodecode_title') or 'RhodeCode' |
|
171 | 171 | |
|
172 | 172 | base_response = HTTPInternalServerError() |
|
173 | 173 | # prefer original exception for the response since it may have headers set |
|
174 | 174 | if isinstance(exception, HTTPException): |
|
175 | 175 | base_response = exception |
|
176 | 176 | elif isinstance(exception, VCSCommunicationError): |
|
177 | 177 | base_response = VCSServerUnavailable() |
|
178 | 178 | |
|
179 | 179 | if is_http_error(base_response): |
|
180 | 180 | log.exception( |
|
181 | 181 | 'error occurred handling this request for path: %s', request.path) |
|
182 | 182 | |
|
183 | 183 | statsd = request.registry.statsd |
|
184 | 184 | if statsd and base_response.status_code > 499: |
|
185 |
statsd.incr('rhodecode_exception_total', |
|
|
185 | statsd.incr('rhodecode_exception_total', | |
|
186 | tags=["exc_source:web", "type:{}".format(base_response.status_code)]) | |
|
186 | 187 | |
|
187 | 188 | error_explanation = base_response.explanation or str(base_response) |
|
188 | 189 | if base_response.status_code == 404: |
|
189 | 190 | error_explanation += " Optionally you don't have permission to access this page." |
|
190 | 191 | c = AttributeDict() |
|
191 | 192 | c.error_message = base_response.status |
|
192 | 193 | c.error_explanation = error_explanation |
|
193 | 194 | c.visual = AttributeDict() |
|
194 | 195 | |
|
195 | 196 | c.visual.rhodecode_support_url = ( |
|
196 | 197 | request.registry.settings.get('rhodecode_support_url') or |
|
197 | 198 | request.route_url('rhodecode_support') |
|
198 | 199 | ) |
|
199 | 200 | c.redirect_time = 0 |
|
200 | 201 | c.rhodecode_name = rhodecode_title |
|
201 | 202 | if not c.rhodecode_name: |
|
202 | 203 | c.rhodecode_name = 'Rhodecode' |
|
203 | 204 | |
|
204 | 205 | c.causes = [] |
|
205 | 206 | if is_http_error(base_response): |
|
206 | 207 | c.causes.append('Server is overloaded.') |
|
207 | 208 | c.causes.append('Server database connection is lost.') |
|
208 | 209 | c.causes.append('Server expected unhandled error.') |
|
209 | 210 | |
|
210 | 211 | if hasattr(base_response, 'causes'): |
|
211 | 212 | c.causes = base_response.causes |
|
212 | 213 | |
|
213 | 214 | c.messages = helpers.flash.pop_messages(request=request) |
|
214 | 215 | |
|
215 | 216 | exc_info = sys.exc_info() |
|
216 | 217 | c.exception_id = id(exc_info) |
|
217 | 218 | c.show_exception_id = isinstance(base_response, VCSServerUnavailable) \ |
|
218 | 219 | or base_response.status_code > 499 |
|
219 | 220 | c.exception_id_url = request.route_url( |
|
220 | 221 | 'admin_settings_exception_tracker_show', exception_id=c.exception_id) |
|
221 | 222 | |
|
222 | 223 | if c.show_exception_id: |
|
223 | 224 | store_exception(c.exception_id, exc_info) |
|
224 | 225 | c.exception_debug = str2bool(rhodecode.CONFIG.get('debug')) |
|
225 | 226 | c.exception_config_ini = rhodecode.CONFIG.get('__file__') |
|
226 | 227 | |
|
227 | 228 | response = render_to_response( |
|
228 | 229 | '/errors/error_document.mako', {'c': c, 'h': helpers}, request=request, |
|
229 | 230 | response=base_response) |
|
230 | 231 | |
|
231 | 232 | return response |
|
232 | 233 | |
|
233 | 234 | |
|
234 | 235 | def includeme_first(config): |
|
235 | 236 | # redirect automatic browser favicon.ico requests to correct place |
|
236 | 237 | def favicon_redirect(context, request): |
|
237 | 238 | return HTTPFound( |
|
238 | 239 | request.static_path('rhodecode:public/images/favicon.ico')) |
|
239 | 240 | |
|
240 | 241 | config.add_view(favicon_redirect, route_name='favicon') |
|
241 | 242 | config.add_route('favicon', '/favicon.ico') |
|
242 | 243 | |
|
243 | 244 | def robots_redirect(context, request): |
|
244 | 245 | return HTTPFound( |
|
245 | 246 | request.static_path('rhodecode:public/robots.txt')) |
|
246 | 247 | |
|
247 | 248 | config.add_view(robots_redirect, route_name='robots') |
|
248 | 249 | config.add_route('robots', '/robots.txt') |
|
249 | 250 | |
|
250 | 251 | config.add_static_view( |
|
251 | 252 | '_static/deform', 'deform:static') |
|
252 | 253 | config.add_static_view( |
|
253 | 254 | '_static/rhodecode', path='rhodecode:public', cache_max_age=3600 * 24) |
|
254 | 255 | |
|
255 | 256 | |
|
256 | 257 | def includeme(config, auth_resources=None): |
|
257 | 258 | from rhodecode.lib.celerylib.loader import configure_celery |
|
258 | 259 | log.debug('Initializing main includeme from %s', os.path.basename(__file__)) |
|
259 | 260 | settings = config.registry.settings |
|
260 | 261 | config.set_request_factory(Request) |
|
261 | 262 | |
|
262 | 263 | # plugin information |
|
263 | 264 | config.registry.rhodecode_plugins = collections.OrderedDict() |
|
264 | 265 | |
|
265 | 266 | config.add_directive( |
|
266 | 267 | 'register_rhodecode_plugin', register_rhodecode_plugin) |
|
267 | 268 | |
|
268 | 269 | config.add_directive('configure_celery', configure_celery) |
|
269 | 270 | |
|
270 | 271 | if asbool(settings.get('appenlight', 'false')): |
|
271 | 272 | config.include('appenlight_client.ext.pyramid_tween') |
|
272 | 273 | |
|
273 | 274 | load_all = should_load_all() |
|
274 | 275 | |
|
275 | 276 | # Includes which are required. The application would fail without them. |
|
276 | 277 | config.include('pyramid_mako') |
|
277 | 278 | config.include('rhodecode.lib.rc_beaker') |
|
278 | 279 | config.include('rhodecode.lib.rc_cache') |
|
279 | 280 | config.include('rhodecode.apps._base.navigation') |
|
280 | 281 | config.include('rhodecode.apps._base.subscribers') |
|
281 | 282 | config.include('rhodecode.tweens') |
|
282 | 283 | config.include('rhodecode.authentication') |
|
283 | 284 | |
|
284 | 285 | if load_all: |
|
285 | 286 | ce_auth_resources = [ |
|
286 | 287 | 'rhodecode.authentication.plugins.auth_crowd', |
|
287 | 288 | 'rhodecode.authentication.plugins.auth_headers', |
|
288 | 289 | 'rhodecode.authentication.plugins.auth_jasig_cas', |
|
289 | 290 | 'rhodecode.authentication.plugins.auth_ldap', |
|
290 | 291 | 'rhodecode.authentication.plugins.auth_pam', |
|
291 | 292 | 'rhodecode.authentication.plugins.auth_rhodecode', |
|
292 | 293 | 'rhodecode.authentication.plugins.auth_token', |
|
293 | 294 | ] |
|
294 | 295 | |
|
295 | 296 | # load CE authentication plugins |
|
296 | 297 | |
|
297 | 298 | if auth_resources: |
|
298 | 299 | ce_auth_resources.extend(auth_resources) |
|
299 | 300 | |
|
300 | 301 | for resource in ce_auth_resources: |
|
301 | 302 | config.include(resource) |
|
302 | 303 | |
|
303 | 304 | # Auto discover authentication plugins and include their configuration. |
|
304 | 305 | if asbool(settings.get('auth_plugin.import_legacy_plugins', 'true')): |
|
305 | 306 | from rhodecode.authentication import discover_legacy_plugins |
|
306 | 307 | discover_legacy_plugins(config) |
|
307 | 308 | |
|
308 | 309 | # apps |
|
309 | 310 | if load_all: |
|
310 | 311 | config.include('rhodecode.api') |
|
311 | 312 | config.include('rhodecode.apps._base') |
|
312 | 313 | config.include('rhodecode.apps.hovercards') |
|
313 | 314 | config.include('rhodecode.apps.ops') |
|
314 | 315 | config.include('rhodecode.apps.channelstream') |
|
315 | 316 | config.include('rhodecode.apps.file_store') |
|
316 | 317 | config.include('rhodecode.apps.admin') |
|
317 | 318 | config.include('rhodecode.apps.login') |
|
318 | 319 | config.include('rhodecode.apps.home') |
|
319 | 320 | config.include('rhodecode.apps.journal') |
|
320 | 321 | |
|
321 | 322 | config.include('rhodecode.apps.repository') |
|
322 | 323 | config.include('rhodecode.apps.repo_group') |
|
323 | 324 | config.include('rhodecode.apps.user_group') |
|
324 | 325 | config.include('rhodecode.apps.search') |
|
325 | 326 | config.include('rhodecode.apps.user_profile') |
|
326 | 327 | config.include('rhodecode.apps.user_group_profile') |
|
327 | 328 | config.include('rhodecode.apps.my_account') |
|
328 | 329 | config.include('rhodecode.apps.gist') |
|
329 | 330 | |
|
330 | 331 | config.include('rhodecode.apps.svn_support') |
|
331 | 332 | config.include('rhodecode.apps.ssh_support') |
|
332 | 333 | config.include('rhodecode.apps.debug_style') |
|
333 | 334 | |
|
334 | 335 | if load_all: |
|
335 | 336 | config.include('rhodecode.integrations') |
|
336 | 337 | |
|
337 | 338 | config.add_route('rhodecode_support', 'https://rhodecode.com/help/', static=True) |
|
338 | 339 | config.add_translation_dirs('rhodecode:i18n/') |
|
339 | 340 | settings['default_locale_name'] = settings.get('lang', 'en') |
|
340 | 341 | |
|
341 | 342 | # Add subscribers. |
|
342 | 343 | if load_all: |
|
343 | 344 | config.add_subscriber(scan_repositories_if_enabled, |
|
344 | 345 | pyramid.events.ApplicationCreated) |
|
345 | 346 | config.add_subscriber(write_metadata_if_needed, |
|
346 | 347 | pyramid.events.ApplicationCreated) |
|
347 | 348 | config.add_subscriber(write_usage_data, |
|
348 | 349 | pyramid.events.ApplicationCreated) |
|
349 | 350 | config.add_subscriber(write_js_routes_if_enabled, |
|
350 | 351 | pyramid.events.ApplicationCreated) |
|
351 | 352 | |
|
352 | 353 | # request custom methods |
|
353 | 354 | config.add_request_method( |
|
354 | 355 | 'rhodecode.lib.partial_renderer.get_partial_renderer', |
|
355 | 356 | 'get_partial_renderer') |
|
356 | 357 | |
|
357 | 358 | config.add_request_method( |
|
358 | 359 | 'rhodecode.lib.request_counter.get_request_counter', |
|
359 | 360 | 'request_count') |
|
360 | 361 | |
|
361 | 362 | # Set the authorization policy. |
|
362 | 363 | authz_policy = ACLAuthorizationPolicy() |
|
363 | 364 | config.set_authorization_policy(authz_policy) |
|
364 | 365 | |
|
365 | 366 | # Set the default renderer for HTML templates to mako. |
|
366 | 367 | config.add_mako_renderer('.html') |
|
367 | 368 | |
|
368 | 369 | config.add_renderer( |
|
369 | 370 | name='json_ext', |
|
370 | 371 | factory='rhodecode.lib.ext_json_renderer.pyramid_ext_json') |
|
371 | 372 | |
|
372 | 373 | config.add_renderer( |
|
373 | 374 | name='string_html', |
|
374 | 375 | factory='rhodecode.lib.string_renderer.html') |
|
375 | 376 | |
|
376 | 377 | # include RhodeCode plugins |
|
377 | 378 | includes = aslist(settings.get('rhodecode.includes', [])) |
|
378 | 379 | for inc in includes: |
|
379 | 380 | config.include(inc) |
|
380 | 381 | |
|
381 | 382 | # custom not found view, if our pyramid app doesn't know how to handle |
|
382 | 383 | # the request pass it to potential VCS handling ap |
|
383 | 384 | config.add_notfound_view(not_found_view) |
|
384 | 385 | if not settings.get('debugtoolbar.enabled', False): |
|
385 | 386 | # disabled debugtoolbar handle all exceptions via the error_handlers |
|
386 | 387 | config.add_view(error_handler, context=Exception) |
|
387 | 388 | |
|
388 | 389 | # all errors including 403/404/50X |
|
389 | 390 | config.add_view(error_handler, context=HTTPError) |
|
390 | 391 | |
|
391 | 392 | |
|
392 | 393 | def wrap_app_in_wsgi_middlewares(pyramid_app, config): |
|
393 | 394 | """ |
|
394 | 395 | Apply outer WSGI middlewares around the application. |
|
395 | 396 | """ |
|
396 | 397 | registry = config.registry |
|
397 | 398 | settings = registry.settings |
|
398 | 399 | |
|
399 | 400 | # enable https redirects based on HTTP_X_URL_SCHEME set by proxy |
|
400 | 401 | pyramid_app = HttpsFixup(pyramid_app, settings) |
|
401 | 402 | |
|
402 | 403 | pyramid_app, _ae_client = wrap_in_appenlight_if_enabled( |
|
403 | 404 | pyramid_app, settings) |
|
404 | 405 | registry.ae_client = _ae_client |
|
405 | 406 | |
|
406 | 407 | if settings['gzip_responses']: |
|
407 | 408 | pyramid_app = make_gzip_middleware( |
|
408 | 409 | pyramid_app, settings, compress_level=1) |
|
409 | 410 | |
|
410 | 411 | # this should be the outer most middleware in the wsgi stack since |
|
411 | 412 | # middleware like Routes make database calls |
|
412 | 413 | def pyramid_app_with_cleanup(environ, start_response): |
|
413 | 414 | try: |
|
414 | 415 | return pyramid_app(environ, start_response) |
|
415 | 416 | finally: |
|
416 | 417 | # Dispose current database session and rollback uncommitted |
|
417 | 418 | # transactions. |
|
418 | 419 | meta.Session.remove() |
|
419 | 420 | |
|
420 | 421 | # In a single threaded mode server, on non sqlite db we should have |
|
421 | 422 | # '0 Current Checked out connections' at the end of a request, |
|
422 | 423 | # if not, then something, somewhere is leaving a connection open |
|
423 | 424 | pool = meta.Base.metadata.bind.engine.pool |
|
424 | 425 | log.debug('sa pool status: %s', pool.status()) |
|
425 | 426 | log.debug('Request processing finalized') |
|
426 | 427 | |
|
427 | 428 | return pyramid_app_with_cleanup |
|
428 | 429 | |
|
429 | 430 | |
|
430 | 431 | def sanitize_settings_and_apply_defaults(global_config, settings): |
|
431 | 432 | """ |
|
432 | 433 | Applies settings defaults and does all type conversion. |
|
433 | 434 | |
|
434 | 435 | We would move all settings parsing and preparation into this place, so that |
|
435 | 436 | we have only one place left which deals with this part. The remaining parts |
|
436 | 437 | of the application would start to rely fully on well prepared settings. |
|
437 | 438 | |
|
438 | 439 | This piece would later be split up per topic to avoid a big fat monster |
|
439 | 440 | function. |
|
440 | 441 | """ |
|
441 | 442 | |
|
442 | 443 | settings.setdefault('rhodecode.edition', 'Community Edition') |
|
443 | 444 | settings.setdefault('rhodecode.edition_id', 'CE') |
|
444 | 445 | |
|
445 | 446 | if 'mako.default_filters' not in settings: |
|
446 | 447 | # set custom default filters if we don't have it defined |
|
447 | 448 | settings['mako.imports'] = 'from rhodecode.lib.base import h_filter' |
|
448 | 449 | settings['mako.default_filters'] = 'h_filter' |
|
449 | 450 | |
|
450 | 451 | if 'mako.directories' not in settings: |
|
451 | 452 | mako_directories = settings.setdefault('mako.directories', [ |
|
452 | 453 | # Base templates of the original application |
|
453 | 454 | 'rhodecode:templates', |
|
454 | 455 | ]) |
|
455 | 456 | log.debug( |
|
456 | 457 | "Using the following Mako template directories: %s", |
|
457 | 458 | mako_directories) |
|
458 | 459 | |
|
459 | 460 | # NOTE(marcink): fix redis requirement for schema of connection since 3.X |
|
460 | 461 | if 'beaker.session.type' in settings and settings['beaker.session.type'] == 'ext:redis': |
|
461 | 462 | raw_url = settings['beaker.session.url'] |
|
462 | 463 | if not raw_url.startswith(('redis://', 'rediss://', 'unix://')): |
|
463 | 464 | settings['beaker.session.url'] = 'redis://' + raw_url |
|
464 | 465 | |
|
465 | 466 | # Default includes, possible to change as a user |
|
466 | 467 | pyramid_includes = settings.setdefault('pyramid.includes', []) |
|
467 | 468 | log.debug( |
|
468 | 469 | "Using the following pyramid.includes: %s", |
|
469 | 470 | pyramid_includes) |
|
470 | 471 | |
|
471 | 472 | # TODO: johbo: Re-think this, usually the call to config.include |
|
472 | 473 | # should allow to pass in a prefix. |
|
473 | 474 | settings.setdefault('rhodecode.api.url', '/_admin/api') |
|
474 | 475 | settings.setdefault('__file__', global_config.get('__file__')) |
|
475 | 476 | |
|
476 | 477 | # Sanitize generic settings. |
|
477 | 478 | _list_setting(settings, 'default_encoding', 'UTF-8') |
|
478 | 479 | _bool_setting(settings, 'is_test', 'false') |
|
479 | 480 | _bool_setting(settings, 'gzip_responses', 'false') |
|
480 | 481 | |
|
481 | 482 | # Call split out functions that sanitize settings for each topic. |
|
482 | 483 | _sanitize_appenlight_settings(settings) |
|
483 | 484 | _sanitize_vcs_settings(settings) |
|
484 | 485 | _sanitize_cache_settings(settings) |
|
485 | 486 | |
|
486 | 487 | # configure instance id |
|
487 | 488 | config_utils.set_instance_id(settings) |
|
488 | 489 | |
|
489 | 490 | return settings |
|
490 | 491 | |
|
491 | 492 | |
|
492 | 493 | def enable_debug(): |
|
493 | 494 | """ |
|
494 | 495 | Helper to enable debug on running instance |
|
495 | 496 | :return: |
|
496 | 497 | """ |
|
497 | 498 | import tempfile |
|
498 | 499 | import textwrap |
|
499 | 500 | import logging.config |
|
500 | 501 | |
|
501 | 502 | ini_template = textwrap.dedent(""" |
|
502 | 503 | ##################################### |
|
503 | 504 | ### DEBUG LOGGING CONFIGURATION #### |
|
504 | 505 | ##################################### |
|
505 | 506 | [loggers] |
|
506 | 507 | keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper |
|
507 | 508 | |
|
508 | 509 | [handlers] |
|
509 | 510 | keys = console, console_sql |
|
510 | 511 | |
|
511 | 512 | [formatters] |
|
512 | 513 | keys = generic, color_formatter, color_formatter_sql |
|
513 | 514 | |
|
514 | 515 | ############# |
|
515 | 516 | ## LOGGERS ## |
|
516 | 517 | ############# |
|
517 | 518 | [logger_root] |
|
518 | 519 | level = NOTSET |
|
519 | 520 | handlers = console |
|
520 | 521 | |
|
521 | 522 | [logger_sqlalchemy] |
|
522 | 523 | level = INFO |
|
523 | 524 | handlers = console_sql |
|
524 | 525 | qualname = sqlalchemy.engine |
|
525 | 526 | propagate = 0 |
|
526 | 527 | |
|
527 | 528 | [logger_beaker] |
|
528 | 529 | level = DEBUG |
|
529 | 530 | handlers = |
|
530 | 531 | qualname = beaker.container |
|
531 | 532 | propagate = 1 |
|
532 | 533 | |
|
533 | 534 | [logger_rhodecode] |
|
534 | 535 | level = DEBUG |
|
535 | 536 | handlers = |
|
536 | 537 | qualname = rhodecode |
|
537 | 538 | propagate = 1 |
|
538 | 539 | |
|
539 | 540 | [logger_ssh_wrapper] |
|
540 | 541 | level = DEBUG |
|
541 | 542 | handlers = |
|
542 | 543 | qualname = ssh_wrapper |
|
543 | 544 | propagate = 1 |
|
544 | 545 | |
|
545 | 546 | [logger_celery] |
|
546 | 547 | level = DEBUG |
|
547 | 548 | handlers = |
|
548 | 549 | qualname = celery |
|
549 | 550 | |
|
550 | 551 | |
|
551 | 552 | ############## |
|
552 | 553 | ## HANDLERS ## |
|
553 | 554 | ############## |
|
554 | 555 | |
|
555 | 556 | [handler_console] |
|
556 | 557 | class = StreamHandler |
|
557 | 558 | args = (sys.stderr, ) |
|
558 | 559 | level = DEBUG |
|
559 | 560 | formatter = color_formatter |
|
560 | 561 | |
|
561 | 562 | [handler_console_sql] |
|
562 | 563 | # "level = DEBUG" logs SQL queries and results. |
|
563 | 564 | # "level = INFO" logs SQL queries. |
|
564 | 565 | # "level = WARN" logs neither. (Recommended for production systems.) |
|
565 | 566 | class = StreamHandler |
|
566 | 567 | args = (sys.stderr, ) |
|
567 | 568 | level = WARN |
|
568 | 569 | formatter = color_formatter_sql |
|
569 | 570 | |
|
570 | 571 | ################ |
|
571 | 572 | ## FORMATTERS ## |
|
572 | 573 | ################ |
|
573 | 574 | |
|
574 | 575 | [formatter_generic] |
|
575 | 576 | class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter |
|
576 | 577 | format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s | %(req_id)s |
|
577 | 578 | datefmt = %Y-%m-%d %H:%M:%S |
|
578 | 579 | |
|
579 | 580 | [formatter_color_formatter] |
|
580 | 581 | class = rhodecode.lib.logging_formatter.ColorRequestTrackingFormatter |
|
581 | 582 | format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s | %(req_id)s |
|
582 | 583 | datefmt = %Y-%m-%d %H:%M:%S |
|
583 | 584 | |
|
584 | 585 | [formatter_color_formatter_sql] |
|
585 | 586 | class = rhodecode.lib.logging_formatter.ColorFormatterSql |
|
586 | 587 | format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s |
|
587 | 588 | datefmt = %Y-%m-%d %H:%M:%S |
|
588 | 589 | """) |
|
589 | 590 | |
|
590 | 591 | with tempfile.NamedTemporaryFile(prefix='rc_debug_logging_', suffix='.ini', |
|
591 | 592 | delete=False) as f: |
|
592 | 593 | log.info('Saved Temporary DEBUG config at %s', f.name) |
|
593 | 594 | f.write(ini_template) |
|
594 | 595 | |
|
595 | 596 | logging.config.fileConfig(f.name) |
|
596 | 597 | log.debug('DEBUG MODE ON') |
|
597 | 598 | os.remove(f.name) |
|
598 | 599 | |
|
599 | 600 | |
|
600 | 601 | def _sanitize_appenlight_settings(settings): |
|
601 | 602 | _bool_setting(settings, 'appenlight', 'false') |
|
602 | 603 | |
|
603 | 604 | |
|
604 | 605 | def _sanitize_vcs_settings(settings): |
|
605 | 606 | """ |
|
606 | 607 | Applies settings defaults and does type conversion for all VCS related |
|
607 | 608 | settings. |
|
608 | 609 | """ |
|
609 | 610 | _string_setting(settings, 'vcs.svn.compatible_version', '') |
|
610 | 611 | _string_setting(settings, 'vcs.hooks.protocol', 'http') |
|
611 | 612 | _string_setting(settings, 'vcs.hooks.host', '127.0.0.1') |
|
612 | 613 | _string_setting(settings, 'vcs.scm_app_implementation', 'http') |
|
613 | 614 | _string_setting(settings, 'vcs.server', '') |
|
614 | 615 | _string_setting(settings, 'vcs.server.protocol', 'http') |
|
615 | 616 | _bool_setting(settings, 'startup.import_repos', 'false') |
|
616 | 617 | _bool_setting(settings, 'vcs.hooks.direct_calls', 'false') |
|
617 | 618 | _bool_setting(settings, 'vcs.server.enable', 'true') |
|
618 | 619 | _bool_setting(settings, 'vcs.start_server', 'false') |
|
619 | 620 | _list_setting(settings, 'vcs.backends', 'hg, git, svn') |
|
620 | 621 | _int_setting(settings, 'vcs.connection_timeout', 3600) |
|
621 | 622 | |
|
622 | 623 | # Support legacy values of vcs.scm_app_implementation. Legacy |
|
623 | 624 | # configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or |
|
624 | 625 | # disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'. |
|
625 | 626 | scm_app_impl = settings['vcs.scm_app_implementation'] |
|
626 | 627 | if scm_app_impl in ['rhodecode.lib.middleware.utils.scm_app_http', 'vcsserver.scm_app']: |
|
627 | 628 | settings['vcs.scm_app_implementation'] = 'http' |
|
628 | 629 | |
|
629 | 630 | |
|
630 | 631 | def _sanitize_cache_settings(settings): |
|
631 | 632 | temp_store = tempfile.gettempdir() |
|
632 | 633 | default_cache_dir = os.path.join(temp_store, 'rc_cache') |
|
633 | 634 | |
|
634 | 635 | # save default, cache dir, and use it for all backends later. |
|
635 | 636 | default_cache_dir = _string_setting( |
|
636 | 637 | settings, |
|
637 | 638 | 'cache_dir', |
|
638 | 639 | default_cache_dir, lower=False, default_when_empty=True) |
|
639 | 640 | |
|
640 | 641 | # ensure we have our dir created |
|
641 | 642 | if not os.path.isdir(default_cache_dir): |
|
642 | 643 | os.makedirs(default_cache_dir, mode=0o755) |
|
643 | 644 | |
|
644 | 645 | # exception store cache |
|
645 | 646 | _string_setting( |
|
646 | 647 | settings, |
|
647 | 648 | 'exception_tracker.store_path', |
|
648 | 649 | temp_store, lower=False, default_when_empty=True) |
|
649 | 650 | _bool_setting( |
|
650 | 651 | settings, |
|
651 | 652 | 'exception_tracker.send_email', |
|
652 | 653 | 'false') |
|
653 | 654 | _string_setting( |
|
654 | 655 | settings, |
|
655 | 656 | 'exception_tracker.email_prefix', |
|
656 | 657 | '[RHODECODE ERROR]', lower=False, default_when_empty=True) |
|
657 | 658 | |
|
658 | 659 | # cache_perms |
|
659 | 660 | _string_setting( |
|
660 | 661 | settings, |
|
661 | 662 | 'rc_cache.cache_perms.backend', |
|
662 | 663 | 'dogpile.cache.rc.file_namespace', lower=False) |
|
663 | 664 | _int_setting( |
|
664 | 665 | settings, |
|
665 | 666 | 'rc_cache.cache_perms.expiration_time', |
|
666 | 667 | 60) |
|
667 | 668 | _string_setting( |
|
668 | 669 | settings, |
|
669 | 670 | 'rc_cache.cache_perms.arguments.filename', |
|
670 | 671 | os.path.join(default_cache_dir, 'rc_cache_1'), lower=False) |
|
671 | 672 | |
|
672 | 673 | # cache_repo |
|
673 | 674 | _string_setting( |
|
674 | 675 | settings, |
|
675 | 676 | 'rc_cache.cache_repo.backend', |
|
676 | 677 | 'dogpile.cache.rc.file_namespace', lower=False) |
|
677 | 678 | _int_setting( |
|
678 | 679 | settings, |
|
679 | 680 | 'rc_cache.cache_repo.expiration_time', |
|
680 | 681 | 60) |
|
681 | 682 | _string_setting( |
|
682 | 683 | settings, |
|
683 | 684 | 'rc_cache.cache_repo.arguments.filename', |
|
684 | 685 | os.path.join(default_cache_dir, 'rc_cache_2'), lower=False) |
|
685 | 686 | |
|
686 | 687 | # cache_license |
|
687 | 688 | _string_setting( |
|
688 | 689 | settings, |
|
689 | 690 | 'rc_cache.cache_license.backend', |
|
690 | 691 | 'dogpile.cache.rc.file_namespace', lower=False) |
|
691 | 692 | _int_setting( |
|
692 | 693 | settings, |
|
693 | 694 | 'rc_cache.cache_license.expiration_time', |
|
694 | 695 | 5*60) |
|
695 | 696 | _string_setting( |
|
696 | 697 | settings, |
|
697 | 698 | 'rc_cache.cache_license.arguments.filename', |
|
698 | 699 | os.path.join(default_cache_dir, 'rc_cache_3'), lower=False) |
|
699 | 700 | |
|
700 | 701 | # cache_repo_longterm memory, 96H |
|
701 | 702 | _string_setting( |
|
702 | 703 | settings, |
|
703 | 704 | 'rc_cache.cache_repo_longterm.backend', |
|
704 | 705 | 'dogpile.cache.rc.memory_lru', lower=False) |
|
705 | 706 | _int_setting( |
|
706 | 707 | settings, |
|
707 | 708 | 'rc_cache.cache_repo_longterm.expiration_time', |
|
708 | 709 | 345600) |
|
709 | 710 | _int_setting( |
|
710 | 711 | settings, |
|
711 | 712 | 'rc_cache.cache_repo_longterm.max_size', |
|
712 | 713 | 10000) |
|
713 | 714 | |
|
714 | 715 | # sql_cache_short |
|
715 | 716 | _string_setting( |
|
716 | 717 | settings, |
|
717 | 718 | 'rc_cache.sql_cache_short.backend', |
|
718 | 719 | 'dogpile.cache.rc.memory_lru', lower=False) |
|
719 | 720 | _int_setting( |
|
720 | 721 | settings, |
|
721 | 722 | 'rc_cache.sql_cache_short.expiration_time', |
|
722 | 723 | 30) |
|
723 | 724 | _int_setting( |
|
724 | 725 | settings, |
|
725 | 726 | 'rc_cache.sql_cache_short.max_size', |
|
726 | 727 | 10000) |
|
727 | 728 | |
|
728 | 729 | |
|
729 | 730 | def _int_setting(settings, name, default): |
|
730 | 731 | settings[name] = int(settings.get(name, default)) |
|
731 | 732 | return settings[name] |
|
732 | 733 | |
|
733 | 734 | |
|
734 | 735 | def _bool_setting(settings, name, default): |
|
735 | 736 | input_val = settings.get(name, default) |
|
736 | 737 | if isinstance(input_val, unicode): |
|
737 | 738 | input_val = input_val.encode('utf8') |
|
738 | 739 | settings[name] = asbool(input_val) |
|
739 | 740 | return settings[name] |
|
740 | 741 | |
|
741 | 742 | |
|
742 | 743 | def _list_setting(settings, name, default): |
|
743 | 744 | raw_value = settings.get(name, default) |
|
744 | 745 | |
|
745 | 746 | old_separator = ',' |
|
746 | 747 | if old_separator in raw_value: |
|
747 | 748 | # If we get a comma separated list, pass it to our own function. |
|
748 | 749 | settings[name] = rhodecode_aslist(raw_value, sep=old_separator) |
|
749 | 750 | else: |
|
750 | 751 | # Otherwise we assume it uses pyramids space/newline separation. |
|
751 | 752 | settings[name] = aslist(raw_value) |
|
752 | 753 | return settings[name] |
|
753 | 754 | |
|
754 | 755 | |
|
755 | 756 | def _string_setting(settings, name, default, lower=True, default_when_empty=False): |
|
756 | 757 | value = settings.get(name, default) |
|
757 | 758 | |
|
758 | 759 | if default_when_empty and not value: |
|
759 | 760 | # use default value when value is empty |
|
760 | 761 | value = default |
|
761 | 762 | |
|
762 | 763 | if lower: |
|
763 | 764 | value = value.lower() |
|
764 | 765 | settings[name] = value |
|
765 | 766 | return settings[name] |
|
766 | 767 | |
|
767 | 768 | |
|
768 | 769 | def _substitute_values(mapping, substitutions): |
|
769 | 770 | result = {} |
|
770 | 771 | |
|
771 | 772 | try: |
|
772 | 773 | for key, value in mapping.items(): |
|
773 | 774 | # initialize without substitution first |
|
774 | 775 | result[key] = value |
|
775 | 776 | |
|
776 | 777 | # Note: Cannot use regular replacements, since they would clash |
|
777 | 778 | # with the implementation of ConfigParser. Using "format" instead. |
|
778 | 779 | try: |
|
779 | 780 | result[key] = value.format(**substitutions) |
|
780 | 781 | except KeyError as e: |
|
781 | 782 | env_var = '{}'.format(e.args[0]) |
|
782 | 783 | |
|
783 | 784 | msg = 'Failed to substitute: `{key}={{{var}}}` with environment entry. ' \ |
|
784 | 785 | 'Make sure your environment has {var} set, or remove this ' \ |
|
785 | 786 | 'variable from config file'.format(key=key, var=env_var) |
|
786 | 787 | |
|
787 | 788 | if env_var.startswith('ENV_'): |
|
788 | 789 | raise ValueError(msg) |
|
789 | 790 | else: |
|
790 | 791 | log.warning(msg) |
|
791 | 792 | |
|
792 | 793 | except ValueError as e: |
|
793 | 794 | log.warning('Failed to substitute ENV variable: %s', e) |
|
794 | 795 | result = mapping |
|
795 | 796 | |
|
796 | 797 | return result |
@@ -1,310 +1,310 b'' | |||
|
1 | 1 | # -*- coding: utf-8 -*- |
|
2 | 2 | |
|
3 | 3 | # Copyright (C) 2010-2020 RhodeCode GmbH |
|
4 | 4 | # |
|
5 | 5 | # This program is free software: you can redistribute it and/or modify |
|
6 | 6 | # it under the terms of the GNU Affero General Public License, version 3 |
|
7 | 7 | # (only), as published by the Free Software Foundation. |
|
8 | 8 | # |
|
9 | 9 | # This program is distributed in the hope that it will be useful, |
|
10 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
11 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
12 | 12 | # GNU General Public License for more details. |
|
13 | 13 | # |
|
14 | 14 | # You should have received a copy of the GNU Affero General Public License |
|
15 | 15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
16 | 16 | # |
|
17 | 17 | # This program is dual-licensed. If you wish to learn more about the |
|
18 | 18 | # RhodeCode Enterprise Edition, including its added features, Support services, |
|
19 | 19 | # and proprietary license terms, please see https://rhodecode.com/licenses/ |
|
20 | 20 | """ |
|
21 | 21 | Celery loader, run with:: |
|
22 | 22 | |
|
23 | 23 | celery worker \ |
|
24 | 24 | --beat \ |
|
25 | 25 | --app rhodecode.lib.celerylib.loader \ |
|
26 | 26 | --scheduler rhodecode.lib.celerylib.scheduler.RcScheduler \ |
|
27 | 27 | --loglevel DEBUG --ini=._dev/dev.ini |
|
28 | 28 | """ |
|
29 | 29 | import os |
|
30 | 30 | import logging |
|
31 | 31 | import importlib |
|
32 | 32 | |
|
33 | 33 | from celery import Celery |
|
34 | 34 | from celery import signals |
|
35 | 35 | from celery import Task |
|
36 | 36 | from celery import exceptions # pragma: no cover |
|
37 | 37 | from kombu.serialization import register |
|
38 | 38 | from pyramid.threadlocal import get_current_request |
|
39 | 39 | |
|
40 | 40 | import rhodecode |
|
41 | 41 | |
|
42 | 42 | from rhodecode.lib.auth import AuthUser |
|
43 | 43 | from rhodecode.lib.celerylib.utils import get_ini_config, parse_ini_vars, ping_db |
|
44 | 44 | from rhodecode.lib.ext_json import json |
|
45 | 45 | from rhodecode.lib.pyramid_utils import bootstrap, setup_logging, prepare_request |
|
46 | 46 | from rhodecode.lib.utils2 import str2bool |
|
47 | 47 | from rhodecode.model import meta |
|
48 | 48 | |
|
49 | 49 | |
|
50 | 50 | register('json_ext', json.dumps, json.loads, |
|
51 | 51 | content_type='application/x-json-ext', |
|
52 | 52 | content_encoding='utf-8') |
|
53 | 53 | |
|
54 | 54 | log = logging.getLogger('celery.rhodecode.loader') |
|
55 | 55 | |
|
56 | 56 | |
|
57 | 57 | def add_preload_arguments(parser): |
|
58 | 58 | parser.add_argument( |
|
59 | 59 | '--ini', default=None, |
|
60 | 60 | help='Path to ini configuration file.' |
|
61 | 61 | ) |
|
62 | 62 | parser.add_argument( |
|
63 | 63 | '--ini-var', default=None, |
|
64 | 64 | help='Comma separated list of key=value to pass to ini.' |
|
65 | 65 | ) |
|
66 | 66 | |
|
67 | 67 | |
|
68 | 68 | def get_logger(obj): |
|
69 | 69 | custom_log = logging.getLogger( |
|
70 | 70 | 'rhodecode.task.{}'.format(obj.__class__.__name__)) |
|
71 | 71 | |
|
72 | 72 | if rhodecode.CELERY_ENABLED: |
|
73 | 73 | try: |
|
74 | 74 | custom_log = obj.get_logger() |
|
75 | 75 | except Exception: |
|
76 | 76 | pass |
|
77 | 77 | |
|
78 | 78 | return custom_log |
|
79 | 79 | |
|
80 | 80 | |
|
81 | 81 | imports = ['rhodecode.lib.celerylib.tasks'] |
|
82 | 82 | |
|
83 | 83 | try: |
|
84 | 84 | # try if we have EE tasks available |
|
85 | 85 | importlib.import_module('rc_ee') |
|
86 | 86 | imports.append('rc_ee.lib.celerylib.tasks') |
|
87 | 87 | except ImportError: |
|
88 | 88 | pass |
|
89 | 89 | |
|
90 | 90 | |
|
91 | 91 | base_celery_config = { |
|
92 | 92 | 'result_backend': 'rpc://', |
|
93 | 93 | 'result_expires': 60 * 60 * 24, |
|
94 | 94 | 'result_persistent': True, |
|
95 | 95 | 'imports': imports, |
|
96 | 96 | 'worker_max_tasks_per_child': 100, |
|
97 | 97 | 'accept_content': ['json_ext'], |
|
98 | 98 | 'task_serializer': 'json_ext', |
|
99 | 99 | 'result_serializer': 'json_ext', |
|
100 | 100 | 'worker_hijack_root_logger': False, |
|
101 | 101 | 'database_table_names': { |
|
102 | 102 | 'task': 'beat_taskmeta', |
|
103 | 103 | 'group': 'beat_groupmeta', |
|
104 | 104 | } |
|
105 | 105 | } |
|
106 | 106 | # init main celery app |
|
107 | 107 | celery_app = Celery() |
|
108 | 108 | celery_app.user_options['preload'].add(add_preload_arguments) |
|
109 | 109 | ini_file_glob = None |
|
110 | 110 | |
|
111 | 111 | |
|
112 | 112 | @signals.setup_logging.connect |
|
113 | 113 | def setup_logging_callback(**kwargs): |
|
114 | 114 | setup_logging(ini_file_glob) |
|
115 | 115 | |
|
116 | 116 | |
|
117 | 117 | @signals.user_preload_options.connect |
|
118 | 118 | def on_preload_parsed(options, **kwargs): |
|
119 | 119 | ini_location = options['ini'] |
|
120 | 120 | ini_vars = options['ini_var'] |
|
121 | 121 | celery_app.conf['INI_PYRAMID'] = options['ini'] |
|
122 | 122 | |
|
123 | 123 | if ini_location is None: |
|
124 | 124 | print('You must provide the paste --ini argument') |
|
125 | 125 | exit(-1) |
|
126 | 126 | |
|
127 | 127 | options = None |
|
128 | 128 | if ini_vars is not None: |
|
129 | 129 | options = parse_ini_vars(ini_vars) |
|
130 | 130 | |
|
131 | 131 | global ini_file_glob |
|
132 | 132 | ini_file_glob = ini_location |
|
133 | 133 | |
|
134 | 134 | log.debug('Bootstrapping RhodeCode application...') |
|
135 | 135 | env = bootstrap(ini_location, options=options) |
|
136 | 136 | |
|
137 | 137 | setup_celery_app( |
|
138 | 138 | app=env['app'], root=env['root'], request=env['request'], |
|
139 | 139 | registry=env['registry'], closer=env['closer'], |
|
140 | 140 | ini_location=ini_location) |
|
141 | 141 | |
|
142 | 142 | # fix the global flag even if it's disabled via .ini file because this |
|
143 | 143 | # is a worker code that doesn't need this to be disabled. |
|
144 | 144 | rhodecode.CELERY_ENABLED = True |
|
145 | 145 | |
|
146 | 146 | |
|
147 | 147 | @signals.task_prerun.connect |
|
148 | 148 | def task_prerun_signal(task_id, task, args, **kwargs): |
|
149 | 149 | ping_db() |
|
150 | 150 | |
|
151 | 151 | |
|
152 | 152 | @signals.task_success.connect |
|
153 | 153 | def task_success_signal(result, **kwargs): |
|
154 | 154 | meta.Session.commit() |
|
155 | 155 | closer = celery_app.conf['PYRAMID_CLOSER'] |
|
156 | 156 | if closer: |
|
157 | 157 | closer() |
|
158 | 158 | |
|
159 | 159 | |
|
160 | 160 | @signals.task_retry.connect |
|
161 | 161 | def task_retry_signal( |
|
162 | 162 | request, reason, einfo, **kwargs): |
|
163 | 163 | meta.Session.remove() |
|
164 | 164 | closer = celery_app.conf['PYRAMID_CLOSER'] |
|
165 | 165 | if closer: |
|
166 | 166 | closer() |
|
167 | 167 | |
|
168 | 168 | |
|
169 | 169 | @signals.task_failure.connect |
|
170 | 170 | def task_failure_signal( |
|
171 | 171 | task_id, exception, args, kwargs, traceback, einfo, **kargs): |
|
172 | 172 | from rhodecode.lib.exc_tracking import store_exception |
|
173 | 173 | from rhodecode.lib.statsd_client import StatsdClient |
|
174 | 174 | |
|
175 | 175 | meta.Session.remove() |
|
176 | 176 | |
|
177 | 177 | # simulate sys.exc_info() |
|
178 | 178 | exc_info = (einfo.type, einfo.exception, einfo.tb) |
|
179 | 179 | store_exception(id(exc_info), exc_info, prefix='rhodecode-celery') |
|
180 | 180 | statsd = StatsdClient.statsd |
|
181 | 181 | if statsd: |
|
182 | statsd.incr('rhodecode_exception_total', tags=["exc_source:celery"]) | |
|
182 | statsd.incr('rhodecode_exception_total', tags=["exc_source:celery", "type:{}".format(einfo.type)]) | |
|
183 | 183 | |
|
184 | 184 | closer = celery_app.conf['PYRAMID_CLOSER'] |
|
185 | 185 | if closer: |
|
186 | 186 | closer() |
|
187 | 187 | |
|
188 | 188 | |
|
189 | 189 | @signals.task_revoked.connect |
|
190 | 190 | def task_revoked_signal( |
|
191 | 191 | request, terminated, signum, expired, **kwargs): |
|
192 | 192 | closer = celery_app.conf['PYRAMID_CLOSER'] |
|
193 | 193 | if closer: |
|
194 | 194 | closer() |
|
195 | 195 | |
|
196 | 196 | |
|
197 | 197 | def setup_celery_app(app, root, request, registry, closer, ini_location): |
|
198 | 198 | ini_dir = os.path.dirname(os.path.abspath(ini_location)) |
|
199 | 199 | celery_config = base_celery_config |
|
200 | 200 | celery_config.update({ |
|
201 | 201 | # store celerybeat scheduler db where the .ini file is |
|
202 | 202 | 'beat_schedule_filename': os.path.join(ini_dir, 'celerybeat-schedule'), |
|
203 | 203 | }) |
|
204 | 204 | ini_settings = get_ini_config(ini_location) |
|
205 | 205 | log.debug('Got custom celery conf: %s', ini_settings) |
|
206 | 206 | |
|
207 | 207 | celery_config.update(ini_settings) |
|
208 | 208 | celery_app.config_from_object(celery_config) |
|
209 | 209 | |
|
210 | 210 | celery_app.conf.update({'PYRAMID_APP': app}) |
|
211 | 211 | celery_app.conf.update({'PYRAMID_ROOT': root}) |
|
212 | 212 | celery_app.conf.update({'PYRAMID_REQUEST': request}) |
|
213 | 213 | celery_app.conf.update({'PYRAMID_REGISTRY': registry}) |
|
214 | 214 | celery_app.conf.update({'PYRAMID_CLOSER': closer}) |
|
215 | 215 | |
|
216 | 216 | |
|
217 | 217 | def configure_celery(config, ini_location): |
|
218 | 218 | """ |
|
219 | 219 | Helper that is called from our application creation logic. It gives |
|
220 | 220 | connection info into running webapp and allows execution of tasks from |
|
221 | 221 | RhodeCode itself |
|
222 | 222 | """ |
|
223 | 223 | # store some globals into rhodecode |
|
224 | 224 | rhodecode.CELERY_ENABLED = str2bool( |
|
225 | 225 | config.registry.settings.get('use_celery')) |
|
226 | 226 | if rhodecode.CELERY_ENABLED: |
|
227 | 227 | log.info('Configuring celery based on `%s` file', ini_location) |
|
228 | 228 | setup_celery_app( |
|
229 | 229 | app=None, root=None, request=None, registry=config.registry, |
|
230 | 230 | closer=None, ini_location=ini_location) |
|
231 | 231 | |
|
232 | 232 | |
|
233 | 233 | def maybe_prepare_env(req): |
|
234 | 234 | environ = {} |
|
235 | 235 | try: |
|
236 | 236 | environ.update({ |
|
237 | 237 | 'PATH_INFO': req.environ['PATH_INFO'], |
|
238 | 238 | 'SCRIPT_NAME': req.environ['SCRIPT_NAME'], |
|
239 | 239 | 'HTTP_HOST':req.environ.get('HTTP_HOST', req.environ['SERVER_NAME']), |
|
240 | 240 | 'SERVER_NAME': req.environ['SERVER_NAME'], |
|
241 | 241 | 'SERVER_PORT': req.environ['SERVER_PORT'], |
|
242 | 242 | 'wsgi.url_scheme': req.environ['wsgi.url_scheme'], |
|
243 | 243 | }) |
|
244 | 244 | except Exception: |
|
245 | 245 | pass |
|
246 | 246 | |
|
247 | 247 | return environ |
|
248 | 248 | |
|
249 | 249 | |
|
250 | 250 | class RequestContextTask(Task): |
|
251 | 251 | """ |
|
252 | 252 | This is a celery task which will create a rhodecode app instance context |
|
253 | 253 | for the task, patch pyramid with the original request |
|
254 | 254 | that created the task and also add the user to the context. |
|
255 | 255 | """ |
|
256 | 256 | |
|
257 | 257 | def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, |
|
258 | 258 | link=None, link_error=None, shadow=None, **options): |
|
259 | 259 | """ queue the job to run (we are in web request context here) """ |
|
260 | 260 | |
|
261 | 261 | req = get_current_request() |
|
262 | 262 | |
|
263 | 263 | # web case |
|
264 | 264 | if hasattr(req, 'user'): |
|
265 | 265 | ip_addr = req.user.ip_addr |
|
266 | 266 | user_id = req.user.user_id |
|
267 | 267 | |
|
268 | 268 | # api case |
|
269 | 269 | elif hasattr(req, 'rpc_user'): |
|
270 | 270 | ip_addr = req.rpc_user.ip_addr |
|
271 | 271 | user_id = req.rpc_user.user_id |
|
272 | 272 | else: |
|
273 | 273 | raise Exception( |
|
274 | 274 | 'Unable to fetch required data from request: {}. \n' |
|
275 | 275 | 'This task is required to be executed from context of ' |
|
276 | 276 | 'request in a webapp'.format(repr(req))) |
|
277 | 277 | |
|
278 | 278 | if req: |
|
279 | 279 | # we hook into kwargs since it is the only way to pass our data to |
|
280 | 280 | # the celery worker |
|
281 | 281 | environ = maybe_prepare_env(req) |
|
282 | 282 | options['headers'] = options.get('headers', {}) |
|
283 | 283 | options['headers'].update({ |
|
284 | 284 | 'rhodecode_proxy_data': { |
|
285 | 285 | 'environ': environ, |
|
286 | 286 | 'auth_user': { |
|
287 | 287 | 'ip_addr': ip_addr, |
|
288 | 288 | 'user_id': user_id |
|
289 | 289 | }, |
|
290 | 290 | } |
|
291 | 291 | }) |
|
292 | 292 | |
|
293 | 293 | return super(RequestContextTask, self).apply_async( |
|
294 | 294 | args, kwargs, task_id, producer, link, link_error, shadow, **options) |
|
295 | 295 | |
|
296 | 296 | def __call__(self, *args, **kwargs): |
|
297 | 297 | """ rebuild the context and then run task on celery worker """ |
|
298 | 298 | |
|
299 | 299 | proxy_data = getattr(self.request, 'rhodecode_proxy_data', None) |
|
300 | 300 | if not proxy_data: |
|
301 | 301 | return super(RequestContextTask, self).__call__(*args, **kwargs) |
|
302 | 302 | |
|
303 | 303 | log.debug('using celery proxy data to run task: %r', proxy_data) |
|
304 | 304 | # re-inject and register threadlocals for proper routing support |
|
305 | 305 | request = prepare_request(proxy_data['environ']) |
|
306 | 306 | request.user = AuthUser(user_id=proxy_data['auth_user']['user_id'], |
|
307 | 307 | ip_addr=proxy_data['auth_user']['ip_addr']) |
|
308 | 308 | |
|
309 | 309 | return super(RequestContextTask, self).__call__(*args, **kwargs) |
|
310 | 310 |
@@ -1,89 +1,90 b'' | |||
|
1 | 1 | # -*- coding: utf-8 -*- |
|
2 | 2 | |
|
3 | 3 | # Copyright (C) 2016-2020 RhodeCode GmbH |
|
4 | 4 | # |
|
5 | 5 | # This program is free software: you can redistribute it and/or modify |
|
6 | 6 | # it under the terms of the GNU Affero General Public License, version 3 |
|
7 | 7 | # (only), as published by the Free Software Foundation. |
|
8 | 8 | # |
|
9 | 9 | # This program is distributed in the hope that it will be useful, |
|
10 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
11 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
12 | 12 | # GNU General Public License for more details. |
|
13 | 13 | # |
|
14 | 14 | # You should have received a copy of the GNU Affero General Public License |
|
15 | 15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
16 | 16 | # |
|
17 | 17 | # This program is dual-licensed. If you wish to learn more about the |
|
18 | 18 | # RhodeCode Enterprise Edition, including its added features, Support services, |
|
19 | 19 | # and proprietary license terms, please see https://rhodecode.com/licenses/ |
|
20 | 20 | |
|
21 | 21 | import time |
|
22 | 22 | import logging |
|
23 | 23 | |
|
24 | 24 | import rhodecode |
|
25 | 25 | from rhodecode.lib.auth import AuthUser |
|
26 | 26 | from rhodecode.lib.base import get_ip_addr, get_access_path, get_user_agent |
|
27 | 27 | from rhodecode.lib.utils2 import safe_str, get_current_rhodecode_user |
|
28 | 28 | |
|
29 | 29 | |
|
30 | 30 | log = logging.getLogger(__name__) |
|
31 | 31 | |
|
32 | 32 | |
|
33 | 33 | class RequestWrapperTween(object): |
|
34 | 34 | def __init__(self, handler, registry): |
|
35 | 35 | self.handler = handler |
|
36 | 36 | self.registry = registry |
|
37 | 37 | |
|
38 | 38 | # one-time configuration code goes here |
|
39 | 39 | |
|
40 | 40 | def _get_user_info(self, request): |
|
41 | 41 | user = get_current_rhodecode_user(request) |
|
42 | 42 | if not user: |
|
43 | 43 | user = AuthUser.repr_user(ip=get_ip_addr(request.environ)) |
|
44 | 44 | return user |
|
45 | 45 | |
|
46 | 46 | def __call__(self, request): |
|
47 | 47 | start = time.time() |
|
48 | 48 | log.debug('Starting request time measurement') |
|
49 | 49 | try: |
|
50 | 50 | response = self.handler(request) |
|
51 | 51 | finally: |
|
52 | 52 | count = request.request_count() |
|
53 | 53 | _ver_ = rhodecode.__version__ |
|
54 | 54 | _path = safe_str(get_access_path(request.environ)) |
|
55 | 55 | _auth_user = self._get_user_info(request) |
|
56 | 56 | user_id = getattr(_auth_user, 'user_id', _auth_user) |
|
57 | 57 | total = time.time() - start |
|
58 | 58 | log.info( |
|
59 | 59 | 'Req[%4s] %s %s Request to %s time: %.4fs [%s], RhodeCode %s', |
|
60 | 60 | count, _auth_user, request.environ.get('REQUEST_METHOD'), |
|
61 | 61 | _path, total, get_user_agent(request. environ), _ver_ |
|
62 | 62 | ) |
|
63 | 63 | |
|
64 | 64 | statsd = request.registry.statsd |
|
65 | 65 | if statsd: |
|
66 | match_route = request.matched_route.name if request.matched_route else _path | |
|
66 | 67 | resp_code = response.status_code |
|
67 | 68 | elapsed_time_ms = 1000.0 * total |
|
68 | 69 | statsd.timing( |
|
69 | 70 | 'rhodecode_req_timing', elapsed_time_ms, |
|
70 | 71 | tags=[ |
|
71 |
"view_name:{}".format( |
|
|
72 | "view_name:{}".format(match_route), | |
|
72 | 73 | #"user:{}".format(user_id), |
|
73 | 74 | "code:{}".format(resp_code) |
|
74 | 75 | ] |
|
75 | 76 | ) |
|
76 | 77 | statsd.incr( |
|
77 | 78 | 'rhodecode_req_total', tags=[ |
|
78 |
"view_name:{}".format( |
|
|
79 | "view_name:{}".format(match_route), | |
|
79 | 80 | #"user:{}".format(user_id), |
|
80 | 81 | "code:{}".format(resp_code) |
|
81 | 82 | ]) |
|
82 | 83 | |
|
83 | 84 | return response |
|
84 | 85 | |
|
85 | 86 | |
|
86 | 87 | def includeme(config): |
|
87 | 88 | config.add_tween( |
|
88 | 89 | 'rhodecode.lib.middleware.request_wrapper.RequestWrapperTween', |
|
89 | 90 | ) |
General Comments 0
You need to be logged in to leave comments.
Login now