##// END OF EJS Templates
caches: use single default cache dir for all backends....
marcink -
r3008:ab171c02 default
parent child Browse files
Show More
@@ -1,556 +1,575 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # Copyright (C) 2010-2018 RhodeCode GmbH
4 4 #
5 5 # This program is free software: you can redistribute it and/or modify
6 6 # it under the terms of the GNU Affero General Public License, version 3
7 7 # (only), as published by the Free Software Foundation.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU Affero General Public License
15 15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 16 #
17 17 # This program is dual-licensed. If you wish to learn more about the
18 18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20
21 21 import os
22 22 import sys
23 23 import logging
24 24 import collections
25 25 import tempfile
26 26
27 27 from paste.gzipper import make_gzip_middleware
28 28 import pyramid.events
29 29 from pyramid.wsgi import wsgiapp
30 30 from pyramid.authorization import ACLAuthorizationPolicy
31 31 from pyramid.config import Configurator
32 32 from pyramid.settings import asbool, aslist
33 33 from pyramid.httpexceptions import (
34 34 HTTPException, HTTPError, HTTPInternalServerError, HTTPFound, HTTPNotFound)
35 35 from pyramid.renderers import render_to_response
36 36
37 37 from rhodecode.model import meta
38 38 from rhodecode.config import patches
39 39 from rhodecode.config import utils as config_utils
40 40 from rhodecode.config.environment import load_pyramid_environment
41 41
42 42 import rhodecode.events
43 43 from rhodecode.lib.middleware.vcs import VCSMiddleware
44 44 from rhodecode.lib.request import Request
45 45 from rhodecode.lib.vcs import VCSCommunicationError
46 46 from rhodecode.lib.exceptions import VCSServerUnavailable
47 47 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
48 48 from rhodecode.lib.middleware.https_fixup import HttpsFixup
49 49 from rhodecode.lib.celerylib.loader import configure_celery
50 50 from rhodecode.lib.plugins.utils import register_rhodecode_plugin
51 51 from rhodecode.lib.utils2 import aslist as rhodecode_aslist, AttributeDict
52 52 from rhodecode.lib.exc_tracking import store_exception
53 53 from rhodecode.subscribers import (
54 54 scan_repositories_if_enabled, write_js_routes_if_enabled,
55 55 write_metadata_if_needed, inject_app_settings)
56 56
57 57
58 58 log = logging.getLogger(__name__)
59 59
60 60
61 61 def is_http_error(response):
62 62 # error which should have traceback
63 63 return response.status_code > 499
64 64
65 65
66 66 def make_pyramid_app(global_config, **settings):
67 67 """
68 68 Constructs the WSGI application based on Pyramid.
69 69
70 70 Specials:
71 71
72 72 * The application can also be integrated like a plugin via the call to
73 73 `includeme`. This is accompanied with the other utility functions which
74 74 are called. Changing this should be done with great care to not break
75 75 cases when these fragments are assembled from another place.
76 76
77 77 """
78 78
79 79 # Allows to use format style "{ENV_NAME}" placeholders in the configuration. It
80 80 # will be replaced by the value of the environment variable "NAME" in this case.
81 81 environ = {
82 82 'ENV_{}'.format(key): value for key, value in os.environ.items()}
83 83
84 84 global_config = _substitute_values(global_config, environ)
85 85 settings = _substitute_values(settings, environ)
86 86
87 87 sanitize_settings_and_apply_defaults(settings)
88 88
89 89 config = Configurator(settings=settings)
90 90
91 91 # Apply compatibility patches
92 92 patches.inspect_getargspec()
93 93
94 94 load_pyramid_environment(global_config, settings)
95 95
96 96 # Static file view comes first
97 97 includeme_first(config)
98 98
99 99 includeme(config)
100 100
101 101 pyramid_app = config.make_wsgi_app()
102 102 pyramid_app = wrap_app_in_wsgi_middlewares(pyramid_app, config)
103 103 pyramid_app.config = config
104 104
105 105 config.configure_celery(global_config['__file__'])
106 106 # creating the app uses a connection - return it after we are done
107 107 meta.Session.remove()
108 108
109 109 log.info('Pyramid app %s created and configured.', pyramid_app)
110 110 return pyramid_app
111 111
112 112
113 113 def not_found_view(request):
114 114 """
115 115 This creates the view which should be registered as not-found-view to
116 116 pyramid.
117 117 """
118 118
119 119 if not getattr(request, 'vcs_call', None):
120 120 # handle like regular case with our error_handler
121 121 return error_handler(HTTPNotFound(), request)
122 122
123 123 # handle not found view as a vcs call
124 124 settings = request.registry.settings
125 125 ae_client = getattr(request, 'ae_client', None)
126 126 vcs_app = VCSMiddleware(
127 127 HTTPNotFound(), request.registry, settings,
128 128 appenlight_client=ae_client)
129 129
130 130 return wsgiapp(vcs_app)(None, request)
131 131
132 132
133 133 def error_handler(exception, request):
134 134 import rhodecode
135 135 from rhodecode.lib import helpers
136 136
137 137 rhodecode_title = rhodecode.CONFIG.get('rhodecode_title') or 'RhodeCode'
138 138
139 139 base_response = HTTPInternalServerError()
140 140 # prefer original exception for the response since it may have headers set
141 141 if isinstance(exception, HTTPException):
142 142 base_response = exception
143 143 elif isinstance(exception, VCSCommunicationError):
144 144 base_response = VCSServerUnavailable()
145 145
146 146 if is_http_error(base_response):
147 147 log.exception(
148 148 'error occurred handling this request for path: %s', request.path)
149 149
150 150 error_explanation = base_response.explanation or str(base_response)
151 151 if base_response.status_code == 404:
152 152 error_explanation += " Or you don't have permission to access it."
153 153 c = AttributeDict()
154 154 c.error_message = base_response.status
155 155 c.error_explanation = error_explanation
156 156 c.visual = AttributeDict()
157 157
158 158 c.visual.rhodecode_support_url = (
159 159 request.registry.settings.get('rhodecode_support_url') or
160 160 request.route_url('rhodecode_support')
161 161 )
162 162 c.redirect_time = 0
163 163 c.rhodecode_name = rhodecode_title
164 164 if not c.rhodecode_name:
165 165 c.rhodecode_name = 'Rhodecode'
166 166
167 167 c.causes = []
168 168 if is_http_error(base_response):
169 169 c.causes.append('Server is overloaded.')
170 170 c.causes.append('Server database connection is lost.')
171 171 c.causes.append('Server expected unhandled error.')
172 172
173 173 if hasattr(base_response, 'causes'):
174 174 c.causes = base_response.causes
175 175
176 176 c.messages = helpers.flash.pop_messages(request=request)
177 177
178 178 exc_info = sys.exc_info()
179 179 c.exception_id = id(exc_info)
180 180 c.show_exception_id = isinstance(base_response, VCSServerUnavailable) \
181 181 or base_response.status_code > 499
182 182 c.exception_id_url = request.route_url(
183 183 'admin_settings_exception_tracker_show', exception_id=c.exception_id)
184 184
185 185 if c.show_exception_id:
186 186 store_exception(c.exception_id, exc_info)
187 187
188 188 response = render_to_response(
189 189 '/errors/error_document.mako', {'c': c, 'h': helpers}, request=request,
190 190 response=base_response)
191 191
192 192 return response
193 193
194 194
195 195 def includeme_first(config):
196 196 # redirect automatic browser favicon.ico requests to correct place
197 197 def favicon_redirect(context, request):
198 198 return HTTPFound(
199 199 request.static_path('rhodecode:public/images/favicon.ico'))
200 200
201 201 config.add_view(favicon_redirect, route_name='favicon')
202 202 config.add_route('favicon', '/favicon.ico')
203 203
204 204 def robots_redirect(context, request):
205 205 return HTTPFound(
206 206 request.static_path('rhodecode:public/robots.txt'))
207 207
208 208 config.add_view(robots_redirect, route_name='robots')
209 209 config.add_route('robots', '/robots.txt')
210 210
211 211 config.add_static_view(
212 212 '_static/deform', 'deform:static')
213 213 config.add_static_view(
214 214 '_static/rhodecode', path='rhodecode:public', cache_max_age=3600 * 24)
215 215
216 216
217 217 def includeme(config):
218 218 settings = config.registry.settings
219 219 config.set_request_factory(Request)
220 220
221 221 # plugin information
222 222 config.registry.rhodecode_plugins = collections.OrderedDict()
223 223
224 224 config.add_directive(
225 225 'register_rhodecode_plugin', register_rhodecode_plugin)
226 226
227 227 config.add_directive('configure_celery', configure_celery)
228 228
229 229 if asbool(settings.get('appenlight', 'false')):
230 230 config.include('appenlight_client.ext.pyramid_tween')
231 231
232 232 # Includes which are required. The application would fail without them.
233 233 config.include('pyramid_mako')
234 234 config.include('pyramid_beaker')
235 235 config.include('rhodecode.lib.rc_cache')
236 236
237 237 config.include('rhodecode.authentication')
238 238 config.include('rhodecode.integrations')
239 239
240 240 # apps
241 241 config.include('rhodecode.apps._base')
242 242 config.include('rhodecode.apps.ops')
243 243
244 244 config.include('rhodecode.apps.admin')
245 245 config.include('rhodecode.apps.channelstream')
246 246 config.include('rhodecode.apps.login')
247 247 config.include('rhodecode.apps.home')
248 248 config.include('rhodecode.apps.journal')
249 249 config.include('rhodecode.apps.repository')
250 250 config.include('rhodecode.apps.repo_group')
251 251 config.include('rhodecode.apps.user_group')
252 252 config.include('rhodecode.apps.search')
253 253 config.include('rhodecode.apps.user_profile')
254 254 config.include('rhodecode.apps.user_group_profile')
255 255 config.include('rhodecode.apps.my_account')
256 256 config.include('rhodecode.apps.svn_support')
257 257 config.include('rhodecode.apps.ssh_support')
258 258 config.include('rhodecode.apps.gist')
259 259
260 260 config.include('rhodecode.apps.debug_style')
261 261 config.include('rhodecode.tweens')
262 262 config.include('rhodecode.api')
263 263
264 264 config.add_route(
265 265 'rhodecode_support', 'https://rhodecode.com/help/', static=True)
266 266
267 267 config.add_translation_dirs('rhodecode:i18n/')
268 268 settings['default_locale_name'] = settings.get('lang', 'en')
269 269
270 270 # Add subscribers.
271 271 config.add_subscriber(inject_app_settings,
272 272 pyramid.events.ApplicationCreated)
273 273 config.add_subscriber(scan_repositories_if_enabled,
274 274 pyramid.events.ApplicationCreated)
275 275 config.add_subscriber(write_metadata_if_needed,
276 276 pyramid.events.ApplicationCreated)
277 277 config.add_subscriber(write_js_routes_if_enabled,
278 278 pyramid.events.ApplicationCreated)
279 279
280 280 # request custom methods
281 281 config.add_request_method(
282 282 'rhodecode.lib.partial_renderer.get_partial_renderer',
283 283 'get_partial_renderer')
284 284
285 285 # Set the authorization policy.
286 286 authz_policy = ACLAuthorizationPolicy()
287 287 config.set_authorization_policy(authz_policy)
288 288
289 289 # Set the default renderer for HTML templates to mako.
290 290 config.add_mako_renderer('.html')
291 291
292 292 config.add_renderer(
293 293 name='json_ext',
294 294 factory='rhodecode.lib.ext_json_renderer.pyramid_ext_json')
295 295
296 296 # include RhodeCode plugins
297 297 includes = aslist(settings.get('rhodecode.includes', []))
298 298 for inc in includes:
299 299 config.include(inc)
300 300
301 301 # custom not found view, if our pyramid app doesn't know how to handle
302 302 # the request pass it to potential VCS handling ap
303 303 config.add_notfound_view(not_found_view)
304 304 if not settings.get('debugtoolbar.enabled', False):
305 305 # disabled debugtoolbar handle all exceptions via the error_handlers
306 306 config.add_view(error_handler, context=Exception)
307 307
308 308 # all errors including 403/404/50X
309 309 config.add_view(error_handler, context=HTTPError)
310 310
311 311
312 312 def wrap_app_in_wsgi_middlewares(pyramid_app, config):
313 313 """
314 314 Apply outer WSGI middlewares around the application.
315 315 """
316 316 registry = config.registry
317 317 settings = registry.settings
318 318
319 319 # enable https redirects based on HTTP_X_URL_SCHEME set by proxy
320 320 pyramid_app = HttpsFixup(pyramid_app, settings)
321 321
322 322 pyramid_app, _ae_client = wrap_in_appenlight_if_enabled(
323 323 pyramid_app, settings)
324 324 registry.ae_client = _ae_client
325 325
326 326 if settings['gzip_responses']:
327 327 pyramid_app = make_gzip_middleware(
328 328 pyramid_app, settings, compress_level=1)
329 329
330 330 # this should be the outer most middleware in the wsgi stack since
331 331 # middleware like Routes make database calls
332 332 def pyramid_app_with_cleanup(environ, start_response):
333 333 try:
334 334 return pyramid_app(environ, start_response)
335 335 finally:
336 336 # Dispose current database session and rollback uncommitted
337 337 # transactions.
338 338 meta.Session.remove()
339 339
340 340 # In a single threaded mode server, on non sqlite db we should have
341 341 # '0 Current Checked out connections' at the end of a request,
342 342 # if not, then something, somewhere is leaving a connection open
343 343 pool = meta.Base.metadata.bind.engine.pool
344 344 log.debug('sa pool status: %s', pool.status())
345 345 log.debug('Request processing finalized')
346 346
347 347 return pyramid_app_with_cleanup
348 348
349 349
350 350 def sanitize_settings_and_apply_defaults(settings):
351 351 """
352 352 Applies settings defaults and does all type conversion.
353 353
354 354 We would move all settings parsing and preparation into this place, so that
355 355 we have only one place left which deals with this part. The remaining parts
356 356 of the application would start to rely fully on well prepared settings.
357 357
358 358 This piece would later be split up per topic to avoid a big fat monster
359 359 function.
360 360 """
361 361
362 362 settings.setdefault('rhodecode.edition', 'Community Edition')
363 363
364 364 if 'mako.default_filters' not in settings:
365 365 # set custom default filters if we don't have it defined
366 366 settings['mako.imports'] = 'from rhodecode.lib.base import h_filter'
367 367 settings['mako.default_filters'] = 'h_filter'
368 368
369 369 if 'mako.directories' not in settings:
370 370 mako_directories = settings.setdefault('mako.directories', [
371 371 # Base templates of the original application
372 372 'rhodecode:templates',
373 373 ])
374 374 log.debug(
375 375 "Using the following Mako template directories: %s",
376 376 mako_directories)
377 377
378 378 # Default includes, possible to change as a user
379 379 pyramid_includes = settings.setdefault('pyramid.includes', [
380 380 'rhodecode.lib.middleware.request_wrapper',
381 381 ])
382 382 log.debug(
383 383 "Using the following pyramid.includes: %s",
384 384 pyramid_includes)
385 385
386 386 # TODO: johbo: Re-think this, usually the call to config.include
387 387 # should allow to pass in a prefix.
388 388 settings.setdefault('rhodecode.api.url', '/_admin/api')
389 389
390 390 # Sanitize generic settings.
391 391 _list_setting(settings, 'default_encoding', 'UTF-8')
392 392 _bool_setting(settings, 'is_test', 'false')
393 393 _bool_setting(settings, 'gzip_responses', 'false')
394 394
395 395 # Call split out functions that sanitize settings for each topic.
396 396 _sanitize_appenlight_settings(settings)
397 397 _sanitize_vcs_settings(settings)
398 398 _sanitize_cache_settings(settings)
399 399
400 400 # configure instance id
401 401 config_utils.set_instance_id(settings)
402 402
403 403 return settings
404 404
405 405
406 406 def _sanitize_appenlight_settings(settings):
407 407 _bool_setting(settings, 'appenlight', 'false')
408 408
409 409
410 410 def _sanitize_vcs_settings(settings):
411 411 """
412 412 Applies settings defaults and does type conversion for all VCS related
413 413 settings.
414 414 """
415 415 _string_setting(settings, 'vcs.svn.compatible_version', '')
416 416 _string_setting(settings, 'git_rev_filter', '--all')
417 417 _string_setting(settings, 'vcs.hooks.protocol', 'http')
418 418 _string_setting(settings, 'vcs.hooks.host', '127.0.0.1')
419 419 _string_setting(settings, 'vcs.scm_app_implementation', 'http')
420 420 _string_setting(settings, 'vcs.server', '')
421 421 _string_setting(settings, 'vcs.server.log_level', 'debug')
422 422 _string_setting(settings, 'vcs.server.protocol', 'http')
423 423 _bool_setting(settings, 'startup.import_repos', 'false')
424 424 _bool_setting(settings, 'vcs.hooks.direct_calls', 'false')
425 425 _bool_setting(settings, 'vcs.server.enable', 'true')
426 426 _bool_setting(settings, 'vcs.start_server', 'false')
427 427 _list_setting(settings, 'vcs.backends', 'hg, git, svn')
428 428 _int_setting(settings, 'vcs.connection_timeout', 3600)
429 429
430 430 # Support legacy values of vcs.scm_app_implementation. Legacy
431 431 # configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or
432 432 # disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'.
433 433 scm_app_impl = settings['vcs.scm_app_implementation']
434 434 if scm_app_impl in ['rhodecode.lib.middleware.utils.scm_app_http', 'vcsserver.scm_app']:
435 435 settings['vcs.scm_app_implementation'] = 'http'
436 436
437 437
438 438 def _sanitize_cache_settings(settings):
439 _string_setting(settings, 'cache_dir',
440 os.path.join(tempfile.gettempdir(), 'rc_cache'))
439 default_cache_dir = os.path.join(tempfile.gettempdir(), 'rc_cache')
440
441 # save default, cache dir, and use it for all backends later.
442 default_cache_dir = _string_setting(
443 settings,
444 'cache_dir',
445 default_cache_dir, lower=False, default_when_empty=True)
446
447 # ensure we have our dir created
448 if not os.path.isdir(default_cache_dir):
449 os.makedirs(default_cache_dir, mode=0755)
450
441 451 # cache_perms
442 452 _string_setting(
443 453 settings,
444 454 'rc_cache.cache_perms.backend',
445 'dogpile.cache.rc.file_namespace')
455 'dogpile.cache.rc.file_namespace', lower=False)
446 456 _int_setting(
447 457 settings,
448 458 'rc_cache.cache_perms.expiration_time',
449 459 60)
450 460 _string_setting(
451 461 settings,
452 462 'rc_cache.cache_perms.arguments.filename',
453 os.path.join(tempfile.gettempdir(), 'rc_cache_1'))
463 os.path.join(default_cache_dir, 'rc_cache_1'), lower=False)
454 464
455 465 # cache_repo
456 466 _string_setting(
457 467 settings,
458 468 'rc_cache.cache_repo.backend',
459 'dogpile.cache.rc.file_namespace')
469 'dogpile.cache.rc.file_namespace', lower=False)
460 470 _int_setting(
461 471 settings,
462 472 'rc_cache.cache_repo.expiration_time',
463 473 60)
464 474 _string_setting(
465 475 settings,
466 476 'rc_cache.cache_repo.arguments.filename',
467 os.path.join(tempfile.gettempdir(), 'rc_cache_2'))
477 os.path.join(default_cache_dir, 'rc_cache_2'), lower=False)
468 478
469 479 # cache_license
470 480 _string_setting(
471 481 settings,
472 482 'rc_cache.cache_license.backend',
473 'dogpile.cache.rc.file_namespace')
483 'dogpile.cache.rc.file_namespace', lower=False)
474 484 _int_setting(
475 485 settings,
476 486 'rc_cache.cache_license.expiration_time',
477 487 5*60)
478 488 _string_setting(
479 489 settings,
480 490 'rc_cache.cache_license.arguments.filename',
481 os.path.join(tempfile.gettempdir(), 'rc_cache_3'))
491 os.path.join(default_cache_dir, 'rc_cache_3'), lower=False)
482 492
483 493 # cache_repo_longterm memory, 96H
484 494 _string_setting(
485 495 settings,
486 496 'rc_cache.cache_repo_longterm.backend',
487 'dogpile.cache.rc.memory_lru')
497 'dogpile.cache.rc.memory_lru', lower=False)
488 498 _int_setting(
489 499 settings,
490 500 'rc_cache.cache_repo_longterm.expiration_time',
491 501 345600)
492 502 _int_setting(
493 503 settings,
494 504 'rc_cache.cache_repo_longterm.max_size',
495 505 10000)
496 506
497 507 # sql_cache_short
498 508 _string_setting(
499 509 settings,
500 510 'rc_cache.sql_cache_short.backend',
501 'dogpile.cache.rc.memory_lru')
511 'dogpile.cache.rc.memory_lru', lower=False)
502 512 _int_setting(
503 513 settings,
504 514 'rc_cache.sql_cache_short.expiration_time',
505 515 30)
506 516 _int_setting(
507 517 settings,
508 518 'rc_cache.sql_cache_short.max_size',
509 519 10000)
510 520
511 521
512 522 def _int_setting(settings, name, default):
513 523 settings[name] = int(settings.get(name, default))
524 return settings[name]
514 525
515 526
516 527 def _bool_setting(settings, name, default):
517 528 input_val = settings.get(name, default)
518 529 if isinstance(input_val, unicode):
519 530 input_val = input_val.encode('utf8')
520 531 settings[name] = asbool(input_val)
532 return settings[name]
521 533
522 534
523 535 def _list_setting(settings, name, default):
524 536 raw_value = settings.get(name, default)
525 537
526 538 old_separator = ','
527 539 if old_separator in raw_value:
528 540 # If we get a comma separated list, pass it to our own function.
529 541 settings[name] = rhodecode_aslist(raw_value, sep=old_separator)
530 542 else:
531 543 # Otherwise we assume it uses pyramids space/newline separation.
532 544 settings[name] = aslist(raw_value)
545 return settings[name]
533 546
534 547
535 def _string_setting(settings, name, default, lower=True):
548 def _string_setting(settings, name, default, lower=True, default_when_empty=False):
536 549 value = settings.get(name, default)
550
551 if default_when_empty and not value:
552 # use default value when value is empty
553 value = default
554
537 555 if lower:
538 556 value = value.lower()
539 557 settings[name] = value
558 return settings[name]
540 559
541 560
542 561 def _substitute_values(mapping, substitutions):
543 562
544 563 try:
545 564 result = {
546 565 # Note: Cannot use regular replacements, since they would clash
547 566 # with the implementation of ConfigParser. Using "format" instead.
548 567 key: value.format(**substitutions)
549 568 for key, value in mapping.items()
550 569 }
551 570 except KeyError as e:
552 571 raise ValueError(
553 572 'Failed to substitute env variable: {}. '
554 573 'Make sure you have specified this env variable without ENV_ prefix'.format(e))
555 574
556 575 return result
General Comments 0
You need to be logged in to leave comments. Login now