middleware.py
502 lines
| 18.3 KiB
| text/x-python
|
PythonLexer
r1 | # -*- coding: utf-8 -*- | |||
r1271 | # Copyright (C) 2010-2017 RhodeCode GmbH | |||
r1 | # | |||
# This program is free software: you can redistribute it and/or modify | ||||
# it under the terms of the GNU Affero General Public License, version 3 | ||||
# (only), as published by the Free Software Foundation. | ||||
# | ||||
# This program is distributed in the hope that it will be useful, | ||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of | ||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||||
# GNU General Public License for more details. | ||||
# | ||||
# You should have received a copy of the GNU Affero General Public License | ||||
# along with this program. If not, see <http://www.gnu.org/licenses/>. | ||||
# | ||||
# This program is dual-licensed. If you wish to learn more about the | ||||
# RhodeCode Enterprise Edition, including its added features, Support services, | ||||
# and proprietary license terms, please see https://rhodecode.com/licenses/ | ||||
""" | ||||
Pylons middleware initialization | ||||
""" | ||||
import logging | ||||
r526 | from collections import OrderedDict | |||
r1 | ||||
from paste.registry import RegistryManager | ||||
from paste.gzipper import make_gzip_middleware | ||||
from pylons.wsgiapp import PylonsApp | ||||
from pyramid.authorization import ACLAuthorizationPolicy | ||||
from pyramid.config import Configurator | ||||
from pyramid.settings import asbool, aslist | ||||
from pyramid.wsgi import wsgiapp | ||||
|
r945 | from pyramid.httpexceptions import ( | ||
r1499 | HTTPException, HTTPError, HTTPInternalServerError, HTTPFound) | |||
|
r580 | from pyramid.events import ApplicationCreated | ||
|
r595 | from pyramid.renderers import render_to_response | ||
r1 | from routes.middleware import RoutesMiddleware | |||
import routes.util | ||||
import rhodecode | ||||
r669 | from rhodecode.model import meta | |||
r121 | from rhodecode.config import patches | |||
r463 | from rhodecode.config.routing import STATIC_FILE_PREFIX | |||
r121 | from rhodecode.config.environment import ( | |||
load_environment, load_pyramid_environment) | ||||
r1 | from rhodecode.lib.middleware import csrf | |||
from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled | ||||
|
r945 | from rhodecode.lib.middleware.error_handling import ( | ||
PylonsErrorHandlingMiddleware) | ||||
r1 | from rhodecode.lib.middleware.https_fixup import HttpsFixup | |||
from rhodecode.lib.middleware.vcs import VCSMiddleware | ||||
from rhodecode.lib.plugins.utils import register_rhodecode_plugin | ||||
|
r604 | from rhodecode.lib.utils2 import aslist as rhodecode_aslist | ||
r1392 | from rhodecode.subscribers import ( | |||
scan_repositories_if_enabled, write_metadata_if_needed) | ||||
r1 | ||||
log = logging.getLogger(__name__) | ||||
r463 | # this is used to avoid avoid the route lookup overhead in routesmiddleware | |||
# for certain routes which won't go to pylons to - eg. static files, debugger | ||||
# it is only needed for the pylons migration and can be removed once complete | ||||
class SkippableRoutesMiddleware(RoutesMiddleware): | ||||
""" Routes middleware that allows you to skip prefixes """ | ||||
def __init__(self, *args, **kw): | ||||
self.skip_prefixes = kw.pop('skip_prefixes', []) | ||||
super(SkippableRoutesMiddleware, self).__init__(*args, **kw) | ||||
def __call__(self, environ, start_response): | ||||
for prefix in self.skip_prefixes: | ||||
if environ['PATH_INFO'].startswith(prefix): | ||||
r526 | # added to avoid the case when a missing /_static route falls | |||
# through to pylons and causes an exception as pylons is | ||||
# expecting wsgiorg.routingargs to be set in the environ | ||||
# by RoutesMiddleware. | ||||
if 'wsgiorg.routing_args' not in environ: | ||||
environ['wsgiorg.routing_args'] = (None, {}) | ||||
r463 | return self.app(environ, start_response) | |||
return super(SkippableRoutesMiddleware, self).__call__( | ||||
environ, start_response) | ||||
|
r595 | def make_app(global_conf, static_files=True, **app_conf): | ||
r1 | """Create a Pylons WSGI application and return it | |||
``global_conf`` | ||||
The inherited configuration for this application. Normally from | ||||
the [DEFAULT] section of the Paste ini file. | ||||
``app_conf`` | ||||
The application's local configuration. Normally specified in | ||||
the [app:<name>] section of the Paste ini file (where <name> | ||||
defaults to main). | ||||
""" | ||||
# Apply compatibility patches | ||||
patches.kombu_1_5_1_python_2_7_11() | ||||
patches.inspect_getargspec() | ||||
# Configure the Pylons environment | ||||
config = load_environment(global_conf, app_conf) | ||||
# The Pylons WSGI app | ||||
app = PylonsApp(config=config) | ||||
if rhodecode.is_test: | ||||
app = csrf.CSRFDetector(app) | ||||
expected_origin = config.get('expected_origin') | ||||
if expected_origin: | ||||
# The API can be accessed from other Origins. | ||||
app = csrf.OriginChecker(app, expected_origin, | ||||
skip_urls=[routes.util.url_for('api')]) | ||||
# Establish the Registry for this application | ||||
app = RegistryManager(app) | ||||
app.config = config | ||||
return app | ||||
def make_pyramid_app(global_config, **settings): | ||||
""" | ||||
Constructs the WSGI application based on Pyramid and wraps the Pylons based | ||||
application. | ||||
Specials: | ||||
* We migrate from Pylons to Pyramid. While doing this, we keep both | ||||
frameworks functional. This involves moving some WSGI middlewares around | ||||
and providing access to some data internals, so that the old code is | ||||
still functional. | ||||
* The application can also be integrated like a plugin via the call to | ||||
`includeme`. This is accompanied with the other utility functions which | ||||
are called. Changing this should be done with great care to not break | ||||
cases when these fragments are assembled from another place. | ||||
""" | ||||
# The edition string should be available in pylons too, so we add it here | ||||
# before copying the settings. | ||||
settings.setdefault('rhodecode.edition', 'Community Edition') | ||||
# As long as our Pylons application does expect "unprepared" settings, make | ||||
# sure that we keep an unmodified copy. This avoids unintentional change of | ||||
# behavior in the old application. | ||||
settings_pylons = settings.copy() | ||||
sanitize_settings_and_apply_defaults(settings) | ||||
config = Configurator(settings=settings) | ||||
add_pylons_compat_data(config.registry, global_config, settings_pylons) | ||||
r116 | ||||
r121 | load_pyramid_environment(global_config, settings) | |||
r116 | ||||
r463 | includeme_first(config) | |||
r1 | includeme(config) | |||
pyramid_app = config.make_wsgi_app() | ||||
pyramid_app = wrap_app_in_wsgi_middlewares(pyramid_app, config) | ||||
r619 | pyramid_app.config = config | |||
r669 | ||||
# creating the app uses a connection - return it after we are done | ||||
meta.Session.remove() | ||||
r1 | return pyramid_app | |||
|
r581 | def make_not_found_view(config): | ||
""" | ||||
|
r595 | This creates the view which should be registered as not-found-view to | ||
|
r581 | pyramid. Basically it contains of the old pylons app, converted to a view. | ||
Additionally it is wrapped by some other middlewares. | ||||
""" | ||||
settings = config.registry.settings | ||||
|
r595 | vcs_server_enabled = settings['vcs.server.enable'] | ||
|
r581 | |||
# Make pylons app from unprepared settings. | ||||
pylons_app = make_app( | ||||
config.registry._pylons_compat_global_config, | ||||
**config.registry._pylons_compat_settings) | ||||
config.registry._pylons_compat_config = pylons_app.config | ||||
|
r595 | # Appenlight monitoring. | ||
pylons_app, appenlight_client = wrap_in_appenlight_if_enabled( | ||||
pylons_app, settings) | ||||
|
r946 | # The pylons app is executed inside of the pyramid 404 exception handler. | ||
# Exceptions which are raised inside of it are not handled by pyramid | ||||
# again. Therefore we add a middleware that invokes the error handler in | ||||
# case of an exception or error response. This way we return proper error | ||||
# HTML pages in case of an error. | ||||
|
r945 | reraise = (settings.get('debugtoolbar.enabled', False) or | ||
rhodecode.disable_error_handler) | ||||
pylons_app = PylonsErrorHandlingMiddleware( | ||||
pylons_app, error_handler, reraise) | ||||
|
r609 | |||
|
r948 | # The VCSMiddleware shall operate like a fallback if pyramid doesn't find a | ||
# view to handle the request. Therefore it is wrapped around the pylons | ||||
# app. It has to be outside of the error handling otherwise error responses | ||||
# from the vcsserver are converted to HTML error pages. This confuses the | ||||
# command line tools and the user won't get a meaningful error message. | ||||
if vcs_server_enabled: | ||||
pylons_app = VCSMiddleware( | ||||
pylons_app, settings, appenlight_client, registry=config.registry) | ||||
|
r945 | # Convert WSGI app to pyramid view and return it. | ||
return wsgiapp(pylons_app) | ||||
|
r581 | |||
r1 | def add_pylons_compat_data(registry, global_config, settings): | |||
""" | ||||
Attach data to the registry to support the Pylons integration. | ||||
""" | ||||
registry._pylons_compat_global_config = global_config | ||||
registry._pylons_compat_settings = settings | ||||
r194 | def error_handler(exception, request): | |||
r1496 | import rhodecode | |||
r187 | from rhodecode.lib.utils2 import AttributeDict | |||
r1496 | rhodecode_title = rhodecode.CONFIG.get('rhodecode_title') or 'RhodeCode' | |||
r187 | ||||
r194 | base_response = HTTPInternalServerError() | |||
# prefer original exception for the response since it may have headers set | ||||
r1499 | if isinstance(exception, HTTPException): | |||
r194 | base_response = exception | |||
r1314 | def is_http_error(response): | |||
# error which should have traceback | ||||
return response.status_code > 499 | ||||
if is_http_error(base_response): | ||||
log.exception( | ||||
'error occurred handling this request for path: %s', request.path) | ||||
r187 | c = AttributeDict() | |||
r194 | c.error_message = base_response.status | |||
c.error_explanation = base_response.explanation or str(base_response) | ||||
r187 | c.visual = AttributeDict() | |||
c.visual.rhodecode_support_url = ( | ||||
request.registry.settings.get('rhodecode_support_url') or | ||||
request.route_url('rhodecode_support') | ||||
) | ||||
c.redirect_time = 0 | ||||
r1496 | c.rhodecode_name = rhodecode_title | |||
r187 | if not c.rhodecode_name: | |||
c.rhodecode_name = 'Rhodecode' | ||||
r683 | c.causes = [] | |||
if hasattr(base_response, 'causes'): | ||||
c.causes = base_response.causes | ||||
r187 | response = render_to_response( | |||
r1282 | '/errors/error_document.mako', {'c': c}, request=request, | |||
r190 | response=base_response) | |||
r187 | return response | |||
r1 | def includeme(config): | |||
settings = config.registry.settings | ||||
r470 | # plugin information | |||
r526 | config.registry.rhodecode_plugins = OrderedDict() | |||
r470 | ||||
config.add_directive( | ||||
'register_rhodecode_plugin', register_rhodecode_plugin) | ||||
r194 | if asbool(settings.get('appenlight', 'false')): | |||
config.include('appenlight_client.ext.pyramid_tween') | ||||
r1 | # Includes which are required. The application would fail without them. | |||
config.include('pyramid_mako') | ||||
config.include('pyramid_beaker') | ||||
r1503 | ||||
r1 | config.include('rhodecode.authentication') | |||
r411 | config.include('rhodecode.integrations') | |||
r1502 | ||||
# apps | ||||
r1503 | config.include('rhodecode.apps.admin') | |||
r1504 | config.include('rhodecode.apps.channelstream') | |||
r1501 | config.include('rhodecode.apps.login') | |||
r1502 | config.include('rhodecode.apps.user_profile') | |||
r1505 | config.include('rhodecode.apps.my_account') | |||
r1502 | ||||
r1 | config.include('rhodecode.tweens') | |||
config.include('rhodecode.api') | ||||
|
r561 | config.include('rhodecode.svn_support') | ||
r1503 | ||||
r187 | config.add_route( | |||
'rhodecode_support', 'https://rhodecode.com/help/', static=True) | ||||
r1 | ||||
r1303 | config.add_translation_dirs('rhodecode:i18n/') | |||
settings['default_locale_name'] = settings.get('lang', 'en') | ||||
|
r580 | # Add subscribers. | ||
config.add_subscriber(scan_repositories_if_enabled, ApplicationCreated) | ||||
r1392 | config.add_subscriber(write_metadata_if_needed, ApplicationCreated) | |||
|
r580 | |||
r1 | # Set the authorization policy. | |||
authz_policy = ACLAuthorizationPolicy() | ||||
config.set_authorization_policy(authz_policy) | ||||
# Set the default renderer for HTML templates to mako. | ||||
config.add_mako_renderer('.html') | ||||
# include RhodeCode plugins | ||||
includes = aslist(settings.get('rhodecode.includes', [])) | ||||
for inc in includes: | ||||
config.include(inc) | ||||
# This is the glue which allows us to migrate in chunks. By registering the | ||||
# pylons based application as the "Not Found" view in Pyramid, we will | ||||
# fallback to the old application each time the new one does not yet know | ||||
# how to handle a request. | ||||
|
r581 | config.add_notfound_view(make_not_found_view(config)) | ||
r1 | ||||
r449 | if not settings.get('debugtoolbar.enabled', False): | |||
r194 | # if no toolbar, then any exception gets caught and rendered | |||
r449 | config.add_view(error_handler, context=Exception) | |||
config.add_view(error_handler, context=HTTPError) | ||||
r1 | ||||
r463 | def includeme_first(config): | |||
r455 | # redirect automatic browser favicon.ico requests to correct place | |||
def favicon_redirect(context, request): | ||||
r620 | return HTTPFound( | |||
r577 | request.static_path('rhodecode:public/images/favicon.ico')) | |||
r463 | ||||
r455 | config.add_view(favicon_redirect, route_name='favicon') | |||
config.add_route('favicon', '/favicon.ico') | ||||
r1316 | def robots_redirect(context, request): | |||
return HTTPFound( | ||||
request.static_path('rhodecode:public/robots.txt')) | ||||
config.add_view(robots_redirect, route_name='robots') | ||||
config.add_route('robots', '/robots.txt') | ||||
r466 | config.add_static_view( | |||
r518 | '_static/deform', 'deform:static') | |||
config.add_static_view( | ||||
r522 | '_static/rhodecode', path='rhodecode:public', cache_max_age=3600 * 24) | |||
r463 | ||||
|
r598 | |||
r1 | def wrap_app_in_wsgi_middlewares(pyramid_app, config): | |||
""" | ||||
Apply outer WSGI middlewares around the application. | ||||
Part of this has been moved up from the Pylons layer, so that the | ||||
data is also available if old Pylons code is hit through an already ported | ||||
view. | ||||
""" | ||||
settings = config.registry.settings | ||||
r181 | # enable https redirects based on HTTP_X_URL_SCHEME set by proxy | |||
pyramid_app = HttpsFixup(pyramid_app, settings) | ||||
r187 | # Add RoutesMiddleware to support the pylons compatibility tween during | |||
r1 | # migration to pyramid. | |||
r463 | pyramid_app = SkippableRoutesMiddleware( | |||
pyramid_app, config.registry._pylons_compat_config['routes.map'], | ||||
skip_prefixes=(STATIC_FILE_PREFIX, '/_debug_toolbar')) | ||||
r1 | ||||
|
r599 | pyramid_app, _ = wrap_in_appenlight_if_enabled(pyramid_app, settings) | ||
r194 | ||||
|
r598 | if settings['gzip_responses']: | ||
r1 | pyramid_app = make_gzip_middleware( | |||
pyramid_app, settings, compress_level=1) | ||||
r669 | # this should be the outer most middleware in the wsgi stack since | |||
# middleware like Routes make database calls | ||||
def pyramid_app_with_cleanup(environ, start_response): | ||||
try: | ||||
return pyramid_app(environ, start_response) | ||||
finally: | ||||
# Dispose current database session and rollback uncommitted | ||||
# transactions. | ||||
meta.Session.remove() | ||||
# In a single threaded mode server, on non sqlite db we should have | ||||
# '0 Current Checked out connections' at the end of a request, | ||||
# if not, then something, somewhere is leaving a connection open | ||||
pool = meta.Base.metadata.bind.engine.pool | ||||
log.debug('sa pool status: %s', pool.status()) | ||||
return pyramid_app_with_cleanup | ||||
r1 | ||||
def sanitize_settings_and_apply_defaults(settings): | ||||
""" | ||||
Applies settings defaults and does all type conversion. | ||||
We would move all settings parsing and preparation into this place, so that | ||||
we have only one place left which deals with this part. The remaining parts | ||||
of the application would start to rely fully on well prepared settings. | ||||
This piece would later be split up per topic to avoid a big fat monster | ||||
function. | ||||
""" | ||||
# Pyramid's mako renderer has to search in the templates folder so that the | ||||
# old templates still work. Ported and new templates are expected to use | ||||
# real asset specifications for the includes. | ||||
mako_directories = settings.setdefault('mako.directories', [ | ||||
# Base templates of the original Pylons application | ||||
'rhodecode:templates', | ||||
]) | ||||
log.debug( | ||||
"Using the following Mako template directories: %s", | ||||
mako_directories) | ||||
# Default includes, possible to change as a user | ||||
pyramid_includes = settings.setdefault('pyramid.includes', [ | ||||
'rhodecode.lib.middleware.request_wrapper', | ||||
]) | ||||
log.debug( | ||||
"Using the following pyramid.includes: %s", | ||||
pyramid_includes) | ||||
# TODO: johbo: Re-think this, usually the call to config.include | ||||
# should allow to pass in a prefix. | ||||
settings.setdefault('rhodecode.api.url', '/_admin/api') | ||||
|
r598 | # Sanitize generic settings. | ||
|
r586 | _list_setting(settings, 'default_encoding', 'UTF-8') | ||
r118 | _bool_setting(settings, 'is_test', 'false') | |||
|
r598 | _bool_setting(settings, 'gzip_responses', 'false') | ||
r1 | ||||
|
r585 | # Call split out functions that sanitize settings for each topic. | ||
|
r598 | _sanitize_appenlight_settings(settings) | ||
|
r585 | _sanitize_vcs_settings(settings) | ||
r1 | return settings | |||
|
r598 | def _sanitize_appenlight_settings(settings): | ||
_bool_setting(settings, 'appenlight', 'false') | ||||
|
r585 | def _sanitize_vcs_settings(settings): | ||
""" | ||||
Applies settings defaults and does type conversion for all VCS related | ||||
settings. | ||||
""" | ||||
_string_setting(settings, 'vcs.svn.compatible_version', '') | ||||
_string_setting(settings, 'git_rev_filter', '--all') | ||||
|
r958 | _string_setting(settings, 'vcs.hooks.protocol', 'http') | ||
_string_setting(settings, 'vcs.scm_app_implementation', 'http') | ||||
|
r585 | _string_setting(settings, 'vcs.server', '') | ||
_string_setting(settings, 'vcs.server.log_level', 'debug') | ||||
|
r958 | _string_setting(settings, 'vcs.server.protocol', 'http') | ||
|
r585 | _bool_setting(settings, 'startup.import_repos', 'false') | ||
_bool_setting(settings, 'vcs.hooks.direct_calls', 'false') | ||||
_bool_setting(settings, 'vcs.server.enable', 'true') | ||||
_bool_setting(settings, 'vcs.start_server', 'false') | ||||
_list_setting(settings, 'vcs.backends', 'hg, git, svn') | ||||
|
r623 | _int_setting(settings, 'vcs.connection_timeout', 3600) | ||
|
r963 | # Support legacy values of vcs.scm_app_implementation. Legacy | ||
# configurations may use 'rhodecode.lib.middleware.utils.scm_app_http' | ||||
# which is now mapped to 'http'. | ||||
scm_app_impl = settings['vcs.scm_app_implementation'] | ||||
if scm_app_impl == 'rhodecode.lib.middleware.utils.scm_app_http': | ||||
settings['vcs.scm_app_implementation'] = 'http' | ||||
|
r623 | |||
def _int_setting(settings, name, default): | ||||
settings[name] = int(settings.get(name, default)) | ||||
|
r585 | |||
r1 | def _bool_setting(settings, name, default): | |||
|
r605 | input = settings.get(name, default) | ||
if isinstance(input, unicode): | ||||
input = input.encode('utf8') | ||||
settings[name] = asbool(input) | ||||
|
r583 | |||
def _list_setting(settings, name, default): | ||||
raw_value = settings.get(name, default) | ||||
|
r604 | old_separator = ',' | ||
if old_separator in raw_value: | ||||
# If we get a comma separated list, pass it to our own function. | ||||
settings[name] = rhodecode_aslist(raw_value, sep=old_separator) | ||||
else: | ||||
# Otherwise we assume it uses pyramids space/newline separation. | ||||
settings[name] = aslist(raw_value) | ||||
|
r584 | |||
|
r1003 | def _string_setting(settings, name, default, lower=True): | ||
value = settings.get(name, default) | ||||
if lower: | ||||
value = value.lower() | ||||
settings[name] = value | ||||