rhodecode/config/middleware.py

# Copyright (C) 2010-2020 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/

import os
import sys
import collections
import tempfile
import time
import logging.config

from paste.gzipper import make_gzip_middleware
import pyramid.events
from pyramid.wsgi import wsgiapp
from pyramid.authorization import ACLAuthorizationPolicy
from pyramid.config import Configurator
from pyramid.settings import asbool, aslist
from pyramid.httpexceptions import (
    HTTPException, HTTPError, HTTPInternalServerError, HTTPFound, HTTPNotFound)
from pyramid.renderers import render_to_response

from rhodecode.model import meta
from rhodecode.config import patches
from rhodecode.config import utils as config_utils
from rhodecode.config.settings_maker import SettingsMaker
from rhodecode.config.environment import load_pyramid_environment

import rhodecode.events
from rhodecode.lib.middleware.vcs import VCSMiddleware
from rhodecode.lib.request import Request
from rhodecode.lib.vcs import VCSCommunicationError
from rhodecode.lib.exceptions import VCSServerUnavailable
from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
from rhodecode.lib.middleware.https_fixup import HttpsFixup
from rhodecode.lib.plugins.utils import register_rhodecode_plugin
from rhodecode.lib.utils2 import AttributeDict
from rhodecode.lib.exc_tracking import store_exception
from rhodecode.subscribers import (
    scan_repositories_if_enabled, write_js_routes_if_enabled,
    write_metadata_if_needed, write_usage_data)
from rhodecode.lib.statsd_client import StatsdClient

log = logging.getLogger(__name__)


def is_http_error(response):
    # error which should have traceback
    return response.status_code > 499


def should_load_all():
    """
    Returns if all application components should be loaded. In some cases it's
    desired to skip apps loading for faster shell script execution
    """
    ssh_cmd = os.environ.get('RC_CMD_SSH_WRAPPER')
    if ssh_cmd:
        return False

    return True


def make_pyramid_app(global_config, **settings):
    """
    Constructs the WSGI application based on Pyramid.

    Specials:

    * The application can also be integrated like a plugin via the call to
      `includeme`. This is accompanied with the other utility functions which
      are called. Changing this should be done with great care to not break
      cases when these fragments are assembled from another place.

    """
    start_time = time.time()
    log.info('Pyramid app config starting')

    sanitize_settings_and_apply_defaults(global_config, settings)

    # init and bootstrap StatsdClient
    StatsdClient.setup(settings)

    config = Configurator(settings=settings)
    # Init our statsd at very start
    config.registry.statsd = StatsdClient.statsd

    # Apply compatibility patches
    patches.inspect_getargspec()

    load_pyramid_environment(global_config, settings)

    # Static file view comes first
    includeme_first(config)

    includeme(config)

    pyramid_app = config.make_wsgi_app()
    pyramid_app = wrap_app_in_wsgi_middlewares(pyramid_app, config)
    pyramid_app.config = config

    celery_settings = get_celery_config(settings)
    config.configure_celery(celery_settings)

    # creating the app uses a connection - return it after we are done
    meta.Session.remove()

    total_time = time.time() - start_time
    log.info('Pyramid app created and configured in %.2fs', total_time)
    return pyramid_app


def get_celery_config(settings):
    """
    Converts basic ini configuration into celery 4.X options
    """

    def key_converter(key_name):
        pref = 'celery.'
        if key_name.startswith(pref):
            return key_name[len(pref):].replace('.', '_').lower()

    def type_converter(parsed_key, value):
        # cast to int
        if value.isdigit():
            return int(value)

        # cast to bool
        if value.lower() in ['true', 'false', 'True', 'False']:
            return value.lower() == 'true'
        return value

    celery_config = {}
    for k, v in settings.items():
        pref = 'celery.'
        if k.startswith(pref):
            celery_config[key_converter(k)] = type_converter(key_converter(k), v)

    # TODO:rethink if we want to support celerybeat based file config, probably NOT
    # beat_config = {}
    # for section in parser.sections():
    #     if section.startswith('celerybeat:'):
    #         name = section.split(':', 1)[1]
    #         beat_config[name] = get_beat_config(parser, section)

    # final compose of settings
    celery_settings = {}

    if celery_config:
        celery_settings.update(celery_config)
    # if beat_config:
    #     celery_settings.update({'beat_schedule': beat_config})

    return celery_settings


def not_found_view(request):
    """
    This creates the view which should be registered as not-found-view to
    pyramid.
    """

    if not getattr(request, 'vcs_call', None):
        # handle like regular case with our error_handler
        return error_handler(HTTPNotFound(), request)

    # handle not found view as a vcs call
    settings = request.registry.settings
    ae_client = getattr(request, 'ae_client', None)
    vcs_app = VCSMiddleware(
        HTTPNotFound(), request.registry, settings,
        appenlight_client=ae_client)

    return wsgiapp(vcs_app)(None, request)


def error_handler(exception, request):
    import rhodecode
    from rhodecode.lib import helpers
    from rhodecode.lib.utils2 import str2bool

    rhodecode_title = rhodecode.CONFIG.get('rhodecode_title') or 'RhodeCode'

    base_response = HTTPInternalServerError()
    # prefer original exception for the response since it may have headers set
    if isinstance(exception, HTTPException):
        base_response = exception
    elif isinstance(exception, VCSCommunicationError):
        base_response = VCSServerUnavailable()

    if is_http_error(base_response):
        log.exception(
            'error occurred handling this request for path: %s', request.path)

    error_explanation = base_response.explanation or str(base_response)
    if base_response.status_code == 404:
        error_explanation += " Optionally you don't have permission to access this page."
    c = AttributeDict()
    c.error_message = base_response.status
    c.error_explanation = error_explanation
    c.visual = AttributeDict()

    c.visual.rhodecode_support_url = (
        request.registry.settings.get('rhodecode_support_url') or
        request.route_url('rhodecode_support')
    )
    c.redirect_time = 0
    c.rhodecode_name = rhodecode_title
    if not c.rhodecode_name:
        c.rhodecode_name = 'Rhodecode'

    c.causes = []
    if is_http_error(base_response):
        c.causes.append('Server is overloaded.')
        c.causes.append('Server database connection is lost.')
        c.causes.append('Server expected unhandled error.')

    if hasattr(base_response, 'causes'):
        c.causes = base_response.causes

    c.messages = helpers.flash.pop_messages(request=request)
    exc_info = sys.exc_info()
    c.exception_id = id(exc_info)
    c.show_exception_id = isinstance(base_response, VCSServerUnavailable) \
                          or base_response.status_code > 499
    c.exception_id_url = request.route_url(
        'admin_settings_exception_tracker_show', exception_id=c.exception_id)

    debug_mode = rhodecode.ConfigGet().get_bool('debug')
    if c.show_exception_id:
        store_exception(c.exception_id, exc_info)
    c.exception_debug = debug_mode
    c.exception_config_ini = rhodecode.CONFIG.get('__file__')

    if debug_mode:
        try:
            from rich.traceback import install
            install(show_locals=True)
            log.debug('Installing rich tracebacks...')
        except ImportError:
            pass

    response = render_to_response(
        '/errors/error_document.mako', {'c': c, 'h': helpers}, request=request,
        response=base_response)

    statsd = request.registry.statsd
    if statsd and base_response.status_code > 499:
        exc_type = "{}.{}".format(exception.__class__.__module__, exception.__class__.__name__)
        statsd.incr('rhodecode_exception_total',
                    tags=["exc_source:web",
                          "http_code:{}".format(base_response.status_code),
                          "type:{}".format(exc_type)])

    return response


def includeme_first(config):
    # redirect automatic browser favicon.ico requests to correct place
    def favicon_redirect(context, request):
        return HTTPFound(
            request.static_path('rhodecode:public/images/favicon.ico'))

    config.add_view(favicon_redirect, route_name='favicon')
    config.add_route('favicon', '/favicon.ico')

    def robots_redirect(context, request):
        return HTTPFound(
            request.static_path('rhodecode:public/robots.txt'))

    config.add_view(robots_redirect, route_name='robots')
    config.add_route('robots', '/robots.txt')

    config.add_static_view(
        '_static/deform', 'deform:static')
    config.add_static_view(
        '_static/rhodecode', path='rhodecode:public', cache_max_age=3600 * 24)


ce_auth_resources = [
    'rhodecode.authentication.plugins.auth_crowd',
    'rhodecode.authentication.plugins.auth_headers',
    'rhodecode.authentication.plugins.auth_jasig_cas',
    'rhodecode.authentication.plugins.auth_ldap',
    'rhodecode.authentication.plugins.auth_pam',
    'rhodecode.authentication.plugins.auth_rhodecode',
    'rhodecode.authentication.plugins.auth_token',
]


def includeme(config, auth_resources=None):
    from rhodecode.lib.celerylib.loader import configure_celery
    log.debug('Initializing main includeme from %s', os.path.basename(__file__))
    settings = config.registry.settings
    config.set_request_factory(Request)

    # plugin information
    config.registry.rhodecode_plugins = collections.OrderedDict()

    config.add_directive(
        'register_rhodecode_plugin', register_rhodecode_plugin)

    config.add_directive('configure_celery', configure_celery)

    if settings.get('appenlight', False):
        config.include('appenlight_client.ext.pyramid_tween')

    load_all = should_load_all()

    # Includes which are required. The application would fail without them.
    config.include('pyramid_mako')
    config.include('rhodecode.lib.rc_beaker')
    config.include('rhodecode.lib.rc_cache')
    config.include('rhodecode.lib.rc_cache.archive_cache')

    config.include('rhodecode.apps._base.navigation')
    config.include('rhodecode.apps._base.subscribers')
    config.include('rhodecode.tweens')
    config.include('rhodecode.authentication')

    if load_all:

        # load CE authentication plugins

        if auth_resources:
            ce_auth_resources.extend(auth_resources)

        for resource in ce_auth_resources:
            config.include(resource)

        # Auto discover authentication plugins and include their configuration.
        if asbool(settings.get('auth_plugin.import_legacy_plugins', 'true')):
            from rhodecode.authentication import discover_legacy_plugins
            discover_legacy_plugins(config)

    # apps
    if load_all:
        log.debug('Starting config.include() calls')
        config.include('rhodecode.api.includeme')
        config.include('rhodecode.apps._base.includeme')
        config.include('rhodecode.apps._base.navigation.includeme')
        config.include('rhodecode.apps._base.subscribers.includeme')
        config.include('rhodecode.apps.hovercards.includeme')
        config.include('rhodecode.apps.ops.includeme')
        config.include('rhodecode.apps.channelstream.includeme')
        config.include('rhodecode.apps.file_store.includeme')
        config.include('rhodecode.apps.admin.includeme')
        config.include('rhodecode.apps.login.includeme')
        config.include('rhodecode.apps.home.includeme')
        config.include('rhodecode.apps.journal.includeme')

        config.include('rhodecode.apps.repository.includeme')
        config.include('rhodecode.apps.repo_group.includeme')
        config.include('rhodecode.apps.user_group.includeme')
        config.include('rhodecode.apps.search.includeme')
        config.include('rhodecode.apps.user_profile.includeme')
        config.include('rhodecode.apps.user_group_profile.includeme')
        config.include('rhodecode.apps.my_account.includeme')
        config.include('rhodecode.apps.gist.includeme')

        config.include('rhodecode.apps.svn_support.includeme')
        config.include('rhodecode.apps.ssh_support.includeme')
        config.include('rhodecode.apps.debug_style')

    if load_all:
        config.include('rhodecode.integrations.includeme')
        config.include('rhodecode.integrations.routes.includeme')

    config.add_route('rhodecode_support', 'https://rhodecode.com/help/', static=True)
    settings['default_locale_name'] = settings.get('lang', 'en')
    config.add_translation_dirs('rhodecode:i18n/')

    # Add subscribers.
    if load_all:
        log.debug('Adding subscribers....')
        config.add_subscriber(scan_repositories_if_enabled,
                              pyramid.events.ApplicationCreated)
        config.add_subscriber(write_metadata_if_needed,
                              pyramid.events.ApplicationCreated)
        config.add_subscriber(write_usage_data,
                              pyramid.events.ApplicationCreated)
        config.add_subscriber(write_js_routes_if_enabled,
                              pyramid.events.ApplicationCreated)


    # Set the default renderer for HTML templates to mako.
    config.add_mako_renderer('.html')

    config.add_renderer(
        name='json_ext',
        factory='rhodecode.lib.ext_json_renderer.pyramid_ext_json')

    config.add_renderer(
        name='string_html',
        factory='rhodecode.lib.string_renderer.html')

    # include RhodeCode plugins
    includes = aslist(settings.get('rhodecode.includes', []))
    log.debug('processing rhodecode.includes data...')
    for inc in includes:
        config.include(inc)

    # custom not found view, if our pyramid app doesn't know how to handle
    # the request pass it to potential VCS handling ap
    config.add_notfound_view(not_found_view)
    if not settings.get('debugtoolbar.enabled', False):
        # disabled debugtoolbar handle all exceptions via the error_handlers
        config.add_view(error_handler, context=Exception)

    # all errors including 403/404/50X
    config.add_view(error_handler, context=HTTPError)


def wrap_app_in_wsgi_middlewares(pyramid_app, config):
    """
    Apply outer WSGI middlewares around the application.
    """
    registry = config.registry
    settings = registry.settings

    # enable https redirects based on HTTP_X_URL_SCHEME set by proxy
    pyramid_app = HttpsFixup(pyramid_app, settings)

    pyramid_app, _ae_client = wrap_in_appenlight_if_enabled(
        pyramid_app, settings)
    registry.ae_client = _ae_client

    if settings['gzip_responses']:
        pyramid_app = make_gzip_middleware(
            pyramid_app, settings, compress_level=1)

    # this should be the outer most middleware in the wsgi stack since
    # middleware like Routes make database calls
    def pyramid_app_with_cleanup(environ, start_response):
        start = time.time()
        try:
            return pyramid_app(environ, start_response)
        finally:
            # Dispose current database session and rollback uncommitted
            # transactions.
            meta.Session.remove()

            # In a single threaded mode server, on non sqlite db we should have
            # '0 Current Checked out connections' at the end of a request,
            # if not, then something, somewhere is leaving a connection open
            pool = meta.get_engine().pool
            log.debug('sa pool status: %s', pool.status())
            total = time.time() - start
            log.debug('Request processing finalized: %.4fs', total)

    return pyramid_app_with_cleanup


def sanitize_settings_and_apply_defaults(global_config, settings):
    """
    Applies settings defaults and does all type conversion.

    We would move all settings parsing and preparation into this place, so that
    we have only one place left which deals with this part. The remaining parts
    of the application would start to rely fully on well prepared settings.

    This piece would later be split up per topic to avoid a big fat monster
    function.
    """

    global_settings_maker = SettingsMaker(global_config)
    global_settings_maker.make_setting('debug', default=False, parser='bool')
    debug_enabled = asbool(global_config.get('debug'))

    settings_maker = SettingsMaker(settings)

    settings_maker.make_setting(
        'logging.autoconfigure',
        default=False,
        parser='bool')

    logging_conf = os.path.join(os.path.dirname(global_config.get('__file__')), 'logging.ini')
    settings_maker.enable_logging(logging_conf, level='INFO' if debug_enabled else 'DEBUG')

    # Default includes, possible to change as a user
    pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
    log.debug(
        "Using the following pyramid.includes: %s",
        pyramid_includes)

    settings_maker.make_setting('rhodecode.edition', 'Community Edition')
    settings_maker.make_setting('rhodecode.edition_id', 'CE')

    if 'mako.default_filters' not in settings:
        # set custom default filters if we don't have it defined
        settings['mako.imports'] = 'from rhodecode.lib.base import h_filter'
        settings['mako.default_filters'] = 'h_filter'

    if 'mako.directories' not in settings:
        mako_directories = settings.setdefault('mako.directories', [
            # Base templates of the original application
            'rhodecode:templates',
        ])
        log.debug(
            "Using the following Mako template directories: %s",
            mako_directories)

    # NOTE(marcink): fix redis requirement for schema of connection since 3.X
    if 'beaker.session.type' in settings and settings['beaker.session.type'] == 'ext:redis':
        raw_url = settings['beaker.session.url']
        if not raw_url.startswith(('redis://', 'rediss://', 'unix://')):
            settings['beaker.session.url'] = 'redis://' + raw_url

    settings_maker.make_setting('__file__', global_config.get('__file__'))

    # TODO: johbo: Re-think this, usually the call to config.include
    # should allow to pass in a prefix.
    settings_maker.make_setting('rhodecode.api.url', '/_admin/api')

    # Sanitize generic settings.
    settings_maker.make_setting('default_encoding', 'UTF-8', parser='list')
    settings_maker.make_setting('is_test', False, parser='bool')
    settings_maker.make_setting('gzip_responses', False, parser='bool')

    # statsd
    settings_maker.make_setting('statsd.enabled', False, parser='bool')
    settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
    settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
    settings_maker.make_setting('statsd.statsd_prefix', '')
    settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')

    settings_maker.make_setting('vcs.svn.compatible_version', '')
    settings_maker.make_setting('vcs.hooks.protocol', 'http')
    settings_maker.make_setting('vcs.hooks.host', '*')
    settings_maker.make_setting('vcs.scm_app_implementation', 'http')
    settings_maker.make_setting('vcs.server', '')
    settings_maker.make_setting('vcs.server.protocol', 'http')
    settings_maker.make_setting('vcs.server.enable', 'true', parser='bool')
    settings_maker.make_setting('startup.import_repos', 'false', parser='bool')
    settings_maker.make_setting('vcs.hooks.direct_calls', 'false', parser='bool')
    settings_maker.make_setting('vcs.start_server', 'false', parser='bool')
    settings_maker.make_setting('vcs.backends', 'hg, git, svn', parser='list')
    settings_maker.make_setting('vcs.connection_timeout', 3600, parser='int')

    settings_maker.make_setting('vcs.methods.cache', True, parser='bool')

    # Support legacy values of vcs.scm_app_implementation. Legacy
    # configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or
    # disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'.
    scm_app_impl = settings['vcs.scm_app_implementation']
    if scm_app_impl in ['rhodecode.lib.middleware.utils.scm_app_http', 'vcsserver.scm_app']:
        settings['vcs.scm_app_implementation'] = 'http'

    settings_maker.make_setting('appenlight', False, parser='bool')

    temp_store = tempfile.gettempdir()
    tmp_cache_dir = os.path.join(temp_store, 'rc_cache')

    # save default, cache dir, and use it for all backends later.
    default_cache_dir = settings_maker.make_setting(
        'cache_dir',
        default=tmp_cache_dir, default_when_empty=True,
        parser='dir:ensured')

    # exception store cache
    settings_maker.make_setting(
        'exception_tracker.store_path',
        default=os.path.join(default_cache_dir, 'exc_store'), default_when_empty=True,
        parser='dir:ensured'
    )

    settings_maker.make_setting(
        'celerybeat-schedule.path',
        default=os.path.join(default_cache_dir, 'celerybeat_schedule', 'celerybeat-schedule.db'), default_when_empty=True,
        parser='file:ensured'
    )

    settings_maker.make_setting('exception_tracker.send_email', False, parser='bool')
    settings_maker.make_setting('exception_tracker.email_prefix', '[RHODECODE ERROR]', default_when_empty=True)

    # cache_general
    settings_maker.make_setting('rc_cache.cache_general.backend', 'dogpile.cache.rc.file_namespace')
    settings_maker.make_setting('rc_cache.cache_general.expiration_time', 60 * 60 * 12, parser='int')
    settings_maker.make_setting('rc_cache.cache_general.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_general.db'))

    # cache_perms
    settings_maker.make_setting('rc_cache.cache_perms.backend', 'dogpile.cache.rc.file_namespace')
    settings_maker.make_setting('rc_cache.cache_perms.expiration_time', 60 * 60, parser='int')
    settings_maker.make_setting('rc_cache.cache_perms.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_perms_db'))

    # cache_repo
    settings_maker.make_setting('rc_cache.cache_repo.backend', 'dogpile.cache.rc.file_namespace')
    settings_maker.make_setting('rc_cache.cache_repo.expiration_time', 60 * 60 * 24 * 30, parser='int')
    settings_maker.make_setting('rc_cache.cache_repo.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_repo_db'))

    # cache_license
    settings_maker.make_setting('rc_cache.cache_license.backend', 'dogpile.cache.rc.file_namespace')
    settings_maker.make_setting('rc_cache.cache_license.expiration_time', 60 * 5, parser='int')
    settings_maker.make_setting('rc_cache.cache_license.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_license_db'))

    # cache_repo_longterm memory, 96H
    settings_maker.make_setting('rc_cache.cache_repo_longterm.backend', 'dogpile.cache.rc.memory_lru')
    settings_maker.make_setting('rc_cache.cache_repo_longterm.expiration_time', 345600, parser='int')
    settings_maker.make_setting('rc_cache.cache_repo_longterm.max_size', 10000, parser='int')

    # sql_cache_short
    settings_maker.make_setting('rc_cache.sql_cache_short.backend', 'dogpile.cache.rc.memory_lru')
    settings_maker.make_setting('rc_cache.sql_cache_short.expiration_time', 30, parser='int')
    settings_maker.make_setting('rc_cache.sql_cache_short.max_size', 10000, parser='int')

    # archive_cache
    settings_maker.make_setting('archive_cache.store_dir', os.path.join(default_cache_dir, 'archive_cache'), default_when_empty=True,)
    settings_maker.make_setting('archive_cache.cache_size_gb', 10, parser='float')
    settings_maker.make_setting('archive_cache.cache_shards', 10, parser='int')

    settings_maker.env_expand()

    # configure instance id
    config_utils.set_instance_id(settings)

    return settings