rhodecode/config/config_maker.py

# Copyright (C) 2010-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/

import os
import tempfile
import logging

from pyramid.settings import asbool

from rhodecode.config.settings_maker import SettingsMaker
from rhodecode.config import utils as config_utils

log = logging.getLogger(__name__)


def sanitize_settings_and_apply_defaults(global_config, settings):
    """
    Applies settings defaults and does all type conversion.

    We would move all settings parsing and preparation into this place, so that
    we have only one place left which deals with this part. The remaining parts
    of the application would start to rely fully on well-prepared settings.

    This piece would later be split up per topic to avoid a big fat monster
    function.
    """
    jn = os.path.join

    global_settings_maker = SettingsMaker(global_config)
    global_settings_maker.make_setting('debug', default=False, parser='bool')
    debug_enabled = asbool(global_config.get('debug'))

    settings_maker = SettingsMaker(settings)

    settings_maker.make_setting(
        'logging.autoconfigure',
        default=False,
        parser='bool')

    logging_conf = jn(os.path.dirname(global_config.get('__file__')), 'logging.ini')
    settings_maker.enable_logging(logging_conf, level='INFO' if debug_enabled else 'DEBUG')

    # Default includes, possible to change as a user
    pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
    log.debug(
        "Using the following pyramid.includes: %s",
        pyramid_includes)

    settings_maker.make_setting('rhodecode.edition', 'Community Edition')
    settings_maker.make_setting('rhodecode.edition_id', 'CE')

    if 'mako.default_filters' not in settings:
        # set custom default filters if we don't have it defined
        settings['mako.imports'] = 'from rhodecode.lib.base import h_filter'
        settings['mako.default_filters'] = 'h_filter'

    if 'mako.directories' not in settings:
        mako_directories = settings.setdefault('mako.directories', [
            # Base templates of the original application
            'rhodecode:templates',
        ])
        log.debug(
            "Using the following Mako template directories: %s",
            mako_directories)

    # NOTE(marcink): fix redis requirement for schema of connection since 3.X
    if 'beaker.session.type' in settings and settings['beaker.session.type'] == 'ext:redis':
        raw_url = settings['beaker.session.url']
        if not raw_url.startswith(('redis://', 'rediss://', 'unix://')):
            settings['beaker.session.url'] = 'redis://' + raw_url

    settings_maker.make_setting('__file__', global_config.get('__file__'))

    # TODO: johbo: Re-think this, usually the call to config.include
    # should allow to pass in a prefix.
    settings_maker.make_setting('rhodecode.api.url', '/_admin/api')

    # Sanitize generic settings.
    settings_maker.make_setting('default_encoding', 'UTF-8', parser='list')
    settings_maker.make_setting('gzip_responses', False, parser='bool')
    settings_maker.make_setting('startup.import_repos', 'false', parser='bool')

    # statsd
    settings_maker.make_setting('statsd.enabled', False, parser='bool')
    settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
    settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
    settings_maker.make_setting('statsd.statsd_prefix', '')
    settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')

    settings_maker.make_setting('vcs.svn.compatible_version', '')
    settings_maker.make_setting('vcs.svn.redis_conn', 'redis://redis:6379/0')
    settings_maker.make_setting('vcs.svn.proxy.enabled', True, parser='bool')
    settings_maker.make_setting('vcs.svn.proxy.host', 'http://svn:8090', parser='string')
    settings_maker.make_setting('vcs.hooks.protocol.v2', 'celery')
    settings_maker.make_setting('vcs.hooks.host', '*')
    settings_maker.make_setting('vcs.scm_app_implementation', 'http')
    settings_maker.make_setting('vcs.server', '')
    settings_maker.make_setting('vcs.server.protocol', 'http')
    settings_maker.make_setting('vcs.server.enable', 'true', parser='bool')
    settings_maker.make_setting('vcs.hooks.direct_calls', 'false', parser='bool')
    settings_maker.make_setting('vcs.start_server', 'false', parser='bool')
    settings_maker.make_setting('vcs.backends', 'hg, git, svn', parser='list')
    settings_maker.make_setting('vcs.connection_timeout', 3600, parser='int')
    settings_maker.make_setting('vcs.git.lfs.storage_location', '/var/opt/rhodecode_repo_store/.cache/git_lfs_store')
    settings_maker.make_setting('vcs.hg.largefiles.storage_location',
                                '/var/opt/rhodecode_repo_store/.cache/hg_largefiles_store')

    settings_maker.make_setting('vcs.methods.cache', True, parser='bool')

    # repo_store path
    settings_maker.make_setting('repo_store.path', '/var/opt/rhodecode_repo_store')
    # Support legacy values of vcs.scm_app_implementation. Legacy
    # configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or
    # disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'.
    scm_app_impl = settings['vcs.scm_app_implementation']
    if scm_app_impl in ['rhodecode.lib.middleware.utils.scm_app_http', 'vcsserver.scm_app']:
        settings['vcs.scm_app_implementation'] = 'http'

    settings_maker.make_setting('appenlight', False, parser='bool')

    temp_store = tempfile.gettempdir()
    tmp_cache_dir = jn(temp_store, 'rc_cache')

    # save default, cache dir, and use it for all backends later.
    default_cache_dir = settings_maker.make_setting(
        'cache_dir',
        default=tmp_cache_dir, default_when_empty=True,
        parser='dir:ensured')

    # exception store cache
    settings_maker.make_setting(
        'exception_tracker.store_path',
        default=jn(default_cache_dir, 'exc_store'), default_when_empty=True,
        parser='dir:ensured'
    )

    settings_maker.make_setting(
        'celerybeat-schedule.path',
        default=jn(default_cache_dir, 'celerybeat_schedule', 'celerybeat-schedule.db'), default_when_empty=True,
        parser='file:ensured'
    )

    # celery
    broker_url = settings_maker.make_setting('celery.broker_url', 'redis://redis:6379/8')
    settings_maker.make_setting('celery.result_backend', broker_url)

    settings_maker.make_setting('exception_tracker.send_email', False, parser='bool')
    settings_maker.make_setting('exception_tracker.email_prefix', '[RHODECODE ERROR]', default_when_empty=True)

    # sessions, ensure file since no-value is memory
    settings_maker.make_setting('beaker.session.type', 'file')
    settings_maker.make_setting('beaker.session.data_dir',  jn(default_cache_dir, 'session_data'))

    # cache_general
    settings_maker.make_setting('rc_cache.cache_general.backend', 'dogpile.cache.rc.file_namespace')
    settings_maker.make_setting('rc_cache.cache_general.expiration_time', 60 * 60 * 12, parser='int')
    settings_maker.make_setting('rc_cache.cache_general.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_general.db'))

    # cache_perms
    settings_maker.make_setting('rc_cache.cache_perms.backend', 'dogpile.cache.rc.file_namespace')
    settings_maker.make_setting('rc_cache.cache_perms.expiration_time', 60 * 60, parser='int')
    settings_maker.make_setting('rc_cache.cache_perms.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_perms_db'))

    # cache_repo
    settings_maker.make_setting('rc_cache.cache_repo.backend', 'dogpile.cache.rc.file_namespace')
    settings_maker.make_setting('rc_cache.cache_repo.expiration_time', 60 * 60 * 24 * 30, parser='int')
    settings_maker.make_setting('rc_cache.cache_repo.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_repo_db'))

    # cache_license
    settings_maker.make_setting('rc_cache.cache_license.backend', 'dogpile.cache.rc.file_namespace')
    settings_maker.make_setting('rc_cache.cache_license.expiration_time', 60 * 5, parser='int')
    settings_maker.make_setting('rc_cache.cache_license.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_license_db'))

    # cache_repo_longterm memory, 96H
    settings_maker.make_setting('rc_cache.cache_repo_longterm.backend', 'dogpile.cache.rc.memory_lru')
    settings_maker.make_setting('rc_cache.cache_repo_longterm.expiration_time', 345600, parser='int')
    settings_maker.make_setting('rc_cache.cache_repo_longterm.max_size', 10000, parser='int')

    # sql_cache_short
    settings_maker.make_setting('rc_cache.sql_cache_short.backend', 'dogpile.cache.rc.memory_lru')
    settings_maker.make_setting('rc_cache.sql_cache_short.expiration_time', 30, parser='int')
    settings_maker.make_setting('rc_cache.sql_cache_short.max_size', 10000, parser='int')

    # archive_cache
    settings_maker.make_setting('archive_cache.locking.url', 'redis://redis:6379/1')
    settings_maker.make_setting('archive_cache.backend.type', 'filesystem')

    settings_maker.make_setting('archive_cache.filesystem.store_dir', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,)
    settings_maker.make_setting('archive_cache.filesystem.cache_shards', 8, parser='int')
    settings_maker.make_setting('archive_cache.filesystem.cache_size_gb', 10, parser='float')
    settings_maker.make_setting('archive_cache.filesystem.eviction_policy', 'least-recently-stored')

    settings_maker.make_setting('archive_cache.filesystem.retry', False, parser='bool')
    settings_maker.make_setting('archive_cache.filesystem.retry_backoff', 1, parser='int')
    settings_maker.make_setting('archive_cache.filesystem.retry_attempts', 10, parser='int')

    settings_maker.make_setting('archive_cache.objectstore.url', 'http://s3-minio:9000', default_when_empty=True,)
    settings_maker.make_setting('archive_cache.objectstore.key', '')
    settings_maker.make_setting('archive_cache.objectstore.secret', '')
    settings_maker.make_setting('archive_cache.objectstore.region', 'eu-central-1')
    settings_maker.make_setting('archive_cache.objectstore.bucket', 'rhodecode-archive-cache', default_when_empty=True,)
    settings_maker.make_setting('archive_cache.objectstore.bucket_shards', 8, parser='int')

    settings_maker.make_setting('archive_cache.objectstore.cache_size_gb', 10, parser='float')
    settings_maker.make_setting('archive_cache.objectstore.eviction_policy', 'least-recently-stored')

    settings_maker.make_setting('archive_cache.objectstore.retry', False, parser='bool')
    settings_maker.make_setting('archive_cache.objectstore.retry_backoff', 1, parser='int')
    settings_maker.make_setting('archive_cache.objectstore.retry_attempts', 10, parser='int')

    settings_maker.env_expand()

    # configure instance id
    config_utils.set_instance_id(settings)

    return settings