caches: remove settings cache to use invalidation context as it's slowing down much usage

rhodecode/config/middleware.py

@@ -1,614 +1,619 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import sys
 import collections
 import tempfile
 import time
 import logging.config
 
 from paste.gzipper import make_gzip_middleware
 import pyramid.events
 from pyramid.wsgi import wsgiapp
 from pyramid.authorization import ACLAuthorizationPolicy
 from pyramid.config import Configurator
 from pyramid.settings import asbool, aslist
 from pyramid.httpexceptions import (
     HTTPException, HTTPError, HTTPInternalServerError, HTTPFound, HTTPNotFound)
 from pyramid.renderers import render_to_response
 
 from rhodecode.model import meta
 from rhodecode.config import patches
 from rhodecode.config import utils as config_utils
 from rhodecode.config.settings_maker import SettingsMaker
 from rhodecode.config.environment import load_pyramid_environment
 
 import rhodecode.events
 from rhodecode.lib.middleware.vcs import VCSMiddleware
 from rhodecode.lib.request import Request
 from rhodecode.lib.vcs import VCSCommunicationError
 from rhodecode.lib.exceptions import VCSServerUnavailable
 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
 from rhodecode.lib.middleware.https_fixup import HttpsFixup
 from rhodecode.lib.plugins.utils import register_rhodecode_plugin
 from rhodecode.lib.utils2 import AttributeDict
 from rhodecode.lib.exc_tracking import store_exception
 from rhodecode.subscribers import (
     scan_repositories_if_enabled, write_js_routes_if_enabled,
     write_metadata_if_needed, write_usage_data)
 from rhodecode.lib.statsd_client import StatsdClient
 
 log = logging.getLogger(__name__)
 
 
 def is_http_error(response):
     # error which should have traceback
     return response.status_code > 499
 
 
 def should_load_all():
     """
     Returns if all application components should be loaded. In some cases it's
     desired to skip apps loading for faster shell script execution
     """
     ssh_cmd = os.environ.get('RC_CMD_SSH_WRAPPER')
     if ssh_cmd:
         return False
 
     return True
 
 
 def make_pyramid_app(global_config, **settings):
     """
     Constructs the WSGI application based on Pyramid.
 
     Specials:
 
     * The application can also be integrated like a plugin via the call to
       `includeme`. This is accompanied with the other utility functions which
       are called. Changing this should be done with great care to not break
       cases when these fragments are assembled from another place.
 
     """
     start_time = time.time()
     log.info('Pyramid app config starting')
 
     sanitize_settings_and_apply_defaults(global_config, settings)
 
     # init and bootstrap StatsdClient
     StatsdClient.setup(settings)
 
     config = Configurator(settings=settings)
     # Init our statsd at very start
     config.registry.statsd = StatsdClient.statsd
 
     # Apply compatibility patches
     patches.inspect_getargspec()
 
     load_pyramid_environment(global_config, settings)
 
     # Static file view comes first
     includeme_first(config)
 
     includeme(config)
 
     pyramid_app = config.make_wsgi_app()
     pyramid_app = wrap_app_in_wsgi_middlewares(pyramid_app, config)
     pyramid_app.config = config
 
     celery_settings = get_celery_config(settings)
     config.configure_celery(celery_settings)
 
     # creating the app uses a connection - return it after we are done
     meta.Session.remove()
 
     total_time = time.time() - start_time
     log.info('Pyramid app `%s` created and configured in %.2fs',
              getattr(pyramid_app, 'func_name', 'pyramid_app'), total_time)
     return pyramid_app
 
 
 def get_celery_config(settings):
     """
     Converts basic ini configuration into celery 4.X options
     """
 
     def key_converter(key_name):
         pref = 'celery.'
         if key_name.startswith(pref):
             return key_name[len(pref):].replace('.', '_').lower()
 
     def type_converter(parsed_key, value):
         # cast to int
         if value.isdigit():
             return int(value)
 
         # cast to bool
         if value.lower() in ['true', 'false', 'True', 'False']:
             return value.lower() == 'true'
         return value
 
     celery_config = {}
     for k, v in settings.items():
         pref = 'celery.'
         if k.startswith(pref):
             celery_config[key_converter(k)] = type_converter(key_converter(k), v)
 
     # TODO:rethink if we want to support celerybeat based file config, probably NOT
     # beat_config = {}
     # for section in parser.sections():
     #     if section.startswith('celerybeat:'):
     #         name = section.split(':', 1)[1]
     #         beat_config[name] = get_beat_config(parser, section)
 
     # final compose of settings
     celery_settings = {}
 
     if celery_config:
         celery_settings.update(celery_config)
     # if beat_config:
     #     celery_settings.update({'beat_schedule': beat_config})
 
     return celery_settings
 
 
 def not_found_view(request):
     """
     This creates the view which should be registered as not-found-view to
     pyramid.
     """
 
     if not getattr(request, 'vcs_call', None):
         # handle like regular case with our error_handler
         return error_handler(HTTPNotFound(), request)
 
     # handle not found view as a vcs call
     settings = request.registry.settings
     ae_client = getattr(request, 'ae_client', None)
     vcs_app = VCSMiddleware(
         HTTPNotFound(), request.registry, settings,
         appenlight_client=ae_client)
 
     return wsgiapp(vcs_app)(None, request)
 
 
 def error_handler(exception, request):
     import rhodecode
     from rhodecode.lib import helpers
     from rhodecode.lib.utils2 import str2bool
 
     rhodecode_title = rhodecode.CONFIG.get('rhodecode_title') or 'RhodeCode'
 
     base_response = HTTPInternalServerError()
     # prefer original exception for the response since it may have headers set
     if isinstance(exception, HTTPException):
         base_response = exception
     elif isinstance(exception, VCSCommunicationError):
         base_response = VCSServerUnavailable()
 
     if is_http_error(base_response):
         log.exception(
             'error occurred handling this request for path: %s', request.path)
 
     error_explanation = base_response.explanation or str(base_response)
     if base_response.status_code == 404:
         error_explanation += " Optionally you don't have permission to access this page."
     c = AttributeDict()
     c.error_message = base_response.status
     c.error_explanation = error_explanation
     c.visual = AttributeDict()
 
     c.visual.rhodecode_support_url = (
         request.registry.settings.get('rhodecode_support_url') or
         request.route_url('rhodecode_support')
     )
     c.redirect_time = 0
     c.rhodecode_name = rhodecode_title
     if not c.rhodecode_name:
         c.rhodecode_name = 'Rhodecode'
 
     c.causes = []
     if is_http_error(base_response):
         c.causes.append('Server is overloaded.')
         c.causes.append('Server database connection is lost.')
         c.causes.append('Server expected unhandled error.')
 
     if hasattr(base_response, 'causes'):
         c.causes = base_response.causes
 
     c.messages = helpers.flash.pop_messages(request=request)
 
     exc_info = sys.exc_info()
     c.exception_id = id(exc_info)
     c.show_exception_id = isinstance(base_response, VCSServerUnavailable) \
                           or base_response.status_code > 499
     c.exception_id_url = request.route_url(
         'admin_settings_exception_tracker_show', exception_id=c.exception_id)
 
     if c.show_exception_id:
         store_exception(c.exception_id, exc_info)
     c.exception_debug = str2bool(rhodecode.CONFIG.get('debug'))
     c.exception_config_ini = rhodecode.CONFIG.get('__file__')
 
     response = render_to_response(
         '/errors/error_document.mako', {'c': c, 'h': helpers}, request=request,
         response=base_response)
 
     statsd = request.registry.statsd
     if statsd and base_response.status_code > 499:
         exc_type = "{}.{}".format(exception.__class__.__module__, exception.__class__.__name__)
         statsd.incr('rhodecode_exception_total',
                     tags=["exc_source:web",
                           "http_code:{}".format(base_response.status_code),
                           "type:{}".format(exc_type)])
 
     return response
 
 
 def includeme_first(config):
     # redirect automatic browser favicon.ico requests to correct place
     def favicon_redirect(context, request):
         return HTTPFound(
             request.static_path('rhodecode:public/images/favicon.ico'))
 
     config.add_view(favicon_redirect, route_name='favicon')
     config.add_route('favicon', '/favicon.ico')
 
     def robots_redirect(context, request):
         return HTTPFound(
             request.static_path('rhodecode:public/robots.txt'))
 
     config.add_view(robots_redirect, route_name='robots')
     config.add_route('robots', '/robots.txt')
 
     config.add_static_view(
         '_static/deform', 'deform:static')
     config.add_static_view(
         '_static/rhodecode', path='rhodecode:public', cache_max_age=3600 * 24)
 
 
 def includeme(config, auth_resources=None):
     from rhodecode.lib.celerylib.loader import configure_celery
     log.debug('Initializing main includeme from %s', os.path.basename(__file__))
     settings = config.registry.settings
     config.set_request_factory(Request)
 
     # plugin information
     config.registry.rhodecode_plugins = collections.OrderedDict()
 
     config.add_directive(
         'register_rhodecode_plugin', register_rhodecode_plugin)
 
     config.add_directive('configure_celery', configure_celery)
 
     if settings.get('appenlight', False):
         config.include('appenlight_client.ext.pyramid_tween')
 
     load_all = should_load_all()
 
     # Includes which are required. The application would fail without them.
     config.include('pyramid_mako')
     config.include('rhodecode.lib.rc_beaker')
     config.include('rhodecode.lib.rc_cache')
     config.include('rhodecode.apps._base.navigation')
     config.include('rhodecode.apps._base.subscribers')
     config.include('rhodecode.tweens')
     config.include('rhodecode.authentication')
 
     if load_all:
         ce_auth_resources = [
             'rhodecode.authentication.plugins.auth_crowd',
             'rhodecode.authentication.plugins.auth_headers',
             'rhodecode.authentication.plugins.auth_jasig_cas',
             'rhodecode.authentication.plugins.auth_ldap',
             'rhodecode.authentication.plugins.auth_pam',
             'rhodecode.authentication.plugins.auth_rhodecode',
             'rhodecode.authentication.plugins.auth_token',
         ]
 
         # load CE authentication plugins
 
         if auth_resources:
             ce_auth_resources.extend(auth_resources)
 
         for resource in ce_auth_resources:
             config.include(resource)
 
         # Auto discover authentication plugins and include their configuration.
         if asbool(settings.get('auth_plugin.import_legacy_plugins', 'true')):
             from rhodecode.authentication import discover_legacy_plugins
             discover_legacy_plugins(config)
 
     # apps
     if load_all:
         config.include('rhodecode.api')
         config.include('rhodecode.apps._base')
         config.include('rhodecode.apps.hovercards')
         config.include('rhodecode.apps.ops')
         config.include('rhodecode.apps.channelstream')
         config.include('rhodecode.apps.file_store')
         config.include('rhodecode.apps.admin')
         config.include('rhodecode.apps.login')
         config.include('rhodecode.apps.home')
         config.include('rhodecode.apps.journal')
 
         config.include('rhodecode.apps.repository')
         config.include('rhodecode.apps.repo_group')
         config.include('rhodecode.apps.user_group')
         config.include('rhodecode.apps.search')
         config.include('rhodecode.apps.user_profile')
         config.include('rhodecode.apps.user_group_profile')
         config.include('rhodecode.apps.my_account')
         config.include('rhodecode.apps.gist')
 
         config.include('rhodecode.apps.svn_support')
         config.include('rhodecode.apps.ssh_support')
         config.include('rhodecode.apps.debug_style')
 
     if load_all:
         config.include('rhodecode.integrations')
 
     config.add_route('rhodecode_support', 'https://rhodecode.com/help/', static=True)
     config.add_translation_dirs('rhodecode:i18n/')
     settings['default_locale_name'] = settings.get('lang', 'en')
 
     # Add subscribers.
     if load_all:
         config.add_subscriber(scan_repositories_if_enabled,
                               pyramid.events.ApplicationCreated)
         config.add_subscriber(write_metadata_if_needed,
                               pyramid.events.ApplicationCreated)
         config.add_subscriber(write_usage_data,
                               pyramid.events.ApplicationCreated)
         config.add_subscriber(write_js_routes_if_enabled,
                               pyramid.events.ApplicationCreated)
 
     # request custom methods
     config.add_request_method(
         'rhodecode.lib.partial_renderer.get_partial_renderer',
         'get_partial_renderer')
 
     config.add_request_method(
         'rhodecode.lib.request_counter.get_request_counter',
         'request_count')
 
     # Set the authorization policy.
     authz_policy = ACLAuthorizationPolicy()
     config.set_authorization_policy(authz_policy)
 
     # Set the default renderer for HTML templates to mako.
     config.add_mako_renderer('.html')
 
     config.add_renderer(
         name='json_ext',
         factory='rhodecode.lib.ext_json_renderer.pyramid_ext_json')
 
     config.add_renderer(
         name='string_html',
         factory='rhodecode.lib.string_renderer.html')
 
     # include RhodeCode plugins
     includes = aslist(settings.get('rhodecode.includes', []))
     for inc in includes:
         config.include(inc)
 
     # custom not found view, if our pyramid app doesn't know how to handle
     # the request pass it to potential VCS handling ap
     config.add_notfound_view(not_found_view)
     if not settings.get('debugtoolbar.enabled', False):
         # disabled debugtoolbar handle all exceptions via the error_handlers
         config.add_view(error_handler, context=Exception)
 
     # all errors including 403/404/50X
     config.add_view(error_handler, context=HTTPError)
 
 
 def wrap_app_in_wsgi_middlewares(pyramid_app, config):
     """
     Apply outer WSGI middlewares around the application.
     """
     registry = config.registry
     settings = registry.settings
 
     # enable https redirects based on HTTP_X_URL_SCHEME set by proxy
     pyramid_app = HttpsFixup(pyramid_app, settings)
 
     pyramid_app, _ae_client = wrap_in_appenlight_if_enabled(
         pyramid_app, settings)
     registry.ae_client = _ae_client
 
     if settings['gzip_responses']:
         pyramid_app = make_gzip_middleware(
             pyramid_app, settings, compress_level=1)
 
     # this should be the outer most middleware in the wsgi stack since
     # middleware like Routes make database calls
     def pyramid_app_with_cleanup(environ, start_response):
         try:
             return pyramid_app(environ, start_response)
         finally:
             # Dispose current database session and rollback uncommitted
             # transactions.
             meta.Session.remove()
 
             # In a single threaded mode server, on non sqlite db we should have
             # '0 Current Checked out connections' at the end of a request,
             # if not, then something, somewhere is leaving a connection open
             pool = meta.Base.metadata.bind.engine.pool
             log.debug('sa pool status: %s', pool.status())
             log.debug('Request processing finalized')
 
     return pyramid_app_with_cleanup
 
 
 def sanitize_settings_and_apply_defaults(global_config, settings):
     """
     Applies settings defaults and does all type conversion.
 
     We would move all settings parsing and preparation into this place, so that
     we have only one place left which deals with this part. The remaining parts
     of the application would start to rely fully on well prepared settings.
 
     This piece would later be split up per topic to avoid a big fat monster
     function.
     """
 
     global_settings_maker = SettingsMaker(global_config)
     global_settings_maker.make_setting('debug', default=False, parser='bool')
     debug_enabled = asbool(global_config.get('debug'))
 
     settings_maker = SettingsMaker(settings)
 
     settings_maker.make_setting(
         'logging.autoconfigure',
         default=True,
         parser='bool')
 
     logging_conf = os.path.join(os.path.dirname(global_config.get('__file__')), 'logging.ini')
     settings_maker.enable_logging(logging_conf, level='INFO' if debug_enabled else 'DEBUG')
 
     # Default includes, possible to change as a user
     pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
     log.debug(
         "Using the following pyramid.includes: %s",
         pyramid_includes)
 
     settings_maker.make_setting('rhodecode.edition', 'Community Edition')
     settings_maker.make_setting('rhodecode.edition_id', 'CE')
 
     if 'mako.default_filters' not in settings:
         # set custom default filters if we don't have it defined
         settings['mako.imports'] = 'from rhodecode.lib.base import h_filter'
         settings['mako.default_filters'] = 'h_filter'
 
     if 'mako.directories' not in settings:
         mako_directories = settings.setdefault('mako.directories', [
             # Base templates of the original application
             'rhodecode:templates',
         ])
         log.debug(
             "Using the following Mako template directories: %s",
             mako_directories)
 
     # NOTE(marcink): fix redis requirement for schema of connection since 3.X
     if 'beaker.session.type' in settings and settings['beaker.session.type'] == 'ext:redis':
         raw_url = settings['beaker.session.url']
         if not raw_url.startswith(('redis://', 'rediss://', 'unix://')):
             settings['beaker.session.url'] = 'redis://' + raw_url
 
     settings_maker.make_setting('__file__', global_config.get('__file__'))
 
     # TODO: johbo: Re-think this, usually the call to config.include
     # should allow to pass in a prefix.
     settings_maker.make_setting('rhodecode.api.url', '/_admin/api')
 
     # Sanitize generic settings.
     settings_maker.make_setting('default_encoding', 'UTF-8', parser='list')
     settings_maker.make_setting('is_test', False, parser='bool')
     settings_maker.make_setting('gzip_responses', False, parser='bool')
 
     # statsd
     settings_maker.make_setting('statsd.enabled', False, parser='bool')
     settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
     settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
     settings_maker.make_setting('statsd.statsd_prefix', '')
     settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
 
     settings_maker.make_setting('vcs.svn.compatible_version', '')
     settings_maker.make_setting('vcs.hooks.protocol', 'http')
     settings_maker.make_setting('vcs.hooks.host', '127.0.0.1')
     settings_maker.make_setting('vcs.scm_app_implementation', 'http')
     settings_maker.make_setting('vcs.server', '')
     settings_maker.make_setting('vcs.server.protocol', 'http')
     settings_maker.make_setting('startup.import_repos', 'false', parser='bool')
     settings_maker.make_setting('vcs.hooks.direct_calls', 'false', parser='bool')
     settings_maker.make_setting('vcs.server.enable', 'true', parser='bool')
     settings_maker.make_setting('vcs.start_server', 'false', parser='bool')
     settings_maker.make_setting('vcs.backends', 'hg, git, svn', parser='list')
     settings_maker.make_setting('vcs.connection_timeout', 3600, parser='int')
 
     settings_maker.make_setting('vcs.methods.cache', True, parser='bool')
 
     # Support legacy values of vcs.scm_app_implementation. Legacy
     # configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or
     # disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'.
     scm_app_impl = settings['vcs.scm_app_implementation']
     if scm_app_impl in ['rhodecode.lib.middleware.utils.scm_app_http', 'vcsserver.scm_app']:
         settings['vcs.scm_app_implementation'] = 'http'
 
     settings_maker.make_setting('appenlight', False, parser='bool')
 
     temp_store = tempfile.gettempdir()
     default_cache_dir = os.path.join(temp_store, 'rc_cache')
 
     # save default, cache dir, and use it for all backends later.
     default_cache_dir = settings_maker.make_setting(
         'cache_dir',
         default=default_cache_dir, default_when_empty=True,
         parser='dir:ensured')
 
     # exception store cache
     settings_maker.make_setting(
         'exception_tracker.store_path',
         default=os.path.join(default_cache_dir, 'exc_store'), default_when_empty=True,
         parser='dir:ensured'
     )
 
     settings_maker.make_setting(
         'celerybeat-schedule.path',
         default=os.path.join(default_cache_dir, 'celerybeat_schedule', 'celerybeat-schedule.db'), default_when_empty=True,
         parser='file:ensured'
     )
 
     settings_maker.make_setting('exception_tracker.send_email', False, parser='bool')
     settings_maker.make_setting('exception_tracker.email_prefix', '[RHODECODE ERROR]', default_when_empty=True)
 
+    # cache_general
+    settings_maker.make_setting('rc_cache.cache_general.backend', 'dogpile.cache.rc.file_namespace')
+    settings_maker.make_setting('rc_cache.cache_general.expiration_time', 60 * 60 * 12, parser='int')
+    settings_maker.make_setting('rc_cache.cache_general.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_general.db'))
+
     # cache_perms
     settings_maker.make_setting('rc_cache.cache_perms.backend', 'dogpile.cache.rc.file_namespace')
-    settings_maker.make_setting('rc_cache.cache_perms.expiration_time', 60, parser='int')
+    settings_maker.make_setting('rc_cache.cache_perms.expiration_time', 60 * 60, parser='int')
     settings_maker.make_setting('rc_cache.cache_perms.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_perms.db'))
 
     # cache_repo
     settings_maker.make_setting('rc_cache.cache_repo.backend', 'dogpile.cache.rc.file_namespace')
-    settings_maker.make_setting('rc_cache.cache_repo.expiration_time', 60, parser='int')
+    settings_maker.make_setting('rc_cache.cache_repo.expiration_time', 60 * 60 * 24 * 30, parser='int')
     settings_maker.make_setting('rc_cache.cache_repo.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_repo.db'))
 
     # cache_license
     settings_maker.make_setting('rc_cache.cache_license.backend', 'dogpile.cache.rc.file_namespace')
-    settings_maker.make_setting('rc_cache.cache_license.expiration_time', 5*60, parser='int')
+    settings_maker.make_setting('rc_cache.cache_license.expiration_time', 60 * 5, parser='int')
     settings_maker.make_setting('rc_cache.cache_license.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_license.db'))
 
     # cache_repo_longterm memory, 96H
     settings_maker.make_setting('rc_cache.cache_repo_longterm.backend', 'dogpile.cache.rc.memory_lru')
     settings_maker.make_setting('rc_cache.cache_repo_longterm.expiration_time', 345600, parser='int')
     settings_maker.make_setting('rc_cache.cache_repo_longterm.max_size', 10000, parser='int')
 
     # sql_cache_short
     settings_maker.make_setting('rc_cache.sql_cache_short.backend', 'dogpile.cache.rc.memory_lru')
     settings_maker.make_setting('rc_cache.sql_cache_short.expiration_time', 30, parser='int')
     settings_maker.make_setting('rc_cache.sql_cache_short.max_size', 10000, parser='int')
 
     settings_maker.env_expand()
 
     # configure instance id
     config_utils.set_instance_id(settings)
 
     return settings

rhodecode/model/settings.py

@@ -1,920 +1,918 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import hashlib
 import logging
 import re
 from collections import namedtuple
 from functools import wraps
 import bleach
 from pyramid.threadlocal import get_current_request, get_current_registry
 
 from rhodecode.lib import rc_cache
 from rhodecode.lib.utils2 import (
     Optional, AttributeDict, safe_str, remove_prefix, str2bool)
 from rhodecode.lib.vcs.backends import base
+from rhodecode.lib.statsd_client import StatsdClient
 from rhodecode.model import BaseModel
 from rhodecode.model.db import (
     RepoRhodeCodeUi, RepoRhodeCodeSetting, RhodeCodeUi, RhodeCodeSetting, CacheKey)
 from rhodecode.model.meta import Session
 
 
 log = logging.getLogger(__name__)
 
 
 UiSetting = namedtuple(
     'UiSetting', ['section', 'key', 'value', 'active'])
 
 SOCIAL_PLUGINS_LIST = ['github', 'bitbucket', 'twitter', 'google']
 
 
 class SettingNotFound(Exception):
     def __init__(self, setting_id):
         msg = 'Setting `{}` is not found'.format(setting_id)
         super(SettingNotFound, self).__init__(msg)
 
 
 class SettingsModel(BaseModel):
     BUILTIN_HOOKS = (
         RhodeCodeUi.HOOK_REPO_SIZE, RhodeCodeUi.HOOK_PUSH,
         RhodeCodeUi.HOOK_PRE_PUSH, RhodeCodeUi.HOOK_PRETX_PUSH,
         RhodeCodeUi.HOOK_PULL, RhodeCodeUi.HOOK_PRE_PULL,
         RhodeCodeUi.HOOK_PUSH_KEY,)
     HOOKS_SECTION = 'hooks'
 
     def __init__(self, sa=None, repo=None):
         self.repo = repo
         self.UiDbModel = RepoRhodeCodeUi if repo else RhodeCodeUi
         self.SettingsDbModel = (
             RepoRhodeCodeSetting if repo else RhodeCodeSetting)
         super(SettingsModel, self).__init__(sa)
 
     def get_ui_by_key(self, key):
         q = self.UiDbModel.query()
         q = q.filter(self.UiDbModel.ui_key == key)
         q = self._filter_by_repo(RepoRhodeCodeUi, q)
         return q.scalar()
 
     def get_ui_by_section(self, section):
         q = self.UiDbModel.query()
         q = q.filter(self.UiDbModel.ui_section == section)
         q = self._filter_by_repo(RepoRhodeCodeUi, q)
         return q.all()
 
     def get_ui_by_section_and_key(self, section, key):
         q = self.UiDbModel.query()
         q = q.filter(self.UiDbModel.ui_section == section)
         q = q.filter(self.UiDbModel.ui_key == key)
         q = self._filter_by_repo(RepoRhodeCodeUi, q)
         return q.scalar()
 
     def get_ui(self, section=None, key=None):
         q = self.UiDbModel.query()
         q = self._filter_by_repo(RepoRhodeCodeUi, q)
 
         if section:
             q = q.filter(self.UiDbModel.ui_section == section)
         if key:
             q = q.filter(self.UiDbModel.ui_key == key)
 
         # TODO: mikhail: add caching
         result = [
             UiSetting(
                 section=safe_str(r.ui_section), key=safe_str(r.ui_key),
                 value=safe_str(r.ui_value), active=r.ui_active
             )
             for r in q.all()
         ]
         return result
 
     def get_builtin_hooks(self):
         q = self.UiDbModel.query()
         q = q.filter(self.UiDbModel.ui_key.in_(self.BUILTIN_HOOKS))
         return self._get_hooks(q)
 
     def get_custom_hooks(self):
         q = self.UiDbModel.query()
         q = q.filter(~self.UiDbModel.ui_key.in_(self.BUILTIN_HOOKS))
         return self._get_hooks(q)
 
     def create_ui_section_value(self, section, val, key=None, active=True):
         new_ui = self.UiDbModel()
         new_ui.ui_section = section
         new_ui.ui_value = val
         new_ui.ui_active = active
 
         repository_id = ''
         if self.repo:
             repo = self._get_repo(self.repo)
             repository_id = repo.repo_id
             new_ui.repository_id = repository_id
 
         if not key:
             # keys are unique so they need appended info
             if self.repo:
                 key = hashlib.sha1(
                     '{}{}{}'.format(section, val, repository_id)).hexdigest()
             else:
                 key = hashlib.sha1('{}{}'.format(section, val)).hexdigest()
 
         new_ui.ui_key = key
 
         Session().add(new_ui)
         return new_ui
 
     def create_or_update_hook(self, key, value):
         ui = (
             self.get_ui_by_section_and_key(self.HOOKS_SECTION, key) or
             self.UiDbModel())
         ui.ui_section = self.HOOKS_SECTION
         ui.ui_active = True
         ui.ui_key = key
         ui.ui_value = value
 
         if self.repo:
             repo = self._get_repo(self.repo)
             repository_id = repo.repo_id
             ui.repository_id = repository_id
 
         Session().add(ui)
         return ui
 
     def delete_ui(self, id_):
         ui = self.UiDbModel.get(id_)
         if not ui:
             raise SettingNotFound(id_)
         Session().delete(ui)
 
     def get_setting_by_name(self, name):
         q = self._get_settings_query()
         q = q.filter(self.SettingsDbModel.app_settings_name == name)
         return q.scalar()
 
     def create_or_update_setting(
             self, name, val=Optional(''), type_=Optional('unicode')):
         """
         Creates or updates RhodeCode setting. If updates is triggered it will
         only update parameters that are explicityl set Optional instance will
         be skipped
 
         :param name:
         :param val:
         :param type_:
         :return:
         """
 
         res = self.get_setting_by_name(name)
         repo = self._get_repo(self.repo) if self.repo else None
 
         if not res:
             val = Optional.extract(val)
             type_ = Optional.extract(type_)
 
             args = (
                 (repo.repo_id, name, val, type_)
                 if repo else (name, val, type_))
             res = self.SettingsDbModel(*args)
 
         else:
             if self.repo:
                 res.repository_id = repo.repo_id
 
             res.app_settings_name = name
             if not isinstance(type_, Optional):
                 # update if set
                 res.app_settings_type = type_
             if not isinstance(val, Optional):
                 # update if set
                 res.app_settings_value = val
 
         Session().add(res)
         return res
 
+    def get_cache_region(self):
+        repo = self._get_repo(self.repo) if self.repo else None
+        cache_key = "repo.{}".format(repo.repo_id) if repo else "general_settings"
+        cache_namespace_uid = 'cache_settings.{}'.format(cache_key)
+        region = rc_cache.get_or_create_region('cache_general', cache_namespace_uid)
+        return region, cache_key
+
     def invalidate_settings_cache(self):
-        invalidation_namespace = CacheKey.SETTINGS_INVALIDATION_NAMESPACE
-        CacheKey.set_invalidate(invalidation_namespace)
+        from rhodecode.authentication.base import get_authn_registry
+
+        region, cache_key = self.get_cache_region()
+        log.debug('Invalidation cache region %s for cache_key: %s', region, cache_key)
+        region.invalidate()
+        registry = get_current_registry()
+        if registry:
+            authn_registry = get_authn_registry(registry)
+            if authn_registry:
+                authn_registry.invalidate_plugins_for_auth()
 
     def get_all_settings(self, cache=False, from_request=True):
-        from rhodecode.authentication.base import get_authn_registry
-
         # defines if we use GLOBAL, or PER_REPO
         repo = self._get_repo(self.repo) if self.repo else None
-        key = "settings_repo.{}".format(repo.repo_id) if repo else "settings_app"
 
         # initially try the requests context, this is the fastest
         # we only fetch global config
         if from_request:
             request = get_current_request()
 
             if request and not repo and hasattr(request, 'call_context') and hasattr(request.call_context, 'rc_config'):
                 rc_config = request.call_context.rc_config
                 if rc_config:
                     return rc_config
 
-        region = rc_cache.get_or_create_region('sql_cache_short')
-        invalidation_namespace = CacheKey.SETTINGS_INVALIDATION_NAMESPACE
+        region, cache_key = self.get_cache_region()
 
         @region.conditional_cache_on_arguments(condition=cache)
         def _get_all_settings(name, key):
             q = self._get_settings_query()
             if not q:
                 raise Exception('Could not get application settings !')
 
             settings = {
-                'rhodecode_' + result.app_settings_name: result.app_settings_value
-                for result in q
+                'rhodecode_' + res.app_settings_name: res.app_settings_value
+                for res in q
             }
             return settings
 
-        inv_context_manager = rc_cache.InvalidationContext(
-            uid='cache_settings', invalidation_namespace=invalidation_namespace)
-        with inv_context_manager as invalidation_context:
-            # check for stored invalidation signal, and maybe purge the cache
-            # before computing it again
-            if invalidation_context.should_invalidate():
-                # NOTE:(marcink) we flush the whole sql_cache_short region, because it
-                # reads different settings etc. It's little too much but those caches
-                # are anyway very short lived and it's a safest way.
-                region = rc_cache.get_or_create_region('sql_cache_short')
-                region.invalidate()
-                registry = get_current_registry()
-                if registry:
-                    authn_registry = get_authn_registry(registry)
-                    if authn_registry:
-                        authn_registry.invalidate_plugins_for_auth()
+        statsd = StatsdClient.statsd
+        with statsd.timer('rhodecode_settings_timing.histogram', auto_send=True) as tmr:
+            result = _get_all_settings('rhodecode_settings', cache_key)
 
-            result = _get_all_settings('rhodecode_settings', key)
-            log.debug('Fetching app settings for key: %s took: %.4fs', key,
-                      inv_context_manager.compute_time)
+        compute_time = tmr.ms / 1000.
+        log.debug('Fetching app settings for key: %s took: %.4fs', cache_key, compute_time)
 
         return result
 
     def get_auth_settings(self):
         q = self._get_settings_query()
         q = q.filter(
             self.SettingsDbModel.app_settings_name.startswith('auth_'))
         rows = q.all()
         auth_settings = {
             row.app_settings_name: row.app_settings_value for row in rows}
         return auth_settings
 
     def get_auth_plugins(self):
         auth_plugins = self.get_setting_by_name("auth_plugins")
         return auth_plugins.app_settings_value
 
     def get_default_repo_settings(self, strip_prefix=False):
         q = self._get_settings_query()
         q = q.filter(
             self.SettingsDbModel.app_settings_name.startswith('default_'))
         rows = q.all()
 
         result = {}
         for row in rows:
             key = row.app_settings_name
             if strip_prefix:
                 key = remove_prefix(key, prefix='default_')
             result.update({key: row.app_settings_value})
         return result
 
     def get_repo(self):
         repo = self._get_repo(self.repo)
         if not repo:
             raise Exception(
                 'Repository `{}` cannot be found inside the database'.format(
                     self.repo))
         return repo
 
     def _filter_by_repo(self, model, query):
         if self.repo:
             repo = self.get_repo()
             query = query.filter(model.repository_id == repo.repo_id)
         return query
 
     def _get_hooks(self, query):
         query = query.filter(self.UiDbModel.ui_section == self.HOOKS_SECTION)
         query = self._filter_by_repo(RepoRhodeCodeUi, query)
         return query.all()
 
     def _get_settings_query(self):
         q = self.SettingsDbModel.query()
         return self._filter_by_repo(RepoRhodeCodeSetting, q)
 
     def list_enabled_social_plugins(self, settings):
         enabled = []
         for plug in SOCIAL_PLUGINS_LIST:
             if str2bool(settings.get('rhodecode_auth_{}_enabled'.format(plug)
                                      )):
                 enabled.append(plug)
         return enabled
 
 
 def assert_repo_settings(func):
     @wraps(func)
     def _wrapper(self, *args, **kwargs):
         if not self.repo_settings:
             raise Exception('Repository is not specified')
         return func(self, *args, **kwargs)
     return _wrapper
 
 
 class IssueTrackerSettingsModel(object):
     INHERIT_SETTINGS = 'inherit_issue_tracker_settings'
     SETTINGS_PREFIX = 'issuetracker_'
 
     def __init__(self, sa=None, repo=None):
         self.global_settings = SettingsModel(sa=sa)
         self.repo_settings = SettingsModel(sa=sa, repo=repo) if repo else None
 
     @property
     def inherit_global_settings(self):
         if not self.repo_settings:
             return True
         setting = self.repo_settings.get_setting_by_name(self.INHERIT_SETTINGS)
         return setting.app_settings_value if setting else True
 
     @inherit_global_settings.setter
     def inherit_global_settings(self, value):
         if self.repo_settings:
             settings = self.repo_settings.create_or_update_setting(
                 self.INHERIT_SETTINGS, value, type_='bool')
             Session().add(settings)
 
     def _get_keyname(self, key, uid, prefix=''):
         return '{0}{1}{2}_{3}'.format(
             prefix, self.SETTINGS_PREFIX, key, uid)
 
     def _make_dict_for_settings(self, qs):
         prefix_match = self._get_keyname('pat', '', 'rhodecode_')
 
         issuetracker_entries = {}
         # create keys
         for k, v in qs.items():
             if k.startswith(prefix_match):
                 uid = k[len(prefix_match):]
                 issuetracker_entries[uid] = None
 
         def url_cleaner(input_str):
             input_str = input_str.replace('"', '').replace("'", '')
             input_str = bleach.clean(input_str, strip=True)
             return input_str
 
         # populate
         for uid in issuetracker_entries:
             url_data = qs.get(self._get_keyname('url', uid, 'rhodecode_'))
 
             pat = qs.get(self._get_keyname('pat', uid, 'rhodecode_'))
             try:
                 pat_compiled = re.compile(r'%s' % pat)
             except re.error:
                 pat_compiled = None
 
             issuetracker_entries[uid] = AttributeDict({
                 'pat': pat,
                 'pat_compiled': pat_compiled,
                 'url': url_cleaner(
                     qs.get(self._get_keyname('url', uid, 'rhodecode_')) or ''),
                 'pref': bleach.clean(
                     qs.get(self._get_keyname('pref', uid, 'rhodecode_')) or ''),
                 'desc': qs.get(
                     self._get_keyname('desc', uid, 'rhodecode_')),
             })
 
         return issuetracker_entries
 
     def get_global_settings(self, cache=False):
         """
         Returns list of global issue tracker settings
         """
         defaults = self.global_settings.get_all_settings(cache=cache)
         settings = self._make_dict_for_settings(defaults)
         return settings
 
     def get_repo_settings(self, cache=False):
         """
         Returns list of issue tracker settings per repository
         """
         if not self.repo_settings:
             raise Exception('Repository is not specified')
         all_settings = self.repo_settings.get_all_settings(cache=cache)
         settings = self._make_dict_for_settings(all_settings)
         return settings
 
     def get_settings(self, cache=False):
         if self.inherit_global_settings:
             return self.get_global_settings(cache=cache)
         else:
             return self.get_repo_settings(cache=cache)
 
     def delete_entries(self, uid):
         if self.repo_settings:
             all_patterns = self.get_repo_settings()
             settings_model = self.repo_settings
         else:
             all_patterns = self.get_global_settings()
             settings_model = self.global_settings
         entries = all_patterns.get(uid, [])
 
         for del_key in entries:
             setting_name = self._get_keyname(del_key, uid)
             entry = settings_model.get_setting_by_name(setting_name)
             if entry:
                 Session().delete(entry)
 
         Session().commit()
 
     def create_or_update_setting(
             self, name, val=Optional(''), type_=Optional('unicode')):
         if self.repo_settings:
             setting = self.repo_settings.create_or_update_setting(
                 name, val, type_)
         else:
             setting = self.global_settings.create_or_update_setting(
                 name, val, type_)
         return setting
 
 
 class VcsSettingsModel(object):
 
     INHERIT_SETTINGS = 'inherit_vcs_settings'
     GENERAL_SETTINGS = (
         'use_outdated_comments',
         'pr_merge_enabled',
         'hg_use_rebase_for_merging',
         'hg_close_branch_before_merging',
         'git_use_rebase_for_merging',
         'git_close_branch_before_merging',
         'diff_cache',
     )
 
     HOOKS_SETTINGS = (
         ('hooks', 'changegroup.repo_size'),
         ('hooks', 'changegroup.push_logger'),
         ('hooks', 'outgoing.pull_logger'),
     )
     HG_SETTINGS = (
         ('extensions', 'largefiles'),
         ('phases', 'publish'),
         ('extensions', 'evolve'),
         ('extensions', 'topic'),
         ('experimental', 'evolution'),
         ('experimental', 'evolution.exchange'),
     )
     GIT_SETTINGS = (
         ('vcs_git_lfs', 'enabled'),
     )
     GLOBAL_HG_SETTINGS = (
         ('extensions', 'largefiles'),
         ('largefiles', 'usercache'),
         ('phases', 'publish'),
         ('extensions', 'hgsubversion'),
         ('extensions', 'evolve'),
         ('extensions', 'topic'),
         ('experimental', 'evolution'),
         ('experimental', 'evolution.exchange'),
     )
 
     GLOBAL_GIT_SETTINGS = (
         ('vcs_git_lfs', 'enabled'),
         ('vcs_git_lfs', 'store_location')
     )
 
     GLOBAL_SVN_SETTINGS = (
         ('vcs_svn_proxy', 'http_requests_enabled'),
         ('vcs_svn_proxy', 'http_server_url')
     )
 
     SVN_BRANCH_SECTION = 'vcs_svn_branch'
     SVN_TAG_SECTION = 'vcs_svn_tag'
     SSL_SETTING = ('web', 'push_ssl')
     PATH_SETTING = ('paths', '/')
 
     def __init__(self, sa=None, repo=None):
         self.global_settings = SettingsModel(sa=sa)
         self.repo_settings = SettingsModel(sa=sa, repo=repo) if repo else None
         self._ui_settings = (
             self.HG_SETTINGS + self.GIT_SETTINGS + self.HOOKS_SETTINGS)
         self._svn_sections = (self.SVN_BRANCH_SECTION, self.SVN_TAG_SECTION)
 
     @property
     @assert_repo_settings
     def inherit_global_settings(self):
         setting = self.repo_settings.get_setting_by_name(self.INHERIT_SETTINGS)
         return setting.app_settings_value if setting else True
 
     @inherit_global_settings.setter
     @assert_repo_settings
     def inherit_global_settings(self, value):
         self.repo_settings.create_or_update_setting(
             self.INHERIT_SETTINGS, value, type_='bool')
 
     def get_global_svn_branch_patterns(self):
         return self.global_settings.get_ui_by_section(self.SVN_BRANCH_SECTION)
 
     @assert_repo_settings
     def get_repo_svn_branch_patterns(self):
         return self.repo_settings.get_ui_by_section(self.SVN_BRANCH_SECTION)
 
     def get_global_svn_tag_patterns(self):
         return self.global_settings.get_ui_by_section(self.SVN_TAG_SECTION)
 
     @assert_repo_settings
     def get_repo_svn_tag_patterns(self):
         return self.repo_settings.get_ui_by_section(self.SVN_TAG_SECTION)
 
     def get_global_settings(self):
         return self._collect_all_settings(global_=True)
 
     @assert_repo_settings
     def get_repo_settings(self):
         return self._collect_all_settings(global_=False)
 
     @assert_repo_settings
     def get_repo_settings_inherited(self):
         global_settings = self.get_global_settings()
         global_settings.update(self.get_repo_settings())
         return global_settings
 
     @assert_repo_settings
     def create_or_update_repo_settings(
             self, data, inherit_global_settings=False):
         from rhodecode.model.scm import ScmModel
 
         self.inherit_global_settings = inherit_global_settings
 
         repo = self.repo_settings.get_repo()
         if not inherit_global_settings:
             if repo.repo_type == 'svn':
                 self.create_repo_svn_settings(data)
             else:
                 self.create_or_update_repo_hook_settings(data)
                 self.create_or_update_repo_pr_settings(data)
 
             if repo.repo_type == 'hg':
                 self.create_or_update_repo_hg_settings(data)
 
             if repo.repo_type == 'git':
                 self.create_or_update_repo_git_settings(data)
 
         ScmModel().mark_for_invalidation(repo.repo_name, delete=True)
 
     @assert_repo_settings
     def create_or_update_repo_hook_settings(self, data):
         for section, key in self.HOOKS_SETTINGS:
             data_key = self._get_form_ui_key(section, key)
             if data_key not in data:
                 raise ValueError(
                     'The given data does not contain {} key'.format(data_key))
 
             active = data.get(data_key)
             repo_setting = self.repo_settings.get_ui_by_section_and_key(
                 section, key)
             if not repo_setting:
                 global_setting = self.global_settings.\
                     get_ui_by_section_and_key(section, key)
                 self.repo_settings.create_ui_section_value(
                     section, global_setting.ui_value, key=key, active=active)
             else:
                 repo_setting.ui_active = active
                 Session().add(repo_setting)
 
     def update_global_hook_settings(self, data):
         for section, key in self.HOOKS_SETTINGS:
             data_key = self._get_form_ui_key(section, key)
             if data_key not in data:
                 raise ValueError(
                     'The given data does not contain {} key'.format(data_key))
             active = data.get(data_key)
             repo_setting = self.global_settings.get_ui_by_section_and_key(
                 section, key)
             repo_setting.ui_active = active
             Session().add(repo_setting)
 
     @assert_repo_settings
     def create_or_update_repo_pr_settings(self, data):
         return self._create_or_update_general_settings(
             self.repo_settings, data)
 
     def create_or_update_global_pr_settings(self, data):
         return self._create_or_update_general_settings(
             self.global_settings, data)
 
     @assert_repo_settings
     def create_repo_svn_settings(self, data):
         return self._create_svn_settings(self.repo_settings, data)
 
     def _set_evolution(self, settings, is_enabled):
         if is_enabled:
             # if evolve is active set evolution=all
 
             self._create_or_update_ui(
                 settings, *('experimental', 'evolution'), value='all',
                 active=True)
             self._create_or_update_ui(
                 settings, *('experimental', 'evolution.exchange'), value='yes',
                 active=True)
             # if evolve is active set topics server support
             self._create_or_update_ui(
                 settings, *('extensions', 'topic'), value='',
                 active=True)
 
         else:
             self._create_or_update_ui(
                 settings, *('experimental', 'evolution'), value='',
                 active=False)
             self._create_or_update_ui(
                 settings, *('experimental', 'evolution.exchange'), value='no',
                 active=False)
             self._create_or_update_ui(
                 settings, *('extensions', 'topic'), value='',
                 active=False)
 
     @assert_repo_settings
     def create_or_update_repo_hg_settings(self, data):
         largefiles, phases, evolve = \
             self.HG_SETTINGS[:3]
         largefiles_key, phases_key, evolve_key = \
             self._get_settings_keys(self.HG_SETTINGS[:3], data)
 
         self._create_or_update_ui(
             self.repo_settings, *largefiles, value='',
             active=data[largefiles_key])
         self._create_or_update_ui(
             self.repo_settings, *evolve, value='',
             active=data[evolve_key])
         self._set_evolution(self.repo_settings, is_enabled=data[evolve_key])
 
         self._create_or_update_ui(
             self.repo_settings, *phases, value=safe_str(data[phases_key]))
 
     def create_or_update_global_hg_settings(self, data):
         largefiles, largefiles_store, phases, hgsubversion, evolve \
             = self.GLOBAL_HG_SETTINGS[:5]
         largefiles_key, largefiles_store_key, phases_key, subversion_key, evolve_key \
             = self._get_settings_keys(self.GLOBAL_HG_SETTINGS[:5], data)
 
         self._create_or_update_ui(
             self.global_settings, *largefiles, value='',
             active=data[largefiles_key])
         self._create_or_update_ui(
             self.global_settings, *largefiles_store, value=data[largefiles_store_key])
         self._create_or_update_ui(
             self.global_settings, *phases, value=safe_str(data[phases_key]))
         self._create_or_update_ui(
             self.global_settings, *hgsubversion, active=data[subversion_key])
         self._create_or_update_ui(
             self.global_settings, *evolve, value='',
             active=data[evolve_key])
         self._set_evolution(self.global_settings, is_enabled=data[evolve_key])
 
     def create_or_update_repo_git_settings(self, data):
         # NOTE(marcink): # comma makes unpack work properly
         lfs_enabled, \
             = self.GIT_SETTINGS
 
         lfs_enabled_key, \
             = self._get_settings_keys(self.GIT_SETTINGS, data)
 
         self._create_or_update_ui(
             self.repo_settings, *lfs_enabled, value=data[lfs_enabled_key],
             active=data[lfs_enabled_key])
 
     def create_or_update_global_git_settings(self, data):
         lfs_enabled, lfs_store_location \
             = self.GLOBAL_GIT_SETTINGS
         lfs_enabled_key, lfs_store_location_key \
             = self._get_settings_keys(self.GLOBAL_GIT_SETTINGS, data)
 
         self._create_or_update_ui(
             self.global_settings, *lfs_enabled, value=data[lfs_enabled_key],
             active=data[lfs_enabled_key])
         self._create_or_update_ui(
             self.global_settings, *lfs_store_location,
             value=data[lfs_store_location_key])
 
     def create_or_update_global_svn_settings(self, data):
         # branch/tags patterns
         self._create_svn_settings(self.global_settings, data)
 
         http_requests_enabled, http_server_url = self.GLOBAL_SVN_SETTINGS
         http_requests_enabled_key, http_server_url_key = self._get_settings_keys(
             self.GLOBAL_SVN_SETTINGS, data)
 
         self._create_or_update_ui(
             self.global_settings, *http_requests_enabled,
             value=safe_str(data[http_requests_enabled_key]))
         self._create_or_update_ui(
             self.global_settings, *http_server_url,
             value=data[http_server_url_key])
 
     def update_global_ssl_setting(self, value):
         self._create_or_update_ui(
             self.global_settings, *self.SSL_SETTING, value=value)
 
     def update_global_path_setting(self, value):
         self._create_or_update_ui(
             self.global_settings, *self.PATH_SETTING, value=value)
 
     @assert_repo_settings
     def delete_repo_svn_pattern(self, id_):
         ui = self.repo_settings.UiDbModel.get(id_)
         if ui and ui.repository.repo_name == self.repo_settings.repo:
             # only delete if it's the same repo as initialized settings
             self.repo_settings.delete_ui(id_)
         else:
             # raise error as if we wouldn't find this option
             self.repo_settings.delete_ui(-1)
 
     def delete_global_svn_pattern(self, id_):
         self.global_settings.delete_ui(id_)
 
     @assert_repo_settings
     def get_repo_ui_settings(self, section=None, key=None):
         global_uis = self.global_settings.get_ui(section, key)
         repo_uis = self.repo_settings.get_ui(section, key)
 
         filtered_repo_uis = self._filter_ui_settings(repo_uis)
         filtered_repo_uis_keys = [
             (s.section, s.key) for s in filtered_repo_uis]
 
         def _is_global_ui_filtered(ui):
             return (
                 (ui.section, ui.key) in filtered_repo_uis_keys
                 or ui.section in self._svn_sections)
 
         filtered_global_uis = [
             ui for ui in global_uis if not _is_global_ui_filtered(ui)]
 
         return filtered_global_uis + filtered_repo_uis
 
     def get_global_ui_settings(self, section=None, key=None):
         return self.global_settings.get_ui(section, key)
 
     def get_ui_settings_as_config_obj(self, section=None, key=None):
         config = base.Config()
 
         ui_settings = self.get_ui_settings(section=section, key=key)
 
         for entry in ui_settings:
             config.set(entry.section, entry.key, entry.value)
 
         return config
 
     def get_ui_settings(self, section=None, key=None):
         if not self.repo_settings or self.inherit_global_settings:
             return self.get_global_ui_settings(section, key)
         else:
             return self.get_repo_ui_settings(section, key)
 
     def get_svn_patterns(self, section=None):
         if not self.repo_settings:
             return self.get_global_ui_settings(section)
         else:
             return self.get_repo_ui_settings(section)
 
     @assert_repo_settings
     def get_repo_general_settings(self):
         global_settings = self.global_settings.get_all_settings()
         repo_settings = self.repo_settings.get_all_settings()
         filtered_repo_settings = self._filter_general_settings(repo_settings)
         global_settings.update(filtered_repo_settings)
         return global_settings
 
     def get_global_general_settings(self):
         return self.global_settings.get_all_settings()
 
     def get_general_settings(self):
         if not self.repo_settings or self.inherit_global_settings:
             return self.get_global_general_settings()
         else:
             return self.get_repo_general_settings()
 
     def get_repos_location(self):
         return self.global_settings.get_ui_by_key('/').ui_value
 
     def _filter_ui_settings(self, settings):
         filtered_settings = [
             s for s in settings if self._should_keep_setting(s)]
         return filtered_settings
 
     def _should_keep_setting(self, setting):
         keep = (
             (setting.section, setting.key) in self._ui_settings or
             setting.section in self._svn_sections)
         return keep
 
     def _filter_general_settings(self, settings):
         keys = ['rhodecode_{}'.format(key) for key in self.GENERAL_SETTINGS]
         return {
             k: settings[k]
             for k in settings if k in keys}
 
     def _collect_all_settings(self, global_=False):
         settings = self.global_settings if global_ else self.repo_settings
         result = {}
 
         for section, key in self._ui_settings:
             ui = settings.get_ui_by_section_and_key(section, key)
             result_key = self._get_form_ui_key(section, key)
 
             if ui:
                 if section in ('hooks', 'extensions'):
                     result[result_key] = ui.ui_active
                 elif result_key in ['vcs_git_lfs_enabled']:
                     result[result_key] = ui.ui_active
                 else:
                     result[result_key] = ui.ui_value
 
         for name in self.GENERAL_SETTINGS:
             setting = settings.get_setting_by_name(name)
             if setting:
                 result_key = 'rhodecode_{}'.format(name)
                 result[result_key] = setting.app_settings_value
 
         return result
 
     def _get_form_ui_key(self, section, key):
         return '{section}_{key}'.format(
             section=section, key=key.replace('.', '_'))
 
     def _create_or_update_ui(
             self, settings, section, key, value=None, active=None):
         ui = settings.get_ui_by_section_and_key(section, key)
         if not ui:
             active = True if active is None else active
             settings.create_ui_section_value(
                 section, value, key=key, active=active)
         else:
             if active is not None:
                 ui.ui_active = active
             if value is not None:
                 ui.ui_value = value
             Session().add(ui)
 
     def _create_svn_settings(self, settings, data):
         svn_settings = {
             'new_svn_branch': self.SVN_BRANCH_SECTION,
             'new_svn_tag': self.SVN_TAG_SECTION
         }
         for key in svn_settings:
             if data.get(key):
                 settings.create_ui_section_value(svn_settings[key], data[key])
 
     def _create_or_update_general_settings(self, settings, data):
         for name in self.GENERAL_SETTINGS:
             data_key = 'rhodecode_{}'.format(name)
             if data_key not in data:
                 raise ValueError(
                     'The given data does not contain {} key'.format(data_key))
             setting = settings.create_or_update_setting(
                 name, data[data_key], 'bool')
             Session().add(setting)
 
     def _get_settings_keys(self, settings, data):
         data_keys = [self._get_form_ui_key(*s) for s in settings]
         for data_key in data_keys:
             if data_key not in data:
                 raise ValueError(
                     'The given data does not contain {} key'.format(data_key))
         return data_keys
 
     def create_largeobjects_dirs_if_needed(self, repo_store_path):
         """
         This is subscribed to the `pyramid.events.ApplicationCreated` event. It
         does a repository scan if enabled in the settings.
         """
 
         from rhodecode.lib.vcs.backends.hg import largefiles_store
         from rhodecode.lib.vcs.backends.git import lfs_store
 
         paths = [
             largefiles_store(repo_store_path),
             lfs_store(repo_store_path)]
 
         for path in paths:
             if os.path.isdir(path):
                 continue
             if os.path.isfile(path):
                 continue
             # not a file nor dir, we try to create it
             try:
                 os.makedirs(path)
             except Exception:
                 log.warning('Failed to create largefiles dir:%s', path)