caches: remove settings cache to use invalidation context as it's slowing down much usage

rhodecode/config/middleware.py

@@ -581,19 +581,24 b' def sanitize_settings_and_apply_defaults'
     settings_maker.make_setting('exception_tracker.send_email', False, parser='bool')
     settings_maker.make_setting('exception_tracker.email_prefix', '[RHODECODE ERROR]', default_when_empty=True)
 
+    # cache_general
+    settings_maker.make_setting('rc_cache.cache_general.backend', 'dogpile.cache.rc.file_namespace')
+    settings_maker.make_setting('rc_cache.cache_general.expiration_time', 60 * 60 * 12, parser='int')
+    settings_maker.make_setting('rc_cache.cache_general.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_general.db'))
+
     # cache_perms
     settings_maker.make_setting('rc_cache.cache_perms.backend', 'dogpile.cache.rc.file_namespace')
-    settings_maker.make_setting('rc_cache.cache_perms.expiration_time', 60, parser='int')
+    settings_maker.make_setting('rc_cache.cache_perms.expiration_time', 60 * 60, parser='int')
     settings_maker.make_setting('rc_cache.cache_perms.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_perms.db'))
 
     # cache_repo
     settings_maker.make_setting('rc_cache.cache_repo.backend', 'dogpile.cache.rc.file_namespace')
-    settings_maker.make_setting('rc_cache.cache_repo.expiration_time', 60, parser='int')
+    settings_maker.make_setting('rc_cache.cache_repo.expiration_time', 60 * 60 * 24 * 30, parser='int')
     settings_maker.make_setting('rc_cache.cache_repo.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_repo.db'))
 
     # cache_license
     settings_maker.make_setting('rc_cache.cache_license.backend', 'dogpile.cache.rc.file_namespace')
-    settings_maker.make_setting('rc_cache.cache_license.expiration_time', 5*60, parser='int')
+    settings_maker.make_setting('rc_cache.cache_license.expiration_time', 60 * 5, parser='int')
     settings_maker.make_setting('rc_cache.cache_license.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_license.db'))
 
     # cache_repo_longterm memory, 96H

rhodecode/model/settings.py

@@ -31,6 +31,7 b' from rhodecode.lib import rc_cache'
 from rhodecode.lib.utils2 import (
     Optional, AttributeDict, safe_str, remove_prefix, str2bool)
 from rhodecode.lib.vcs.backends import base
+from rhodecode.lib.statsd_client import StatsdClient
 from rhodecode.model import BaseModel
 from rhodecode.model.db import (
     RepoRhodeCodeUi, RepoRhodeCodeSetting, RhodeCodeUi, RhodeCodeSetting, CacheKey)
@@ -208,16 +209,28 b' class SettingsModel(BaseModel):'
         Session().add(res)
         return res
 
+    def get_cache_region(self):
+        repo = self._get_repo(self.repo) if self.repo else None
+        cache_key = "repo.{}".format(repo.repo_id) if repo else "general_settings"
+        cache_namespace_uid = 'cache_settings.{}'.format(cache_key)
+        region = rc_cache.get_or_create_region('cache_general', cache_namespace_uid)
+        return region, cache_key
+
     def invalidate_settings_cache(self):
-        invalidation_namespace = CacheKey.SETTINGS_INVALIDATION_NAMESPACE
-        CacheKey.set_invalidate(invalidation_namespace)
+        from rhodecode.authentication.base import get_authn_registry
+
+        region, cache_key = self.get_cache_region()
+        log.debug('Invalidation cache region %s for cache_key: %s', region, cache_key)
+        region.invalidate()
+        registry = get_current_registry()
+        if registry:
+            authn_registry = get_authn_registry(registry)
+            if authn_registry:
+                authn_registry.invalidate_plugins_for_auth()
 
     def get_all_settings(self, cache=False, from_request=True):
-        from rhodecode.authentication.base import get_authn_registry
-
         # defines if we use GLOBAL, or PER_REPO
         repo = self._get_repo(self.repo) if self.repo else None
-        key = "settings_repo.{}".format(repo.repo_id) if repo else "settings_app"
 
         # initially try the requests context, this is the fastest
         # we only fetch global config
@@ -229,8 +242,7 b' class SettingsModel(BaseModel):'
                 if rc_config:
                     return rc_config
 
-        region = rc_cache.get_or_create_region('sql_cache_short')
-        invalidation_namespace = CacheKey.SETTINGS_INVALIDATION_NAMESPACE
+        region, cache_key = self.get_cache_region()
 
         @region.conditional_cache_on_arguments(condition=cache)
         def _get_all_settings(name, key):
@@ -239,31 +251,17 b' class SettingsModel(BaseModel):'
                 raise Exception('Could not get application settings !')
 
             settings = {
-                'rhodecode_' + result.app_settings_name: result.app_settings_value
-                for result in q
+                'rhodecode_' + res.app_settings_name: res.app_settings_value
+                for res in q
             }
             return settings
 
-        inv_context_manager = rc_cache.InvalidationContext(
-            uid='cache_settings', invalidation_namespace=invalidation_namespace)
-        with inv_context_manager as invalidation_context:
-            # check for stored invalidation signal, and maybe purge the cache
-            # before computing it again
-            if invalidation_context.should_invalidate():
-                # NOTE:(marcink) we flush the whole sql_cache_short region, because it
-                # reads different settings etc. It's little too much but those caches
-                # are anyway very short lived and it's a safest way.
-                region = rc_cache.get_or_create_region('sql_cache_short')
-                region.invalidate()
-                registry = get_current_registry()
-                if registry:
-                    authn_registry = get_authn_registry(registry)
-                    if authn_registry:
-                        authn_registry.invalidate_plugins_for_auth()
+        statsd = StatsdClient.statsd
+        with statsd.timer('rhodecode_settings_timing.histogram', auto_send=True) as tmr:
+            result = _get_all_settings('rhodecode_settings', cache_key)
 
-            result = _get_all_settings('rhodecode_settings', key)
-            log.debug('Fetching app settings for key: %s took: %.4fs', key,
-                      inv_context_manager.compute_time)
+        compute_time = tmr.ms / 1000.
+        log.debug('Fetching app settings for key: %s took: %.4fs', cache_key, compute_time)
 
         return result