Authentication: cache plugins for auth and their settings in the auth_registry.

- Before that change on each requests 4x we loaded plugins for authentication, this hit
many caches, db invalidation context and loaded the plugins logic each time. This was a heavy performance hit
for SVN and other backends as they needed to load that plugins many many times
- Since Authentication plugins almost never change, we'll not store the plugins listed for authentication into
the authnregistry for each process
- For AuthPlugins settings we now also flush plugins settings, and authnregistry cached plugins

rhodecode/apps/login/tests/test_login.py

@@ -108,7 +108,8 b' class TestLoginController(object):'
 
     def test_login_regular_forbidden_when_super_admin_restriction(self):
         from rhodecode.authentication.plugins.auth_rhodecode import RhodeCodeAuthPlugin
-        with fixture.auth_restriction(RhodeCodeAuthPlugin.AUTH_RESTRICTION_SUPER_ADMIN):
+        with fixture.auth_restriction(self.app._pyramid_registry,
+                                      RhodeCodeAuthPlugin.AUTH_RESTRICTION_SUPER_ADMIN):
             response = self.app.post(route_path('login'),
                                      {'username': 'test_regular',
                                       'password': 'test12'})
@@ -118,7 +119,8 b' class TestLoginController(object):'
 
     def test_login_regular_forbidden_when_scope_restriction(self):
         from rhodecode.authentication.plugins.auth_rhodecode import RhodeCodeAuthPlugin
-        with fixture.scope_restriction(RhodeCodeAuthPlugin.AUTH_RESTRICTION_SCOPE_VCS):
+        with fixture.scope_restriction(self.app._pyramid_registry,
+                                       RhodeCodeAuthPlugin.AUTH_RESTRICTION_SCOPE_VCS):
             response = self.app.post(route_path('login'),
                                      {'username': 'test_regular',
                                       'password': 'test12'})

rhodecode/authentication/base.py

@@ -150,6 +150,7 b' class RhodeCodeAuthPluginBase(object):'
 
     def __init__(self, plugin_id):
         self._plugin_id = plugin_id
+        self._settings = {}
 
     def __str__(self):
         return self.get_id()
@@ -226,17 +227,26 b' class RhodeCodeAuthPluginBase(object):'
         """
         return AuthnPluginSettingsSchemaBase()
 
-    def get_settings(self):
-        """
-        Returns the plugin settings as dictionary.
-        """
+    def _propagate_settings(self, raw_settings):
         settings = {}
-        raw_settings = SettingsModel().get_all_settings()
         for node in self.get_settings_schema():
             settings[node.name] = self.get_setting_by_name(
                 node.name, plugin_cached_settings=raw_settings)
         return settings
 
+    def get_settings(self, use_cache=True):
+        """
+        Returns the plugin settings as dictionary.
+        """
+        if self._settings != {} and use_cache:
+            return self._settings
+
+        raw_settings = SettingsModel().get_all_settings()
+        settings = self._propagate_settings(raw_settings)
+
+        self._settings = settings
+        return self._settings
+
     def get_setting_by_name(self, name, default=None, plugin_cached_settings=None):
         """
         Returns a plugin setting by name.
@@ -667,7 +677,7 b' def loadplugin(plugin_id):'
 
 def get_authn_registry(registry=None):
     registry = registry or get_current_registry()
-    authn_registry = registry.getUtility(IAuthnPluginRegistry)
+    authn_registry = registry.queryUtility(IAuthnPluginRegistry)
     return authn_registry
 
 
@@ -690,6 +700,7 b' def authenticate(username, password, env'
     headers_only = environ and not (username and password)
 
     authn_registry = get_authn_registry(registry)
+
     plugins_to_check = authn_registry.get_plugins_for_authentication()
     log.debug('Starting ordered authentication chain using %s plugins',
               [x.name for x in plugins_to_check])

rhodecode/authentication/registry.py

@@ -38,6 +38,7 b' class AuthenticationPluginRegistry(objec'
 
     def __init__(self, settings):
         self._plugins = {}
+        self._plugins_for_auth = None
         self._fallback_plugin = settings.get(self.fallback_plugin_key, None)
 
     def add_authn_plugin(self, config, plugin):
@@ -63,6 +64,10 b' class AuthenticationPluginRegistry(objec'
             if plugin.uid == plugin_uid:
                 return plugin
 
+    def invalidate_plugins_for_auth(self):
+        log.debug('Invalidating cached plugins for authentication')
+        self._plugins_for_auth = None
+
     def get_plugins_for_authentication(self):
         """
         Returns a list of plugins which should be consulted when authenticating
@@ -70,6 +75,9 b' class AuthenticationPluginRegistry(objec'
         Additionally it includes the fallback plugin from the INI file, if
         `rhodecode.auth_plugin_fallback` is set to a plugin ID.
         """
+        if self._plugins_for_auth is not None:
+            return self._plugins_for_auth
+
         plugins = []
 
         # Add all enabled and active plugins to the list. We iterate over the
@@ -80,6 +88,9 b' class AuthenticationPluginRegistry(objec'
             plugin = self.get_plugin(plugin_id)
             if plugin is not None and plugin.is_active(
                     plugin_cached_settings=raw_settings):
+
+                # inject settings into plugin, we can re-use the DB fetched settings here
+                plugin._settings = plugin._propagate_settings(raw_settings)
                 plugins.append(plugin)
 
         # Add the fallback plugin from ini file.
@@ -89,6 +100,8 b' class AuthenticationPluginRegistry(objec'
                 self._fallback_plugin)
             plugin = self.get_plugin(self._fallback_plugin)
             if plugin is not None and plugin not in plugins:
+                plugin._settings = plugin._propagate_settings(raw_settings)
                 plugins.append(plugin)
 
-        return plugins
+        self._plugins_for_auth = plugins
+        return self._plugins_for_auth

rhodecode/authentication/views.py

@@ -99,11 +99,12 b' class AuthnPluginViewBase(BaseAppView):'
         for name, value in valid_data.items():
             self.plugin.create_or_update_setting(name, value)
         Session().commit()
+        SettingsModel().invalidate_settings_cache()
 
         # Display success message and redirect.
         h.flash(_('Auth settings updated successfully.'), category='success')
-        redirect_to = self.request.resource_path(
-            self.context, route_name='auth_home')
+        redirect_to = self.request.resource_path(self.context, route_name='auth_home')
+
         return HTTPFound(redirect_to)
 
 
@@ -159,7 +160,7 b' class AuthSettingsView(BaseAppView):'
                 'auth_plugins', plugins)
             Session().add(setting)
             Session().commit()
-
+            SettingsModel().invalidate_settings_cache()
             h.flash(_('Auth settings updated successfully.'), category='success')
         except formencode.Invalid as errors:
             e = errors.error_dict or {}
@@ -174,6 +175,6 b' class AuthSettingsView(BaseAppView):'
             h.flash(_('Error occurred during update of auth settings.'),
                     category='error')
 
-        redirect_to = self.request.resource_path(
-            self.context, route_name='auth_home')
+        redirect_to = self.request.resource_path(self.context, route_name='auth_home')
+
         return HTTPFound(redirect_to)

rhodecode/lib/base.py

@@ -211,8 +211,9 b' def vcs_operation_context('
 class BasicAuth(AuthBasicAuthenticator):
 
     def __init__(self, realm, authfunc, registry, auth_http_code=None,
-                 initial_call_detection=False, acl_repo_name=None):
+                 initial_call_detection=False, acl_repo_name=None, rc_realm=''):
         self.realm = realm
+        self.rc_realm = rc_realm
         self.initial_call = initial_call_detection
         self.authfunc = authfunc
         self.registry = registry
@@ -227,7 +228,7 b' class BasicAuth(AuthBasicAuthenticator):'
             return HTTPForbidden
 
     def get_rc_realm(self):
-        return safe_str(self.registry.rhodecode_settings.get('rhodecode_realm'))
+        return safe_str(self.rc_realm)
 
     def build_authentication(self):
         head = WWW_AUTHENTICATE.tuples('Basic realm="%s"' % self.realm)

rhodecode/lib/middleware/simplevcs.py

@@ -133,15 +133,16 b' class SimpleVCS(object):'
         self.config = config
         # re-populated by specialized middleware
         self.repo_vcs_config = base.Config()
-        self.rhodecode_settings = SettingsModel().get_all_settings(cache=True)
 
-        registry.rhodecode_settings = self.rhodecode_settings
+        rc_settings = SettingsModel().get_all_settings(cache=True, from_request=False)
+        realm = rc_settings.get('rhodecode_realm') or 'RhodeCode AUTH'
+
         # authenticate this VCS request using authfunc
         auth_ret_code_detection = \
             str2bool(self.config.get('auth_ret_code_detection', False))
         self.authenticate = BasicAuth(
             '', authenticate, registry, config.get('auth_ret_code'),
-            auth_ret_code_detection)
+            auth_ret_code_detection, rc_realm=realm)
         self.ip_addr = '0.0.0.0'
 
     @LazyProperty

rhodecode/model/settings.py

@@ -25,7 +25,7 b' import re'
 from collections import namedtuple
 from functools import wraps
 import bleach
-from pyramid.threadlocal import get_current_request
+from pyramid.threadlocal import get_current_request, get_current_registry
 
 from rhodecode.lib import rc_cache
 from rhodecode.lib.utils2 import (
@@ -213,6 +213,8 b' class SettingsModel(BaseModel):'
         CacheKey.set_invalidate(invalidation_namespace)
 
     def get_all_settings(self, cache=False, from_request=True):
+        from rhodecode.authentication.base import get_authn_registry
+
         # defines if we use GLOBAL, or PER_REPO
         repo = self._get_repo(self.repo) if self.repo else None
         key = "settings_repo.{}".format(repo.repo_id) if repo else "settings_app"
@@ -253,6 +255,11 b' class SettingsModel(BaseModel):'
                 # are anyway very short lived and it's a safest way.
                 region = rc_cache.get_or_create_region('sql_cache_short')
                 region.invalidate()
+                registry = get_current_registry()
+                if registry:
+                    authn_registry = get_authn_registry(registry)
+                    if authn_registry:
+                        authn_registry.invalidate_plugins_for_auth()
 
             result = _get_all_settings('rhodecode_settings', key)
             log.debug('Fetching app settings for key: %s took: %.4fs', key,

rhodecode/tests/fixture.py

@@ -29,6 +29,7 b' import shutil'
 
 import configobj
 
+from rhodecode.model.settings import SettingsModel
 from rhodecode.tests import *
 from rhodecode.model.db import Repository, User, RepoGroup, UserGroup, Gist, UserEmailMap
 from rhodecode.model.meta import Session
@@ -122,7 +123,7 b' class Fixture(object):'
 
         return context()
 
-    def auth_restriction(self, auth_restriction):
+    def auth_restriction(self, registry, auth_restriction):
         """
         Context process for changing the builtin rhodecode plugin auth restrictions.
         Use like:
@@ -135,26 +136,26 b' class Fixture(object):'
 
         class context(object):
             def _get_pluing(self):
-                plugin_id = 'egg:rhodecode-enterprise-ce#{}'.format(
-                    RhodeCodeAuthPlugin.uid)
+                plugin_id = 'egg:rhodecode-enterprise-ce#{}'.format(RhodeCodeAuthPlugin.uid)
                 plugin = RhodeCodeAuthPlugin(plugin_id)
                 return plugin
 
             def __enter__(self):
                 plugin = self._get_pluing()
-                plugin.create_or_update_setting(
-                    'auth_restriction', auth_restriction)
+                plugin.create_or_update_setting('auth_restriction', auth_restriction)
                 Session().commit()
+                SettingsModel().invalidate_settings_cache()
 
             def __exit__(self, exc_type, exc_val, exc_tb):
                 plugin = self._get_pluing()
                 plugin.create_or_update_setting(
                     'auth_restriction', RhodeCodeAuthPlugin.AUTH_RESTRICTION_NONE)
                 Session().commit()
+                SettingsModel().invalidate_settings_cache()
 
         return context()
 
-    def scope_restriction(self, scope_restriction):
+    def scope_restriction(self, registry, scope_restriction):
         """
         Context process for changing the builtin rhodecode plugin scope restrictions.
         Use like:
@@ -167,22 +168,22 b' class Fixture(object):'
 
         class context(object):
             def _get_pluing(self):
-                plugin_id = 'egg:rhodecode-enterprise-ce#{}'.format(
-                    RhodeCodeAuthPlugin.uid)
+                plugin_id = 'egg:rhodecode-enterprise-ce#{}'.format(RhodeCodeAuthPlugin.uid)
                 plugin = RhodeCodeAuthPlugin(plugin_id)
                 return plugin
 
             def __enter__(self):
                 plugin = self._get_pluing()
-                plugin.create_or_update_setting(
-                    'scope_restriction', scope_restriction)
+                plugin.create_or_update_setting('scope_restriction', scope_restriction)
                 Session().commit()
+                SettingsModel().invalidate_settings_cache()
 
             def __exit__(self, exc_type, exc_val, exc_tb):
                 plugin = self._get_pluing()
                 plugin.create_or_update_setting(
                     'scope_restriction', RhodeCodeAuthPlugin.AUTH_RESTRICTION_SCOPE_ALL)
                 Session().commit()
+                SettingsModel().invalidate_settings_cache()
 
         return context()
 

rhodecode/tests/vcs_operations/conftest.py

@@ -209,6 +209,8 b' def enable_auth_plugins(request, baseapp'
                 Session().add(setting)
             Session().commit()
 
+        SettingsModel().invalidate_settings_cache()
+
     def cleanup():
         _enable_plugins(['egg:rhodecode-enterprise-ce#rhodecode'])