auth: use cache_ttl from a plugin to also cache permissions.

- this gives a 30% speed increase in operations like svn commit
- generally uses the same mechanismy like auth cache to cache permissions for faster
access to vcs commands

rhodecode/authentication/base.py

@@ -37,7 +37,7 b' from rhodecode.authentication.interface '
 from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase
 from rhodecode.lib import caches
 from rhodecode.lib.auth import PasswordGenerator, _RhodeCodeCryptoBCrypt
-from rhodecode.lib.utils2 import md5_safe, safe_int
+from rhodecode.lib.utils2 import safe_int
 from rhodecode.lib.utils2 import safe_str
 from rhodecode.model.db import User
 from rhodecode.model.meta import Session
@@ -423,11 +423,14 b' class RhodeCodeAuthPluginBase(object):'
         """
         auth = self.auth(userobj, username, passwd, settings, **kwargs)
         if auth:
+            auth['_plugin'] = self.name
+            auth['_ttl_cache'] = self.get_ttl_cache(settings)
             # check if hash should be migrated ?
             new_hash = auth.get('_hash_migrate')
             if new_hash:
                 self._migrate_hash_to_bcrypt(username, passwd, new_hash)
             return self._validate_auth_return(auth)
+
         return auth
 
     def _migrate_hash_to_bcrypt(self, username, password, new_hash):
@@ -450,6 +453,19 b' class RhodeCodeAuthPluginBase(object):'
                 raise Exception('Missing %s attribute from returned data' % k)
         return ret
 
+    def get_ttl_cache(self, settings=None):
+        plugin_settings = settings or self.get_settings()
+        cache_ttl = 0
+
+        if isinstance(self.AUTH_CACHE_TTL, (int, long)):
+            # plugin cache set inside is more important than the settings value
+            cache_ttl = self.AUTH_CACHE_TTL
+        elif plugin_settings.get('cache_ttl'):
+            cache_ttl = safe_int(plugin_settings.get('cache_ttl'), 0)
+
+        plugin_cache_active = bool(cache_ttl and cache_ttl > 0)
+        return plugin_cache_active, cache_ttl
+
 
 class RhodeCodeExternalAuthPlugin(RhodeCodeAuthPluginBase):
 
@@ -578,6 +594,11 b' def get_auth_cache_manager(custom_ttl=No'
         'auth_plugins', 'rhodecode.authentication', custom_ttl)
 
 
+def get_perms_cache_manager(custom_ttl=None):
+    return caches.get_cache_manager(
+        'auth_plugins', 'rhodecode.permissions', custom_ttl)
+
+
 def authenticate(username, password, environ=None, auth_type=None,
                  skip_missing=False, registry=None, acl_repo_name=None):
     """
@@ -633,25 +654,19 b' def authenticate(username, password, env'
         log.info('Authenticating user `%s` using %s plugin',
                  display_user, plugin.get_id())
 
-        _cache_ttl = 0
-
-        if isinstance(plugin.AUTH_CACHE_TTL, (int, long)):
-            # plugin cache set inside is more important than the settings value
-            _cache_ttl = plugin.AUTH_CACHE_TTL
-        elif plugin_settings.get('cache_ttl'):
-            _cache_ttl = safe_int(plugin_settings.get('cache_ttl'), 0)
-
-        plugin_cache_active = bool(_cache_ttl and _cache_ttl > 0)
+        plugin_cache_active, cache_ttl = plugin.get_ttl_cache(plugin_settings)
 
         # get instance of cache manager configured for a namespace
-        cache_manager = get_auth_cache_manager(custom_ttl=_cache_ttl)
+        cache_manager = get_auth_cache_manager(custom_ttl=cache_ttl)
 
         log.debug('AUTH_CACHE_TTL for plugin `%s` active: %s (TTL: %s)',
-                  plugin.get_id(), plugin_cache_active, _cache_ttl)
+                  plugin.get_id(), plugin_cache_active, cache_ttl)
 
         # for environ based password can be empty, but then the validation is
         # on the server that fills in the env data needed for authentication
-        _password_hash = md5_safe(plugin.name + username + (password or ''))
+
+        _password_hash = caches.compute_key_from_params(
+            plugin.name, username, (password or ''))
 
         # _authenticate is a wrapper for .auth() method of plugin.
         # it checks if .auth() sends proper data.
@@ -667,11 +682,13 b' def authenticate(username, password, env'
             This function is used internally in Cache of Beaker to calculate
             Results
             """
+            log.debug('auth: calculating password access now...')
             return plugin._authenticate(
                 user, username, password, plugin_settings,
                 environ=environ or {})
 
         if plugin_cache_active:
+            log.debug('Trying to fetch cached auth by %s', _password_hash[:6])
             plugin_user = cache_manager.get(
                 _password_hash, createfunc=auth_func)
         else:
@@ -680,7 +697,7 b' def authenticate(username, password, env'
         auth_time = time.time() - start
         log.debug('Authentication for plugin `%s` completed in %.3fs, '
                   'expiration time of fetched cache %.1fs.',
-                  plugin.get_id(), auth_time, _cache_ttl)
+                  plugin.get_id(), auth_time, cache_ttl)
 
         log.debug('PLUGIN USER DATA: %s', plugin_user)
 
@@ -690,6 +707,8 b' def authenticate(username, password, env'
         # we failed to Auth because .auth() method didn't return proper user
         log.debug("User `%s` failed to authenticate against %s",
                   display_user, plugin.get_id())
+
+    # case when we failed to authenticate against all defined plugins
     return None
 
 

rhodecode/authentication/schema.py

@@ -40,11 +40,10 b' class AuthnPluginSettingsSchemaBase(cola'
     cache_ttl = colander.SchemaNode(
         colander.Int(),
         default=0,
-        description=_('Amount of seconds to cache the authentication response'
-                      'call for this plugin. \n'
-                      'Useful for long calls like LDAP to improve the '
-                      'performance of the authentication system '
-                      '(0 means disabled).'),
+        description=_('Amount of seconds to cache the authentication and '
+                      'permissions check response call for this plugin. \n'
+                      'Useful for expensive calls like LDAP to improve the '
+                      'performance of the system (0 means disabled).'),
         missing=0,
         title=_('Auth Cache TTL'),
         validator=colander.Range(min=0, max=None),

rhodecode/lib/base.py

@@ -277,10 +277,11 b' class BasicAuth(AuthBasicAuthenticator):'
         _parts = auth.split(':', 1)
         if len(_parts) == 2:
             username, password = _parts
-            if self.authfunc(
+            auth_data = self.authfunc(
                     username, password, environ, VCS_TYPE,
-                    registry=self.registry, acl_repo_name=self.acl_repo_name):
-                return username
+                    registry=self.registry, acl_repo_name=self.acl_repo_name)
+            if auth_data:
+                return {'username': username, 'auth_data': auth_data}
             if username and password:
                 # we mark that we actually executed authentication once, at
                 # that point we can use the alternative auth code

rhodecode/lib/middleware/simplevcs.py

@@ -24,17 +24,20 b" It's implemented with basic auth functio"
 """
 
 import os
+import re
 import logging
 import importlib
-import re
 from functools import wraps
 
+import time
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
 from webob.exc import (
     HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError)
 
 import rhodecode
-from rhodecode.authentication.base import authenticate, VCS_TYPE
+from rhodecode.authentication.base import (
+    authenticate, get_perms_cache_manager, VCS_TYPE)
+from rhodecode.lib import caches
 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
 from rhodecode.lib.base import (
     BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
@@ -44,8 +47,7 b' from rhodecode.lib.exceptions import ('
 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
 from rhodecode.lib.middleware import appenlight
 from rhodecode.lib.middleware.utils import scm_app_http
-from rhodecode.lib.utils import (
-    is_valid_repo, get_rhodecode_base_path, SLUG_RE)
+from rhodecode.lib.utils import is_valid_repo, SLUG_RE
 from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool, safe_unicode
 from rhodecode.lib.vcs.conf import settings as vcs_settings
 from rhodecode.lib.vcs.backends import base
@@ -242,40 +244,81 b' class SimpleVCS(object):'
     def is_shadow_repo_dir(self):
         return os.path.isdir(self.vcs_repo_name)
 
-    def _check_permission(self, action, user, repo_name, ip_addr=None):
+    def _check_permission(self, action, user, repo_name, ip_addr=None,
+                          plugin_id='', plugin_cache_active=False, cache_ttl=0):
         """
         Checks permissions using action (push/pull) user and repository
-        name
+        name. If plugin_cache and ttl is set it will use the plugin which
+        authenticated the user to store the cached permissions result for N
+        amount of seconds as in cache_ttl
 
         :param action: push or pull action
         :param user: user instance
         :param repo_name: repository name
         """
+
+        # get instance of cache manager configured for a namespace
+        cache_manager = get_perms_cache_manager(custom_ttl=cache_ttl)
+        log.debug('AUTH_CACHE_TTL for permissions `%s` active: %s (TTL: %s)',
+                  plugin_id, plugin_cache_active, cache_ttl)
+
+        # for environ based password can be empty, but then the validation is
+        # on the server that fills in the env data needed for authentication
+        _perm_calc_hash = caches.compute_key_from_params(
+            plugin_id, action, user.user_id, repo_name, ip_addr)
+
+        # _authenticate is a wrapper for .auth() method of plugin.
+        # it checks if .auth() sends proper data.
+        # For RhodeCodeExternalAuthPlugin it also maps users to
+        # Database and maps the attributes returned from .auth()
+        # to RhodeCode database. If this function returns data
+        # then auth is correct.
+        start = time.time()
+        log.debug('Running plugin `%s` permissions check', plugin_id)
+
+        def perm_func():
+            """
+            This function is used internally in Cache of Beaker to calculate
+            Results
+            """
+            log.debug('auth: calculating permission access now...')
         # check IP
         inherit = user.inherit_default_permissions
-        ip_allowed = AuthUser.check_ip_allowed(user.user_id, ip_addr,
-                                               inherit_from_default=inherit)
+            ip_allowed = AuthUser.check_ip_allowed(
+                user.user_id, ip_addr, inherit_from_default=inherit)
         if ip_allowed:
             log.info('Access for IP:%s allowed', ip_addr)
         else:
             return False
 
         if action == 'push':
-            if not HasPermissionAnyMiddleware('repository.write',
-                                              'repository.admin')(user,
-                                                                  repo_name):
+                perms = ('repository.write', 'repository.admin')
+                if not HasPermissionAnyMiddleware(*perms)(user, repo_name):
                 return False
 
         else:
             # any other action need at least read permission
-            if not HasPermissionAnyMiddleware('repository.read',
-                                              'repository.write',
-                                              'repository.admin')(user,
-                                                                  repo_name):
+                perms = (
+                    'repository.read', 'repository.write', 'repository.admin')
+                if not HasPermissionAnyMiddleware(*perms)(user, repo_name):
                 return False
 
         return True
 
+        if plugin_cache_active:
+            log.debug('Trying to fetch cached perms by %s', _perm_calc_hash[:6])
+            perm_result = cache_manager.get(
+                _perm_calc_hash, createfunc=perm_func)
+        else:
+            perm_result = perm_func()
+
+        auth_time = time.time() - start
+        log.debug('Permissions for plugin `%s` completed in %.3fs, '
+                  'expiration time of fetched cache %.1fs.',
+                  plugin_id, auth_time, cache_ttl)
+
+        return perm_result
+
     def _check_ssl(self, environ, start_response):
         """
         Checks the SSL check flag and returns False if SSL is not present
@@ -376,26 +419,41 b' class SimpleVCS(object):'
                 if pre_auth and pre_auth.get('username'):
                     username = pre_auth['username']
                 log.debug('PRE-AUTH got %s as username', username)
+                if pre_auth:
+                    log.debug('PRE-AUTH successful from %s',
+                              pre_auth.get('auth_data', {}).get('_plugin'))
 
                 # If not authenticated by the container, running basic auth
                 # before inject the calling repo_name for special scope checks
                 self.authenticate.acl_repo_name = self.acl_repo_name
+
+                plugin_cache_active, cache_ttl = False, 0
+                plugin = None
                 if not username:
                     self.authenticate.realm = self.authenticate.get_rc_realm()
 
                     try:
-                        result = self.authenticate(environ)
+                        auth_result = self.authenticate(environ)
                     except (UserCreationError, NotAllowedToCreateUserError) as e:
                         log.error(e)
                         reason = safe_str(e)
                         return HTTPNotAcceptable(reason)(environ, start_response)
 
-                    if isinstance(result, str):
+                    if isinstance(auth_result, dict):
                         AUTH_TYPE.update(environ, 'basic')
-                        REMOTE_USER.update(environ, result)
-                        username = result
+                        REMOTE_USER.update(environ, auth_result['username'])
+                        username = auth_result['username']
+                        plugin = auth_result.get('auth_data', {}).get('_plugin')
+                        log.info(
+                            'MAIN-AUTH successful for user `%s` from %s plugin',
+                            username, plugin)
+
+                        plugin_cache_active, cache_ttl = auth_result.get(
+                            'auth_data', {}).get('_ttl_cache') or (False, 0)
                     else:
-                        return result.wsgi_application(environ, start_response)
+                        return auth_result.wsgi_application(
+                            environ, start_response)
+
 
                 # ==============================================================
                 # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
@@ -417,12 +475,13 b' class SimpleVCS(object):'
 
                 # check permissions for this repository
                 perm = self._check_permission(
-                    action, user, self.acl_repo_name, ip_addr)
+                    action, user, self.acl_repo_name, ip_addr,
+                    plugin, plugin_cache_active, cache_ttl)
                 if not perm:
                     return HTTPForbidden()(environ, start_response)
 
         # extras are injected into UI object and later available
-        # in hooks executed by rhodecode
+        # in hooks executed by RhodeCode
         check_locking = _should_check_locking(environ.get('QUERY_STRING'))
         extras = vcs_operation_context(
             environ, repo_name=self.acl_repo_name, username=username,

rhodecode/model/db.py

@@ -918,7 +918,7 b' class User(Base, BaseModel):'
         """Update user lastactivity"""
         self.last_activity = datetime.datetime.now()
         Session().add(self)
-        log.debug('updated user %s lastactivity', self.username)
+        log.debug('updated user `%s` last activity', self.username)
 
     def update_password(self, new_password):
         from rhodecode.lib.auth import get_crypt_password

rhodecode/tests/lib/auth_modules/test_auth_modules.py

@@ -27,8 +27,13 b' from rhodecode.authentication.plugins.au'
 from rhodecode.model import db
 
 
+class TestAuthPlugin(RhodeCodeAuthPluginBase):
+
+    def name(self):
+        return 'stub_auth'
+
 def test_authenticate_returns_from_auth(stub_auth_data):
-    plugin = RhodeCodeAuthPluginBase('stub_id')
+    plugin = TestAuthPlugin('stub_id')
     with mock.patch.object(plugin, 'auth') as auth_mock:
         auth_mock.return_value = stub_auth_data
         result = plugin._authenticate(mock.Mock(), 'test', 'password', {})
@@ -37,7 +42,7 b' def test_authenticate_returns_from_auth('
 
 def test_authenticate_returns_empty_auth_data():
     auth_data = {}
-    plugin = RhodeCodeAuthPluginBase('stub_id')
+    plugin = TestAuthPlugin('stub_id')
     with mock.patch.object(plugin, 'auth') as auth_mock:
         auth_mock.return_value = auth_data
         result = plugin._authenticate(mock.Mock(), 'test', 'password', {})
@@ -46,7 +51,7 b' def test_authenticate_returns_empty_auth'
 
 def test_authenticate_skips_hash_migration_if_mismatch(stub_auth_data):
     stub_auth_data['_hash_migrate'] = 'new-hash'
-    plugin = RhodeCodeAuthPluginBase('stub_id')
+    plugin = TestAuthPlugin('stub_id')
     with mock.patch.object(plugin, 'auth') as auth_mock:
         auth_mock.return_value = stub_auth_data
         result = plugin._authenticate(mock.Mock(), 'test', 'password', {})
@@ -60,7 +65,7 b' def test_authenticate_migrates_to_new_ha'
     new_password = b'new-password'
     new_hash = _RhodeCodeCryptoBCrypt().hash_create(new_password)
     stub_auth_data['_hash_migrate'] = new_hash
-    plugin = RhodeCodeAuthPluginBase('stub_id')
+    plugin = TestAuthPlugin('stub_id')
     with mock.patch.object(plugin, 'auth') as auth_mock:
         auth_mock.return_value = stub_auth_data
         result = plugin._authenticate(