# HG changeset patch # User Marcin Kuzminski # Date 2017-10-07 08:02:39 # Node ID 574d07a8f69d5c982246fb3d835370a38d9c7a2d # Parent 6de97439953fcd5f31d40fcf43ab5c5626eec08a auth: use cache_ttl from a plugin to also cache permissions. - this gives a 30% speed increase in operations like svn commit - generally uses the same mechanismy like auth cache to cache permissions for faster access to vcs commands diff --git a/rhodecode/authentication/base.py b/rhodecode/authentication/base.py --- a/rhodecode/authentication/base.py +++ b/rhodecode/authentication/base.py @@ -37,7 +37,7 @@ from rhodecode.authentication.interface from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase from rhodecode.lib import caches from rhodecode.lib.auth import PasswordGenerator, _RhodeCodeCryptoBCrypt -from rhodecode.lib.utils2 import md5_safe, safe_int +from rhodecode.lib.utils2 import safe_int from rhodecode.lib.utils2 import safe_str from rhodecode.model.db import User from rhodecode.model.meta import Session @@ -423,11 +423,14 @@ class RhodeCodeAuthPluginBase(object): """ auth = self.auth(userobj, username, passwd, settings, **kwargs) if auth: + auth['_plugin'] = self.name + auth['_ttl_cache'] = self.get_ttl_cache(settings) # check if hash should be migrated ? new_hash = auth.get('_hash_migrate') if new_hash: self._migrate_hash_to_bcrypt(username, passwd, new_hash) return self._validate_auth_return(auth) + return auth def _migrate_hash_to_bcrypt(self, username, password, new_hash): @@ -450,6 +453,19 @@ class RhodeCodeAuthPluginBase(object): raise Exception('Missing %s attribute from returned data' % k) return ret + def get_ttl_cache(self, settings=None): + plugin_settings = settings or self.get_settings() + cache_ttl = 0 + + if isinstance(self.AUTH_CACHE_TTL, (int, long)): + # plugin cache set inside is more important than the settings value + cache_ttl = self.AUTH_CACHE_TTL + elif plugin_settings.get('cache_ttl'): + cache_ttl = safe_int(plugin_settings.get('cache_ttl'), 0) + + plugin_cache_active = bool(cache_ttl and cache_ttl > 0) + return plugin_cache_active, cache_ttl + class RhodeCodeExternalAuthPlugin(RhodeCodeAuthPluginBase): @@ -578,6 +594,11 @@ def get_auth_cache_manager(custom_ttl=No 'auth_plugins', 'rhodecode.authentication', custom_ttl) +def get_perms_cache_manager(custom_ttl=None): + return caches.get_cache_manager( + 'auth_plugins', 'rhodecode.permissions', custom_ttl) + + def authenticate(username, password, environ=None, auth_type=None, skip_missing=False, registry=None, acl_repo_name=None): """ @@ -633,25 +654,19 @@ def authenticate(username, password, env log.info('Authenticating user `%s` using %s plugin', display_user, plugin.get_id()) - _cache_ttl = 0 - - if isinstance(plugin.AUTH_CACHE_TTL, (int, long)): - # plugin cache set inside is more important than the settings value - _cache_ttl = plugin.AUTH_CACHE_TTL - elif plugin_settings.get('cache_ttl'): - _cache_ttl = safe_int(plugin_settings.get('cache_ttl'), 0) - - plugin_cache_active = bool(_cache_ttl and _cache_ttl > 0) + plugin_cache_active, cache_ttl = plugin.get_ttl_cache(plugin_settings) # get instance of cache manager configured for a namespace - cache_manager = get_auth_cache_manager(custom_ttl=_cache_ttl) + cache_manager = get_auth_cache_manager(custom_ttl=cache_ttl) log.debug('AUTH_CACHE_TTL for plugin `%s` active: %s (TTL: %s)', - plugin.get_id(), plugin_cache_active, _cache_ttl) + plugin.get_id(), plugin_cache_active, cache_ttl) # for environ based password can be empty, but then the validation is # on the server that fills in the env data needed for authentication - _password_hash = md5_safe(plugin.name + username + (password or '')) + + _password_hash = caches.compute_key_from_params( + plugin.name, username, (password or '')) # _authenticate is a wrapper for .auth() method of plugin. # it checks if .auth() sends proper data. @@ -667,11 +682,13 @@ def authenticate(username, password, env This function is used internally in Cache of Beaker to calculate Results """ + log.debug('auth: calculating password access now...') return plugin._authenticate( user, username, password, plugin_settings, environ=environ or {}) if plugin_cache_active: + log.debug('Trying to fetch cached auth by %s', _password_hash[:6]) plugin_user = cache_manager.get( _password_hash, createfunc=auth_func) else: @@ -680,7 +697,7 @@ def authenticate(username, password, env auth_time = time.time() - start log.debug('Authentication for plugin `%s` completed in %.3fs, ' 'expiration time of fetched cache %.1fs.', - plugin.get_id(), auth_time, _cache_ttl) + plugin.get_id(), auth_time, cache_ttl) log.debug('PLUGIN USER DATA: %s', plugin_user) @@ -690,6 +707,8 @@ def authenticate(username, password, env # we failed to Auth because .auth() method didn't return proper user log.debug("User `%s` failed to authenticate against %s", display_user, plugin.get_id()) + + # case when we failed to authenticate against all defined plugins return None diff --git a/rhodecode/authentication/schema.py b/rhodecode/authentication/schema.py --- a/rhodecode/authentication/schema.py +++ b/rhodecode/authentication/schema.py @@ -40,11 +40,10 @@ class AuthnPluginSettingsSchemaBase(cola cache_ttl = colander.SchemaNode( colander.Int(), default=0, - description=_('Amount of seconds to cache the authentication response' - 'call for this plugin. \n' - 'Useful for long calls like LDAP to improve the ' - 'performance of the authentication system ' - '(0 means disabled).'), + description=_('Amount of seconds to cache the authentication and ' + 'permissions check response call for this plugin. \n' + 'Useful for expensive calls like LDAP to improve the ' + 'performance of the system (0 means disabled).'), missing=0, title=_('Auth Cache TTL'), validator=colander.Range(min=0, max=None), diff --git a/rhodecode/lib/base.py b/rhodecode/lib/base.py --- a/rhodecode/lib/base.py +++ b/rhodecode/lib/base.py @@ -277,10 +277,11 @@ class BasicAuth(AuthBasicAuthenticator): _parts = auth.split(':', 1) if len(_parts) == 2: username, password = _parts - if self.authfunc( + auth_data = self.authfunc( username, password, environ, VCS_TYPE, - registry=self.registry, acl_repo_name=self.acl_repo_name): - return username + registry=self.registry, acl_repo_name=self.acl_repo_name) + if auth_data: + return {'username': username, 'auth_data': auth_data} if username and password: # we mark that we actually executed authentication once, at # that point we can use the alternative auth code diff --git a/rhodecode/lib/middleware/simplevcs.py b/rhodecode/lib/middleware/simplevcs.py --- a/rhodecode/lib/middleware/simplevcs.py +++ b/rhodecode/lib/middleware/simplevcs.py @@ -24,17 +24,20 @@ It's implemented with basic auth functio """ import os +import re import logging import importlib -import re from functools import wraps +import time from paste.httpheaders import REMOTE_USER, AUTH_TYPE from webob.exc import ( HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError) import rhodecode -from rhodecode.authentication.base import authenticate, VCS_TYPE +from rhodecode.authentication.base import ( + authenticate, get_perms_cache_manager, VCS_TYPE) +from rhodecode.lib import caches from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware from rhodecode.lib.base import ( BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context) @@ -44,8 +47,7 @@ from rhodecode.lib.exceptions import ( from rhodecode.lib.hooks_daemon import prepare_callback_daemon from rhodecode.lib.middleware import appenlight from rhodecode.lib.middleware.utils import scm_app_http -from rhodecode.lib.utils import ( - is_valid_repo, get_rhodecode_base_path, SLUG_RE) +from rhodecode.lib.utils import is_valid_repo, SLUG_RE from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool, safe_unicode from rhodecode.lib.vcs.conf import settings as vcs_settings from rhodecode.lib.vcs.backends import base @@ -242,39 +244,80 @@ class SimpleVCS(object): def is_shadow_repo_dir(self): return os.path.isdir(self.vcs_repo_name) - def _check_permission(self, action, user, repo_name, ip_addr=None): + def _check_permission(self, action, user, repo_name, ip_addr=None, + plugin_id='', plugin_cache_active=False, cache_ttl=0): """ Checks permissions using action (push/pull) user and repository - name + name. If plugin_cache and ttl is set it will use the plugin which + authenticated the user to store the cached permissions result for N + amount of seconds as in cache_ttl :param action: push or pull action :param user: user instance :param repo_name: repository name """ - # check IP - inherit = user.inherit_default_permissions - ip_allowed = AuthUser.check_ip_allowed(user.user_id, ip_addr, - inherit_from_default=inherit) - if ip_allowed: - log.info('Access for IP:%s allowed', ip_addr) - else: - return False + + # get instance of cache manager configured for a namespace + cache_manager = get_perms_cache_manager(custom_ttl=cache_ttl) + log.debug('AUTH_CACHE_TTL for permissions `%s` active: %s (TTL: %s)', + plugin_id, plugin_cache_active, cache_ttl) + + # for environ based password can be empty, but then the validation is + # on the server that fills in the env data needed for authentication + _perm_calc_hash = caches.compute_key_from_params( + plugin_id, action, user.user_id, repo_name, ip_addr) - if action == 'push': - if not HasPermissionAnyMiddleware('repository.write', - 'repository.admin')(user, - repo_name): + # _authenticate is a wrapper for .auth() method of plugin. + # it checks if .auth() sends proper data. + # For RhodeCodeExternalAuthPlugin it also maps users to + # Database and maps the attributes returned from .auth() + # to RhodeCode database. If this function returns data + # then auth is correct. + start = time.time() + log.debug('Running plugin `%s` permissions check', plugin_id) + + def perm_func(): + """ + This function is used internally in Cache of Beaker to calculate + Results + """ + log.debug('auth: calculating permission access now...') + # check IP + inherit = user.inherit_default_permissions + ip_allowed = AuthUser.check_ip_allowed( + user.user_id, ip_addr, inherit_from_default=inherit) + if ip_allowed: + log.info('Access for IP:%s allowed', ip_addr) + else: return False + if action == 'push': + perms = ('repository.write', 'repository.admin') + if not HasPermissionAnyMiddleware(*perms)(user, repo_name): + return False + + else: + # any other action need at least read permission + perms = ( + 'repository.read', 'repository.write', 'repository.admin') + if not HasPermissionAnyMiddleware(*perms)(user, repo_name): + return False + + return True + + if plugin_cache_active: + log.debug('Trying to fetch cached perms by %s', _perm_calc_hash[:6]) + perm_result = cache_manager.get( + _perm_calc_hash, createfunc=perm_func) else: - # any other action need at least read permission - if not HasPermissionAnyMiddleware('repository.read', - 'repository.write', - 'repository.admin')(user, - repo_name): - return False + perm_result = perm_func() - return True + auth_time = time.time() - start + log.debug('Permissions for plugin `%s` completed in %.3fs, ' + 'expiration time of fetched cache %.1fs.', + plugin_id, auth_time, cache_ttl) + + return perm_result def _check_ssl(self, environ, start_response): """ @@ -376,26 +419,41 @@ class SimpleVCS(object): if pre_auth and pre_auth.get('username'): username = pre_auth['username'] log.debug('PRE-AUTH got %s as username', username) + if pre_auth: + log.debug('PRE-AUTH successful from %s', + pre_auth.get('auth_data', {}).get('_plugin')) # If not authenticated by the container, running basic auth # before inject the calling repo_name for special scope checks self.authenticate.acl_repo_name = self.acl_repo_name + + plugin_cache_active, cache_ttl = False, 0 + plugin = None if not username: self.authenticate.realm = self.authenticate.get_rc_realm() try: - result = self.authenticate(environ) + auth_result = self.authenticate(environ) except (UserCreationError, NotAllowedToCreateUserError) as e: log.error(e) reason = safe_str(e) return HTTPNotAcceptable(reason)(environ, start_response) - if isinstance(result, str): + if isinstance(auth_result, dict): AUTH_TYPE.update(environ, 'basic') - REMOTE_USER.update(environ, result) - username = result + REMOTE_USER.update(environ, auth_result['username']) + username = auth_result['username'] + plugin = auth_result.get('auth_data', {}).get('_plugin') + log.info( + 'MAIN-AUTH successful for user `%s` from %s plugin', + username, plugin) + + plugin_cache_active, cache_ttl = auth_result.get( + 'auth_data', {}).get('_ttl_cache') or (False, 0) else: - return result.wsgi_application(environ, start_response) + return auth_result.wsgi_application( + environ, start_response) + # ============================================================== # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME @@ -417,12 +475,13 @@ class SimpleVCS(object): # check permissions for this repository perm = self._check_permission( - action, user, self.acl_repo_name, ip_addr) + action, user, self.acl_repo_name, ip_addr, + plugin, plugin_cache_active, cache_ttl) if not perm: return HTTPForbidden()(environ, start_response) # extras are injected into UI object and later available - # in hooks executed by rhodecode + # in hooks executed by RhodeCode check_locking = _should_check_locking(environ.get('QUERY_STRING')) extras = vcs_operation_context( environ, repo_name=self.acl_repo_name, username=username, diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py --- a/rhodecode/model/db.py +++ b/rhodecode/model/db.py @@ -918,7 +918,7 @@ class User(Base, BaseModel): """Update user lastactivity""" self.last_activity = datetime.datetime.now() Session().add(self) - log.debug('updated user %s lastactivity', self.username) + log.debug('updated user `%s` last activity', self.username) def update_password(self, new_password): from rhodecode.lib.auth import get_crypt_password diff --git a/rhodecode/tests/lib/auth_modules/test_auth_modules.py b/rhodecode/tests/lib/auth_modules/test_auth_modules.py --- a/rhodecode/tests/lib/auth_modules/test_auth_modules.py +++ b/rhodecode/tests/lib/auth_modules/test_auth_modules.py @@ -27,8 +27,13 @@ from rhodecode.authentication.plugins.au from rhodecode.model import db +class TestAuthPlugin(RhodeCodeAuthPluginBase): + + def name(self): + return 'stub_auth' + def test_authenticate_returns_from_auth(stub_auth_data): - plugin = RhodeCodeAuthPluginBase('stub_id') + plugin = TestAuthPlugin('stub_id') with mock.patch.object(plugin, 'auth') as auth_mock: auth_mock.return_value = stub_auth_data result = plugin._authenticate(mock.Mock(), 'test', 'password', {}) @@ -37,7 +42,7 @@ def test_authenticate_returns_from_auth( def test_authenticate_returns_empty_auth_data(): auth_data = {} - plugin = RhodeCodeAuthPluginBase('stub_id') + plugin = TestAuthPlugin('stub_id') with mock.patch.object(plugin, 'auth') as auth_mock: auth_mock.return_value = auth_data result = plugin._authenticate(mock.Mock(), 'test', 'password', {}) @@ -46,7 +51,7 @@ def test_authenticate_returns_empty_auth def test_authenticate_skips_hash_migration_if_mismatch(stub_auth_data): stub_auth_data['_hash_migrate'] = 'new-hash' - plugin = RhodeCodeAuthPluginBase('stub_id') + plugin = TestAuthPlugin('stub_id') with mock.patch.object(plugin, 'auth') as auth_mock: auth_mock.return_value = stub_auth_data result = plugin._authenticate(mock.Mock(), 'test', 'password', {}) @@ -60,7 +65,7 @@ def test_authenticate_migrates_to_new_ha new_password = b'new-password' new_hash = _RhodeCodeCryptoBCrypt().hash_create(new_password) stub_auth_data['_hash_migrate'] = new_hash - plugin = RhodeCodeAuthPluginBase('stub_id') + plugin = TestAuthPlugin('stub_id') with mock.patch.object(plugin, 'auth') as auth_mock: auth_mock.return_value = stub_auth_data result = plugin._authenticate(