diff --git a/configs/development.ini b/configs/development.ini --- a/configs/development.ini +++ b/configs/development.ini @@ -320,12 +320,7 @@ cache_dir = %(here)s/data beaker.cache.data_dir = %(here)s/data/cache/beaker_data beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock -beaker.cache.regions = short_term, long_term, sql_cache_short, auth_plugins, repo_cache_long - -# used for caching user permissions -beaker.cache.short_term.type = file -beaker.cache.short_term.expire = 0 -beaker.cache.short_term.key_length = 256 +beaker.cache.regions = long_term, sql_cache_short, repo_cache_long beaker.cache.long_term.type = memory beaker.cache.long_term.expire = 36000 @@ -335,16 +330,6 @@ beaker.cache.sql_cache_short.type = memo beaker.cache.sql_cache_short.expire = 10 beaker.cache.sql_cache_short.key_length = 256 -## default is memory cache, configure only if required -## using multi-node or multi-worker setup -#beaker.cache.auth_plugins.type = ext:database -#beaker.cache.auth_plugins.lock_dir = %(here)s/data/cache/auth_plugin_lock -#beaker.cache.auth_plugins.url = postgresql://postgres:secret@localhost/rhodecode -#beaker.cache.auth_plugins.url = mysql://root:secret@127.0.0.1/rhodecode -#beaker.cache.auth_plugins.sa.pool_recycle = 3600 -#beaker.cache.auth_plugins.sa.pool_size = 10 -#beaker.cache.auth_plugins.sa.max_overflow = 0 - beaker.cache.repo_cache_long.type = memorylru_base beaker.cache.repo_cache_long.max_items = 4096 beaker.cache.repo_cache_long.expire = 2592000 diff --git a/configs/production.ini b/configs/production.ini --- a/configs/production.ini +++ b/configs/production.ini @@ -295,12 +295,7 @@ cache_dir = %(here)s/data beaker.cache.data_dir = %(here)s/data/cache/beaker_data beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock -beaker.cache.regions = short_term, long_term, sql_cache_short, auth_plugins, repo_cache_long - -# used for caching user permissions -beaker.cache.short_term.type = file -beaker.cache.short_term.expire = 0 -beaker.cache.short_term.key_length = 256 +beaker.cache.regions = long_term, sql_cache_short, repo_cache_long beaker.cache.long_term.type = memory beaker.cache.long_term.expire = 36000 @@ -310,16 +305,6 @@ beaker.cache.sql_cache_short.type = memo beaker.cache.sql_cache_short.expire = 10 beaker.cache.sql_cache_short.key_length = 256 -## default is memory cache, configure only if required -## using multi-node or multi-worker setup -#beaker.cache.auth_plugins.type = ext:database -#beaker.cache.auth_plugins.lock_dir = %(here)s/data/cache/auth_plugin_lock -#beaker.cache.auth_plugins.url = postgresql://postgres:secret@localhost/rhodecode -#beaker.cache.auth_plugins.url = mysql://root:secret@127.0.0.1/rhodecode -#beaker.cache.auth_plugins.sa.pool_recycle = 3600 -#beaker.cache.auth_plugins.sa.pool_size = 10 -#beaker.cache.auth_plugins.sa.max_overflow = 0 - beaker.cache.repo_cache_long.type = memorylru_base beaker.cache.repo_cache_long.max_items = 4096 beaker.cache.repo_cache_long.expire = 2592000 diff --git a/rhodecode/apps/admin/__init__.py b/rhodecode/apps/admin/__init__.py --- a/rhodecode/apps/admin/__init__.py +++ b/rhodecode/apps/admin/__init__.py @@ -356,6 +356,16 @@ def admin_routes(config): name='edit_user_audit_logs', pattern='/users/{user_id:\d+}/edit/audit', user_route=True) + # user caches + config.add_route( + name='edit_user_caches', + pattern='/users/{user_id:\d+}/edit/caches', + user_route=True) + config.add_route( + name='edit_user_caches_update', + pattern='/users/{user_id:\d+}/edit/caches/update', + user_route=True) + # user-groups admin config.add_route( name='user_groups', diff --git a/rhodecode/apps/admin/views/users.py b/rhodecode/apps/admin/views/users.py --- a/rhodecode/apps/admin/views/users.py +++ b/rhodecode/apps/admin/views/users.py @@ -34,7 +34,7 @@ from rhodecode.authentication.plugins im from rhodecode.events import trigger from rhodecode.model.db import true -from rhodecode.lib import audit_logger +from rhodecode.lib import audit_logger, rc_cache from rhodecode.lib.exceptions import ( UserCreationError, UserOwnsReposException, UserOwnsRepoGroupsException, UserOwnsUserGroupsException, DefaultUserException) @@ -1198,3 +1198,57 @@ class UsersView(UserAppView): perm_user = self.db_user.AuthUser(ip_addr=self.request.remote_addr) return perm_user.permissions + + def _get_user_cache_keys(self, cache_namespace_uid, keys): + user_keys = [] + for k in sorted(keys): + if k.startswith(cache_namespace_uid): + user_keys.append(k) + return user_keys + + @LoginRequired() + @HasPermissionAllDecorator('hg.admin') + @view_config( + route_name='edit_user_caches', request_method='GET', + renderer='rhodecode:templates/admin/users/user_edit.mako') + def user_caches(self): + _ = self.request.translate + c = self.load_default_context() + c.user = self.db_user + + c.active = 'caches' + c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr) + + cache_namespace_uid = 'cache_user_auth.{}'.format(self.db_user.user_id) + c.region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid) + c.backend = c.region.backend + c.user_keys = self._get_user_cache_keys( + cache_namespace_uid, c.region.backend.list_keys()) + + return self._get_template_context(c) + + @LoginRequired() + @HasPermissionAllDecorator('hg.admin') + @CSRFRequired() + @view_config( + route_name='edit_user_caches_update', request_method='POST') + def user_caches_update(self): + _ = self.request.translate + c = self.load_default_context() + c.user = self.db_user + + c.active = 'caches' + c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr) + + cache_namespace_uid = 'cache_user_auth.{}'.format(self.db_user.user_id) + c.region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid) + + c.user_keys = self._get_user_cache_keys( + cache_namespace_uid, c.region.backend.list_keys()) + for k in c.user_keys: + c.region.delete(k) + + h.flash(_("Deleted {} cache keys").format(len(c.user_keys)), category='success') + + return HTTPFound(h.route_path( + 'edit_user_caches', user_id=c.user.user_id)) diff --git a/rhodecode/authentication/base.py b/rhodecode/authentication/base.py --- a/rhodecode/authentication/base.py +++ b/rhodecode/authentication/base.py @@ -35,7 +35,7 @@ from pyramid.threadlocal import get_curr from rhodecode.authentication.interface import IAuthnPluginRegistry from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase -from rhodecode.lib import caches +from rhodecode.lib import caches, rc_cache from rhodecode.lib.auth import PasswordGenerator, _RhodeCodeCryptoBCrypt from rhodecode.lib.utils2 import safe_int, safe_str from rhodecode.lib.exceptions import LdapConnectionError @@ -633,22 +633,6 @@ def get_authn_registry(registry=None): return authn_registry -def get_auth_cache_manager(custom_ttl=None, suffix=None): - cache_name = 'rhodecode.authentication' - if suffix: - cache_name = 'rhodecode.authentication.{}'.format(suffix) - return caches.get_cache_manager( - 'auth_plugins', cache_name, custom_ttl) - - -def get_perms_cache_manager(custom_ttl=None, suffix=None): - cache_name = 'rhodecode.permissions' - if suffix: - cache_name = 'rhodecode.permissions.{}'.format(suffix) - return caches.get_cache_manager( - 'auth_plugins', cache_name, custom_ttl) - - def authenticate(username, password, environ=None, auth_type=None, skip_missing=False, registry=None, acl_repo_name=None): """ @@ -707,45 +691,35 @@ def authenticate(username, password, env plugin_cache_active, cache_ttl = plugin.get_ttl_cache(plugin_settings) - # get instance of cache manager configured for a namespace - cache_manager = get_auth_cache_manager( - custom_ttl=cache_ttl, suffix=user.user_id if user else '') - log.debug('AUTH_CACHE_TTL for plugin `%s` active: %s (TTL: %s)', plugin.get_id(), plugin_cache_active, cache_ttl) - # for environ based password can be empty, but then the validation is - # on the server that fills in the env data needed for authentication - - _password_hash = caches.compute_key_from_params( - plugin.name, username, (password or '')) + user_id = user.user_id + cache_namespace_uid = 'cache_user_auth.{}'.format(user_id) + region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid) - # _authenticate is a wrapper for .auth() method of plugin. - # it checks if .auth() sends proper data. - # For RhodeCodeExternalAuthPlugin it also maps users to - # Database and maps the attributes returned from .auth() - # to RhodeCode database. If this function returns data - # then auth is correct. - start = time.time() - log.debug('Running plugin `%s` _authenticate method', plugin.get_id()) + @region.cache_on_arguments(namespace=cache_namespace_uid, + expiration_time=cache_ttl, + should_cache_fn=lambda v: plugin_cache_active) + def compute_auth( + cache_name, plugin_name, username, password): - def auth_func(): - """ - This function is used internally in Cache of Beaker to calculate - Results - """ - log.debug('auth: calculating password access now...') + # _authenticate is a wrapper for .auth() method of plugin. + # it checks if .auth() sends proper data. + # For RhodeCodeExternalAuthPlugin it also maps users to + # Database and maps the attributes returned from .auth() + # to RhodeCode database. If this function returns data + # then auth is correct. + log.debug('Running plugin `%s` _authenticate method ' + 'using username and password', plugin.get_id()) return plugin._authenticate( user, username, password, plugin_settings, environ=environ or {}) - if plugin_cache_active: - log.debug('Trying to fetch cached auth by pwd hash `...%s`', - _password_hash[:6]) - plugin_user = cache_manager.get( - _password_hash, createfunc=auth_func) - else: - plugin_user = auth_func() + start = time.time() + # for environ based auth, password can be empty, but then the validation is + # on the server that fills in the env data needed for authentication + plugin_user = compute_auth('auth', plugin.name, username, (password or '')) auth_time = time.time() - start log.debug('Authentication for plugin `%s` completed in %.3fs, ' diff --git a/rhodecode/authentication/views.py b/rhodecode/authentication/views.py --- a/rhodecode/authentication/views.py +++ b/rhodecode/authentication/views.py @@ -27,8 +27,7 @@ from pyramid.renderers import render from pyramid.response import Response from rhodecode.apps._base import BaseAppView -from rhodecode.authentication.base import ( - get_auth_cache_manager, get_perms_cache_manager, get_authn_registry) +from rhodecode.authentication.base import get_authn_registry from rhodecode.lib import helpers as h from rhodecode.lib.auth import ( LoginRequired, HasPermissionAllDecorator, CSRFRequired) @@ -102,16 +101,6 @@ class AuthnPluginViewBase(BaseAppView): self.plugin.create_or_update_setting(name, value) Session().commit() - # cleanup cache managers in case of change for plugin - # TODO(marcink): because we can register multiple namespaces - # we should at some point figure out how to retrieve ALL namespace - # cache managers and clear them... - cache_manager = get_auth_cache_manager() - clear_cache_manager(cache_manager) - - cache_manager = get_perms_cache_manager() - clear_cache_manager(cache_manager) - # Display success message and redirect. h.flash(_('Auth settings updated successfully.'), category='success') redirect_to = self.request.resource_path( diff --git a/rhodecode/config/middleware.py b/rhodecode/config/middleware.py --- a/rhodecode/config/middleware.py +++ b/rhodecode/config/middleware.py @@ -22,6 +22,7 @@ import os import logging import traceback import collections +import tempfile from paste.gzipper import make_gzip_middleware from pyramid.wsgi import wsgiapp @@ -220,6 +221,7 @@ def includeme(config): config.include('pyramid_mako') config.include('pyramid_beaker') config.include('rhodecode.lib.caches') + config.include('rhodecode.lib.rc_cache') config.include('rhodecode.authentication') config.include('rhodecode.integrations') @@ -382,6 +384,7 @@ def sanitize_settings_and_apply_defaults # Call split out functions that sanitize settings for each topic. _sanitize_appenlight_settings(settings) _sanitize_vcs_settings(settings) + _sanitize_cache_settings(settings) # configure instance id config_utils.set_instance_id(settings) @@ -421,6 +424,18 @@ def _sanitize_vcs_settings(settings): settings['vcs.scm_app_implementation'] = 'http' +def _sanitize_cache_settings(settings): + _string_setting(settings, 'cache_dir', + os.path.join(tempfile.gettempdir(), 'rc_cache')) + + _string_setting(settings, 'rc_cache.cache_perms.backend', + 'dogpile.cache.rc.file_namespace') + _int_setting(settings, 'rc_cache.cache_perms.expiration_time', + 60) + _string_setting(settings, 'rc_cache.cache_perms.arguments.filename', + os.path.join(tempfile.gettempdir(), 'rc_cache_1')) + + def _int_setting(settings, name, default): settings[name] = int(settings.get(name, default)) diff --git a/rhodecode/lib/auth.py b/rhodecode/lib/auth.py --- a/rhodecode/lib/auth.py +++ b/rhodecode/lib/auth.py @@ -48,7 +48,7 @@ from rhodecode.model.user import UserMod from rhodecode.model.db import ( User, Repository, Permission, UserToPerm, UserGroupToPerm, UserGroupMember, UserIpMap, UserApiKeys, RepoGroup, UserGroup) -from rhodecode.lib import caches +from rhodecode.lib import rc_cache from rhodecode.lib.utils2 import safe_unicode, aslist, safe_str, md5, safe_int, sha1 from rhodecode.lib.utils import ( get_repo_slug, get_repo_group_slug, get_user_group_slug) @@ -976,21 +976,18 @@ class AuthUser(object): obj = Repository.get_by_repo_name(scope['repo_name']) if obj: scope['repo_id'] = obj.repo_id - _scope = { - 'repo_id': -1, - 'user_group_id': -1, - 'repo_group_id': -1, - } - _scope.update(scope) - cache_key = "_".join(map(safe_str, reduce(lambda a, b: a+b, - _scope.items()))) - if cache_key not in self._permissions_scoped_cache: - # store in cache to mimic how the @LazyProperty works, - # the difference here is that we use the unique key calculated - # from params and values - res = self.get_perms(user=self, cache=False, scope=_scope) - self._permissions_scoped_cache[cache_key] = res - return self._permissions_scoped_cache[cache_key] + _scope = collections.OrderedDict() + _scope['repo_id'] = -1 + _scope['user_group_id'] = -1 + _scope['repo_group_id'] = -1 + + for k in sorted(scope.keys()): + _scope[k] = scope[k] + + # store in cache to mimic how the @LazyProperty works, + # the difference here is that we use the unique key calculated + # from params and values + return self.get_perms(user=self, cache=False, scope=_scope) def get_instance(self): return User.get(self.user_id) @@ -1072,16 +1069,27 @@ class AuthUser(object): user_inherit_default_permissions = user.inherit_default_permissions cache_seconds = safe_int( - rhodecode.CONFIG.get('beaker.cache.short_term.expire')) + rhodecode.CONFIG.get('rc_cache.cache_perms.expiration_time')) + cache_on = cache or cache_seconds > 0 log.debug( 'Computing PERMISSION tree for user %s scope `%s` ' - 'with caching: %s[%ss]' % (user, scope, cache_on, cache_seconds)) + 'with caching: %s[TTL: %ss]' % (user, scope, cache_on, cache_seconds or 0)) + + cache_namespace_uid = 'cache_user_auth.{}'.format(user_id) + region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid) + + @region.cache_on_arguments(namespace=cache_namespace_uid, + should_cache_fn=lambda v: cache_on) + def compute_perm_tree(cache_name, + user_id, scope, user_is_admin,user_inherit_default_permissions, + explicit, algo, calculate_super_admin): + return _cached_perms_data( + user_id, scope, user_is_admin, user_inherit_default_permissions, + explicit, algo, calculate_super_admin) + start = time.time() - compute = caches.conditional_cache( - 'short_term', 'cache_desc.{}'.format(user_id), - condition=cache_on, func=_cached_perms_data) - result = compute(user_id, scope, user_is_admin, + result = compute_perm_tree('permissions', user_id, scope, user_is_admin, user_inherit_default_permissions, explicit, algo, calculate_super_admin) @@ -1091,6 +1099,7 @@ class AuthUser(object): total = time.time() - start log.debug('PERMISSION tree for user %s computed in %.3fs: %s' % ( user, total, result_repr)) + return result @property @@ -1153,10 +1162,7 @@ class AuthUser(object): return [x.repo_id for x in RepoList(qry, perm_set=perm_def)] - compute = caches.conditional_cache( - 'short_term', 'repo_acl_ids.{}'.format(self.user_id), - condition=cache, func=_cached_repo_acl) - return compute(self.user_id, perms, name_filter) + return _cached_repo_acl(self.user_id, perms, name_filter) def repo_group_acl_ids(self, perms=None, name_filter=None, cache=False): """ @@ -1179,10 +1185,7 @@ class AuthUser(object): return [x.group_id for x in RepoGroupList(qry, perm_set=perm_def)] - compute = caches.conditional_cache( - 'short_term', 'repo_group_acl_ids.{}'.format(self.user_id), - condition=cache, func=_cached_repo_group_acl) - return compute(self.user_id, perms, name_filter) + return _cached_repo_group_acl(self.user_id, perms, name_filter) def user_group_acl_ids(self, perms=None, name_filter=None, cache=False): """ @@ -1205,10 +1208,7 @@ class AuthUser(object): return [x.users_group_id for x in UserGroupList(qry, perm_set=perm_def)] - compute = caches.conditional_cache( - 'short_term', 'user_group_acl_ids.{}'.format(self.user_id), - condition=cache, func=_cached_user_group_acl) - return compute(self.user_id, perms, name_filter) + return _cached_user_group_acl(self.user_id, perms, name_filter) @property def ip_allowed(self): diff --git a/rhodecode/lib/base.py b/rhodecode/lib/base.py --- a/rhodecode/lib/base.py +++ b/rhodecode/lib/base.py @@ -505,6 +505,7 @@ def bootstrap_config(request): config.include('pyramid_mako') config.include('pyramid_beaker') config.include('rhodecode.lib.caches') + config.include('rhodecode.lib.rc_cache') add_events_routes(config) diff --git a/rhodecode/lib/caches.py b/rhodecode/lib/caches.py --- a/rhodecode/lib/caches.py +++ b/rhodecode/lib/caches.py @@ -96,7 +96,7 @@ def get_cache_manager(region_name, cache Creates a Beaker cache manager. Such instance can be used like that:: _namespace = caches.get_repo_namespace_key(caches.XXX, repo_name) - cache_manager = caches.get_cache_manager('repo_cache_long', _namespace) + cache_manager = caches.get_cache_manager('some_namespace_name', _namespace) _cache_key = caches.compute_key_from_params(repo_name, commit.raw_id) def heavy_compute(): ... @@ -121,7 +121,7 @@ def get_cache_manager(region_name, cache def clear_cache_manager(cache_manager): """ namespace = 'foobar' - cache_manager = get_cache_manager('repo_cache_long', namespace) + cache_manager = get_cache_manager('some_namespace_name', namespace) clear_cache_manager(cache_manager) """ diff --git a/rhodecode/lib/middleware/simplevcs.py b/rhodecode/lib/middleware/simplevcs.py --- a/rhodecode/lib/middleware/simplevcs.py +++ b/rhodecode/lib/middleware/simplevcs.py @@ -39,9 +39,8 @@ from pyramid.httpexceptions import ( from zope.cachedescriptors.property import Lazy as LazyProperty import rhodecode -from rhodecode.authentication.base import ( - authenticate, get_perms_cache_manager, VCS_TYPE, loadplugin) -from rhodecode.lib import caches +from rhodecode.authentication.base import authenticate, VCS_TYPE, loadplugin +from rhodecode.lib import caches, rc_cache from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware from rhodecode.lib.base import ( BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context) @@ -311,36 +310,24 @@ class SimpleVCS(object): :param repo_name: repository name """ - # get instance of cache manager configured for a namespace - cache_manager = get_perms_cache_manager( - custom_ttl=cache_ttl, suffix=user.user_id) log.debug('AUTH_CACHE_TTL for permissions `%s` active: %s (TTL: %s)', plugin_id, plugin_cache_active, cache_ttl) - # for environ based password can be empty, but then the validation is - # on the server that fills in the env data needed for authentication - _perm_calc_hash = caches.compute_key_from_params( - plugin_id, action, user.user_id, repo_name, ip_addr) + user_id = user.user_id + cache_namespace_uid = 'cache_user_auth.{}'.format(user_id) + region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid) - # _authenticate is a wrapper for .auth() method of plugin. - # it checks if .auth() sends proper data. - # For RhodeCodeExternalAuthPlugin it also maps users to - # Database and maps the attributes returned from .auth() - # to RhodeCode database. If this function returns data - # then auth is correct. - start = time.time() - log.debug('Running plugin `%s` permissions check', plugin_id) + @region.cache_on_arguments(namespace=cache_namespace_uid, + expiration_time=cache_ttl, + should_cache_fn=lambda v: plugin_cache_active) + def compute_perm_vcs( + cache_name, plugin_id, action, user_id, repo_name, ip_addr): - def perm_func(): - """ - This function is used internally in Cache of Beaker to calculate - Results - """ log.debug('auth: calculating permission access now...') # check IP inherit = user.inherit_default_permissions ip_allowed = AuthUser.check_ip_allowed( - user.user_id, ip_addr, inherit_from_default=inherit) + user_id, ip_addr, inherit_from_default=inherit) if ip_allowed: log.info('Access for IP:%s allowed', ip_addr) else: @@ -360,12 +347,13 @@ class SimpleVCS(object): return True - if plugin_cache_active: - log.debug('Trying to fetch cached perms by %s', _perm_calc_hash[:6]) - perm_result = cache_manager.get( - _perm_calc_hash, createfunc=perm_func) - else: - perm_result = perm_func() + start = time.time() + log.debug('Running plugin `%s` permissions check', plugin_id) + + # for environ based auth, password can be empty, but then the validation is + # on the server that fills in the env data needed for authentication + perm_result = compute_perm_vcs( + 'vcs_permissions', plugin_id, action, user.user_id, repo_name, ip_addr) auth_time = time.time() - start log.debug('Permissions for plugin `%s` completed in %.3fs, ' diff --git a/rhodecode/lib/rc_cache/__init__.py b/rhodecode/lib/rc_cache/__init__.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/rc_cache/__init__.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2015-2018 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +from dogpile.cache import register_backend +from dogpile.cache import make_region + +register_backend( + "dogpile.cache.rc.memory_lru", "rhodecode.lib.rc_cache.backends", + "LRUMemoryBackend") + +register_backend( + "dogpile.cache.rc.file_namespace", "rhodecode.lib.rc_cache.backends", + "FileNamespaceBackend") + +register_backend( + "dogpile.cache.rc.redis", "rhodecode.lib.rc_cache.backends", + "RedisPickleBackend") + + +from . import region_meta +from .utils import get_default_cache_settings, key_generator, get_or_create_region + + +def configure_dogpile_cache(settings): + cache_dir = settings.get('cache_dir') + if cache_dir: + region_meta.dogpile_config_defaults['cache_dir'] = cache_dir + + rc_cache_data = get_default_cache_settings(settings, prefixes=['rc_cache.']) + + # inspect available namespaces + avail_regions = set() + for key in rc_cache_data.keys(): + namespace_name = key.split('.', 1)[0] + avail_regions.add(namespace_name) + + # register them into namespace + for region_name in avail_regions: + new_region = make_region( + name=region_name, + function_key_generator=key_generator + ) + + new_region.configure_from_config(settings, 'rc_cache.{}.'.format(region_name)) + region_meta.dogpile_cache_regions[region_name] = new_region + + +def includeme(config): + configure_dogpile_cache(config.registry.settings) diff --git a/rhodecode/lib/rc_cache/backends.py b/rhodecode/lib/rc_cache/backends.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/rc_cache/backends.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2015-2018 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +from dogpile.cache.backends import memory as memory_backend +from dogpile.cache.backends import file as file_backend +from dogpile.cache.backends import redis as redis_backend +from dogpile.cache.backends.file import NO_VALUE, compat + +from rhodecode.lib.memory_lru_debug import LRUDict + +_default_max_size = 1024 + + +class LRUMemoryBackend(memory_backend.MemoryBackend): + + def __init__(self, arguments): + max_size = arguments.pop('max_size', _default_max_size) + arguments['cache_dict'] = LRUDict(max_size) + super(LRUMemoryBackend, self).__init__(arguments) + + +class Serializer(object): + def _dumps(self, value): + return compat.pickle.dumps(value) + + def _loads(self, value): + return compat.pickle.loads(value) + + +class FileNamespaceBackend(Serializer, file_backend.DBMBackend): + + def __init__(self, arguments): + super(FileNamespaceBackend, self).__init__(arguments) + + def list_keys(self): + with self._dbm_file(True) as dbm: + return dbm.keys() + + def get_store(self): + return self.filename + + def get(self, key): + with self._dbm_file(False) as dbm: + if hasattr(dbm, 'get'): + value = dbm.get(key, NO_VALUE) + else: + # gdbm objects lack a .get method + try: + value = dbm[key] + except KeyError: + value = NO_VALUE + if value is not NO_VALUE: + value = self._loads(value) + return value + + def set(self, key, value): + with self._dbm_file(True) as dbm: + dbm[key] = self._dumps(value) + + def set_multi(self, mapping): + with self._dbm_file(True) as dbm: + for key, value in mapping.items(): + dbm[key] = self._dumps(value) + + +class RedisPickleBackend(Serializer, redis_backend.RedisBackend): + def list_keys(self): + return self.client.keys() + + def get_store(self): + return self.client.connection_pool + + def set(self, key, value): + if self.redis_expiration_time: + self.client.setex(key, self.redis_expiration_time, + self._dumps(value)) + else: + self.client.set(key, self._dumps(value)) + + def set_multi(self, mapping): + mapping = dict( + (k, self._dumps(v)) + for k, v in mapping.items() + ) + + if not self.redis_expiration_time: + self.client.mset(mapping) + else: + pipe = self.client.pipeline() + for key, value in mapping.items(): + pipe.setex(key, self.redis_expiration_time, value) + pipe.execute() diff --git a/rhodecode/lib/rc_cache/region_meta.py b/rhodecode/lib/rc_cache/region_meta.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/rc_cache/region_meta.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2015-2018 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ +import os +import tempfile + +dogpile_config_defaults = { + 'cache_dir': os.path.join(tempfile.gettempdir(), 'rc_cache') +} + +# GLOBAL TO STORE ALL REGISTERED REGIONS +dogpile_cache_regions = {} diff --git a/rhodecode/lib/rc_cache/utils.py b/rhodecode/lib/rc_cache/utils.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/rc_cache/utils.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2015-2018 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ +import os +import logging +from dogpile.cache import make_region + +from rhodecode.lib.utils import safe_str, sha1 +from . import region_meta + +log = logging.getLogger(__name__) + + +def get_default_cache_settings(settings, prefixes=None): + prefixes = prefixes or [] + cache_settings = {} + for key in settings.keys(): + for prefix in prefixes: + if key.startswith(prefix): + name = key.split(prefix)[1].strip() + val = settings[key] + if isinstance(val, basestring): + val = val.strip() + cache_settings[name] = val + return cache_settings + + +def compute_key_from_params(*args): + """ + Helper to compute key from given params to be used in cache manager + """ + return sha1("_".join(map(safe_str, args))) + + +def key_generator(namespace, fn): + fname = fn.__name__ + + def generate_key(*args): + namespace_pref = namespace or 'default' + arg_key = compute_key_from_params(*args) + final_key = "{}:{}_{}".format(namespace_pref, fname, arg_key) + + return final_key + + return generate_key + + +def get_or_create_region(region_name, region_namespace=None): + from rhodecode.lib.rc_cache.backends import FileNamespaceBackend + region_obj = region_meta.dogpile_cache_regions.get(region_name) + if not region_obj: + raise EnvironmentError( + 'Region `{}` not in configured: {}.'.format( + region_name, region_meta.dogpile_cache_regions.keys())) + + region_uid_name = '{}:{}'.format(region_name, region_namespace) + if isinstance(region_obj.actual_backend, FileNamespaceBackend): + region_exist = region_meta.dogpile_cache_regions.get(region_namespace) + if region_exist: + log.debug('Using already configured region: %s', region_namespace) + return region_exist + cache_dir = region_meta.dogpile_config_defaults['cache_dir'] + expiration_time = region_obj.expiration_time + + if not os.path.isdir(cache_dir): + os.makedirs(cache_dir) + new_region = make_region( + name=region_uid_name, function_key_generator=key_generator + ) + namespace_filename = os.path.join( + cache_dir, "{}.cache.dbm".format(region_namespace)) + # special type that allows 1db per namespace + new_region.configure( + backend='dogpile.cache.rc.file_namespace', + expiration_time=expiration_time, + arguments={"filename": namespace_filename} + ) + + # create and save in region caches + log.debug('configuring new region: %s',region_uid_name) + region_obj = region_meta.dogpile_cache_regions[region_namespace] = new_region + + return region_obj diff --git a/rhodecode/public/js/rhodecode/routes.js b/rhodecode/public/js/rhodecode/routes.js --- a/rhodecode/public/js/rhodecode/routes.js +++ b/rhodecode/public/js/rhodecode/routes.js @@ -119,6 +119,8 @@ function registerRCRoutes() { pyroutes.register('edit_user_groups_management', '/_admin/users/%(user_id)s/edit/groups_management', ['user_id']); pyroutes.register('edit_user_groups_management_updates', '/_admin/users/%(user_id)s/edit/edit_user_groups_management/updates', ['user_id']); pyroutes.register('edit_user_audit_logs', '/_admin/users/%(user_id)s/edit/audit', ['user_id']); + pyroutes.register('edit_user_caches', '/_admin/users/%(user_id)s/edit/caches', ['user_id']); + pyroutes.register('edit_user_caches_update', '/_admin/users/%(user_id)s/edit/caches/update', ['user_id']); pyroutes.register('user_groups', '/_admin/user_groups', []); pyroutes.register('user_groups_data', '/_admin/user_groups_data', []); pyroutes.register('user_groups_new', '/_admin/user_groups/new', []); diff --git a/rhodecode/templates/admin/users/user_edit.mako b/rhodecode/templates/admin/users/user_edit.mako --- a/rhodecode/templates/admin/users/user_edit.mako +++ b/rhodecode/templates/admin/users/user_edit.mako @@ -45,6 +45,7 @@
  • ${_('Ip Whitelist')}
  • ${_('User Groups Management')}
  • ${_('Audit logs')}
    • +
  • ${_('Caches')}
    • diff --git a/rhodecode/templates/admin/users/user_edit_caches.mako b/rhodecode/templates/admin/users/user_edit_caches.mako new file mode 100644 --- /dev/null +++ b/rhodecode/templates/admin/users/user_edit_caches.mako @@ -0,0 +1,29 @@ +<%namespace name="base" file="/base/base.mako"/> + +
    +
    +

    ${_('Caches')}

    +
    +
    + 
    +region: ${c.region.name}
+backend: ${c.region.actual_backend.__class__}
+store: ${c.region.actual_backend.get_store()}
+
+% for k in c.user_keys:
+ - ${k}
+% endfor
+
    + + ${h.secure_form(h.route_path('edit_user_caches_update', user_id=c.user.user_id), request=request)} +
    +
    + ${h.submit('reset_cache_%s' % c.user.user_id, _('Invalidate user cache'),class_="btn btn-small",onclick="return confirm('"+_('Confirm to invalidate user cache')+"');")} +
    +
    + ${h.end_form()} + +
    +
    + + diff --git a/rhodecode/tests/lib/test_caches.py b/rhodecode/tests/lib/test_caches.py --- a/rhodecode/tests/lib/test_caches.py +++ b/rhodecode/tests/lib/test_caches.py @@ -22,36 +22,24 @@ import time import pytest -from rhodecode.lib import caches -from rhodecode.lib.memory_lru_debug import MemoryLRUNamespaceManagerBase +from rhodecode.lib import rc_cache -class TestCacheManager(object): +@pytest.mark.usefixtures( 'app') +class TestCaches(object): - @pytest.mark.parametrize('repo_name', [ - ('',), - (u'',), - (u'ac',), - ('ac', ), - (u'ęćc',), - ('ąac',), + def test_cache_decorator_init_not_configured(self): + with pytest.raises(EnvironmentError): + rc_cache.get_or_create_region('dontexist') + + @pytest.mark.parametrize('region_name', [ + 'cache_perms', u'cache_perms', ]) - def test_cache_manager_init(self, repo_name): - cache_manager_instance = caches.get_cache_manager( - repo_name, 'my_cache') - assert cache_manager_instance - - def test_cache_manager_init_undefined_namespace(self): - cache_manager_instance = caches.get_cache_manager( - 'repo_cache_long_undefined', 'my_cache') - assert cache_manager_instance - - def_config = caches.DEFAULT_CACHE_MANAGER_CONFIG.copy() - def_config.pop('type') - assert cache_manager_instance.nsargs == def_config - - assert isinstance( - cache_manager_instance.namespace, MemoryLRUNamespaceManagerBase) + def test_cache_decorator_init(self, region_name): + namespace = region_name + cache_region = rc_cache.get_or_create_region( + region_name, region_namespace=namespace) + assert cache_region @pytest.mark.parametrize('example_input', [ ('',), @@ -62,45 +50,60 @@ class TestCacheManager(object): (u'/ac', ), ]) def test_cache_manager_create_key(self, example_input): - key = caches.compute_key_from_params(*example_input) + key = rc_cache.utils.compute_key_from_params(*example_input) assert key - def test_store_and_invalidate_value_from_manager(self): - cache_manger_instance = caches.get_cache_manager( - 'repo_cache_long', 'my_cache_store') + @pytest.mark.parametrize('example_namespace', [ + 'namespace', None + ]) + @pytest.mark.parametrize('example_input', [ + ('',), + (u'/ac',), + (u'/ac', 1, 2, object()), + (u'/ęćc', 1, 2, object()), + ('/ąac',), + (u'/ac', ), + ]) + def test_cache_keygen(self, example_input, example_namespace): + def func_wrapped(): + return 1 + func = rc_cache.utils.key_generator(example_namespace, func_wrapped) + key = func(*example_input) + assert key - def compute(): + def test_store_value_in_cache(self): + cache_region = rc_cache.get_or_create_region('cache_perms') + # make sure we empty the cache now + for key in cache_region.backend.list_keys(): + cache_region.delete(key) + + assert cache_region.backend.list_keys() == [] + + @cache_region.cache_on_arguments(expiration_time=5) + def compute(key): return time.time() - added_keys = [] - for i in xrange(3): - _cache_key = caches.compute_key_from_params('foo_bar', 'p%s' % i) - added_keys.append(_cache_key) - for x in xrange(10): - cache_manger_instance.get(_cache_key, createfunc=compute) + for x in range(10): + compute(x) + + assert len(set(cache_region.backend.list_keys())) == 10 - for key in added_keys: - assert cache_manger_instance[key] - - caches.clear_cache_manager(cache_manger_instance) - - for key in added_keys: - assert key not in cache_manger_instance - assert len(cache_manger_instance.namespace.keys()) == 0 + def test_store_and_get_value_from_region(self): + cache_region = rc_cache.get_or_create_region('cache_perms') + # make sure we empty the cache now + for key in cache_region.backend.list_keys(): + cache_region.delete(key) + assert cache_region.backend.list_keys() == [] - def test_store_and_get_value_from_manager(self): - cache_manger_instance = caches.get_cache_manager( - 'repo_cache_long', 'my_cache_store') - - _cache_key = caches.compute_key_from_params('foo_bar', 'multicall') + @cache_region.cache_on_arguments(expiration_time=5) + def compute(key): + return time.time() - def compute(): - return time.time() + result = set() + for x in range(10): + ret = compute('x') + result.add(ret) - result = set() - for x in xrange(10): - ret = cache_manger_instance.get(_cache_key, createfunc=compute) - result.add(ret) - - # once computed we have only one value after executing it 10x - assert len(result) == 1 + # once computed we have only one value (the same from cache) + # after executing it 10x + assert len(result) == 1 diff --git a/rhodecode/tests/rhodecode.ini b/rhodecode/tests/rhodecode.ini --- a/rhodecode/tests/rhodecode.ini +++ b/rhodecode/tests/rhodecode.ini @@ -292,11 +292,7 @@ cache_dir = %(here)s/data beaker.cache.data_dir = %(here)s/rc/data/cache/beaker_data beaker.cache.lock_dir = %(here)s/rc/data/cache/beaker_lock -beaker.cache.regions = short_term, long_term, sql_cache_short, auth_plugins, repo_cache_long - -beaker.cache.short_term.type = file -beaker.cache.short_term.expire = 0 -beaker.cache.short_term.key_length = 256 +beaker.cache.regions = long_term, sql_cache_short, repo_cache_long beaker.cache.long_term.type = memory beaker.cache.long_term.expire = 36000 @@ -306,16 +302,6 @@ beaker.cache.sql_cache_short.type = memo beaker.cache.sql_cache_short.expire = 1 beaker.cache.sql_cache_short.key_length = 256 -## default is memory cache, configure only if required -## using multi-node or multi-worker setup -#beaker.cache.auth_plugins.type = memory -#beaker.cache.auth_plugins.lock_dir = %(here)s/data/cache/auth_plugin_lock -#beaker.cache.auth_plugins.url = postgresql://postgres:secret@localhost/rhodecode -#beaker.cache.auth_plugins.url = mysql://root:secret@127.0.0.1/rhodecode -#beaker.cache.auth_plugins.sa.pool_recycle = 3600 -#beaker.cache.auth_plugins.sa.pool_size = 10 -#beaker.cache.auth_plugins.sa.max_overflow = 0 - beaker.cache.repo_cache_long.type = memorylru_base beaker.cache.repo_cache_long.max_items = 4096 beaker.cache.repo_cache_long.expire = 2592000 @@ -327,6 +313,16 @@ beaker.cache.repo_cache_long.expire = 25 #beaker.cache.repo_cache_long.expire = 1209600 #beaker.cache.repo_cache_long.key_length = 256 + +##################################### +### DOGPILE CACHE #### +##################################### + +## permission tree cache settings +rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace +rc_cache.cache_perms.expiration_time = 0 +rc_cache.cache_perms.arguments.filename = /tmp/rc_cache_1 + #################################### ### BEAKER SESSION #### ####################################