diff --git a/configs/development.ini b/configs/development.ini
--- a/configs/development.ini
+++ b/configs/development.ini
@@ -320,12 +320,7 @@ cache_dir = %(here)s/data
beaker.cache.data_dir = %(here)s/data/cache/beaker_data
beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock
-beaker.cache.regions = short_term, long_term, sql_cache_short, auth_plugins, repo_cache_long
-
-# used for caching user permissions
-beaker.cache.short_term.type = file
-beaker.cache.short_term.expire = 0
-beaker.cache.short_term.key_length = 256
+beaker.cache.regions = long_term, sql_cache_short, repo_cache_long
beaker.cache.long_term.type = memory
beaker.cache.long_term.expire = 36000
@@ -335,16 +330,6 @@ beaker.cache.sql_cache_short.type = memo
beaker.cache.sql_cache_short.expire = 10
beaker.cache.sql_cache_short.key_length = 256
-## default is memory cache, configure only if required
-## using multi-node or multi-worker setup
-#beaker.cache.auth_plugins.type = ext:database
-#beaker.cache.auth_plugins.lock_dir = %(here)s/data/cache/auth_plugin_lock
-#beaker.cache.auth_plugins.url = postgresql://postgres:secret@localhost/rhodecode
-#beaker.cache.auth_plugins.url = mysql://root:secret@127.0.0.1/rhodecode
-#beaker.cache.auth_plugins.sa.pool_recycle = 3600
-#beaker.cache.auth_plugins.sa.pool_size = 10
-#beaker.cache.auth_plugins.sa.max_overflow = 0
-
beaker.cache.repo_cache_long.type = memorylru_base
beaker.cache.repo_cache_long.max_items = 4096
beaker.cache.repo_cache_long.expire = 2592000
diff --git a/configs/production.ini b/configs/production.ini
--- a/configs/production.ini
+++ b/configs/production.ini
@@ -295,12 +295,7 @@ cache_dir = %(here)s/data
beaker.cache.data_dir = %(here)s/data/cache/beaker_data
beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock
-beaker.cache.regions = short_term, long_term, sql_cache_short, auth_plugins, repo_cache_long
-
-# used for caching user permissions
-beaker.cache.short_term.type = file
-beaker.cache.short_term.expire = 0
-beaker.cache.short_term.key_length = 256
+beaker.cache.regions = long_term, sql_cache_short, repo_cache_long
beaker.cache.long_term.type = memory
beaker.cache.long_term.expire = 36000
@@ -310,16 +305,6 @@ beaker.cache.sql_cache_short.type = memo
beaker.cache.sql_cache_short.expire = 10
beaker.cache.sql_cache_short.key_length = 256
-## default is memory cache, configure only if required
-## using multi-node or multi-worker setup
-#beaker.cache.auth_plugins.type = ext:database
-#beaker.cache.auth_plugins.lock_dir = %(here)s/data/cache/auth_plugin_lock
-#beaker.cache.auth_plugins.url = postgresql://postgres:secret@localhost/rhodecode
-#beaker.cache.auth_plugins.url = mysql://root:secret@127.0.0.1/rhodecode
-#beaker.cache.auth_plugins.sa.pool_recycle = 3600
-#beaker.cache.auth_plugins.sa.pool_size = 10
-#beaker.cache.auth_plugins.sa.max_overflow = 0
-
beaker.cache.repo_cache_long.type = memorylru_base
beaker.cache.repo_cache_long.max_items = 4096
beaker.cache.repo_cache_long.expire = 2592000
diff --git a/rhodecode/apps/admin/__init__.py b/rhodecode/apps/admin/__init__.py
--- a/rhodecode/apps/admin/__init__.py
+++ b/rhodecode/apps/admin/__init__.py
@@ -356,6 +356,16 @@ def admin_routes(config):
name='edit_user_audit_logs',
pattern='/users/{user_id:\d+}/edit/audit', user_route=True)
+ # user caches
+ config.add_route(
+ name='edit_user_caches',
+ pattern='/users/{user_id:\d+}/edit/caches',
+ user_route=True)
+ config.add_route(
+ name='edit_user_caches_update',
+ pattern='/users/{user_id:\d+}/edit/caches/update',
+ user_route=True)
+
# user-groups admin
config.add_route(
name='user_groups',
diff --git a/rhodecode/apps/admin/views/users.py b/rhodecode/apps/admin/views/users.py
--- a/rhodecode/apps/admin/views/users.py
+++ b/rhodecode/apps/admin/views/users.py
@@ -34,7 +34,7 @@ from rhodecode.authentication.plugins im
from rhodecode.events import trigger
from rhodecode.model.db import true
-from rhodecode.lib import audit_logger
+from rhodecode.lib import audit_logger, rc_cache
from rhodecode.lib.exceptions import (
UserCreationError, UserOwnsReposException, UserOwnsRepoGroupsException,
UserOwnsUserGroupsException, DefaultUserException)
@@ -1198,3 +1198,57 @@ class UsersView(UserAppView):
perm_user = self.db_user.AuthUser(ip_addr=self.request.remote_addr)
return perm_user.permissions
+
+ def _get_user_cache_keys(self, cache_namespace_uid, keys):
+ user_keys = []
+ for k in sorted(keys):
+ if k.startswith(cache_namespace_uid):
+ user_keys.append(k)
+ return user_keys
+
+ @LoginRequired()
+ @HasPermissionAllDecorator('hg.admin')
+ @view_config(
+ route_name='edit_user_caches', request_method='GET',
+ renderer='rhodecode:templates/admin/users/user_edit.mako')
+ def user_caches(self):
+ _ = self.request.translate
+ c = self.load_default_context()
+ c.user = self.db_user
+
+ c.active = 'caches'
+ c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
+
+ cache_namespace_uid = 'cache_user_auth.{}'.format(self.db_user.user_id)
+ c.region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
+ c.backend = c.region.backend
+ c.user_keys = self._get_user_cache_keys(
+ cache_namespace_uid, c.region.backend.list_keys())
+
+ return self._get_template_context(c)
+
+ @LoginRequired()
+ @HasPermissionAllDecorator('hg.admin')
+ @CSRFRequired()
+ @view_config(
+ route_name='edit_user_caches_update', request_method='POST')
+ def user_caches_update(self):
+ _ = self.request.translate
+ c = self.load_default_context()
+ c.user = self.db_user
+
+ c.active = 'caches'
+ c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
+
+ cache_namespace_uid = 'cache_user_auth.{}'.format(self.db_user.user_id)
+ c.region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
+
+ c.user_keys = self._get_user_cache_keys(
+ cache_namespace_uid, c.region.backend.list_keys())
+ for k in c.user_keys:
+ c.region.delete(k)
+
+ h.flash(_("Deleted {} cache keys").format(len(c.user_keys)), category='success')
+
+ return HTTPFound(h.route_path(
+ 'edit_user_caches', user_id=c.user.user_id))
diff --git a/rhodecode/authentication/base.py b/rhodecode/authentication/base.py
--- a/rhodecode/authentication/base.py
+++ b/rhodecode/authentication/base.py
@@ -35,7 +35,7 @@ from pyramid.threadlocal import get_curr
from rhodecode.authentication.interface import IAuthnPluginRegistry
from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase
-from rhodecode.lib import caches
+from rhodecode.lib import caches, rc_cache
from rhodecode.lib.auth import PasswordGenerator, _RhodeCodeCryptoBCrypt
from rhodecode.lib.utils2 import safe_int, safe_str
from rhodecode.lib.exceptions import LdapConnectionError
@@ -633,22 +633,6 @@ def get_authn_registry(registry=None):
return authn_registry
-def get_auth_cache_manager(custom_ttl=None, suffix=None):
- cache_name = 'rhodecode.authentication'
- if suffix:
- cache_name = 'rhodecode.authentication.{}'.format(suffix)
- return caches.get_cache_manager(
- 'auth_plugins', cache_name, custom_ttl)
-
-
-def get_perms_cache_manager(custom_ttl=None, suffix=None):
- cache_name = 'rhodecode.permissions'
- if suffix:
- cache_name = 'rhodecode.permissions.{}'.format(suffix)
- return caches.get_cache_manager(
- 'auth_plugins', cache_name, custom_ttl)
-
-
def authenticate(username, password, environ=None, auth_type=None,
skip_missing=False, registry=None, acl_repo_name=None):
"""
@@ -707,45 +691,35 @@ def authenticate(username, password, env
plugin_cache_active, cache_ttl = plugin.get_ttl_cache(plugin_settings)
- # get instance of cache manager configured for a namespace
- cache_manager = get_auth_cache_manager(
- custom_ttl=cache_ttl, suffix=user.user_id if user else '')
-
log.debug('AUTH_CACHE_TTL for plugin `%s` active: %s (TTL: %s)',
plugin.get_id(), plugin_cache_active, cache_ttl)
- # for environ based password can be empty, but then the validation is
- # on the server that fills in the env data needed for authentication
-
- _password_hash = caches.compute_key_from_params(
- plugin.name, username, (password or ''))
+ user_id = user.user_id
+ cache_namespace_uid = 'cache_user_auth.{}'.format(user_id)
+ region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
- # _authenticate is a wrapper for .auth() method of plugin.
- # it checks if .auth() sends proper data.
- # For RhodeCodeExternalAuthPlugin it also maps users to
- # Database and maps the attributes returned from .auth()
- # to RhodeCode database. If this function returns data
- # then auth is correct.
- start = time.time()
- log.debug('Running plugin `%s` _authenticate method', plugin.get_id())
+ @region.cache_on_arguments(namespace=cache_namespace_uid,
+ expiration_time=cache_ttl,
+ should_cache_fn=lambda v: plugin_cache_active)
+ def compute_auth(
+ cache_name, plugin_name, username, password):
- def auth_func():
- """
- This function is used internally in Cache of Beaker to calculate
- Results
- """
- log.debug('auth: calculating password access now...')
+ # _authenticate is a wrapper for .auth() method of plugin.
+ # it checks if .auth() sends proper data.
+ # For RhodeCodeExternalAuthPlugin it also maps users to
+ # Database and maps the attributes returned from .auth()
+ # to RhodeCode database. If this function returns data
+ # then auth is correct.
+ log.debug('Running plugin `%s` _authenticate method '
+ 'using username and password', plugin.get_id())
return plugin._authenticate(
user, username, password, plugin_settings,
environ=environ or {})
- if plugin_cache_active:
- log.debug('Trying to fetch cached auth by pwd hash `...%s`',
- _password_hash[:6])
- plugin_user = cache_manager.get(
- _password_hash, createfunc=auth_func)
- else:
- plugin_user = auth_func()
+ start = time.time()
+ # for environ based auth, password can be empty, but then the validation is
+ # on the server that fills in the env data needed for authentication
+ plugin_user = compute_auth('auth', plugin.name, username, (password or ''))
auth_time = time.time() - start
log.debug('Authentication for plugin `%s` completed in %.3fs, '
diff --git a/rhodecode/authentication/views.py b/rhodecode/authentication/views.py
--- a/rhodecode/authentication/views.py
+++ b/rhodecode/authentication/views.py
@@ -27,8 +27,7 @@ from pyramid.renderers import render
from pyramid.response import Response
from rhodecode.apps._base import BaseAppView
-from rhodecode.authentication.base import (
- get_auth_cache_manager, get_perms_cache_manager, get_authn_registry)
+from rhodecode.authentication.base import get_authn_registry
from rhodecode.lib import helpers as h
from rhodecode.lib.auth import (
LoginRequired, HasPermissionAllDecorator, CSRFRequired)
@@ -102,16 +101,6 @@ class AuthnPluginViewBase(BaseAppView):
self.plugin.create_or_update_setting(name, value)
Session().commit()
- # cleanup cache managers in case of change for plugin
- # TODO(marcink): because we can register multiple namespaces
- # we should at some point figure out how to retrieve ALL namespace
- # cache managers and clear them...
- cache_manager = get_auth_cache_manager()
- clear_cache_manager(cache_manager)
-
- cache_manager = get_perms_cache_manager()
- clear_cache_manager(cache_manager)
-
# Display success message and redirect.
h.flash(_('Auth settings updated successfully.'), category='success')
redirect_to = self.request.resource_path(
diff --git a/rhodecode/config/middleware.py b/rhodecode/config/middleware.py
--- a/rhodecode/config/middleware.py
+++ b/rhodecode/config/middleware.py
@@ -22,6 +22,7 @@ import os
import logging
import traceback
import collections
+import tempfile
from paste.gzipper import make_gzip_middleware
from pyramid.wsgi import wsgiapp
@@ -220,6 +221,7 @@ def includeme(config):
config.include('pyramid_mako')
config.include('pyramid_beaker')
config.include('rhodecode.lib.caches')
+ config.include('rhodecode.lib.rc_cache')
config.include('rhodecode.authentication')
config.include('rhodecode.integrations')
@@ -382,6 +384,7 @@ def sanitize_settings_and_apply_defaults
# Call split out functions that sanitize settings for each topic.
_sanitize_appenlight_settings(settings)
_sanitize_vcs_settings(settings)
+ _sanitize_cache_settings(settings)
# configure instance id
config_utils.set_instance_id(settings)
@@ -421,6 +424,18 @@ def _sanitize_vcs_settings(settings):
settings['vcs.scm_app_implementation'] = 'http'
+def _sanitize_cache_settings(settings):
+ _string_setting(settings, 'cache_dir',
+ os.path.join(tempfile.gettempdir(), 'rc_cache'))
+
+ _string_setting(settings, 'rc_cache.cache_perms.backend',
+ 'dogpile.cache.rc.file_namespace')
+ _int_setting(settings, 'rc_cache.cache_perms.expiration_time',
+ 60)
+ _string_setting(settings, 'rc_cache.cache_perms.arguments.filename',
+ os.path.join(tempfile.gettempdir(), 'rc_cache_1'))
+
+
def _int_setting(settings, name, default):
settings[name] = int(settings.get(name, default))
diff --git a/rhodecode/lib/auth.py b/rhodecode/lib/auth.py
--- a/rhodecode/lib/auth.py
+++ b/rhodecode/lib/auth.py
@@ -48,7 +48,7 @@ from rhodecode.model.user import UserMod
from rhodecode.model.db import (
User, Repository, Permission, UserToPerm, UserGroupToPerm, UserGroupMember,
UserIpMap, UserApiKeys, RepoGroup, UserGroup)
-from rhodecode.lib import caches
+from rhodecode.lib import rc_cache
from rhodecode.lib.utils2 import safe_unicode, aslist, safe_str, md5, safe_int, sha1
from rhodecode.lib.utils import (
get_repo_slug, get_repo_group_slug, get_user_group_slug)
@@ -976,21 +976,18 @@ class AuthUser(object):
obj = Repository.get_by_repo_name(scope['repo_name'])
if obj:
scope['repo_id'] = obj.repo_id
- _scope = {
- 'repo_id': -1,
- 'user_group_id': -1,
- 'repo_group_id': -1,
- }
- _scope.update(scope)
- cache_key = "_".join(map(safe_str, reduce(lambda a, b: a+b,
- _scope.items())))
- if cache_key not in self._permissions_scoped_cache:
- # store in cache to mimic how the @LazyProperty works,
- # the difference here is that we use the unique key calculated
- # from params and values
- res = self.get_perms(user=self, cache=False, scope=_scope)
- self._permissions_scoped_cache[cache_key] = res
- return self._permissions_scoped_cache[cache_key]
+ _scope = collections.OrderedDict()
+ _scope['repo_id'] = -1
+ _scope['user_group_id'] = -1
+ _scope['repo_group_id'] = -1
+
+ for k in sorted(scope.keys()):
+ _scope[k] = scope[k]
+
+ # store in cache to mimic how the @LazyProperty works,
+ # the difference here is that we use the unique key calculated
+ # from params and values
+ return self.get_perms(user=self, cache=False, scope=_scope)
def get_instance(self):
return User.get(self.user_id)
@@ -1072,16 +1069,27 @@ class AuthUser(object):
user_inherit_default_permissions = user.inherit_default_permissions
cache_seconds = safe_int(
- rhodecode.CONFIG.get('beaker.cache.short_term.expire'))
+ rhodecode.CONFIG.get('rc_cache.cache_perms.expiration_time'))
+
cache_on = cache or cache_seconds > 0
log.debug(
'Computing PERMISSION tree for user %s scope `%s` '
- 'with caching: %s[%ss]' % (user, scope, cache_on, cache_seconds))
+ 'with caching: %s[TTL: %ss]' % (user, scope, cache_on, cache_seconds or 0))
+
+ cache_namespace_uid = 'cache_user_auth.{}'.format(user_id)
+ region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
+
+ @region.cache_on_arguments(namespace=cache_namespace_uid,
+ should_cache_fn=lambda v: cache_on)
+ def compute_perm_tree(cache_name,
+ user_id, scope, user_is_admin,user_inherit_default_permissions,
+ explicit, algo, calculate_super_admin):
+ return _cached_perms_data(
+ user_id, scope, user_is_admin, user_inherit_default_permissions,
+ explicit, algo, calculate_super_admin)
+
start = time.time()
- compute = caches.conditional_cache(
- 'short_term', 'cache_desc.{}'.format(user_id),
- condition=cache_on, func=_cached_perms_data)
- result = compute(user_id, scope, user_is_admin,
+ result = compute_perm_tree('permissions', user_id, scope, user_is_admin,
user_inherit_default_permissions, explicit, algo,
calculate_super_admin)
@@ -1091,6 +1099,7 @@ class AuthUser(object):
total = time.time() - start
log.debug('PERMISSION tree for user %s computed in %.3fs: %s' % (
user, total, result_repr))
+
return result
@property
@@ -1153,10 +1162,7 @@ class AuthUser(object):
return [x.repo_id for x in
RepoList(qry, perm_set=perm_def)]
- compute = caches.conditional_cache(
- 'short_term', 'repo_acl_ids.{}'.format(self.user_id),
- condition=cache, func=_cached_repo_acl)
- return compute(self.user_id, perms, name_filter)
+ return _cached_repo_acl(self.user_id, perms, name_filter)
def repo_group_acl_ids(self, perms=None, name_filter=None, cache=False):
"""
@@ -1179,10 +1185,7 @@ class AuthUser(object):
return [x.group_id for x in
RepoGroupList(qry, perm_set=perm_def)]
- compute = caches.conditional_cache(
- 'short_term', 'repo_group_acl_ids.{}'.format(self.user_id),
- condition=cache, func=_cached_repo_group_acl)
- return compute(self.user_id, perms, name_filter)
+ return _cached_repo_group_acl(self.user_id, perms, name_filter)
def user_group_acl_ids(self, perms=None, name_filter=None, cache=False):
"""
@@ -1205,10 +1208,7 @@ class AuthUser(object):
return [x.users_group_id for x in
UserGroupList(qry, perm_set=perm_def)]
- compute = caches.conditional_cache(
- 'short_term', 'user_group_acl_ids.{}'.format(self.user_id),
- condition=cache, func=_cached_user_group_acl)
- return compute(self.user_id, perms, name_filter)
+ return _cached_user_group_acl(self.user_id, perms, name_filter)
@property
def ip_allowed(self):
diff --git a/rhodecode/lib/base.py b/rhodecode/lib/base.py
--- a/rhodecode/lib/base.py
+++ b/rhodecode/lib/base.py
@@ -505,6 +505,7 @@ def bootstrap_config(request):
config.include('pyramid_mako')
config.include('pyramid_beaker')
config.include('rhodecode.lib.caches')
+ config.include('rhodecode.lib.rc_cache')
add_events_routes(config)
diff --git a/rhodecode/lib/caches.py b/rhodecode/lib/caches.py
--- a/rhodecode/lib/caches.py
+++ b/rhodecode/lib/caches.py
@@ -96,7 +96,7 @@ def get_cache_manager(region_name, cache
Creates a Beaker cache manager. Such instance can be used like that::
_namespace = caches.get_repo_namespace_key(caches.XXX, repo_name)
- cache_manager = caches.get_cache_manager('repo_cache_long', _namespace)
+ cache_manager = caches.get_cache_manager('some_namespace_name', _namespace)
_cache_key = caches.compute_key_from_params(repo_name, commit.raw_id)
def heavy_compute():
...
@@ -121,7 +121,7 @@ def get_cache_manager(region_name, cache
def clear_cache_manager(cache_manager):
"""
namespace = 'foobar'
- cache_manager = get_cache_manager('repo_cache_long', namespace)
+ cache_manager = get_cache_manager('some_namespace_name', namespace)
clear_cache_manager(cache_manager)
"""
diff --git a/rhodecode/lib/middleware/simplevcs.py b/rhodecode/lib/middleware/simplevcs.py
--- a/rhodecode/lib/middleware/simplevcs.py
+++ b/rhodecode/lib/middleware/simplevcs.py
@@ -39,9 +39,8 @@ from pyramid.httpexceptions import (
from zope.cachedescriptors.property import Lazy as LazyProperty
import rhodecode
-from rhodecode.authentication.base import (
- authenticate, get_perms_cache_manager, VCS_TYPE, loadplugin)
-from rhodecode.lib import caches
+from rhodecode.authentication.base import authenticate, VCS_TYPE, loadplugin
+from rhodecode.lib import caches, rc_cache
from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
from rhodecode.lib.base import (
BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
@@ -311,36 +310,24 @@ class SimpleVCS(object):
:param repo_name: repository name
"""
- # get instance of cache manager configured for a namespace
- cache_manager = get_perms_cache_manager(
- custom_ttl=cache_ttl, suffix=user.user_id)
log.debug('AUTH_CACHE_TTL for permissions `%s` active: %s (TTL: %s)',
plugin_id, plugin_cache_active, cache_ttl)
- # for environ based password can be empty, but then the validation is
- # on the server that fills in the env data needed for authentication
- _perm_calc_hash = caches.compute_key_from_params(
- plugin_id, action, user.user_id, repo_name, ip_addr)
+ user_id = user.user_id
+ cache_namespace_uid = 'cache_user_auth.{}'.format(user_id)
+ region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
- # _authenticate is a wrapper for .auth() method of plugin.
- # it checks if .auth() sends proper data.
- # For RhodeCodeExternalAuthPlugin it also maps users to
- # Database and maps the attributes returned from .auth()
- # to RhodeCode database. If this function returns data
- # then auth is correct.
- start = time.time()
- log.debug('Running plugin `%s` permissions check', plugin_id)
+ @region.cache_on_arguments(namespace=cache_namespace_uid,
+ expiration_time=cache_ttl,
+ should_cache_fn=lambda v: plugin_cache_active)
+ def compute_perm_vcs(
+ cache_name, plugin_id, action, user_id, repo_name, ip_addr):
- def perm_func():
- """
- This function is used internally in Cache of Beaker to calculate
- Results
- """
log.debug('auth: calculating permission access now...')
# check IP
inherit = user.inherit_default_permissions
ip_allowed = AuthUser.check_ip_allowed(
- user.user_id, ip_addr, inherit_from_default=inherit)
+ user_id, ip_addr, inherit_from_default=inherit)
if ip_allowed:
log.info('Access for IP:%s allowed', ip_addr)
else:
@@ -360,12 +347,13 @@ class SimpleVCS(object):
return True
- if plugin_cache_active:
- log.debug('Trying to fetch cached perms by %s', _perm_calc_hash[:6])
- perm_result = cache_manager.get(
- _perm_calc_hash, createfunc=perm_func)
- else:
- perm_result = perm_func()
+ start = time.time()
+ log.debug('Running plugin `%s` permissions check', plugin_id)
+
+ # for environ based auth, password can be empty, but then the validation is
+ # on the server that fills in the env data needed for authentication
+ perm_result = compute_perm_vcs(
+ 'vcs_permissions', plugin_id, action, user.user_id, repo_name, ip_addr)
auth_time = time.time() - start
log.debug('Permissions for plugin `%s` completed in %.3fs, '
diff --git a/rhodecode/lib/rc_cache/__init__.py b/rhodecode/lib/rc_cache/__init__.py
new file mode 100644
--- /dev/null
+++ b/rhodecode/lib/rc_cache/__init__.py
@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2015-2018 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+from dogpile.cache import register_backend
+from dogpile.cache import make_region
+
+register_backend(
+ "dogpile.cache.rc.memory_lru", "rhodecode.lib.rc_cache.backends",
+ "LRUMemoryBackend")
+
+register_backend(
+ "dogpile.cache.rc.file_namespace", "rhodecode.lib.rc_cache.backends",
+ "FileNamespaceBackend")
+
+register_backend(
+ "dogpile.cache.rc.redis", "rhodecode.lib.rc_cache.backends",
+ "RedisPickleBackend")
+
+
+from . import region_meta
+from .utils import get_default_cache_settings, key_generator, get_or_create_region
+
+
+def configure_dogpile_cache(settings):
+ cache_dir = settings.get('cache_dir')
+ if cache_dir:
+ region_meta.dogpile_config_defaults['cache_dir'] = cache_dir
+
+ rc_cache_data = get_default_cache_settings(settings, prefixes=['rc_cache.'])
+
+ # inspect available namespaces
+ avail_regions = set()
+ for key in rc_cache_data.keys():
+ namespace_name = key.split('.', 1)[0]
+ avail_regions.add(namespace_name)
+
+ # register them into namespace
+ for region_name in avail_regions:
+ new_region = make_region(
+ name=region_name,
+ function_key_generator=key_generator
+ )
+
+ new_region.configure_from_config(settings, 'rc_cache.{}.'.format(region_name))
+ region_meta.dogpile_cache_regions[region_name] = new_region
+
+
+def includeme(config):
+ configure_dogpile_cache(config.registry.settings)
diff --git a/rhodecode/lib/rc_cache/backends.py b/rhodecode/lib/rc_cache/backends.py
new file mode 100644
--- /dev/null
+++ b/rhodecode/lib/rc_cache/backends.py
@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2015-2018 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+from dogpile.cache.backends import memory as memory_backend
+from dogpile.cache.backends import file as file_backend
+from dogpile.cache.backends import redis as redis_backend
+from dogpile.cache.backends.file import NO_VALUE, compat
+
+from rhodecode.lib.memory_lru_debug import LRUDict
+
+_default_max_size = 1024
+
+
+class LRUMemoryBackend(memory_backend.MemoryBackend):
+
+ def __init__(self, arguments):
+ max_size = arguments.pop('max_size', _default_max_size)
+ arguments['cache_dict'] = LRUDict(max_size)
+ super(LRUMemoryBackend, self).__init__(arguments)
+
+
+class Serializer(object):
+ def _dumps(self, value):
+ return compat.pickle.dumps(value)
+
+ def _loads(self, value):
+ return compat.pickle.loads(value)
+
+
+class FileNamespaceBackend(Serializer, file_backend.DBMBackend):
+
+ def __init__(self, arguments):
+ super(FileNamespaceBackend, self).__init__(arguments)
+
+ def list_keys(self):
+ with self._dbm_file(True) as dbm:
+ return dbm.keys()
+
+ def get_store(self):
+ return self.filename
+
+ def get(self, key):
+ with self._dbm_file(False) as dbm:
+ if hasattr(dbm, 'get'):
+ value = dbm.get(key, NO_VALUE)
+ else:
+ # gdbm objects lack a .get method
+ try:
+ value = dbm[key]
+ except KeyError:
+ value = NO_VALUE
+ if value is not NO_VALUE:
+ value = self._loads(value)
+ return value
+
+ def set(self, key, value):
+ with self._dbm_file(True) as dbm:
+ dbm[key] = self._dumps(value)
+
+ def set_multi(self, mapping):
+ with self._dbm_file(True) as dbm:
+ for key, value in mapping.items():
+ dbm[key] = self._dumps(value)
+
+
+class RedisPickleBackend(Serializer, redis_backend.RedisBackend):
+ def list_keys(self):
+ return self.client.keys()
+
+ def get_store(self):
+ return self.client.connection_pool
+
+ def set(self, key, value):
+ if self.redis_expiration_time:
+ self.client.setex(key, self.redis_expiration_time,
+ self._dumps(value))
+ else:
+ self.client.set(key, self._dumps(value))
+
+ def set_multi(self, mapping):
+ mapping = dict(
+ (k, self._dumps(v))
+ for k, v in mapping.items()
+ )
+
+ if not self.redis_expiration_time:
+ self.client.mset(mapping)
+ else:
+ pipe = self.client.pipeline()
+ for key, value in mapping.items():
+ pipe.setex(key, self.redis_expiration_time, value)
+ pipe.execute()
diff --git a/rhodecode/lib/rc_cache/region_meta.py b/rhodecode/lib/rc_cache/region_meta.py
new file mode 100644
--- /dev/null
+++ b/rhodecode/lib/rc_cache/region_meta.py
@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2015-2018 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import os
+import tempfile
+
+dogpile_config_defaults = {
+ 'cache_dir': os.path.join(tempfile.gettempdir(), 'rc_cache')
+}
+
+# GLOBAL TO STORE ALL REGISTERED REGIONS
+dogpile_cache_regions = {}
diff --git a/rhodecode/lib/rc_cache/utils.py b/rhodecode/lib/rc_cache/utils.py
new file mode 100644
--- /dev/null
+++ b/rhodecode/lib/rc_cache/utils.py
@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2015-2018 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import os
+import logging
+from dogpile.cache import make_region
+
+from rhodecode.lib.utils import safe_str, sha1
+from . import region_meta
+
+log = logging.getLogger(__name__)
+
+
+def get_default_cache_settings(settings, prefixes=None):
+ prefixes = prefixes or []
+ cache_settings = {}
+ for key in settings.keys():
+ for prefix in prefixes:
+ if key.startswith(prefix):
+ name = key.split(prefix)[1].strip()
+ val = settings[key]
+ if isinstance(val, basestring):
+ val = val.strip()
+ cache_settings[name] = val
+ return cache_settings
+
+
+def compute_key_from_params(*args):
+ """
+ Helper to compute key from given params to be used in cache manager
+ """
+ return sha1("_".join(map(safe_str, args)))
+
+
+def key_generator(namespace, fn):
+ fname = fn.__name__
+
+ def generate_key(*args):
+ namespace_pref = namespace or 'default'
+ arg_key = compute_key_from_params(*args)
+ final_key = "{}:{}_{}".format(namespace_pref, fname, arg_key)
+
+ return final_key
+
+ return generate_key
+
+
+def get_or_create_region(region_name, region_namespace=None):
+ from rhodecode.lib.rc_cache.backends import FileNamespaceBackend
+ region_obj = region_meta.dogpile_cache_regions.get(region_name)
+ if not region_obj:
+ raise EnvironmentError(
+ 'Region `{}` not in configured: {}.'.format(
+ region_name, region_meta.dogpile_cache_regions.keys()))
+
+ region_uid_name = '{}:{}'.format(region_name, region_namespace)
+ if isinstance(region_obj.actual_backend, FileNamespaceBackend):
+ region_exist = region_meta.dogpile_cache_regions.get(region_namespace)
+ if region_exist:
+ log.debug('Using already configured region: %s', region_namespace)
+ return region_exist
+ cache_dir = region_meta.dogpile_config_defaults['cache_dir']
+ expiration_time = region_obj.expiration_time
+
+ if not os.path.isdir(cache_dir):
+ os.makedirs(cache_dir)
+ new_region = make_region(
+ name=region_uid_name, function_key_generator=key_generator
+ )
+ namespace_filename = os.path.join(
+ cache_dir, "{}.cache.dbm".format(region_namespace))
+ # special type that allows 1db per namespace
+ new_region.configure(
+ backend='dogpile.cache.rc.file_namespace',
+ expiration_time=expiration_time,
+ arguments={"filename": namespace_filename}
+ )
+
+ # create and save in region caches
+ log.debug('configuring new region: %s',region_uid_name)
+ region_obj = region_meta.dogpile_cache_regions[region_namespace] = new_region
+
+ return region_obj
diff --git a/rhodecode/public/js/rhodecode/routes.js b/rhodecode/public/js/rhodecode/routes.js
--- a/rhodecode/public/js/rhodecode/routes.js
+++ b/rhodecode/public/js/rhodecode/routes.js
@@ -119,6 +119,8 @@ function registerRCRoutes() {
pyroutes.register('edit_user_groups_management', '/_admin/users/%(user_id)s/edit/groups_management', ['user_id']);
pyroutes.register('edit_user_groups_management_updates', '/_admin/users/%(user_id)s/edit/edit_user_groups_management/updates', ['user_id']);
pyroutes.register('edit_user_audit_logs', '/_admin/users/%(user_id)s/edit/audit', ['user_id']);
+ pyroutes.register('edit_user_caches', '/_admin/users/%(user_id)s/edit/caches', ['user_id']);
+ pyroutes.register('edit_user_caches_update', '/_admin/users/%(user_id)s/edit/caches/update', ['user_id']);
pyroutes.register('user_groups', '/_admin/user_groups', []);
pyroutes.register('user_groups_data', '/_admin/user_groups_data', []);
pyroutes.register('user_groups_new', '/_admin/user_groups/new', []);
diff --git a/rhodecode/templates/admin/users/user_edit.mako b/rhodecode/templates/admin/users/user_edit.mako
--- a/rhodecode/templates/admin/users/user_edit.mako
+++ b/rhodecode/templates/admin/users/user_edit.mako
@@ -45,6 +45,7 @@
<li class="${'active' if c.active=='ips' else ''}"><a href="${h.route_path('edit_user_ips', user_id=c.user.user_id)}">${_('Ip Whitelist')}</a></li>
<li class="${'active' if c.active=='groups' else ''}"><a href="${h.route_path('edit_user_groups_management', user_id=c.user.user_id)}">${_('User Groups Management')}</a></li>
<li class="${'active' if c.active=='audit' else ''}"><a href="${h.route_path('edit_user_audit_logs', user_id=c.user.user_id)}">${_('Audit logs')}</a></li>
+ <li class="${'active' if c.active=='caches' else ''}"><a href="${h.route_path('edit_user_caches', user_id=c.user.user_id)}">${_('Caches')}</a></li>
</ul>
</div>
diff --git a/rhodecode/templates/admin/users/user_edit_caches.mako b/rhodecode/templates/admin/users/user_edit_caches.mako
new file mode 100644
--- /dev/null
+++ b/rhodecode/templates/admin/users/user_edit_caches.mako
@@ -0,0 +1,29 @@
+<%namespace name="base" file="/base/base.mako"/>
+
+<div class="panel panel-default">
+ <div class="panel-heading">
+ <h3 class="panel-title">${_('Caches')}</h3>
+ </div>
+ <div class="panel-body">
+ <pre>
+region: ${c.region.name}
+backend: ${c.region.actual_backend.__class__}
+store: ${c.region.actual_backend.get_store()}
+
+% for k in c.user_keys:
+ - ${k}
+% endfor
+ </pre>
+
+ ${h.secure_form(h.route_path('edit_user_caches_update', user_id=c.user.user_id), request=request)}
+ <div class="form">
+ <div class="fields">
+ ${h.submit('reset_cache_%s' % c.user.user_id, _('Invalidate user cache'),class_="btn btn-small",onclick="return confirm('"+_('Confirm to invalidate user cache')+"');")}
+ </div>
+ </div>
+ ${h.end_form()}
+
+ </div>
+</div>
+
+
diff --git a/rhodecode/tests/lib/test_caches.py b/rhodecode/tests/lib/test_caches.py
--- a/rhodecode/tests/lib/test_caches.py
+++ b/rhodecode/tests/lib/test_caches.py
@@ -22,36 +22,24 @@ import time
import pytest
-from rhodecode.lib import caches
-from rhodecode.lib.memory_lru_debug import MemoryLRUNamespaceManagerBase
+from rhodecode.lib import rc_cache
-class TestCacheManager(object):
+@pytest.mark.usefixtures( 'app')
+class TestCaches(object):
- @pytest.mark.parametrize('repo_name', [
- ('',),
- (u'',),
- (u'ac',),
- ('ac', ),
- (u'ęćc',),
- ('ąac',),
+ def test_cache_decorator_init_not_configured(self):
+ with pytest.raises(EnvironmentError):
+ rc_cache.get_or_create_region('dontexist')
+
+ @pytest.mark.parametrize('region_name', [
+ 'cache_perms', u'cache_perms',
])
- def test_cache_manager_init(self, repo_name):
- cache_manager_instance = caches.get_cache_manager(
- repo_name, 'my_cache')
- assert cache_manager_instance
-
- def test_cache_manager_init_undefined_namespace(self):
- cache_manager_instance = caches.get_cache_manager(
- 'repo_cache_long_undefined', 'my_cache')
- assert cache_manager_instance
-
- def_config = caches.DEFAULT_CACHE_MANAGER_CONFIG.copy()
- def_config.pop('type')
- assert cache_manager_instance.nsargs == def_config
-
- assert isinstance(
- cache_manager_instance.namespace, MemoryLRUNamespaceManagerBase)
+ def test_cache_decorator_init(self, region_name):
+ namespace = region_name
+ cache_region = rc_cache.get_or_create_region(
+ region_name, region_namespace=namespace)
+ assert cache_region
@pytest.mark.parametrize('example_input', [
('',),
@@ -62,45 +50,60 @@ class TestCacheManager(object):
(u'/ac', ),
])
def test_cache_manager_create_key(self, example_input):
- key = caches.compute_key_from_params(*example_input)
+ key = rc_cache.utils.compute_key_from_params(*example_input)
assert key
- def test_store_and_invalidate_value_from_manager(self):
- cache_manger_instance = caches.get_cache_manager(
- 'repo_cache_long', 'my_cache_store')
+ @pytest.mark.parametrize('example_namespace', [
+ 'namespace', None
+ ])
+ @pytest.mark.parametrize('example_input', [
+ ('',),
+ (u'/ac',),
+ (u'/ac', 1, 2, object()),
+ (u'/ęćc', 1, 2, object()),
+ ('/ąac',),
+ (u'/ac', ),
+ ])
+ def test_cache_keygen(self, example_input, example_namespace):
+ def func_wrapped():
+ return 1
+ func = rc_cache.utils.key_generator(example_namespace, func_wrapped)
+ key = func(*example_input)
+ assert key
- def compute():
+ def test_store_value_in_cache(self):
+ cache_region = rc_cache.get_or_create_region('cache_perms')
+ # make sure we empty the cache now
+ for key in cache_region.backend.list_keys():
+ cache_region.delete(key)
+
+ assert cache_region.backend.list_keys() == []
+
+ @cache_region.cache_on_arguments(expiration_time=5)
+ def compute(key):
return time.time()
- added_keys = []
- for i in xrange(3):
- _cache_key = caches.compute_key_from_params('foo_bar', 'p%s' % i)
- added_keys.append(_cache_key)
- for x in xrange(10):
- cache_manger_instance.get(_cache_key, createfunc=compute)
+ for x in range(10):
+ compute(x)
+
+ assert len(set(cache_region.backend.list_keys())) == 10
- for key in added_keys:
- assert cache_manger_instance[key]
-
- caches.clear_cache_manager(cache_manger_instance)
-
- for key in added_keys:
- assert key not in cache_manger_instance
- assert len(cache_manger_instance.namespace.keys()) == 0
+ def test_store_and_get_value_from_region(self):
+ cache_region = rc_cache.get_or_create_region('cache_perms')
+ # make sure we empty the cache now
+ for key in cache_region.backend.list_keys():
+ cache_region.delete(key)
+ assert cache_region.backend.list_keys() == []
- def test_store_and_get_value_from_manager(self):
- cache_manger_instance = caches.get_cache_manager(
- 'repo_cache_long', 'my_cache_store')
-
- _cache_key = caches.compute_key_from_params('foo_bar', 'multicall')
+ @cache_region.cache_on_arguments(expiration_time=5)
+ def compute(key):
+ return time.time()
- def compute():
- return time.time()
+ result = set()
+ for x in range(10):
+ ret = compute('x')
+ result.add(ret)
- result = set()
- for x in xrange(10):
- ret = cache_manger_instance.get(_cache_key, createfunc=compute)
- result.add(ret)
-
- # once computed we have only one value after executing it 10x
- assert len(result) == 1
+ # once computed we have only one value (the same from cache)
+ # after executing it 10x
+ assert len(result) == 1
diff --git a/rhodecode/tests/rhodecode.ini b/rhodecode/tests/rhodecode.ini
--- a/rhodecode/tests/rhodecode.ini
+++ b/rhodecode/tests/rhodecode.ini
@@ -292,11 +292,7 @@ cache_dir = %(here)s/data
beaker.cache.data_dir = %(here)s/rc/data/cache/beaker_data
beaker.cache.lock_dir = %(here)s/rc/data/cache/beaker_lock
-beaker.cache.regions = short_term, long_term, sql_cache_short, auth_plugins, repo_cache_long
-
-beaker.cache.short_term.type = file
-beaker.cache.short_term.expire = 0
-beaker.cache.short_term.key_length = 256
+beaker.cache.regions = long_term, sql_cache_short, repo_cache_long
beaker.cache.long_term.type = memory
beaker.cache.long_term.expire = 36000
@@ -306,16 +302,6 @@ beaker.cache.sql_cache_short.type = memo
beaker.cache.sql_cache_short.expire = 1
beaker.cache.sql_cache_short.key_length = 256
-## default is memory cache, configure only if required
-## using multi-node or multi-worker setup
-#beaker.cache.auth_plugins.type = memory
-#beaker.cache.auth_plugins.lock_dir = %(here)s/data/cache/auth_plugin_lock
-#beaker.cache.auth_plugins.url = postgresql://postgres:secret@localhost/rhodecode
-#beaker.cache.auth_plugins.url = mysql://root:secret@127.0.0.1/rhodecode
-#beaker.cache.auth_plugins.sa.pool_recycle = 3600
-#beaker.cache.auth_plugins.sa.pool_size = 10
-#beaker.cache.auth_plugins.sa.max_overflow = 0
-
beaker.cache.repo_cache_long.type = memorylru_base
beaker.cache.repo_cache_long.max_items = 4096
beaker.cache.repo_cache_long.expire = 2592000
@@ -327,6 +313,16 @@ beaker.cache.repo_cache_long.expire = 25
#beaker.cache.repo_cache_long.expire = 1209600
#beaker.cache.repo_cache_long.key_length = 256
+
+#####################################
+### DOGPILE CACHE ####
+#####################################
+
+## permission tree cache settings
+rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace
+rc_cache.cache_perms.expiration_time = 0
+rc_cache.cache_perms.arguments.filename = /tmp/rc_cache_1
+
####################################
### BEAKER SESSION ####
####################################