caches: rewrite of auth/permission caches to dogpile.

rhodecode/lib/rc_cache/__init__.py

@@ -0,0 +1,66 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2015-2018 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+from dogpile.cache import register_backend
+from dogpile.cache import make_region
+
+register_backend(
+    "dogpile.cache.rc.memory_lru", "rhodecode.lib.rc_cache.backends",
+    "LRUMemoryBackend")
+
+register_backend(
+    "dogpile.cache.rc.file_namespace", "rhodecode.lib.rc_cache.backends",
+    "FileNamespaceBackend")
+
+register_backend(
+    "dogpile.cache.rc.redis", "rhodecode.lib.rc_cache.backends",
+    "RedisPickleBackend")
+
+
+from . import region_meta
+from .utils import get_default_cache_settings, key_generator, get_or_create_region
+
+
+def configure_dogpile_cache(settings):
+    cache_dir = settings.get('cache_dir')
+    if cache_dir:
+        region_meta.dogpile_config_defaults['cache_dir'] = cache_dir
+
+    rc_cache_data = get_default_cache_settings(settings, prefixes=['rc_cache.'])
+
+    # inspect available namespaces
+    avail_regions = set()
+    for key in rc_cache_data.keys():
+        namespace_name = key.split('.', 1)[0]
+        avail_regions.add(namespace_name)
+
+    # register them into namespace
+    for region_name in avail_regions:
+        new_region = make_region(
+            name=region_name,
+            function_key_generator=key_generator
+        )
+
+        new_region.configure_from_config(settings, 'rc_cache.{}.'.format(region_name))
+        region_meta.dogpile_cache_regions[region_name] = new_region
+
+
+def includeme(config):
+    configure_dogpile_cache(config.registry.settings)

rhodecode/lib/rc_cache/backends.py

@@ -0,0 +1,109 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2015-2018 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+from dogpile.cache.backends import memory as memory_backend
+from dogpile.cache.backends import file as file_backend
+from dogpile.cache.backends import redis as redis_backend
+from dogpile.cache.backends.file import NO_VALUE, compat
+
+from rhodecode.lib.memory_lru_debug import LRUDict
+
+_default_max_size = 1024
+
+
+class LRUMemoryBackend(memory_backend.MemoryBackend):
+
+    def __init__(self, arguments):
+        max_size = arguments.pop('max_size', _default_max_size)
+        arguments['cache_dict'] = LRUDict(max_size)
+        super(LRUMemoryBackend, self).__init__(arguments)
+
+
+class Serializer(object):
+    def _dumps(self, value):
+        return compat.pickle.dumps(value)
+
+    def _loads(self, value):
+        return compat.pickle.loads(value)
+
+
+class FileNamespaceBackend(Serializer, file_backend.DBMBackend):
+
+    def __init__(self, arguments):
+        super(FileNamespaceBackend, self).__init__(arguments)
+
+    def list_keys(self):
+        with self._dbm_file(True) as dbm:
+            return dbm.keys()
+
+    def get_store(self):
+        return self.filename
+
+    def get(self, key):
+        with self._dbm_file(False) as dbm:
+            if hasattr(dbm, 'get'):
+                value = dbm.get(key, NO_VALUE)
+            else:
+                # gdbm objects lack a .get method
+                try:
+                    value = dbm[key]
+                except KeyError:
+                    value = NO_VALUE
+            if value is not NO_VALUE:
+                value = self._loads(value)
+            return value
+
+    def set(self, key, value):
+        with self._dbm_file(True) as dbm:
+            dbm[key] = self._dumps(value)
+
+    def set_multi(self, mapping):
+        with self._dbm_file(True) as dbm:
+            for key, value in mapping.items():
+                dbm[key] = self._dumps(value)
+
+
+class RedisPickleBackend(Serializer, redis_backend.RedisBackend):
+    def list_keys(self):
+        return self.client.keys()
+
+    def get_store(self):
+        return self.client.connection_pool
+
+    def set(self, key, value):
+        if self.redis_expiration_time:
+            self.client.setex(key, self.redis_expiration_time,
+                              self._dumps(value))
+        else:
+            self.client.set(key, self._dumps(value))
+
+    def set_multi(self, mapping):
+        mapping = dict(
+            (k, self._dumps(v))
+            for k, v in mapping.items()
+        )
+
+        if not self.redis_expiration_time:
+            self.client.mset(mapping)
+        else:
+            pipe = self.client.pipeline()
+            for key, value in mapping.items():
+                pipe.setex(key, self.redis_expiration_time, value)
+            pipe.execute()

rhodecode/lib/rc_cache/region_meta.py

@@ -0,0 +1,28 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2015-2018 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import os
+import tempfile
+
+dogpile_config_defaults = {
+    'cache_dir': os.path.join(tempfile.gettempdir(), 'rc_cache')
+}
+
+# GLOBAL TO STORE ALL REGISTERED REGIONS
+dogpile_cache_regions = {}

rhodecode/lib/rc_cache/utils.py

@@ -0,0 +1,99 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2015-2018 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import os
+import logging
+from dogpile.cache import make_region
+
+from rhodecode.lib.utils import safe_str, sha1
+from . import region_meta
+
+log = logging.getLogger(__name__)
+
+
+def get_default_cache_settings(settings, prefixes=None):
+    prefixes = prefixes or []
+    cache_settings = {}
+    for key in settings.keys():
+        for prefix in prefixes:
+            if key.startswith(prefix):
+                name = key.split(prefix)[1].strip()
+                val = settings[key]
+                if isinstance(val, basestring):
+                    val = val.strip()
+                cache_settings[name] = val
+    return cache_settings
+
+
+def compute_key_from_params(*args):
+    """
+    Helper to compute key from given params to be used in cache manager
+    """
+    return sha1("_".join(map(safe_str, args)))
+
+
+def key_generator(namespace, fn):
+    fname = fn.__name__
+
+    def generate_key(*args):
+        namespace_pref = namespace or 'default'
+        arg_key = compute_key_from_params(*args)
+        final_key = "{}:{}_{}".format(namespace_pref, fname, arg_key)
+
+        return final_key
+
+    return generate_key
+
+
+def get_or_create_region(region_name, region_namespace=None):
+    from rhodecode.lib.rc_cache.backends import FileNamespaceBackend
+    region_obj = region_meta.dogpile_cache_regions.get(region_name)
+    if not region_obj:
+        raise EnvironmentError(
+            'Region `{}` not in configured: {}.'.format(
+                region_name, region_meta.dogpile_cache_regions.keys()))
+
+    region_uid_name = '{}:{}'.format(region_name, region_namespace)
+    if isinstance(region_obj.actual_backend, FileNamespaceBackend):
+        region_exist = region_meta.dogpile_cache_regions.get(region_namespace)
+        if region_exist:
+            log.debug('Using already configured region: %s', region_namespace)
+            return region_exist
+        cache_dir = region_meta.dogpile_config_defaults['cache_dir']
+        expiration_time = region_obj.expiration_time
+
+        if not os.path.isdir(cache_dir):
+            os.makedirs(cache_dir)
+        new_region = make_region(
+            name=region_uid_name, function_key_generator=key_generator
+        )
+        namespace_filename = os.path.join(
+            cache_dir, "{}.cache.dbm".format(region_namespace))
+        # special type that allows 1db per namespace
+        new_region.configure(
+            backend='dogpile.cache.rc.file_namespace',
+            expiration_time=expiration_time,
+            arguments={"filename": namespace_filename}
+        )
+
+        # create and save in region caches
+        log.debug('configuring new region: %s',region_uid_name)
+        region_obj = region_meta.dogpile_cache_regions[region_namespace] = new_region
+
+    return region_obj

rhodecode/templates/admin/users/user_edit_caches.mako

@@ -0,0 +1,29 b''
+<%namespace name="base" file="/base/base.mako"/>
+
+<div class="panel panel-default">
+    <div class="panel-heading">
+        <h3 class="panel-title">${_('Caches')}</h3>
+    </div>
+    <div class="panel-body">
+        <pre>
+region: ${c.region.name}
+backend: ${c.region.actual_backend.__class__}
+store: ${c.region.actual_backend.get_store()}
+
+% for k in c.user_keys:
+ - ${k}
+% endfor
+        </pre>
+
+    ${h.secure_form(h.route_path('edit_user_caches_update', user_id=c.user.user_id), request=request)}
+    <div class="form">
+       <div class="fields">
+           ${h.submit('reset_cache_%s' % c.user.user_id, _('Invalidate user cache'),class_="btn btn-small",onclick="return confirm('"+_('Confirm to invalidate user cache')+"');")}
+       </div>
+    </div>
+    ${h.end_form()}
+
+    </div>
+</div>
+
+

configs/development.ini

@@ -320,12 +320,7 b' cache_dir = %(here)s/data'
 beaker.cache.data_dir = %(here)s/data/cache/beaker_data
 beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock
 
-beaker.cache.regions = short_term, long_term, sql_cache_short, auth_plugins, repo_cache_long
-
-# used for caching user permissions
-beaker.cache.short_term.type = file
-beaker.cache.short_term.expire = 0
-beaker.cache.short_term.key_length = 256
+beaker.cache.regions = long_term, sql_cache_short, repo_cache_long
 
 beaker.cache.long_term.type = memory
 beaker.cache.long_term.expire = 36000
@@ -335,16 +330,6 b' beaker.cache.sql_cache_short.type = memo'
 beaker.cache.sql_cache_short.expire = 10
 beaker.cache.sql_cache_short.key_length = 256
 
-## default is memory cache, configure only if required
-## using multi-node or multi-worker setup
-#beaker.cache.auth_plugins.type = ext:database
-#beaker.cache.auth_plugins.lock_dir = %(here)s/data/cache/auth_plugin_lock
-#beaker.cache.auth_plugins.url = postgresql://postgres:secret@localhost/rhodecode
-#beaker.cache.auth_plugins.url = mysql://root:secret@127.0.0.1/rhodecode
-#beaker.cache.auth_plugins.sa.pool_recycle = 3600
-#beaker.cache.auth_plugins.sa.pool_size = 10
-#beaker.cache.auth_plugins.sa.max_overflow = 0
-
 beaker.cache.repo_cache_long.type = memorylru_base
 beaker.cache.repo_cache_long.max_items = 4096
 beaker.cache.repo_cache_long.expire = 2592000

configs/production.ini

@@ -295,12 +295,7 b' cache_dir = %(here)s/data'
 beaker.cache.data_dir = %(here)s/data/cache/beaker_data
 beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock
 
-beaker.cache.regions = short_term, long_term, sql_cache_short, auth_plugins, repo_cache_long
-
-# used for caching user permissions
-beaker.cache.short_term.type = file
-beaker.cache.short_term.expire = 0
-beaker.cache.short_term.key_length = 256
+beaker.cache.regions = long_term, sql_cache_short, repo_cache_long
 
 beaker.cache.long_term.type = memory
 beaker.cache.long_term.expire = 36000
@@ -310,16 +305,6 b' beaker.cache.sql_cache_short.type = memo'
 beaker.cache.sql_cache_short.expire = 10
 beaker.cache.sql_cache_short.key_length = 256
 
-## default is memory cache, configure only if required
-## using multi-node or multi-worker setup
-#beaker.cache.auth_plugins.type = ext:database
-#beaker.cache.auth_plugins.lock_dir = %(here)s/data/cache/auth_plugin_lock
-#beaker.cache.auth_plugins.url = postgresql://postgres:secret@localhost/rhodecode
-#beaker.cache.auth_plugins.url = mysql://root:secret@127.0.0.1/rhodecode
-#beaker.cache.auth_plugins.sa.pool_recycle = 3600
-#beaker.cache.auth_plugins.sa.pool_size = 10
-#beaker.cache.auth_plugins.sa.max_overflow = 0
-
 beaker.cache.repo_cache_long.type = memorylru_base
 beaker.cache.repo_cache_long.max_items = 4096
 beaker.cache.repo_cache_long.expire = 2592000

rhodecode/apps/admin/__init__.py

@@ -356,6 +356,16 b' def admin_routes(config):'
         name='edit_user_audit_logs',
         pattern='/users/{user_id:\d+}/edit/audit', user_route=True)
 
+    # user caches
+    config.add_route(
+        name='edit_user_caches',
+        pattern='/users/{user_id:\d+}/edit/caches',
+        user_route=True)
+    config.add_route(
+        name='edit_user_caches_update',
+        pattern='/users/{user_id:\d+}/edit/caches/update',
+        user_route=True)
+
     # user-groups admin
     config.add_route(
         name='user_groups',

rhodecode/apps/admin/views/users.py

@@ -34,7 +34,7 b' from rhodecode.authentication.plugins im'
 from rhodecode.events import trigger
 from rhodecode.model.db import true
 
-from rhodecode.lib import audit_logger
+from rhodecode.lib import audit_logger, rc_cache
 from rhodecode.lib.exceptions import (
     UserCreationError, UserOwnsReposException, UserOwnsRepoGroupsException,
     UserOwnsUserGroupsException, DefaultUserException)
@@ -1198,3 +1198,57 b' class UsersView(UserAppView):'
         perm_user = self.db_user.AuthUser(ip_addr=self.request.remote_addr)
 
         return perm_user.permissions
+
+    def _get_user_cache_keys(self, cache_namespace_uid, keys):
+        user_keys = []
+        for k in sorted(keys):
+            if k.startswith(cache_namespace_uid):
+                user_keys.append(k)
+        return user_keys
+
+    @LoginRequired()
+    @HasPermissionAllDecorator('hg.admin')
+    @view_config(
+        route_name='edit_user_caches', request_method='GET',
+        renderer='rhodecode:templates/admin/users/user_edit.mako')
+    def user_caches(self):
+        _ = self.request.translate
+        c = self.load_default_context()
+        c.user = self.db_user
+
+        c.active = 'caches'
+        c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
+
+        cache_namespace_uid = 'cache_user_auth.{}'.format(self.db_user.user_id)
+        c.region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
+        c.backend = c.region.backend
+        c.user_keys = self._get_user_cache_keys(
+            cache_namespace_uid, c.region.backend.list_keys())
+
+        return self._get_template_context(c)
+
+    @LoginRequired()
+    @HasPermissionAllDecorator('hg.admin')
+    @CSRFRequired()
+    @view_config(
+        route_name='edit_user_caches_update', request_method='POST')
+    def user_caches_update(self):
+        _ = self.request.translate
+        c = self.load_default_context()
+        c.user = self.db_user
+
+        c.active = 'caches'
+        c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
+
+        cache_namespace_uid = 'cache_user_auth.{}'.format(self.db_user.user_id)
+        c.region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
+
+        c.user_keys = self._get_user_cache_keys(
+            cache_namespace_uid, c.region.backend.list_keys())
+        for k in c.user_keys:
+            c.region.delete(k)
+
+        h.flash(_("Deleted {} cache keys").format(len(c.user_keys)), category='success')
+
+        return HTTPFound(h.route_path(
+            'edit_user_caches', user_id=c.user.user_id))

rhodecode/authentication/base.py

@@ -35,7 +35,7 b' from pyramid.threadlocal import get_curr'
 
 from rhodecode.authentication.interface import IAuthnPluginRegistry
 from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase
-from rhodecode.lib import caches
+from rhodecode.lib import caches, rc_cache
 from rhodecode.lib.auth import PasswordGenerator, _RhodeCodeCryptoBCrypt
 from rhodecode.lib.utils2 import safe_int, safe_str
 from rhodecode.lib.exceptions import LdapConnectionError
@@ -633,22 +633,6 b' def get_authn_registry(registry=None):'
     return authn_registry
 
 
-def get_auth_cache_manager(custom_ttl=None, suffix=None):
-    cache_name = 'rhodecode.authentication'
-    if suffix:
-        cache_name = 'rhodecode.authentication.{}'.format(suffix)
-    return caches.get_cache_manager(
-        'auth_plugins', cache_name, custom_ttl)
-
-
-def get_perms_cache_manager(custom_ttl=None, suffix=None):
-    cache_name = 'rhodecode.permissions'
-    if suffix:
-        cache_name = 'rhodecode.permissions.{}'.format(suffix)
-    return caches.get_cache_manager(
-        'auth_plugins', cache_name, custom_ttl)
-
-
 def authenticate(username, password, environ=None, auth_type=None,
                  skip_missing=False, registry=None, acl_repo_name=None):
     """
@@ -707,45 +691,35 b' def authenticate(username, password, env'
 
         plugin_cache_active, cache_ttl = plugin.get_ttl_cache(plugin_settings)
 
-        # get instance of cache manager configured for a namespace
-        cache_manager = get_auth_cache_manager(
-            custom_ttl=cache_ttl, suffix=user.user_id if user else '')
-
         log.debug('AUTH_CACHE_TTL for plugin `%s` active: %s (TTL: %s)',
                   plugin.get_id(), plugin_cache_active, cache_ttl)
 
-        # for environ based password can be empty, but then the validation is
-        # on the server that fills in the env data needed for authentication
-
-        _password_hash = caches.compute_key_from_params(
-            plugin.name, username, (password or ''))
+        user_id = user.user_id
+        cache_namespace_uid = 'cache_user_auth.{}'.format(user_id)
+        region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
 
-        # _authenticate is a wrapper for .auth() method of plugin.
-        # it checks if .auth() sends proper data.
-        # For RhodeCodeExternalAuthPlugin it also maps users to
-        # Database and maps the attributes returned from .auth()
-        # to RhodeCode database. If this function returns data
-        # then auth is correct.
-        start = time.time()
-        log.debug('Running plugin `%s` _authenticate method', plugin.get_id())
+        @region.cache_on_arguments(namespace=cache_namespace_uid,
+                                   expiration_time=cache_ttl,
+                                   should_cache_fn=lambda v: plugin_cache_active)
+        def compute_auth(
+                cache_name, plugin_name, username, password):
 
-        def auth_func():
-            """
-            This function is used internally in Cache of Beaker to calculate
-            Results
-            """
-            log.debug('auth: calculating password access now...')
+            # _authenticate is a wrapper for .auth() method of plugin.
+            # it checks if .auth() sends proper data.
+            # For RhodeCodeExternalAuthPlugin it also maps users to
+            # Database and maps the attributes returned from .auth()
+            # to RhodeCode database. If this function returns data
+            # then auth is correct.
+            log.debug('Running plugin `%s` _authenticate method '
+                      'using username and password', plugin.get_id())
             return plugin._authenticate(
                 user, username, password, plugin_settings,
                 environ=environ or {})
 
-        if plugin_cache_active:
-            log.debug('Trying to fetch cached auth by pwd hash `...%s`',
-                      _password_hash[:6])
-            plugin_user = cache_manager.get(
-                _password_hash, createfunc=auth_func)
-        else:
-            plugin_user = auth_func()
+        start = time.time()
+        # for environ based auth, password can be empty, but then the validation is
+        # on the server that fills in the env data needed for authentication
+        plugin_user = compute_auth('auth', plugin.name, username, (password or ''))
 
         auth_time = time.time() - start
         log.debug('Authentication for plugin `%s` completed in %.3fs, '

rhodecode/authentication/views.py

@@ -27,8 +27,7 b' from pyramid.renderers import render'
 from pyramid.response import Response
 
 from rhodecode.apps._base import BaseAppView
-from rhodecode.authentication.base import (
-    get_auth_cache_manager, get_perms_cache_manager, get_authn_registry)
+from rhodecode.authentication.base import get_authn_registry
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import (
     LoginRequired, HasPermissionAllDecorator, CSRFRequired)
@@ -102,16 +101,6 b' class AuthnPluginViewBase(BaseAppView):'
             self.plugin.create_or_update_setting(name, value)
         Session().commit()
 
-        # cleanup cache managers in case of change for plugin
-        # TODO(marcink): because we can register multiple namespaces
-        # we should at some point figure out how to retrieve ALL namespace
-        # cache managers and clear them...
-        cache_manager = get_auth_cache_manager()
-        clear_cache_manager(cache_manager)
-
-        cache_manager = get_perms_cache_manager()
-        clear_cache_manager(cache_manager)
-
         # Display success message and redirect.
         h.flash(_('Auth settings updated successfully.'), category='success')
         redirect_to = self.request.resource_path(

rhodecode/config/middleware.py

@@ -22,6 +22,7 b' import os'
 import logging
 import traceback
 import collections
+import tempfile
 
 from paste.gzipper import make_gzip_middleware
 from pyramid.wsgi import wsgiapp
@@ -220,6 +221,7 b' def includeme(config):'
     config.include('pyramid_mako')
     config.include('pyramid_beaker')
     config.include('rhodecode.lib.caches')
+    config.include('rhodecode.lib.rc_cache')
 
     config.include('rhodecode.authentication')
     config.include('rhodecode.integrations')
@@ -382,6 +384,7 b' def sanitize_settings_and_apply_defaults'
     # Call split out functions that sanitize settings for each topic.
     _sanitize_appenlight_settings(settings)
     _sanitize_vcs_settings(settings)
+    _sanitize_cache_settings(settings)
 
     # configure instance id
     config_utils.set_instance_id(settings)
@@ -421,6 +424,18 b' def _sanitize_vcs_settings(settings):'
         settings['vcs.scm_app_implementation'] = 'http'
 
 
+def _sanitize_cache_settings(settings):
+    _string_setting(settings, 'cache_dir',
+                    os.path.join(tempfile.gettempdir(), 'rc_cache'))
+
+    _string_setting(settings, 'rc_cache.cache_perms.backend',
+                    'dogpile.cache.rc.file_namespace')
+    _int_setting(settings, 'rc_cache.cache_perms.expiration_time',
+                 60)
+    _string_setting(settings, 'rc_cache.cache_perms.arguments.filename',
+                    os.path.join(tempfile.gettempdir(), 'rc_cache_1'))
+
+
 def _int_setting(settings, name, default):
     settings[name] = int(settings.get(name, default))
 

rhodecode/lib/auth.py

@@ -48,7 +48,7 b' from rhodecode.model.user import UserMod'
 from rhodecode.model.db import (
     User, Repository, Permission, UserToPerm, UserGroupToPerm, UserGroupMember,
     UserIpMap, UserApiKeys, RepoGroup, UserGroup)
-from rhodecode.lib import caches
+from rhodecode.lib import rc_cache
 from rhodecode.lib.utils2 import safe_unicode, aslist, safe_str, md5, safe_int, sha1
 from rhodecode.lib.utils import (
     get_repo_slug, get_repo_group_slug, get_user_group_slug)
@@ -976,21 +976,18 b' class AuthUser(object):'
             obj = Repository.get_by_repo_name(scope['repo_name'])
             if obj:
                 scope['repo_id'] = obj.repo_id
-        _scope = {
-            'repo_id': -1,
-            'user_group_id': -1,
-            'repo_group_id': -1,
-        }
-        _scope.update(scope)
-        cache_key = "_".join(map(safe_str, reduce(lambda a, b: a+b,
-                                                  _scope.items())))
-        if cache_key not in self._permissions_scoped_cache:
-            # store in cache to mimic how the @LazyProperty works,
-            # the difference here is that we use the unique key calculated
-            # from params and values
-            res = self.get_perms(user=self, cache=False, scope=_scope)
-            self._permissions_scoped_cache[cache_key] = res
-        return self._permissions_scoped_cache[cache_key]
+        _scope = collections.OrderedDict()
+        _scope['repo_id'] = -1
+        _scope['user_group_id'] = -1
+        _scope['repo_group_id'] = -1
+
+        for k in sorted(scope.keys()):
+            _scope[k] = scope[k]
+
+        # store in cache to mimic how the @LazyProperty works,
+        # the difference here is that we use the unique key calculated
+        # from params and values
+        return self.get_perms(user=self, cache=False, scope=_scope)
 
     def get_instance(self):
         return User.get(self.user_id)
@@ -1072,16 +1069,27 b' class AuthUser(object):'
         user_inherit_default_permissions = user.inherit_default_permissions
 
         cache_seconds = safe_int(
-            rhodecode.CONFIG.get('beaker.cache.short_term.expire'))
+            rhodecode.CONFIG.get('rc_cache.cache_perms.expiration_time'))
+
         cache_on = cache or cache_seconds > 0
         log.debug(
             'Computing PERMISSION tree for user %s scope `%s` '
-            'with caching: %s[%ss]' % (user, scope, cache_on, cache_seconds))
+            'with caching: %s[TTL: %ss]' % (user, scope, cache_on, cache_seconds or 0))
+
+        cache_namespace_uid = 'cache_user_auth.{}'.format(user_id)
+        region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
+
+        @region.cache_on_arguments(namespace=cache_namespace_uid,
+                                   should_cache_fn=lambda v: cache_on)
+        def compute_perm_tree(cache_name,
+                user_id, scope, user_is_admin,user_inherit_default_permissions,
+                explicit, algo, calculate_super_admin):
+            return _cached_perms_data(
+                user_id, scope, user_is_admin, user_inherit_default_permissions,
+                explicit, algo, calculate_super_admin)
+
         start = time.time()
-        compute = caches.conditional_cache(
-            'short_term', 'cache_desc.{}'.format(user_id),
-            condition=cache_on, func=_cached_perms_data)
-        result = compute(user_id, scope, user_is_admin,
+        result = compute_perm_tree('permissions', user_id, scope, user_is_admin,
                          user_inherit_default_permissions, explicit, algo,
                          calculate_super_admin)
 
@@ -1091,6 +1099,7 b' class AuthUser(object):'
         total = time.time() - start
         log.debug('PERMISSION tree for user %s computed in %.3fs: %s' % (
             user, total, result_repr))
+
         return result
 
     @property
@@ -1153,10 +1162,7 b' class AuthUser(object):'
             return [x.repo_id for x in
                     RepoList(qry, perm_set=perm_def)]
 
-        compute = caches.conditional_cache(
-            'short_term', 'repo_acl_ids.{}'.format(self.user_id),
-            condition=cache, func=_cached_repo_acl)
-        return compute(self.user_id, perms, name_filter)
+        return _cached_repo_acl(self.user_id, perms, name_filter)
 
     def repo_group_acl_ids(self, perms=None, name_filter=None, cache=False):
         """
@@ -1179,10 +1185,7 b' class AuthUser(object):'
             return [x.group_id for x in
                     RepoGroupList(qry, perm_set=perm_def)]
 
-        compute = caches.conditional_cache(
-            'short_term', 'repo_group_acl_ids.{}'.format(self.user_id),
-            condition=cache, func=_cached_repo_group_acl)
-        return compute(self.user_id, perms, name_filter)
+        return _cached_repo_group_acl(self.user_id, perms, name_filter)
 
     def user_group_acl_ids(self, perms=None, name_filter=None, cache=False):
         """
@@ -1205,10 +1208,7 b' class AuthUser(object):'
             return [x.users_group_id for x in
                     UserGroupList(qry, perm_set=perm_def)]
 
-        compute = caches.conditional_cache(
-            'short_term', 'user_group_acl_ids.{}'.format(self.user_id),
-            condition=cache, func=_cached_user_group_acl)
-        return compute(self.user_id, perms, name_filter)
+        return _cached_user_group_acl(self.user_id, perms, name_filter)
 
     @property
     def ip_allowed(self):

rhodecode/lib/base.py

@@ -505,6 +505,7 b' def bootstrap_config(request):'
     config.include('pyramid_mako')
     config.include('pyramid_beaker')
     config.include('rhodecode.lib.caches')
+    config.include('rhodecode.lib.rc_cache')
 
     add_events_routes(config)
 

rhodecode/lib/caches.py

@@ -96,7 +96,7 b' def get_cache_manager(region_name, cache'
     Creates a Beaker cache manager. Such instance can be used like that::
 
     _namespace = caches.get_repo_namespace_key(caches.XXX, repo_name)
-    cache_manager = caches.get_cache_manager('repo_cache_long', _namespace)
+    cache_manager = caches.get_cache_manager('some_namespace_name', _namespace)
     _cache_key = caches.compute_key_from_params(repo_name, commit.raw_id)
     def heavy_compute():
         ...
@@ -121,7 +121,7 b' def get_cache_manager(region_name, cache'
 def clear_cache_manager(cache_manager):
     """
     namespace = 'foobar'
-    cache_manager = get_cache_manager('repo_cache_long', namespace)
+    cache_manager = get_cache_manager('some_namespace_name', namespace)
     clear_cache_manager(cache_manager)
     """
 

rhodecode/lib/middleware/simplevcs.py

@@ -39,9 +39,8 b' from pyramid.httpexceptions import ('
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 import rhodecode
-from rhodecode.authentication.base import (
-    authenticate, get_perms_cache_manager, VCS_TYPE, loadplugin)
-from rhodecode.lib import caches
+from rhodecode.authentication.base import authenticate, VCS_TYPE, loadplugin
+from rhodecode.lib import caches, rc_cache
 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
 from rhodecode.lib.base import (
     BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
@@ -311,36 +310,24 b' class SimpleVCS(object):'
         :param repo_name: repository name
         """
 
-        # get instance of cache manager configured for a namespace
-        cache_manager = get_perms_cache_manager(
-            custom_ttl=cache_ttl, suffix=user.user_id)
         log.debug('AUTH_CACHE_TTL for permissions `%s` active: %s (TTL: %s)',
                   plugin_id, plugin_cache_active, cache_ttl)
 
-        # for environ based password can be empty, but then the validation is
-        # on the server that fills in the env data needed for authentication
-        _perm_calc_hash = caches.compute_key_from_params(
-            plugin_id, action, user.user_id, repo_name, ip_addr)
+        user_id = user.user_id
+        cache_namespace_uid = 'cache_user_auth.{}'.format(user_id)
+        region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
 
-        # _authenticate is a wrapper for .auth() method of plugin.
-        # it checks if .auth() sends proper data.
-        # For RhodeCodeExternalAuthPlugin it also maps users to
-        # Database and maps the attributes returned from .auth()
-        # to RhodeCode database. If this function returns data
-        # then auth is correct.
-        start = time.time()
-        log.debug('Running plugin `%s` permissions check', plugin_id)
+        @region.cache_on_arguments(namespace=cache_namespace_uid,
+                                   expiration_time=cache_ttl,
+                                   should_cache_fn=lambda v: plugin_cache_active)
+        def compute_perm_vcs(
+                cache_name, plugin_id, action, user_id, repo_name, ip_addr):
 
-        def perm_func():
-            """
-            This function is used internally in Cache of Beaker to calculate
-            Results
-            """
             log.debug('auth: calculating permission access now...')
             # check IP
             inherit = user.inherit_default_permissions
             ip_allowed = AuthUser.check_ip_allowed(
-                user.user_id, ip_addr, inherit_from_default=inherit)
+                user_id, ip_addr, inherit_from_default=inherit)
             if ip_allowed:
                 log.info('Access for IP:%s allowed', ip_addr)
             else:
@@ -360,12 +347,13 b' class SimpleVCS(object):'
 
             return True
 
-        if plugin_cache_active:
-            log.debug('Trying to fetch cached perms by %s', _perm_calc_hash[:6])
-            perm_result = cache_manager.get(
-                _perm_calc_hash, createfunc=perm_func)
-        else:
-            perm_result = perm_func()
+        start = time.time()
+        log.debug('Running plugin `%s` permissions check', plugin_id)
+
+        # for environ based auth, password can be empty, but then the validation is
+        # on the server that fills in the env data needed for authentication
+        perm_result = compute_perm_vcs(
+            'vcs_permissions', plugin_id, action, user.user_id, repo_name, ip_addr)
 
         auth_time = time.time() - start
         log.debug('Permissions for plugin `%s` completed in %.3fs, '

rhodecode/public/js/rhodecode/routes.js

@@ -119,6 +119,8 b' function registerRCRoutes() {'
     pyroutes.register('edit_user_groups_management', '/_admin/users/%(user_id)s/edit/groups_management', ['user_id']);
     pyroutes.register('edit_user_groups_management_updates', '/_admin/users/%(user_id)s/edit/edit_user_groups_management/updates', ['user_id']);
     pyroutes.register('edit_user_audit_logs', '/_admin/users/%(user_id)s/edit/audit', ['user_id']);
+    pyroutes.register('edit_user_caches', '/_admin/users/%(user_id)s/edit/caches', ['user_id']);
+    pyroutes.register('edit_user_caches_update', '/_admin/users/%(user_id)s/edit/caches/update', ['user_id']);
     pyroutes.register('user_groups', '/_admin/user_groups', []);
     pyroutes.register('user_groups_data', '/_admin/user_groups_data', []);
     pyroutes.register('user_groups_new', '/_admin/user_groups/new', []);

rhodecode/templates/admin/users/user_edit.mako

@@ -45,6 +45,7 b''
           <li class="${'active' if c.active=='ips' else ''}"><a href="${h.route_path('edit_user_ips', user_id=c.user.user_id)}">${_('Ip Whitelist')}</a></li>
           <li class="${'active' if c.active=='groups' else ''}"><a href="${h.route_path('edit_user_groups_management', user_id=c.user.user_id)}">${_('User Groups Management')}</a></li>
           <li class="${'active' if c.active=='audit' else ''}"><a href="${h.route_path('edit_user_audit_logs', user_id=c.user.user_id)}">${_('Audit logs')}</a></li>
+          <li class="${'active' if c.active=='caches' else ''}"><a href="${h.route_path('edit_user_caches', user_id=c.user.user_id)}">${_('Caches')}</a></li>
         </ul>
     </div>
 

rhodecode/tests/lib/test_caches.py

@@ -22,36 +22,24 b' import time'
 
 import pytest
 
-from rhodecode.lib import caches
-from rhodecode.lib.memory_lru_debug import MemoryLRUNamespaceManagerBase
+from rhodecode.lib import rc_cache
 
 
-class TestCacheManager(object):
[e-mail]( 'app')
+class TestCaches(object):
 
-    @pytest.mark.parametrize('repo_name', [
-        ('',),
-        (u'',),
-        (u'ac',),
-        ('ac', ),
-        (u'ęćc',),
-        ('ąac',),
+    def test_cache_decorator_init_not_configured(self):
+        with pytest.raises(EnvironmentError):
+            rc_cache.get_or_create_region('dontexist')
+
+    @pytest.mark.parametrize('region_name', [
+        'cache_perms', u'cache_perms',
     ])
-    def test_cache_manager_init(self, repo_name):
-        cache_manager_instance = caches.get_cache_manager(
-            repo_name, 'my_cache')
-        assert cache_manager_instance
-
-    def test_cache_manager_init_undefined_namespace(self):
-        cache_manager_instance = caches.get_cache_manager(
-            'repo_cache_long_undefined', 'my_cache')
-        assert cache_manager_instance
-
-        def_config = caches.DEFAULT_CACHE_MANAGER_CONFIG.copy()
-        def_config.pop('type')
-        assert cache_manager_instance.nsargs == def_config
-
-        assert isinstance(
-            cache_manager_instance.namespace, MemoryLRUNamespaceManagerBase)
+    def test_cache_decorator_init(self, region_name):
+        namespace = region_name
+        cache_region = rc_cache.get_or_create_region(
+            region_name, region_namespace=namespace)
+        assert cache_region
 
     @pytest.mark.parametrize('example_input', [
         ('',),
@@ -62,45 +50,60 b' class TestCacheManager(object):'
         (u'/ac', ),
     ])
     def test_cache_manager_create_key(self, example_input):
-        key = caches.compute_key_from_params(*example_input)
+        key = rc_cache.utils.compute_key_from_params(*example_input)
         assert key
 
-    def test_store_and_invalidate_value_from_manager(self):
-        cache_manger_instance = caches.get_cache_manager(
-            'repo_cache_long', 'my_cache_store')
+    @pytest.mark.parametrize('example_namespace', [
+        'namespace', None
+    ])
+    @pytest.mark.parametrize('example_input', [
+        ('',),
+        (u'/ac',),
+        (u'/ac', 1, 2, object()),
+        (u'/ęćc', 1, 2, object()),
+        ('/ąac',),
+        (u'/ac', ),
+    ])
+    def test_cache_keygen(self, example_input, example_namespace):
+        def func_wrapped():
+            return 1
+        func = rc_cache.utils.key_generator(example_namespace, func_wrapped)
+        key = func(*example_input)
+        assert key
 
-        def compute():
+    def test_store_value_in_cache(self):
+        cache_region = rc_cache.get_or_create_region('cache_perms')
+        # make sure we empty the cache now
+        for key in cache_region.backend.list_keys():
+            cache_region.delete(key)
+
+        assert cache_region.backend.list_keys() == []
+
+        @cache_region.cache_on_arguments(expiration_time=5)
+        def compute(key):
             return time.time()
 
-        added_keys = []
-        for i in xrange(3):
-            _cache_key = caches.compute_key_from_params('foo_bar', 'p%s' % i)
-            added_keys.append(_cache_key)
-            for x in xrange(10):
-                cache_manger_instance.get(_cache_key, createfunc=compute)
+        for x in range(10):
+            compute(x)
+
+        assert len(set(cache_region.backend.list_keys())) == 10
 
-        for key in added_keys:
-            assert cache_manger_instance[key]
-
-        caches.clear_cache_manager(cache_manger_instance)
-
-        for key in added_keys:
-            assert key not in cache_manger_instance
-        assert len(cache_manger_instance.namespace.keys()) == 0
+    def test_store_and_get_value_from_region(self):
+        cache_region = rc_cache.get_or_create_region('cache_perms')
+        # make sure we empty the cache now
+        for key in cache_region.backend.list_keys():
+            cache_region.delete(key)
+        assert cache_region.backend.list_keys() == []
 
-    def test_store_and_get_value_from_manager(self):
-            cache_manger_instance = caches.get_cache_manager(
-                'repo_cache_long', 'my_cache_store')
-
-            _cache_key = caches.compute_key_from_params('foo_bar', 'multicall')
+        @cache_region.cache_on_arguments(expiration_time=5)
+        def compute(key):
+            return time.time()
 
-            def compute():
-                return time.time()
+        result = set()
+        for x in range(10):
+            ret = compute('x')
+            result.add(ret)
 
-            result = set()
-            for x in xrange(10):
-                ret = cache_manger_instance.get(_cache_key, createfunc=compute)
-                result.add(ret)
-
-            # once computed we have only one value after executing it 10x
-            assert len(result) == 1
+        # once computed we have only one value (the same from cache)
+        # after executing it 10x
+        assert len(result) == 1

rhodecode/tests/rhodecode.ini

@@ -292,11 +292,7 b' cache_dir = %(here)s/data'
 beaker.cache.data_dir = %(here)s/rc/data/cache/beaker_data
 beaker.cache.lock_dir = %(here)s/rc/data/cache/beaker_lock
 
-beaker.cache.regions = short_term, long_term, sql_cache_short, auth_plugins, repo_cache_long
-
-beaker.cache.short_term.type = file
-beaker.cache.short_term.expire = 0
-beaker.cache.short_term.key_length = 256
+beaker.cache.regions = long_term, sql_cache_short, repo_cache_long
 
 beaker.cache.long_term.type = memory
 beaker.cache.long_term.expire = 36000
@@ -306,16 +302,6 b' beaker.cache.sql_cache_short.type = memo'
 beaker.cache.sql_cache_short.expire = 1
 beaker.cache.sql_cache_short.key_length = 256
 
-## default is memory cache, configure only if required
-## using multi-node or multi-worker setup
-#beaker.cache.auth_plugins.type = memory
-#beaker.cache.auth_plugins.lock_dir = %(here)s/data/cache/auth_plugin_lock
-#beaker.cache.auth_plugins.url = postgresql://postgres:secret@localhost/rhodecode
-#beaker.cache.auth_plugins.url = mysql://root:secret@127.0.0.1/rhodecode
-#beaker.cache.auth_plugins.sa.pool_recycle = 3600
-#beaker.cache.auth_plugins.sa.pool_size = 10
-#beaker.cache.auth_plugins.sa.max_overflow = 0
-
 beaker.cache.repo_cache_long.type = memorylru_base
 beaker.cache.repo_cache_long.max_items = 4096
 beaker.cache.repo_cache_long.expire = 2592000
@@ -327,6 +313,16 b' beaker.cache.repo_cache_long.expire = 25'
 #beaker.cache.repo_cache_long.expire = 1209600
 #beaker.cache.repo_cache_long.key_length = 256
 
+
+#####################################
+###         DOGPILE CACHE        ####
+#####################################
+
+## permission tree cache settings
+rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace
+rc_cache.cache_perms.expiration_time = 0
+rc_cache.cache_perms.arguments.filename = /tmp/rc_cache_1
+
 ####################################
 ###       BEAKER SESSION        ####
 ####################################