##// END OF EJS Templates
caches: rewrite of auth/permission caches to dogpile.
marcink -
r2845:3d5ae486 default
parent child Browse files
Show More
@@ -0,0 +1,66 b''
1 # -*- coding: utf-8 -*-
2
3 # Copyright (C) 2015-2018 RhodeCode GmbH
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
21 from dogpile.cache import register_backend
22 from dogpile.cache import make_region
23
24 register_backend(
25 "dogpile.cache.rc.memory_lru", "rhodecode.lib.rc_cache.backends",
26 "LRUMemoryBackend")
27
28 register_backend(
29 "dogpile.cache.rc.file_namespace", "rhodecode.lib.rc_cache.backends",
30 "FileNamespaceBackend")
31
32 register_backend(
33 "dogpile.cache.rc.redis", "rhodecode.lib.rc_cache.backends",
34 "RedisPickleBackend")
35
36
37 from . import region_meta
38 from .utils import get_default_cache_settings, key_generator, get_or_create_region
39
40
41 def configure_dogpile_cache(settings):
42 cache_dir = settings.get('cache_dir')
43 if cache_dir:
44 region_meta.dogpile_config_defaults['cache_dir'] = cache_dir
45
46 rc_cache_data = get_default_cache_settings(settings, prefixes=['rc_cache.'])
47
48 # inspect available namespaces
49 avail_regions = set()
50 for key in rc_cache_data.keys():
51 namespace_name = key.split('.', 1)[0]
52 avail_regions.add(namespace_name)
53
54 # register them into namespace
55 for region_name in avail_regions:
56 new_region = make_region(
57 name=region_name,
58 function_key_generator=key_generator
59 )
60
61 new_region.configure_from_config(settings, 'rc_cache.{}.'.format(region_name))
62 region_meta.dogpile_cache_regions[region_name] = new_region
63
64
65 def includeme(config):
66 configure_dogpile_cache(config.registry.settings)
@@ -0,0 +1,109 b''
1 # -*- coding: utf-8 -*-
2
3 # Copyright (C) 2015-2018 RhodeCode GmbH
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
21 from dogpile.cache.backends import memory as memory_backend
22 from dogpile.cache.backends import file as file_backend
23 from dogpile.cache.backends import redis as redis_backend
24 from dogpile.cache.backends.file import NO_VALUE, compat
25
26 from rhodecode.lib.memory_lru_debug import LRUDict
27
28 _default_max_size = 1024
29
30
31 class LRUMemoryBackend(memory_backend.MemoryBackend):
32
33 def __init__(self, arguments):
34 max_size = arguments.pop('max_size', _default_max_size)
35 arguments['cache_dict'] = LRUDict(max_size)
36 super(LRUMemoryBackend, self).__init__(arguments)
37
38
39 class Serializer(object):
40 def _dumps(self, value):
41 return compat.pickle.dumps(value)
42
43 def _loads(self, value):
44 return compat.pickle.loads(value)
45
46
47 class FileNamespaceBackend(Serializer, file_backend.DBMBackend):
48
49 def __init__(self, arguments):
50 super(FileNamespaceBackend, self).__init__(arguments)
51
52 def list_keys(self):
53 with self._dbm_file(True) as dbm:
54 return dbm.keys()
55
56 def get_store(self):
57 return self.filename
58
59 def get(self, key):
60 with self._dbm_file(False) as dbm:
61 if hasattr(dbm, 'get'):
62 value = dbm.get(key, NO_VALUE)
63 else:
64 # gdbm objects lack a .get method
65 try:
66 value = dbm[key]
67 except KeyError:
68 value = NO_VALUE
69 if value is not NO_VALUE:
70 value = self._loads(value)
71 return value
72
73 def set(self, key, value):
74 with self._dbm_file(True) as dbm:
75 dbm[key] = self._dumps(value)
76
77 def set_multi(self, mapping):
78 with self._dbm_file(True) as dbm:
79 for key, value in mapping.items():
80 dbm[key] = self._dumps(value)
81
82
83 class RedisPickleBackend(Serializer, redis_backend.RedisBackend):
84 def list_keys(self):
85 return self.client.keys()
86
87 def get_store(self):
88 return self.client.connection_pool
89
90 def set(self, key, value):
91 if self.redis_expiration_time:
92 self.client.setex(key, self.redis_expiration_time,
93 self._dumps(value))
94 else:
95 self.client.set(key, self._dumps(value))
96
97 def set_multi(self, mapping):
98 mapping = dict(
99 (k, self._dumps(v))
100 for k, v in mapping.items()
101 )
102
103 if not self.redis_expiration_time:
104 self.client.mset(mapping)
105 else:
106 pipe = self.client.pipeline()
107 for key, value in mapping.items():
108 pipe.setex(key, self.redis_expiration_time, value)
109 pipe.execute()
@@ -0,0 +1,28 b''
1 # -*- coding: utf-8 -*-
2
3 # Copyright (C) 2015-2018 RhodeCode GmbH
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 import os
21 import tempfile
22
23 dogpile_config_defaults = {
24 'cache_dir': os.path.join(tempfile.gettempdir(), 'rc_cache')
25 }
26
27 # GLOBAL TO STORE ALL REGISTERED REGIONS
28 dogpile_cache_regions = {}
@@ -0,0 +1,99 b''
1 # -*- coding: utf-8 -*-
2
3 # Copyright (C) 2015-2018 RhodeCode GmbH
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 import os
21 import logging
22 from dogpile.cache import make_region
23
24 from rhodecode.lib.utils import safe_str, sha1
25 from . import region_meta
26
27 log = logging.getLogger(__name__)
28
29
30 def get_default_cache_settings(settings, prefixes=None):
31 prefixes = prefixes or []
32 cache_settings = {}
33 for key in settings.keys():
34 for prefix in prefixes:
35 if key.startswith(prefix):
36 name = key.split(prefix)[1].strip()
37 val = settings[key]
38 if isinstance(val, basestring):
39 val = val.strip()
40 cache_settings[name] = val
41 return cache_settings
42
43
44 def compute_key_from_params(*args):
45 """
46 Helper to compute key from given params to be used in cache manager
47 """
48 return sha1("_".join(map(safe_str, args)))
49
50
51 def key_generator(namespace, fn):
52 fname = fn.__name__
53
54 def generate_key(*args):
55 namespace_pref = namespace or 'default'
56 arg_key = compute_key_from_params(*args)
57 final_key = "{}:{}_{}".format(namespace_pref, fname, arg_key)
58
59 return final_key
60
61 return generate_key
62
63
64 def get_or_create_region(region_name, region_namespace=None):
65 from rhodecode.lib.rc_cache.backends import FileNamespaceBackend
66 region_obj = region_meta.dogpile_cache_regions.get(region_name)
67 if not region_obj:
68 raise EnvironmentError(
69 'Region `{}` not in configured: {}.'.format(
70 region_name, region_meta.dogpile_cache_regions.keys()))
71
72 region_uid_name = '{}:{}'.format(region_name, region_namespace)
73 if isinstance(region_obj.actual_backend, FileNamespaceBackend):
74 region_exist = region_meta.dogpile_cache_regions.get(region_namespace)
75 if region_exist:
76 log.debug('Using already configured region: %s', region_namespace)
77 return region_exist
78 cache_dir = region_meta.dogpile_config_defaults['cache_dir']
79 expiration_time = region_obj.expiration_time
80
81 if not os.path.isdir(cache_dir):
82 os.makedirs(cache_dir)
83 new_region = make_region(
84 name=region_uid_name, function_key_generator=key_generator
85 )
86 namespace_filename = os.path.join(
87 cache_dir, "{}.cache.dbm".format(region_namespace))
88 # special type that allows 1db per namespace
89 new_region.configure(
90 backend='dogpile.cache.rc.file_namespace',
91 expiration_time=expiration_time,
92 arguments={"filename": namespace_filename}
93 )
94
95 # create and save in region caches
96 log.debug('configuring new region: %s',region_uid_name)
97 region_obj = region_meta.dogpile_cache_regions[region_namespace] = new_region
98
99 return region_obj
@@ -0,0 +1,29 b''
1 <%namespace name="base" file="/base/base.mako"/>
2
3 <div class="panel panel-default">
4 <div class="panel-heading">
5 <h3 class="panel-title">${_('Caches')}</h3>
6 </div>
7 <div class="panel-body">
8 <pre>
9 region: ${c.region.name}
10 backend: ${c.region.actual_backend.__class__}
11 store: ${c.region.actual_backend.get_store()}
12
13 % for k in c.user_keys:
14 - ${k}
15 % endfor
16 </pre>
17
18 ${h.secure_form(h.route_path('edit_user_caches_update', user_id=c.user.user_id), request=request)}
19 <div class="form">
20 <div class="fields">
21 ${h.submit('reset_cache_%s' % c.user.user_id, _('Invalidate user cache'),class_="btn btn-small",onclick="return confirm('"+_('Confirm to invalidate user cache')+"');")}
22 </div>
23 </div>
24 ${h.end_form()}
25
26 </div>
27 </div>
28
29
@@ -320,12 +320,7 b' cache_dir = %(here)s/data'
320 beaker.cache.data_dir = %(here)s/data/cache/beaker_data
320 beaker.cache.data_dir = %(here)s/data/cache/beaker_data
321 beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock
321 beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock
322
322
323 beaker.cache.regions = short_term, long_term, sql_cache_short, auth_plugins, repo_cache_long
323 beaker.cache.regions = long_term, sql_cache_short, repo_cache_long
324
325 # used for caching user permissions
326 beaker.cache.short_term.type = file
327 beaker.cache.short_term.expire = 0
328 beaker.cache.short_term.key_length = 256
329
324
330 beaker.cache.long_term.type = memory
325 beaker.cache.long_term.type = memory
331 beaker.cache.long_term.expire = 36000
326 beaker.cache.long_term.expire = 36000
@@ -335,16 +330,6 b' beaker.cache.sql_cache_short.type = memo'
335 beaker.cache.sql_cache_short.expire = 10
330 beaker.cache.sql_cache_short.expire = 10
336 beaker.cache.sql_cache_short.key_length = 256
331 beaker.cache.sql_cache_short.key_length = 256
337
332
338 ## default is memory cache, configure only if required
339 ## using multi-node or multi-worker setup
340 #beaker.cache.auth_plugins.type = ext:database
341 #beaker.cache.auth_plugins.lock_dir = %(here)s/data/cache/auth_plugin_lock
342 #beaker.cache.auth_plugins.url = postgresql://postgres:secret@localhost/rhodecode
343 #beaker.cache.auth_plugins.url = mysql://root:secret@127.0.0.1/rhodecode
344 #beaker.cache.auth_plugins.sa.pool_recycle = 3600
345 #beaker.cache.auth_plugins.sa.pool_size = 10
346 #beaker.cache.auth_plugins.sa.max_overflow = 0
347
348 beaker.cache.repo_cache_long.type = memorylru_base
333 beaker.cache.repo_cache_long.type = memorylru_base
349 beaker.cache.repo_cache_long.max_items = 4096
334 beaker.cache.repo_cache_long.max_items = 4096
350 beaker.cache.repo_cache_long.expire = 2592000
335 beaker.cache.repo_cache_long.expire = 2592000
@@ -295,12 +295,7 b' cache_dir = %(here)s/data'
295 beaker.cache.data_dir = %(here)s/data/cache/beaker_data
295 beaker.cache.data_dir = %(here)s/data/cache/beaker_data
296 beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock
296 beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock
297
297
298 beaker.cache.regions = short_term, long_term, sql_cache_short, auth_plugins, repo_cache_long
298 beaker.cache.regions = long_term, sql_cache_short, repo_cache_long
299
300 # used for caching user permissions
301 beaker.cache.short_term.type = file
302 beaker.cache.short_term.expire = 0
303 beaker.cache.short_term.key_length = 256
304
299
305 beaker.cache.long_term.type = memory
300 beaker.cache.long_term.type = memory
306 beaker.cache.long_term.expire = 36000
301 beaker.cache.long_term.expire = 36000
@@ -310,16 +305,6 b' beaker.cache.sql_cache_short.type = memo'
310 beaker.cache.sql_cache_short.expire = 10
305 beaker.cache.sql_cache_short.expire = 10
311 beaker.cache.sql_cache_short.key_length = 256
306 beaker.cache.sql_cache_short.key_length = 256
312
307
313 ## default is memory cache, configure only if required
314 ## using multi-node or multi-worker setup
315 #beaker.cache.auth_plugins.type = ext:database
316 #beaker.cache.auth_plugins.lock_dir = %(here)s/data/cache/auth_plugin_lock
317 #beaker.cache.auth_plugins.url = postgresql://postgres:secret@localhost/rhodecode
318 #beaker.cache.auth_plugins.url = mysql://root:secret@127.0.0.1/rhodecode
319 #beaker.cache.auth_plugins.sa.pool_recycle = 3600
320 #beaker.cache.auth_plugins.sa.pool_size = 10
321 #beaker.cache.auth_plugins.sa.max_overflow = 0
322
323 beaker.cache.repo_cache_long.type = memorylru_base
308 beaker.cache.repo_cache_long.type = memorylru_base
324 beaker.cache.repo_cache_long.max_items = 4096
309 beaker.cache.repo_cache_long.max_items = 4096
325 beaker.cache.repo_cache_long.expire = 2592000
310 beaker.cache.repo_cache_long.expire = 2592000
@@ -356,6 +356,16 b' def admin_routes(config):'
356 name='edit_user_audit_logs',
356 name='edit_user_audit_logs',
357 pattern='/users/{user_id:\d+}/edit/audit', user_route=True)
357 pattern='/users/{user_id:\d+}/edit/audit', user_route=True)
358
358
359 # user caches
360 config.add_route(
361 name='edit_user_caches',
362 pattern='/users/{user_id:\d+}/edit/caches',
363 user_route=True)
364 config.add_route(
365 name='edit_user_caches_update',
366 pattern='/users/{user_id:\d+}/edit/caches/update',
367 user_route=True)
368
359 # user-groups admin
369 # user-groups admin
360 config.add_route(
370 config.add_route(
361 name='user_groups',
371 name='user_groups',
@@ -34,7 +34,7 b' from rhodecode.authentication.plugins im'
34 from rhodecode.events import trigger
34 from rhodecode.events import trigger
35 from rhodecode.model.db import true
35 from rhodecode.model.db import true
36
36
37 from rhodecode.lib import audit_logger
37 from rhodecode.lib import audit_logger, rc_cache
38 from rhodecode.lib.exceptions import (
38 from rhodecode.lib.exceptions import (
39 UserCreationError, UserOwnsReposException, UserOwnsRepoGroupsException,
39 UserCreationError, UserOwnsReposException, UserOwnsRepoGroupsException,
40 UserOwnsUserGroupsException, DefaultUserException)
40 UserOwnsUserGroupsException, DefaultUserException)
@@ -1198,3 +1198,57 b' class UsersView(UserAppView):'
1198 perm_user = self.db_user.AuthUser(ip_addr=self.request.remote_addr)
1198 perm_user = self.db_user.AuthUser(ip_addr=self.request.remote_addr)
1199
1199
1200 return perm_user.permissions
1200 return perm_user.permissions
1201
1202 def _get_user_cache_keys(self, cache_namespace_uid, keys):
1203 user_keys = []
1204 for k in sorted(keys):
1205 if k.startswith(cache_namespace_uid):
1206 user_keys.append(k)
1207 return user_keys
1208
1209 @LoginRequired()
1210 @HasPermissionAllDecorator('hg.admin')
1211 @view_config(
1212 route_name='edit_user_caches', request_method='GET',
1213 renderer='rhodecode:templates/admin/users/user_edit.mako')
1214 def user_caches(self):
1215 _ = self.request.translate
1216 c = self.load_default_context()
1217 c.user = self.db_user
1218
1219 c.active = 'caches'
1220 c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
1221
1222 cache_namespace_uid = 'cache_user_auth.{}'.format(self.db_user.user_id)
1223 c.region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
1224 c.backend = c.region.backend
1225 c.user_keys = self._get_user_cache_keys(
1226 cache_namespace_uid, c.region.backend.list_keys())
1227
1228 return self._get_template_context(c)
1229
1230 @LoginRequired()
1231 @HasPermissionAllDecorator('hg.admin')
1232 @CSRFRequired()
1233 @view_config(
1234 route_name='edit_user_caches_update', request_method='POST')
1235 def user_caches_update(self):
1236 _ = self.request.translate
1237 c = self.load_default_context()
1238 c.user = self.db_user
1239
1240 c.active = 'caches'
1241 c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
1242
1243 cache_namespace_uid = 'cache_user_auth.{}'.format(self.db_user.user_id)
1244 c.region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
1245
1246 c.user_keys = self._get_user_cache_keys(
1247 cache_namespace_uid, c.region.backend.list_keys())
1248 for k in c.user_keys:
1249 c.region.delete(k)
1250
1251 h.flash(_("Deleted {} cache keys").format(len(c.user_keys)), category='success')
1252
1253 return HTTPFound(h.route_path(
1254 'edit_user_caches', user_id=c.user.user_id))
@@ -35,7 +35,7 b' from pyramid.threadlocal import get_curr'
35
35
36 from rhodecode.authentication.interface import IAuthnPluginRegistry
36 from rhodecode.authentication.interface import IAuthnPluginRegistry
37 from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase
37 from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase
38 from rhodecode.lib import caches
38 from rhodecode.lib import caches, rc_cache
39 from rhodecode.lib.auth import PasswordGenerator, _RhodeCodeCryptoBCrypt
39 from rhodecode.lib.auth import PasswordGenerator, _RhodeCodeCryptoBCrypt
40 from rhodecode.lib.utils2 import safe_int, safe_str
40 from rhodecode.lib.utils2 import safe_int, safe_str
41 from rhodecode.lib.exceptions import LdapConnectionError
41 from rhodecode.lib.exceptions import LdapConnectionError
@@ -633,22 +633,6 b' def get_authn_registry(registry=None):'
633 return authn_registry
633 return authn_registry
634
634
635
635
636 def get_auth_cache_manager(custom_ttl=None, suffix=None):
637 cache_name = 'rhodecode.authentication'
638 if suffix:
639 cache_name = 'rhodecode.authentication.{}'.format(suffix)
640 return caches.get_cache_manager(
641 'auth_plugins', cache_name, custom_ttl)
642
643
644 def get_perms_cache_manager(custom_ttl=None, suffix=None):
645 cache_name = 'rhodecode.permissions'
646 if suffix:
647 cache_name = 'rhodecode.permissions.{}'.format(suffix)
648 return caches.get_cache_manager(
649 'auth_plugins', cache_name, custom_ttl)
650
651
652 def authenticate(username, password, environ=None, auth_type=None,
636 def authenticate(username, password, environ=None, auth_type=None,
653 skip_missing=False, registry=None, acl_repo_name=None):
637 skip_missing=False, registry=None, acl_repo_name=None):
654 """
638 """
@@ -707,45 +691,35 b' def authenticate(username, password, env'
707
691
708 plugin_cache_active, cache_ttl = plugin.get_ttl_cache(plugin_settings)
692 plugin_cache_active, cache_ttl = plugin.get_ttl_cache(plugin_settings)
709
693
710 # get instance of cache manager configured for a namespace
711 cache_manager = get_auth_cache_manager(
712 custom_ttl=cache_ttl, suffix=user.user_id if user else '')
713
714 log.debug('AUTH_CACHE_TTL for plugin `%s` active: %s (TTL: %s)',
694 log.debug('AUTH_CACHE_TTL for plugin `%s` active: %s (TTL: %s)',
715 plugin.get_id(), plugin_cache_active, cache_ttl)
695 plugin.get_id(), plugin_cache_active, cache_ttl)
716
696
717 # for environ based password can be empty, but then the validation is
697 user_id = user.user_id
718 # on the server that fills in the env data needed for authentication
698 cache_namespace_uid = 'cache_user_auth.{}'.format(user_id)
719
699 region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
720 _password_hash = caches.compute_key_from_params(
721 plugin.name, username, (password or ''))
722
700
723 # _authenticate is a wrapper for .auth() method of plugin.
701 @region.cache_on_arguments(namespace=cache_namespace_uid,
724 # it checks if .auth() sends proper data.
702 expiration_time=cache_ttl,
725 # For RhodeCodeExternalAuthPlugin it also maps users to
703 should_cache_fn=lambda v: plugin_cache_active)
726 # Database and maps the attributes returned from .auth()
704 def compute_auth(
727 # to RhodeCode database. If this function returns data
705 cache_name, plugin_name, username, password):
728 # then auth is correct.
729 start = time.time()
730 log.debug('Running plugin `%s` _authenticate method', plugin.get_id())
731
706
732 def auth_func():
707 # _authenticate is a wrapper for .auth() method of plugin.
733 """
708 # it checks if .auth() sends proper data.
734 This function is used internally in Cache of Beaker to calculate
709 # For RhodeCodeExternalAuthPlugin it also maps users to
735 Results
710 # Database and maps the attributes returned from .auth()
736 """
711 # to RhodeCode database. If this function returns data
737 log.debug('auth: calculating password access now...')
712 # then auth is correct.
713 log.debug('Running plugin `%s` _authenticate method '
714 'using username and password', plugin.get_id())
738 return plugin._authenticate(
715 return plugin._authenticate(
739 user, username, password, plugin_settings,
716 user, username, password, plugin_settings,
740 environ=environ or {})
717 environ=environ or {})
741
718
742 if plugin_cache_active:
719 start = time.time()
743 log.debug('Trying to fetch cached auth by pwd hash `...%s`',
720 # for environ based auth, password can be empty, but then the validation is
744 _password_hash[:6])
721 # on the server that fills in the env data needed for authentication
745 plugin_user = cache_manager.get(
722 plugin_user = compute_auth('auth', plugin.name, username, (password or ''))
746 _password_hash, createfunc=auth_func)
747 else:
748 plugin_user = auth_func()
749
723
750 auth_time = time.time() - start
724 auth_time = time.time() - start
751 log.debug('Authentication for plugin `%s` completed in %.3fs, '
725 log.debug('Authentication for plugin `%s` completed in %.3fs, '
@@ -27,8 +27,7 b' from pyramid.renderers import render'
27 from pyramid.response import Response
27 from pyramid.response import Response
28
28
29 from rhodecode.apps._base import BaseAppView
29 from rhodecode.apps._base import BaseAppView
30 from rhodecode.authentication.base import (
30 from rhodecode.authentication.base import get_authn_registry
31 get_auth_cache_manager, get_perms_cache_manager, get_authn_registry)
32 from rhodecode.lib import helpers as h
31 from rhodecode.lib import helpers as h
33 from rhodecode.lib.auth import (
32 from rhodecode.lib.auth import (
34 LoginRequired, HasPermissionAllDecorator, CSRFRequired)
33 LoginRequired, HasPermissionAllDecorator, CSRFRequired)
@@ -102,16 +101,6 b' class AuthnPluginViewBase(BaseAppView):'
102 self.plugin.create_or_update_setting(name, value)
101 self.plugin.create_or_update_setting(name, value)
103 Session().commit()
102 Session().commit()
104
103
105 # cleanup cache managers in case of change for plugin
106 # TODO(marcink): because we can register multiple namespaces
107 # we should at some point figure out how to retrieve ALL namespace
108 # cache managers and clear them...
109 cache_manager = get_auth_cache_manager()
110 clear_cache_manager(cache_manager)
111
112 cache_manager = get_perms_cache_manager()
113 clear_cache_manager(cache_manager)
114
115 # Display success message and redirect.
104 # Display success message and redirect.
116 h.flash(_('Auth settings updated successfully.'), category='success')
105 h.flash(_('Auth settings updated successfully.'), category='success')
117 redirect_to = self.request.resource_path(
106 redirect_to = self.request.resource_path(
@@ -22,6 +22,7 b' import os'
22 import logging
22 import logging
23 import traceback
23 import traceback
24 import collections
24 import collections
25 import tempfile
25
26
26 from paste.gzipper import make_gzip_middleware
27 from paste.gzipper import make_gzip_middleware
27 from pyramid.wsgi import wsgiapp
28 from pyramid.wsgi import wsgiapp
@@ -220,6 +221,7 b' def includeme(config):'
220 config.include('pyramid_mako')
221 config.include('pyramid_mako')
221 config.include('pyramid_beaker')
222 config.include('pyramid_beaker')
222 config.include('rhodecode.lib.caches')
223 config.include('rhodecode.lib.caches')
224 config.include('rhodecode.lib.rc_cache')
223
225
224 config.include('rhodecode.authentication')
226 config.include('rhodecode.authentication')
225 config.include('rhodecode.integrations')
227 config.include('rhodecode.integrations')
@@ -382,6 +384,7 b' def sanitize_settings_and_apply_defaults'
382 # Call split out functions that sanitize settings for each topic.
384 # Call split out functions that sanitize settings for each topic.
383 _sanitize_appenlight_settings(settings)
385 _sanitize_appenlight_settings(settings)
384 _sanitize_vcs_settings(settings)
386 _sanitize_vcs_settings(settings)
387 _sanitize_cache_settings(settings)
385
388
386 # configure instance id
389 # configure instance id
387 config_utils.set_instance_id(settings)
390 config_utils.set_instance_id(settings)
@@ -421,6 +424,18 b' def _sanitize_vcs_settings(settings):'
421 settings['vcs.scm_app_implementation'] = 'http'
424 settings['vcs.scm_app_implementation'] = 'http'
422
425
423
426
427 def _sanitize_cache_settings(settings):
428 _string_setting(settings, 'cache_dir',
429 os.path.join(tempfile.gettempdir(), 'rc_cache'))
430
431 _string_setting(settings, 'rc_cache.cache_perms.backend',
432 'dogpile.cache.rc.file_namespace')
433 _int_setting(settings, 'rc_cache.cache_perms.expiration_time',
434 60)
435 _string_setting(settings, 'rc_cache.cache_perms.arguments.filename',
436 os.path.join(tempfile.gettempdir(), 'rc_cache_1'))
437
438
424 def _int_setting(settings, name, default):
439 def _int_setting(settings, name, default):
425 settings[name] = int(settings.get(name, default))
440 settings[name] = int(settings.get(name, default))
426
441
@@ -48,7 +48,7 b' from rhodecode.model.user import UserMod'
48 from rhodecode.model.db import (
48 from rhodecode.model.db import (
49 User, Repository, Permission, UserToPerm, UserGroupToPerm, UserGroupMember,
49 User, Repository, Permission, UserToPerm, UserGroupToPerm, UserGroupMember,
50 UserIpMap, UserApiKeys, RepoGroup, UserGroup)
50 UserIpMap, UserApiKeys, RepoGroup, UserGroup)
51 from rhodecode.lib import caches
51 from rhodecode.lib import rc_cache
52 from rhodecode.lib.utils2 import safe_unicode, aslist, safe_str, md5, safe_int, sha1
52 from rhodecode.lib.utils2 import safe_unicode, aslist, safe_str, md5, safe_int, sha1
53 from rhodecode.lib.utils import (
53 from rhodecode.lib.utils import (
54 get_repo_slug, get_repo_group_slug, get_user_group_slug)
54 get_repo_slug, get_repo_group_slug, get_user_group_slug)
@@ -976,21 +976,18 b' class AuthUser(object):'
976 obj = Repository.get_by_repo_name(scope['repo_name'])
976 obj = Repository.get_by_repo_name(scope['repo_name'])
977 if obj:
977 if obj:
978 scope['repo_id'] = obj.repo_id
978 scope['repo_id'] = obj.repo_id
979 _scope = {
979 _scope = collections.OrderedDict()
980 'repo_id': -1,
980 _scope['repo_id'] = -1
981 'user_group_id': -1,
981 _scope['user_group_id'] = -1
982 'repo_group_id': -1,
982 _scope['repo_group_id'] = -1
983 }
983
984 _scope.update(scope)
984 for k in sorted(scope.keys()):
985 cache_key = "_".join(map(safe_str, reduce(lambda a, b: a+b,
985 _scope[k] = scope[k]
986 _scope.items())))
986
987 if cache_key not in self._permissions_scoped_cache:
987 # store in cache to mimic how the @LazyProperty works,
988 # store in cache to mimic how the @LazyProperty works,
988 # the difference here is that we use the unique key calculated
989 # the difference here is that we use the unique key calculated
989 # from params and values
990 # from params and values
990 return self.get_perms(user=self, cache=False, scope=_scope)
991 res = self.get_perms(user=self, cache=False, scope=_scope)
992 self._permissions_scoped_cache[cache_key] = res
993 return self._permissions_scoped_cache[cache_key]
994
991
995 def get_instance(self):
992 def get_instance(self):
996 return User.get(self.user_id)
993 return User.get(self.user_id)
@@ -1072,16 +1069,27 b' class AuthUser(object):'
1072 user_inherit_default_permissions = user.inherit_default_permissions
1069 user_inherit_default_permissions = user.inherit_default_permissions
1073
1070
1074 cache_seconds = safe_int(
1071 cache_seconds = safe_int(
1075 rhodecode.CONFIG.get('beaker.cache.short_term.expire'))
1072 rhodecode.CONFIG.get('rc_cache.cache_perms.expiration_time'))
1073
1076 cache_on = cache or cache_seconds > 0
1074 cache_on = cache or cache_seconds > 0
1077 log.debug(
1075 log.debug(
1078 'Computing PERMISSION tree for user %s scope `%s` '
1076 'Computing PERMISSION tree for user %s scope `%s` '
1079 'with caching: %s[%ss]' % (user, scope, cache_on, cache_seconds))
1077 'with caching: %s[TTL: %ss]' % (user, scope, cache_on, cache_seconds or 0))
1078
1079 cache_namespace_uid = 'cache_user_auth.{}'.format(user_id)
1080 region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
1081
1082 @region.cache_on_arguments(namespace=cache_namespace_uid,
1083 should_cache_fn=lambda v: cache_on)
1084 def compute_perm_tree(cache_name,
1085 user_id, scope, user_is_admin,user_inherit_default_permissions,
1086 explicit, algo, calculate_super_admin):
1087 return _cached_perms_data(
1088 user_id, scope, user_is_admin, user_inherit_default_permissions,
1089 explicit, algo, calculate_super_admin)
1090
1080 start = time.time()
1091 start = time.time()
1081 compute = caches.conditional_cache(
1092 result = compute_perm_tree('permissions', user_id, scope, user_is_admin,
1082 'short_term', 'cache_desc.{}'.format(user_id),
1083 condition=cache_on, func=_cached_perms_data)
1084 result = compute(user_id, scope, user_is_admin,
1085 user_inherit_default_permissions, explicit, algo,
1093 user_inherit_default_permissions, explicit, algo,
1086 calculate_super_admin)
1094 calculate_super_admin)
1087
1095
@@ -1091,6 +1099,7 b' class AuthUser(object):'
1091 total = time.time() - start
1099 total = time.time() - start
1092 log.debug('PERMISSION tree for user %s computed in %.3fs: %s' % (
1100 log.debug('PERMISSION tree for user %s computed in %.3fs: %s' % (
1093 user, total, result_repr))
1101 user, total, result_repr))
1102
1094 return result
1103 return result
1095
1104
1096 @property
1105 @property
@@ -1153,10 +1162,7 b' class AuthUser(object):'
1153 return [x.repo_id for x in
1162 return [x.repo_id for x in
1154 RepoList(qry, perm_set=perm_def)]
1163 RepoList(qry, perm_set=perm_def)]
1155
1164
1156 compute = caches.conditional_cache(
1165 return _cached_repo_acl(self.user_id, perms, name_filter)
1157 'short_term', 'repo_acl_ids.{}'.format(self.user_id),
1158 condition=cache, func=_cached_repo_acl)
1159 return compute(self.user_id, perms, name_filter)
1160
1166
1161 def repo_group_acl_ids(self, perms=None, name_filter=None, cache=False):
1167 def repo_group_acl_ids(self, perms=None, name_filter=None, cache=False):
1162 """
1168 """
@@ -1179,10 +1185,7 b' class AuthUser(object):'
1179 return [x.group_id for x in
1185 return [x.group_id for x in
1180 RepoGroupList(qry, perm_set=perm_def)]
1186 RepoGroupList(qry, perm_set=perm_def)]
1181
1187
1182 compute = caches.conditional_cache(
1188 return _cached_repo_group_acl(self.user_id, perms, name_filter)
1183 'short_term', 'repo_group_acl_ids.{}'.format(self.user_id),
1184 condition=cache, func=_cached_repo_group_acl)
1185 return compute(self.user_id, perms, name_filter)
1186
1189
1187 def user_group_acl_ids(self, perms=None, name_filter=None, cache=False):
1190 def user_group_acl_ids(self, perms=None, name_filter=None, cache=False):
1188 """
1191 """
@@ -1205,10 +1208,7 b' class AuthUser(object):'
1205 return [x.users_group_id for x in
1208 return [x.users_group_id for x in
1206 UserGroupList(qry, perm_set=perm_def)]
1209 UserGroupList(qry, perm_set=perm_def)]
1207
1210
1208 compute = caches.conditional_cache(
1211 return _cached_user_group_acl(self.user_id, perms, name_filter)
1209 'short_term', 'user_group_acl_ids.{}'.format(self.user_id),
1210 condition=cache, func=_cached_user_group_acl)
1211 return compute(self.user_id, perms, name_filter)
1212
1212
1213 @property
1213 @property
1214 def ip_allowed(self):
1214 def ip_allowed(self):
@@ -505,6 +505,7 b' def bootstrap_config(request):'
505 config.include('pyramid_mako')
505 config.include('pyramid_mako')
506 config.include('pyramid_beaker')
506 config.include('pyramid_beaker')
507 config.include('rhodecode.lib.caches')
507 config.include('rhodecode.lib.caches')
508 config.include('rhodecode.lib.rc_cache')
508
509
509 add_events_routes(config)
510 add_events_routes(config)
510
511
@@ -96,7 +96,7 b' def get_cache_manager(region_name, cache'
96 Creates a Beaker cache manager. Such instance can be used like that::
96 Creates a Beaker cache manager. Such instance can be used like that::
97
97
98 _namespace = caches.get_repo_namespace_key(caches.XXX, repo_name)
98 _namespace = caches.get_repo_namespace_key(caches.XXX, repo_name)
99 cache_manager = caches.get_cache_manager('repo_cache_long', _namespace)
99 cache_manager = caches.get_cache_manager('some_namespace_name', _namespace)
100 _cache_key = caches.compute_key_from_params(repo_name, commit.raw_id)
100 _cache_key = caches.compute_key_from_params(repo_name, commit.raw_id)
101 def heavy_compute():
101 def heavy_compute():
102 ...
102 ...
@@ -121,7 +121,7 b' def get_cache_manager(region_name, cache'
121 def clear_cache_manager(cache_manager):
121 def clear_cache_manager(cache_manager):
122 """
122 """
123 namespace = 'foobar'
123 namespace = 'foobar'
124 cache_manager = get_cache_manager('repo_cache_long', namespace)
124 cache_manager = get_cache_manager('some_namespace_name', namespace)
125 clear_cache_manager(cache_manager)
125 clear_cache_manager(cache_manager)
126 """
126 """
127
127
@@ -39,9 +39,8 b' from pyramid.httpexceptions import ('
39 from zope.cachedescriptors.property import Lazy as LazyProperty
39 from zope.cachedescriptors.property import Lazy as LazyProperty
40
40
41 import rhodecode
41 import rhodecode
42 from rhodecode.authentication.base import (
42 from rhodecode.authentication.base import authenticate, VCS_TYPE, loadplugin
43 authenticate, get_perms_cache_manager, VCS_TYPE, loadplugin)
43 from rhodecode.lib import caches, rc_cache
44 from rhodecode.lib import caches
45 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
44 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
46 from rhodecode.lib.base import (
45 from rhodecode.lib.base import (
47 BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
46 BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
@@ -311,36 +310,24 b' class SimpleVCS(object):'
311 :param repo_name: repository name
310 :param repo_name: repository name
312 """
311 """
313
312
314 # get instance of cache manager configured for a namespace
315 cache_manager = get_perms_cache_manager(
316 custom_ttl=cache_ttl, suffix=user.user_id)
317 log.debug('AUTH_CACHE_TTL for permissions `%s` active: %s (TTL: %s)',
313 log.debug('AUTH_CACHE_TTL for permissions `%s` active: %s (TTL: %s)',
318 plugin_id, plugin_cache_active, cache_ttl)
314 plugin_id, plugin_cache_active, cache_ttl)
319
315
320 # for environ based password can be empty, but then the validation is
316 user_id = user.user_id
321 # on the server that fills in the env data needed for authentication
317 cache_namespace_uid = 'cache_user_auth.{}'.format(user_id)
322 _perm_calc_hash = caches.compute_key_from_params(
318 region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
323 plugin_id, action, user.user_id, repo_name, ip_addr)
324
319
325 # _authenticate is a wrapper for .auth() method of plugin.
320 @region.cache_on_arguments(namespace=cache_namespace_uid,
326 # it checks if .auth() sends proper data.
321 expiration_time=cache_ttl,
327 # For RhodeCodeExternalAuthPlugin it also maps users to
322 should_cache_fn=lambda v: plugin_cache_active)
328 # Database and maps the attributes returned from .auth()
323 def compute_perm_vcs(
329 # to RhodeCode database. If this function returns data
324 cache_name, plugin_id, action, user_id, repo_name, ip_addr):
330 # then auth is correct.
331 start = time.time()
332 log.debug('Running plugin `%s` permissions check', plugin_id)
333
325
334 def perm_func():
335 """
336 This function is used internally in Cache of Beaker to calculate
337 Results
338 """
339 log.debug('auth: calculating permission access now...')
326 log.debug('auth: calculating permission access now...')
340 # check IP
327 # check IP
341 inherit = user.inherit_default_permissions
328 inherit = user.inherit_default_permissions
342 ip_allowed = AuthUser.check_ip_allowed(
329 ip_allowed = AuthUser.check_ip_allowed(
343 user.user_id, ip_addr, inherit_from_default=inherit)
330 user_id, ip_addr, inherit_from_default=inherit)
344 if ip_allowed:
331 if ip_allowed:
345 log.info('Access for IP:%s allowed', ip_addr)
332 log.info('Access for IP:%s allowed', ip_addr)
346 else:
333 else:
@@ -360,12 +347,13 b' class SimpleVCS(object):'
360
347
361 return True
348 return True
362
349
363 if plugin_cache_active:
350 start = time.time()
364 log.debug('Trying to fetch cached perms by %s', _perm_calc_hash[:6])
351 log.debug('Running plugin `%s` permissions check', plugin_id)
365 perm_result = cache_manager.get(
352
366 _perm_calc_hash, createfunc=perm_func)
353 # for environ based auth, password can be empty, but then the validation is
367 else:
354 # on the server that fills in the env data needed for authentication
368 perm_result = perm_func()
355 perm_result = compute_perm_vcs(
356 'vcs_permissions', plugin_id, action, user.user_id, repo_name, ip_addr)
369
357
370 auth_time = time.time() - start
358 auth_time = time.time() - start
371 log.debug('Permissions for plugin `%s` completed in %.3fs, '
359 log.debug('Permissions for plugin `%s` completed in %.3fs, '
@@ -119,6 +119,8 b' function registerRCRoutes() {'
119 pyroutes.register('edit_user_groups_management', '/_admin/users/%(user_id)s/edit/groups_management', ['user_id']);
119 pyroutes.register('edit_user_groups_management', '/_admin/users/%(user_id)s/edit/groups_management', ['user_id']);
120 pyroutes.register('edit_user_groups_management_updates', '/_admin/users/%(user_id)s/edit/edit_user_groups_management/updates', ['user_id']);
120 pyroutes.register('edit_user_groups_management_updates', '/_admin/users/%(user_id)s/edit/edit_user_groups_management/updates', ['user_id']);
121 pyroutes.register('edit_user_audit_logs', '/_admin/users/%(user_id)s/edit/audit', ['user_id']);
121 pyroutes.register('edit_user_audit_logs', '/_admin/users/%(user_id)s/edit/audit', ['user_id']);
122 pyroutes.register('edit_user_caches', '/_admin/users/%(user_id)s/edit/caches', ['user_id']);
123 pyroutes.register('edit_user_caches_update', '/_admin/users/%(user_id)s/edit/caches/update', ['user_id']);
122 pyroutes.register('user_groups', '/_admin/user_groups', []);
124 pyroutes.register('user_groups', '/_admin/user_groups', []);
123 pyroutes.register('user_groups_data', '/_admin/user_groups_data', []);
125 pyroutes.register('user_groups_data', '/_admin/user_groups_data', []);
124 pyroutes.register('user_groups_new', '/_admin/user_groups/new', []);
126 pyroutes.register('user_groups_new', '/_admin/user_groups/new', []);
@@ -45,6 +45,7 b''
45 <li class="${'active' if c.active=='ips' else ''}"><a href="${h.route_path('edit_user_ips', user_id=c.user.user_id)}">${_('Ip Whitelist')}</a></li>
45 <li class="${'active' if c.active=='ips' else ''}"><a href="${h.route_path('edit_user_ips', user_id=c.user.user_id)}">${_('Ip Whitelist')}</a></li>
46 <li class="${'active' if c.active=='groups' else ''}"><a href="${h.route_path('edit_user_groups_management', user_id=c.user.user_id)}">${_('User Groups Management')}</a></li>
46 <li class="${'active' if c.active=='groups' else ''}"><a href="${h.route_path('edit_user_groups_management', user_id=c.user.user_id)}">${_('User Groups Management')}</a></li>
47 <li class="${'active' if c.active=='audit' else ''}"><a href="${h.route_path('edit_user_audit_logs', user_id=c.user.user_id)}">${_('Audit logs')}</a></li>
47 <li class="${'active' if c.active=='audit' else ''}"><a href="${h.route_path('edit_user_audit_logs', user_id=c.user.user_id)}">${_('Audit logs')}</a></li>
48 <li class="${'active' if c.active=='caches' else ''}"><a href="${h.route_path('edit_user_caches', user_id=c.user.user_id)}">${_('Caches')}</a></li>
48 </ul>
49 </ul>
49 </div>
50 </div>
50
51
@@ -22,36 +22,24 b' import time'
22
22
23 import pytest
23 import pytest
24
24
25 from rhodecode.lib import caches
25 from rhodecode.lib import rc_cache
26 from rhodecode.lib.memory_lru_debug import MemoryLRUNamespaceManagerBase
27
26
28
27
29 class TestCacheManager(object):
28 @pytest.mark.usefixtures( 'app')
29 class TestCaches(object):
30
30
31 @pytest.mark.parametrize('repo_name', [
31 def test_cache_decorator_init_not_configured(self):
32 ('',),
32 with pytest.raises(EnvironmentError):
33 (u'',),
33 rc_cache.get_or_create_region('dontexist')
34 (u'ac',),
34
35 ('ac', ),
35 @pytest.mark.parametrize('region_name', [
36 (u'Δ™Δ‡c',),
36 'cache_perms', u'cache_perms',
37 ('Δ…ac',),
38 ])
37 ])
39 def test_cache_manager_init(self, repo_name):
38 def test_cache_decorator_init(self, region_name):
40 cache_manager_instance = caches.get_cache_manager(
39 namespace = region_name
41 repo_name, 'my_cache')
40 cache_region = rc_cache.get_or_create_region(
42 assert cache_manager_instance
41 region_name, region_namespace=namespace)
43
42 assert cache_region
44 def test_cache_manager_init_undefined_namespace(self):
45 cache_manager_instance = caches.get_cache_manager(
46 'repo_cache_long_undefined', 'my_cache')
47 assert cache_manager_instance
48
49 def_config = caches.DEFAULT_CACHE_MANAGER_CONFIG.copy()
50 def_config.pop('type')
51 assert cache_manager_instance.nsargs == def_config
52
53 assert isinstance(
54 cache_manager_instance.namespace, MemoryLRUNamespaceManagerBase)
55
43
56 @pytest.mark.parametrize('example_input', [
44 @pytest.mark.parametrize('example_input', [
57 ('',),
45 ('',),
@@ -62,45 +50,60 b' class TestCacheManager(object):'
62 (u'/ac', ),
50 (u'/ac', ),
63 ])
51 ])
64 def test_cache_manager_create_key(self, example_input):
52 def test_cache_manager_create_key(self, example_input):
65 key = caches.compute_key_from_params(*example_input)
53 key = rc_cache.utils.compute_key_from_params(*example_input)
66 assert key
54 assert key
67
55
68 def test_store_and_invalidate_value_from_manager(self):
56 @pytest.mark.parametrize('example_namespace', [
69 cache_manger_instance = caches.get_cache_manager(
57 'namespace', None
70 'repo_cache_long', 'my_cache_store')
58 ])
59 @pytest.mark.parametrize('example_input', [
60 ('',),
61 (u'/ac',),
62 (u'/ac', 1, 2, object()),
63 (u'/Δ™Δ‡c', 1, 2, object()),
64 ('/Δ…ac',),
65 (u'/ac', ),
66 ])
67 def test_cache_keygen(self, example_input, example_namespace):
68 def func_wrapped():
69 return 1
70 func = rc_cache.utils.key_generator(example_namespace, func_wrapped)
71 key = func(*example_input)
72 assert key
71
73
72 def compute():
74 def test_store_value_in_cache(self):
75 cache_region = rc_cache.get_or_create_region('cache_perms')
76 # make sure we empty the cache now
77 for key in cache_region.backend.list_keys():
78 cache_region.delete(key)
79
80 assert cache_region.backend.list_keys() == []
81
82 @cache_region.cache_on_arguments(expiration_time=5)
83 def compute(key):
73 return time.time()
84 return time.time()
74
85
75 added_keys = []
86 for x in range(10):
76 for i in xrange(3):
87 compute(x)
77 _cache_key = caches.compute_key_from_params('foo_bar', 'p%s' % i)
88
78 added_keys.append(_cache_key)
89 assert len(set(cache_region.backend.list_keys())) == 10
79 for x in xrange(10):
80 cache_manger_instance.get(_cache_key, createfunc=compute)
81
90
82 for key in added_keys:
91 def test_store_and_get_value_from_region(self):
83 assert cache_manger_instance[key]
92 cache_region = rc_cache.get_or_create_region('cache_perms')
84
93 # make sure we empty the cache now
85 caches.clear_cache_manager(cache_manger_instance)
94 for key in cache_region.backend.list_keys():
86
95 cache_region.delete(key)
87 for key in added_keys:
96 assert cache_region.backend.list_keys() == []
88 assert key not in cache_manger_instance
89 assert len(cache_manger_instance.namespace.keys()) == 0
90
97
91 def test_store_and_get_value_from_manager(self):
98 @cache_region.cache_on_arguments(expiration_time=5)
92 cache_manger_instance = caches.get_cache_manager(
99 def compute(key):
93 'repo_cache_long', 'my_cache_store')
100 return time.time()
94
95 _cache_key = caches.compute_key_from_params('foo_bar', 'multicall')
96
101
97 def compute():
102 result = set()
98 return time.time()
103 for x in range(10):
104 ret = compute('x')
105 result.add(ret)
99
106
100 result = set()
107 # once computed we have only one value (the same from cache)
101 for x in xrange(10):
108 # after executing it 10x
102 ret = cache_manger_instance.get(_cache_key, createfunc=compute)
109 assert len(result) == 1
103 result.add(ret)
104
105 # once computed we have only one value after executing it 10x
106 assert len(result) == 1
@@ -292,11 +292,7 b' cache_dir = %(here)s/data'
292 beaker.cache.data_dir = %(here)s/rc/data/cache/beaker_data
292 beaker.cache.data_dir = %(here)s/rc/data/cache/beaker_data
293 beaker.cache.lock_dir = %(here)s/rc/data/cache/beaker_lock
293 beaker.cache.lock_dir = %(here)s/rc/data/cache/beaker_lock
294
294
295 beaker.cache.regions = short_term, long_term, sql_cache_short, auth_plugins, repo_cache_long
295 beaker.cache.regions = long_term, sql_cache_short, repo_cache_long
296
297 beaker.cache.short_term.type = file
298 beaker.cache.short_term.expire = 0
299 beaker.cache.short_term.key_length = 256
300
296
301 beaker.cache.long_term.type = memory
297 beaker.cache.long_term.type = memory
302 beaker.cache.long_term.expire = 36000
298 beaker.cache.long_term.expire = 36000
@@ -306,16 +302,6 b' beaker.cache.sql_cache_short.type = memo'
306 beaker.cache.sql_cache_short.expire = 1
302 beaker.cache.sql_cache_short.expire = 1
307 beaker.cache.sql_cache_short.key_length = 256
303 beaker.cache.sql_cache_short.key_length = 256
308
304
309 ## default is memory cache, configure only if required
310 ## using multi-node or multi-worker setup
311 #beaker.cache.auth_plugins.type = memory
312 #beaker.cache.auth_plugins.lock_dir = %(here)s/data/cache/auth_plugin_lock
313 #beaker.cache.auth_plugins.url = postgresql://postgres:secret@localhost/rhodecode
314 #beaker.cache.auth_plugins.url = mysql://root:secret@127.0.0.1/rhodecode
315 #beaker.cache.auth_plugins.sa.pool_recycle = 3600
316 #beaker.cache.auth_plugins.sa.pool_size = 10
317 #beaker.cache.auth_plugins.sa.max_overflow = 0
318
319 beaker.cache.repo_cache_long.type = memorylru_base
305 beaker.cache.repo_cache_long.type = memorylru_base
320 beaker.cache.repo_cache_long.max_items = 4096
306 beaker.cache.repo_cache_long.max_items = 4096
321 beaker.cache.repo_cache_long.expire = 2592000
307 beaker.cache.repo_cache_long.expire = 2592000
@@ -327,6 +313,16 b' beaker.cache.repo_cache_long.expire = 25'
327 #beaker.cache.repo_cache_long.expire = 1209600
313 #beaker.cache.repo_cache_long.expire = 1209600
328 #beaker.cache.repo_cache_long.key_length = 256
314 #beaker.cache.repo_cache_long.key_length = 256
329
315
316
317 #####################################
318 ### DOGPILE CACHE ####
319 #####################################
320
321 ## permission tree cache settings
322 rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace
323 rc_cache.cache_perms.expiration_time = 0
324 rc_cache.cache_perms.arguments.filename = /tmp/rc_cache_1
325
330 ####################################
326 ####################################
331 ### BEAKER SESSION ####
327 ### BEAKER SESSION ####
332 ####################################
328 ####################################
General Comments 0
You need to be logged in to leave comments. Login now