##// END OF EJS Templates
caches: rewrite of auth/permission caches to dogpile.
marcink -
r2845:3d5ae486 default
parent child Browse files
Show More
@@ -0,0 +1,66 b''
1 # -*- coding: utf-8 -*-
2
3 # Copyright (C) 2015-2018 RhodeCode GmbH
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
21 from dogpile.cache import register_backend
22 from dogpile.cache import make_region
23
24 register_backend(
25 "dogpile.cache.rc.memory_lru", "rhodecode.lib.rc_cache.backends",
26 "LRUMemoryBackend")
27
28 register_backend(
29 "dogpile.cache.rc.file_namespace", "rhodecode.lib.rc_cache.backends",
30 "FileNamespaceBackend")
31
32 register_backend(
33 "dogpile.cache.rc.redis", "rhodecode.lib.rc_cache.backends",
34 "RedisPickleBackend")
35
36
37 from . import region_meta
38 from .utils import get_default_cache_settings, key_generator, get_or_create_region
39
40
41 def configure_dogpile_cache(settings):
42 cache_dir = settings.get('cache_dir')
43 if cache_dir:
44 region_meta.dogpile_config_defaults['cache_dir'] = cache_dir
45
46 rc_cache_data = get_default_cache_settings(settings, prefixes=['rc_cache.'])
47
48 # inspect available namespaces
49 avail_regions = set()
50 for key in rc_cache_data.keys():
51 namespace_name = key.split('.', 1)[0]
52 avail_regions.add(namespace_name)
53
54 # register them into namespace
55 for region_name in avail_regions:
56 new_region = make_region(
57 name=region_name,
58 function_key_generator=key_generator
59 )
60
61 new_region.configure_from_config(settings, 'rc_cache.{}.'.format(region_name))
62 region_meta.dogpile_cache_regions[region_name] = new_region
63
64
65 def includeme(config):
66 configure_dogpile_cache(config.registry.settings)
@@ -0,0 +1,109 b''
1 # -*- coding: utf-8 -*-
2
3 # Copyright (C) 2015-2018 RhodeCode GmbH
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
21 from dogpile.cache.backends import memory as memory_backend
22 from dogpile.cache.backends import file as file_backend
23 from dogpile.cache.backends import redis as redis_backend
24 from dogpile.cache.backends.file import NO_VALUE, compat
25
26 from rhodecode.lib.memory_lru_debug import LRUDict
27
28 _default_max_size = 1024
29
30
31 class LRUMemoryBackend(memory_backend.MemoryBackend):
32
33 def __init__(self, arguments):
34 max_size = arguments.pop('max_size', _default_max_size)
35 arguments['cache_dict'] = LRUDict(max_size)
36 super(LRUMemoryBackend, self).__init__(arguments)
37
38
39 class Serializer(object):
40 def _dumps(self, value):
41 return compat.pickle.dumps(value)
42
43 def _loads(self, value):
44 return compat.pickle.loads(value)
45
46
47 class FileNamespaceBackend(Serializer, file_backend.DBMBackend):
48
49 def __init__(self, arguments):
50 super(FileNamespaceBackend, self).__init__(arguments)
51
52 def list_keys(self):
53 with self._dbm_file(True) as dbm:
54 return dbm.keys()
55
56 def get_store(self):
57 return self.filename
58
59 def get(self, key):
60 with self._dbm_file(False) as dbm:
61 if hasattr(dbm, 'get'):
62 value = dbm.get(key, NO_VALUE)
63 else:
64 # gdbm objects lack a .get method
65 try:
66 value = dbm[key]
67 except KeyError:
68 value = NO_VALUE
69 if value is not NO_VALUE:
70 value = self._loads(value)
71 return value
72
73 def set(self, key, value):
74 with self._dbm_file(True) as dbm:
75 dbm[key] = self._dumps(value)
76
77 def set_multi(self, mapping):
78 with self._dbm_file(True) as dbm:
79 for key, value in mapping.items():
80 dbm[key] = self._dumps(value)
81
82
83 class RedisPickleBackend(Serializer, redis_backend.RedisBackend):
84 def list_keys(self):
85 return self.client.keys()
86
87 def get_store(self):
88 return self.client.connection_pool
89
90 def set(self, key, value):
91 if self.redis_expiration_time:
92 self.client.setex(key, self.redis_expiration_time,
93 self._dumps(value))
94 else:
95 self.client.set(key, self._dumps(value))
96
97 def set_multi(self, mapping):
98 mapping = dict(
99 (k, self._dumps(v))
100 for k, v in mapping.items()
101 )
102
103 if not self.redis_expiration_time:
104 self.client.mset(mapping)
105 else:
106 pipe = self.client.pipeline()
107 for key, value in mapping.items():
108 pipe.setex(key, self.redis_expiration_time, value)
109 pipe.execute()
@@ -0,0 +1,28 b''
1 # -*- coding: utf-8 -*-
2
3 # Copyright (C) 2015-2018 RhodeCode GmbH
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 import os
21 import tempfile
22
23 dogpile_config_defaults = {
24 'cache_dir': os.path.join(tempfile.gettempdir(), 'rc_cache')
25 }
26
27 # GLOBAL TO STORE ALL REGISTERED REGIONS
28 dogpile_cache_regions = {}
@@ -0,0 +1,99 b''
1 # -*- coding: utf-8 -*-
2
3 # Copyright (C) 2015-2018 RhodeCode GmbH
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 import os
21 import logging
22 from dogpile.cache import make_region
23
24 from rhodecode.lib.utils import safe_str, sha1
25 from . import region_meta
26
27 log = logging.getLogger(__name__)
28
29
30 def get_default_cache_settings(settings, prefixes=None):
31 prefixes = prefixes or []
32 cache_settings = {}
33 for key in settings.keys():
34 for prefix in prefixes:
35 if key.startswith(prefix):
36 name = key.split(prefix)[1].strip()
37 val = settings[key]
38 if isinstance(val, basestring):
39 val = val.strip()
40 cache_settings[name] = val
41 return cache_settings
42
43
44 def compute_key_from_params(*args):
45 """
46 Helper to compute key from given params to be used in cache manager
47 """
48 return sha1("_".join(map(safe_str, args)))
49
50
51 def key_generator(namespace, fn):
52 fname = fn.__name__
53
54 def generate_key(*args):
55 namespace_pref = namespace or 'default'
56 arg_key = compute_key_from_params(*args)
57 final_key = "{}:{}_{}".format(namespace_pref, fname, arg_key)
58
59 return final_key
60
61 return generate_key
62
63
64 def get_or_create_region(region_name, region_namespace=None):
65 from rhodecode.lib.rc_cache.backends import FileNamespaceBackend
66 region_obj = region_meta.dogpile_cache_regions.get(region_name)
67 if not region_obj:
68 raise EnvironmentError(
69 'Region `{}` not in configured: {}.'.format(
70 region_name, region_meta.dogpile_cache_regions.keys()))
71
72 region_uid_name = '{}:{}'.format(region_name, region_namespace)
73 if isinstance(region_obj.actual_backend, FileNamespaceBackend):
74 region_exist = region_meta.dogpile_cache_regions.get(region_namespace)
75 if region_exist:
76 log.debug('Using already configured region: %s', region_namespace)
77 return region_exist
78 cache_dir = region_meta.dogpile_config_defaults['cache_dir']
79 expiration_time = region_obj.expiration_time
80
81 if not os.path.isdir(cache_dir):
82 os.makedirs(cache_dir)
83 new_region = make_region(
84 name=region_uid_name, function_key_generator=key_generator
85 )
86 namespace_filename = os.path.join(
87 cache_dir, "{}.cache.dbm".format(region_namespace))
88 # special type that allows 1db per namespace
89 new_region.configure(
90 backend='dogpile.cache.rc.file_namespace',
91 expiration_time=expiration_time,
92 arguments={"filename": namespace_filename}
93 )
94
95 # create and save in region caches
96 log.debug('configuring new region: %s',region_uid_name)
97 region_obj = region_meta.dogpile_cache_regions[region_namespace] = new_region
98
99 return region_obj
@@ -0,0 +1,29 b''
1 <%namespace name="base" file="/base/base.mako"/>
2
3 <div class="panel panel-default">
4 <div class="panel-heading">
5 <h3 class="panel-title">${_('Caches')}</h3>
6 </div>
7 <div class="panel-body">
8 <pre>
9 region: ${c.region.name}
10 backend: ${c.region.actual_backend.__class__}
11 store: ${c.region.actual_backend.get_store()}
12
13 % for k in c.user_keys:
14 - ${k}
15 % endfor
16 </pre>
17
18 ${h.secure_form(h.route_path('edit_user_caches_update', user_id=c.user.user_id), request=request)}
19 <div class="form">
20 <div class="fields">
21 ${h.submit('reset_cache_%s' % c.user.user_id, _('Invalidate user cache'),class_="btn btn-small",onclick="return confirm('"+_('Confirm to invalidate user cache')+"');")}
22 </div>
23 </div>
24 ${h.end_form()}
25
26 </div>
27 </div>
28
29
@@ -320,12 +320,7 b' cache_dir = %(here)s/data'
320 320 beaker.cache.data_dir = %(here)s/data/cache/beaker_data
321 321 beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock
322 322
323 beaker.cache.regions = short_term, long_term, sql_cache_short, auth_plugins, repo_cache_long
324
325 # used for caching user permissions
326 beaker.cache.short_term.type = file
327 beaker.cache.short_term.expire = 0
328 beaker.cache.short_term.key_length = 256
323 beaker.cache.regions = long_term, sql_cache_short, repo_cache_long
329 324
330 325 beaker.cache.long_term.type = memory
331 326 beaker.cache.long_term.expire = 36000
@@ -335,16 +330,6 b' beaker.cache.sql_cache_short.type = memo'
335 330 beaker.cache.sql_cache_short.expire = 10
336 331 beaker.cache.sql_cache_short.key_length = 256
337 332
338 ## default is memory cache, configure only if required
339 ## using multi-node or multi-worker setup
340 #beaker.cache.auth_plugins.type = ext:database
341 #beaker.cache.auth_plugins.lock_dir = %(here)s/data/cache/auth_plugin_lock
342 #beaker.cache.auth_plugins.url = postgresql://postgres:secret@localhost/rhodecode
343 #beaker.cache.auth_plugins.url = mysql://root:secret@127.0.0.1/rhodecode
344 #beaker.cache.auth_plugins.sa.pool_recycle = 3600
345 #beaker.cache.auth_plugins.sa.pool_size = 10
346 #beaker.cache.auth_plugins.sa.max_overflow = 0
347
348 333 beaker.cache.repo_cache_long.type = memorylru_base
349 334 beaker.cache.repo_cache_long.max_items = 4096
350 335 beaker.cache.repo_cache_long.expire = 2592000
@@ -295,12 +295,7 b' cache_dir = %(here)s/data'
295 295 beaker.cache.data_dir = %(here)s/data/cache/beaker_data
296 296 beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock
297 297
298 beaker.cache.regions = short_term, long_term, sql_cache_short, auth_plugins, repo_cache_long
299
300 # used for caching user permissions
301 beaker.cache.short_term.type = file
302 beaker.cache.short_term.expire = 0
303 beaker.cache.short_term.key_length = 256
298 beaker.cache.regions = long_term, sql_cache_short, repo_cache_long
304 299
305 300 beaker.cache.long_term.type = memory
306 301 beaker.cache.long_term.expire = 36000
@@ -310,16 +305,6 b' beaker.cache.sql_cache_short.type = memo'
310 305 beaker.cache.sql_cache_short.expire = 10
311 306 beaker.cache.sql_cache_short.key_length = 256
312 307
313 ## default is memory cache, configure only if required
314 ## using multi-node or multi-worker setup
315 #beaker.cache.auth_plugins.type = ext:database
316 #beaker.cache.auth_plugins.lock_dir = %(here)s/data/cache/auth_plugin_lock
317 #beaker.cache.auth_plugins.url = postgresql://postgres:secret@localhost/rhodecode
318 #beaker.cache.auth_plugins.url = mysql://root:secret@127.0.0.1/rhodecode
319 #beaker.cache.auth_plugins.sa.pool_recycle = 3600
320 #beaker.cache.auth_plugins.sa.pool_size = 10
321 #beaker.cache.auth_plugins.sa.max_overflow = 0
322
323 308 beaker.cache.repo_cache_long.type = memorylru_base
324 309 beaker.cache.repo_cache_long.max_items = 4096
325 310 beaker.cache.repo_cache_long.expire = 2592000
@@ -356,6 +356,16 b' def admin_routes(config):'
356 356 name='edit_user_audit_logs',
357 357 pattern='/users/{user_id:\d+}/edit/audit', user_route=True)
358 358
359 # user caches
360 config.add_route(
361 name='edit_user_caches',
362 pattern='/users/{user_id:\d+}/edit/caches',
363 user_route=True)
364 config.add_route(
365 name='edit_user_caches_update',
366 pattern='/users/{user_id:\d+}/edit/caches/update',
367 user_route=True)
368
359 369 # user-groups admin
360 370 config.add_route(
361 371 name='user_groups',
@@ -34,7 +34,7 b' from rhodecode.authentication.plugins im'
34 34 from rhodecode.events import trigger
35 35 from rhodecode.model.db import true
36 36
37 from rhodecode.lib import audit_logger
37 from rhodecode.lib import audit_logger, rc_cache
38 38 from rhodecode.lib.exceptions import (
39 39 UserCreationError, UserOwnsReposException, UserOwnsRepoGroupsException,
40 40 UserOwnsUserGroupsException, DefaultUserException)
@@ -1198,3 +1198,57 b' class UsersView(UserAppView):'
1198 1198 perm_user = self.db_user.AuthUser(ip_addr=self.request.remote_addr)
1199 1199
1200 1200 return perm_user.permissions
1201
1202 def _get_user_cache_keys(self, cache_namespace_uid, keys):
1203 user_keys = []
1204 for k in sorted(keys):
1205 if k.startswith(cache_namespace_uid):
1206 user_keys.append(k)
1207 return user_keys
1208
1209 @LoginRequired()
1210 @HasPermissionAllDecorator('hg.admin')
1211 @view_config(
1212 route_name='edit_user_caches', request_method='GET',
1213 renderer='rhodecode:templates/admin/users/user_edit.mako')
1214 def user_caches(self):
1215 _ = self.request.translate
1216 c = self.load_default_context()
1217 c.user = self.db_user
1218
1219 c.active = 'caches'
1220 c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
1221
1222 cache_namespace_uid = 'cache_user_auth.{}'.format(self.db_user.user_id)
1223 c.region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
1224 c.backend = c.region.backend
1225 c.user_keys = self._get_user_cache_keys(
1226 cache_namespace_uid, c.region.backend.list_keys())
1227
1228 return self._get_template_context(c)
1229
1230 @LoginRequired()
1231 @HasPermissionAllDecorator('hg.admin')
1232 @CSRFRequired()
1233 @view_config(
1234 route_name='edit_user_caches_update', request_method='POST')
1235 def user_caches_update(self):
1236 _ = self.request.translate
1237 c = self.load_default_context()
1238 c.user = self.db_user
1239
1240 c.active = 'caches'
1241 c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
1242
1243 cache_namespace_uid = 'cache_user_auth.{}'.format(self.db_user.user_id)
1244 c.region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
1245
1246 c.user_keys = self._get_user_cache_keys(
1247 cache_namespace_uid, c.region.backend.list_keys())
1248 for k in c.user_keys:
1249 c.region.delete(k)
1250
1251 h.flash(_("Deleted {} cache keys").format(len(c.user_keys)), category='success')
1252
1253 return HTTPFound(h.route_path(
1254 'edit_user_caches', user_id=c.user.user_id))
@@ -35,7 +35,7 b' from pyramid.threadlocal import get_curr'
35 35
36 36 from rhodecode.authentication.interface import IAuthnPluginRegistry
37 37 from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase
38 from rhodecode.lib import caches
38 from rhodecode.lib import caches, rc_cache
39 39 from rhodecode.lib.auth import PasswordGenerator, _RhodeCodeCryptoBCrypt
40 40 from rhodecode.lib.utils2 import safe_int, safe_str
41 41 from rhodecode.lib.exceptions import LdapConnectionError
@@ -633,22 +633,6 b' def get_authn_registry(registry=None):'
633 633 return authn_registry
634 634
635 635
636 def get_auth_cache_manager(custom_ttl=None, suffix=None):
637 cache_name = 'rhodecode.authentication'
638 if suffix:
639 cache_name = 'rhodecode.authentication.{}'.format(suffix)
640 return caches.get_cache_manager(
641 'auth_plugins', cache_name, custom_ttl)
642
643
644 def get_perms_cache_manager(custom_ttl=None, suffix=None):
645 cache_name = 'rhodecode.permissions'
646 if suffix:
647 cache_name = 'rhodecode.permissions.{}'.format(suffix)
648 return caches.get_cache_manager(
649 'auth_plugins', cache_name, custom_ttl)
650
651
652 636 def authenticate(username, password, environ=None, auth_type=None,
653 637 skip_missing=False, registry=None, acl_repo_name=None):
654 638 """
@@ -707,18 +691,18 b' def authenticate(username, password, env'
707 691
708 692 plugin_cache_active, cache_ttl = plugin.get_ttl_cache(plugin_settings)
709 693
710 # get instance of cache manager configured for a namespace
711 cache_manager = get_auth_cache_manager(
712 custom_ttl=cache_ttl, suffix=user.user_id if user else '')
713
714 694 log.debug('AUTH_CACHE_TTL for plugin `%s` active: %s (TTL: %s)',
715 695 plugin.get_id(), plugin_cache_active, cache_ttl)
716 696
717 # for environ based password can be empty, but then the validation is
718 # on the server that fills in the env data needed for authentication
697 user_id = user.user_id
698 cache_namespace_uid = 'cache_user_auth.{}'.format(user_id)
699 region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
719 700
720 _password_hash = caches.compute_key_from_params(
721 plugin.name, username, (password or ''))
701 @region.cache_on_arguments(namespace=cache_namespace_uid,
702 expiration_time=cache_ttl,
703 should_cache_fn=lambda v: plugin_cache_active)
704 def compute_auth(
705 cache_name, plugin_name, username, password):
722 706
723 707 # _authenticate is a wrapper for .auth() method of plugin.
724 708 # it checks if .auth() sends proper data.
@@ -726,26 +710,16 b' def authenticate(username, password, env'
726 710 # Database and maps the attributes returned from .auth()
727 711 # to RhodeCode database. If this function returns data
728 712 # then auth is correct.
729 start = time.time()
730 log.debug('Running plugin `%s` _authenticate method', plugin.get_id())
731
732 def auth_func():
733 """
734 This function is used internally in Cache of Beaker to calculate
735 Results
736 """
737 log.debug('auth: calculating password access now...')
713 log.debug('Running plugin `%s` _authenticate method '
714 'using username and password', plugin.get_id())
738 715 return plugin._authenticate(
739 716 user, username, password, plugin_settings,
740 717 environ=environ or {})
741 718
742 if plugin_cache_active:
743 log.debug('Trying to fetch cached auth by pwd hash `...%s`',
744 _password_hash[:6])
745 plugin_user = cache_manager.get(
746 _password_hash, createfunc=auth_func)
747 else:
748 plugin_user = auth_func()
719 start = time.time()
720 # for environ based auth, password can be empty, but then the validation is
721 # on the server that fills in the env data needed for authentication
722 plugin_user = compute_auth('auth', plugin.name, username, (password or ''))
749 723
750 724 auth_time = time.time() - start
751 725 log.debug('Authentication for plugin `%s` completed in %.3fs, '
@@ -27,8 +27,7 b' from pyramid.renderers import render'
27 27 from pyramid.response import Response
28 28
29 29 from rhodecode.apps._base import BaseAppView
30 from rhodecode.authentication.base import (
31 get_auth_cache_manager, get_perms_cache_manager, get_authn_registry)
30 from rhodecode.authentication.base import get_authn_registry
32 31 from rhodecode.lib import helpers as h
33 32 from rhodecode.lib.auth import (
34 33 LoginRequired, HasPermissionAllDecorator, CSRFRequired)
@@ -102,16 +101,6 b' class AuthnPluginViewBase(BaseAppView):'
102 101 self.plugin.create_or_update_setting(name, value)
103 102 Session().commit()
104 103
105 # cleanup cache managers in case of change for plugin
106 # TODO(marcink): because we can register multiple namespaces
107 # we should at some point figure out how to retrieve ALL namespace
108 # cache managers and clear them...
109 cache_manager = get_auth_cache_manager()
110 clear_cache_manager(cache_manager)
111
112 cache_manager = get_perms_cache_manager()
113 clear_cache_manager(cache_manager)
114
115 104 # Display success message and redirect.
116 105 h.flash(_('Auth settings updated successfully.'), category='success')
117 106 redirect_to = self.request.resource_path(
@@ -22,6 +22,7 b' import os'
22 22 import logging
23 23 import traceback
24 24 import collections
25 import tempfile
25 26
26 27 from paste.gzipper import make_gzip_middleware
27 28 from pyramid.wsgi import wsgiapp
@@ -220,6 +221,7 b' def includeme(config):'
220 221 config.include('pyramid_mako')
221 222 config.include('pyramid_beaker')
222 223 config.include('rhodecode.lib.caches')
224 config.include('rhodecode.lib.rc_cache')
223 225
224 226 config.include('rhodecode.authentication')
225 227 config.include('rhodecode.integrations')
@@ -382,6 +384,7 b' def sanitize_settings_and_apply_defaults'
382 384 # Call split out functions that sanitize settings for each topic.
383 385 _sanitize_appenlight_settings(settings)
384 386 _sanitize_vcs_settings(settings)
387 _sanitize_cache_settings(settings)
385 388
386 389 # configure instance id
387 390 config_utils.set_instance_id(settings)
@@ -421,6 +424,18 b' def _sanitize_vcs_settings(settings):'
421 424 settings['vcs.scm_app_implementation'] = 'http'
422 425
423 426
427 def _sanitize_cache_settings(settings):
428 _string_setting(settings, 'cache_dir',
429 os.path.join(tempfile.gettempdir(), 'rc_cache'))
430
431 _string_setting(settings, 'rc_cache.cache_perms.backend',
432 'dogpile.cache.rc.file_namespace')
433 _int_setting(settings, 'rc_cache.cache_perms.expiration_time',
434 60)
435 _string_setting(settings, 'rc_cache.cache_perms.arguments.filename',
436 os.path.join(tempfile.gettempdir(), 'rc_cache_1'))
437
438
424 439 def _int_setting(settings, name, default):
425 440 settings[name] = int(settings.get(name, default))
426 441
@@ -48,7 +48,7 b' from rhodecode.model.user import UserMod'
48 48 from rhodecode.model.db import (
49 49 User, Repository, Permission, UserToPerm, UserGroupToPerm, UserGroupMember,
50 50 UserIpMap, UserApiKeys, RepoGroup, UserGroup)
51 from rhodecode.lib import caches
51 from rhodecode.lib import rc_cache
52 52 from rhodecode.lib.utils2 import safe_unicode, aslist, safe_str, md5, safe_int, sha1
53 53 from rhodecode.lib.utils import (
54 54 get_repo_slug, get_repo_group_slug, get_user_group_slug)
@@ -976,21 +976,18 b' class AuthUser(object):'
976 976 obj = Repository.get_by_repo_name(scope['repo_name'])
977 977 if obj:
978 978 scope['repo_id'] = obj.repo_id
979 _scope = {
980 'repo_id': -1,
981 'user_group_id': -1,
982 'repo_group_id': -1,
983 }
984 _scope.update(scope)
985 cache_key = "_".join(map(safe_str, reduce(lambda a, b: a+b,
986 _scope.items())))
987 if cache_key not in self._permissions_scoped_cache:
979 _scope = collections.OrderedDict()
980 _scope['repo_id'] = -1
981 _scope['user_group_id'] = -1
982 _scope['repo_group_id'] = -1
983
984 for k in sorted(scope.keys()):
985 _scope[k] = scope[k]
986
988 987 # store in cache to mimic how the @LazyProperty works,
989 988 # the difference here is that we use the unique key calculated
990 989 # from params and values
991 res = self.get_perms(user=self, cache=False, scope=_scope)
992 self._permissions_scoped_cache[cache_key] = res
993 return self._permissions_scoped_cache[cache_key]
990 return self.get_perms(user=self, cache=False, scope=_scope)
994 991
995 992 def get_instance(self):
996 993 return User.get(self.user_id)
@@ -1072,16 +1069,27 b' class AuthUser(object):'
1072 1069 user_inherit_default_permissions = user.inherit_default_permissions
1073 1070
1074 1071 cache_seconds = safe_int(
1075 rhodecode.CONFIG.get('beaker.cache.short_term.expire'))
1072 rhodecode.CONFIG.get('rc_cache.cache_perms.expiration_time'))
1073
1076 1074 cache_on = cache or cache_seconds > 0
1077 1075 log.debug(
1078 1076 'Computing PERMISSION tree for user %s scope `%s` '
1079 'with caching: %s[%ss]' % (user, scope, cache_on, cache_seconds))
1077 'with caching: %s[TTL: %ss]' % (user, scope, cache_on, cache_seconds or 0))
1078
1079 cache_namespace_uid = 'cache_user_auth.{}'.format(user_id)
1080 region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
1081
1082 @region.cache_on_arguments(namespace=cache_namespace_uid,
1083 should_cache_fn=lambda v: cache_on)
1084 def compute_perm_tree(cache_name,
1085 user_id, scope, user_is_admin,user_inherit_default_permissions,
1086 explicit, algo, calculate_super_admin):
1087 return _cached_perms_data(
1088 user_id, scope, user_is_admin, user_inherit_default_permissions,
1089 explicit, algo, calculate_super_admin)
1090
1080 1091 start = time.time()
1081 compute = caches.conditional_cache(
1082 'short_term', 'cache_desc.{}'.format(user_id),
1083 condition=cache_on, func=_cached_perms_data)
1084 result = compute(user_id, scope, user_is_admin,
1092 result = compute_perm_tree('permissions', user_id, scope, user_is_admin,
1085 1093 user_inherit_default_permissions, explicit, algo,
1086 1094 calculate_super_admin)
1087 1095
@@ -1091,6 +1099,7 b' class AuthUser(object):'
1091 1099 total = time.time() - start
1092 1100 log.debug('PERMISSION tree for user %s computed in %.3fs: %s' % (
1093 1101 user, total, result_repr))
1102
1094 1103 return result
1095 1104
1096 1105 @property
@@ -1153,10 +1162,7 b' class AuthUser(object):'
1153 1162 return [x.repo_id for x in
1154 1163 RepoList(qry, perm_set=perm_def)]
1155 1164
1156 compute = caches.conditional_cache(
1157 'short_term', 'repo_acl_ids.{}'.format(self.user_id),
1158 condition=cache, func=_cached_repo_acl)
1159 return compute(self.user_id, perms, name_filter)
1165 return _cached_repo_acl(self.user_id, perms, name_filter)
1160 1166
1161 1167 def repo_group_acl_ids(self, perms=None, name_filter=None, cache=False):
1162 1168 """
@@ -1179,10 +1185,7 b' class AuthUser(object):'
1179 1185 return [x.group_id for x in
1180 1186 RepoGroupList(qry, perm_set=perm_def)]
1181 1187
1182 compute = caches.conditional_cache(
1183 'short_term', 'repo_group_acl_ids.{}'.format(self.user_id),
1184 condition=cache, func=_cached_repo_group_acl)
1185 return compute(self.user_id, perms, name_filter)
1188 return _cached_repo_group_acl(self.user_id, perms, name_filter)
1186 1189
1187 1190 def user_group_acl_ids(self, perms=None, name_filter=None, cache=False):
1188 1191 """
@@ -1205,10 +1208,7 b' class AuthUser(object):'
1205 1208 return [x.users_group_id for x in
1206 1209 UserGroupList(qry, perm_set=perm_def)]
1207 1210
1208 compute = caches.conditional_cache(
1209 'short_term', 'user_group_acl_ids.{}'.format(self.user_id),
1210 condition=cache, func=_cached_user_group_acl)
1211 return compute(self.user_id, perms, name_filter)
1211 return _cached_user_group_acl(self.user_id, perms, name_filter)
1212 1212
1213 1213 @property
1214 1214 def ip_allowed(self):
@@ -505,6 +505,7 b' def bootstrap_config(request):'
505 505 config.include('pyramid_mako')
506 506 config.include('pyramid_beaker')
507 507 config.include('rhodecode.lib.caches')
508 config.include('rhodecode.lib.rc_cache')
508 509
509 510 add_events_routes(config)
510 511
@@ -96,7 +96,7 b' def get_cache_manager(region_name, cache'
96 96 Creates a Beaker cache manager. Such instance can be used like that::
97 97
98 98 _namespace = caches.get_repo_namespace_key(caches.XXX, repo_name)
99 cache_manager = caches.get_cache_manager('repo_cache_long', _namespace)
99 cache_manager = caches.get_cache_manager('some_namespace_name', _namespace)
100 100 _cache_key = caches.compute_key_from_params(repo_name, commit.raw_id)
101 101 def heavy_compute():
102 102 ...
@@ -121,7 +121,7 b' def get_cache_manager(region_name, cache'
121 121 def clear_cache_manager(cache_manager):
122 122 """
123 123 namespace = 'foobar'
124 cache_manager = get_cache_manager('repo_cache_long', namespace)
124 cache_manager = get_cache_manager('some_namespace_name', namespace)
125 125 clear_cache_manager(cache_manager)
126 126 """
127 127
@@ -39,9 +39,8 b' from pyramid.httpexceptions import ('
39 39 from zope.cachedescriptors.property import Lazy as LazyProperty
40 40
41 41 import rhodecode
42 from rhodecode.authentication.base import (
43 authenticate, get_perms_cache_manager, VCS_TYPE, loadplugin)
44 from rhodecode.lib import caches
42 from rhodecode.authentication.base import authenticate, VCS_TYPE, loadplugin
43 from rhodecode.lib import caches, rc_cache
45 44 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
46 45 from rhodecode.lib.base import (
47 46 BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
@@ -311,36 +310,24 b' class SimpleVCS(object):'
311 310 :param repo_name: repository name
312 311 """
313 312
314 # get instance of cache manager configured for a namespace
315 cache_manager = get_perms_cache_manager(
316 custom_ttl=cache_ttl, suffix=user.user_id)
317 313 log.debug('AUTH_CACHE_TTL for permissions `%s` active: %s (TTL: %s)',
318 314 plugin_id, plugin_cache_active, cache_ttl)
319 315
320 # for environ based password can be empty, but then the validation is
321 # on the server that fills in the env data needed for authentication
322 _perm_calc_hash = caches.compute_key_from_params(
323 plugin_id, action, user.user_id, repo_name, ip_addr)
316 user_id = user.user_id
317 cache_namespace_uid = 'cache_user_auth.{}'.format(user_id)
318 region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
324 319
325 # _authenticate is a wrapper for .auth() method of plugin.
326 # it checks if .auth() sends proper data.
327 # For RhodeCodeExternalAuthPlugin it also maps users to
328 # Database and maps the attributes returned from .auth()
329 # to RhodeCode database. If this function returns data
330 # then auth is correct.
331 start = time.time()
332 log.debug('Running plugin `%s` permissions check', plugin_id)
320 @region.cache_on_arguments(namespace=cache_namespace_uid,
321 expiration_time=cache_ttl,
322 should_cache_fn=lambda v: plugin_cache_active)
323 def compute_perm_vcs(
324 cache_name, plugin_id, action, user_id, repo_name, ip_addr):
333 325
334 def perm_func():
335 """
336 This function is used internally in Cache of Beaker to calculate
337 Results
338 """
339 326 log.debug('auth: calculating permission access now...')
340 327 # check IP
341 328 inherit = user.inherit_default_permissions
342 329 ip_allowed = AuthUser.check_ip_allowed(
343 user.user_id, ip_addr, inherit_from_default=inherit)
330 user_id, ip_addr, inherit_from_default=inherit)
344 331 if ip_allowed:
345 332 log.info('Access for IP:%s allowed', ip_addr)
346 333 else:
@@ -360,12 +347,13 b' class SimpleVCS(object):'
360 347
361 348 return True
362 349
363 if plugin_cache_active:
364 log.debug('Trying to fetch cached perms by %s', _perm_calc_hash[:6])
365 perm_result = cache_manager.get(
366 _perm_calc_hash, createfunc=perm_func)
367 else:
368 perm_result = perm_func()
350 start = time.time()
351 log.debug('Running plugin `%s` permissions check', plugin_id)
352
353 # for environ based auth, password can be empty, but then the validation is
354 # on the server that fills in the env data needed for authentication
355 perm_result = compute_perm_vcs(
356 'vcs_permissions', plugin_id, action, user.user_id, repo_name, ip_addr)
369 357
370 358 auth_time = time.time() - start
371 359 log.debug('Permissions for plugin `%s` completed in %.3fs, '
@@ -119,6 +119,8 b' function registerRCRoutes() {'
119 119 pyroutes.register('edit_user_groups_management', '/_admin/users/%(user_id)s/edit/groups_management', ['user_id']);
120 120 pyroutes.register('edit_user_groups_management_updates', '/_admin/users/%(user_id)s/edit/edit_user_groups_management/updates', ['user_id']);
121 121 pyroutes.register('edit_user_audit_logs', '/_admin/users/%(user_id)s/edit/audit', ['user_id']);
122 pyroutes.register('edit_user_caches', '/_admin/users/%(user_id)s/edit/caches', ['user_id']);
123 pyroutes.register('edit_user_caches_update', '/_admin/users/%(user_id)s/edit/caches/update', ['user_id']);
122 124 pyroutes.register('user_groups', '/_admin/user_groups', []);
123 125 pyroutes.register('user_groups_data', '/_admin/user_groups_data', []);
124 126 pyroutes.register('user_groups_new', '/_admin/user_groups/new', []);
@@ -45,6 +45,7 b''
45 45 <li class="${'active' if c.active=='ips' else ''}"><a href="${h.route_path('edit_user_ips', user_id=c.user.user_id)}">${_('Ip Whitelist')}</a></li>
46 46 <li class="${'active' if c.active=='groups' else ''}"><a href="${h.route_path('edit_user_groups_management', user_id=c.user.user_id)}">${_('User Groups Management')}</a></li>
47 47 <li class="${'active' if c.active=='audit' else ''}"><a href="${h.route_path('edit_user_audit_logs', user_id=c.user.user_id)}">${_('Audit logs')}</a></li>
48 <li class="${'active' if c.active=='caches' else ''}"><a href="${h.route_path('edit_user_caches', user_id=c.user.user_id)}">${_('Caches')}</a></li>
48 49 </ul>
49 50 </div>
50 51
@@ -22,36 +22,24 b' import time'
22 22
23 23 import pytest
24 24
25 from rhodecode.lib import caches
26 from rhodecode.lib.memory_lru_debug import MemoryLRUNamespaceManagerBase
25 from rhodecode.lib import rc_cache
27 26
28 27
29 class TestCacheManager(object):
28 @pytest.mark.usefixtures( 'app')
29 class TestCaches(object):
30 30
31 @pytest.mark.parametrize('repo_name', [
32 ('',),
33 (u'',),
34 (u'ac',),
35 ('ac', ),
36 (u'Δ™Δ‡c',),
37 ('Δ…ac',),
31 def test_cache_decorator_init_not_configured(self):
32 with pytest.raises(EnvironmentError):
33 rc_cache.get_or_create_region('dontexist')
34
35 @pytest.mark.parametrize('region_name', [
36 'cache_perms', u'cache_perms',
38 37 ])
39 def test_cache_manager_init(self, repo_name):
40 cache_manager_instance = caches.get_cache_manager(
41 repo_name, 'my_cache')
42 assert cache_manager_instance
43
44 def test_cache_manager_init_undefined_namespace(self):
45 cache_manager_instance = caches.get_cache_manager(
46 'repo_cache_long_undefined', 'my_cache')
47 assert cache_manager_instance
48
49 def_config = caches.DEFAULT_CACHE_MANAGER_CONFIG.copy()
50 def_config.pop('type')
51 assert cache_manager_instance.nsargs == def_config
52
53 assert isinstance(
54 cache_manager_instance.namespace, MemoryLRUNamespaceManagerBase)
38 def test_cache_decorator_init(self, region_name):
39 namespace = region_name
40 cache_region = rc_cache.get_or_create_region(
41 region_name, region_namespace=namespace)
42 assert cache_region
55 43
56 44 @pytest.mark.parametrize('example_input', [
57 45 ('',),
@@ -62,45 +50,60 b' class TestCacheManager(object):'
62 50 (u'/ac', ),
63 51 ])
64 52 def test_cache_manager_create_key(self, example_input):
65 key = caches.compute_key_from_params(*example_input)
53 key = rc_cache.utils.compute_key_from_params(*example_input)
66 54 assert key
67 55
68 def test_store_and_invalidate_value_from_manager(self):
69 cache_manger_instance = caches.get_cache_manager(
70 'repo_cache_long', 'my_cache_store')
56 @pytest.mark.parametrize('example_namespace', [
57 'namespace', None
58 ])
59 @pytest.mark.parametrize('example_input', [
60 ('',),
61 (u'/ac',),
62 (u'/ac', 1, 2, object()),
63 (u'/Δ™Δ‡c', 1, 2, object()),
64 ('/Δ…ac',),
65 (u'/ac', ),
66 ])
67 def test_cache_keygen(self, example_input, example_namespace):
68 def func_wrapped():
69 return 1
70 func = rc_cache.utils.key_generator(example_namespace, func_wrapped)
71 key = func(*example_input)
72 assert key
71 73
72 def compute():
74 def test_store_value_in_cache(self):
75 cache_region = rc_cache.get_or_create_region('cache_perms')
76 # make sure we empty the cache now
77 for key in cache_region.backend.list_keys():
78 cache_region.delete(key)
79
80 assert cache_region.backend.list_keys() == []
81
82 @cache_region.cache_on_arguments(expiration_time=5)
83 def compute(key):
73 84 return time.time()
74 85
75 added_keys = []
76 for i in xrange(3):
77 _cache_key = caches.compute_key_from_params('foo_bar', 'p%s' % i)
78 added_keys.append(_cache_key)
79 for x in xrange(10):
80 cache_manger_instance.get(_cache_key, createfunc=compute)
86 for x in range(10):
87 compute(x)
81 88
82 for key in added_keys:
83 assert cache_manger_instance[key]
84
85 caches.clear_cache_manager(cache_manger_instance)
89 assert len(set(cache_region.backend.list_keys())) == 10
86 90
87 for key in added_keys:
88 assert key not in cache_manger_instance
89 assert len(cache_manger_instance.namespace.keys()) == 0
91 def test_store_and_get_value_from_region(self):
92 cache_region = rc_cache.get_or_create_region('cache_perms')
93 # make sure we empty the cache now
94 for key in cache_region.backend.list_keys():
95 cache_region.delete(key)
96 assert cache_region.backend.list_keys() == []
90 97
91 def test_store_and_get_value_from_manager(self):
92 cache_manger_instance = caches.get_cache_manager(
93 'repo_cache_long', 'my_cache_store')
94
95 _cache_key = caches.compute_key_from_params('foo_bar', 'multicall')
96
97 def compute():
98 @cache_region.cache_on_arguments(expiration_time=5)
99 def compute(key):
98 100 return time.time()
99 101
100 102 result = set()
101 for x in xrange(10):
102 ret = cache_manger_instance.get(_cache_key, createfunc=compute)
103 for x in range(10):
104 ret = compute('x')
103 105 result.add(ret)
104 106
105 # once computed we have only one value after executing it 10x
107 # once computed we have only one value (the same from cache)
108 # after executing it 10x
106 109 assert len(result) == 1
@@ -292,11 +292,7 b' cache_dir = %(here)s/data'
292 292 beaker.cache.data_dir = %(here)s/rc/data/cache/beaker_data
293 293 beaker.cache.lock_dir = %(here)s/rc/data/cache/beaker_lock
294 294
295 beaker.cache.regions = short_term, long_term, sql_cache_short, auth_plugins, repo_cache_long
296
297 beaker.cache.short_term.type = file
298 beaker.cache.short_term.expire = 0
299 beaker.cache.short_term.key_length = 256
295 beaker.cache.regions = long_term, sql_cache_short, repo_cache_long
300 296
301 297 beaker.cache.long_term.type = memory
302 298 beaker.cache.long_term.expire = 36000
@@ -306,16 +302,6 b' beaker.cache.sql_cache_short.type = memo'
306 302 beaker.cache.sql_cache_short.expire = 1
307 303 beaker.cache.sql_cache_short.key_length = 256
308 304
309 ## default is memory cache, configure only if required
310 ## using multi-node or multi-worker setup
311 #beaker.cache.auth_plugins.type = memory
312 #beaker.cache.auth_plugins.lock_dir = %(here)s/data/cache/auth_plugin_lock
313 #beaker.cache.auth_plugins.url = postgresql://postgres:secret@localhost/rhodecode
314 #beaker.cache.auth_plugins.url = mysql://root:secret@127.0.0.1/rhodecode
315 #beaker.cache.auth_plugins.sa.pool_recycle = 3600
316 #beaker.cache.auth_plugins.sa.pool_size = 10
317 #beaker.cache.auth_plugins.sa.max_overflow = 0
318
319 305 beaker.cache.repo_cache_long.type = memorylru_base
320 306 beaker.cache.repo_cache_long.max_items = 4096
321 307 beaker.cache.repo_cache_long.expire = 2592000
@@ -327,6 +313,16 b' beaker.cache.repo_cache_long.expire = 25'
327 313 #beaker.cache.repo_cache_long.expire = 1209600
328 314 #beaker.cache.repo_cache_long.key_length = 256
329 315
316
317 #####################################
318 ### DOGPILE CACHE ####
319 #####################################
320
321 ## permission tree cache settings
322 rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace
323 rc_cache.cache_perms.expiration_time = 0
324 rc_cache.cache_perms.arguments.filename = /tmp/rc_cache_1
325
330 326 ####################################
331 327 ### BEAKER SESSION ####
332 328 ####################################
General Comments 0
You need to be logged in to leave comments. Login now