Show More
@@ -0,0 +1,66 b'' | |||||
|
1 | # -*- coding: utf-8 -*- | |||
|
2 | ||||
|
3 | # Copyright (C) 2015-2018 RhodeCode GmbH | |||
|
4 | # | |||
|
5 | # This program is free software: you can redistribute it and/or modify | |||
|
6 | # it under the terms of the GNU Affero General Public License, version 3 | |||
|
7 | # (only), as published by the Free Software Foundation. | |||
|
8 | # | |||
|
9 | # This program is distributed in the hope that it will be useful, | |||
|
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |||
|
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |||
|
12 | # GNU General Public License for more details. | |||
|
13 | # | |||
|
14 | # You should have received a copy of the GNU Affero General Public License | |||
|
15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |||
|
16 | # | |||
|
17 | # This program is dual-licensed. If you wish to learn more about the | |||
|
18 | # RhodeCode Enterprise Edition, including its added features, Support services, | |||
|
19 | # and proprietary license terms, please see https://rhodecode.com/licenses/ | |||
|
20 | ||||
|
21 | from dogpile.cache import register_backend | |||
|
22 | from dogpile.cache import make_region | |||
|
23 | ||||
|
24 | register_backend( | |||
|
25 | "dogpile.cache.rc.memory_lru", "rhodecode.lib.rc_cache.backends", | |||
|
26 | "LRUMemoryBackend") | |||
|
27 | ||||
|
28 | register_backend( | |||
|
29 | "dogpile.cache.rc.file_namespace", "rhodecode.lib.rc_cache.backends", | |||
|
30 | "FileNamespaceBackend") | |||
|
31 | ||||
|
32 | register_backend( | |||
|
33 | "dogpile.cache.rc.redis", "rhodecode.lib.rc_cache.backends", | |||
|
34 | "RedisPickleBackend") | |||
|
35 | ||||
|
36 | ||||
|
37 | from . import region_meta | |||
|
38 | from .utils import get_default_cache_settings, key_generator, get_or_create_region | |||
|
39 | ||||
|
40 | ||||
|
41 | def configure_dogpile_cache(settings): | |||
|
42 | cache_dir = settings.get('cache_dir') | |||
|
43 | if cache_dir: | |||
|
44 | region_meta.dogpile_config_defaults['cache_dir'] = cache_dir | |||
|
45 | ||||
|
46 | rc_cache_data = get_default_cache_settings(settings, prefixes=['rc_cache.']) | |||
|
47 | ||||
|
48 | # inspect available namespaces | |||
|
49 | avail_regions = set() | |||
|
50 | for key in rc_cache_data.keys(): | |||
|
51 | namespace_name = key.split('.', 1)[0] | |||
|
52 | avail_regions.add(namespace_name) | |||
|
53 | ||||
|
54 | # register them into namespace | |||
|
55 | for region_name in avail_regions: | |||
|
56 | new_region = make_region( | |||
|
57 | name=region_name, | |||
|
58 | function_key_generator=key_generator | |||
|
59 | ) | |||
|
60 | ||||
|
61 | new_region.configure_from_config(settings, 'rc_cache.{}.'.format(region_name)) | |||
|
62 | region_meta.dogpile_cache_regions[region_name] = new_region | |||
|
63 | ||||
|
64 | ||||
|
65 | def includeme(config): | |||
|
66 | configure_dogpile_cache(config.registry.settings) |
@@ -0,0 +1,109 b'' | |||||
|
1 | # -*- coding: utf-8 -*- | |||
|
2 | ||||
|
3 | # Copyright (C) 2015-2018 RhodeCode GmbH | |||
|
4 | # | |||
|
5 | # This program is free software: you can redistribute it and/or modify | |||
|
6 | # it under the terms of the GNU Affero General Public License, version 3 | |||
|
7 | # (only), as published by the Free Software Foundation. | |||
|
8 | # | |||
|
9 | # This program is distributed in the hope that it will be useful, | |||
|
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |||
|
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |||
|
12 | # GNU General Public License for more details. | |||
|
13 | # | |||
|
14 | # You should have received a copy of the GNU Affero General Public License | |||
|
15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |||
|
16 | # | |||
|
17 | # This program is dual-licensed. If you wish to learn more about the | |||
|
18 | # RhodeCode Enterprise Edition, including its added features, Support services, | |||
|
19 | # and proprietary license terms, please see https://rhodecode.com/licenses/ | |||
|
20 | ||||
|
21 | from dogpile.cache.backends import memory as memory_backend | |||
|
22 | from dogpile.cache.backends import file as file_backend | |||
|
23 | from dogpile.cache.backends import redis as redis_backend | |||
|
24 | from dogpile.cache.backends.file import NO_VALUE, compat | |||
|
25 | ||||
|
26 | from rhodecode.lib.memory_lru_debug import LRUDict | |||
|
27 | ||||
|
28 | _default_max_size = 1024 | |||
|
29 | ||||
|
30 | ||||
|
31 | class LRUMemoryBackend(memory_backend.MemoryBackend): | |||
|
32 | ||||
|
33 | def __init__(self, arguments): | |||
|
34 | max_size = arguments.pop('max_size', _default_max_size) | |||
|
35 | arguments['cache_dict'] = LRUDict(max_size) | |||
|
36 | super(LRUMemoryBackend, self).__init__(arguments) | |||
|
37 | ||||
|
38 | ||||
|
39 | class Serializer(object): | |||
|
40 | def _dumps(self, value): | |||
|
41 | return compat.pickle.dumps(value) | |||
|
42 | ||||
|
43 | def _loads(self, value): | |||
|
44 | return compat.pickle.loads(value) | |||
|
45 | ||||
|
46 | ||||
|
47 | class FileNamespaceBackend(Serializer, file_backend.DBMBackend): | |||
|
48 | ||||
|
49 | def __init__(self, arguments): | |||
|
50 | super(FileNamespaceBackend, self).__init__(arguments) | |||
|
51 | ||||
|
52 | def list_keys(self): | |||
|
53 | with self._dbm_file(True) as dbm: | |||
|
54 | return dbm.keys() | |||
|
55 | ||||
|
56 | def get_store(self): | |||
|
57 | return self.filename | |||
|
58 | ||||
|
59 | def get(self, key): | |||
|
60 | with self._dbm_file(False) as dbm: | |||
|
61 | if hasattr(dbm, 'get'): | |||
|
62 | value = dbm.get(key, NO_VALUE) | |||
|
63 | else: | |||
|
64 | # gdbm objects lack a .get method | |||
|
65 | try: | |||
|
66 | value = dbm[key] | |||
|
67 | except KeyError: | |||
|
68 | value = NO_VALUE | |||
|
69 | if value is not NO_VALUE: | |||
|
70 | value = self._loads(value) | |||
|
71 | return value | |||
|
72 | ||||
|
73 | def set(self, key, value): | |||
|
74 | with self._dbm_file(True) as dbm: | |||
|
75 | dbm[key] = self._dumps(value) | |||
|
76 | ||||
|
77 | def set_multi(self, mapping): | |||
|
78 | with self._dbm_file(True) as dbm: | |||
|
79 | for key, value in mapping.items(): | |||
|
80 | dbm[key] = self._dumps(value) | |||
|
81 | ||||
|
82 | ||||
|
83 | class RedisPickleBackend(Serializer, redis_backend.RedisBackend): | |||
|
84 | def list_keys(self): | |||
|
85 | return self.client.keys() | |||
|
86 | ||||
|
87 | def get_store(self): | |||
|
88 | return self.client.connection_pool | |||
|
89 | ||||
|
90 | def set(self, key, value): | |||
|
91 | if self.redis_expiration_time: | |||
|
92 | self.client.setex(key, self.redis_expiration_time, | |||
|
93 | self._dumps(value)) | |||
|
94 | else: | |||
|
95 | self.client.set(key, self._dumps(value)) | |||
|
96 | ||||
|
97 | def set_multi(self, mapping): | |||
|
98 | mapping = dict( | |||
|
99 | (k, self._dumps(v)) | |||
|
100 | for k, v in mapping.items() | |||
|
101 | ) | |||
|
102 | ||||
|
103 | if not self.redis_expiration_time: | |||
|
104 | self.client.mset(mapping) | |||
|
105 | else: | |||
|
106 | pipe = self.client.pipeline() | |||
|
107 | for key, value in mapping.items(): | |||
|
108 | pipe.setex(key, self.redis_expiration_time, value) | |||
|
109 | pipe.execute() |
@@ -0,0 +1,28 b'' | |||||
|
1 | # -*- coding: utf-8 -*- | |||
|
2 | ||||
|
3 | # Copyright (C) 2015-2018 RhodeCode GmbH | |||
|
4 | # | |||
|
5 | # This program is free software: you can redistribute it and/or modify | |||
|
6 | # it under the terms of the GNU Affero General Public License, version 3 | |||
|
7 | # (only), as published by the Free Software Foundation. | |||
|
8 | # | |||
|
9 | # This program is distributed in the hope that it will be useful, | |||
|
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |||
|
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |||
|
12 | # GNU General Public License for more details. | |||
|
13 | # | |||
|
14 | # You should have received a copy of the GNU Affero General Public License | |||
|
15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |||
|
16 | # | |||
|
17 | # This program is dual-licensed. If you wish to learn more about the | |||
|
18 | # RhodeCode Enterprise Edition, including its added features, Support services, | |||
|
19 | # and proprietary license terms, please see https://rhodecode.com/licenses/ | |||
|
20 | import os | |||
|
21 | import tempfile | |||
|
22 | ||||
|
23 | dogpile_config_defaults = { | |||
|
24 | 'cache_dir': os.path.join(tempfile.gettempdir(), 'rc_cache') | |||
|
25 | } | |||
|
26 | ||||
|
27 | # GLOBAL TO STORE ALL REGISTERED REGIONS | |||
|
28 | dogpile_cache_regions = {} |
@@ -0,0 +1,99 b'' | |||||
|
1 | # -*- coding: utf-8 -*- | |||
|
2 | ||||
|
3 | # Copyright (C) 2015-2018 RhodeCode GmbH | |||
|
4 | # | |||
|
5 | # This program is free software: you can redistribute it and/or modify | |||
|
6 | # it under the terms of the GNU Affero General Public License, version 3 | |||
|
7 | # (only), as published by the Free Software Foundation. | |||
|
8 | # | |||
|
9 | # This program is distributed in the hope that it will be useful, | |||
|
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |||
|
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |||
|
12 | # GNU General Public License for more details. | |||
|
13 | # | |||
|
14 | # You should have received a copy of the GNU Affero General Public License | |||
|
15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |||
|
16 | # | |||
|
17 | # This program is dual-licensed. If you wish to learn more about the | |||
|
18 | # RhodeCode Enterprise Edition, including its added features, Support services, | |||
|
19 | # and proprietary license terms, please see https://rhodecode.com/licenses/ | |||
|
20 | import os | |||
|
21 | import logging | |||
|
22 | from dogpile.cache import make_region | |||
|
23 | ||||
|
24 | from rhodecode.lib.utils import safe_str, sha1 | |||
|
25 | from . import region_meta | |||
|
26 | ||||
|
27 | log = logging.getLogger(__name__) | |||
|
28 | ||||
|
29 | ||||
|
30 | def get_default_cache_settings(settings, prefixes=None): | |||
|
31 | prefixes = prefixes or [] | |||
|
32 | cache_settings = {} | |||
|
33 | for key in settings.keys(): | |||
|
34 | for prefix in prefixes: | |||
|
35 | if key.startswith(prefix): | |||
|
36 | name = key.split(prefix)[1].strip() | |||
|
37 | val = settings[key] | |||
|
38 | if isinstance(val, basestring): | |||
|
39 | val = val.strip() | |||
|
40 | cache_settings[name] = val | |||
|
41 | return cache_settings | |||
|
42 | ||||
|
43 | ||||
|
44 | def compute_key_from_params(*args): | |||
|
45 | """ | |||
|
46 | Helper to compute key from given params to be used in cache manager | |||
|
47 | """ | |||
|
48 | return sha1("_".join(map(safe_str, args))) | |||
|
49 | ||||
|
50 | ||||
|
51 | def key_generator(namespace, fn): | |||
|
52 | fname = fn.__name__ | |||
|
53 | ||||
|
54 | def generate_key(*args): | |||
|
55 | namespace_pref = namespace or 'default' | |||
|
56 | arg_key = compute_key_from_params(*args) | |||
|
57 | final_key = "{}:{}_{}".format(namespace_pref, fname, arg_key) | |||
|
58 | ||||
|
59 | return final_key | |||
|
60 | ||||
|
61 | return generate_key | |||
|
62 | ||||
|
63 | ||||
|
64 | def get_or_create_region(region_name, region_namespace=None): | |||
|
65 | from rhodecode.lib.rc_cache.backends import FileNamespaceBackend | |||
|
66 | region_obj = region_meta.dogpile_cache_regions.get(region_name) | |||
|
67 | if not region_obj: | |||
|
68 | raise EnvironmentError( | |||
|
69 | 'Region `{}` not in configured: {}.'.format( | |||
|
70 | region_name, region_meta.dogpile_cache_regions.keys())) | |||
|
71 | ||||
|
72 | region_uid_name = '{}:{}'.format(region_name, region_namespace) | |||
|
73 | if isinstance(region_obj.actual_backend, FileNamespaceBackend): | |||
|
74 | region_exist = region_meta.dogpile_cache_regions.get(region_namespace) | |||
|
75 | if region_exist: | |||
|
76 | log.debug('Using already configured region: %s', region_namespace) | |||
|
77 | return region_exist | |||
|
78 | cache_dir = region_meta.dogpile_config_defaults['cache_dir'] | |||
|
79 | expiration_time = region_obj.expiration_time | |||
|
80 | ||||
|
81 | if not os.path.isdir(cache_dir): | |||
|
82 | os.makedirs(cache_dir) | |||
|
83 | new_region = make_region( | |||
|
84 | name=region_uid_name, function_key_generator=key_generator | |||
|
85 | ) | |||
|
86 | namespace_filename = os.path.join( | |||
|
87 | cache_dir, "{}.cache.dbm".format(region_namespace)) | |||
|
88 | # special type that allows 1db per namespace | |||
|
89 | new_region.configure( | |||
|
90 | backend='dogpile.cache.rc.file_namespace', | |||
|
91 | expiration_time=expiration_time, | |||
|
92 | arguments={"filename": namespace_filename} | |||
|
93 | ) | |||
|
94 | ||||
|
95 | # create and save in region caches | |||
|
96 | log.debug('configuring new region: %s',region_uid_name) | |||
|
97 | region_obj = region_meta.dogpile_cache_regions[region_namespace] = new_region | |||
|
98 | ||||
|
99 | return region_obj |
@@ -0,0 +1,29 b'' | |||||
|
1 | <%namespace name="base" file="/base/base.mako"/> | |||
|
2 | ||||
|
3 | <div class="panel panel-default"> | |||
|
4 | <div class="panel-heading"> | |||
|
5 | <h3 class="panel-title">${_('Caches')}</h3> | |||
|
6 | </div> | |||
|
7 | <div class="panel-body"> | |||
|
8 | <pre> | |||
|
9 | region: ${c.region.name} | |||
|
10 | backend: ${c.region.actual_backend.__class__} | |||
|
11 | store: ${c.region.actual_backend.get_store()} | |||
|
12 | ||||
|
13 | % for k in c.user_keys: | |||
|
14 | - ${k} | |||
|
15 | % endfor | |||
|
16 | </pre> | |||
|
17 | ||||
|
18 | ${h.secure_form(h.route_path('edit_user_caches_update', user_id=c.user.user_id), request=request)} | |||
|
19 | <div class="form"> | |||
|
20 | <div class="fields"> | |||
|
21 | ${h.submit('reset_cache_%s' % c.user.user_id, _('Invalidate user cache'),class_="btn btn-small",onclick="return confirm('"+_('Confirm to invalidate user cache')+"');")} | |||
|
22 | </div> | |||
|
23 | </div> | |||
|
24 | ${h.end_form()} | |||
|
25 | ||||
|
26 | </div> | |||
|
27 | </div> | |||
|
28 | ||||
|
29 |
@@ -320,12 +320,7 b' cache_dir = %(here)s/data' | |||||
320 | beaker.cache.data_dir = %(here)s/data/cache/beaker_data |
|
320 | beaker.cache.data_dir = %(here)s/data/cache/beaker_data | |
321 | beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock |
|
321 | beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock | |
322 |
|
322 | |||
323 |
beaker.cache.regions = |
|
323 | beaker.cache.regions = long_term, sql_cache_short, repo_cache_long | |
324 |
|
||||
325 | # used for caching user permissions |
|
|||
326 | beaker.cache.short_term.type = file |
|
|||
327 | beaker.cache.short_term.expire = 0 |
|
|||
328 | beaker.cache.short_term.key_length = 256 |
|
|||
329 |
|
324 | |||
330 | beaker.cache.long_term.type = memory |
|
325 | beaker.cache.long_term.type = memory | |
331 | beaker.cache.long_term.expire = 36000 |
|
326 | beaker.cache.long_term.expire = 36000 | |
@@ -335,16 +330,6 b' beaker.cache.sql_cache_short.type = memo' | |||||
335 | beaker.cache.sql_cache_short.expire = 10 |
|
330 | beaker.cache.sql_cache_short.expire = 10 | |
336 | beaker.cache.sql_cache_short.key_length = 256 |
|
331 | beaker.cache.sql_cache_short.key_length = 256 | |
337 |
|
332 | |||
338 | ## default is memory cache, configure only if required |
|
|||
339 | ## using multi-node or multi-worker setup |
|
|||
340 | #beaker.cache.auth_plugins.type = ext:database |
|
|||
341 | #beaker.cache.auth_plugins.lock_dir = %(here)s/data/cache/auth_plugin_lock |
|
|||
342 | #beaker.cache.auth_plugins.url = postgresql://postgres:secret@localhost/rhodecode |
|
|||
343 | #beaker.cache.auth_plugins.url = mysql://root:secret@127.0.0.1/rhodecode |
|
|||
344 | #beaker.cache.auth_plugins.sa.pool_recycle = 3600 |
|
|||
345 | #beaker.cache.auth_plugins.sa.pool_size = 10 |
|
|||
346 | #beaker.cache.auth_plugins.sa.max_overflow = 0 |
|
|||
347 |
|
||||
348 | beaker.cache.repo_cache_long.type = memorylru_base |
|
333 | beaker.cache.repo_cache_long.type = memorylru_base | |
349 | beaker.cache.repo_cache_long.max_items = 4096 |
|
334 | beaker.cache.repo_cache_long.max_items = 4096 | |
350 | beaker.cache.repo_cache_long.expire = 2592000 |
|
335 | beaker.cache.repo_cache_long.expire = 2592000 |
@@ -295,12 +295,7 b' cache_dir = %(here)s/data' | |||||
295 | beaker.cache.data_dir = %(here)s/data/cache/beaker_data |
|
295 | beaker.cache.data_dir = %(here)s/data/cache/beaker_data | |
296 | beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock |
|
296 | beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock | |
297 |
|
297 | |||
298 |
beaker.cache.regions = |
|
298 | beaker.cache.regions = long_term, sql_cache_short, repo_cache_long | |
299 |
|
||||
300 | # used for caching user permissions |
|
|||
301 | beaker.cache.short_term.type = file |
|
|||
302 | beaker.cache.short_term.expire = 0 |
|
|||
303 | beaker.cache.short_term.key_length = 256 |
|
|||
304 |
|
299 | |||
305 | beaker.cache.long_term.type = memory |
|
300 | beaker.cache.long_term.type = memory | |
306 | beaker.cache.long_term.expire = 36000 |
|
301 | beaker.cache.long_term.expire = 36000 | |
@@ -310,16 +305,6 b' beaker.cache.sql_cache_short.type = memo' | |||||
310 | beaker.cache.sql_cache_short.expire = 10 |
|
305 | beaker.cache.sql_cache_short.expire = 10 | |
311 | beaker.cache.sql_cache_short.key_length = 256 |
|
306 | beaker.cache.sql_cache_short.key_length = 256 | |
312 |
|
307 | |||
313 | ## default is memory cache, configure only if required |
|
|||
314 | ## using multi-node or multi-worker setup |
|
|||
315 | #beaker.cache.auth_plugins.type = ext:database |
|
|||
316 | #beaker.cache.auth_plugins.lock_dir = %(here)s/data/cache/auth_plugin_lock |
|
|||
317 | #beaker.cache.auth_plugins.url = postgresql://postgres:secret@localhost/rhodecode |
|
|||
318 | #beaker.cache.auth_plugins.url = mysql://root:secret@127.0.0.1/rhodecode |
|
|||
319 | #beaker.cache.auth_plugins.sa.pool_recycle = 3600 |
|
|||
320 | #beaker.cache.auth_plugins.sa.pool_size = 10 |
|
|||
321 | #beaker.cache.auth_plugins.sa.max_overflow = 0 |
|
|||
322 |
|
||||
323 | beaker.cache.repo_cache_long.type = memorylru_base |
|
308 | beaker.cache.repo_cache_long.type = memorylru_base | |
324 | beaker.cache.repo_cache_long.max_items = 4096 |
|
309 | beaker.cache.repo_cache_long.max_items = 4096 | |
325 | beaker.cache.repo_cache_long.expire = 2592000 |
|
310 | beaker.cache.repo_cache_long.expire = 2592000 |
@@ -356,6 +356,16 b' def admin_routes(config):' | |||||
356 | name='edit_user_audit_logs', |
|
356 | name='edit_user_audit_logs', | |
357 | pattern='/users/{user_id:\d+}/edit/audit', user_route=True) |
|
357 | pattern='/users/{user_id:\d+}/edit/audit', user_route=True) | |
358 |
|
358 | |||
|
359 | # user caches | |||
|
360 | config.add_route( | |||
|
361 | name='edit_user_caches', | |||
|
362 | pattern='/users/{user_id:\d+}/edit/caches', | |||
|
363 | user_route=True) | |||
|
364 | config.add_route( | |||
|
365 | name='edit_user_caches_update', | |||
|
366 | pattern='/users/{user_id:\d+}/edit/caches/update', | |||
|
367 | user_route=True) | |||
|
368 | ||||
359 | # user-groups admin |
|
369 | # user-groups admin | |
360 | config.add_route( |
|
370 | config.add_route( | |
361 | name='user_groups', |
|
371 | name='user_groups', |
@@ -34,7 +34,7 b' from rhodecode.authentication.plugins im' | |||||
34 | from rhodecode.events import trigger |
|
34 | from rhodecode.events import trigger | |
35 | from rhodecode.model.db import true |
|
35 | from rhodecode.model.db import true | |
36 |
|
36 | |||
37 | from rhodecode.lib import audit_logger |
|
37 | from rhodecode.lib import audit_logger, rc_cache | |
38 | from rhodecode.lib.exceptions import ( |
|
38 | from rhodecode.lib.exceptions import ( | |
39 | UserCreationError, UserOwnsReposException, UserOwnsRepoGroupsException, |
|
39 | UserCreationError, UserOwnsReposException, UserOwnsRepoGroupsException, | |
40 | UserOwnsUserGroupsException, DefaultUserException) |
|
40 | UserOwnsUserGroupsException, DefaultUserException) | |
@@ -1198,3 +1198,57 b' class UsersView(UserAppView):' | |||||
1198 | perm_user = self.db_user.AuthUser(ip_addr=self.request.remote_addr) |
|
1198 | perm_user = self.db_user.AuthUser(ip_addr=self.request.remote_addr) | |
1199 |
|
1199 | |||
1200 | return perm_user.permissions |
|
1200 | return perm_user.permissions | |
|
1201 | ||||
|
1202 | def _get_user_cache_keys(self, cache_namespace_uid, keys): | |||
|
1203 | user_keys = [] | |||
|
1204 | for k in sorted(keys): | |||
|
1205 | if k.startswith(cache_namespace_uid): | |||
|
1206 | user_keys.append(k) | |||
|
1207 | return user_keys | |||
|
1208 | ||||
|
1209 | @LoginRequired() | |||
|
1210 | @HasPermissionAllDecorator('hg.admin') | |||
|
1211 | @view_config( | |||
|
1212 | route_name='edit_user_caches', request_method='GET', | |||
|
1213 | renderer='rhodecode:templates/admin/users/user_edit.mako') | |||
|
1214 | def user_caches(self): | |||
|
1215 | _ = self.request.translate | |||
|
1216 | c = self.load_default_context() | |||
|
1217 | c.user = self.db_user | |||
|
1218 | ||||
|
1219 | c.active = 'caches' | |||
|
1220 | c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr) | |||
|
1221 | ||||
|
1222 | cache_namespace_uid = 'cache_user_auth.{}'.format(self.db_user.user_id) | |||
|
1223 | c.region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid) | |||
|
1224 | c.backend = c.region.backend | |||
|
1225 | c.user_keys = self._get_user_cache_keys( | |||
|
1226 | cache_namespace_uid, c.region.backend.list_keys()) | |||
|
1227 | ||||
|
1228 | return self._get_template_context(c) | |||
|
1229 | ||||
|
1230 | @LoginRequired() | |||
|
1231 | @HasPermissionAllDecorator('hg.admin') | |||
|
1232 | @CSRFRequired() | |||
|
1233 | @view_config( | |||
|
1234 | route_name='edit_user_caches_update', request_method='POST') | |||
|
1235 | def user_caches_update(self): | |||
|
1236 | _ = self.request.translate | |||
|
1237 | c = self.load_default_context() | |||
|
1238 | c.user = self.db_user | |||
|
1239 | ||||
|
1240 | c.active = 'caches' | |||
|
1241 | c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr) | |||
|
1242 | ||||
|
1243 | cache_namespace_uid = 'cache_user_auth.{}'.format(self.db_user.user_id) | |||
|
1244 | c.region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid) | |||
|
1245 | ||||
|
1246 | c.user_keys = self._get_user_cache_keys( | |||
|
1247 | cache_namespace_uid, c.region.backend.list_keys()) | |||
|
1248 | for k in c.user_keys: | |||
|
1249 | c.region.delete(k) | |||
|
1250 | ||||
|
1251 | h.flash(_("Deleted {} cache keys").format(len(c.user_keys)), category='success') | |||
|
1252 | ||||
|
1253 | return HTTPFound(h.route_path( | |||
|
1254 | 'edit_user_caches', user_id=c.user.user_id)) |
@@ -35,7 +35,7 b' from pyramid.threadlocal import get_curr' | |||||
35 |
|
35 | |||
36 | from rhodecode.authentication.interface import IAuthnPluginRegistry |
|
36 | from rhodecode.authentication.interface import IAuthnPluginRegistry | |
37 | from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase |
|
37 | from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase | |
38 | from rhodecode.lib import caches |
|
38 | from rhodecode.lib import caches, rc_cache | |
39 | from rhodecode.lib.auth import PasswordGenerator, _RhodeCodeCryptoBCrypt |
|
39 | from rhodecode.lib.auth import PasswordGenerator, _RhodeCodeCryptoBCrypt | |
40 | from rhodecode.lib.utils2 import safe_int, safe_str |
|
40 | from rhodecode.lib.utils2 import safe_int, safe_str | |
41 | from rhodecode.lib.exceptions import LdapConnectionError |
|
41 | from rhodecode.lib.exceptions import LdapConnectionError | |
@@ -633,22 +633,6 b' def get_authn_registry(registry=None):' | |||||
633 | return authn_registry |
|
633 | return authn_registry | |
634 |
|
634 | |||
635 |
|
635 | |||
636 | def get_auth_cache_manager(custom_ttl=None, suffix=None): |
|
|||
637 | cache_name = 'rhodecode.authentication' |
|
|||
638 | if suffix: |
|
|||
639 | cache_name = 'rhodecode.authentication.{}'.format(suffix) |
|
|||
640 | return caches.get_cache_manager( |
|
|||
641 | 'auth_plugins', cache_name, custom_ttl) |
|
|||
642 |
|
||||
643 |
|
||||
644 | def get_perms_cache_manager(custom_ttl=None, suffix=None): |
|
|||
645 | cache_name = 'rhodecode.permissions' |
|
|||
646 | if suffix: |
|
|||
647 | cache_name = 'rhodecode.permissions.{}'.format(suffix) |
|
|||
648 | return caches.get_cache_manager( |
|
|||
649 | 'auth_plugins', cache_name, custom_ttl) |
|
|||
650 |
|
||||
651 |
|
||||
652 | def authenticate(username, password, environ=None, auth_type=None, |
|
636 | def authenticate(username, password, environ=None, auth_type=None, | |
653 | skip_missing=False, registry=None, acl_repo_name=None): |
|
637 | skip_missing=False, registry=None, acl_repo_name=None): | |
654 | """ |
|
638 | """ | |
@@ -707,45 +691,35 b' def authenticate(username, password, env' | |||||
707 |
|
691 | |||
708 | plugin_cache_active, cache_ttl = plugin.get_ttl_cache(plugin_settings) |
|
692 | plugin_cache_active, cache_ttl = plugin.get_ttl_cache(plugin_settings) | |
709 |
|
693 | |||
710 | # get instance of cache manager configured for a namespace |
|
|||
711 | cache_manager = get_auth_cache_manager( |
|
|||
712 | custom_ttl=cache_ttl, suffix=user.user_id if user else '') |
|
|||
713 |
|
||||
714 | log.debug('AUTH_CACHE_TTL for plugin `%s` active: %s (TTL: %s)', |
|
694 | log.debug('AUTH_CACHE_TTL for plugin `%s` active: %s (TTL: %s)', | |
715 | plugin.get_id(), plugin_cache_active, cache_ttl) |
|
695 | plugin.get_id(), plugin_cache_active, cache_ttl) | |
716 |
|
696 | |||
717 | # for environ based password can be empty, but then the validation is |
|
697 | user_id = user.user_id | |
718 | # on the server that fills in the env data needed for authentication |
|
698 | cache_namespace_uid = 'cache_user_auth.{}'.format(user_id) | |
719 |
|
699 | region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid) | ||
720 | _password_hash = caches.compute_key_from_params( |
|
|||
721 | plugin.name, username, (password or '')) |
|
|||
722 |
|
700 | |||
723 | # _authenticate is a wrapper for .auth() method of plugin. |
|
701 | @region.cache_on_arguments(namespace=cache_namespace_uid, | |
724 | # it checks if .auth() sends proper data. |
|
702 | expiration_time=cache_ttl, | |
725 | # For RhodeCodeExternalAuthPlugin it also maps users to |
|
703 | should_cache_fn=lambda v: plugin_cache_active) | |
726 | # Database and maps the attributes returned from .auth() |
|
704 | def compute_auth( | |
727 | # to RhodeCode database. If this function returns data |
|
705 | cache_name, plugin_name, username, password): | |
728 | # then auth is correct. |
|
|||
729 | start = time.time() |
|
|||
730 | log.debug('Running plugin `%s` _authenticate method', plugin.get_id()) |
|
|||
731 |
|
706 | |||
732 | def auth_func(): |
|
707 | # _authenticate is a wrapper for .auth() method of plugin. | |
733 | """ |
|
708 | # it checks if .auth() sends proper data. | |
734 | This function is used internally in Cache of Beaker to calculate |
|
709 | # For RhodeCodeExternalAuthPlugin it also maps users to | |
735 | Results |
|
710 | # Database and maps the attributes returned from .auth() | |
736 | """ |
|
711 | # to RhodeCode database. If this function returns data | |
737 | log.debug('auth: calculating password access now...') |
|
712 | # then auth is correct. | |
|
713 | log.debug('Running plugin `%s` _authenticate method ' | |||
|
714 | 'using username and password', plugin.get_id()) | |||
738 | return plugin._authenticate( |
|
715 | return plugin._authenticate( | |
739 | user, username, password, plugin_settings, |
|
716 | user, username, password, plugin_settings, | |
740 | environ=environ or {}) |
|
717 | environ=environ or {}) | |
741 |
|
718 | |||
742 | if plugin_cache_active: |
|
719 | start = time.time() | |
743 | log.debug('Trying to fetch cached auth by pwd hash `...%s`', |
|
720 | # for environ based auth, password can be empty, but then the validation is | |
744 | _password_hash[:6]) |
|
721 | # on the server that fills in the env data needed for authentication | |
745 | plugin_user = cache_manager.get( |
|
722 | plugin_user = compute_auth('auth', plugin.name, username, (password or '')) | |
746 | _password_hash, createfunc=auth_func) |
|
|||
747 | else: |
|
|||
748 | plugin_user = auth_func() |
|
|||
749 |
|
723 | |||
750 | auth_time = time.time() - start |
|
724 | auth_time = time.time() - start | |
751 | log.debug('Authentication for plugin `%s` completed in %.3fs, ' |
|
725 | log.debug('Authentication for plugin `%s` completed in %.3fs, ' |
@@ -27,8 +27,7 b' from pyramid.renderers import render' | |||||
27 | from pyramid.response import Response |
|
27 | from pyramid.response import Response | |
28 |
|
28 | |||
29 | from rhodecode.apps._base import BaseAppView |
|
29 | from rhodecode.apps._base import BaseAppView | |
30 |
from rhodecode.authentication.base import |
|
30 | from rhodecode.authentication.base import get_authn_registry | |
31 | get_auth_cache_manager, get_perms_cache_manager, get_authn_registry) |
|
|||
32 | from rhodecode.lib import helpers as h |
|
31 | from rhodecode.lib import helpers as h | |
33 | from rhodecode.lib.auth import ( |
|
32 | from rhodecode.lib.auth import ( | |
34 | LoginRequired, HasPermissionAllDecorator, CSRFRequired) |
|
33 | LoginRequired, HasPermissionAllDecorator, CSRFRequired) | |
@@ -102,16 +101,6 b' class AuthnPluginViewBase(BaseAppView):' | |||||
102 | self.plugin.create_or_update_setting(name, value) |
|
101 | self.plugin.create_or_update_setting(name, value) | |
103 | Session().commit() |
|
102 | Session().commit() | |
104 |
|
103 | |||
105 | # cleanup cache managers in case of change for plugin |
|
|||
106 | # TODO(marcink): because we can register multiple namespaces |
|
|||
107 | # we should at some point figure out how to retrieve ALL namespace |
|
|||
108 | # cache managers and clear them... |
|
|||
109 | cache_manager = get_auth_cache_manager() |
|
|||
110 | clear_cache_manager(cache_manager) |
|
|||
111 |
|
||||
112 | cache_manager = get_perms_cache_manager() |
|
|||
113 | clear_cache_manager(cache_manager) |
|
|||
114 |
|
||||
115 | # Display success message and redirect. |
|
104 | # Display success message and redirect. | |
116 | h.flash(_('Auth settings updated successfully.'), category='success') |
|
105 | h.flash(_('Auth settings updated successfully.'), category='success') | |
117 | redirect_to = self.request.resource_path( |
|
106 | redirect_to = self.request.resource_path( |
@@ -22,6 +22,7 b' import os' | |||||
22 | import logging |
|
22 | import logging | |
23 | import traceback |
|
23 | import traceback | |
24 | import collections |
|
24 | import collections | |
|
25 | import tempfile | |||
25 |
|
26 | |||
26 | from paste.gzipper import make_gzip_middleware |
|
27 | from paste.gzipper import make_gzip_middleware | |
27 | from pyramid.wsgi import wsgiapp |
|
28 | from pyramid.wsgi import wsgiapp | |
@@ -220,6 +221,7 b' def includeme(config):' | |||||
220 | config.include('pyramid_mako') |
|
221 | config.include('pyramid_mako') | |
221 | config.include('pyramid_beaker') |
|
222 | config.include('pyramid_beaker') | |
222 | config.include('rhodecode.lib.caches') |
|
223 | config.include('rhodecode.lib.caches') | |
|
224 | config.include('rhodecode.lib.rc_cache') | |||
223 |
|
225 | |||
224 | config.include('rhodecode.authentication') |
|
226 | config.include('rhodecode.authentication') | |
225 | config.include('rhodecode.integrations') |
|
227 | config.include('rhodecode.integrations') | |
@@ -382,6 +384,7 b' def sanitize_settings_and_apply_defaults' | |||||
382 | # Call split out functions that sanitize settings for each topic. |
|
384 | # Call split out functions that sanitize settings for each topic. | |
383 | _sanitize_appenlight_settings(settings) |
|
385 | _sanitize_appenlight_settings(settings) | |
384 | _sanitize_vcs_settings(settings) |
|
386 | _sanitize_vcs_settings(settings) | |
|
387 | _sanitize_cache_settings(settings) | |||
385 |
|
388 | |||
386 | # configure instance id |
|
389 | # configure instance id | |
387 | config_utils.set_instance_id(settings) |
|
390 | config_utils.set_instance_id(settings) | |
@@ -421,6 +424,18 b' def _sanitize_vcs_settings(settings):' | |||||
421 | settings['vcs.scm_app_implementation'] = 'http' |
|
424 | settings['vcs.scm_app_implementation'] = 'http' | |
422 |
|
425 | |||
423 |
|
426 | |||
|
427 | def _sanitize_cache_settings(settings): | |||
|
428 | _string_setting(settings, 'cache_dir', | |||
|
429 | os.path.join(tempfile.gettempdir(), 'rc_cache')) | |||
|
430 | ||||
|
431 | _string_setting(settings, 'rc_cache.cache_perms.backend', | |||
|
432 | 'dogpile.cache.rc.file_namespace') | |||
|
433 | _int_setting(settings, 'rc_cache.cache_perms.expiration_time', | |||
|
434 | 60) | |||
|
435 | _string_setting(settings, 'rc_cache.cache_perms.arguments.filename', | |||
|
436 | os.path.join(tempfile.gettempdir(), 'rc_cache_1')) | |||
|
437 | ||||
|
438 | ||||
424 | def _int_setting(settings, name, default): |
|
439 | def _int_setting(settings, name, default): | |
425 | settings[name] = int(settings.get(name, default)) |
|
440 | settings[name] = int(settings.get(name, default)) | |
426 |
|
441 |
@@ -48,7 +48,7 b' from rhodecode.model.user import UserMod' | |||||
48 | from rhodecode.model.db import ( |
|
48 | from rhodecode.model.db import ( | |
49 | User, Repository, Permission, UserToPerm, UserGroupToPerm, UserGroupMember, |
|
49 | User, Repository, Permission, UserToPerm, UserGroupToPerm, UserGroupMember, | |
50 | UserIpMap, UserApiKeys, RepoGroup, UserGroup) |
|
50 | UserIpMap, UserApiKeys, RepoGroup, UserGroup) | |
51 |
from rhodecode.lib import cache |
|
51 | from rhodecode.lib import rc_cache | |
52 | from rhodecode.lib.utils2 import safe_unicode, aslist, safe_str, md5, safe_int, sha1 |
|
52 | from rhodecode.lib.utils2 import safe_unicode, aslist, safe_str, md5, safe_int, sha1 | |
53 | from rhodecode.lib.utils import ( |
|
53 | from rhodecode.lib.utils import ( | |
54 | get_repo_slug, get_repo_group_slug, get_user_group_slug) |
|
54 | get_repo_slug, get_repo_group_slug, get_user_group_slug) | |
@@ -976,21 +976,18 b' class AuthUser(object):' | |||||
976 | obj = Repository.get_by_repo_name(scope['repo_name']) |
|
976 | obj = Repository.get_by_repo_name(scope['repo_name']) | |
977 | if obj: |
|
977 | if obj: | |
978 | scope['repo_id'] = obj.repo_id |
|
978 | scope['repo_id'] = obj.repo_id | |
979 | _scope = { |
|
979 | _scope = collections.OrderedDict() | |
980 |
|
|
980 | _scope['repo_id'] = -1 | |
981 |
|
|
981 | _scope['user_group_id'] = -1 | |
982 |
|
|
982 | _scope['repo_group_id'] = -1 | |
983 | } |
|
983 | ||
984 | _scope.update(scope) |
|
984 | for k in sorted(scope.keys()): | |
985 | cache_key = "_".join(map(safe_str, reduce(lambda a, b: a+b, |
|
985 | _scope[k] = scope[k] | |
986 | _scope.items()))) |
|
986 | ||
987 | if cache_key not in self._permissions_scoped_cache: |
|
987 | # store in cache to mimic how the @LazyProperty works, | |
988 | # store in cache to mimic how the @LazyProperty works, |
|
988 | # the difference here is that we use the unique key calculated | |
989 | # the difference here is that we use the unique key calculated |
|
989 | # from params and values | |
990 | # from params and values |
|
990 | return self.get_perms(user=self, cache=False, scope=_scope) | |
991 | res = self.get_perms(user=self, cache=False, scope=_scope) |
|
|||
992 | self._permissions_scoped_cache[cache_key] = res |
|
|||
993 | return self._permissions_scoped_cache[cache_key] |
|
|||
994 |
|
991 | |||
995 | def get_instance(self): |
|
992 | def get_instance(self): | |
996 | return User.get(self.user_id) |
|
993 | return User.get(self.user_id) | |
@@ -1072,16 +1069,27 b' class AuthUser(object):' | |||||
1072 | user_inherit_default_permissions = user.inherit_default_permissions |
|
1069 | user_inherit_default_permissions = user.inherit_default_permissions | |
1073 |
|
1070 | |||
1074 | cache_seconds = safe_int( |
|
1071 | cache_seconds = safe_int( | |
1075 |
rhodecode.CONFIG.get(' |
|
1072 | rhodecode.CONFIG.get('rc_cache.cache_perms.expiration_time')) | |
|
1073 | ||||
1076 | cache_on = cache or cache_seconds > 0 |
|
1074 | cache_on = cache or cache_seconds > 0 | |
1077 | log.debug( |
|
1075 | log.debug( | |
1078 | 'Computing PERMISSION tree for user %s scope `%s` ' |
|
1076 | 'Computing PERMISSION tree for user %s scope `%s` ' | |
1079 | 'with caching: %s[%ss]' % (user, scope, cache_on, cache_seconds)) |
|
1077 | 'with caching: %s[TTL: %ss]' % (user, scope, cache_on, cache_seconds or 0)) | |
|
1078 | ||||
|
1079 | cache_namespace_uid = 'cache_user_auth.{}'.format(user_id) | |||
|
1080 | region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid) | |||
|
1081 | ||||
|
1082 | @region.cache_on_arguments(namespace=cache_namespace_uid, | |||
|
1083 | should_cache_fn=lambda v: cache_on) | |||
|
1084 | def compute_perm_tree(cache_name, | |||
|
1085 | user_id, scope, user_is_admin,user_inherit_default_permissions, | |||
|
1086 | explicit, algo, calculate_super_admin): | |||
|
1087 | return _cached_perms_data( | |||
|
1088 | user_id, scope, user_is_admin, user_inherit_default_permissions, | |||
|
1089 | explicit, algo, calculate_super_admin) | |||
|
1090 | ||||
1080 | start = time.time() |
|
1091 | start = time.time() | |
1081 | compute = caches.conditional_cache( |
|
1092 | result = compute_perm_tree('permissions', user_id, scope, user_is_admin, | |
1082 | 'short_term', 'cache_desc.{}'.format(user_id), |
|
|||
1083 | condition=cache_on, func=_cached_perms_data) |
|
|||
1084 | result = compute(user_id, scope, user_is_admin, |
|
|||
1085 | user_inherit_default_permissions, explicit, algo, |
|
1093 | user_inherit_default_permissions, explicit, algo, | |
1086 | calculate_super_admin) |
|
1094 | calculate_super_admin) | |
1087 |
|
1095 | |||
@@ -1091,6 +1099,7 b' class AuthUser(object):' | |||||
1091 | total = time.time() - start |
|
1099 | total = time.time() - start | |
1092 | log.debug('PERMISSION tree for user %s computed in %.3fs: %s' % ( |
|
1100 | log.debug('PERMISSION tree for user %s computed in %.3fs: %s' % ( | |
1093 | user, total, result_repr)) |
|
1101 | user, total, result_repr)) | |
|
1102 | ||||
1094 | return result |
|
1103 | return result | |
1095 |
|
1104 | |||
1096 | @property |
|
1105 | @property | |
@@ -1153,10 +1162,7 b' class AuthUser(object):' | |||||
1153 | return [x.repo_id for x in |
|
1162 | return [x.repo_id for x in | |
1154 | RepoList(qry, perm_set=perm_def)] |
|
1163 | RepoList(qry, perm_set=perm_def)] | |
1155 |
|
1164 | |||
1156 | compute = caches.conditional_cache( |
|
1165 | return _cached_repo_acl(self.user_id, perms, name_filter) | |
1157 | 'short_term', 'repo_acl_ids.{}'.format(self.user_id), |
|
|||
1158 | condition=cache, func=_cached_repo_acl) |
|
|||
1159 | return compute(self.user_id, perms, name_filter) |
|
|||
1160 |
|
1166 | |||
1161 | def repo_group_acl_ids(self, perms=None, name_filter=None, cache=False): |
|
1167 | def repo_group_acl_ids(self, perms=None, name_filter=None, cache=False): | |
1162 | """ |
|
1168 | """ | |
@@ -1179,10 +1185,7 b' class AuthUser(object):' | |||||
1179 | return [x.group_id for x in |
|
1185 | return [x.group_id for x in | |
1180 | RepoGroupList(qry, perm_set=perm_def)] |
|
1186 | RepoGroupList(qry, perm_set=perm_def)] | |
1181 |
|
1187 | |||
1182 | compute = caches.conditional_cache( |
|
1188 | return _cached_repo_group_acl(self.user_id, perms, name_filter) | |
1183 | 'short_term', 'repo_group_acl_ids.{}'.format(self.user_id), |
|
|||
1184 | condition=cache, func=_cached_repo_group_acl) |
|
|||
1185 | return compute(self.user_id, perms, name_filter) |
|
|||
1186 |
|
1189 | |||
1187 | def user_group_acl_ids(self, perms=None, name_filter=None, cache=False): |
|
1190 | def user_group_acl_ids(self, perms=None, name_filter=None, cache=False): | |
1188 | """ |
|
1191 | """ | |
@@ -1205,10 +1208,7 b' class AuthUser(object):' | |||||
1205 | return [x.users_group_id for x in |
|
1208 | return [x.users_group_id for x in | |
1206 | UserGroupList(qry, perm_set=perm_def)] |
|
1209 | UserGroupList(qry, perm_set=perm_def)] | |
1207 |
|
1210 | |||
1208 | compute = caches.conditional_cache( |
|
1211 | return _cached_user_group_acl(self.user_id, perms, name_filter) | |
1209 | 'short_term', 'user_group_acl_ids.{}'.format(self.user_id), |
|
|||
1210 | condition=cache, func=_cached_user_group_acl) |
|
|||
1211 | return compute(self.user_id, perms, name_filter) |
|
|||
1212 |
|
1212 | |||
1213 | @property |
|
1213 | @property | |
1214 | def ip_allowed(self): |
|
1214 | def ip_allowed(self): |
@@ -505,6 +505,7 b' def bootstrap_config(request):' | |||||
505 | config.include('pyramid_mako') |
|
505 | config.include('pyramid_mako') | |
506 | config.include('pyramid_beaker') |
|
506 | config.include('pyramid_beaker') | |
507 | config.include('rhodecode.lib.caches') |
|
507 | config.include('rhodecode.lib.caches') | |
|
508 | config.include('rhodecode.lib.rc_cache') | |||
508 |
|
509 | |||
509 | add_events_routes(config) |
|
510 | add_events_routes(config) | |
510 |
|
511 |
@@ -96,7 +96,7 b' def get_cache_manager(region_name, cache' | |||||
96 | Creates a Beaker cache manager. Such instance can be used like that:: |
|
96 | Creates a Beaker cache manager. Such instance can be used like that:: | |
97 |
|
97 | |||
98 | _namespace = caches.get_repo_namespace_key(caches.XXX, repo_name) |
|
98 | _namespace = caches.get_repo_namespace_key(caches.XXX, repo_name) | |
99 |
cache_manager = caches.get_cache_manager(' |
|
99 | cache_manager = caches.get_cache_manager('some_namespace_name', _namespace) | |
100 | _cache_key = caches.compute_key_from_params(repo_name, commit.raw_id) |
|
100 | _cache_key = caches.compute_key_from_params(repo_name, commit.raw_id) | |
101 | def heavy_compute(): |
|
101 | def heavy_compute(): | |
102 | ... |
|
102 | ... | |
@@ -121,7 +121,7 b' def get_cache_manager(region_name, cache' | |||||
121 | def clear_cache_manager(cache_manager): |
|
121 | def clear_cache_manager(cache_manager): | |
122 | """ |
|
122 | """ | |
123 | namespace = 'foobar' |
|
123 | namespace = 'foobar' | |
124 |
cache_manager = get_cache_manager(' |
|
124 | cache_manager = get_cache_manager('some_namespace_name', namespace) | |
125 | clear_cache_manager(cache_manager) |
|
125 | clear_cache_manager(cache_manager) | |
126 | """ |
|
126 | """ | |
127 |
|
127 |
@@ -39,9 +39,8 b' from pyramid.httpexceptions import (' | |||||
39 | from zope.cachedescriptors.property import Lazy as LazyProperty |
|
39 | from zope.cachedescriptors.property import Lazy as LazyProperty | |
40 |
|
40 | |||
41 | import rhodecode |
|
41 | import rhodecode | |
42 |
from rhodecode.authentication.base import |
|
42 | from rhodecode.authentication.base import authenticate, VCS_TYPE, loadplugin | |
43 | authenticate, get_perms_cache_manager, VCS_TYPE, loadplugin) |
|
43 | from rhodecode.lib import caches, rc_cache | |
44 | from rhodecode.lib import caches |
|
|||
45 | from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware |
|
44 | from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware | |
46 | from rhodecode.lib.base import ( |
|
45 | from rhodecode.lib.base import ( | |
47 | BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context) |
|
46 | BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context) | |
@@ -311,36 +310,24 b' class SimpleVCS(object):' | |||||
311 | :param repo_name: repository name |
|
310 | :param repo_name: repository name | |
312 | """ |
|
311 | """ | |
313 |
|
312 | |||
314 | # get instance of cache manager configured for a namespace |
|
|||
315 | cache_manager = get_perms_cache_manager( |
|
|||
316 | custom_ttl=cache_ttl, suffix=user.user_id) |
|
|||
317 | log.debug('AUTH_CACHE_TTL for permissions `%s` active: %s (TTL: %s)', |
|
313 | log.debug('AUTH_CACHE_TTL for permissions `%s` active: %s (TTL: %s)', | |
318 | plugin_id, plugin_cache_active, cache_ttl) |
|
314 | plugin_id, plugin_cache_active, cache_ttl) | |
319 |
|
315 | |||
320 | # for environ based password can be empty, but then the validation is |
|
316 | user_id = user.user_id | |
321 | # on the server that fills in the env data needed for authentication |
|
317 | cache_namespace_uid = 'cache_user_auth.{}'.format(user_id) | |
322 | _perm_calc_hash = caches.compute_key_from_params( |
|
318 | region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid) | |
323 | plugin_id, action, user.user_id, repo_name, ip_addr) |
|
|||
324 |
|
319 | |||
325 | # _authenticate is a wrapper for .auth() method of plugin. |
|
320 | @region.cache_on_arguments(namespace=cache_namespace_uid, | |
326 | # it checks if .auth() sends proper data. |
|
321 | expiration_time=cache_ttl, | |
327 | # For RhodeCodeExternalAuthPlugin it also maps users to |
|
322 | should_cache_fn=lambda v: plugin_cache_active) | |
328 | # Database and maps the attributes returned from .auth() |
|
323 | def compute_perm_vcs( | |
329 | # to RhodeCode database. If this function returns data |
|
324 | cache_name, plugin_id, action, user_id, repo_name, ip_addr): | |
330 | # then auth is correct. |
|
|||
331 | start = time.time() |
|
|||
332 | log.debug('Running plugin `%s` permissions check', plugin_id) |
|
|||
333 |
|
325 | |||
334 | def perm_func(): |
|
|||
335 | """ |
|
|||
336 | This function is used internally in Cache of Beaker to calculate |
|
|||
337 | Results |
|
|||
338 | """ |
|
|||
339 | log.debug('auth: calculating permission access now...') |
|
326 | log.debug('auth: calculating permission access now...') | |
340 | # check IP |
|
327 | # check IP | |
341 | inherit = user.inherit_default_permissions |
|
328 | inherit = user.inherit_default_permissions | |
342 | ip_allowed = AuthUser.check_ip_allowed( |
|
329 | ip_allowed = AuthUser.check_ip_allowed( | |
343 |
|
|
330 | user_id, ip_addr, inherit_from_default=inherit) | |
344 | if ip_allowed: |
|
331 | if ip_allowed: | |
345 | log.info('Access for IP:%s allowed', ip_addr) |
|
332 | log.info('Access for IP:%s allowed', ip_addr) | |
346 | else: |
|
333 | else: | |
@@ -360,12 +347,13 b' class SimpleVCS(object):' | |||||
360 |
|
347 | |||
361 | return True |
|
348 | return True | |
362 |
|
349 | |||
363 | if plugin_cache_active: |
|
350 | start = time.time() | |
364 | log.debug('Trying to fetch cached perms by %s', _perm_calc_hash[:6]) |
|
351 | log.debug('Running plugin `%s` permissions check', plugin_id) | |
365 | perm_result = cache_manager.get( |
|
352 | ||
366 | _perm_calc_hash, createfunc=perm_func) |
|
353 | # for environ based auth, password can be empty, but then the validation is | |
367 | else: |
|
354 | # on the server that fills in the env data needed for authentication | |
368 |
|
|
355 | perm_result = compute_perm_vcs( | |
|
356 | 'vcs_permissions', plugin_id, action, user.user_id, repo_name, ip_addr) | |||
369 |
|
357 | |||
370 | auth_time = time.time() - start |
|
358 | auth_time = time.time() - start | |
371 | log.debug('Permissions for plugin `%s` completed in %.3fs, ' |
|
359 | log.debug('Permissions for plugin `%s` completed in %.3fs, ' |
@@ -119,6 +119,8 b' function registerRCRoutes() {' | |||||
119 | pyroutes.register('edit_user_groups_management', '/_admin/users/%(user_id)s/edit/groups_management', ['user_id']); |
|
119 | pyroutes.register('edit_user_groups_management', '/_admin/users/%(user_id)s/edit/groups_management', ['user_id']); | |
120 | pyroutes.register('edit_user_groups_management_updates', '/_admin/users/%(user_id)s/edit/edit_user_groups_management/updates', ['user_id']); |
|
120 | pyroutes.register('edit_user_groups_management_updates', '/_admin/users/%(user_id)s/edit/edit_user_groups_management/updates', ['user_id']); | |
121 | pyroutes.register('edit_user_audit_logs', '/_admin/users/%(user_id)s/edit/audit', ['user_id']); |
|
121 | pyroutes.register('edit_user_audit_logs', '/_admin/users/%(user_id)s/edit/audit', ['user_id']); | |
|
122 | pyroutes.register('edit_user_caches', '/_admin/users/%(user_id)s/edit/caches', ['user_id']); | |||
|
123 | pyroutes.register('edit_user_caches_update', '/_admin/users/%(user_id)s/edit/caches/update', ['user_id']); | |||
122 | pyroutes.register('user_groups', '/_admin/user_groups', []); |
|
124 | pyroutes.register('user_groups', '/_admin/user_groups', []); | |
123 | pyroutes.register('user_groups_data', '/_admin/user_groups_data', []); |
|
125 | pyroutes.register('user_groups_data', '/_admin/user_groups_data', []); | |
124 | pyroutes.register('user_groups_new', '/_admin/user_groups/new', []); |
|
126 | pyroutes.register('user_groups_new', '/_admin/user_groups/new', []); |
@@ -45,6 +45,7 b'' | |||||
45 | <li class="${'active' if c.active=='ips' else ''}"><a href="${h.route_path('edit_user_ips', user_id=c.user.user_id)}">${_('Ip Whitelist')}</a></li> |
|
45 | <li class="${'active' if c.active=='ips' else ''}"><a href="${h.route_path('edit_user_ips', user_id=c.user.user_id)}">${_('Ip Whitelist')}</a></li> | |
46 | <li class="${'active' if c.active=='groups' else ''}"><a href="${h.route_path('edit_user_groups_management', user_id=c.user.user_id)}">${_('User Groups Management')}</a></li> |
|
46 | <li class="${'active' if c.active=='groups' else ''}"><a href="${h.route_path('edit_user_groups_management', user_id=c.user.user_id)}">${_('User Groups Management')}</a></li> | |
47 | <li class="${'active' if c.active=='audit' else ''}"><a href="${h.route_path('edit_user_audit_logs', user_id=c.user.user_id)}">${_('Audit logs')}</a></li> |
|
47 | <li class="${'active' if c.active=='audit' else ''}"><a href="${h.route_path('edit_user_audit_logs', user_id=c.user.user_id)}">${_('Audit logs')}</a></li> | |
|
48 | <li class="${'active' if c.active=='caches' else ''}"><a href="${h.route_path('edit_user_caches', user_id=c.user.user_id)}">${_('Caches')}</a></li> | |||
48 | </ul> |
|
49 | </ul> | |
49 | </div> |
|
50 | </div> | |
50 |
|
51 |
@@ -22,36 +22,24 b' import time' | |||||
22 |
|
22 | |||
23 | import pytest |
|
23 | import pytest | |
24 |
|
24 | |||
25 |
from rhodecode.lib import cache |
|
25 | from rhodecode.lib import rc_cache | |
26 | from rhodecode.lib.memory_lru_debug import MemoryLRUNamespaceManagerBase |
|
|||
27 |
|
26 | |||
28 |
|
27 | |||
29 | class TestCacheManager(object): |
|
28 | @pytest.mark.usefixtures( 'app') | |
|
29 | class TestCaches(object): | |||
30 |
|
30 | |||
31 | @pytest.mark.parametrize('repo_name', [ |
|
31 | def test_cache_decorator_init_not_configured(self): | |
32 | ('',), |
|
32 | with pytest.raises(EnvironmentError): | |
33 | (u'',), |
|
33 | rc_cache.get_or_create_region('dontexist') | |
34 | (u'ac',), |
|
34 | ||
35 | ('ac', ), |
|
35 | @pytest.mark.parametrize('region_name', [ | |
36 | (u'ΔΔc',), |
|
36 | 'cache_perms', u'cache_perms', | |
37 | ('Δ ac',), |
|
|||
38 | ]) |
|
37 | ]) | |
39 |
def test_cache_ |
|
38 | def test_cache_decorator_init(self, region_name): | |
40 | cache_manager_instance = caches.get_cache_manager( |
|
39 | namespace = region_name | |
41 | repo_name, 'my_cache') |
|
40 | cache_region = rc_cache.get_or_create_region( | |
42 | assert cache_manager_instance |
|
41 | region_name, region_namespace=namespace) | |
43 |
|
42 | assert cache_region | ||
44 | def test_cache_manager_init_undefined_namespace(self): |
|
|||
45 | cache_manager_instance = caches.get_cache_manager( |
|
|||
46 | 'repo_cache_long_undefined', 'my_cache') |
|
|||
47 | assert cache_manager_instance |
|
|||
48 |
|
||||
49 | def_config = caches.DEFAULT_CACHE_MANAGER_CONFIG.copy() |
|
|||
50 | def_config.pop('type') |
|
|||
51 | assert cache_manager_instance.nsargs == def_config |
|
|||
52 |
|
||||
53 | assert isinstance( |
|
|||
54 | cache_manager_instance.namespace, MemoryLRUNamespaceManagerBase) |
|
|||
55 |
|
43 | |||
56 | @pytest.mark.parametrize('example_input', [ |
|
44 | @pytest.mark.parametrize('example_input', [ | |
57 | ('',), |
|
45 | ('',), | |
@@ -62,45 +50,60 b' class TestCacheManager(object):' | |||||
62 | (u'/ac', ), |
|
50 | (u'/ac', ), | |
63 | ]) |
|
51 | ]) | |
64 | def test_cache_manager_create_key(self, example_input): |
|
52 | def test_cache_manager_create_key(self, example_input): | |
65 | key = caches.compute_key_from_params(*example_input) |
|
53 | key = rc_cache.utils.compute_key_from_params(*example_input) | |
66 | assert key |
|
54 | assert key | |
67 |
|
55 | |||
68 | def test_store_and_invalidate_value_from_manager(self): |
|
56 | @pytest.mark.parametrize('example_namespace', [ | |
69 | cache_manger_instance = caches.get_cache_manager( |
|
57 | 'namespace', None | |
70 | 'repo_cache_long', 'my_cache_store') |
|
58 | ]) | |
|
59 | @pytest.mark.parametrize('example_input', [ | |||
|
60 | ('',), | |||
|
61 | (u'/ac',), | |||
|
62 | (u'/ac', 1, 2, object()), | |||
|
63 | (u'/ΔΔc', 1, 2, object()), | |||
|
64 | ('/Δ ac',), | |||
|
65 | (u'/ac', ), | |||
|
66 | ]) | |||
|
67 | def test_cache_keygen(self, example_input, example_namespace): | |||
|
68 | def func_wrapped(): | |||
|
69 | return 1 | |||
|
70 | func = rc_cache.utils.key_generator(example_namespace, func_wrapped) | |||
|
71 | key = func(*example_input) | |||
|
72 | assert key | |||
71 |
|
73 | |||
72 | def compute(): |
|
74 | def test_store_value_in_cache(self): | |
|
75 | cache_region = rc_cache.get_or_create_region('cache_perms') | |||
|
76 | # make sure we empty the cache now | |||
|
77 | for key in cache_region.backend.list_keys(): | |||
|
78 | cache_region.delete(key) | |||
|
79 | ||||
|
80 | assert cache_region.backend.list_keys() == [] | |||
|
81 | ||||
|
82 | @cache_region.cache_on_arguments(expiration_time=5) | |||
|
83 | def compute(key): | |||
73 | return time.time() |
|
84 | return time.time() | |
74 |
|
85 | |||
75 | added_keys = [] |
|
86 | for x in range(10): | |
76 | for i in xrange(3): |
|
87 | compute(x) | |
77 | _cache_key = caches.compute_key_from_params('foo_bar', 'p%s' % i) |
|
88 | ||
78 | added_keys.append(_cache_key) |
|
89 | assert len(set(cache_region.backend.list_keys())) == 10 | |
79 | for x in xrange(10): |
|
|||
80 | cache_manger_instance.get(_cache_key, createfunc=compute) |
|
|||
81 |
|
90 | |||
82 | for key in added_keys: |
|
91 | def test_store_and_get_value_from_region(self): | |
83 | assert cache_manger_instance[key] |
|
92 | cache_region = rc_cache.get_or_create_region('cache_perms') | |
84 |
|
93 | # make sure we empty the cache now | ||
85 | caches.clear_cache_manager(cache_manger_instance) |
|
94 | for key in cache_region.backend.list_keys(): | |
86 |
|
95 | cache_region.delete(key) | ||
87 | for key in added_keys: |
|
96 | assert cache_region.backend.list_keys() == [] | |
88 | assert key not in cache_manger_instance |
|
|||
89 | assert len(cache_manger_instance.namespace.keys()) == 0 |
|
|||
90 |
|
97 | |||
91 | def test_store_and_get_value_from_manager(self): |
|
98 | @cache_region.cache_on_arguments(expiration_time=5) | |
92 | cache_manger_instance = caches.get_cache_manager( |
|
99 | def compute(key): | |
93 | 'repo_cache_long', 'my_cache_store') |
|
100 | return time.time() | |
94 |
|
||||
95 | _cache_key = caches.compute_key_from_params('foo_bar', 'multicall') |
|
|||
96 |
|
101 | |||
97 | def compute(): |
|
102 | result = set() | |
98 | return time.time() |
|
103 | for x in range(10): | |
|
104 | ret = compute('x') | |||
|
105 | result.add(ret) | |||
99 |
|
106 | |||
100 | result = set() |
|
107 | # once computed we have only one value (the same from cache) | |
101 | for x in xrange(10): |
|
108 | # after executing it 10x | |
102 | ret = cache_manger_instance.get(_cache_key, createfunc=compute) |
|
109 | assert len(result) == 1 | |
103 | result.add(ret) |
|
|||
104 |
|
||||
105 | # once computed we have only one value after executing it 10x |
|
|||
106 | assert len(result) == 1 |
|
@@ -292,11 +292,7 b' cache_dir = %(here)s/data' | |||||
292 | beaker.cache.data_dir = %(here)s/rc/data/cache/beaker_data |
|
292 | beaker.cache.data_dir = %(here)s/rc/data/cache/beaker_data | |
293 | beaker.cache.lock_dir = %(here)s/rc/data/cache/beaker_lock |
|
293 | beaker.cache.lock_dir = %(here)s/rc/data/cache/beaker_lock | |
294 |
|
294 | |||
295 |
beaker.cache.regions = |
|
295 | beaker.cache.regions = long_term, sql_cache_short, repo_cache_long | |
296 |
|
||||
297 | beaker.cache.short_term.type = file |
|
|||
298 | beaker.cache.short_term.expire = 0 |
|
|||
299 | beaker.cache.short_term.key_length = 256 |
|
|||
300 |
|
296 | |||
301 | beaker.cache.long_term.type = memory |
|
297 | beaker.cache.long_term.type = memory | |
302 | beaker.cache.long_term.expire = 36000 |
|
298 | beaker.cache.long_term.expire = 36000 | |
@@ -306,16 +302,6 b' beaker.cache.sql_cache_short.type = memo' | |||||
306 | beaker.cache.sql_cache_short.expire = 1 |
|
302 | beaker.cache.sql_cache_short.expire = 1 | |
307 | beaker.cache.sql_cache_short.key_length = 256 |
|
303 | beaker.cache.sql_cache_short.key_length = 256 | |
308 |
|
304 | |||
309 | ## default is memory cache, configure only if required |
|
|||
310 | ## using multi-node or multi-worker setup |
|
|||
311 | #beaker.cache.auth_plugins.type = memory |
|
|||
312 | #beaker.cache.auth_plugins.lock_dir = %(here)s/data/cache/auth_plugin_lock |
|
|||
313 | #beaker.cache.auth_plugins.url = postgresql://postgres:secret@localhost/rhodecode |
|
|||
314 | #beaker.cache.auth_plugins.url = mysql://root:secret@127.0.0.1/rhodecode |
|
|||
315 | #beaker.cache.auth_plugins.sa.pool_recycle = 3600 |
|
|||
316 | #beaker.cache.auth_plugins.sa.pool_size = 10 |
|
|||
317 | #beaker.cache.auth_plugins.sa.max_overflow = 0 |
|
|||
318 |
|
||||
319 | beaker.cache.repo_cache_long.type = memorylru_base |
|
305 | beaker.cache.repo_cache_long.type = memorylru_base | |
320 | beaker.cache.repo_cache_long.max_items = 4096 |
|
306 | beaker.cache.repo_cache_long.max_items = 4096 | |
321 | beaker.cache.repo_cache_long.expire = 2592000 |
|
307 | beaker.cache.repo_cache_long.expire = 2592000 | |
@@ -327,6 +313,16 b' beaker.cache.repo_cache_long.expire = 25' | |||||
327 | #beaker.cache.repo_cache_long.expire = 1209600 |
|
313 | #beaker.cache.repo_cache_long.expire = 1209600 | |
328 | #beaker.cache.repo_cache_long.key_length = 256 |
|
314 | #beaker.cache.repo_cache_long.key_length = 256 | |
329 |
|
315 | |||
|
316 | ||||
|
317 | ##################################### | |||
|
318 | ### DOGPILE CACHE #### | |||
|
319 | ##################################### | |||
|
320 | ||||
|
321 | ## permission tree cache settings | |||
|
322 | rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace | |||
|
323 | rc_cache.cache_perms.expiration_time = 0 | |||
|
324 | rc_cache.cache_perms.arguments.filename = /tmp/rc_cache_1 | |||
|
325 | ||||
330 | #################################### |
|
326 | #################################### | |
331 | ### BEAKER SESSION #### |
|
327 | ### BEAKER SESSION #### | |
332 | #################################### |
|
328 | #################################### |
General Comments 0
You need to be logged in to leave comments.
Login now