##// END OF EJS Templates
caches: rewrite of auth/permission caches to dogpile.
marcink -
r2845:3d5ae486 default
parent child
Show More
@@ -0,0 +1,66
1 # -*- coding: utf-8 -*-
2
3 # Copyright (C) 2015-2018 RhodeCode GmbH
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
21 from dogpile.cache import register_backend
22 from dogpile.cache import make_region
23
24 register_backend(
25 "dogpile.cache.rc.memory_lru", "rhodecode.lib.rc_cache.backends",
26 "LRUMemoryBackend")
27
28 register_backend(
29 "dogpile.cache.rc.file_namespace", "rhodecode.lib.rc_cache.backends",
30 "FileNamespaceBackend")
31
32 register_backend(
33 "dogpile.cache.rc.redis", "rhodecode.lib.rc_cache.backends",
34 "RedisPickleBackend")
35
36
37 from . import region_meta
38 from .utils import get_default_cache_settings, key_generator, get_or_create_region
39
40
41 def configure_dogpile_cache(settings):
42 cache_dir = settings.get('cache_dir')
43 if cache_dir:
44 region_meta.dogpile_config_defaults['cache_dir'] = cache_dir
45
46 rc_cache_data = get_default_cache_settings(settings, prefixes=['rc_cache.'])
47
48 # inspect available namespaces
49 avail_regions = set()
50 for key in rc_cache_data.keys():
51 namespace_name = key.split('.', 1)[0]
52 avail_regions.add(namespace_name)
53
54 # register them into namespace
55 for region_name in avail_regions:
56 new_region = make_region(
57 name=region_name,
58 function_key_generator=key_generator
59 )
60
61 new_region.configure_from_config(settings, 'rc_cache.{}.'.format(region_name))
62 region_meta.dogpile_cache_regions[region_name] = new_region
63
64
65 def includeme(config):
66 configure_dogpile_cache(config.registry.settings)
@@ -0,0 +1,109
1 # -*- coding: utf-8 -*-
2
3 # Copyright (C) 2015-2018 RhodeCode GmbH
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
21 from dogpile.cache.backends import memory as memory_backend
22 from dogpile.cache.backends import file as file_backend
23 from dogpile.cache.backends import redis as redis_backend
24 from dogpile.cache.backends.file import NO_VALUE, compat
25
26 from rhodecode.lib.memory_lru_debug import LRUDict
27
28 _default_max_size = 1024
29
30
31 class LRUMemoryBackend(memory_backend.MemoryBackend):
32
33 def __init__(self, arguments):
34 max_size = arguments.pop('max_size', _default_max_size)
35 arguments['cache_dict'] = LRUDict(max_size)
36 super(LRUMemoryBackend, self).__init__(arguments)
37
38
39 class Serializer(object):
40 def _dumps(self, value):
41 return compat.pickle.dumps(value)
42
43 def _loads(self, value):
44 return compat.pickle.loads(value)
45
46
47 class FileNamespaceBackend(Serializer, file_backend.DBMBackend):
48
49 def __init__(self, arguments):
50 super(FileNamespaceBackend, self).__init__(arguments)
51
52 def list_keys(self):
53 with self._dbm_file(True) as dbm:
54 return dbm.keys()
55
56 def get_store(self):
57 return self.filename
58
59 def get(self, key):
60 with self._dbm_file(False) as dbm:
61 if hasattr(dbm, 'get'):
62 value = dbm.get(key, NO_VALUE)
63 else:
64 # gdbm objects lack a .get method
65 try:
66 value = dbm[key]
67 except KeyError:
68 value = NO_VALUE
69 if value is not NO_VALUE:
70 value = self._loads(value)
71 return value
72
73 def set(self, key, value):
74 with self._dbm_file(True) as dbm:
75 dbm[key] = self._dumps(value)
76
77 def set_multi(self, mapping):
78 with self._dbm_file(True) as dbm:
79 for key, value in mapping.items():
80 dbm[key] = self._dumps(value)
81
82
83 class RedisPickleBackend(Serializer, redis_backend.RedisBackend):
84 def list_keys(self):
85 return self.client.keys()
86
87 def get_store(self):
88 return self.client.connection_pool
89
90 def set(self, key, value):
91 if self.redis_expiration_time:
92 self.client.setex(key, self.redis_expiration_time,
93 self._dumps(value))
94 else:
95 self.client.set(key, self._dumps(value))
96
97 def set_multi(self, mapping):
98 mapping = dict(
99 (k, self._dumps(v))
100 for k, v in mapping.items()
101 )
102
103 if not self.redis_expiration_time:
104 self.client.mset(mapping)
105 else:
106 pipe = self.client.pipeline()
107 for key, value in mapping.items():
108 pipe.setex(key, self.redis_expiration_time, value)
109 pipe.execute()
@@ -0,0 +1,28
1 # -*- coding: utf-8 -*-
2
3 # Copyright (C) 2015-2018 RhodeCode GmbH
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 import os
21 import tempfile
22
23 dogpile_config_defaults = {
24 'cache_dir': os.path.join(tempfile.gettempdir(), 'rc_cache')
25 }
26
27 # GLOBAL TO STORE ALL REGISTERED REGIONS
28 dogpile_cache_regions = {}
@@ -0,0 +1,99
1 # -*- coding: utf-8 -*-
2
3 # Copyright (C) 2015-2018 RhodeCode GmbH
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 import os
21 import logging
22 from dogpile.cache import make_region
23
24 from rhodecode.lib.utils import safe_str, sha1
25 from . import region_meta
26
27 log = logging.getLogger(__name__)
28
29
30 def get_default_cache_settings(settings, prefixes=None):
31 prefixes = prefixes or []
32 cache_settings = {}
33 for key in settings.keys():
34 for prefix in prefixes:
35 if key.startswith(prefix):
36 name = key.split(prefix)[1].strip()
37 val = settings[key]
38 if isinstance(val, basestring):
39 val = val.strip()
40 cache_settings[name] = val
41 return cache_settings
42
43
44 def compute_key_from_params(*args):
45 """
46 Helper to compute key from given params to be used in cache manager
47 """
48 return sha1("_".join(map(safe_str, args)))
49
50
51 def key_generator(namespace, fn):
52 fname = fn.__name__
53
54 def generate_key(*args):
55 namespace_pref = namespace or 'default'
56 arg_key = compute_key_from_params(*args)
57 final_key = "{}:{}_{}".format(namespace_pref, fname, arg_key)
58
59 return final_key
60
61 return generate_key
62
63
64 def get_or_create_region(region_name, region_namespace=None):
65 from rhodecode.lib.rc_cache.backends import FileNamespaceBackend
66 region_obj = region_meta.dogpile_cache_regions.get(region_name)
67 if not region_obj:
68 raise EnvironmentError(
69 'Region `{}` not in configured: {}.'.format(
70 region_name, region_meta.dogpile_cache_regions.keys()))
71
72 region_uid_name = '{}:{}'.format(region_name, region_namespace)
73 if isinstance(region_obj.actual_backend, FileNamespaceBackend):
74 region_exist = region_meta.dogpile_cache_regions.get(region_namespace)
75 if region_exist:
76 log.debug('Using already configured region: %s', region_namespace)
77 return region_exist
78 cache_dir = region_meta.dogpile_config_defaults['cache_dir']
79 expiration_time = region_obj.expiration_time
80
81 if not os.path.isdir(cache_dir):
82 os.makedirs(cache_dir)
83 new_region = make_region(
84 name=region_uid_name, function_key_generator=key_generator
85 )
86 namespace_filename = os.path.join(
87 cache_dir, "{}.cache.dbm".format(region_namespace))
88 # special type that allows 1db per namespace
89 new_region.configure(
90 backend='dogpile.cache.rc.file_namespace',
91 expiration_time=expiration_time,
92 arguments={"filename": namespace_filename}
93 )
94
95 # create and save in region caches
96 log.debug('configuring new region: %s',region_uid_name)
97 region_obj = region_meta.dogpile_cache_regions[region_namespace] = new_region
98
99 return region_obj
@@ -0,0 +1,29
1 <%namespace name="base" file="/base/base.mako"/>
2
3 <div class="panel panel-default">
4 <div class="panel-heading">
5 <h3 class="panel-title">${_('Caches')}</h3>
6 </div>
7 <div class="panel-body">
8 <pre>
9 region: ${c.region.name}
10 backend: ${c.region.actual_backend.__class__}
11 store: ${c.region.actual_backend.get_store()}
12
13 % for k in c.user_keys:
14 - ${k}
15 % endfor
16 </pre>
17
18 ${h.secure_form(h.route_path('edit_user_caches_update', user_id=c.user.user_id), request=request)}
19 <div class="form">
20 <div class="fields">
21 ${h.submit('reset_cache_%s' % c.user.user_id, _('Invalidate user cache'),class_="btn btn-small",onclick="return confirm('"+_('Confirm to invalidate user cache')+"');")}
22 </div>
23 </div>
24 ${h.end_form()}
25
26 </div>
27 </div>
28
29
@@ -320,12 +320,7 cache_dir = %(here)s/data
320 beaker.cache.data_dir = %(here)s/data/cache/beaker_data
320 beaker.cache.data_dir = %(here)s/data/cache/beaker_data
321 beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock
321 beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock
322
322
323 beaker.cache.regions = short_term, long_term, sql_cache_short, auth_plugins, repo_cache_long
323 beaker.cache.regions = long_term, sql_cache_short, repo_cache_long
324
325 # used for caching user permissions
326 beaker.cache.short_term.type = file
327 beaker.cache.short_term.expire = 0
328 beaker.cache.short_term.key_length = 256
329
324
330 beaker.cache.long_term.type = memory
325 beaker.cache.long_term.type = memory
331 beaker.cache.long_term.expire = 36000
326 beaker.cache.long_term.expire = 36000
@@ -335,16 +330,6 beaker.cache.sql_cache_short.type = memo
335 beaker.cache.sql_cache_short.expire = 10
330 beaker.cache.sql_cache_short.expire = 10
336 beaker.cache.sql_cache_short.key_length = 256
331 beaker.cache.sql_cache_short.key_length = 256
337
332
338 ## default is memory cache, configure only if required
339 ## using multi-node or multi-worker setup
340 #beaker.cache.auth_plugins.type = ext:database
341 #beaker.cache.auth_plugins.lock_dir = %(here)s/data/cache/auth_plugin_lock
342 #beaker.cache.auth_plugins.url = postgresql://postgres:secret@localhost/rhodecode
343 #beaker.cache.auth_plugins.url = mysql://root:secret@127.0.0.1/rhodecode
344 #beaker.cache.auth_plugins.sa.pool_recycle = 3600
345 #beaker.cache.auth_plugins.sa.pool_size = 10
346 #beaker.cache.auth_plugins.sa.max_overflow = 0
347
348 beaker.cache.repo_cache_long.type = memorylru_base
333 beaker.cache.repo_cache_long.type = memorylru_base
349 beaker.cache.repo_cache_long.max_items = 4096
334 beaker.cache.repo_cache_long.max_items = 4096
350 beaker.cache.repo_cache_long.expire = 2592000
335 beaker.cache.repo_cache_long.expire = 2592000
@@ -295,12 +295,7 cache_dir = %(here)s/data
295 beaker.cache.data_dir = %(here)s/data/cache/beaker_data
295 beaker.cache.data_dir = %(here)s/data/cache/beaker_data
296 beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock
296 beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock
297
297
298 beaker.cache.regions = short_term, long_term, sql_cache_short, auth_plugins, repo_cache_long
298 beaker.cache.regions = long_term, sql_cache_short, repo_cache_long
299
300 # used for caching user permissions
301 beaker.cache.short_term.type = file
302 beaker.cache.short_term.expire = 0
303 beaker.cache.short_term.key_length = 256
304
299
305 beaker.cache.long_term.type = memory
300 beaker.cache.long_term.type = memory
306 beaker.cache.long_term.expire = 36000
301 beaker.cache.long_term.expire = 36000
@@ -310,16 +305,6 beaker.cache.sql_cache_short.type = memo
310 beaker.cache.sql_cache_short.expire = 10
305 beaker.cache.sql_cache_short.expire = 10
311 beaker.cache.sql_cache_short.key_length = 256
306 beaker.cache.sql_cache_short.key_length = 256
312
307
313 ## default is memory cache, configure only if required
314 ## using multi-node or multi-worker setup
315 #beaker.cache.auth_plugins.type = ext:database
316 #beaker.cache.auth_plugins.lock_dir = %(here)s/data/cache/auth_plugin_lock
317 #beaker.cache.auth_plugins.url = postgresql://postgres:secret@localhost/rhodecode
318 #beaker.cache.auth_plugins.url = mysql://root:secret@127.0.0.1/rhodecode
319 #beaker.cache.auth_plugins.sa.pool_recycle = 3600
320 #beaker.cache.auth_plugins.sa.pool_size = 10
321 #beaker.cache.auth_plugins.sa.max_overflow = 0
322
323 beaker.cache.repo_cache_long.type = memorylru_base
308 beaker.cache.repo_cache_long.type = memorylru_base
324 beaker.cache.repo_cache_long.max_items = 4096
309 beaker.cache.repo_cache_long.max_items = 4096
325 beaker.cache.repo_cache_long.expire = 2592000
310 beaker.cache.repo_cache_long.expire = 2592000
@@ -356,6 +356,16 def admin_routes(config):
356 name='edit_user_audit_logs',
356 name='edit_user_audit_logs',
357 pattern='/users/{user_id:\d+}/edit/audit', user_route=True)
357 pattern='/users/{user_id:\d+}/edit/audit', user_route=True)
358
358
359 # user caches
360 config.add_route(
361 name='edit_user_caches',
362 pattern='/users/{user_id:\d+}/edit/caches',
363 user_route=True)
364 config.add_route(
365 name='edit_user_caches_update',
366 pattern='/users/{user_id:\d+}/edit/caches/update',
367 user_route=True)
368
359 # user-groups admin
369 # user-groups admin
360 config.add_route(
370 config.add_route(
361 name='user_groups',
371 name='user_groups',
@@ -34,7 +34,7 from rhodecode.authentication.plugins im
34 from rhodecode.events import trigger
34 from rhodecode.events import trigger
35 from rhodecode.model.db import true
35 from rhodecode.model.db import true
36
36
37 from rhodecode.lib import audit_logger
37 from rhodecode.lib import audit_logger, rc_cache
38 from rhodecode.lib.exceptions import (
38 from rhodecode.lib.exceptions import (
39 UserCreationError, UserOwnsReposException, UserOwnsRepoGroupsException,
39 UserCreationError, UserOwnsReposException, UserOwnsRepoGroupsException,
40 UserOwnsUserGroupsException, DefaultUserException)
40 UserOwnsUserGroupsException, DefaultUserException)
@@ -1198,3 +1198,57 class UsersView(UserAppView):
1198 perm_user = self.db_user.AuthUser(ip_addr=self.request.remote_addr)
1198 perm_user = self.db_user.AuthUser(ip_addr=self.request.remote_addr)
1199
1199
1200 return perm_user.permissions
1200 return perm_user.permissions
1201
1202 def _get_user_cache_keys(self, cache_namespace_uid, keys):
1203 user_keys = []
1204 for k in sorted(keys):
1205 if k.startswith(cache_namespace_uid):
1206 user_keys.append(k)
1207 return user_keys
1208
1209 @LoginRequired()
1210 @HasPermissionAllDecorator('hg.admin')
1211 @view_config(
1212 route_name='edit_user_caches', request_method='GET',
1213 renderer='rhodecode:templates/admin/users/user_edit.mako')
1214 def user_caches(self):
1215 _ = self.request.translate
1216 c = self.load_default_context()
1217 c.user = self.db_user
1218
1219 c.active = 'caches'
1220 c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
1221
1222 cache_namespace_uid = 'cache_user_auth.{}'.format(self.db_user.user_id)
1223 c.region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
1224 c.backend = c.region.backend
1225 c.user_keys = self._get_user_cache_keys(
1226 cache_namespace_uid, c.region.backend.list_keys())
1227
1228 return self._get_template_context(c)
1229
1230 @LoginRequired()
1231 @HasPermissionAllDecorator('hg.admin')
1232 @CSRFRequired()
1233 @view_config(
1234 route_name='edit_user_caches_update', request_method='POST')
1235 def user_caches_update(self):
1236 _ = self.request.translate
1237 c = self.load_default_context()
1238 c.user = self.db_user
1239
1240 c.active = 'caches'
1241 c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
1242
1243 cache_namespace_uid = 'cache_user_auth.{}'.format(self.db_user.user_id)
1244 c.region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
1245
1246 c.user_keys = self._get_user_cache_keys(
1247 cache_namespace_uid, c.region.backend.list_keys())
1248 for k in c.user_keys:
1249 c.region.delete(k)
1250
1251 h.flash(_("Deleted {} cache keys").format(len(c.user_keys)), category='success')
1252
1253 return HTTPFound(h.route_path(
1254 'edit_user_caches', user_id=c.user.user_id))
@@ -35,7 +35,7 from pyramid.threadlocal import get_curr
35
35
36 from rhodecode.authentication.interface import IAuthnPluginRegistry
36 from rhodecode.authentication.interface import IAuthnPluginRegistry
37 from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase
37 from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase
38 from rhodecode.lib import caches
38 from rhodecode.lib import caches, rc_cache
39 from rhodecode.lib.auth import PasswordGenerator, _RhodeCodeCryptoBCrypt
39 from rhodecode.lib.auth import PasswordGenerator, _RhodeCodeCryptoBCrypt
40 from rhodecode.lib.utils2 import safe_int, safe_str
40 from rhodecode.lib.utils2 import safe_int, safe_str
41 from rhodecode.lib.exceptions import LdapConnectionError
41 from rhodecode.lib.exceptions import LdapConnectionError
@@ -633,22 +633,6 def get_authn_registry(registry=None):
633 return authn_registry
633 return authn_registry
634
634
635
635
636 def get_auth_cache_manager(custom_ttl=None, suffix=None):
637 cache_name = 'rhodecode.authentication'
638 if suffix:
639 cache_name = 'rhodecode.authentication.{}'.format(suffix)
640 return caches.get_cache_manager(
641 'auth_plugins', cache_name, custom_ttl)
642
643
644 def get_perms_cache_manager(custom_ttl=None, suffix=None):
645 cache_name = 'rhodecode.permissions'
646 if suffix:
647 cache_name = 'rhodecode.permissions.{}'.format(suffix)
648 return caches.get_cache_manager(
649 'auth_plugins', cache_name, custom_ttl)
650
651
652 def authenticate(username, password, environ=None, auth_type=None,
636 def authenticate(username, password, environ=None, auth_type=None,
653 skip_missing=False, registry=None, acl_repo_name=None):
637 skip_missing=False, registry=None, acl_repo_name=None):
654 """
638 """
@@ -707,45 +691,35 def authenticate(username, password, env
707
691
708 plugin_cache_active, cache_ttl = plugin.get_ttl_cache(plugin_settings)
692 plugin_cache_active, cache_ttl = plugin.get_ttl_cache(plugin_settings)
709
693
710 # get instance of cache manager configured for a namespace
711 cache_manager = get_auth_cache_manager(
712 custom_ttl=cache_ttl, suffix=user.user_id if user else '')
713
714 log.debug('AUTH_CACHE_TTL for plugin `%s` active: %s (TTL: %s)',
694 log.debug('AUTH_CACHE_TTL for plugin `%s` active: %s (TTL: %s)',
715 plugin.get_id(), plugin_cache_active, cache_ttl)
695 plugin.get_id(), plugin_cache_active, cache_ttl)
716
696
717 # for environ based password can be empty, but then the validation is
697 user_id = user.user_id
718 # on the server that fills in the env data needed for authentication
698 cache_namespace_uid = 'cache_user_auth.{}'.format(user_id)
719
699 region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
720 _password_hash = caches.compute_key_from_params(
721 plugin.name, username, (password or ''))
722
700
723 # _authenticate is a wrapper for .auth() method of plugin.
701 @region.cache_on_arguments(namespace=cache_namespace_uid,
724 # it checks if .auth() sends proper data.
702 expiration_time=cache_ttl,
725 # For RhodeCodeExternalAuthPlugin it also maps users to
703 should_cache_fn=lambda v: plugin_cache_active)
726 # Database and maps the attributes returned from .auth()
704 def compute_auth(
727 # to RhodeCode database. If this function returns data
705 cache_name, plugin_name, username, password):
728 # then auth is correct.
729 start = time.time()
730 log.debug('Running plugin `%s` _authenticate method', plugin.get_id())
731
706
732 def auth_func():
707 # _authenticate is a wrapper for .auth() method of plugin.
733 """
708 # it checks if .auth() sends proper data.
734 This function is used internally in Cache of Beaker to calculate
709 # For RhodeCodeExternalAuthPlugin it also maps users to
735 Results
710 # Database and maps the attributes returned from .auth()
736 """
711 # to RhodeCode database. If this function returns data
737 log.debug('auth: calculating password access now...')
712 # then auth is correct.
713 log.debug('Running plugin `%s` _authenticate method '
714 'using username and password', plugin.get_id())
738 return plugin._authenticate(
715 return plugin._authenticate(
739 user, username, password, plugin_settings,
716 user, username, password, plugin_settings,
740 environ=environ or {})
717 environ=environ or {})
741
718
742 if plugin_cache_active:
719 start = time.time()
743 log.debug('Trying to fetch cached auth by pwd hash `...%s`',
720 # for environ based auth, password can be empty, but then the validation is
744 _password_hash[:6])
721 # on the server that fills in the env data needed for authentication
745 plugin_user = cache_manager.get(
722 plugin_user = compute_auth('auth', plugin.name, username, (password or ''))
746 _password_hash, createfunc=auth_func)
747 else:
748 plugin_user = auth_func()
749
723
750 auth_time = time.time() - start
724 auth_time = time.time() - start
751 log.debug('Authentication for plugin `%s` completed in %.3fs, '
725 log.debug('Authentication for plugin `%s` completed in %.3fs, '
@@ -27,8 +27,7 from pyramid.renderers import render
27 from pyramid.response import Response
27 from pyramid.response import Response
28
28
29 from rhodecode.apps._base import BaseAppView
29 from rhodecode.apps._base import BaseAppView
30 from rhodecode.authentication.base import (
30 from rhodecode.authentication.base import get_authn_registry
31 get_auth_cache_manager, get_perms_cache_manager, get_authn_registry)
32 from rhodecode.lib import helpers as h
31 from rhodecode.lib import helpers as h
33 from rhodecode.lib.auth import (
32 from rhodecode.lib.auth import (
34 LoginRequired, HasPermissionAllDecorator, CSRFRequired)
33 LoginRequired, HasPermissionAllDecorator, CSRFRequired)
@@ -102,16 +101,6 class AuthnPluginViewBase(BaseAppView):
102 self.plugin.create_or_update_setting(name, value)
101 self.plugin.create_or_update_setting(name, value)
103 Session().commit()
102 Session().commit()
104
103
105 # cleanup cache managers in case of change for plugin
106 # TODO(marcink): because we can register multiple namespaces
107 # we should at some point figure out how to retrieve ALL namespace
108 # cache managers and clear them...
109 cache_manager = get_auth_cache_manager()
110 clear_cache_manager(cache_manager)
111
112 cache_manager = get_perms_cache_manager()
113 clear_cache_manager(cache_manager)
114
115 # Display success message and redirect.
104 # Display success message and redirect.
116 h.flash(_('Auth settings updated successfully.'), category='success')
105 h.flash(_('Auth settings updated successfully.'), category='success')
117 redirect_to = self.request.resource_path(
106 redirect_to = self.request.resource_path(
@@ -22,6 +22,7 import os
22 import logging
22 import logging
23 import traceback
23 import traceback
24 import collections
24 import collections
25 import tempfile
25
26
26 from paste.gzipper import make_gzip_middleware
27 from paste.gzipper import make_gzip_middleware
27 from pyramid.wsgi import wsgiapp
28 from pyramid.wsgi import wsgiapp
@@ -220,6 +221,7 def includeme(config):
220 config.include('pyramid_mako')
221 config.include('pyramid_mako')
221 config.include('pyramid_beaker')
222 config.include('pyramid_beaker')
222 config.include('rhodecode.lib.caches')
223 config.include('rhodecode.lib.caches')
224 config.include('rhodecode.lib.rc_cache')
223
225
224 config.include('rhodecode.authentication')
226 config.include('rhodecode.authentication')
225 config.include('rhodecode.integrations')
227 config.include('rhodecode.integrations')
@@ -382,6 +384,7 def sanitize_settings_and_apply_defaults
382 # Call split out functions that sanitize settings for each topic.
384 # Call split out functions that sanitize settings for each topic.
383 _sanitize_appenlight_settings(settings)
385 _sanitize_appenlight_settings(settings)
384 _sanitize_vcs_settings(settings)
386 _sanitize_vcs_settings(settings)
387 _sanitize_cache_settings(settings)
385
388
386 # configure instance id
389 # configure instance id
387 config_utils.set_instance_id(settings)
390 config_utils.set_instance_id(settings)
@@ -421,6 +424,18 def _sanitize_vcs_settings(settings):
421 settings['vcs.scm_app_implementation'] = 'http'
424 settings['vcs.scm_app_implementation'] = 'http'
422
425
423
426
427 def _sanitize_cache_settings(settings):
428 _string_setting(settings, 'cache_dir',
429 os.path.join(tempfile.gettempdir(), 'rc_cache'))
430
431 _string_setting(settings, 'rc_cache.cache_perms.backend',
432 'dogpile.cache.rc.file_namespace')
433 _int_setting(settings, 'rc_cache.cache_perms.expiration_time',
434 60)
435 _string_setting(settings, 'rc_cache.cache_perms.arguments.filename',
436 os.path.join(tempfile.gettempdir(), 'rc_cache_1'))
437
438
424 def _int_setting(settings, name, default):
439 def _int_setting(settings, name, default):
425 settings[name] = int(settings.get(name, default))
440 settings[name] = int(settings.get(name, default))
426
441
@@ -48,7 +48,7 from rhodecode.model.user import UserMod
48 from rhodecode.model.db import (
48 from rhodecode.model.db import (
49 User, Repository, Permission, UserToPerm, UserGroupToPerm, UserGroupMember,
49 User, Repository, Permission, UserToPerm, UserGroupToPerm, UserGroupMember,
50 UserIpMap, UserApiKeys, RepoGroup, UserGroup)
50 UserIpMap, UserApiKeys, RepoGroup, UserGroup)
51 from rhodecode.lib import caches
51 from rhodecode.lib import rc_cache
52 from rhodecode.lib.utils2 import safe_unicode, aslist, safe_str, md5, safe_int, sha1
52 from rhodecode.lib.utils2 import safe_unicode, aslist, safe_str, md5, safe_int, sha1
53 from rhodecode.lib.utils import (
53 from rhodecode.lib.utils import (
54 get_repo_slug, get_repo_group_slug, get_user_group_slug)
54 get_repo_slug, get_repo_group_slug, get_user_group_slug)
@@ -976,21 +976,18 class AuthUser(object):
976 obj = Repository.get_by_repo_name(scope['repo_name'])
976 obj = Repository.get_by_repo_name(scope['repo_name'])
977 if obj:
977 if obj:
978 scope['repo_id'] = obj.repo_id
978 scope['repo_id'] = obj.repo_id
979 _scope = {
979 _scope = collections.OrderedDict()
980 'repo_id': -1,
980 _scope['repo_id'] = -1
981 'user_group_id': -1,
981 _scope['user_group_id'] = -1
982 'repo_group_id': -1,
982 _scope['repo_group_id'] = -1
983 }
983
984 _scope.update(scope)
984 for k in sorted(scope.keys()):
985 cache_key = "_".join(map(safe_str, reduce(lambda a, b: a+b,
985 _scope[k] = scope[k]
986 _scope.items())))
986
987 if cache_key not in self._permissions_scoped_cache:
987 # store in cache to mimic how the @LazyProperty works,
988 # store in cache to mimic how the @LazyProperty works,
988 # the difference here is that we use the unique key calculated
989 # the difference here is that we use the unique key calculated
989 # from params and values
990 # from params and values
990 return self.get_perms(user=self, cache=False, scope=_scope)
991 res = self.get_perms(user=self, cache=False, scope=_scope)
992 self._permissions_scoped_cache[cache_key] = res
993 return self._permissions_scoped_cache[cache_key]
994
991
995 def get_instance(self):
992 def get_instance(self):
996 return User.get(self.user_id)
993 return User.get(self.user_id)
@@ -1072,16 +1069,27 class AuthUser(object):
1072 user_inherit_default_permissions = user.inherit_default_permissions
1069 user_inherit_default_permissions = user.inherit_default_permissions
1073
1070
1074 cache_seconds = safe_int(
1071 cache_seconds = safe_int(
1075 rhodecode.CONFIG.get('beaker.cache.short_term.expire'))
1072 rhodecode.CONFIG.get('rc_cache.cache_perms.expiration_time'))
1073
1076 cache_on = cache or cache_seconds > 0
1074 cache_on = cache or cache_seconds > 0
1077 log.debug(
1075 log.debug(
1078 'Computing PERMISSION tree for user %s scope `%s` '
1076 'Computing PERMISSION tree for user %s scope `%s` '
1079 'with caching: %s[%ss]' % (user, scope, cache_on, cache_seconds))
1077 'with caching: %s[TTL: %ss]' % (user, scope, cache_on, cache_seconds or 0))
1078
1079 cache_namespace_uid = 'cache_user_auth.{}'.format(user_id)
1080 region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
1081
1082 @region.cache_on_arguments(namespace=cache_namespace_uid,
1083 should_cache_fn=lambda v: cache_on)
1084 def compute_perm_tree(cache_name,
1085 user_id, scope, user_is_admin,user_inherit_default_permissions,
1086 explicit, algo, calculate_super_admin):
1087 return _cached_perms_data(
1088 user_id, scope, user_is_admin, user_inherit_default_permissions,
1089 explicit, algo, calculate_super_admin)
1090
1080 start = time.time()
1091 start = time.time()
1081 compute = caches.conditional_cache(
1092 result = compute_perm_tree('permissions', user_id, scope, user_is_admin,
1082 'short_term', 'cache_desc.{}'.format(user_id),
1083 condition=cache_on, func=_cached_perms_data)
1084 result = compute(user_id, scope, user_is_admin,
1085 user_inherit_default_permissions, explicit, algo,
1093 user_inherit_default_permissions, explicit, algo,
1086 calculate_super_admin)
1094 calculate_super_admin)
1087
1095
@@ -1091,6 +1099,7 class AuthUser(object):
1091 total = time.time() - start
1099 total = time.time() - start
1092 log.debug('PERMISSION tree for user %s computed in %.3fs: %s' % (
1100 log.debug('PERMISSION tree for user %s computed in %.3fs: %s' % (
1093 user, total, result_repr))
1101 user, total, result_repr))
1102
1094 return result
1103 return result
1095
1104
1096 @property
1105 @property
@@ -1153,10 +1162,7 class AuthUser(object):
1153 return [x.repo_id for x in
1162 return [x.repo_id for x in
1154 RepoList(qry, perm_set=perm_def)]
1163 RepoList(qry, perm_set=perm_def)]
1155
1164
1156 compute = caches.conditional_cache(
1165 return _cached_repo_acl(self.user_id, perms, name_filter)
1157 'short_term', 'repo_acl_ids.{}'.format(self.user_id),
1158 condition=cache, func=_cached_repo_acl)
1159 return compute(self.user_id, perms, name_filter)
1160