diff --git a/configs/development.ini b/configs/development.ini --- a/configs/development.ini +++ b/configs/development.ini @@ -308,47 +308,72 @@ celery.max_tasks_per_child = 100 ## tasks will never be sent to the queue, but executed locally instead. celery.task_always_eager = false +##################################### +### DOGPILE CACHE #### +##################################### +## Default cache dir for caches. Putting this into a ramdisk +## can boost performance, eg. /tmpfs/data_ramdisk, however this might require lots +## of space +cache_dir = /tmp/rcdev/data + +## cache settings for permission tree, auth TTL. +rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace +rc_cache.cache_perms.expiration_time = 300 +rc_cache.cache_perms.arguments.filename = /tmp/rc_cache_1 + +## redis backend with distributed locks +#rc_cache.cache_perms.backend = dogpile.cache.rc.redis +#rc_cache.cache_perms.expiration_time = 300 +#rc_cache.cache_perms.arguments.host = localhost +#rc_cache.cache_perms.arguments.port = 6379 +#rc_cache.cache_perms.arguments.db = 0 +#rc_cache.cache_perms.arguments.redis_expiration_time = 7200 +#rc_cache.cache_perms.arguments.distributed_lock = true + + +rc_cache.cache_repo.backend = dogpile.cache.rc.file_namespace +rc_cache.cache_repo.expiration_time = 2592000 +rc_cache.cache_repo.arguments.filename = /tmp/rc_cache_2 + +## redis backend with distributed locks +#rc_cache.cache_repo.backend = dogpile.cache.rc.redis +#rc_cache.cache_repo.expiration_time = 2592000 +## this needs to be greater then expiration_time +#rc_cache.cache_repo.arguments.redis_expiration_time = 2678400 +#rc_cache.cache_repo.arguments.host = localhost +#rc_cache.cache_repo.arguments.port = 6379 +#rc_cache.cache_repo.arguments.db = 1 +#rc_cache.cache_repo.arguments.distributed_lock = true + + #################################### ### BEAKER CACHE #### #################################### -# default cache dir for templates. Putting this into a ramdisk -## can boost performance, eg. %(here)s/data_ramdisk -cache_dir = %(here)s/data ## locking and default file storage for Beaker. Putting this into a ramdisk ## can boost performance, eg. %(here)s/data_ramdisk/cache/beaker_data beaker.cache.data_dir = %(here)s/data/cache/beaker_data beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock -beaker.cache.regions = long_term, sql_cache_short, repo_cache_long +beaker.cache.regions = long_term, sql_cache_short -beaker.cache.long_term.type = memory -beaker.cache.long_term.expire = 36000 +beaker.cache.long_term.type = memorylru_base +beaker.cache.long_term.expire = 172800 beaker.cache.long_term.key_length = 256 -beaker.cache.sql_cache_short.type = memory +beaker.cache.sql_cache_short.type = memorylru_base beaker.cache.sql_cache_short.expire = 10 beaker.cache.sql_cache_short.key_length = 256 -beaker.cache.repo_cache_long.type = memorylru_base -beaker.cache.repo_cache_long.max_items = 4096 -beaker.cache.repo_cache_long.expire = 2592000 - -## default is memorylru_base cache, configure only if required -## using multi-node or multi-worker setup -#beaker.cache.repo_cache_long.type = ext:memcached -#beaker.cache.repo_cache_long.url = localhost:11211 -#beaker.cache.repo_cache_long.expire = 1209600 -#beaker.cache.repo_cache_long.key_length = 256 #################################### ### BEAKER SESSION #### #################################### ## .session.type is type of storage options for the session, current allowed -## types are file, ext:memcached, ext:database, and memory (default). +## types are file, ext:memcached, ext:redis, ext:database, and memory (default). beaker.session.type = file -beaker.session.data_dir = %(here)s/data/sessions/data +beaker.session.data_dir = %(here)s/data/sessions ## db based session, fast, and allows easy management over logged in users #beaker.session.type = ext:database @@ -496,6 +521,8 @@ debug_style = true #sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30 #sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode #sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode +#sqlalchemy.db1.url = mysql+pymysql://root:qweqwe@localhost/rhodecode + sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30 # see sqlalchemy docs for other advanced settings @@ -515,6 +542,9 @@ sqlalchemy.db1.convert_unicode = true ## which defaults to five. #sqlalchemy.db1.max_overflow = 10 +## Connection check ping, used to detect broken database connections +## could be enabled to better handle cases if MySQL has gone away errors +#sqlalchemy.db1.ping_connection = true ################## ### VCS CONFIG ### @@ -624,7 +654,7 @@ custom.conf = 1 ### LOGGING CONFIGURATION #### ################################ [loggers] -keys = root, sqlalchemy, beaker, rhodecode, ssh_wrapper, celery +keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper [handlers] keys = console, console_sql @@ -680,9 +710,12 @@ level = DEBUG formatter = color_formatter [handler_console_sql] +# "level = DEBUG" logs SQL queries and results. +# "level = INFO" logs SQL queries. +# "level = WARN" logs neither. (Recommended for production systems.) class = StreamHandler args = (sys.stderr, ) -level = DEBUG +level = WARN formatter = color_formatter_sql ################ diff --git a/configs/production.ini b/configs/production.ini --- a/configs/production.ini +++ b/configs/production.ini @@ -283,47 +283,72 @@ celery.max_tasks_per_child = 100 ## tasks will never be sent to the queue, but executed locally instead. celery.task_always_eager = false +##################################### +### DOGPILE CACHE #### +##################################### +## Default cache dir for caches. Putting this into a ramdisk +## can boost performance, eg. /tmpfs/data_ramdisk, however this might require lots +## of space +cache_dir = /tmp/rcdev/data + +## cache settings for permission tree, auth TTL. +rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace +rc_cache.cache_perms.expiration_time = 300 +rc_cache.cache_perms.arguments.filename = /tmp/rc_cache_1 + +## redis backend with distributed locks +#rc_cache.cache_perms.backend = dogpile.cache.rc.redis +#rc_cache.cache_perms.expiration_time = 300 +#rc_cache.cache_perms.arguments.host = localhost +#rc_cache.cache_perms.arguments.port = 6379 +#rc_cache.cache_perms.arguments.db = 0 +#rc_cache.cache_perms.arguments.redis_expiration_time = 7200 +#rc_cache.cache_perms.arguments.distributed_lock = true + + +rc_cache.cache_repo.backend = dogpile.cache.rc.file_namespace +rc_cache.cache_repo.expiration_time = 2592000 +rc_cache.cache_repo.arguments.filename = /tmp/rc_cache_2 + +## redis backend with distributed locks +#rc_cache.cache_repo.backend = dogpile.cache.rc.redis +#rc_cache.cache_repo.expiration_time = 2592000 +## this needs to be greater then expiration_time +#rc_cache.cache_repo.arguments.redis_expiration_time = 2678400 +#rc_cache.cache_repo.arguments.host = localhost +#rc_cache.cache_repo.arguments.port = 6379 +#rc_cache.cache_repo.arguments.db = 1 +#rc_cache.cache_repo.arguments.distributed_lock = true + + #################################### ### BEAKER CACHE #### #################################### -# default cache dir for templates. Putting this into a ramdisk -## can boost performance, eg. %(here)s/data_ramdisk -cache_dir = %(here)s/data ## locking and default file storage for Beaker. Putting this into a ramdisk ## can boost performance, eg. %(here)s/data_ramdisk/cache/beaker_data beaker.cache.data_dir = %(here)s/data/cache/beaker_data beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock -beaker.cache.regions = long_term, sql_cache_short, repo_cache_long +beaker.cache.regions = long_term, sql_cache_short beaker.cache.long_term.type = memory -beaker.cache.long_term.expire = 36000 +beaker.cache.long_term.expire = 172800 beaker.cache.long_term.key_length = 256 beaker.cache.sql_cache_short.type = memory beaker.cache.sql_cache_short.expire = 10 beaker.cache.sql_cache_short.key_length = 256 -beaker.cache.repo_cache_long.type = memorylru_base -beaker.cache.repo_cache_long.max_items = 4096 -beaker.cache.repo_cache_long.expire = 2592000 - -## default is memorylru_base cache, configure only if required -## using multi-node or multi-worker setup -#beaker.cache.repo_cache_long.type = ext:memcached -#beaker.cache.repo_cache_long.url = localhost:11211 -#beaker.cache.repo_cache_long.expire = 1209600 -#beaker.cache.repo_cache_long.key_length = 256 #################################### ### BEAKER SESSION #### #################################### ## .session.type is type of storage options for the session, current allowed -## types are file, ext:memcached, ext:database, and memory (default). +## types are file, ext:memcached, ext:redis, ext:database, and memory (default). beaker.session.type = file -beaker.session.data_dir = %(here)s/data/sessions/data +beaker.session.data_dir = %(here)s/data/sessions ## db based session, fast, and allows easy management over logged in users #beaker.session.type = ext:database @@ -466,6 +491,8 @@ set debug = false #sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30 #sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode #sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode +#sqlalchemy.db1.url = mysql+pymysql://root:qweqwe@localhost/rhodecode + sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode # see sqlalchemy docs for other advanced settings @@ -485,6 +512,9 @@ sqlalchemy.db1.convert_unicode = true ## which defaults to five. #sqlalchemy.db1.max_overflow = 10 +## Connection check ping, used to detect broken database connections +## could be enabled to better handle cases if MySQL has gone away errors +#sqlalchemy.db1.ping_connection = true ################## ### VCS CONFIG ### @@ -593,7 +623,7 @@ custom.conf = 1 ### LOGGING CONFIGURATION #### ################################ [loggers] -keys = root, sqlalchemy, beaker, rhodecode, ssh_wrapper, celery +keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper [handlers] keys = console, console_sql @@ -649,6 +679,9 @@ level = INFO formatter = generic [handler_console_sql] +# "level = DEBUG" logs SQL queries and results. +# "level = INFO" logs SQL queries. +# "level = WARN" logs neither. (Recommended for production systems.) class = StreamHandler args = (sys.stderr, ) level = WARN diff --git a/rhodecode/apps/admin/views/users.py b/rhodecode/apps/admin/views/users.py --- a/rhodecode/apps/admin/views/users.py +++ b/rhodecode/apps/admin/views/users.py @@ -1199,13 +1199,6 @@ class UsersView(UserAppView): return perm_user.permissions - def _get_user_cache_keys(self, cache_namespace_uid, keys): - user_keys = [] - for k in sorted(keys): - if k.startswith(cache_namespace_uid): - user_keys.append(k) - return user_keys - @LoginRequired() @HasPermissionAllDecorator('hg.admin') @view_config( @@ -1222,8 +1215,7 @@ class UsersView(UserAppView): cache_namespace_uid = 'cache_user_auth.{}'.format(self.db_user.user_id) c.region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid) c.backend = c.region.backend - c.user_keys = self._get_user_cache_keys( - cache_namespace_uid, c.region.backend.list_keys()) + c.user_keys = sorted(c.region.backend.list_keys(prefix=cache_namespace_uid)) return self._get_template_context(c) @@ -1241,14 +1233,9 @@ class UsersView(UserAppView): c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr) cache_namespace_uid = 'cache_user_auth.{}'.format(self.db_user.user_id) - c.region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid) + del_keys = rc_cache.clear_cache_namespace('cache_perms', cache_namespace_uid) - c.user_keys = self._get_user_cache_keys( - cache_namespace_uid, c.region.backend.list_keys()) - for k in c.user_keys: - c.region.delete(k) - - h.flash(_("Deleted {} cache keys").format(len(c.user_keys)), category='success') + h.flash(_("Deleted {} cache keys").format(del_keys), category='success') return HTTPFound(h.route_path( 'edit_user_caches', user_id=c.user.user_id)) diff --git a/rhodecode/apps/repository/views/repo_caches.py b/rhodecode/apps/repository/views/repo_caches.py --- a/rhodecode/apps/repository/views/repo_caches.py +++ b/rhodecode/apps/repository/views/repo_caches.py @@ -27,7 +27,7 @@ from pyramid.view import view_config from rhodecode.apps._base import RepoAppView from rhodecode.lib.auth import ( LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired) -from rhodecode.lib import helpers as h +from rhodecode.lib import helpers as h, rc_cache from rhodecode.lib import system_info from rhodecode.model.meta import Session from rhodecode.model.scm import ScmModel @@ -54,6 +54,12 @@ class RepoCachesView(RepoAppView): if os.path.isdir(cached_diffs_dir): c.cached_diff_size = system_info.get_storage_size(cached_diffs_dir) c.shadow_repos = c.rhodecode_db_repo.shadow_repos() + + cache_namespace_uid = 'cache_repo.{}'.format(self.db_repo.repo_id) + c.region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid) + c.backend = c.region.backend + c.repo_keys = sorted(c.region.backend.list_keys(prefix=cache_namespace_uid)) + return self._get_template_context(c) @LoginRequired() @@ -68,7 +74,9 @@ class RepoCachesView(RepoAppView): try: ScmModel().mark_for_invalidation(self.db_repo_name, delete=True) + Session().commit() + h.flash(_('Cache invalidation successful'), category='success') except Exception: diff --git a/rhodecode/apps/repository/views/repo_files.py b/rhodecode/apps/repository/views/repo_files.py --- a/rhodecode/apps/repository/views/repo_files.py +++ b/rhodecode/apps/repository/views/repo_files.py @@ -33,7 +33,7 @@ from pyramid.response import Response from rhodecode.apps._base import RepoAppView from rhodecode.controllers.utils import parse_path_ref -from rhodecode.lib import diffs, helpers as h, caches +from rhodecode.lib import diffs, helpers as h, caches, rc_cache from rhodecode.lib import audit_logger from rhodecode.lib.exceptions import NonRelativePathError from rhodecode.lib.codeblocks import ( @@ -187,32 +187,25 @@ class RepoFilesView(RepoAppView): # check if commit is a branch name or branch hash return commit_id in valid_heads - def _get_tree_cache_manager(self, namespace_type): - _namespace = caches.get_repo_namespace_key( - namespace_type, self.db_repo_name) - return caches.get_cache_manager('repo_cache_long', _namespace) + def _get_tree_at_commit( + self, c, commit_id, f_path, full_load=False): + + repo_id = self.db_repo.repo_id - def _get_tree_at_commit( - self, c, commit_id, f_path, full_load=False, force=False): - def _cached_tree(): - log.debug('Generating cached file tree for %s, %s, %s', - self.db_repo_name, commit_id, f_path) + cache_namespace_uid = 'cache_repo.{}'.format(repo_id) + region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid) + + @region.cache_on_arguments(namespace=cache_namespace_uid) + def compute_file_tree(repo_id, commit_id, f_path, full_load): + log.debug('Generating cached file tree for repo_id: %s, %s, %s', + repo_id, commit_id, f_path) c.full_load = full_load return render( 'rhodecode:templates/files/files_browser_tree.mako', self._get_template_context(c), self.request) - cache_manager = self._get_tree_cache_manager(caches.FILE_TREE) - - cache_key = caches.compute_key_from_params( - self.db_repo_name, commit_id, f_path) - - if force: - # we want to force recompute of caches - cache_manager.remove_value(cache_key) - - return cache_manager.get(cache_key, createfunc=_cached_tree) + return compute_file_tree(self.db_repo.repo_id, commit_id, f_path, full_load) def _get_archive_spec(self, fname): log.debug('Detecting archive spec for: `%s`', fname) @@ -664,12 +657,8 @@ class RepoFilesView(RepoAppView): c.file = dir_node c.commit = commit - # using force=True here, make a little trick. We flush the cache and - # compute it using the same key as without previous full_load, so now - # the fully loaded tree is now returned instead of partial, - # and we store this in caches html = self._get_tree_at_commit( - c, commit.raw_id, dir_node.path, full_load=True, force=True) + c, commit.raw_id, dir_node.path, full_load=True) return Response(html) @@ -784,10 +773,15 @@ class RepoFilesView(RepoAppView): return response - def _get_nodelist_at_commit(self, repo_name, commit_id, f_path): - def _cached_nodes(): - log.debug('Generating cached nodelist for %s, %s, %s', - repo_name, commit_id, f_path) + def _get_nodelist_at_commit(self, repo_name, repo_id, commit_id, f_path): + + cache_namespace_uid = 'cache_repo.{}'.format(repo_id) + region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid) + + @region.cache_on_arguments(namespace=cache_namespace_uid) + def compute_file_search(repo_id, commit_id, f_path): + log.debug('Generating cached nodelist for repo_id:%s, %s, %s', + repo_id, commit_id, f_path) try: _d, _f = ScmModel().get_nodes( repo_name, commit_id, f_path, flat=False) @@ -799,12 +793,7 @@ class RepoFilesView(RepoAppView): commit_id='tip', f_path='/')) return _d + _f - cache_manager = self._get_tree_cache_manager( - caches.FILE_SEARCH_TREE_META) - - cache_key = caches.compute_key_from_params( - repo_name, commit_id, f_path) - return cache_manager.get(cache_key, createfunc=_cached_nodes) + return compute_file_search(self.db_repo.repo_id, commit_id, f_path) @LoginRequired() @HasRepoPermissionAnyDecorator( @@ -819,7 +808,7 @@ class RepoFilesView(RepoAppView): commit = self._get_commit_or_redirect(commit_id) metadata = self._get_nodelist_at_commit( - self.db_repo_name, commit.raw_id, f_path) + self.db_repo_name, self.db_repo.repo_id, commit.raw_id, f_path) return {'nodes': metadata} def _create_references( diff --git a/rhodecode/apps/repository/views/repo_summary.py b/rhodecode/apps/repository/views/repo_summary.py --- a/rhodecode/apps/repository/views/repo_summary.py +++ b/rhodecode/apps/repository/views/repo_summary.py @@ -27,15 +27,14 @@ from beaker.cache import cache_region from rhodecode.controllers import utils from rhodecode.apps._base import RepoAppView from rhodecode.config.conf import (LANGUAGES_EXTENSIONS_MAP) -from rhodecode.lib import caches, helpers as h -from rhodecode.lib.helpers import RepoPage +from rhodecode.lib import helpers as h, rc_cache from rhodecode.lib.utils2 import safe_str, safe_int from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator from rhodecode.lib.markup_renderer import MarkupRenderer, relative_links from rhodecode.lib.ext_json import json from rhodecode.lib.vcs.backends.base import EmptyCommit -from rhodecode.lib.vcs.exceptions import CommitError, EmptyRepositoryError, \ - CommitDoesNotExistError +from rhodecode.lib.vcs.exceptions import ( + CommitError, EmptyRepositoryError, CommitDoesNotExistError) from rhodecode.model.db import Statistics, CacheKey, User from rhodecode.model.meta import Session from rhodecode.model.repo import ReadmeFinder @@ -134,7 +133,7 @@ class RepoSummaryView(RepoAppView): except EmptyRepositoryError: collection = self.rhodecode_vcs_repo - c.repo_commits = RepoPage( + c.repo_commits = h.RepoPage( collection, page=p, items_per_page=size, url=url_generator) page_ids = [x.raw_id for x in c.repo_commits] c.comments = self.db_repo.get_comments(page_ids) @@ -247,16 +246,14 @@ class RepoSummaryView(RepoAppView): renderer='json_ext') def repo_stats(self): commit_id = self.get_request_commit_id() + show_stats = bool(self.db_repo.enable_statistics) + repo_id = self.db_repo.repo_id - _namespace = caches.get_repo_namespace_key( - caches.SUMMARY_STATS, self.db_repo_name) - show_stats = bool(self.db_repo.enable_statistics) - cache_manager = caches.get_cache_manager( - 'repo_cache_long', _namespace) - _cache_key = caches.compute_key_from_params( - self.db_repo_name, commit_id, show_stats) + cache_namespace_uid = 'cache_repo.{}'.format(repo_id) + region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid) - def compute_stats(): + @region.cache_on_arguments(namespace=cache_namespace_uid) + def compute_stats(repo_id, commit_id, show_stats): code_stats = {} size = 0 try: @@ -279,7 +276,7 @@ class RepoSummaryView(RepoAppView): return {'size': h.format_byte_size_binary(size), 'code_stats': code_stats} - stats = cache_manager.get(_cache_key, createfunc=compute_stats) + stats = compute_stats(self.db_repo.repo_id, commit_id, show_stats) return stats @LoginRequired() diff --git a/rhodecode/config/middleware.py b/rhodecode/config/middleware.py --- a/rhodecode/config/middleware.py +++ b/rhodecode/config/middleware.py @@ -435,6 +435,13 @@ def _sanitize_cache_settings(settings): _string_setting(settings, 'rc_cache.cache_perms.arguments.filename', os.path.join(tempfile.gettempdir(), 'rc_cache_1')) + _string_setting(settings, 'rc_cache.cache_repo.backend', + 'dogpile.cache.rc.file_namespace') + _int_setting(settings, 'rc_cache.cache_repo.expiration_time', + 60) + _string_setting(settings, 'rc_cache.cache_repo.arguments.filename', + os.path.join(tempfile.gettempdir(), 'rc_cache_2')) + def _int_setting(settings, name, default): settings[name] = int(settings.get(name, default)) diff --git a/rhodecode/lib/caches.py b/rhodecode/lib/caches.py --- a/rhodecode/lib/caches.py +++ b/rhodecode/lib/caches.py @@ -31,13 +31,6 @@ from rhodecode.model.db import Session, log = logging.getLogger(__name__) -FILE_TREE = 'cache_file_tree' -FILE_TREE_META = 'cache_file_tree_metadata' -FILE_SEARCH_TREE_META = 'cache_file_search_metadata' -SUMMARY_STATS = 'cache_summary_stats' - -# This list of caches gets purged when invalidation happens -USED_REPO_CACHES = (FILE_TREE, FILE_SEARCH_TREE_META) DEFAULT_CACHE_MANAGER_CONFIG = { 'type': 'memorylru_base', @@ -129,14 +122,6 @@ def clear_cache_manager(cache_manager): cache_manager.clear() -def clear_repo_caches(repo_name): - # invalidate cache manager for this repo - for prefix in USED_REPO_CACHES: - namespace = get_repo_namespace_key(prefix, repo_name) - cache_manager = get_cache_manager('repo_cache_long', namespace) - clear_cache_manager(cache_manager) - - def compute_key_from_params(*args): """ Helper to compute key from given params to be used in cache manager @@ -148,60 +133,6 @@ def get_repo_namespace_key(prefix, repo_ return '{0}_{1}'.format(prefix, compute_key_from_params(repo_name)) -def conditional_cache(region, cache_namespace, condition, func): - """ - Conditional caching function use like:: - def _c(arg): - # heavy computation function - return data - - # depending on the condition the compute is wrapped in cache or not - compute = conditional_cache('short_term', 'cache_namespace_id', - condition=True, func=func) - return compute(arg) - - :param region: name of cache region - :param cache_namespace: cache namespace - :param condition: condition for cache to be triggered, and - return data cached - :param func: wrapped heavy function to compute - - """ - wrapped = func - if condition: - log.debug('conditional_cache: True, wrapping call of ' - 'func: %s into %s region cache', region, func) - - def _cache_wrap(region_name, cache_namespace): - """Return a caching wrapper""" - - def decorate(func): - @functools.wraps(func) - def cached(*args, **kwargs): - if kwargs: - raise AttributeError( - 'Usage of kwargs is not allowed. ' - 'Use only positional arguments in wrapped function') - manager = get_cache_manager(region_name, cache_namespace) - cache_key = compute_key_from_params(*args) - - def go(): - return func(*args, **kwargs) - - # save org function name - go.__name__ = '_cached_%s' % (func.__name__,) - - return manager.get(cache_key, createfunc=go) - return cached - - return decorate - - cached_region = _cache_wrap(region, cache_namespace) - wrapped = cached_region(func) - - return wrapped - - class ActiveRegionCache(object): def __init__(self, context): self.context = context diff --git a/rhodecode/lib/rc_cache/__init__.py b/rhodecode/lib/rc_cache/__init__.py --- a/rhodecode/lib/rc_cache/__init__.py +++ b/rhodecode/lib/rc_cache/__init__.py @@ -35,7 +35,9 @@ register_backend( from . import region_meta -from .utils import get_default_cache_settings, key_generator, get_or_create_region +from .utils import ( + get_default_cache_settings, key_generator, get_or_create_region, + clear_cache_namespace) def configure_dogpile_cache(settings): diff --git a/rhodecode/lib/rc_cache/backends.py b/rhodecode/lib/rc_cache/backends.py --- a/rhodecode/lib/rc_cache/backends.py +++ b/rhodecode/lib/rc_cache/backends.py @@ -49,9 +49,18 @@ class FileNamespaceBackend(Serializer, f def __init__(self, arguments): super(FileNamespaceBackend, self).__init__(arguments) - def list_keys(self): + def list_keys(self, prefix=''): + def cond(v): + if not prefix: + return True + + if v.startswith(prefix): + return True + return False + with self._dbm_file(True) as dbm: - return dbm.keys() + + return filter(cond, dbm.keys()) def get_store(self): return self.filename @@ -81,8 +90,10 @@ class FileNamespaceBackend(Serializer, f class RedisPickleBackend(Serializer, redis_backend.RedisBackend): - def list_keys(self): - return self.client.keys() + def list_keys(self, prefix=''): + if prefix: + prefix = prefix + '*' + return self.client.keys(prefix) def get_store(self): return self.client.connection_pool diff --git a/rhodecode/lib/rc_cache/utils.py b/rhodecode/lib/rc_cache/utils.py --- a/rhodecode/lib/rc_cache/utils.py +++ b/rhodecode/lib/rc_cache/utils.py @@ -97,3 +97,11 @@ def get_or_create_region(region_name, re region_obj = region_meta.dogpile_cache_regions[region_namespace] = new_region return region_obj + + +def clear_cache_namespace(cache_region, cache_namespace_uid): + region = get_or_create_region(cache_region, cache_namespace_uid) + cache_keys = region.backend.list_keys(prefix=cache_namespace_uid) + for k in cache_keys: + region.delete(k) + return len(cache_keys) diff --git a/rhodecode/model/scm.py b/rhodecode/model/scm.py --- a/rhodecode/model/scm.py +++ b/rhodecode/model/scm.py @@ -38,7 +38,7 @@ from rhodecode.lib.vcs import get_backen from rhodecode.lib.vcs.exceptions import RepositoryError, NodeNotChangedError from rhodecode.lib.vcs.nodes import FileNode from rhodecode.lib.vcs.backends.base import EmptyCommit -from rhodecode.lib import helpers as h +from rhodecode.lib import helpers as h, rc_cache from rhodecode.lib.auth import ( HasRepoPermissionAny, HasRepoGroupPermissionAny, HasUserGroupPermissionAny) @@ -267,16 +267,19 @@ class ScmModel(BaseModel): :param repo_name: the repo_name for which caches should be marked invalid, or deleted :param delete: delete the entry keys instead of setting bool - flag on them + flag on them, and also purge caches used by the dogpile """ CacheKey.set_invalidate(repo_name, delete=delete) repo = Repository.get_by_repo_name(repo_name) if repo: + repo_id = repo.repo_id config = repo._config config.set('extensions', 'largefiles', '') repo.update_commit_cache(config=config, cs_cache=None) - caches.clear_repo_caches(repo_name) + if delete: + cache_namespace_uid = 'cache_repo.{}'.format(repo_id) + rc_cache.clear_cache_namespace('cache_repo', cache_namespace_uid) def toggle_following_repo(self, follow_repo_id, user_id): diff --git a/rhodecode/templates/admin/repos/repo_edit_caches.mako b/rhodecode/templates/admin/repos/repo_edit_caches.mako --- a/rhodecode/templates/admin/repos/repo_edit_caches.mako +++ b/rhodecode/templates/admin/repos/repo_edit_caches.mako @@ -54,6 +54,41 @@
+ Cache keys used for storing cached values of repository stats, + file tree history and file tree search. + Invalidating the cache will remove those entries. +
++region: ${c.region.name} +backend: ${c.region.actual_backend.__class__} +store: ${c.region.actual_backend.get_store()} + + +% if c.repo_keys: +${len(c.repo_keys)} ${_('Show all')} + +% else: + NO KEYS FOUND +% endif + ++ +
++ ${h.secure_form(h.route_path('edit_user_caches_update', user_id=c.user.user_id), request=request)}+ Cache keys used for storing cached values of user permissions and authentication plugin cache. + Invalidating the cache will remove those entries. +
+ +region: ${c.region.name} backend: ${c.region.actual_backend.__class__} store: ${c.region.actual_backend.get_store()} -% for k in c.user_keys: +% if c.user_keys: +${len(c.user_keys)} ${_('Show all')} +- + % endfor + +% else: + NO KEYS FOUND +% endif +