diff --git a/Makefile b/Makefile --- a/Makefile +++ b/Makefile @@ -145,7 +145,7 @@ dev-srv: .PHONY: dev-srv-g ## run gunicorn multi process workers dev-srv-g: - gunicorn --workers=2 --paste .dev/dev.ini --bind=0.0.0.0:10020 --config=.dev/gunicorn_config.py + gunicorn --paste .dev/dev.ini --bind=0.0.0.0:10020 --config=.dev/gunicorn_config.py --timeout=120 # Default command on calling make diff --git a/rhodecode/apps/_base/subscribers.py b/rhodecode/apps/_base/subscribers.py --- a/rhodecode/apps/_base/subscribers.py +++ b/rhodecode/apps/_base/subscribers.py @@ -44,7 +44,7 @@ def trigger_user_permission_flush(event) for user_id in affected_user_ids: for cache_namespace_uid_tmpl in cache_namespaces: cache_namespace_uid = cache_namespace_uid_tmpl.format(user_id) - del_keys = rc_cache.clear_cache_namespace('cache_perms', cache_namespace_uid, method=rc_cache.CLEAR_INVALIDATE) + del_keys = rc_cache.clear_cache_namespace('cache_perms', cache_namespace_uid, method=rc_cache.CLEAR_DELETE) log.debug('Invalidated %s cache keys for user_id: %s and namespace %s', del_keys, user_id, cache_namespace_uid) diff --git a/rhodecode/apps/admin/views/users.py b/rhodecode/apps/admin/views/users.py --- a/rhodecode/apps/admin/views/users.py +++ b/rhodecode/apps/admin/views/users.py @@ -1313,7 +1313,7 @@ class UsersView(UserAppView): c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr) cache_namespace_uid = f'cache_user_auth.{rc_cache.PERMISSIONS_CACHE_VER}.{self.db_user.user_id}' - del_keys = rc_cache.clear_cache_namespace('cache_perms', cache_namespace_uid) + del_keys = rc_cache.clear_cache_namespace('cache_perms', cache_namespace_uid, method=rc_cache.CLEAR_DELETE) h.flash(_("Deleted {} cache keys").format(del_keys), category='success') diff --git a/rhodecode/lib/rc_cache/backends.py b/rhodecode/lib/rc_cache/backends.py --- a/rhodecode/lib/rc_cache/backends.py +++ b/rhodecode/lib/rc_cache/backends.py @@ -74,10 +74,20 @@ class LRUMemoryBackend(memory_backend.Me # we don't care if key isn't there at deletion pass + def list_keys(self, prefix): + return list(self._cache.keys()) + def delete_multi(self, keys): for key in keys: self.delete(key) + def delete_multi_by_prefix(self, prefix): + cache_keys = self.list_keys(prefix=prefix) + num_affected_keys = len(cache_keys) + if num_affected_keys: + self.delete_multi(cache_keys) + return num_affected_keys + class PickleSerializer: serializer: None | Serializer = staticmethod( # type: ignore @@ -178,6 +188,13 @@ class FileNamespaceBackend(PickleSeriali log.error('Failed to fetch DBM keys from DB: %s', self.get_store()) raise + def delete_multi_by_prefix(self, prefix): + cache_keys = self.list_keys(prefix=prefix) + num_affected_keys = len(cache_keys) + if num_affected_keys: + self.delete_multi(cache_keys) + return num_affected_keys + def get_store(self): return self.filename @@ -227,6 +244,25 @@ class BaseRedisBackend(redis_backend.Red prefix = self._get_keys_pattern(prefix) return self.reader_client.keys(prefix) + def delete_multi_by_prefix(self, prefix, use_lua=False): + if use_lua: + # high efficient LUA script to delete ALL keys by prefix... + lua = """local keys = redis.call('keys', ARGV[1]) + for i=1,#keys,5000 do + redis.call('del', unpack(keys, i, math.min(i+(5000-1), #keys))) + end + return #keys""" + num_affected_keys = self.writer_client.eval( + lua, + 0, + f"{prefix}*") + else: + cache_keys = self.list_keys(prefix=prefix) + num_affected_keys = len(cache_keys) + if num_affected_keys: + self.delete_multi(cache_keys) + return num_affected_keys + def get_store(self): return self.reader_client.connection_pool diff --git a/rhodecode/lib/rc_cache/utils.py b/rhodecode/lib/rc_cache/utils.py --- a/rhodecode/lib/rc_cache/utils.py +++ b/rhodecode/lib/rc_cache/utils.py @@ -232,14 +232,15 @@ def get_or_create_region(region_name, re return region_obj -def clear_cache_namespace(cache_region: str | RhodeCodeCacheRegion, cache_namespace_uid: str, method: str): +def clear_cache_namespace(cache_region: str | RhodeCodeCacheRegion, cache_namespace_uid: str, method: str) -> int: from . import CLEAR_DELETE, CLEAR_INVALIDATE if not isinstance(cache_region, RhodeCodeCacheRegion): cache_region = get_or_create_region(cache_region, cache_namespace_uid) - log.debug('clearing cache region: %s with method=%s', cache_region, method) + log.debug('clearing cache region: %s [prefix:%s] with method=%s', + cache_region, cache_namespace_uid, method) - num_affected_keys = None + num_affected_keys = 0 if method == CLEAR_INVALIDATE: # NOTE: The CacheRegion.invalidate() method’s default mode of @@ -248,10 +249,7 @@ def clear_cache_namespace(cache_region: cache_region.invalidate(hard=True) if method == CLEAR_DELETE: - cache_keys = cache_region.backend.list_keys(prefix=cache_namespace_uid) - num_affected_keys = len(cache_keys) - if num_affected_keys: - cache_region.delete_multi(cache_keys) + num_affected_keys = cache_region.backend.delete_multi_by_prefix(prefix=cache_namespace_uid) return num_affected_keys