caches: make sure the global cache namespace prefixes are used.

- fixes problems when porting from old RC to new python3 based and different serialization mech

rhodecode/api/views/repo_api.py

@@ -624,10 +624,10 b' def get_repo_fts_tree(request, apiuser, '
     cache_seconds = rhodecode.ConfigGet().get_int('rc_cache.cache_repo.expiration_time')
     cache_on = cache_seconds > 0
 
-    cache_namespace_uid = f'repo.{repo_id}'
+    cache_namespace_uid = f'repo.{rc_cache.FILE_TREE_CACHE_VER}.{repo_id}'
     rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
 
-    def compute_fts_tree(cache_ver, repo_id, commit_id, root_path):
+    def compute_fts_tree(repo_id, commit_id, root_path):
         return ScmModel().get_fts_data(repo_id, commit_id, root_path)
 
     try:
@@ -648,7 +648,7 b' def get_repo_fts_tree(request, apiuser, '
             'with caching: %s[TTL: %ss]' % (
                 repo_id, commit_id, cache_on, cache_seconds or 0))
 
-        tree_files = compute_fts_tree(rc_cache.FILE_TREE_CACHE_VER, repo_id, commit_id, root_path)
+        tree_files = compute_fts_tree(repo_id, commit_id, root_path)
 
         return tree_files
 

rhodecode/apps/_base/subscribers.py

@@ -26,10 +26,10 b' log = logging.getLogger(__name__)'
 # names of namespaces used for different permission related cached
 # during flush operation we need to take care of all those
 cache_namespaces = [
-    'cache_user_auth.{}',
-    'cache_user_repo_acl_ids.{}',
-    'cache_user_user_group_acl_ids.{}',
-    'cache_user_repo_group_acl_ids.{}'
+    f'cache_user_auth.{rc_cache.PERMISSIONS_CACHE_VER}.{{}}',
+    f'cache_user_repo_acl_ids.{rc_cache.PERMISSIONS_CACHE_VER}.{{}}',
+    f'cache_user_user_group_acl_ids.{rc_cache.PERMISSIONS_CACHE_VER}.{{}}',
+    f'cache_user_repo_group_acl_ids.{rc_cache.PERMISSIONS_CACHE_VER}.{{}}'
 ]
 
 

rhodecode/apps/admin/views/users.py

@@ -1294,7 +1294,7 b' class UsersView(UserAppView):'
         c.active = 'caches'
         c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
 
-        cache_namespace_uid = f'cache_user_auth.{self.db_user.user_id}'
+        cache_namespace_uid = f'cache_user_auth.{rc_cache.PERMISSIONS_CACHE_VER}.{self.db_user.user_id}'
         c.region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
         c.backend = c.region.backend
         c.user_keys = sorted(c.region.backend.list_keys(prefix=cache_namespace_uid))
@@ -1312,7 +1312,7 b' class UsersView(UserAppView):'
         c.active = 'caches'
         c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
 
-        cache_namespace_uid = f'cache_user_auth.{self.db_user.user_id}'
+        cache_namespace_uid = f'cache_user_auth.{rc_cache.PERMISSIONS_CACHE_VER}.{self.db_user.user_id}'
         del_keys = rc_cache.clear_cache_namespace('cache_perms', cache_namespace_uid)
 
         h.flash(_("Deleted {} cache keys").format(del_keys), category='success')

rhodecode/apps/repository/views/repo_files.py

@@ -310,13 +310,13 b' class RepoFilesView(RepoAppView):'
             'with caching: %s[TTL: %ss]' % (
                 repo_id, commit_id, f_path, cache_on, cache_seconds or 0))
 
-        cache_namespace_uid = f'repo.{repo_id}'
+        cache_namespace_uid = f'repo.{rc_cache.FILE_TREE_CACHE_VER}.{repo_id}'
         region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid, condition=cache_on)
-        def compute_file_tree(ver, _name_hash, _repo_id, _commit_id, _f_path, _full_load, _at_rev):
+        def compute_file_tree(_name_hash, _repo_id, _commit_id, _f_path, _full_load, _at_rev):
             log.debug('Generating cached file tree at ver:%s for repo_id: %s, %s, %s',
-                      ver, _repo_id, _commit_id, _f_path)
+                      _repo_id, _commit_id, _f_path)
 
             c.full_load = _full_load
             return render(
@@ -324,8 +324,7 b' class RepoFilesView(RepoAppView):'
                 self._get_template_context(c), self.request, _at_rev)
 
         return compute_file_tree(
-            rc_cache.FILE_TREE_CACHE_VER, self.db_repo.repo_name_hash,
-            self.db_repo.repo_id, commit_id, f_path, full_load, at_rev)
+            self.db_repo.repo_name_hash, self.db_repo.repo_id, commit_id, f_path, full_load, at_rev)
 
     def create_pure_path(self, *parts):
         # Split paths and sanitize them, removing any ../ etc

rhodecode/authentication/base.py

@@ -750,7 +750,7 b' def authenticate(username, password, env'
         user_id = user.user_id if user else 'no-user'
         # don't cache for empty users
         plugin_cache_active = plugin_cache_active and user_id
-        cache_namespace_uid = f'cache_user_auth.{user_id}'
+        cache_namespace_uid = f'cache_user_auth.{rc_cache.PERMISSIONS_CACHE_VER}.{user_id}'
         region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid,

rhodecode/authentication/registry.py

@@ -125,7 +125,7 b' class AuthenticationPluginRegistry(objec'
 
     @classmethod
     def get_cache_region(cls):
-        cache_namespace_uid = 'auth_plugins'
+        cache_namespace_uid = 'auth_plugins.v1'
         region = rc_cache.get_or_create_region('cache_general', cache_namespace_uid)
         return region, cache_namespace_uid
 

rhodecode/lib/auth.py

@@ -1255,7 +1255,7 b' class AuthUser(object):'
             'Computing PERMISSION tree for user %s scope `%s` '
             'with caching: %s[TTL: %ss]', user, scope, cache_on, cache_seconds or 0)
 
-        cache_namespace_uid = f'cache_user_auth.{user_id}'
+        cache_namespace_uid = f'cache_user_auth.{rc_cache.PERMISSIONS_CACHE_VER}.{user_id}'
         region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid,
@@ -1358,7 +1358,7 b' class AuthUser(object):'
 
         log.debug('Computing REPO ACL IDS user %s', self)
 
-        cache_namespace_uid = 'cache_user_repo_acl_ids.{}'.format(self.user_id)
+        cache_namespace_uid = f'cache_user_repo_acl_ids.{rc_cache.PERMISSIONS_CACHE_VER}.{self.user_id}'
         region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid, condition=cache)
@@ -1409,7 +1409,7 b' class AuthUser(object):'
 
         log.debug('Computing REPO GROUP ACL IDS user %s', self)
 
-        cache_namespace_uid = 'cache_user_repo_group_acl_ids.{}'.format(self.user_id)
+        cache_namespace_uid = f'cache_user_repo_group_acl_ids.{rc_cache.PERMISSIONS_CACHE_VER}.{self.user_id}'
         region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid, condition=cache)
@@ -1458,7 +1458,7 b' class AuthUser(object):'
 
         log.debug('Computing USER GROUP ACL IDS user %s', self)
 
-        cache_namespace_uid = 'cache_user_user_group_acl_ids.{}'.format(self.user_id)
+        cache_namespace_uid = f'cache_user_user_group_acl_ids.{rc_cache.PERMISSIONS_CACHE_VER}.{self.user_id}'
         region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid, condition=cache)

rhodecode/lib/middleware/simplevcs.py

@@ -314,7 +314,7 b' class SimpleVCS(object):'
                   plugin_id, plugin_cache_active, cache_ttl)
 
         user_id = user.user_id
-        cache_namespace_uid = 'cache_user_auth.{}'.format(user_id)
+        cache_namespace_uid = f'cache_user_auth.{rc_cache.PERMISSIONS_CACHE_VER}.{user_id}'
         region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid,

rhodecode/lib/rc_cache/__init__.py

@@ -56,9 +56,9 b' register_backend('
 log = logging.getLogger(__name__)
 
 
-FILE_TREE_CACHE_VER = 'v4'
-LICENSE_CACHE_VER = 'v2'
-
+FILE_TREE_CACHE_VER = 'v5'
+LICENSE_CACHE_VER = 'v3'
+PERMISSIONS_CACHE_VER = 'v2'
 
 CLEAR_DELETE = 'delete'
 CLEAR_INVALIDATE = 'invalidate'
@@ -111,7 +111,7 b' def configure_dogpile_cache(settings):'
         if log.isEnabledFor(logging.DEBUG):
             region_args = dict(backend=new_region.actual_backend,
                                region_invalidator=new_region.region_invalidator.__class__)
-            log.debug('dogpile: registering a new region `%s` %s', namespace_name, region_args)
+            log.debug('dogpile: registering a new region key=`%s` args=%s', namespace_name, region_args)
 
         region_meta.dogpile_cache_regions[namespace_name] = new_region
 

rhodecode/lib/rc_cache/utils.py

@@ -28,7 +28,7 b' from dogpile.cache import CacheRegion'
 import rhodecode
 from rhodecode.lib.hash_utils import sha1
 from rhodecode.lib.str_utils import safe_bytes
-from rhodecode.lib.type_utils import str2bool
+from rhodecode.lib.type_utils import str2bool # noqa :required by imports from .utils
 
 from . import region_meta, cache_key_meta
 
@@ -185,6 +185,7 b' def get_or_create_region(region_name, re'
 
     region_uid_name = f'{region_name}:{region_namespace}'
 
+    # Special case for ONLY the FileNamespaceBackend backend. We register one-file-per-region
     if isinstance(region_obj.actual_backend, FileNamespaceBackend):
         if not region_namespace:
             raise ValueError(f'{FileNamespaceBackend} used requires to specify region_namespace param')
@@ -200,7 +201,7 b' def get_or_create_region(region_name, re'
         namespace_cache_dir = cache_dir
 
         # we default the namespace_cache_dir to our default cache dir.
-        # however if this backend is configured with filename= param, we prioritize that
+        # however, if this backend is configured with filename= param, we prioritize that
         # so all caches within that particular region, even those namespaced end up in the same path
         if region_obj.actual_backend.filename:
             namespace_cache_dir = os.path.dirname(region_obj.actual_backend.filename)
@@ -280,7 +281,7 b' class InvalidationContext(object):'
         from rhodecode.lib import rc_cache
 
         cache_namespace_uid = CacheKey.SOME_NAMESPACE.format(1)
-        region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
+        region = rc_cache.get_or_create_region('some_region', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid, condition=True)
         def heavy_compute(cache_name, param1, param2):

rhodecode/model/db.py

@@ -3675,7 +3675,7 b' class CacheKey(Base, BaseModel):'
     CACHE_TYPE_FEED = 'FEED'
 
     # namespaces used to register process/thread aware caches
-    REPO_INVALIDATION_NAMESPACE = 'repo_cache:{repo_id}'
+    REPO_INVALIDATION_NAMESPACE = 'repo_cache.v1:{repo_id}'
 
     cache_id = Column("cache_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     cache_key = Column("cache_key", String(255), nullable=True, unique=None, default=None)

rhodecode/model/settings.py

@@ -210,7 +210,7 b' class SettingsModel(BaseModel):'
 
     def get_cache_region(self):
         repo = self._get_repo(self.repo) if self.repo else None
-        cache_key = f"repo.{repo.repo_id}" if repo else "repo.ALL"
+        cache_key = f"repo.v1.{repo.repo_id}" if repo else "repo.v1.ALL"
         cache_namespace_uid = f'cache_settings.{cache_key}'
         region = rc_cache.get_or_create_region('cache_general', cache_namespace_uid)
         return region, cache_namespace_uid

rhodecode/tests/lib/test_caches.py

@@ -37,8 +37,7 b' class TestCaches(object):'
     ])
     def test_cache_decorator_init(self, region_name):
         namespace = region_name
-        cache_region = rc_cache.get_or_create_region(
-            region_name, region_namespace=namespace)
+        cache_region = rc_cache.get_or_create_region(region_name, region_namespace=namespace)
         assert cache_region
 
     @pytest.mark.parametrize('example_input', [