# HG changeset patch # User Mads Kiilerich # Date 2013-04-03 13:56:12 # Node ID 910ad1ffee9956bba4068a88825b2fbebf2d7854 # Parent d23f2bea2346979a1371034fb2c56ff718c84042 invalidation: merge .invalidate and .set_valid as .test_and_set_valid This simplifies the internal API and avoids leaking CacheInvalidation instances and cache keys. diff --git a/rhodecode/controllers/feed.py b/rhodecode/controllers/feed.py --- a/rhodecode/controllers/feed.py +++ b/rhodecode/controllers/feed.py @@ -141,10 +141,9 @@ class FeedController(BaseRepoController) return feed.writeString('utf-8') key = repo_name + '_ATOM' - inv = CacheInvalidation.invalidate(key) - if inv is not None: + valid = CacheInvalidation.test_and_set_valid(key) + if not valid: region_invalidate(_get_feed_from_cache, None, key) - CacheInvalidation.set_valid(inv.cache_key) return _get_feed_from_cache(key) def rss(self, repo_name): @@ -174,8 +173,7 @@ class FeedController(BaseRepoController) return feed.writeString('utf-8') key = repo_name + '_RSS' - inv = CacheInvalidation.invalidate(key) - if inv is not None: + valid = CacheInvalidation.test_and_set_valid(key) + if not valid: region_invalidate(_get_feed_from_cache, None, key) - CacheInvalidation.set_valid(inv.cache_key) return _get_feed_from_cache(key) diff --git a/rhodecode/controllers/summary.py b/rhodecode/controllers/summary.py --- a/rhodecode/controllers/summary.py +++ b/rhodecode/controllers/summary.py @@ -88,7 +88,6 @@ class SummaryController(BaseRepoControll return download_l - def __get_readme_data(self, db_repo): repo_name = db_repo.repo_name @@ -126,10 +125,9 @@ class SummaryController(BaseRepoControll return readme_data, readme_file key = repo_name + '_README' - inv = CacheInvalidation.invalidate(key) - if inv is not None: + valid = CacheInvalidation.test_and_set_valid(key) + if not valid: region_invalidate(_get_readme_from_cache, None, key) - CacheInvalidation.set_valid(inv.cache_key) return _get_readme_from_cache(key) @LoginRequired() diff --git a/rhodecode/lib/utils.py b/rhodecode/lib/utils.py --- a/rhodecode/lib/utils.py +++ b/rhodecode/lib/utils.py @@ -476,7 +476,7 @@ def repo2db_mapper(initial_repo_list, re # system, this will register all repos and multiple instances cache_key = CacheInvalidation._get_cache_key(name) log.debug("Creating invalidation cache key for %s: %s", name, cache_key) - CacheInvalidation.invalidate(name) + CacheInvalidation.test_and_set_valid(name) sa.commit() removed = [] diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py --- a/rhodecode/model/db.py +++ b/rhodecode/model/db.py @@ -1163,10 +1163,6 @@ class Repository(Base, BaseModel): # SCM CACHE INSTANCE #========================================================================== - @property - def invalidate(self): - return CacheInvalidation.invalidate(self.repo_name) - def set_invalidate(self): """ Mark caches of this repo as invalid. @@ -1184,27 +1180,16 @@ class Repository(Base, BaseModel): return self.scm_instance_cached() return self.__get_instance() - def scm_instance_cached(self, cache_map=None): + def scm_instance_cached(self, valid_cache_keys=None): @cache_region('long_term') def _c(repo_name): return self.__get_instance() rn = self.repo_name - if cache_map: - # get using prefilled cache_map - invalidate_repo = cache_map[self.repo_name] - if invalidate_repo: - invalidate_repo = (None if invalidate_repo.cache_active - else invalidate_repo) - else: - # get from invalidate - invalidate_repo = self.invalidate - - if invalidate_repo is not None: + valid = CacheInvalidation.test_and_set_valid(rn, valid_cache_keys=valid_cache_keys) + if not valid: + log.debug('Cache for %s invalidated, getting new object' % (rn)) region_invalidate(_c, None, rn) - log.debug('Cache for %s invalidated, getting new object' % (rn)) - # update our cache - CacheInvalidation.set_valid(invalidate_repo.cache_key) else: log.debug('Getting obj for %s from cache' % (rn)) return _c(rn) @@ -1825,34 +1810,6 @@ class CacheInvalidation(Base, BaseModel) return "%s%s" % (prefix, key) @classmethod - def invalidate(cls, key): - """ - Returns Invalidation object if the local cache with the given key is invalid, - None otherwise. - """ - repo_name = key - repo_name = remove_suffix(repo_name, '_README') - repo_name = remove_suffix(repo_name, '_RSS') - repo_name = remove_suffix(repo_name, '_ATOM') - - cache_key = cls._get_cache_key(key) - inv_obj = Session().query(cls).filter(cls.cache_key == cache_key).scalar() - if not inv_obj: - try: - inv_obj = CacheInvalidation(cache_key, repo_name) - Session().add(inv_obj) - Session().commit() - except Exception: - log.error(traceback.format_exc()) - Session().rollback() - return - - if not inv_obj.cache_active: - # `cache_active = False` means that this cache - # no longer is valid - return inv_obj - - @classmethod def set_invalidate(cls, repo_name): """ Mark all caches of a repo as invalid in the database. @@ -1871,46 +1828,43 @@ class CacheInvalidation(Base, BaseModel) Session().rollback() @classmethod - def set_valid(cls, cache_key): + def test_and_set_valid(cls, key, valid_cache_keys=None): """ - Mark this cache key as active and currently cached + Mark this cache key as active and currently cached. + Return True if the existing cache registration still was valid. + Return False to indicate that it had been invalidated and caches should be refreshed. """ - inv_obj = cls.query().filter(cls.cache_key == cache_key).scalar() - inv_obj.cache_active = True - Session().add(inv_obj) - Session().commit() + cache_key = cls._get_cache_key(key) + + if valid_cache_keys and cache_key in valid_cache_keys: + return True + + repo_name = key + repo_name = remove_suffix(repo_name, '_README') + repo_name = remove_suffix(repo_name, '_RSS') + repo_name = remove_suffix(repo_name, '_ATOM') + + try: + inv_obj = cls.query().filter(cls.cache_key == cache_key).scalar() + if not inv_obj: + inv_obj = CacheInvalidation(cache_key, repo_name) + was_valid = inv_obj.cache_active + inv_obj.cache_active = True + Session().add(inv_obj) + Session().commit() + return was_valid + except Exception: + log.error(traceback.format_exc()) + Session().rollback() + return False @classmethod - def get_cache_map(cls): - - class cachemapdict(dict): - - def __init__(self, *args, **kwargs): - self.fixkey = kwargs.pop('fixkey', False) - super(cachemapdict, self).__init__(*args, **kwargs) - - def __getattr__(self, name): - cache_key = name - if self.fixkey: - cache_key = cls._get_cache_key(name) - if cache_key in self.__dict__: - return self.__dict__[cache_key] - else: - return self[cache_key] - - def __getitem__(self, name): - cache_key = name - if self.fixkey: - cache_key = cls._get_cache_key(name) - try: - return super(cachemapdict, self).__getitem__(cache_key) - except KeyError: - return None - - cache_map = cachemapdict(fixkey=True) - for obj in cls.query().all(): - cache_map[obj.cache_key] = cachemapdict(obj.get_dict()) - return cache_map + def get_valid_cache_keys(cls): + """ + Return opaque object with information of which caches still are valid + and can be used without checking for invalidation. + """ + return set(inv_obj.cache_key for inv_obj in cls.query().filter(cls.cache_active).all()) class ChangesetComment(Base, BaseModel): diff --git a/rhodecode/model/scm.py b/rhodecode/model/scm.py --- a/rhodecode/model/scm.py +++ b/rhodecode/model/scm.py @@ -97,12 +97,12 @@ class CachedRepoList(object): return '<%s (%s)>' % (self.__class__.__name__, self.__len__()) def __iter__(self): - # pre-propagated cache_map to save executing select statements + # pre-propagated valid_cache_keys to save executing select statements # for each repo - cache_map = CacheInvalidation.get_cache_map() + valid_cache_keys = CacheInvalidation.get_valid_cache_keys() for dbr in self.db_repo_list: - scmr = dbr.scm_instance_cached(cache_map) + scmr = dbr.scm_instance_cached(valid_cache_keys) # check permission at this level if not HasRepoPermissionAny( *self.perm_set)(dbr.repo_name, 'get repo check'):