diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py --- a/rhodecode/model/db.py +++ b/rhodecode/model/db.py @@ -693,18 +693,27 @@ class Repository(Base, BaseModel): def scm_instance(self): return self.__get_instance() - @property - def scm_instance_cached(self): + def scm_instance_cached(self, cache_map=None): @cache_region('long_term') def _c(repo_name): return self.__get_instance() rn = self.repo_name log.debug('Getting cached instance of repo') - inv = self.invalidate - if inv is not None: + + if cache_map: + # get using prefilled cache_map + invalidate_repo = cache_map[self.repo_name] + if invalidate_repo: + invalidate_repo = (None if invalidate_repo.cache_active + else invalidate_repo) + else: + # get from invalidate + invalidate_repo = self.invalidate + + if invalidate_repo is not None: region_invalidate(_c, None, rn) # update our cache - CacheInvalidation.set_valid(inv.cache_key) + CacheInvalidation.set_valid(invalidate_repo.cache_key) return _c(rn) def __get_instance(self): @@ -1072,6 +1081,7 @@ class CacheInvalidation(Base, BaseModel) __tablename__ = 'cache_invalidation' __table_args__ = ( UniqueConstraint('cache_key'), + Index('key_idx', 'cache_key'), {'extend_existing': True, 'mysql_engine':'InnoDB', 'mysql_charset': 'utf8'}, ) @@ -1088,6 +1098,7 @@ class CacheInvalidation(Base, BaseModel) def __unicode__(self): return u"<%s('%s:%s')>" % (self.__class__.__name__, self.cache_id, self.cache_key) + @classmethod def clear_cache(cls): cls.query().delete() @@ -1117,7 +1128,7 @@ class CacheInvalidation(Base, BaseModel) try: inv_obj = CacheInvalidation(key, org_key) Session.add(inv_obj) - Session.commit() + #Session.commit() except Exception: log.error(traceback.format_exc()) Session.rollback() @@ -1174,6 +1185,40 @@ class CacheInvalidation(Base, BaseModel) Session.add(inv_obj) Session.commit() + @classmethod + def get_cache_map(cls): + + class cachemapdict(dict): + + def __init__(self, *args, **kwargs): + fixkey = kwargs.get('fixkey') + if fixkey: + del kwargs['fixkey'] + self.fixkey = fixkey + super(cachemapdict, self).__init__(*args, **kwargs) + + def __getattr__(self, name): + key = name + if self.fixkey: + key, _prefix, _org_key = cls._get_key(key) + if key in self.__dict__: + return self.__dict__[key] + else: + return self[key] + + def __getitem__(self, key): + if self.fixkey: + key, _prefix, _org_key = cls._get_key(key) + try: + return super(cachemapdict, self).__getitem__(key) + except KeyError: + return + + cache_map = cachemapdict(fixkey=True) + for obj in cls.query().all(): + cache_map[obj.cache_key] = cachemapdict(obj.get_dict()) + return cache_map + class ChangesetComment(Base, BaseModel): __tablename__ = 'changeset_comments' diff --git a/rhodecode/model/scm.py b/rhodecode/model/scm.py --- a/rhodecode/model/scm.py +++ b/rhodecode/model/scm.py @@ -77,8 +77,12 @@ class CachedRepoList(object): return '<%s (%s)>' % (self.__class__.__name__, self.__len__()) def __iter__(self): + # pre-propagated cache_map to save executing select statements + # for each repo + cache_map = CacheInvalidation.get_cache_map() + for dbr in self.db_repo_list: - scmr = dbr.scm_instance_cached + scmr = dbr.scm_instance_cached(cache_map) # check permission at this level if not HasRepoPermissionAny( 'repository.read', 'repository.write', 'repository.admin'