Cached readme generation

- cleaned up cache system

development.ini

@@ -91,21 +91,27 b' beaker.cache.regions=super_short_term,sh'
 
 beaker.cache.super_short_term.type=memory
 beaker.cache.super_short_term.expire=10
+beaker.cache.super_short_term.key_length = 256
 
 beaker.cache.short_term.type=memory
 beaker.cache.short_term.expire=60
+beaker.cache.short_term.key_length = 256
 
 beaker.cache.long_term.type=memory
 beaker.cache.long_term.expire=36000
+beaker.cache.long_term.key_length = 256
 
 beaker.cache.sql_cache_short.type=memory
 beaker.cache.sql_cache_short.expire=10
+beaker.cache.sql_cache_short.key_length = 256
 
 beaker.cache.sql_cache_med.type=memory
 beaker.cache.sql_cache_med.expire=360
+beaker.cache.sql_cache_med.key_length = 256
 
 beaker.cache.sql_cache_long.type=file
 beaker.cache.sql_cache_long.expire=3600
+beaker.cache.sql_cache_long.key_length = 256
 
 ####################################
 ###       BEAKER SESSION        ####

production.ini

@@ -91,21 +91,27 b' beaker.cache.regions=super_short_term,sh'
 
 beaker.cache.super_short_term.type=memory
 beaker.cache.super_short_term.expire=10
+beaker.cache.super_short_term.key_length = 256
 
 beaker.cache.short_term.type=memory
 beaker.cache.short_term.expire=60
+beaker.cache.short_term.key_length = 256
 
 beaker.cache.long_term.type=memory
 beaker.cache.long_term.expire=36000
+beaker.cache.long_term.key_length = 256
 
 beaker.cache.sql_cache_short.type=memory
 beaker.cache.sql_cache_short.expire=10
+beaker.cache.sql_cache_short.key_length = 256
 
 beaker.cache.sql_cache_med.type=memory
 beaker.cache.sql_cache_med.expire=360
+beaker.cache.sql_cache_med.key_length = 256
 
 beaker.cache.sql_cache_long.type=file
 beaker.cache.sql_cache_long.expire=3600
+beaker.cache.sql_cache_long.key_length = 256
 
 ####################################
 ###       BEAKER SESSION        ####

rhodecode/config/deployment.ini_tmpl

@@ -91,21 +91,27 b' beaker.cache.regions=super_short_term,sh'
 
 beaker.cache.super_short_term.type=memory
 beaker.cache.super_short_term.expire=10
+beaker.cache.super_short_term.key_length = 256
 
 beaker.cache.short_term.type=memory
 beaker.cache.short_term.expire=60
+beaker.cache.short_term.key_length = 256
 
 beaker.cache.long_term.type=memory
 beaker.cache.long_term.expire=36000
+beaker.cache.long_term.key_length = 256
 
 beaker.cache.sql_cache_short.type=memory
 beaker.cache.sql_cache_short.expire=10
+beaker.cache.sql_cache_short.key_length = 256
 
 beaker.cache.sql_cache_med.type=memory
 beaker.cache.sql_cache_med.expire=360
+beaker.cache.sql_cache_med.key_length = 256
 
 beaker.cache.sql_cache_long.type=file
 beaker.cache.sql_cache_long.expire=3600
+beaker.cache.sql_cache_long.key_length = 256
 
 ####################################
 ###       BEAKER SESSION        ####
@@ -154,6 +160,7 b' sqlalchemy.db1.url = sqlite:///%(here)s/'
 # MySQL
 # sqlalchemy.db1.url = mysql://user:pass@localhost/rhodecode
 
+# see sqlalchemy docs for others
 
 sqlalchemy.db1.echo = false
 sqlalchemy.db1.pool_recycle = 3600

rhodecode/controllers/summary.py

@@ -36,7 +36,9 b' from vcs.exceptions import ChangesetErro'
 from pylons import tmpl_context as c, request, url
 from pylons.i18n.translation import _
 
-from rhodecode.model.db import Statistics
+from beaker.cache import cache_region, region_invalidate
+
+from rhodecode.model.db import Statistics, CacheInvalidation
 from rhodecode.lib import ALL_READMES, ALL_EXTS
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseRepoController, render
@@ -50,7 +52,7 b' from rhodecode.lib.compat import json, O'
 
 log = logging.getLogger(__name__)
 
-README_FILES = [''.join([x[0][0], x[1][0]]) for x in 
+README_FILES = [''.join([x[0][0], x[1][0]]) for x in
                     sorted(list(product(ALL_READMES, ALL_EXTS)),
                            key=lambda y:y[0][1] + y[1][1])]
 
@@ -166,32 +168,43 b' class SummaryController(BaseRepoControll'
         if c.enable_downloads:
             c.download_options = self._get_download_links(c.rhodecode_repo)
 
-        c.readme_data,c.readme_file = self.__get_readme_data()
+        c.readme_data, c.readme_file = self.__get_readme_data(c.rhodecode_repo)
         return render('summary/summary.html')
 
-    def __get_readme_data(self):
-        readme_data = None
-        readme_file = None
-        
-        try:
-            cs = c.rhodecode_repo.get_changeset('tip')
-            renderer = MarkupRenderer()
-            for f in README_FILES:
-                try:
-                    readme = cs.get_node(f)
-                    readme_file = f
-                    readme_data = renderer.render(readme.content, f)
-                    break
-                except NodeDoesNotExistError:
-                    continue
-        except ChangesetError:
-            pass
-        except EmptyRepositoryError:
-            pass
-        except Exception:
-            log.error(traceback.format_exc())        
+    def __get_readme_data(self, repo):
 
-        return readme_data, readme_file
+        @cache_region('long_term')
+        def _get_readme_from_cache(key):
+            readme_data = None
+            readme_file = None
+            log.debug('Fetching readme file')
+            try:
+                cs = repo.get_changeset('tip')
+                renderer = MarkupRenderer()
+                for f in README_FILES:
+                    try:
+                        readme = cs.get_node(f)
+                        readme_file = f
+                        readme_data = renderer.render(readme.content, f)
+                        log.debug('Found readme %s' % readme_file)
+                        break
+                    except NodeDoesNotExistError:
+                        continue
+            except ChangesetError:
+                pass
+            except EmptyRepositoryError:
+                pass
+            except Exception:
+                log.error(traceback.format_exc())
+
+            return readme_data, readme_file
+
+        key = repo.name + '_README'
+        inv = CacheInvalidation.invalidate(key)
+        if inv is not None:
+            region_invalidate(_get_readme_from_cache, None, key)
+            CacheInvalidation.set_valid(inv.cache_key)
+        return _get_readme_from_cache(key)
 
     def _get_download_links(self, repo):
 

rhodecode/model/db.py

@@ -663,29 +663,13 b' class Repository(Base, BaseModel):'
 
     @property
     def invalidate(self):
-        """
-        Returns Invalidation object if this repo should be invalidated
-        None otherwise. `cache_active = False` means that this cache
-        state is not valid and needs to be invalidated
-        """
-        return CacheInvalidation.query()\
-            .filter(CacheInvalidation.cache_key == self.repo_name)\
-            .filter(CacheInvalidation.cache_active == False)\
-            .scalar()
+        return CacheInvalidation.invalidate(self.repo_name)
 
     def set_invalidate(self):
         """
         set a cache for invalidation for this instance
         """
-        inv = CacheInvalidation.query()\
-            .filter(CacheInvalidation.cache_key == self.repo_name)\
-            .scalar()
-
-        if inv is None:
-            inv = CacheInvalidation(self.repo_name)
-        inv.cache_active = True
-        Session.add(inv)
-        Session.commit()
+        CacheInvalidation.set_invalidate(self.repo_name)
 
     @LazyProperty
     def scm_instance(self):
@@ -696,19 +680,13 b' class Repository(Base, BaseModel):'
         @cache_region('long_term')
         def _c(repo_name):
             return self.__get_instance()
-
-        # TODO: remove this trick when beaker 1.6 is released
-        # and have fixed this issue with not supporting unicode keys
-        rn = safe_str(self.repo_name)
+        rn = self.repo_name
 
         inv = self.invalidate
         if inv is not None:
             region_invalidate(_c, None, rn)
             # update our cache
-            inv.cache_active = True
-            Session.add(inv)
-            Session.commit()
-
+            CacheInvalidation.set_valid(inv.cache_key)
         return _c(rn)
 
     def __get_instance(self):
@@ -730,7 +708,7 b' class Repository(Base, BaseModel):'
 
             repo = backend(safe_str(repo_full_path), create=False,
                            baseui=self._ui)
-            #skip hidden web repository
+            # skip hidden web repository
             if repo._get_hidden():
                 return
         else:
@@ -855,7 +833,7 b' class Group(Base, BaseModel):'
 
         :param group_name:
         """
-        path_prefix = (self.parent_group.full_path_splitted if 
+        path_prefix = (self.parent_group.full_path_splitted if
                        self.parent_group else [])
         return Group.url_sep().join(path_prefix + [group_name])
 
@@ -1060,6 +1038,57 b' class CacheInvalidation(Base, BaseModel)'
         return "<%s('%s:%s')>" % (self.__class__.__name__,
                                   self.cache_id, self.cache_key)
 
+    @classmethod
+    def invalidate(cls, key):
+        """
+        Returns Invalidation object if this given key should be invalidated
+        None otherwise. `cache_active = False` means that this cache
+        state is not valid and needs to be invalidated
+        
+        :param key:
+        """
+        return cls.query()\
+                .filter(CacheInvalidation.cache_key == key)\
+                .filter(CacheInvalidation.cache_active == False)\
+                .scalar()
+
+    @classmethod
+    def set_invalidate(cls, key):
+        """
+        Mark this Cache key for invalidation
+        
+        :param key:
+        """
+
+        log.debug('marking %s for invalidation' % key)
+        inv_obj = Session().query(cls)\
+            .filter(cls.cache_key == key).scalar()
+        if inv_obj:
+            inv_obj.cache_active = False
+        else:
+            log.debug('cache key not found in invalidation db -> creating one')
+            inv_obj = CacheInvalidation(key)
+
+        try:
+            Session.add(inv_obj)
+            Session.commit()
+        except Exception:
+            log.error(traceback.format_exc())
+            Session.rollback()
+
+    @classmethod
+    def set_valid(cls, key):
+        """
+        Mark this cache key as active and currently cached
+        
+        :param key:
+        """
+        inv_obj = Session().query(CacheInvalidation)\
+            .filter(CacheInvalidation.cache_key == key).scalar()
+        inv_obj.cache_active = True
+        Session.add(inv_obj)
+        Session.commit()
+
 class DbMigrateVersion(Base, BaseModel):
     __tablename__ = 'db_migrate_version'
     __table_args__ = {'extend_existing':True}

rhodecode/model/scm.py

@@ -197,24 +197,8 b' class ScmModel(BaseModel):'
 
         :param repo_name: this repo that should invalidation take place
         """
-
-        log.debug('marking %s for invalidation', repo_name)
-        cache = self.sa.query(CacheInvalidation)\
-            .filter(CacheInvalidation.cache_key == repo_name).scalar()
-
-        if cache:
-            # mark this cache as inactive
-            cache.cache_active = False
-        else:
-            log.debug('cache key not found in invalidation db -> creating one')
-            cache = CacheInvalidation(repo_name)
-
-        try:
-            self.sa.add(cache)
-            self.sa.commit()
-        except (DatabaseError,):
-            log.error(traceback.format_exc())
-            self.sa.rollback()
+        CacheInvalidation.set_invalidate(repo_name)
+        CacheInvalidation.set_invalidate(repo_name+"_README")
 
     def toggle_following_repo(self, follow_repo_id, user_id):
 
@@ -395,20 +379,5 b' class ScmModel(BaseModel):'
 
         self.mark_for_invalidation(repo_name)
 
-
     def get_unread_journal(self):
         return self.sa.query(UserLog).count()
-
-    def _should_invalidate(self, repo_name):
-        """Looks up database for invalidation signals for this repo_name
-
-        :param repo_name:
-        """
-
-        ret = self.sa.query(CacheInvalidation)\
-            .filter(CacheInvalidation.cache_key == repo_name)\
-            .filter(CacheInvalidation.cache_active == False)\
-            .scalar()
-
-        return ret
-

test.ini

@@ -88,21 +88,27 b' beaker.cache.regions=super_short_term,sh'
 
 beaker.cache.super_short_term.type=memory
 beaker.cache.super_short_term.expire=10
+beaker.cache.super_short_term.key_length = 256
 
 beaker.cache.short_term.type=memory
 beaker.cache.short_term.expire=60
+beaker.cache.short_term.key_length = 256
 
 beaker.cache.long_term.type=memory
 beaker.cache.long_term.expire=36000
+beaker.cache.long_term.key_length = 256
 
 beaker.cache.sql_cache_short.type=memory
 beaker.cache.sql_cache_short.expire=10
+beaker.cache.sql_cache_short.key_length = 256
 
 beaker.cache.sql_cache_med.type=memory
 beaker.cache.sql_cache_med.expire=360
+beaker.cache.sql_cache_med.key_length = 256
 
 beaker.cache.sql_cache_long.type=file
 beaker.cache.sql_cache_long.expire=3600
+beaker.cache.sql_cache_long.key_length = 256
 
 ####################################
 ###       BEAKER SESSION        ####
@@ -143,7 +149,7 b' logview.pylons.util = #eee'
 #########################################################
 sqlalchemy.db1.url = sqlite:///%(here)s/test.db
 #sqlalchemy.db1.url = postgresql://postgres:qwe@localhost/rhodecode_tests
-#sqlalchemy.db1.echo = False
+#sqlalchemy.db1.echo = false
 #sqlalchemy.db1.pool_recycle = 3600
 sqlalchemy.convert_unicode = true