caching: invalidate Repository cache of README and RSS based on latest revision hash in its .changeset_cache

Avoid using the the more heavy and complex CacheInvalidation.

Note that raw_id only is passed to the getter function as cache key, in order
to make sure new data is retrieved whenever the repo changes.

kallithea/controllers/feed.py

@@ -1,168 +1,161 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.controllers.feed
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Feed controller for Kallithea
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 23, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 
 import logging
 
-from beaker.cache import cache_region, region_invalidate
+from beaker.cache import cache_region
 from tg import response
 from tg import tmpl_context as c
 from tg.i18n import ugettext as _
 from webhelpers.feedgenerator import Atom1Feed, Rss201rev2Feed
 
 from kallithea import CONFIG
 from kallithea.lib import helpers as h
 from kallithea.lib.auth import HasRepoPermissionLevelDecorator, LoginRequired
 from kallithea.lib.base import BaseRepoController
 from kallithea.lib.diffs import DiffProcessor
 from kallithea.lib.utils2 import safe_int, safe_unicode, str2bool
-from kallithea.model.db import CacheInvalidation
 
 
 log = logging.getLogger(__name__)
 
 
 language = 'en-us'
 ttl = "5"
 
 
 class FeedController(BaseRepoController):
 
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def _before(self, *args, **kwargs):
         super(FeedController, self)._before(*args, **kwargs)
 
     def _get_title(self, cs):
         return h.shorter(cs.message, 160)
 
     def __get_desc(self, cs):
         desc_msg = [(_('%s committed on %s')
                      % (h.person(cs.author), h.fmt_date(cs.date))) + '<br/>']
         # branches, tags, bookmarks
         for branch in cs.branches:
             desc_msg.append('branch: %s<br/>' % branch)
         for book in cs.bookmarks:
             desc_msg.append('bookmark: %s<br/>' % book)
         for tag in cs.tags:
             desc_msg.append('tag: %s<br/>' % tag)
 
         changes = []
         diff_limit = safe_int(CONFIG.get('rss_cut_off_limit', 32 * 1024))
         raw_diff = cs.diff()
         diff_processor = DiffProcessor(raw_diff,
                                        diff_limit=diff_limit,
                                        inline_diff=False)
 
         for st in diff_processor.parsed:
             st.update({'added': st['stats']['added'],
                        'removed': st['stats']['deleted']})
             changes.append('\n %(operation)s %(filename)s '
                            '(%(added)s lines added, %(removed)s lines removed)'
                             % st)
         if diff_processor.limited_diff:
             changes = changes + ['\n ' +
                                  _('Changeset was too big and was cut off...')]
 
         # rev link
         _url = h.canonical_url('changeset_home', repo_name=c.db_repo.repo_name,
                    revision=cs.raw_id)
         desc_msg.append('changeset: <a href="%s">%s</a>' % (_url, cs.raw_id[:8]))
 
         desc_msg.append('<pre>')
         desc_msg.append(h.urlify_text(cs.message))
         desc_msg.append('\n')
         desc_msg.extend(changes)
         if str2bool(CONFIG.get('rss_include_diff', False)):
             desc_msg.append('\n\n')
             desc_msg.append(raw_diff)
         desc_msg.append('</pre>')
         return map(safe_unicode, desc_msg)
 
     def atom(self, repo_name):
         """Produce an atom-1.0 feed via feedgenerator module"""
 
         @cache_region('long_term', '_get_feed_from_cache')
         def _get_feed_from_cache(*_cache_keys):  # parameters are not really used - only as caching key
             feed = Atom1Feed(
                 title=_('%s %s feed') % (c.site_name, repo_name),
                 link=h.canonical_url('summary_home', repo_name=repo_name),
                 description=_('Changes on %s repository') % repo_name,
                 language=language,
                 ttl=ttl
             )
 
             rss_items_per_page = safe_int(CONFIG.get('rss_items_per_page', 20))
             for cs in reversed(list(c.db_repo_scm_instance[-rss_items_per_page:])):
                 feed.add_item(title=self._get_title(cs),
                               link=h.canonical_url('changeset_home', repo_name=repo_name,
                                        revision=cs.raw_id),
                               author_name=cs.author,
                               description=''.join(self.__get_desc(cs)),
                               pubdate=cs.date,
                               )
 
             response.content_type = feed.mime_type
             return feed.writeString('utf-8')
 
         kind = 'ATOM'
-        valid = CacheInvalidation.test_and_set_valid(repo_name, kind)
-        if not valid:
-            region_invalidate(_get_feed_from_cache, None, '_get_feed_from_cache', repo_name, kind)
-        return _get_feed_from_cache(repo_name, kind)
+        return _get_feed_from_cache(repo_name, kind, c.db_repo.changeset_cache.get('raw_id'))
 
     def rss(self, repo_name):
         """Produce an rss2 feed via feedgenerator module"""
 
         @cache_region('long_term', '_get_feed_from_cache')
         def _get_feed_from_cache(*_cache_keys):  # parameters are not really used - only as caching key
             feed = Rss201rev2Feed(
                 title=_('%s %s feed') % (c.site_name, repo_name),
                 link=h.canonical_url('summary_home', repo_name=repo_name),
                 description=_('Changes on %s repository') % repo_name,
                 language=language,
                 ttl=ttl
             )
 
             rss_items_per_page = safe_int(CONFIG.get('rss_items_per_page', 20))
             for cs in reversed(list(c.db_repo_scm_instance[-rss_items_per_page:])):
                 feed.add_item(title=self._get_title(cs),
                               link=h.canonical_url('changeset_home', repo_name=repo_name,
                                        revision=cs.raw_id),
                               author_name=cs.author,
                               description=''.join(self.__get_desc(cs)),
                               pubdate=cs.date,
                              )
 
             response.content_type = feed.mime_type
             return feed.writeString('utf-8')
 
         kind = 'RSS'
-        valid = CacheInvalidation.test_and_set_valid(repo_name, kind)
-        if not valid:
-            region_invalidate(_get_feed_from_cache, None, '_get_feed_from_cache', repo_name, kind)
-        return _get_feed_from_cache(repo_name, kind)
+        return _get_feed_from_cache(repo_name, kind, c.db_repo.changeset_cache.get('raw_id'))

kallithea/controllers/summary.py

@@ -1,219 +1,216 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.controllers.summary
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Summary controller for Kallithea
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 18, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 import calendar
 import itertools
 import logging
 import traceback
 from datetime import date, timedelta
 from time import mktime
 
-from beaker.cache import cache_region, region_invalidate
+from beaker.cache import cache_region
 from tg import request
 from tg import tmpl_context as c
 from tg.i18n import ugettext as _
 from webob.exc import HTTPBadRequest
 
 from kallithea.config.conf import ALL_EXTS, ALL_READMES, LANGUAGES_EXTENSIONS_MAP
 from kallithea.lib.auth import HasRepoPermissionLevelDecorator, LoginRequired
 from kallithea.lib.base import BaseRepoController, jsonify, render
 from kallithea.lib.celerylib.tasks import get_commits_stats
 from kallithea.lib.compat import json
 from kallithea.lib.markup_renderer import MarkupRenderer
 from kallithea.lib.page import RepoPage
 from kallithea.lib.utils2 import safe_int
 from kallithea.lib.vcs.backends.base import EmptyChangeset
 from kallithea.lib.vcs.exceptions import ChangesetError, EmptyRepositoryError, NodeDoesNotExistError
 from kallithea.lib.vcs.nodes import FileNode
-from kallithea.model.db import CacheInvalidation, Statistics
+from kallithea.model.db import Statistics
 
 
 log = logging.getLogger(__name__)
 
 README_FILES = [''.join([x[0][0], x[1][0]]) for x in
                     sorted(list(itertools.product(ALL_READMES, ALL_EXTS)),
                            key=lambda y:y[0][1] + y[1][1])]
 
 
 class SummaryController(BaseRepoController):
 
     def __get_readme_data(self, db_repo):
         repo_name = db_repo.repo_name
         log.debug('Looking for README file')
 
         @cache_region('long_term', '_get_readme_from_cache')
         def _get_readme_from_cache(*_cache_keys):  # parameters are not really used - only as caching key
             readme_data = None
             readme_file = None
             try:
                 # gets the landing revision! or tip if fails
                 cs = db_repo.get_landing_changeset()
                 if isinstance(cs, EmptyChangeset):
                     raise EmptyRepositoryError()
                 renderer = MarkupRenderer()
                 for f in README_FILES:
                     try:
                         readme = cs.get_node(f)
                         if not isinstance(readme, FileNode):
                             continue
                         readme_file = f
                         log.debug('Found README file `%s` rendering...',
                                   readme_file)
                         readme_data = renderer.render(readme.content,
                                                       filename=f)
                         break
                     except NodeDoesNotExistError:
                         continue
             except ChangesetError:
                 log.error(traceback.format_exc())
                 pass
             except EmptyRepositoryError:
                 pass
 
             return readme_data, readme_file
 
         kind = 'README'
-        valid = CacheInvalidation.test_and_set_valid(repo_name, kind)
-        if not valid:
-            region_invalidate(_get_readme_from_cache, None, '_get_readme_from_cache', repo_name, kind)
-        return _get_readme_from_cache(repo_name, kind)
+        return _get_readme_from_cache(repo_name, kind, c.db_repo.changeset_cache.get('raw_id'))
 
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def index(self, repo_name):
         p = safe_int(request.GET.get('page'), 1)
         size = safe_int(request.GET.get('size'), 10)
         collection = c.db_repo_scm_instance
         c.cs_pagination = RepoPage(collection, page=p, items_per_page=size)
         page_revisions = [x.raw_id for x in list(c.cs_pagination)]
         c.cs_comments = c.db_repo.get_comments(page_revisions)
         c.cs_statuses = c.db_repo.statuses(page_revisions)
 
         c.ssh_repo_url = None
         if request.authuser.is_default_user:
             username = None
         else:
             username = request.authuser.username
             if c.ssh_enabled:
                 c.ssh_repo_url = c.db_repo.clone_url(clone_uri_tmpl=c.clone_ssh_tmpl)
 
         c.clone_repo_url = c.db_repo.clone_url(clone_uri_tmpl=c.clone_uri_tmpl, with_id=False, username=username)
         c.clone_repo_url_id = c.db_repo.clone_url(clone_uri_tmpl=c.clone_uri_tmpl, with_id=True, username=username)
 
         if c.db_repo.enable_statistics:
             c.show_stats = True
         else:
             c.show_stats = False
 
         stats = Statistics.query() \
             .filter(Statistics.repository == c.db_repo) \
             .scalar()
 
         c.stats_percentage = 0
 
         if stats and stats.languages:
             c.no_data = False is c.db_repo.enable_statistics
             lang_stats_d = json.loads(stats.languages)
 
             lang_stats = [(x, {"count": y,
                                "desc": LANGUAGES_EXTENSIONS_MAP.get(x, '?')})
                           for x, y in lang_stats_d.items()]
             lang_stats.sort(key=lambda k: (-k[1]['count'], k[0]))
 
             c.trending_languages = lang_stats[:10]
         else:
             c.no_data = True
             c.trending_languages = []
 
         c.enable_downloads = c.db_repo.enable_downloads
         c.readme_data, c.readme_file = \
             self.__get_readme_data(c.db_repo)
         return render('summary/summary.html')
 
     @LoginRequired()
     @HasRepoPermissionLevelDecorator('read')
     @jsonify
     def repo_size(self, repo_name):
         if request.is_xhr:
             return c.db_repo._repo_size()
         else:
             raise HTTPBadRequest()
 
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def statistics(self, repo_name):
         if c.db_repo.enable_statistics:
             c.show_stats = True
             c.no_data_msg = _('No data ready yet')
         else:
             c.show_stats = False
             c.no_data_msg = _('Statistics are disabled for this repository')
 
         td = date.today() + timedelta(days=1)
         td_1m = td - timedelta(days=calendar.mdays[td.month])
         td_1y = td - timedelta(days=365)
 
         ts_min_m = mktime(td_1m.timetuple())
         ts_min_y = mktime(td_1y.timetuple())
         ts_max_y = mktime(td.timetuple())
         c.ts_min = ts_min_m
         c.ts_max = ts_max_y
 
         stats = Statistics.query() \
             .filter(Statistics.repository == c.db_repo) \
             .scalar()
         c.stats_percentage = 0
         if stats and stats.languages:
             c.no_data = False is c.db_repo.enable_statistics
             lang_stats_d = json.loads(stats.languages)
             c.commit_data = json.loads(stats.commit_activity)
             c.overview_data = json.loads(stats.commit_activity_combined)
 
             lang_stats = ((x, {"count": y,
                                "desc": LANGUAGES_EXTENSIONS_MAP.get(x)})
                           for x, y in lang_stats_d.items())
 
             c.trending_languages = (
                 sorted(lang_stats, reverse=True, key=lambda k: k[1])[:10]
             )
             last_rev = stats.stat_on_revision + 1
             c.repo_last_rev = c.db_repo_scm_instance.count() \
                 if c.db_repo_scm_instance.revisions else 0
             if last_rev == 0 or c.repo_last_rev == 0:
                 pass
             else:
                 c.stats_percentage = '%.2f' % ((float((last_rev)) /
                                                 c.repo_last_rev) * 100)
         else:
             c.commit_data = {}
             c.overview_data = ([[ts_min_y, 0], [ts_max_y, 10]])
             c.trending_languages = {}
             c.no_data = True
 
         recurse_limit = 500  # don't recurse more than 500 times when parsing
         get_commits_stats(c.db_repo.repo_name, ts_min_y, ts_max_y, recurse_limit)
         return render('summary/statistics.html')