summary: use non-memory cache for readme, and cleanup cache for repo stats.

rhodecode/apps/repository/views/repo_summary.py

@@ -1,376 +1,374 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import string
 import time
 
 import rhodecode
 
 from pyramid.view import view_config
 
 from rhodecode.lib.view_utils import get_format_ref_id
 from rhodecode.apps._base import RepoAppView
 from rhodecode.config.conf import (LANGUAGES_EXTENSIONS_MAP)
 from rhodecode.lib import helpers as h, rc_cache
 from rhodecode.lib.utils2 import safe_str, safe_int
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.markup_renderer import MarkupRenderer, relative_links
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.vcs.backends.base import EmptyCommit
 from rhodecode.lib.vcs.exceptions import (
     CommitError, EmptyRepositoryError, CommitDoesNotExistError)
 from rhodecode.model.db import Statistics, CacheKey, User
 from rhodecode.model.meta import Session
 from rhodecode.model.repo import ReadmeFinder
 from rhodecode.model.scm import ScmModel
 
 log = logging.getLogger(__name__)
 
 
 class RepoSummaryView(RepoAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context(include_app_defaults=True)
         c.rhodecode_repo = None
         if not c.repository_requirements_missing:
             c.rhodecode_repo = self.rhodecode_vcs_repo
         return c
 
     def _get_readme_data(self, db_repo, renderer_type):
         log.debug('Looking for README file')
         landing_commit = db_repo.get_landing_commit()
         if isinstance(landing_commit, EmptyCommit):
             return None, None
 
-        cache_namespace_uid = 'cache_repo_instance.{}_{}'.format(
-            db_repo.repo_id, CacheKey.CACHE_TYPE_README)
-        region = rc_cache.get_or_create_region('cache_repo_longterm', cache_namespace_uid)
+        cache_namespace_uid = 'cache_repo.{}'.format(db_repo.repo_id)
+        region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
         start = time.time()
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid)
         def generate_repo_readme(repo_id, commit_id, _repo_name, _renderer_type):
             readme_data = None
             readme_filename = None
 
             commit = db_repo.get_commit(commit_id)
             log.debug("Searching for a README file at commit %s.", commit_id)
             readme_node = ReadmeFinder(_renderer_type).search(commit)
 
             if readme_node:
                 log.debug('Found README node: %s', readme_node)
                 relative_urls = {
                     'raw': h.route_path(
                         'repo_file_raw', repo_name=_repo_name,
                         commit_id=commit.raw_id, f_path=readme_node.path),
                     'standard': h.route_path(
                         'repo_files', repo_name=_repo_name,
                         commit_id=commit.raw_id, f_path=readme_node.path),
                 }
                 readme_data = self._render_readme_or_none(commit, readme_node, relative_urls)
                 readme_filename = readme_node.unicode_path
 
             return readme_data, readme_filename
 
         readme_data, readme_filename = generate_repo_readme(
             db_repo.repo_id, landing_commit.raw_id, db_repo.repo_name, renderer_type,)
         compute_time = time.time() - start
         log.debug('Repo readme generated and computed in %.4fs', compute_time)
         return readme_data, readme_filename
 
     def _render_readme_or_none(self, commit, readme_node, relative_urls):
         log.debug('Found README file `%s` rendering...', readme_node.path)
         renderer = MarkupRenderer()
         try:
             html_source = renderer.render(
                 readme_node.content, filename=readme_node.path)
             if relative_urls:
                 return relative_links(html_source, relative_urls)
             return html_source
         except Exception:
             log.exception(
                 "Exception while trying to render the README")
 
     def _load_commits_context(self, c):
         p = safe_int(self.request.GET.get('page'), 1)
         size = safe_int(self.request.GET.get('size'), 10)
 
         def url_generator(**kw):
             query_params = {
                 'size': size
             }
             query_params.update(kw)
             return h.route_path(
                 'repo_summary_commits',
                 repo_name=c.rhodecode_db_repo.repo_name, _query=query_params)
 
         pre_load = ['author', 'branch', 'date', 'message']
         try:
             collection = self.rhodecode_vcs_repo.get_commits(
                 pre_load=pre_load, translate_tags=False)
         except EmptyRepositoryError:
             collection = self.rhodecode_vcs_repo
 
         c.repo_commits = h.RepoPage(
             collection, page=p, items_per_page=size, url=url_generator)
         page_ids = [x.raw_id for x in c.repo_commits]
         c.comments = self.db_repo.get_comments(page_ids)
         c.statuses = self.db_repo.statuses(page_ids)
 
     def _prepare_and_set_clone_url(self, c):
         username = ''
         if self._rhodecode_user.username != User.DEFAULT_USER:
             username = safe_str(self._rhodecode_user.username)
 
         _def_clone_uri = _def_clone_uri_id = c.clone_uri_tmpl
         _def_clone_uri_ssh = c.clone_uri_ssh_tmpl
 
         if '{repo}' in _def_clone_uri:
             _def_clone_uri_id = _def_clone_uri.replace('{repo}', '_{repoid}')
         elif '{repoid}' in _def_clone_uri:
             _def_clone_uri_id = _def_clone_uri.replace('_{repoid}', '{repo}')
 
         c.clone_repo_url = self.db_repo.clone_url(
             user=username, uri_tmpl=_def_clone_uri)
         c.clone_repo_url_id = self.db_repo.clone_url(
             user=username, uri_tmpl=_def_clone_uri_id)
         c.clone_repo_url_ssh = self.db_repo.clone_url(
             uri_tmpl=_def_clone_uri_ssh, ssh=True)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_summary_commits', request_method='GET',
         renderer='rhodecode:templates/summary/summary_commits.mako')
     def summary_commits(self):
         c = self.load_default_context()
         self._prepare_and_set_clone_url(c)
         self._load_commits_context(c)
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_summary', request_method='GET',
         renderer='rhodecode:templates/summary/summary.mako')
     @view_config(
         route_name='repo_summary_slash', request_method='GET',
         renderer='rhodecode:templates/summary/summary.mako')
     @view_config(
         route_name='repo_summary_explicit', request_method='GET',
         renderer='rhodecode:templates/summary/summary.mako')
     def summary(self):
         c = self.load_default_context()
 
         # Prepare the clone URL
         self._prepare_and_set_clone_url(c)
 
         # update every 5 min
         if self.db_repo.last_commit_cache_update_diff > 60 * 5:
             self.db_repo.update_commit_cache()
 
         # If enabled, get statistics data
 
         c.show_stats = bool(self.db_repo.enable_statistics)
 
         stats = Session().query(Statistics) \
             .filter(Statistics.repository == self.db_repo) \
             .scalar()
 
         c.stats_percentage = 0
 
         if stats and stats.languages:
             c.no_data = False is self.db_repo.enable_statistics
             lang_stats_d = json.loads(stats.languages)
 
             # Sort first by decreasing count and second by the file extension,
             # so we have a consistent output.
             lang_stats_items = sorted(lang_stats_d.iteritems(),
                                       key=lambda k: (-k[1], k[0]))[:10]
             lang_stats = [(x, {"count": y,
                                "desc": LANGUAGES_EXTENSIONS_MAP.get(x)})
                           for x, y in lang_stats_items]
 
             c.trending_languages = json.dumps(lang_stats)
         else:
             c.no_data = True
             c.trending_languages = json.dumps({})
 
         scm_model = ScmModel()
         c.enable_downloads = self.db_repo.enable_downloads
         c.repository_followers = scm_model.get_followers(self.db_repo)
         c.repository_forks = scm_model.get_forks(self.db_repo)
 
         # first interaction with the VCS instance after here...
         if c.repository_requirements_missing:
             self.request.override_renderer = \
                 'rhodecode:templates/summary/missing_requirements.mako'
             return self._get_template_context(c)
 
         c.readme_data, c.readme_file = \
             self._get_readme_data(self.db_repo, c.visual.default_renderer)
 
         # loads the summary commits template context
         self._load_commits_context(c)
 
         return self._get_template_context(c)
 
-    def get_request_commit_id(self):
-        return self.request.matchdict['commit_id']
-
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_stats', request_method='GET',
         renderer='json_ext')
     def repo_stats(self):
-        commit_id = self.get_request_commit_id()
         show_stats = bool(self.db_repo.enable_statistics)
         repo_id = self.db_repo.repo_id
 
-        cache_seconds = safe_int(
-            rhodecode.CONFIG.get('rc_cache.cache_repo.expiration_time'))
+        landing_commit = self.db_repo.get_landing_commit()
+        if isinstance(landing_commit, EmptyCommit):
+            return {'size': 0, 'code_stats': {}}
+
+        cache_seconds = safe_int(rhodecode.CONFIG.get('rc_cache.cache_repo.expiration_time'))
         cache_on = cache_seconds > 0
+
         log.debug(
-            'Computing REPO TREE for repo_id %s commit_id `%s` '
+            'Computing REPO STATS for repo_id %s commit_id `%s` '
             'with caching: %s[TTL: %ss]' % (
-                repo_id, commit_id, cache_on, cache_seconds or 0))
+                repo_id, landing_commit, cache_on, cache_seconds or 0))
 
         cache_namespace_uid = 'cache_repo.{}'.format(repo_id)
         region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid,
                                                condition=cache_on)
-        def compute_stats(repo_id, commit_id, show_stats):
+        def compute_stats(repo_id, commit_id, _show_stats):
             code_stats = {}
             size = 0
             try:
-                scm_instance = self.db_repo.scm_instance()
-                commit = scm_instance.get_commit(commit_id)
+                commit = self.db_repo.get_commit(commit_id)
 
                 for node in commit.get_filenodes_generator():
                     size += node.size
-                    if not show_stats:
+                    if not _show_stats:
                         continue
                     ext = string.lower(node.extension)
                     ext_info = LANGUAGES_EXTENSIONS_MAP.get(ext)
                     if ext_info:
                         if ext in code_stats:
                             code_stats[ext]['count'] += 1
                         else:
                             code_stats[ext] = {"count": 1, "desc": ext_info}
             except (EmptyRepositoryError, CommitDoesNotExistError):
                 pass
             return {'size': h.format_byte_size_binary(size),
                     'code_stats': code_stats}
 
-        stats = compute_stats(self.db_repo.repo_id, commit_id, show_stats)
+        stats = compute_stats(self.db_repo.repo_id, landing_commit.raw_id, show_stats)
         return stats
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_refs_data', request_method='GET',
         renderer='json_ext')
     def repo_refs_data(self):
         _ = self.request.translate
         self.load_default_context()
 
         repo = self.rhodecode_vcs_repo
         refs_to_create = [
             (_("Branch"), repo.branches, 'branch'),
             (_("Tag"), repo.tags, 'tag'),
             (_("Bookmark"), repo.bookmarks, 'book'),
         ]
         res = self._create_reference_data(repo, self.db_repo_name, refs_to_create)
         data = {
             'more': False,
             'results': res
         }
         return data
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_refs_changelog_data', request_method='GET',
         renderer='json_ext')
     def repo_refs_changelog_data(self):
         _ = self.request.translate
         self.load_default_context()
 
         repo = self.rhodecode_vcs_repo
 
         refs_to_create = [
             (_("Branches"), repo.branches, 'branch'),
             (_("Closed branches"), repo.branches_closed, 'branch_closed'),
             # TODO: enable when vcs can handle bookmarks filters
             # (_("Bookmarks"), repo.bookmarks, "book"),
         ]
         res = self._create_reference_data(
             repo, self.db_repo_name, refs_to_create)
         data = {
             'more': False,
             'results': res
         }
         return data
 
     def _create_reference_data(self, repo, full_repo_name, refs_to_create):
         format_ref_id = get_format_ref_id(repo)
 
         result = []
         for title, refs, ref_type in refs_to_create:
             if refs:
                 result.append({
                     'text': title,
                     'children': self._create_reference_items(
                         repo, full_repo_name, refs, ref_type,
                         format_ref_id),
                 })
         return result
 
     def _create_reference_items(self, repo, full_repo_name, refs, ref_type, format_ref_id):
         result = []
         is_svn = h.is_svn(repo)
         for ref_name, raw_id in refs.iteritems():
             files_url = self._create_files_url(
                 repo, full_repo_name, ref_name, raw_id, is_svn)
             result.append({
                 'text': ref_name,
                 'id': format_ref_id(ref_name, raw_id),
                 'raw_id': raw_id,
                 'type': ref_type,
                 'files_url': files_url,
                 'idx': 0,
             })
         return result
 
     def _create_files_url(self, repo, full_repo_name, ref_name, raw_id, is_svn):
         use_commit_id = '/' in ref_name or is_svn
         return h.route_path(
             'repo_files',
             repo_name=full_repo_name,
             f_path=ref_name if is_svn else '',
             commit_id=raw_id if use_commit_id else ref_name,
             _query=dict(at=ref_name))

rhodecode/tests/lib/test_libs.py

@@ -1,687 +1,687 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 Package for testing various lib/helper functions in rhodecode
 """
 
 import datetime
 import string
 import mock
 import pytest
 
 from rhodecode.tests import no_newline_id_generator
 from rhodecode.tests.utils import run_test_concurrently
 
 from rhodecode.lib import rc_cache
 from rhodecode.lib.helpers import InitialsGravatar
 from rhodecode.lib.utils2 import AttributeDict
 
 from rhodecode.model.db import Repository, CacheKey
 
 
 def _urls_for_proto(proto):
     return [
         ('%s://127.0.0.1' % proto, ['%s://' % proto, '127.0.0.1'],
          '%s://127.0.0.1' % proto),
         ('%s://marcink@127.0.0.1' % proto, ['%s://' % proto, '127.0.0.1'],
          '%s://127.0.0.1' % proto),
         ('%s://marcink:pass@127.0.0.1' % proto, ['%s://' % proto, '127.0.0.1'],
          '%s://127.0.0.1' % proto),
         ('%s://127.0.0.1:8080' % proto, ['%s://' % proto, '127.0.0.1', '8080'],
          '%s://127.0.0.1:8080' % proto),
         ('%s://domain.org' % proto, ['%s://' % proto, 'domain.org'],
          '%s://domain.org' % proto),
         ('%s://user:[e-mail]:8080' % proto,
          ['%s://' % proto, 'domain.org', '8080'],
          '%s://domain.org:8080' % proto),
     ]
 
 TEST_URLS = _urls_for_proto('http') + _urls_for_proto('https')
 
 
 @pytest.mark.parametrize("test_url, expected, expected_creds", TEST_URLS)
 def test_uri_filter(test_url, expected, expected_creds):
     from rhodecode.lib.utils2 import uri_filter
     assert uri_filter(test_url) == expected
 
 
 @pytest.mark.parametrize("test_url, expected, expected_creds", TEST_URLS)
 def test_credentials_filter(test_url, expected, expected_creds):
     from rhodecode.lib.utils2 import credentials_filter
     assert credentials_filter(test_url) == expected_creds
 
 
 @pytest.mark.parametrize("str_bool, expected", [
     ('t', True),
     ('true', True),
     ('y', True),
     ('yes', True),
     ('on', True),
     ('1', True),
     ('Y', True),
     ('yeS', True),
     ('Y', True),
     ('TRUE', True),
     ('T', True),
     ('False', False),
     ('F', False),
     ('FALSE', False),
     ('0', False),
     ('-1', False),
     ('', False)
 ])
 def test_str2bool(str_bool, expected):
     from rhodecode.lib.utils2 import str2bool
     assert str2bool(str_bool) == expected
 
 
 @pytest.mark.parametrize("text, expected", reduce(lambda a1,a2:a1+a2, [
 [
 (pref+"", []),
 (pref+"Hi there @marcink", ['marcink']),
 (pref+"Hi there @marcink and @bob", ['bob', 'marcink']),
 (pref+"Hi there @marcink\n", ['marcink']),
 (pref+"Hi there @marcink and @bob\n", ['bob', 'marcink']),
 (pref+"Hi there [e-mail]", []),
 (pref+"Hi there @john.malcovic and @bob\n", ['bob', 'john.malcovic']),
 (pref+"This needs to be reviewed: (@marcink,@john)", ["john", "marcink"]),
 (pref+"This needs to be reviewed: (@marcink, @john)", ["john", "marcink"]),
 (pref+"This needs to be reviewed: [@marcink,@john]", ["john", "marcink"]),
 (pref+"This needs to be reviewed: (@marcink @john)", ["john", "marcink"]),
 (pref+"@john @mary, please review", ["john", "mary"]),
 (pref+"@john,@mary, please review", ["john", "mary"]),
 (pref+"Hej @123, @22john,@mary, please review", ['123', '22john', 'mary']),
 (pref+"@first hi there @marcink here's my email [e-mail] "
  "@lukaszb check @one_more22 it pls @ ttwelve @D[] @one@two@three ", ['first', 'lukaszb', 'marcink', 'one', 'one_more22']),
 (pref+"@MARCIN    @maRCiN @2one_more22 @john please see this http://org.pl", ['2one_more22', 'john', 'MARCIN', 'maRCiN']),
 (pref+"@marian.user just do it @marco-polo and next extract @marco_polo", ['marco-polo', 'marco_polo', 'marian.user']),
 (pref+"user.dot  hej ! not-needed [e-mail]", []),
 (pref+"\n@marcin", ['marcin']),
 ]
 for pref in ['', '\n', 'hi !', '\t', '\n\n']]), ids=no_newline_id_generator)
 def test_mention_extractor(text, expected):
     from rhodecode.lib.utils2 import extract_mentioned_users
     got = extract_mentioned_users(text)
     assert sorted(got, key=lambda x: x.lower()) == got
     assert set(expected) == set(got)
 
 @pytest.mark.parametrize("age_args, expected, kw", [
     ({}, u'just now', {}),
     ({'seconds': -1}, u'1 second ago', {}),
     ({'seconds': -60 * 2}, u'2 minutes ago', {}),
     ({'hours': -1}, u'1 hour ago', {}),
     ({'hours': -24}, u'1 day ago', {}),
     ({'hours': -24 * 5}, u'5 days ago', {}),
     ({'months': -1}, u'1 month ago', {}),
     ({'months': -1, 'days': -2}, u'1 month and 2 days ago', {}),
     ({'years': -1, 'months': -1}, u'1 year and 1 month ago', {}),
     ({}, u'just now', {'short_format': True}),
     ({'seconds': -1}, u'1sec ago', {'short_format': True}),
     ({'seconds': -60 * 2}, u'2min ago', {'short_format': True}),
     ({'hours': -1}, u'1h ago', {'short_format': True}),
     ({'hours': -24}, u'1d ago', {'short_format': True}),
     ({'hours': -24 * 5}, u'5d ago', {'short_format': True}),
     ({'months': -1}, u'1m ago', {'short_format': True}),
     ({'months': -1, 'days': -2}, u'1m, 2d ago', {'short_format': True}),
     ({'years': -1, 'months': -1}, u'1y, 1m ago', {'short_format': True}),
 ])
 def test_age(age_args, expected, kw, baseapp):
     from rhodecode.lib.utils2 import age
     from dateutil import relativedelta
     n = datetime.datetime(year=2012, month=5, day=17)
     delt = lambda *args, **kwargs: relativedelta.relativedelta(*args, **kwargs)
 
     def translate(elem):
         return elem.interpolate()
 
     assert translate(age(n + delt(**age_args), now=n, **kw)) == expected
 
 
 @pytest.mark.parametrize("age_args, expected, kw", [
     ({}, u'just now', {}),
     ({'seconds': 1}, u'in 1 second', {}),
     ({'seconds': 60 * 2}, u'in 2 minutes', {}),
     ({'hours': 1}, u'in 1 hour', {}),
     ({'hours': 24}, u'in 1 day', {}),
     ({'hours': 24 * 5}, u'in 5 days', {}),
     ({'months': 1}, u'in 1 month', {}),
     ({'months': 1, 'days': 1}, u'in 1 month and 1 day', {}),
     ({'years': 1, 'months': 1}, u'in 1 year and 1 month', {}),
     ({}, u'just now', {'short_format': True}),
     ({'seconds': 1}, u'in 1sec', {'short_format': True}),
     ({'seconds': 60 * 2}, u'in 2min', {'short_format': True}),
     ({'hours': 1}, u'in 1h', {'short_format': True}),
     ({'hours': 24}, u'in 1d', {'short_format': True}),
     ({'hours': 24 * 5}, u'in 5d', {'short_format': True}),
     ({'months': 1}, u'in 1m', {'short_format': True}),
     ({'months': 1, 'days': 1}, u'in 1m, 1d', {'short_format': True}),
     ({'years': 1, 'months': 1}, u'in 1y, 1m', {'short_format': True}),
 ])
 def test_age_in_future(age_args, expected, kw, baseapp):
     from rhodecode.lib.utils2 import age
     from dateutil import relativedelta
     n = datetime.datetime(year=2012, month=5, day=17)
     delt = lambda *args, **kwargs: relativedelta.relativedelta(*args, **kwargs)
 
     def translate(elem):
         return elem.interpolate()
 
     assert translate(age(n + delt(**age_args), now=n, **kw)) == expected
 
 
 @pytest.mark.parametrize("sample, expected_tags", [
     # entry
     ((
              ""
      ),
      [
 
      ]),
     # entry
     ((
         "hello world [stale]"
      ),
      [
          ('state', '[stale]'),
      ]),
     # entry
     ((
          "hello world [v2.0.0] [v1.0.0]"
      ),
      [
          ('generic', '[v2.0.0]'),
          ('generic', '[v1.0.0]'),
      ]),
     # entry
     ((
              "he[ll]o wo[rl]d"
      ),
      [
          ('label', '[ll]'),
          ('label', '[rl]'),
      ]),
     # entry
     ((
         "hello world [stale]\n[featured]\n[stale] [dead] [dev]"
      ),
      [
          ('state', '[stale]'),
          ('state', '[featured]'),
          ('state', '[stale]'),
          ('state', '[dead]'),
          ('state', '[dev]'),
      ]),
     # entry
     ((
         "hello world \n\n [stale] \n [url =&gt; [name](http://rc.com)]"
      ),
      [
         ('state', '[stale]'),
         ('url', '[url =&gt; [name](http://rc.com)]'),
      ]),
     # entry
     ((
             "[url =&gt; [linkNameJS](javascript:alert(document.domain))]\n"
             "[url =&gt; [linkNameHTTP](http://rhodecode.com)]\n"
             "[url =&gt; [linkNameHTTPS](https://rhodecode.com)]\n"
             "[url =&gt; [linkNamePath](/repo_group)]\n"
      ),
      [
          ('generic', '[linkNameJS]'),
          ('url', '[url =&gt; [linkNameHTTP](http://rhodecode.com)]'),
          ('url', '[url =&gt; [linkNameHTTPS](https://rhodecode.com)]'),
          ('url', '[url =&gt; [linkNamePath](/repo_group)]'),
      ]),
     # entry
     ((
         "hello pta[tag] gog [[]] [[] sda ero[or]d [me =&gt;>< sa]"
         "[requires] [stale] [see<>=&gt;] [see =&gt; http://url.com]"
         "[requires =&gt; url] [lang =&gt; python] [just a tag] "
         "<html_tag first='abc' attr=\"my.url?attr=&another=\"></html_tag>"
         "[,d] [ =&gt; ULR ] [obsolete] [desc]]"
     ),
      [
         ('label', '[desc]'),
         ('label', '[obsolete]'),
         ('label', '[or]'),
         ('label', '[requires]'),
         ('label', '[tag]'),
         ('state', '[stale]'),
         ('lang', '[lang =&gt; python]'),
         ('ref', '[requires =&gt; url]'),
         ('see', '[see =&gt; http://url.com]'),
 
      ]),
 
 ], ids=no_newline_id_generator)
 def test_metatag_extraction(sample, expected_tags):
     from rhodecode.lib.helpers import extract_metatags
     tags, value = extract_metatags(sample)
     assert sorted(tags) == sorted(expected_tags)
 
 
 @pytest.mark.parametrize("tag_data, expected_html", [
 
     (('state', '[stable]'), '<div class="metatag" tag="state stable">stable</div>'),
     (('state', '[stale]'), '<div class="metatag" tag="state stale">stale</div>'),
     (('state', '[featured]'), '<div class="metatag" tag="state featured">featured</div>'),
     (('state', '[dev]'), '<div class="metatag" tag="state dev">dev</div>'),
     (('state', '[dead]'), '<div class="metatag" tag="state dead">dead</div>'),
 
     (('label', '[personal]'), '<div class="metatag" tag="label">personal</div>'),
     (('generic', '[v2.0.0]'), '<div class="metatag" tag="generic">v2.0.0</div>'),
 
     (('lang', '[lang =&gt; JavaScript]'), '<div class="metatag" tag="lang">JavaScript</div>'),
     (('lang', '[lang =&gt; C++]'), '<div class="metatag" tag="lang">C++</div>'),
     (('lang', '[lang =&gt; C#]'), '<div class="metatag" tag="lang">C#</div>'),
     (('lang', '[lang =&gt; Delphi/Object]'), '<div class="metatag" tag="lang">Delphi/Object</div>'),
     (('lang', '[lang =&gt; Objective-C]'), '<div class="metatag" tag="lang">Objective-C</div>'),
     (('lang', '[lang =&gt; .NET]'), '<div class="metatag" tag="lang">.NET</div>'),
 
     (('license', '[license =&gt; BSD 3-clause]'), '<div class="metatag" tag="license"><a href="http:\/\/www.opensource.org/licenses/BSD 3-clause">BSD 3-clause</a></div>'),
     (('license', '[license =&gt; GPLv3]'), '<div class="metatag" tag="license"><a href="http:\/\/www.opensource.org/licenses/GPLv3">GPLv3</a></div>'),
     (('license', '[license =&gt; MIT]'), '<div class="metatag" tag="license"><a href="http:\/\/www.opensource.org/licenses/MIT">MIT</a></div>'),
     (('license', '[license =&gt; AGPLv3]'), '<div class="metatag" tag="license"><a href="http:\/\/www.opensource.org/licenses/AGPLv3">AGPLv3</a></div>'),
 
     (('ref', '[requires =&gt; RepoName]'), '<div class="metatag" tag="ref requires">requires: <a href="/RepoName">RepoName</a></div>'),
     (('ref', '[recommends =&gt; GroupName]'), '<div class="metatag" tag="ref recommends">recommends: <a href="/GroupName">GroupName</a></div>'),
     (('ref', '[conflicts =&gt; SomeName]'), '<div class="metatag" tag="ref conflicts">conflicts: <a href="/SomeName">SomeName</a></div>'),
     (('ref', '[base =&gt; SomeName]'), '<div class="metatag" tag="ref base">base: <a href="/SomeName">SomeName</a></div>'),
 
     (('see', '[see =&gt; http://rhodecode.com]'), '<div class="metatag" tag="see">see: http://rhodecode.com </div>'),
 
     (('url', '[url =&gt; [linkName](https://rhodecode.com)]'), '<div class="metatag" tag="url"> <a href="https://rhodecode.com">linkName</a> </div>'),
     (('url', '[url =&gt; [example link](https://rhodecode.com)]'), '<div class="metatag" tag="url"> <a href="https://rhodecode.com">example link</a> </div>'),
     (('url', '[url =&gt; [v1.0.0](https://rhodecode.com)]'), '<div class="metatag" tag="url"> <a href="https://rhodecode.com">v1.0.0</a> </div>'),
 
 ])
 def test_metatags_stylize(tag_data, expected_html):
     from rhodecode.lib.helpers import style_metatag
     tag_type,value = tag_data
     assert style_metatag(tag_type, value) == expected_html
 
 
 @pytest.mark.parametrize("tmpl_url, email, expected", [
     ('http://test.com/{email}', '[e-mail]', 'http://test.com/[e-mail]'),
 
     ('http://test.com/{md5email}', '[e-mail]', 'http://test.com/3cb7232fcc48743000cb86d0d5022bd9'),
     ('http://test.com/{md5email}', 'testąć@foo.com', 'http://test.com/978debb907a3c55cd741872ab293ef30'),
 
     ('http://testX.com/{md5email}?s={size}', '[e-mail]', 'http://testX.com/3cb7232fcc48743000cb86d0d5022bd9?s=24'),
     ('http://testX.com/{md5email}?s={size}', 'testąć@foo.com', 'http://testX.com/978debb907a3c55cd741872ab293ef30?s=24'),
 
     ('{scheme}://{netloc}/{md5email}/{size}', '[e-mail]', 'https://server.com/3cb7232fcc48743000cb86d0d5022bd9/24'),
     ('{scheme}://{netloc}/{md5email}/{size}', 'testąć@foo.com', 'https://server.com/978debb907a3c55cd741872ab293ef30/24'),
 
     ('http://test.com/{email}', 'testąć@foo.com', 'http://test.com/testąć@foo.com'),
     ('http://test.com/{email}?size={size}', '[e-mail]', 'http://test.com/[e-mail]?size=24'),
     ('http://test.com/{email}?size={size}', 'testąć@foo.com', 'http://test.com/testąć@foo.com?size=24'),
 ])
 def test_gravatar_url_builder(tmpl_url, email, expected, request_stub):
     from rhodecode.lib.helpers import gravatar_url
 
     def fake_tmpl_context(_url):
         _c = AttributeDict()
         _c.visual = AttributeDict()
         _c.visual.use_gravatar = True
         _c.visual.gravatar_url = _url
         return _c
 
     # mock pyramid.threadlocals
     def fake_get_current_request():
         request_stub.scheme = 'https'
         request_stub.host = 'server.com'
 
         request_stub._call_context = fake_tmpl_context(tmpl_url)
         return request_stub
 
     with mock.patch('rhodecode.lib.helpers.get_current_request',
                     fake_get_current_request):
 
         grav = gravatar_url(email_address=email, size=24)
         assert grav == expected
 
 
 @pytest.mark.parametrize(
     "email, first_name, last_name, expected_initials, expected_color", [
 
         ('[e-mail]', '', '', 'TR', '#8a994d'),
         ('[e-mail]', '', '', 'MK', '#6559b3'),
         # special cases of email
         ('[e-mail]', '', '', 'JD', '#526600'),
         ('[e-mail]', '', '', 'GR', '#990052'),
         ('[e-mail]', 'Guido', 'Van Rossum', 'GR', '#990052'),
 
         ('[e-mail]', '', '', 'RR', '#46598c'),
         ('[e-mail]', 'Nguyễn Thái', 'Tgọc Duy', 'ND', '#665200'),
 
         ('[e-mail]', '', '', 'JF', '#73006b'),
         ('[e-mail]', 'Marcin', 'Kuzminski', 'MK', '#104036'),
         # partials
         ('[e-mail]', 'Marcin', '', 'MR', '#104036'),  # fn+email
         ('[e-mail]', '', 'Kuzminski', 'AK', '#104036'),  # em+ln
         # non-ascii
         ('[e-mail]', 'Marcin', 'Śuzminski', 'MS', '#104036'),
         ('marcin.ś[e-mail]', '', '', 'MS', '#73000f'),
 
         # special cases, LDAP can provide those...
         ('admin@', 'Marcin', 'Śuzminski', 'MS', '#aa00ff'),
         ('marcin.śuzminski', '', '', 'MS', '#402020'),
         ('null', '', '', 'NL', '#8c4646'),
         ('[e-mail]', 'some', '', 'SA', '#664e33')
 ])
 def test_initials_gravatar_pick_of_initials_and_color_algo(
         email, first_name, last_name, expected_initials, expected_color):
     instance = InitialsGravatar(email, first_name, last_name)
     assert instance.get_initials() == expected_initials
     assert instance.str2color(email) == expected_color
 
 
 def test_initials_gravatar_mapping_algo():
     pos = set()
     instance = InitialsGravatar('', '', '')
     iterations = 0
 
     variations = []
     for letter1 in string.ascii_letters:
         for letter2 in string.ascii_letters[::-1][:10]:
             for letter3 in string.ascii_letters[:10]:
                 variations.append(
                     '[e-mail]' % (letter1+letter2+letter3))
 
     max_variations = 4096
     for email in variations[:max_variations]:
         iterations += 1
         pos.add(
             instance.pick_color_bank_index(email,
                                            instance.get_color_bank()))
 
     # we assume that we have match all 256 possible positions,
     # in reasonable amount of different email addresses
     assert len(pos) == 256
     assert iterations == max_variations
 
 
 @pytest.mark.parametrize("tmpl, repo_name, overrides, prefix, expected", [
     (Repository.DEFAULT_CLONE_URI, 'group/repo1', {}, '', 'http://vps1:8000/group/repo1'),
     (Repository.DEFAULT_CLONE_URI, 'group/repo1', {'user': 'marcink'}, '', 'http://marcink@vps1:8000/group/repo1'),
     (Repository.DEFAULT_CLONE_URI, 'group/repo1', {}, '/rc', 'http://vps1:8000/rc/group/repo1'),
     (Repository.DEFAULT_CLONE_URI, 'group/repo1', {'user': 'user'}, '/rc', 'http://user@vps1:8000/rc/group/repo1'),
     (Repository.DEFAULT_CLONE_URI, 'group/repo1', {'user': 'marcink'}, '/rc', 'http://marcink@vps1:8000/rc/group/repo1'),
     (Repository.DEFAULT_CLONE_URI, 'group/repo1', {'user': 'user'}, '/rc/', 'http://user@vps1:8000/rc/group/repo1'),
     (Repository.DEFAULT_CLONE_URI, 'group/repo1', {'user': 'marcink'}, '/rc/', 'http://marcink@vps1:8000/rc/group/repo1'),
     ('{scheme}://{user}@{netloc}/_{repoid}', 'group/repo1', {}, '', 'http://vps1:8000/_23'),
     ('{scheme}://{user}@{netloc}/_{repoid}', 'group/repo1', {'user': 'marcink'}, '', 'http://marcink@vps1:8000/_23'),
     ('http://{user}@{netloc}/_{repoid}', 'group/repo1', {'user': 'marcink'}, '', 'http://marcink@vps1:8000/_23'),
     ('http://{netloc}/_{repoid}', 'group/repo1', {'user': 'marcink'}, '', 'http://vps1:8000/_23'),
     ('https://{user}@proxy1.server.com/{repo}', 'group/repo1', {'user': 'marcink'}, '', 'https://[e-mail]/group/repo1'),
     ('https://{user}@proxy1.server.com/{repo}', 'group/repo1', {}, '', 'https://proxy1.server.com/group/repo1'),
     ('https://proxy1.server.com/{user}/{repo}', 'group/repo1', {'user': 'marcink'}, '', 'https://proxy1.server.com/marcink/group/repo1'),
 ])
 def test_clone_url_generator(tmpl, repo_name, overrides, prefix, expected):
     from rhodecode.lib.utils2 import get_clone_url
 
     class RequestStub(object):
         def request_url(self, name):
             return 'http://vps1:8000' + prefix
 
         def route_url(self, name):
             return self.request_url(name)
 
     clone_url = get_clone_url(
         request=RequestStub(),
         uri_tmpl=tmpl,
         repo_name=repo_name, repo_id=23, **overrides)
     assert clone_url == expected
 
 
 def _quick_url(text, tmpl="""<a class="revision-link" href="%s">%s</a>""", url_=None):
     """
     Changes `some text url[foo]` => `some text <a href="/">foo</a>
 
     :param text:
     """
     import re
     # quickly change expected url[] into a link
     URL_PAT = re.compile(r'(?:url\[)(.+?)(?:\])')
 
     def url_func(match_obj):
         _url = match_obj.groups()[0]
         return tmpl % (url_ or '/some-url', _url)
     return URL_PAT.sub(url_func, text)
 
 
 @pytest.mark.parametrize("sample, expected", [
   ("",
    ""),
   ("git-svn-id: https://svn.apache.org/repos/asf/libcloud/trunk@1441655 13f79535-47bb-0310-9956-ffa450edef68",
    "git-svn-id: https://svn.apache.org/repos/asf/libcloud/trunk@1441655 13f79535-47bb-0310-9956-ffa450edef68"),
   ("from rev 000000000000",
    "from rev url[000000000000]"),
   ("from rev 000000000000123123 also rev 000000000000",
    "from rev url[000000000000123123] also rev url[000000000000]"),
   ("this should-000 00",
    "this should-000 00"),
   ("longtextffffffffff rev 123123123123",
    "longtextffffffffff rev url[123123123123]"),
   ("rev ffffffffffffffffffffffffffffffffffffffffffffffffff",
    "rev ffffffffffffffffffffffffffffffffffffffffffffffffff"),
   ("ffffffffffff some text traalaa",
    "url[ffffffffffff] some text traalaa"),
    ("""Multi line
    123123123123
    some text 123123123123
    sometimes !
    """,
    """Multi line
    url[123123123123]
    some text url[123123123123]
    sometimes !
    """)
 ], ids=no_newline_id_generator)
 def test_urlify_commits(sample, expected):
     def fake_url(self, *args, **kwargs):
         return '/some-url'
 
     expected = _quick_url(expected)
 
     with mock.patch('rhodecode.lib.helpers.route_url', fake_url):
         from rhodecode.lib.helpers import urlify_commits
         assert urlify_commits(sample, 'repo_name') == expected
 
 
 @pytest.mark.parametrize("sample, expected, url_", [
     ("",
      "",
      ""),
     ("https://svn.apache.org/repos",
      "url[https://svn.apache.org/repos]",
      "https://svn.apache.org/repos"),
     ("http://svn.apache.org/repos",
      "url[http://svn.apache.org/repos]",
      "http://svn.apache.org/repos"),
     ("from rev a also rev http://google.com",
      "from rev a also rev url[http://google.com]",
      "http://google.com"),
     ("""Multi line
      https://foo.bar.com
      some text lalala""",
      """Multi line
      url[https://foo.bar.com]
      some text lalala""",
      "https://foo.bar.com")
 ], ids=no_newline_id_generator)
 def test_urlify_test(sample, expected, url_):
     from rhodecode.lib.helpers import urlify_text
     expected = _quick_url(expected, tmpl="""<a href="%s">%s</a>""", url_=url_)
     assert urlify_text(sample) == expected
 
 
 @pytest.mark.parametrize("test, expected", [
   ("", None),
   ("/_2", '2'),
   ("_2", '2'),
   ("/_2/", '2'),
   ("_2/", '2'),
 
   ("/_21", '21'),
   ("_21", '21'),
   ("/_21/", '21'),
   ("_21/", '21'),
 
   ("/_21/foobar", '21'),
   ("_21/121", '21'),
   ("/_21/_12", '21'),
   ("_21/rc/foo", '21'),
 
 ])
 def test_get_repo_by_id(test, expected):
     from rhodecode.model.repo import RepoModel
     _test = RepoModel()._extract_id_from_repo_name(test)
     assert _test == expected
 
 
 def test_invalidation_context(baseapp):
     repo_id = 9999
 
     cache_namespace_uid = 'cache_repo_instance.{}_{}'.format(
-        repo_id, CacheKey.CACHE_TYPE_README)
+        repo_id, CacheKey.CACHE_TYPE_FEED)
     invalidation_namespace = CacheKey.REPO_INVALIDATION_NAMESPACE.format(
         repo_id=repo_id)
     region = rc_cache.get_or_create_region('cache_repo_longterm', cache_namespace_uid)
 
     calls = [1, 2]
 
     @region.conditional_cache_on_arguments(namespace=cache_namespace_uid)
     def _dummy_func(cache_key):
         val = calls.pop(0)
         return 'result:{}'.format(val)
 
     inv_context_manager = rc_cache.InvalidationContext(
         uid=cache_namespace_uid, invalidation_namespace=invalidation_namespace)
 
     # 1st call, fresh caches
     with inv_context_manager as invalidation_context:
         should_invalidate = invalidation_context.should_invalidate()
         if should_invalidate:
             result = _dummy_func.refresh('some-key')
         else:
             result = _dummy_func('some-key')
 
         assert isinstance(invalidation_context, rc_cache.FreshRegionCache)
         assert should_invalidate is True
 
         assert 'result:1' == result
         # should be cached so calling it twice will give the same result !
         result = _dummy_func('some-key')
         assert 'result:1' == result
 
     # 2nd call, we create a new context manager, this should be now key aware, and
     # return an active cache region
     with inv_context_manager as invalidation_context:
         should_invalidate = invalidation_context.should_invalidate()
         assert isinstance(invalidation_context, rc_cache.ActiveRegionCache)
         assert should_invalidate is False
 
     # Mark invalidation
     CacheKey.set_invalidate(invalidation_namespace)
 
     # 3nd call, fresh caches
     with inv_context_manager as invalidation_context:
         should_invalidate = invalidation_context.should_invalidate()
         if should_invalidate:
             result = _dummy_func.refresh('some-key')
         else:
             result = _dummy_func('some-key')
 
         assert isinstance(invalidation_context, rc_cache.FreshRegionCache)
         assert should_invalidate is True
 
         assert 'result:2' == result
 
         # cached again, same result
         result = _dummy_func('some-key')
         assert 'result:2' == result
 
 
 def test_invalidation_context_exception_in_compute(baseapp):
     repo_id = 888
 
     cache_namespace_uid = 'cache_repo_instance.{}_{}'.format(
-        repo_id, CacheKey.CACHE_TYPE_README)
+        repo_id, CacheKey.CACHE_TYPE_FEED)
     invalidation_namespace = CacheKey.REPO_INVALIDATION_NAMESPACE.format(
         repo_id=repo_id)
     region = rc_cache.get_or_create_region('cache_repo_longterm', cache_namespace_uid)
 
     @region.conditional_cache_on_arguments(namespace=cache_namespace_uid)
     def _dummy_func(cache_key):
         raise Exception('Error in cache func')
 
     with pytest.raises(Exception):
         inv_context_manager = rc_cache.InvalidationContext(
             uid=cache_namespace_uid, invalidation_namespace=invalidation_namespace)
 
         # 1st call, fresh caches
         with inv_context_manager as invalidation_context:
             should_invalidate = invalidation_context.should_invalidate()
             if should_invalidate:
                 _dummy_func.refresh('some-key-2')
             else:
                 _dummy_func('some-key-2')
 
 
 @pytest.mark.parametrize('execution_number', range(5))
 def test_cache_invalidation_race_condition(execution_number, baseapp):
     import time
 
     repo_id = 777
 
     cache_namespace_uid = 'cache_repo_instance.{}_{}'.format(
-        repo_id, CacheKey.CACHE_TYPE_README)
+        repo_id, CacheKey.CACHE_TYPE_FEED)
     invalidation_namespace = CacheKey.REPO_INVALIDATION_NAMESPACE.format(
         repo_id=repo_id)
     region = rc_cache.get_or_create_region('cache_repo_longterm', cache_namespace_uid)
 
     @run_test_concurrently(25)
     def test_create_and_delete_cache_keys():
         time.sleep(0.2)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid)
         def _dummy_func(cache_key):
             val = 'async'
             return 'result:{}'.format(val)
 
         inv_context_manager = rc_cache.InvalidationContext(
             uid=cache_namespace_uid, invalidation_namespace=invalidation_namespace)
 
         # 1st call, fresh caches
         with inv_context_manager as invalidation_context:
             should_invalidate = invalidation_context.should_invalidate()
             if should_invalidate:
                 _dummy_func.refresh('some-key-3')
             else:
                 _dummy_func('some-key-3')
 
         # Mark invalidation
         CacheKey.set_invalidate(invalidation_namespace)
 
     test_create_and_delete_cache_keys()