rhodecode/apps/repository/views/repo_summary.py

# -*- coding: utf-8 -*-

# Copyright (C) 2011-2019 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/

import logging
import string
import rhodecode

from pyramid.view import view_config

from rhodecode.lib.view_utils import get_format_ref_id
from rhodecode.apps._base import RepoAppView
from rhodecode.config.conf import (LANGUAGES_EXTENSIONS_MAP)
from rhodecode.lib import helpers as h, rc_cache
from rhodecode.lib.utils2 import safe_str, safe_int
from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
from rhodecode.lib.markup_renderer import MarkupRenderer, relative_links
from rhodecode.lib.ext_json import json
from rhodecode.lib.vcs.backends.base import EmptyCommit
from rhodecode.lib.vcs.exceptions import (
    CommitError, EmptyRepositoryError, CommitDoesNotExistError)
from rhodecode.model.db import Statistics, CacheKey, User
from rhodecode.model.meta import Session
from rhodecode.model.repo import ReadmeFinder
from rhodecode.model.scm import ScmModel

log = logging.getLogger(__name__)


class RepoSummaryView(RepoAppView):

    def load_default_context(self):
        c = self._get_local_tmpl_context(include_app_defaults=True)
        c.rhodecode_repo = None
        if not c.repository_requirements_missing:
            c.rhodecode_repo = self.rhodecode_vcs_repo
        return c

    def _get_readme_data(self, db_repo, renderer_type):

        log.debug('Looking for README file')

        cache_namespace_uid = 'cache_repo_instance.{}_{}'.format(
            db_repo.repo_id, CacheKey.CACHE_TYPE_README)
        invalidation_namespace = CacheKey.REPO_INVALIDATION_NAMESPACE.format(
            repo_id=self.db_repo.repo_id)
        region = rc_cache.get_or_create_region('cache_repo_longterm', cache_namespace_uid)

        @region.conditional_cache_on_arguments(namespace=cache_namespace_uid)
        def generate_repo_readme(repo_id, _repo_name, _renderer_type):
            readme_data = None
            readme_node = None
            readme_filename = None
            commit = self._get_landing_commit_or_none(db_repo)
            if commit:
                log.debug("Searching for a README file.")
                readme_node = ReadmeFinder(_renderer_type).search(commit)
            if readme_node:
                relative_urls = {
                    'raw': h.route_path(
                        'repo_file_raw', repo_name=_repo_name,
                        commit_id=commit.raw_id, f_path=readme_node.path),
                    'standard': h.route_path(
                        'repo_files', repo_name=_repo_name,
                        commit_id=commit.raw_id, f_path=readme_node.path),
                }
                readme_data = self._render_readme_or_none(
                    commit, readme_node, relative_urls)
                readme_filename = readme_node.path
            return readme_data, readme_filename

        inv_context_manager = rc_cache.InvalidationContext(
            uid=cache_namespace_uid, invalidation_namespace=invalidation_namespace)
        with inv_context_manager as invalidation_context:
            args = (db_repo.repo_id, db_repo.repo_name, renderer_type,)
            # re-compute and store cache if we get invalidate signal
            if invalidation_context.should_invalidate():
                instance = generate_repo_readme.refresh(*args)
            else:
                instance = generate_repo_readme(*args)

            log.debug(
                'Repo readme generated and computed in %.3fs',
                inv_context_manager.compute_time)
            return instance

    def _get_landing_commit_or_none(self, db_repo):
        log.debug("Getting the landing commit.")
        try:
            commit = db_repo.get_landing_commit()
            if not isinstance(commit, EmptyCommit):
                return commit
            else:
                log.debug("Repository is empty, no README to render.")
        except CommitError:
            log.exception(
                "Problem getting commit when trying to render the README.")

    def _render_readme_or_none(self, commit, readme_node, relative_urls):
        log.debug(
            'Found README file `%s` rendering...', readme_node.path)
        renderer = MarkupRenderer()
        try:
            html_source = renderer.render(
                readme_node.content, filename=readme_node.path)
            if relative_urls:
                return relative_links(html_source, relative_urls)
            return html_source
        except Exception:
            log.exception(
                "Exception while trying to render the README")

    def _load_commits_context(self, c):
        p = safe_int(self.request.GET.get('page'), 1)
        size = safe_int(self.request.GET.get('size'), 10)

        def url_generator(**kw):
            query_params = {
                'size': size
            }
            query_params.update(kw)
            return h.route_path(
                'repo_summary_commits',
                repo_name=c.rhodecode_db_repo.repo_name, _query=query_params)

        pre_load = ['author', 'branch', 'date', 'message']
        try:
            collection = self.rhodecode_vcs_repo.get_commits(
                pre_load=pre_load, translate_tags=False)
        except EmptyRepositoryError:
            collection = self.rhodecode_vcs_repo

        c.repo_commits = h.RepoPage(
            collection, page=p, items_per_page=size, url=url_generator)
        page_ids = [x.raw_id for x in c.repo_commits]
        c.comments = self.db_repo.get_comments(page_ids)
        c.statuses = self.db_repo.statuses(page_ids)

    def _prepare_and_set_clone_url(self, c):
        username = ''
        if self._rhodecode_user.username != User.DEFAULT_USER:
            username = safe_str(self._rhodecode_user.username)

        _def_clone_uri = _def_clone_uri_id = c.clone_uri_tmpl
        _def_clone_uri_ssh = c.clone_uri_ssh_tmpl

        if '{repo}' in _def_clone_uri:
            _def_clone_uri_id = _def_clone_uri.replace('{repo}', '_{repoid}')
        elif '{repoid}' in _def_clone_uri:
            _def_clone_uri_id = _def_clone_uri.replace('_{repoid}', '{repo}')

        c.clone_repo_url = self.db_repo.clone_url(
            user=username, uri_tmpl=_def_clone_uri)
        c.clone_repo_url_id = self.db_repo.clone_url(
            user=username, uri_tmpl=_def_clone_uri_id)
        c.clone_repo_url_ssh = self.db_repo.clone_url(
            uri_tmpl=_def_clone_uri_ssh, ssh=True)

    @LoginRequired()
    @HasRepoPermissionAnyDecorator(
        'repository.read', 'repository.write', 'repository.admin')
    @view_config(
        route_name='repo_summary_commits', request_method='GET',
        renderer='rhodecode:templates/summary/summary_commits.mako')
    def summary_commits(self):
        c = self.load_default_context()
        self._prepare_and_set_clone_url(c)
        self._load_commits_context(c)
        return self._get_template_context(c)

    @LoginRequired()
    @HasRepoPermissionAnyDecorator(
        'repository.read', 'repository.write', 'repository.admin')
    @view_config(
        route_name='repo_summary', request_method='GET',
        renderer='rhodecode:templates/summary/summary.mako')
    @view_config(
        route_name='repo_summary_slash', request_method='GET',
        renderer='rhodecode:templates/summary/summary.mako')
    @view_config(
        route_name='repo_summary_explicit', request_method='GET',
        renderer='rhodecode:templates/summary/summary.mako')
    def summary(self):
        c = self.load_default_context()

        # Prepare the clone URL
        self._prepare_and_set_clone_url(c)

        # If enabled, get statistics data

        c.show_stats = bool(self.db_repo.enable_statistics)

        stats = Session().query(Statistics) \
            .filter(Statistics.repository == self.db_repo) \
            .scalar()

        c.stats_percentage = 0

        if stats and stats.languages:
            c.no_data = False is self.db_repo.enable_statistics
            lang_stats_d = json.loads(stats.languages)

            # Sort first by decreasing count and second by the file extension,
            # so we have a consistent output.
            lang_stats_items = sorted(lang_stats_d.iteritems(),
                                      key=lambda k: (-k[1], k[0]))[:10]
            lang_stats = [(x, {"count": y,
                               "desc": LANGUAGES_EXTENSIONS_MAP.get(x)})
                          for x, y in lang_stats_items]

            c.trending_languages = json.dumps(lang_stats)
        else:
            c.no_data = True
            c.trending_languages = json.dumps({})

        scm_model = ScmModel()
        c.enable_downloads = self.db_repo.enable_downloads
        c.repository_followers = scm_model.get_followers(self.db_repo)
        c.repository_forks = scm_model.get_forks(self.db_repo)
        c.repository_is_user_following = scm_model.is_following_repo(
            self.db_repo_name, self._rhodecode_user.user_id)

        # first interaction with the VCS instance after here...
        if c.repository_requirements_missing:
            self.request.override_renderer = \
                'rhodecode:templates/summary/missing_requirements.mako'
            return self._get_template_context(c)

        c.readme_data, c.readme_file = \
            self._get_readme_data(self.db_repo, c.visual.default_renderer)

        # loads the summary commits template context
        self._load_commits_context(c)

        return self._get_template_context(c)

    def get_request_commit_id(self):
        return self.request.matchdict['commit_id']

    @LoginRequired()
    @HasRepoPermissionAnyDecorator(
        'repository.read', 'repository.write', 'repository.admin')
    @view_config(
        route_name='repo_stats', request_method='GET',
        renderer='json_ext')
    def repo_stats(self):
        commit_id = self.get_request_commit_id()
        show_stats = bool(self.db_repo.enable_statistics)
        repo_id = self.db_repo.repo_id

        cache_seconds = safe_int(
            rhodecode.CONFIG.get('rc_cache.cache_repo.expiration_time'))
        cache_on = cache_seconds > 0
        log.debug(
            'Computing REPO TREE for repo_id %s commit_id `%s` '
            'with caching: %s[TTL: %ss]' % (
                repo_id, commit_id, cache_on, cache_seconds or 0))

        cache_namespace_uid = 'cache_repo.{}'.format(repo_id)
        region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)

        @region.conditional_cache_on_arguments(namespace=cache_namespace_uid,
                                               condition=cache_on)
        def compute_stats(repo_id, commit_id, show_stats):
            code_stats = {}
            size = 0
            try:
                scm_instance = self.db_repo.scm_instance()
                commit = scm_instance.get_commit(commit_id)

                for node in commit.get_filenodes_generator():
                    size += node.size
                    if not show_stats:
                        continue
                    ext = string.lower(node.extension)
                    ext_info = LANGUAGES_EXTENSIONS_MAP.get(ext)
                    if ext_info:
                        if ext in code_stats:
                            code_stats[ext]['count'] += 1
                        else:
                            code_stats[ext] = {"count": 1, "desc": ext_info}
            except (EmptyRepositoryError, CommitDoesNotExistError):
                pass
            return {'size': h.format_byte_size_binary(size),
                    'code_stats': code_stats}

        stats = compute_stats(self.db_repo.repo_id, commit_id, show_stats)
        return stats

    @LoginRequired()
    @HasRepoPermissionAnyDecorator(
        'repository.read', 'repository.write', 'repository.admin')
    @view_config(
        route_name='repo_refs_data', request_method='GET',
        renderer='json_ext')
    def repo_refs_data(self):
        _ = self.request.translate
        self.load_default_context()

        repo = self.rhodecode_vcs_repo
        refs_to_create = [
            (_("Branch"), repo.branches, 'branch'),
            (_("Tag"), repo.tags, 'tag'),
            (_("Bookmark"), repo.bookmarks, 'book'),
        ]
        res = self._create_reference_data(
            repo, self.db_repo_name, refs_to_create)
        data = {
            'more': False,
            'results': res
        }
        return data

    @LoginRequired()
    @HasRepoPermissionAnyDecorator(
        'repository.read', 'repository.write', 'repository.admin')
    @view_config(
        route_name='repo_refs_changelog_data', request_method='GET',
        renderer='json_ext')
    def repo_refs_changelog_data(self):
        _ = self.request.translate
        self.load_default_context()

        repo = self.rhodecode_vcs_repo

        refs_to_create = [
            (_("Branches"), repo.branches, 'branch'),
            (_("Closed branches"), repo.branches_closed, 'branch_closed'),
            # TODO: enable when vcs can handle bookmarks filters
            # (_("Bookmarks"), repo.bookmarks, "book"),
        ]
        res = self._create_reference_data(
            repo, self.db_repo_name, refs_to_create)
        data = {
            'more': False,
            'results': res
        }
        return data

    def _create_reference_data(self, repo, full_repo_name, refs_to_create):
        format_ref_id = get_format_ref_id(repo)

        result = []
        for title, refs, ref_type in refs_to_create:
            if refs:
                result.append({
                    'text': title,
                    'children': self._create_reference_items(
                        repo, full_repo_name, refs, ref_type,
                        format_ref_id),
                })
        return result

    def _create_reference_items(self, repo, full_repo_name, refs, ref_type, format_ref_id):
        result = []
        is_svn = h.is_svn(repo)
        for ref_name, raw_id in refs.iteritems():
            files_url = self._create_files_url(
                repo, full_repo_name, ref_name, raw_id, is_svn)
            result.append({
                'text': ref_name,
                'id': format_ref_id(ref_name, raw_id),
                'raw_id': raw_id,
                'type': ref_type,
                'files_url': files_url,
                'idx': 0,
            })
        return result

    def _create_files_url(self, repo, full_repo_name, ref_name, raw_id, is_svn):
        use_commit_id = '/' in ref_name or is_svn
        return h.route_path(
            'repo_files',
            repo_name=full_repo_name,
            f_path=ref_name if is_svn else '',
            commit_id=raw_id if use_commit_id else ref_name,
            _query=dict(at=ref_name))