caches: allow cache disable for file tree

rhodecode/apps/_base/__init__.py

@@ -1,694 +1,701 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import time
 import logging
 import operator
 
 from pyramid import compat
 from pyramid.httpexceptions import HTTPFound, HTTPForbidden, HTTPBadRequest
 
 from rhodecode.lib import helpers as h, diffs
 from rhodecode.lib.utils2 import (
-    StrictAttributeDict, safe_int, datetime_to_time, safe_unicode)
+    StrictAttributeDict, str2bool, safe_int, datetime_to_time, safe_unicode)
 from rhodecode.lib.vcs.exceptions import RepositoryRequirementError
 from rhodecode.model import repo
 from rhodecode.model import repo_group
 from rhodecode.model import user_group
 from rhodecode.model import user
 from rhodecode.model.db import User
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.settings import VcsSettingsModel
 
 log = logging.getLogger(__name__)
 
 
 ADMIN_PREFIX = '/_admin'
 STATIC_FILE_PREFIX = '/_static'
 
 URL_NAME_REQUIREMENTS = {
     # group name can have a slash in them, but they must not end with a slash
     'group_name': r'.*?[^/]',
     'repo_group_name': r'.*?[^/]',
     # repo names can have a slash in them, but they must not end with a slash
     'repo_name': r'.*?[^/]',
     # file path eats up everything at the end
     'f_path': r'.*',
     # reference types
     'source_ref_type': '(branch|book|tag|rev|\%\(source_ref_type\)s)',
     'target_ref_type': '(branch|book|tag|rev|\%\(target_ref_type\)s)',
 }
 
 
 def add_route_with_slash(config,name, pattern, **kw):
     config.add_route(name, pattern, **kw)
     if not pattern.endswith('/'):
         config.add_route(name + '_slash', pattern + '/', **kw)
 
 
 def add_route_requirements(route_path, requirements=None):
     """
     Adds regex requirements to pyramid routes using a mapping dict
     e.g::
         add_route_requirements('{repo_name}/settings')
     """
     requirements = requirements or URL_NAME_REQUIREMENTS
     for key, regex in requirements.items():
         route_path = route_path.replace('{%s}' % key, '{%s:%s}' % (key, regex))
     return route_path
 
 
 def get_format_ref_id(repo):
     """Returns a `repo` specific reference formatter function"""
     if h.is_svn(repo):
         return _format_ref_id_svn
     else:
         return _format_ref_id
 
 
 def _format_ref_id(name, raw_id):
     """Default formatting of a given reference `name`"""
     return name
 
 
 def _format_ref_id_svn(name, raw_id):
     """Special way of formatting a reference for Subversion including path"""
     return '%s@%s' % (name, raw_id)
 
 
 class TemplateArgs(StrictAttributeDict):
     pass
 
 
 class BaseAppView(object):
 
     def __init__(self, context, request):
         self.request = request
         self.context = context
         self.session = request.session
         if not hasattr(request, 'user'):
             # NOTE(marcink): edge case, we ended up in matched route
             # but probably of web-app context, e.g API CALL/VCS CALL
             if hasattr(request, 'vcs_call') or hasattr(request, 'rpc_method'):
                 log.warning('Unable to process request `%s` in this scope', request)
                 raise HTTPBadRequest()
 
         self._rhodecode_user = request.user  # auth user
         self._rhodecode_db_user = self._rhodecode_user.get_instance()
         self._maybe_needs_password_change(
             request.matched_route.name, self._rhodecode_db_user)
 
     def _maybe_needs_password_change(self, view_name, user_obj):
         log.debug('Checking if user %s needs password change on view %s',
                   user_obj, view_name)
         skip_user_views = [
             'logout', 'login',
             'my_account_password', 'my_account_password_update'
         ]
 
         if not user_obj:
             return
 
         if user_obj.username == User.DEFAULT_USER:
             return
 
         now = time.time()
         should_change = user_obj.user_data.get('force_password_change')
         change_after = safe_int(should_change) or 0
         if should_change and now > change_after:
             log.debug('User %s requires password change', user_obj)
             h.flash('You are required to change your password', 'warning',
                     ignore_duplicate=True)
 
             if view_name not in skip_user_views:
                 raise HTTPFound(
                     self.request.route_path('my_account_password'))
 
     def _log_creation_exception(self, e, repo_name):
         _ = self.request.translate
         reason = None
         if len(e.args) == 2:
             reason = e.args[1]
 
         if reason == 'INVALID_CERTIFICATE':
             log.exception(
                 'Exception creating a repository: invalid certificate')
             msg = (_('Error creating repository %s: invalid certificate')
                    % repo_name)
         else:
             log.exception("Exception creating a repository")
             msg = (_('Error creating repository %s')
                    % repo_name)
         return msg
 
     def _get_local_tmpl_context(self, include_app_defaults=True):
         c = TemplateArgs()
         c.auth_user = self.request.user
         # TODO(marcink): migrate the usage of c.rhodecode_user to c.auth_user
         c.rhodecode_user = self.request.user
 
         if include_app_defaults:
             from rhodecode.lib.base import attach_context_attributes
             attach_context_attributes(c, self.request, self.request.user.user_id)
 
         return c
 
     def _get_template_context(self, tmpl_args, **kwargs):
 
         local_tmpl_args = {
             'defaults': {},
             'errors': {},
             'c': tmpl_args
         }
         local_tmpl_args.update(kwargs)
         return local_tmpl_args
 
     def load_default_context(self):
         """
         example:
 
         def load_default_context(self):
             c = self._get_local_tmpl_context()
             c.custom_var = 'foobar'
 
             return c
         """
         raise NotImplementedError('Needs implementation in view class')
 
 
 class RepoAppView(BaseAppView):
 
     def __init__(self, context, request):
         super(RepoAppView, self).__init__(context, request)
         self.db_repo = request.db_repo
         self.db_repo_name = self.db_repo.repo_name
         self.db_repo_pull_requests = ScmModel().get_pull_requests(self.db_repo)
 
     def _handle_missing_requirements(self, error):
         log.error(
             'Requirements are missing for repository %s: %s',
             self.db_repo_name, safe_unicode(error))
 
     def _get_local_tmpl_context(self, include_app_defaults=True):
         _ = self.request.translate
         c = super(RepoAppView, self)._get_local_tmpl_context(
             include_app_defaults=include_app_defaults)
 
         # register common vars for this type of view
         c.rhodecode_db_repo = self.db_repo
         c.repo_name = self.db_repo_name
         c.repository_pull_requests = self.db_repo_pull_requests
         self.path_filter = PathFilter(None)
 
         c.repository_requirements_missing = {}
         try:
             self.rhodecode_vcs_repo = self.db_repo.scm_instance()
             if self.rhodecode_vcs_repo:
                 path_perms = self.rhodecode_vcs_repo.get_path_permissions(
                     c.auth_user.username)
                 self.path_filter = PathFilter(path_perms)
         except RepositoryRequirementError as e:
             c.repository_requirements_missing = {'error': str(e)}
             self._handle_missing_requirements(e)
             self.rhodecode_vcs_repo = None
 
         c.path_filter = self.path_filter  # used by atom_feed_entry.mako
 
         if self.rhodecode_vcs_repo is None:
             # unable to fetch this repo as vcs instance, report back to user
             h.flash(_(
                 "The repository `%(repo_name)s` cannot be loaded in filesystem. "
                 "Please check if it exist, or is not damaged.") %
                     {'repo_name': c.repo_name},
                     category='error', ignore_duplicate=True)
             if c.repository_requirements_missing:
                 route = self.request.matched_route.name
                 if route.startswith(('edit_repo', 'repo_summary')):
                     # allow summary and edit repo on missing requirements
                     return c
 
                 raise HTTPFound(
                     h.route_path('repo_summary', repo_name=self.db_repo_name))
 
             else:  # redirect if we don't show missing requirements
                 raise HTTPFound(h.route_path('home'))
 
         c.has_origin_repo_read_perm = False
         if self.db_repo.fork:
             c.has_origin_repo_read_perm = h.HasRepoPermissionAny(
                 'repository.write', 'repository.read', 'repository.admin')(
                 self.db_repo.fork.repo_name, 'summary fork link')
 
         return c
 
     def _get_f_path_unchecked(self, matchdict, default=None):
         """
         Should only be used by redirects, everything else should call _get_f_path
         """
         f_path = matchdict.get('f_path')
         if f_path:
             # fix for multiple initial slashes that causes errors for GIT
             return f_path.lstrip('/')
 
         return default
 
     def _get_f_path(self, matchdict, default=None):
         f_path_match = self._get_f_path_unchecked(matchdict, default)
         return self.path_filter.assert_path_permissions(f_path_match)
 
     def _get_general_setting(self, target_repo, settings_key, default=False):
         settings_model = VcsSettingsModel(repo=target_repo)
         settings = settings_model.get_general_settings()
         return settings.get(settings_key, default)
 
+    def get_recache_flag(self):
+        for flag_name in ['force_recache', 'force-recache', 'no-cache']:
+            flag_val = self.request.GET.get(flag_name)
+            if str2bool(flag_val):
+                return True
+        return False
+
 
 class PathFilter(object):
 
     # Expects and instance of BasePathPermissionChecker or None
     def __init__(self, permission_checker):
         self.permission_checker = permission_checker
 
     def assert_path_permissions(self, path):
         if path and self.permission_checker and not self.permission_checker.has_access(path):
             raise HTTPForbidden()
         return path
 
     def filter_patchset(self, patchset):
         if not self.permission_checker or not patchset:
             return patchset, False
         had_filtered = False
         filtered_patchset = []
         for patch in patchset:
             filename = patch.get('filename', None)
             if not filename or self.permission_checker.has_access(filename):
                 filtered_patchset.append(patch)
             else:
                 had_filtered = True
         if had_filtered:
             if isinstance(patchset, diffs.LimitedDiffContainer):
                 filtered_patchset = diffs.LimitedDiffContainer(patchset.diff_limit, patchset.cur_diff_size, filtered_patchset)
             return filtered_patchset, True
         else:
             return patchset, False
 
     def render_patchset_filtered(self, diffset, patchset, source_ref=None, target_ref=None):
         filtered_patchset, has_hidden_changes = self.filter_patchset(patchset)
         result = diffset.render_patchset(
             filtered_patchset, source_ref=source_ref, target_ref=target_ref)
         result.has_hidden_changes = has_hidden_changes
         return result
 
     def get_raw_patch(self, diff_processor):
         if self.permission_checker is None:
             return diff_processor.as_raw()
         elif self.permission_checker.has_full_access:
             return diff_processor.as_raw()
         else:
             return '# Repository has user-specific filters, raw patch generation is disabled.'
 
     @property
     def is_enabled(self):
         return self.permission_checker is not None
 
 
 class RepoGroupAppView(BaseAppView):
     def __init__(self, context, request):
         super(RepoGroupAppView, self).__init__(context, request)
         self.db_repo_group = request.db_repo_group
         self.db_repo_group_name = self.db_repo_group.group_name
 
     def _get_local_tmpl_context(self, include_app_defaults=True):
         _ = self.request.translate
         c = super(RepoGroupAppView, self)._get_local_tmpl_context(
             include_app_defaults=include_app_defaults)
         c.repo_group = self.db_repo_group
         return c
 
     def _revoke_perms_on_yourself(self, form_result):
         _updates = filter(lambda u: self._rhodecode_user.user_id == int(u[0]),
                           form_result['perm_updates'])
         _additions = filter(lambda u: self._rhodecode_user.user_id == int(u[0]),
                             form_result['perm_additions'])
         _deletions = filter(lambda u: self._rhodecode_user.user_id == int(u[0]),
                             form_result['perm_deletions'])
         admin_perm = 'group.admin'
         if _updates and _updates[0][1] != admin_perm or \
            _additions and _additions[0][1] != admin_perm or \
            _deletions and _deletions[0][1] != admin_perm:
             return True
         return False
 
 
 class UserGroupAppView(BaseAppView):
     def __init__(self, context, request):
         super(UserGroupAppView, self).__init__(context, request)
         self.db_user_group = request.db_user_group
         self.db_user_group_name = self.db_user_group.users_group_name
 
 
 class UserAppView(BaseAppView):
     def __init__(self, context, request):
         super(UserAppView, self).__init__(context, request)
         self.db_user = request.db_user
         self.db_user_id = self.db_user.user_id
 
         _ = self.request.translate
         if not request.db_user_supports_default:
             if self.db_user.username == User.DEFAULT_USER:
                 h.flash(_("Editing user `{}` is disabled.".format(
                     User.DEFAULT_USER)), category='warning')
                 raise HTTPFound(h.route_path('users'))
 
 
 class DataGridAppView(object):
     """
     Common class to have re-usable grid rendering components
     """
 
     def _extract_ordering(self, request, column_map=None):
         column_map = column_map or {}
         column_index = safe_int(request.GET.get('order[0][column]'))
         order_dir = request.GET.get(
             'order[0][dir]', 'desc')
         order_by = request.GET.get(
             'columns[%s][data][sort]' % column_index, 'name_raw')
 
         # translate datatable to DB columns
         order_by = column_map.get(order_by) or order_by
 
         search_q = request.GET.get('search[value]')
         return search_q, order_by, order_dir
 
     def _extract_chunk(self, request):
         start = safe_int(request.GET.get('start'), 0)
         length = safe_int(request.GET.get('length'), 25)
         draw = safe_int(request.GET.get('draw'))
         return draw, start, length
 
     def _get_order_col(self, order_by, model):
         if isinstance(order_by, compat.string_types):
             try:
                 return operator.attrgetter(order_by)(model)
             except AttributeError:
                 return None
         else:
             return order_by
 
 
 class BaseReferencesView(RepoAppView):
     """
     Base for reference view for branches, tags and bookmarks.
     """
     def load_default_context(self):
         c = self._get_local_tmpl_context()
 
 
         return c
 
     def load_refs_context(self, ref_items, partials_template):
         _render = self.request.get_partial_renderer(partials_template)
         pre_load = ["author", "date", "message"]
 
         is_svn = h.is_svn(self.rhodecode_vcs_repo)
         is_hg = h.is_hg(self.rhodecode_vcs_repo)
 
         format_ref_id = get_format_ref_id(self.rhodecode_vcs_repo)
 
         closed_refs = {}
         if is_hg:
             closed_refs = self.rhodecode_vcs_repo.branches_closed
 
         data = []
         for ref_name, commit_id in ref_items:
             commit = self.rhodecode_vcs_repo.get_commit(
                 commit_id=commit_id, pre_load=pre_load)
             closed = ref_name in closed_refs
 
             # TODO: johbo: Unify generation of reference links
             use_commit_id = '/' in ref_name or is_svn
 
             if use_commit_id:
                 files_url = h.route_path(
                     'repo_files',
                     repo_name=self.db_repo_name,
                     f_path=ref_name if is_svn else '',
                     commit_id=commit_id)
 
             else:
                 files_url = h.route_path(
                     'repo_files',
                     repo_name=self.db_repo_name,
                     f_path=ref_name if is_svn else '',
                     commit_id=ref_name,
                     _query=dict(at=ref_name))
 
             data.append({
                 "name": _render('name', ref_name, files_url, closed),
                 "name_raw": ref_name,
                 "date": _render('date', commit.date),
                 "date_raw": datetime_to_time(commit.date),
                 "author": _render('author', commit.author),
                 "commit": _render(
                     'commit', commit.message, commit.raw_id, commit.idx),
                 "commit_raw": commit.idx,
                 "compare": _render(
                     'compare', format_ref_id(ref_name, commit.raw_id)),
             })
 
         return data
 
 
 class RepoRoutePredicate(object):
     def __init__(self, val, config):
         self.val = val
 
     def text(self):
         return 'repo_route = %s' % self.val
 
     phash = text
 
     def __call__(self, info, request):
         if hasattr(request, 'vcs_call'):
             # skip vcs calls
             return
 
         repo_name = info['match']['repo_name']
         repo_model = repo.RepoModel()
 
         by_name_match = repo_model.get_by_repo_name(repo_name, cache=False)
 
         def redirect_if_creating(route_info, db_repo):
             skip_views = ['edit_repo_advanced_delete']
             route = route_info['route']
             # we should skip delete view so we can actually "remove" repositories
             # if they get stuck in creating state.
             if route.name in skip_views:
                 return
 
             if db_repo.repo_state in [repo.Repository.STATE_PENDING]:
                 repo_creating_url = request.route_path(
                     'repo_creating', repo_name=db_repo.repo_name)
                 raise HTTPFound(repo_creating_url)
 
         if by_name_match:
             # register this as request object we can re-use later
             request.db_repo = by_name_match
             redirect_if_creating(info, by_name_match)
             return True
 
         by_id_match = repo_model.get_repo_by_id(repo_name)
         if by_id_match:
             request.db_repo = by_id_match
             redirect_if_creating(info, by_id_match)
             return True
 
         return False
 
 
 class RepoForbidArchivedRoutePredicate(object):
     def __init__(self, val, config):
         self.val = val
 
     def text(self):
         return 'repo_forbid_archived = %s' % self.val
 
     phash = text
 
     def __call__(self, info, request):
         _ = request.translate
         rhodecode_db_repo = request.db_repo
 
         log.debug(
             '%s checking if archived flag for repo for %s',
             self.__class__.__name__, rhodecode_db_repo.repo_name)
 
         if rhodecode_db_repo.archived:
             log.warning('Current view is not supported for archived repo:%s',
                         rhodecode_db_repo.repo_name)
 
             h.flash(
                 h.literal(_('Action not supported for archived repository.')),
                 category='warning')
             summary_url = request.route_path(
                 'repo_summary', repo_name=rhodecode_db_repo.repo_name)
             raise HTTPFound(summary_url)
         return True
 
 
 class RepoTypeRoutePredicate(object):
     def __init__(self, val, config):
         self.val = val or ['hg', 'git', 'svn']
 
     def text(self):
         return 'repo_accepted_type = %s' % self.val
 
     phash = text
 
     def __call__(self, info, request):
         if hasattr(request, 'vcs_call'):
             # skip vcs calls
             return
 
         rhodecode_db_repo = request.db_repo
 
         log.debug(
             '%s checking repo type for %s in %s',
             self.__class__.__name__, rhodecode_db_repo.repo_type, self.val)
 
         if rhodecode_db_repo.repo_type in self.val:
             return True
         else:
             log.warning('Current view is not supported for repo type:%s',
                         rhodecode_db_repo.repo_type)
             return False
 
 
 class RepoGroupRoutePredicate(object):
     def __init__(self, val, config):
         self.val = val
 
     def text(self):
         return 'repo_group_route = %s' % self.val
 
     phash = text
 
     def __call__(self, info, request):
         if hasattr(request, 'vcs_call'):
             # skip vcs calls
             return
 
         repo_group_name = info['match']['repo_group_name']
         repo_group_model = repo_group.RepoGroupModel()
         by_name_match = repo_group_model.get_by_group_name(repo_group_name, cache=False)
 
         if by_name_match:
             # register this as request object we can re-use later
             request.db_repo_group = by_name_match
             return True
 
         return False
 
 
 class UserGroupRoutePredicate(object):
     def __init__(self, val, config):
         self.val = val
 
     def text(self):
         return 'user_group_route = %s' % self.val
 
     phash = text
 
     def __call__(self, info, request):
         if hasattr(request, 'vcs_call'):
             # skip vcs calls
             return
 
         user_group_id = info['match']['user_group_id']
         user_group_model = user_group.UserGroup()
         by_id_match = user_group_model.get(user_group_id, cache=False)
 
         if by_id_match:
             # register this as request object we can re-use later
             request.db_user_group = by_id_match
             return True
 
         return False
 
 
 class UserRoutePredicateBase(object):
     supports_default = None
 
     def __init__(self, val, config):
         self.val = val
 
     def text(self):
         raise NotImplementedError()
 
     def __call__(self, info, request):
         if hasattr(request, 'vcs_call'):
             # skip vcs calls
             return
 
         user_id = info['match']['user_id']
         user_model = user.User()
         by_id_match = user_model.get(user_id, cache=False)
 
         if by_id_match:
             # register this as request object we can re-use later
             request.db_user = by_id_match
             request.db_user_supports_default = self.supports_default
             return True
 
         return False
 
 
 class UserRoutePredicate(UserRoutePredicateBase):
     supports_default = False
 
     def text(self):
         return 'user_route = %s' % self.val
 
     phash = text
 
 
 class UserRouteWithDefaultPredicate(UserRoutePredicateBase):
     supports_default = True
 
     def text(self):
         return 'user_with_default_route = %s' % self.val
 
     phash = text
 
 
 def includeme(config):
     config.add_route_predicate(
         'repo_route', RepoRoutePredicate)
     config.add_route_predicate(
         'repo_accepted_types', RepoTypeRoutePredicate)
     config.add_route_predicate(
         'repo_forbid_when_archived', RepoForbidArchivedRoutePredicate)
     config.add_route_predicate(
         'repo_group_route', RepoGroupRoutePredicate)
     config.add_route_predicate(
         'user_group_route', UserGroupRoutePredicate)
     config.add_route_predicate(
         'user_route_with_default', UserRouteWithDefaultPredicate)
     config.add_route_predicate(
         'user_route', UserRoutePredicate)

rhodecode/apps/repository/views/repo_files.py

@@ -1,1394 +1,1395 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import itertools
 import logging
 import os
 import shutil
 import tempfile
 import collections
 import urllib
 
 from pyramid.httpexceptions import HTTPNotFound, HTTPBadRequest, HTTPFound
 from pyramid.view import view_config
 from pyramid.renderers import render
 from pyramid.response import Response
 
 import rhodecode
 from rhodecode.apps._base import RepoAppView
 
 
 from rhodecode.lib import diffs, helpers as h, rc_cache
 from rhodecode.lib import audit_logger
 from rhodecode.lib.view_utils import parse_path_ref
 from rhodecode.lib.exceptions import NonRelativePathError
 from rhodecode.lib.codeblocks import (
     filenode_as_lines_tokens, filenode_as_annotated_lines_tokens)
 from rhodecode.lib.utils2 import (
     convert_line_endings, detect_mode, safe_str, str2bool, safe_int)
 from rhodecode.lib.auth import (
     LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired)
 from rhodecode.lib.vcs import path as vcspath
 from rhodecode.lib.vcs.backends.base import EmptyCommit
 from rhodecode.lib.vcs.conf import settings
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.lib.vcs.exceptions import (
     RepositoryError, CommitDoesNotExistError, EmptyRepositoryError,
     ImproperArchiveTypeError, VCSError, NodeAlreadyExistsError,
     NodeDoesNotExistError, CommitError, NodeError)
 
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.db import Repository
 
 log = logging.getLogger(__name__)
 
 
 class RepoFilesView(RepoAppView):
 
     @staticmethod
     def adjust_file_path_for_svn(f_path, repo):
         """
         Computes the relative path of `f_path`.
 
         This is mainly based on prefix matching of the recognized tags and
         branches in the underlying repository.
         """
         tags_and_branches = itertools.chain(
             repo.branches.iterkeys(),
             repo.tags.iterkeys())
         tags_and_branches = sorted(tags_and_branches, key=len, reverse=True)
 
         for name in tags_and_branches:
             if f_path.startswith('{}/'.format(name)):
                 f_path = vcspath.relpath(f_path, name)
                 break
         return f_path
 
     def load_default_context(self):
         c = self._get_local_tmpl_context(include_app_defaults=True)
         c.rhodecode_repo = self.rhodecode_vcs_repo
         c.enable_downloads = self.db_repo.enable_downloads
         return c
 
     def _ensure_not_locked(self):
         _ = self.request.translate
 
         repo = self.db_repo
         if repo.enable_locking and repo.locked[0]:
             h.flash(_('This repository has been locked by %s on %s')
                     % (h.person_by_id(repo.locked[0]),
                     h.format_date(h.time_to_datetime(repo.locked[1]))),
                     'warning')
             files_url = h.route_path(
                 'repo_files:default_path',
                 repo_name=self.db_repo_name, commit_id='tip')
             raise HTTPFound(files_url)
 
     def check_branch_permission(self, branch_name):
         _ = self.request.translate
 
         rule, branch_perm = self._rhodecode_user.get_rule_and_branch_permission(
             self.db_repo_name, branch_name)
         if branch_perm and branch_perm not in ['branch.push', 'branch.push_force']:
             h.flash(
                 _('Branch `{}` changes forbidden by rule {}.').format(branch_name, rule),
                 'warning')
             files_url = h.route_path(
                 'repo_files:default_path',
                 repo_name=self.db_repo_name, commit_id='tip')
             raise HTTPFound(files_url)
 
     def _get_commit_and_path(self):
         default_commit_id = self.db_repo.landing_rev[1]
         default_f_path = '/'
 
         commit_id = self.request.matchdict.get(
             'commit_id', default_commit_id)
         f_path = self._get_f_path(self.request.matchdict, default_f_path)
         return commit_id, f_path
 
     def _get_default_encoding(self, c):
         enc_list = getattr(c, 'default_encodings', [])
         return enc_list[0] if enc_list else 'UTF-8'
 
     def _get_commit_or_redirect(self, commit_id, redirect_after=True):
         """
         This is a safe way to get commit. If an error occurs it redirects to
         tip with proper message
 
         :param commit_id: id of commit to fetch
         :param redirect_after: toggle redirection
         """
         _ = self.request.translate
 
         try:
             return self.rhodecode_vcs_repo.get_commit(commit_id)
         except EmptyRepositoryError:
             if not redirect_after:
                 return None
 
             _url = h.route_path(
                 'repo_files_add_file',
                 repo_name=self.db_repo_name, commit_id=0, f_path='',
                 _anchor='edit')
 
             if h.HasRepoPermissionAny(
                     'repository.write', 'repository.admin')(self.db_repo_name):
                 add_new = h.link_to(
                     _('Click here to add a new file.'), _url, class_="alert-link")
             else:
                 add_new = ""
 
             h.flash(h.literal(
                 _('There are no files yet. %s') % add_new), category='warning')
             raise HTTPFound(
                 h.route_path('repo_summary', repo_name=self.db_repo_name))
 
         except (CommitDoesNotExistError, LookupError):
             msg = _('No such commit exists for this repository')
             h.flash(msg, category='error')
             raise HTTPNotFound()
         except RepositoryError as e:
             h.flash(safe_str(h.escape(e)), category='error')
             raise HTTPNotFound()
 
     def _get_filenode_or_redirect(self, commit_obj, path):
         """
         Returns file_node, if error occurs or given path is directory,
         it'll redirect to top level path
         """
         _ = self.request.translate
 
         try:
             file_node = commit_obj.get_node(path)
             if file_node.is_dir():
                 raise RepositoryError('The given path is a directory')
         except CommitDoesNotExistError:
             log.exception('No such commit exists for this repository')
             h.flash(_('No such commit exists for this repository'), category='error')
             raise HTTPNotFound()
         except RepositoryError as e:
             log.warning('Repository error while fetching '
                         'filenode `%s`. Err:%s', path, e)
             h.flash(safe_str(h.escape(e)), category='error')
             raise HTTPNotFound()
 
         return file_node
 
     def _is_valid_head(self, commit_id, repo):
         branch_name = sha_commit_id = ''
         is_head = False
 
         if h.is_svn(repo) and not repo.is_empty():
             # Note: Subversion only has one head.
             if commit_id == repo.get_commit(commit_idx=-1).raw_id:
                 is_head = True
                 return branch_name, sha_commit_id, is_head
 
         for _branch_name, branch_commit_id in repo.branches.items():
             # simple case we pass in branch name, it's a HEAD
             if commit_id == _branch_name:
                 is_head = True
                 branch_name = _branch_name
                 sha_commit_id = branch_commit_id
                 break
             # case when we pass in full sha commit_id, which is a head
             elif commit_id == branch_commit_id:
                 is_head = True
                 branch_name = _branch_name
                 sha_commit_id = branch_commit_id
                 break
 
         # checked branches, means we only need to try to get the branch/commit_sha
         if not repo.is_empty:
             commit = repo.get_commit(commit_id=commit_id)
             if commit:
                 branch_name = commit.branch
                 sha_commit_id = commit.raw_id
 
         return branch_name, sha_commit_id, is_head
 
     def _get_tree_at_commit(
             self, c, commit_id, f_path, full_load=False):
 
         repo_id = self.db_repo.repo_id
+        force_recache = self.get_recache_flag()
 
         cache_seconds = safe_int(
             rhodecode.CONFIG.get('rc_cache.cache_repo.expiration_time'))
-        cache_on = cache_seconds > 0
+        cache_on = not force_recache and cache_seconds > 0
         log.debug(
             'Computing FILE TREE for repo_id %s commit_id `%s` and path `%s`'
             'with caching: %s[TTL: %ss]' % (
                 repo_id, commit_id, f_path, cache_on, cache_seconds or 0))
 
         cache_namespace_uid = 'cache_repo.{}'.format(repo_id)
         region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid,
                                                condition=cache_on)
         def compute_file_tree(repo_id, commit_id, f_path, full_load):
             log.debug('Generating cached file tree for repo_id: %s, %s, %s',
                       repo_id, commit_id, f_path)
 
             c.full_load = full_load
             return render(
                 'rhodecode:templates/files/files_browser_tree.mako',
                 self._get_template_context(c), self.request)
 
         return compute_file_tree(self.db_repo.repo_id, commit_id, f_path, full_load)
 
     def _get_archive_spec(self, fname):
         log.debug('Detecting archive spec for: `%s`', fname)
 
         fileformat = None
         ext = None
         content_type = None
         for a_type, ext_data in settings.ARCHIVE_SPECS.items():
             content_type, extension = ext_data
 
             if fname.endswith(extension):
                 fileformat = a_type
                 log.debug('archive is of type: %s', fileformat)
                 ext = extension
                 break
 
         if not fileformat:
             raise ValueError()
 
         # left over part of whole fname is the commit
         commit_id = fname[:-len(ext)]
 
         return commit_id, ext, fileformat, content_type
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_archivefile', request_method='GET',
         renderer=None)
     def repo_archivefile(self):
         # archive cache config
         from rhodecode import CONFIG
         _ = self.request.translate
         self.load_default_context()
 
         fname = self.request.matchdict['fname']
         subrepos = self.request.GET.get('subrepos') == 'true'
 
         if not self.db_repo.enable_downloads:
             return Response(_('Downloads disabled'))
 
         try:
             commit_id, ext, fileformat, content_type = \
                 self._get_archive_spec(fname)
         except ValueError:
             return Response(_('Unknown archive type for: `{}`').format(
                 h.escape(fname)))
 
         try:
             commit = self.rhodecode_vcs_repo.get_commit(commit_id)
         except CommitDoesNotExistError:
             return Response(_('Unknown commit_id {}').format(
                 h.escape(commit_id)))
         except EmptyRepositoryError:
             return Response(_('Empty repository'))
 
         archive_name = '%s-%s%s%s' % (
             safe_str(self.db_repo_name.replace('/', '_')),
             '-sub' if subrepos else '',
             safe_str(commit.short_id), ext)
 
         use_cached_archive = False
         archive_cache_enabled = CONFIG.get(
             'archive_cache_dir') and not self.request.GET.get('no_cache')
         cached_archive_path = None
 
         if archive_cache_enabled:
             # check if we it's ok to write
             if not os.path.isdir(CONFIG['archive_cache_dir']):
                 os.makedirs(CONFIG['archive_cache_dir'])
             cached_archive_path = os.path.join(
                 CONFIG['archive_cache_dir'], archive_name)
             if os.path.isfile(cached_archive_path):
                 log.debug('Found cached archive in %s', cached_archive_path)
                 fd, archive = None, cached_archive_path
                 use_cached_archive = True
             else:
                 log.debug('Archive %s is not yet cached', archive_name)
 
         if not use_cached_archive:
             # generate new archive
             fd, archive = tempfile.mkstemp()
             log.debug('Creating new temp archive in %s', archive)
             try:
                 commit.archive_repo(archive, kind=fileformat, subrepos=subrepos)
             except ImproperArchiveTypeError:
                 return _('Unknown archive type')
             if archive_cache_enabled:
                 # if we generated the archive and we have cache enabled
                 # let's use this for future
                 log.debug('Storing new archive in %s', cached_archive_path)
                 shutil.move(archive, cached_archive_path)
                 archive = cached_archive_path
 
         # store download action
         audit_logger.store_web(
             'repo.archive.download', action_data={
                 'user_agent': self.request.user_agent,
                 'archive_name': archive_name,
                 'archive_spec': fname,
                 'archive_cached': use_cached_archive},
             user=self._rhodecode_user,
             repo=self.db_repo,
             commit=True
         )
 
         def get_chunked_archive(archive_path):
             with open(archive_path, 'rb') as stream:
                 while True:
                     data = stream.read(16 * 1024)
                     if not data:
                         if fd:  # fd means we used temporary file
                             os.close(fd)
                         if not archive_cache_enabled:
                             log.debug('Destroying temp archive %s', archive_path)
                             os.remove(archive_path)
                         break
                     yield data
 
         response = Response(app_iter=get_chunked_archive(archive))
         response.content_disposition = str(
             'attachment; filename=%s' % archive_name)
         response.content_type = str(content_type)
 
         return response
 
     def _get_file_node(self, commit_id, f_path):
         if commit_id not in ['', None, 'None', '0' * 12, '0' * 40]:
             commit = self.rhodecode_vcs_repo.get_commit(commit_id=commit_id)
             try:
                 node = commit.get_node(f_path)
                 if node.is_dir():
                     raise NodeError('%s path is a %s not a file'
                                     % (node, type(node)))
             except NodeDoesNotExistError:
                 commit = EmptyCommit(
                     commit_id=commit_id,
                     idx=commit.idx,
                     repo=commit.repository,
                     alias=commit.repository.alias,
                     message=commit.message,
                     author=commit.author,
                     date=commit.date)
                 node = FileNode(f_path, '', commit=commit)
         else:
             commit = EmptyCommit(
                 repo=self.rhodecode_vcs_repo,
                 alias=self.rhodecode_vcs_repo.alias)
             node = FileNode(f_path, '', commit=commit)
         return node
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_files_diff', request_method='GET',
         renderer=None)
     def repo_files_diff(self):
         c = self.load_default_context()
         f_path = self._get_f_path(self.request.matchdict)
         diff1 = self.request.GET.get('diff1', '')
         diff2 = self.request.GET.get('diff2', '')
 
         path1, diff1 = parse_path_ref(diff1, default_path=f_path)
 
         ignore_whitespace = str2bool(self.request.GET.get('ignorews'))
         line_context = self.request.GET.get('context', 3)
 
         if not any((diff1, diff2)):
             h.flash(
                 'Need query parameter "diff1" or "diff2" to generate a diff.',
                 category='error')
             raise HTTPBadRequest()
 
         c.action = self.request.GET.get('diff')
         if c.action not in ['download', 'raw']:
             compare_url = h.route_path(
                 'repo_compare',
                 repo_name=self.db_repo_name,
                 source_ref_type='rev',
                 source_ref=diff1,
                 target_repo=self.db_repo_name,
                 target_ref_type='rev',
                 target_ref=diff2,
                 _query=dict(f_path=f_path))
             # redirect to new view if we render diff
             raise HTTPFound(compare_url)
 
         try:
             node1 = self._get_file_node(diff1, path1)
             node2 = self._get_file_node(diff2, f_path)
         except (RepositoryError, NodeError):
             log.exception("Exception while trying to get node from repository")
             raise HTTPFound(
                 h.route_path('repo_files', repo_name=self.db_repo_name,
                              commit_id='tip', f_path=f_path))
 
         if all(isinstance(node.commit, EmptyCommit)
                for node in (node1, node2)):
             raise HTTPNotFound()
 
         c.commit_1 = node1.commit
         c.commit_2 = node2.commit
 
         if c.action == 'download':
             _diff = diffs.get_gitdiff(node1, node2,
                                       ignore_whitespace=ignore_whitespace,
                                       context=line_context)
             diff = diffs.DiffProcessor(_diff, format='gitdiff')
 
             response = Response(self.path_filter.get_raw_patch(diff))
             response.content_type = 'text/plain'
             response.content_disposition = (
                 'attachment; filename=%s_%s_vs_%s.diff' % (f_path, diff1, diff2)
             )
             charset = self._get_default_encoding(c)
             if charset:
                 response.charset = charset
             return response
 
         elif c.action == 'raw':
             _diff = diffs.get_gitdiff(node1, node2,
                                       ignore_whitespace=ignore_whitespace,
                                       context=line_context)
             diff = diffs.DiffProcessor(_diff, format='gitdiff')
 
             response = Response(self.path_filter.get_raw_patch(diff))
             response.content_type = 'text/plain'
             charset = self._get_default_encoding(c)
             if charset:
                 response.charset = charset
             return response
 
         # in case we ever end up here
         raise HTTPNotFound()
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_files_diff_2way_redirect', request_method='GET',
         renderer=None)
     def repo_files_diff_2way_redirect(self):
         """
         Kept only to make OLD links work
         """
         f_path = self._get_f_path_unchecked(self.request.matchdict)
         diff1 = self.request.GET.get('diff1', '')
         diff2 = self.request.GET.get('diff2', '')
 
         if not any((diff1, diff2)):
             h.flash(
                 'Need query parameter "diff1" or "diff2" to generate a diff.',
                 category='error')
             raise HTTPBadRequest()
 
         compare_url = h.route_path(
             'repo_compare',
             repo_name=self.db_repo_name,
             source_ref_type='rev',
             source_ref=diff1,
             target_ref_type='rev',
             target_ref=diff2,
             _query=dict(f_path=f_path, diffmode='sideside',
                         target_repo=self.db_repo_name,))
         raise HTTPFound(compare_url)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_files', request_method='GET',
         renderer=None)
     @view_config(
         route_name='repo_files:default_path', request_method='GET',
         renderer=None)
     @view_config(
         route_name='repo_files:default_commit', request_method='GET',
         renderer=None)
     @view_config(
         route_name='repo_files:rendered', request_method='GET',
         renderer=None)
     @view_config(
         route_name='repo_files:annotated', request_method='GET',
         renderer=None)
     def repo_files(self):
         c = self.load_default_context()
 
         view_name = getattr(self.request.matched_route, 'name', None)
 
         c.annotate = view_name == 'repo_files:annotated'
         # default is false, but .rst/.md files later are auto rendered, we can
         # overwrite auto rendering by setting this GET flag
         c.renderer = view_name == 'repo_files:rendered' or \
                         not self.request.GET.get('no-render', False)
 
         # redirect to given commit_id from form if given
         get_commit_id = self.request.GET.get('at_rev', None)
         if get_commit_id:
             self._get_commit_or_redirect(get_commit_id)
 
         commit_id, f_path = self._get_commit_and_path()
         c.commit = self._get_commit_or_redirect(commit_id)
         c.branch = self.request.GET.get('branch', None)
         c.f_path = f_path
 
         # prev link
         try:
             prev_commit = c.commit.prev(c.branch)
             c.prev_commit = prev_commit
             c.url_prev = h.route_path(
                 'repo_files', repo_name=self.db_repo_name,
                 commit_id=prev_commit.raw_id, f_path=f_path)
             if c.branch:
                 c.url_prev += '?branch=%s' % c.branch
         except (CommitDoesNotExistError, VCSError):
             c.url_prev = '#'
             c.prev_commit = EmptyCommit()
 
         # next link
         try:
             next_commit = c.commit.next(c.branch)
             c.next_commit = next_commit
             c.url_next = h.route_path(
                 'repo_files', repo_name=self.db_repo_name,
                 commit_id=next_commit.raw_id, f_path=f_path)
             if c.branch:
                 c.url_next += '?branch=%s' % c.branch
         except (CommitDoesNotExistError, VCSError):
             c.url_next = '#'
             c.next_commit = EmptyCommit()
 
         # files or dirs
         try:
             c.file = c.commit.get_node(f_path)
             c.file_author = True
             c.file_tree = ''
 
             # load file content
             if c.file.is_file():
                 c.lf_node = c.file.get_largefile_node()
 
                 c.file_source_page = 'true'
                 c.file_last_commit = c.file.last_commit
                 if c.file.size < c.visual.cut_off_limit_diff:
                     if c.annotate:  # annotation has precedence over renderer
                         c.annotated_lines = filenode_as_annotated_lines_tokens(
                             c.file
                         )
                     else:
                         c.renderer = (
                             c.renderer and h.renderer_from_filename(c.file.path)
                         )
                         if not c.renderer:
                             c.lines = filenode_as_lines_tokens(c.file)
 
                 _branch_name, _sha_commit_id, is_head = self._is_valid_head(
                     commit_id, self.rhodecode_vcs_repo)
                 c.on_branch_head = is_head
 
                 branch = c.commit.branch if (
                     c.commit.branch and '/' not in c.commit.branch) else None
                 c.branch_or_raw_id = branch or c.commit.raw_id
                 c.branch_name = c.commit.branch or h.short_id(c.commit.raw_id)
 
                 author = c.file_last_commit.author
                 c.authors = [[
                     h.email(author),
                     h.person(author, 'username_or_name_or_email'),
                     1
                 ]]
 
             else:  # load tree content at path
                 c.file_source_page = 'false'
                 c.authors = []
                 # this loads a simple tree without metadata to speed things up
                 # later via ajax we call repo_nodetree_full and fetch whole
                 c.file_tree = self._get_tree_at_commit(
                     c, c.commit.raw_id, f_path)
 
         except RepositoryError as e:
             h.flash(safe_str(h.escape(e)), category='error')
             raise HTTPNotFound()
 
         if self.request.environ.get('HTTP_X_PJAX'):
             html = render('rhodecode:templates/files/files_pjax.mako',
                           self._get_template_context(c), self.request)
         else:
             html = render('rhodecode:templates/files/files.mako',
                           self._get_template_context(c), self.request)
         return Response(html)
 
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_files:annotated_previous', request_method='GET',
         renderer=None)
     def repo_files_annotated_previous(self):
         self.load_default_context()
 
         commit_id, f_path = self._get_commit_and_path()
         commit = self._get_commit_or_redirect(commit_id)
         prev_commit_id = commit.raw_id
         line_anchor = self.request.GET.get('line_anchor')
         is_file = False
         try:
             _file = commit.get_node(f_path)
             is_file = _file.is_file()
         except (NodeDoesNotExistError, CommitDoesNotExistError, VCSError):
             pass
 
         if is_file:
             history = commit.get_path_history(f_path)
             prev_commit_id = history[1].raw_id \
                 if len(history) > 1 else prev_commit_id
         prev_url = h.route_path(
             'repo_files:annotated', repo_name=self.db_repo_name,
             commit_id=prev_commit_id, f_path=f_path,
             _anchor='L{}'.format(line_anchor))
 
         raise HTTPFound(prev_url)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_nodetree_full', request_method='GET',
         renderer=None, xhr=True)
     @view_config(
         route_name='repo_nodetree_full:default_path', request_method='GET',
         renderer=None, xhr=True)
     def repo_nodetree_full(self):
         """
         Returns rendered html of file tree that contains commit date,
         author, commit_id for the specified combination of
         repo, commit_id and file path
         """
         c = self.load_default_context()
 
         commit_id, f_path = self._get_commit_and_path()
         commit = self._get_commit_or_redirect(commit_id)
         try:
             dir_node = commit.get_node(f_path)
         except RepositoryError as e:
             return Response('error: {}'.format(h.escape(safe_str(e))))
 
         if dir_node.is_file():
             return Response('')
 
         c.file = dir_node
         c.commit = commit
 
         html = self._get_tree_at_commit(
             c, commit.raw_id, dir_node.path, full_load=True)
 
         return Response(html)
 
     def _get_attachement_headers(self, f_path):
         f_name = safe_str(f_path.split(Repository.NAME_SEP)[-1])
         safe_path = f_name.replace('"', '\\"')
         encoded_path = urllib.quote(f_name)
 
         return "attachment; " \
                "filename=\"{}\"; " \
                "filename*=UTF-8\'\'{}".format(safe_path, encoded_path)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_file_raw', request_method='GET',
         renderer=None)
     def repo_file_raw(self):
         """
         Action for show as raw, some mimetypes are "rendered",
         those include images, icons.
         """
         c = self.load_default_context()
 
         commit_id, f_path = self._get_commit_and_path()
         commit = self._get_commit_or_redirect(commit_id)
         file_node = self._get_filenode_or_redirect(commit, f_path)
 
         raw_mimetype_mapping = {
             # map original mimetype to a mimetype used for "show as raw"
             # you can also provide a content-disposition to override the
             # default "attachment" disposition.
             # orig_type: (new_type, new_dispo)
 
             # show images inline:
             # Do not re-add SVG: it is unsafe and permits XSS attacks. One can
             # for example render an SVG with javascript inside or even render
             # HTML.
             'image/x-icon': ('image/x-icon', 'inline'),
             'image/png': ('image/png', 'inline'),
             'image/gif': ('image/gif', 'inline'),
             'image/jpeg': ('image/jpeg', 'inline'),
             'application/pdf': ('application/pdf', 'inline'),
         }
 
         mimetype = file_node.mimetype
         try:
             mimetype, disposition = raw_mimetype_mapping[mimetype]
         except KeyError:
             # we don't know anything special about this, handle it safely
             if file_node.is_binary:
                 # do same as download raw for binary files
                 mimetype, disposition = 'application/octet-stream', 'attachment'
             else:
                 # do not just use the original mimetype, but force text/plain,
                 # otherwise it would serve text/html and that might be unsafe.
                 # Note: underlying vcs library fakes text/plain mimetype if the
                 # mimetype can not be determined and it thinks it is not
                 # binary.This might lead to erroneous text display in some
                 # cases, but helps in other cases, like with text files
                 # without extension.
                 mimetype, disposition = 'text/plain', 'inline'
 
         if disposition == 'attachment':
             disposition = self._get_attachement_headers(f_path)
 
         def stream_node():
             yield file_node.raw_bytes
 
         response = Response(app_iter=stream_node())
         response.content_disposition = disposition
         response.content_type = mimetype
 
         charset = self._get_default_encoding(c)
         if charset:
             response.charset = charset
 
         return response
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_file_download', request_method='GET',
         renderer=None)
     @view_config(
         route_name='repo_file_download:legacy', request_method='GET',
         renderer=None)
     def repo_file_download(self):
         c = self.load_default_context()
 
         commit_id, f_path = self._get_commit_and_path()
         commit = self._get_commit_or_redirect(commit_id)
         file_node = self._get_filenode_or_redirect(commit, f_path)
 
         if self.request.GET.get('lf'):
             # only if lf get flag is passed, we download this file
             # as LFS/Largefile
             lf_node = file_node.get_largefile_node()
             if lf_node:
                 # overwrite our pointer with the REAL large-file
                 file_node = lf_node
 
         disposition = self._get_attachement_headers(f_path)
 
         def stream_node():
             yield file_node.raw_bytes
 
         response = Response(app_iter=stream_node())
         response.content_disposition = disposition
         response.content_type = file_node.mimetype
 
         charset = self._get_default_encoding(c)
         if charset:
             response.charset = charset
 
         return response
 
     def _get_nodelist_at_commit(self, repo_name, repo_id, commit_id, f_path):
 
         cache_seconds = safe_int(
             rhodecode.CONFIG.get('rc_cache.cache_repo.expiration_time'))
         cache_on = cache_seconds > 0
         log.debug(
             'Computing FILE SEARCH for repo_id %s commit_id `%s` and path `%s`'
             'with caching: %s[TTL: %ss]' % (
                 repo_id, commit_id, f_path, cache_on, cache_seconds or 0))
 
         cache_namespace_uid = 'cache_repo.{}'.format(repo_id)
         region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid,
                                                condition=cache_on)
         def compute_file_search(repo_id, commit_id, f_path):
             log.debug('Generating cached nodelist for repo_id:%s, %s, %s',
                       repo_id, commit_id, f_path)
             try:
                 _d, _f = ScmModel().get_nodes(
                     repo_name, commit_id, f_path, flat=False)
             except (RepositoryError, CommitDoesNotExistError, Exception) as e:
                 log.exception(safe_str(e))
                 h.flash(safe_str(h.escape(e)), category='error')
                 raise HTTPFound(h.route_path(
                     'repo_files', repo_name=self.db_repo_name,
                     commit_id='tip', f_path='/'))
             return _d + _f
 
         return compute_file_search(self.db_repo.repo_id, commit_id, f_path)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_files_nodelist', request_method='GET',
         renderer='json_ext', xhr=True)
     def repo_nodelist(self):
         self.load_default_context()
 
         commit_id, f_path = self._get_commit_and_path()
         commit = self._get_commit_or_redirect(commit_id)
 
         metadata = self._get_nodelist_at_commit(
             self.db_repo_name, self.db_repo.repo_id, commit.raw_id, f_path)
         return {'nodes': metadata}
 
     def _create_references(
             self, branches_or_tags, symbolic_reference, f_path):
         items = []
         for name, commit_id in branches_or_tags.items():
             sym_ref = symbolic_reference(commit_id, name, f_path)
             items.append((sym_ref, name))
         return items
 
     def _symbolic_reference(self, commit_id, name, f_path):
         return commit_id
 
     def _symbolic_reference_svn(self, commit_id, name, f_path):
         new_f_path = vcspath.join(name, f_path)
         return u'%s@%s' % (new_f_path, commit_id)
 
     def _get_node_history(self, commit_obj, f_path, commits=None):
         """
         get commit history for given node
 
         :param commit_obj: commit to calculate history
         :param f_path: path for node to calculate history for
         :param commits: if passed don't calculate history and take
             commits defined in this list
         """
         _ = self.request.translate
 
         # calculate history based on tip
         tip = self.rhodecode_vcs_repo.get_commit()
         if commits is None:
             pre_load = ["author", "branch"]
             try:
                 commits = tip.get_path_history(f_path, pre_load=pre_load)
             except (NodeDoesNotExistError, CommitError):
                 # this node is not present at tip!
                 commits = commit_obj.get_path_history(f_path, pre_load=pre_load)
 
         history = []
         commits_group = ([], _("Changesets"))
         for commit in commits:
             branch = ' (%s)' % commit.branch if commit.branch else ''
             n_desc = 'r%s:%s%s' % (commit.idx, commit.short_id, branch)
             commits_group[0].append((commit.raw_id, n_desc,))
         history.append(commits_group)
 
         symbolic_reference = self._symbolic_reference
 
         if self.rhodecode_vcs_repo.alias == 'svn':
             adjusted_f_path = RepoFilesView.adjust_file_path_for_svn(
                 f_path, self.rhodecode_vcs_repo)
             if adjusted_f_path != f_path:
                 log.debug(
                     'Recognized svn tag or branch in file "%s", using svn '
                     'specific symbolic references', f_path)
                 f_path = adjusted_f_path
                 symbolic_reference = self._symbolic_reference_svn
 
         branches = self._create_references(
             self.rhodecode_vcs_repo.branches, symbolic_reference, f_path)
         branches_group = (branches, _("Branches"))
 
         tags = self._create_references(
             self.rhodecode_vcs_repo.tags, symbolic_reference, f_path)
         tags_group = (tags, _("Tags"))
 
         history.append(branches_group)
         history.append(tags_group)
 
         return history, commits
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_file_history', request_method='GET',
         renderer='json_ext')
     def repo_file_history(self):
         self.load_default_context()
 
         commit_id, f_path = self._get_commit_and_path()
         commit = self._get_commit_or_redirect(commit_id)
         file_node = self._get_filenode_or_redirect(commit, f_path)
 
         if file_node.is_file():
             file_history, _hist = self._get_node_history(commit, f_path)
 
             res = []
             for obj in file_history:
                 res.append({
                     'text': obj[1],
                     'children': [{'id': o[0], 'text': o[1]} for o in obj[0]]
                 })
 
             data = {
                 'more': False,
                 'results': res
             }
             return data
 
         log.warning('Cannot fetch history for directory')
         raise HTTPBadRequest()
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_file_authors', request_method='GET',
         renderer='rhodecode:templates/files/file_authors_box.mako')
     def repo_file_authors(self):
         c = self.load_default_context()
 
         commit_id, f_path = self._get_commit_and_path()
         commit = self._get_commit_or_redirect(commit_id)
         file_node = self._get_filenode_or_redirect(commit, f_path)
 
         if not file_node.is_file():
             raise HTTPBadRequest()
 
         c.file_last_commit = file_node.last_commit
         if self.request.GET.get('annotate') == '1':
             # use _hist from annotation if annotation mode is on
             commit_ids = set(x[1] for x in file_node.annotate)
             _hist = (
                 self.rhodecode_vcs_repo.get_commit(commit_id)
                 for commit_id in commit_ids)
         else:
             _f_history, _hist = self._get_node_history(commit, f_path)
         c.file_author = False
 
         unique = collections.OrderedDict()
         for commit in _hist:
             author = commit.author
             if author not in unique:
                 unique[commit.author] = [
                     h.email(author),
                     h.person(author, 'username_or_name_or_email'),
                     1  # counter
                 ]
 
             else:
                 # increase counter
                 unique[commit.author][2] += 1
 
         c.authors = [val for val in unique.values()]
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
     @view_config(
         route_name='repo_files_remove_file', request_method='GET',
         renderer='rhodecode:templates/files/files_delete.mako')
     def repo_files_remove_file(self):
         _ = self.request.translate
         c = self.load_default_context()
         commit_id, f_path = self._get_commit_and_path()
 
         self._ensure_not_locked()
         _branch_name, _sha_commit_id, is_head = \
             self._is_valid_head(commit_id, self.rhodecode_vcs_repo)
 
         if not is_head:
             h.flash(_('You can only delete files with commit '
                       'being a valid branch head.'), category='warning')
             raise HTTPFound(
                 h.route_path('repo_files',
                              repo_name=self.db_repo_name, commit_id='tip',
                              f_path=f_path))
 
         self.check_branch_permission(_branch_name)
         c.commit = self._get_commit_or_redirect(commit_id)
         c.file = self._get_filenode_or_redirect(c.commit, f_path)
 
         c.default_message = _(
             'Deleted file {} via RhodeCode Enterprise').format(f_path)
         c.f_path = f_path
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='repo_files_delete_file', request_method='POST',
         renderer=None)
     def repo_files_delete_file(self):
         _ = self.request.translate
 
         c = self.load_default_context()
         commit_id, f_path = self._get_commit_and_path()
 
         self._ensure_not_locked()
         _branch_name, _sha_commit_id, is_head = \
             self._is_valid_head(commit_id, self.rhodecode_vcs_repo)
 
         if not is_head:
             h.flash(_('You can only delete files with commit '
                       'being a valid branch head.'), category='warning')
             raise HTTPFound(
                 h.route_path('repo_files',
                              repo_name=self.db_repo_name, commit_id='tip',
                              f_path=f_path))
         self.check_branch_permission(_branch_name)
 
         c.commit = self._get_commit_or_redirect(commit_id)
         c.file = self._get_filenode_or_redirect(c.commit, f_path)
 
         c.default_message = _(
             'Deleted file {} via RhodeCode Enterprise').format(f_path)
         c.f_path = f_path
         node_path = f_path
         author = self._rhodecode_db_user.full_contact
         message = self.request.POST.get('message') or c.default_message
         try:
             nodes = {
                 node_path: {
                     'content': ''
                 }
             }
             ScmModel().delete_nodes(
                 user=self._rhodecode_db_user.user_id, repo=self.db_repo,
                 message=message,
                 nodes=nodes,
                 parent_commit=c.commit,
                 author=author,
             )
 
             h.flash(
                 _('Successfully deleted file `{}`').format(
                     h.escape(f_path)), category='success')
         except Exception:
             log.exception('Error during commit operation')
             h.flash(_('Error occurred during commit'), category='error')
         raise HTTPFound(
             h.route_path('repo_commit', repo_name=self.db_repo_name,
                          commit_id='tip'))
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
     @view_config(
         route_name='repo_files_edit_file', request_method='GET',
         renderer='rhodecode:templates/files/files_edit.mako')
     def repo_files_edit_file(self):
         _ = self.request.translate
         c = self.load_default_context()
         commit_id, f_path = self._get_commit_and_path()
 
         self._ensure_not_locked()
         _branch_name, _sha_commit_id, is_head = \
             self._is_valid_head(commit_id, self.rhodecode_vcs_repo)
 
         if not is_head:
             h.flash(_('You can only edit files with commit '
                       'being a valid branch head.'), category='warning')
             raise HTTPFound(
                 h.route_path('repo_files',
                              repo_name=self.db_repo_name, commit_id='tip',
                              f_path=f_path))
         self.check_branch_permission(_branch_name)
 
         c.commit = self._get_commit_or_redirect(commit_id)
         c.file = self._get_filenode_or_redirect(c.commit, f_path)
 
         if c.file.is_binary:
             files_url = h.route_path(
                 'repo_files',
                 repo_name=self.db_repo_name,
                 commit_id=c.commit.raw_id, f_path=f_path)
             raise HTTPFound(files_url)
 
         c.default_message = _(
             'Edited file {} via RhodeCode Enterprise').format(f_path)
         c.f_path = f_path
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='repo_files_update_file', request_method='POST',
         renderer=None)
     def repo_files_update_file(self):
         _ = self.request.translate
         c = self.load_default_context()
         commit_id, f_path = self._get_commit_and_path()
 
         self._ensure_not_locked()
         _branch_name, _sha_commit_id, is_head = \
             self._is_valid_head(commit_id, self.rhodecode_vcs_repo)
 
         if not is_head:
             h.flash(_('You can only edit files with commit '
                       'being a valid branch head.'), category='warning')
             raise HTTPFound(
                 h.route_path('repo_files',
                              repo_name=self.db_repo_name, commit_id='tip',
                              f_path=f_path))
 
         self.check_branch_permission(_branch_name)
 
         c.commit = self._get_commit_or_redirect(commit_id)
         c.file = self._get_filenode_or_redirect(c.commit, f_path)
 
         if c.file.is_binary:
             raise HTTPFound(
                 h.route_path('repo_files',
                              repo_name=self.db_repo_name,
                              commit_id=c.commit.raw_id,
                              f_path=f_path))
 
         c.default_message = _(
             'Edited file {} via RhodeCode Enterprise').format(f_path)
         c.f_path = f_path
         old_content = c.file.content
         sl = old_content.splitlines(1)
         first_line = sl[0] if sl else ''
 
         r_post = self.request.POST
         # line endings:  0 - Unix, 1 - Mac, 2 - DOS
         line_ending_mode = detect_mode(first_line, 0)
         content = convert_line_endings(r_post.get('content', ''), line_ending_mode)
 
         message = r_post.get('message') or c.default_message
         org_f_path = c.file.unicode_path
         filename = r_post['filename']
         org_filename = c.file.name
 
         if content == old_content and filename == org_filename:
             h.flash(_('No changes'), category='warning')
             raise HTTPFound(
                 h.route_path('repo_commit', repo_name=self.db_repo_name,
                              commit_id='tip'))
         try:
             mapping = {
                 org_f_path: {
                     'org_filename': org_f_path,
                     'filename': os.path.join(c.file.dir_path, filename),
                     'content': content,
                     'lexer': '',
                     'op': 'mod',
                     'mode': c.file.mode
                 }
             }
 
             ScmModel().update_nodes(
                 user=self._rhodecode_db_user.user_id,
                 repo=self.db_repo,
                 message=message,
                 nodes=mapping,
                 parent_commit=c.commit,
             )
 
             h.flash(
                 _('Successfully committed changes to file `{}`').format(
                     h.escape(f_path)), category='success')
         except Exception:
             log.exception('Error occurred during commit')
             h.flash(_('Error occurred during commit'), category='error')
         raise HTTPFound(
             h.route_path('repo_commit', repo_name=self.db_repo_name,
                          commit_id='tip'))
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
     @view_config(
         route_name='repo_files_add_file', request_method='GET',
         renderer='rhodecode:templates/files/files_add.mako')
     def repo_files_add_file(self):
         _ = self.request.translate
         c = self.load_default_context()
         commit_id, f_path = self._get_commit_and_path()
 
         self._ensure_not_locked()
 
         c.commit = self._get_commit_or_redirect(commit_id, redirect_after=False)
         if c.commit is None:
             c.commit = EmptyCommit(alias=self.rhodecode_vcs_repo.alias)
         c.default_message = (_('Added file via RhodeCode Enterprise'))
         c.f_path = f_path.lstrip('/')  # ensure not relative path
 
         if self.rhodecode_vcs_repo.is_empty:
             # for empty repository we cannot check for current branch, we rely on
             # c.commit.branch instead
             _branch_name = c.commit.branch
             is_head = True
         else:
             _branch_name, _sha_commit_id, is_head = \
                 self._is_valid_head(commit_id, self.rhodecode_vcs_repo)
 
         if not is_head:
             h.flash(_('You can only add files with commit '
                       'being a valid branch head.'), category='warning')
             raise HTTPFound(
                 h.route_path('repo_files',
                              repo_name=self.db_repo_name, commit_id='tip',
                              f_path=f_path))
 
         self.check_branch_permission(_branch_name)
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='repo_files_create_file', request_method='POST',
         renderer=None)
     def repo_files_create_file(self):
         _ = self.request.translate
         c = self.load_default_context()
         commit_id, f_path = self._get_commit_and_path()
 
         self._ensure_not_locked()
 
         r_post = self.request.POST
 
         c.commit = self._get_commit_or_redirect(
             commit_id, redirect_after=False)
         if c.commit is None:
             c.commit = EmptyCommit(alias=self.rhodecode_vcs_repo.alias)
 
         if self.rhodecode_vcs_repo.is_empty:
             # for empty repository we cannot check for current branch, we rely on
             # c.commit.branch instead
             _branch_name = c.commit.branch
             is_head = True
         else:
             _branch_name, _sha_commit_id, is_head = \
                 self._is_valid_head(commit_id, self.rhodecode_vcs_repo)
 
         if not is_head:
             h.flash(_('You can only add files with commit '
                       'being a valid branch head.'), category='warning')
             raise HTTPFound(
                 h.route_path('repo_files',
                              repo_name=self.db_repo_name, commit_id='tip',
                              f_path=f_path))
 
         self.check_branch_permission(_branch_name)
 
         c.default_message = (_('Added file via RhodeCode Enterprise'))
         c.f_path = f_path
         unix_mode = 0
         content = convert_line_endings(r_post.get('content', ''), unix_mode)
 
         message = r_post.get('message') or c.default_message
         filename = r_post.get('filename')
         location = r_post.get('location', '')  # dir location
         file_obj = r_post.get('upload_file', None)
 
         if file_obj is not None and hasattr(file_obj, 'filename'):
             filename = r_post.get('filename_upload')
             content = file_obj.file
 
             if hasattr(content, 'file'):
                 # non posix systems store real file under file attr
                 content = content.file
 
         if self.rhodecode_vcs_repo.is_empty:
             default_redirect_url = h.route_path(
                 'repo_summary', repo_name=self.db_repo_name)
         else:
             default_redirect_url = h.route_path(
                 'repo_commit', repo_name=self.db_repo_name, commit_id='tip')
 
         # If there's no commit, redirect to repo summary
         if type(c.commit) is EmptyCommit:
             redirect_url = h.route_path(
                 'repo_summary', repo_name=self.db_repo_name)
         else:
             redirect_url = default_redirect_url
 
         if not filename:
             h.flash(_('No filename'), category='warning')
             raise HTTPFound(redirect_url)
 
         # extract the location from filename,
         # allows using foo/bar.txt syntax to create subdirectories
         subdir_loc = filename.rsplit('/', 1)
         if len(subdir_loc) == 2:
             location = os.path.join(location, subdir_loc[0])
 
         # strip all crap out of file, just leave the basename
         filename = os.path.basename(filename)
         node_path = os.path.join(location, filename)
         author = self._rhodecode_db_user.full_contact
 
         try:
             nodes = {
                 node_path: {
                     'content': content
                 }
             }
             ScmModel().create_nodes(
                 user=self._rhodecode_db_user.user_id,
                 repo=self.db_repo,
                 message=message,
                 nodes=nodes,
                 parent_commit=c.commit,
                 author=author,
             )
 
             h.flash(
                 _('Successfully committed new file `{}`').format(
                     h.escape(node_path)), category='success')
         except NonRelativePathError:
             log.exception('Non Relative path found')
             h.flash(_(
                 'The location specified must be a relative path and must not '
                 'contain .. in the path'), category='warning')
             raise HTTPFound(default_redirect_url)
         except (NodeError, NodeAlreadyExistsError) as e:
             h.flash(_(h.escape(e)), category='error')
         except Exception:
             log.exception('Error occurred during commit')
             h.flash(_('Error occurred during commit'), category='error')
 
         raise HTTPFound(default_redirect_url)

rhodecode/apps/repository/views/repo_pull_requests.py

@@ -1,1471 +1,1464 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import collections
 
 import formencode
 import formencode.htmlfill
 import peppercorn
 from pyramid.httpexceptions import (
     HTTPFound, HTTPNotFound, HTTPForbidden, HTTPBadRequest)
 from pyramid.view import view_config
 from pyramid.renderers import render
 
 from rhodecode.apps._base import RepoAppView, DataGridAppView
 
 from rhodecode.lib import helpers as h, diffs, codeblocks, channelstream
 from rhodecode.lib.base import vcs_operation_context
 from rhodecode.lib.diffs import load_cached_diff, cache_diff, diff_cache_exist
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.auth import (
     LoginRequired, HasRepoPermissionAny, HasRepoPermissionAnyDecorator,
     NotAnonymous, CSRFRequired)
 from rhodecode.lib.utils2 import str2bool, safe_str, safe_unicode
 from rhodecode.lib.vcs.backends.base import EmptyCommit, UpdateFailureReason
 from rhodecode.lib.vcs.exceptions import (CommitDoesNotExistError,
     RepositoryRequirementError, EmptyRepositoryError)
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.comment import CommentsModel
 from rhodecode.model.db import (func, or_, PullRequest, PullRequestVersion,
     ChangesetComment, ChangesetStatus, Repository)
 from rhodecode.model.forms import PullRequestForm
 from rhodecode.model.meta import Session
 from rhodecode.model.pull_request import PullRequestModel, MergeCheck
 from rhodecode.model.scm import ScmModel
 
 log = logging.getLogger(__name__)
 
 
 class RepoPullRequestsView(RepoAppView, DataGridAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context(include_app_defaults=True)
         c.REVIEW_STATUS_APPROVED = ChangesetStatus.STATUS_APPROVED
         c.REVIEW_STATUS_REJECTED = ChangesetStatus.STATUS_REJECTED
         # backward compat., we use for OLD PRs a plain renderer
         c.renderer = 'plain'
         return c
 
     def _get_pull_requests_list(
             self, repo_name, source, filter_type, opened_by, statuses):
 
         draw, start, limit = self._extract_chunk(self.request)
         search_q, order_by, order_dir = self._extract_ordering(self.request)
         _render = self.request.get_partial_renderer(
             'rhodecode:templates/data_table/_dt_elements.mako')
 
         # pagination
 
         if filter_type == 'awaiting_review':
             pull_requests = PullRequestModel().get_awaiting_review(
                 repo_name, source=source, opened_by=opened_by,
                 statuses=statuses, offset=start, length=limit,
                 order_by=order_by, order_dir=order_dir)
             pull_requests_total_count = PullRequestModel().count_awaiting_review(
                 repo_name, source=source, statuses=statuses,
                 opened_by=opened_by)
         elif filter_type == 'awaiting_my_review':
             pull_requests = PullRequestModel().get_awaiting_my_review(
                 repo_name, source=source, opened_by=opened_by,
                 user_id=self._rhodecode_user.user_id, statuses=statuses,
                 offset=start, length=limit, order_by=order_by,
                 order_dir=order_dir)
             pull_requests_total_count = PullRequestModel().count_awaiting_my_review(
                 repo_name, source=source, user_id=self._rhodecode_user.user_id,
                 statuses=statuses, opened_by=opened_by)
         else:
             pull_requests = PullRequestModel().get_all(
                 repo_name, source=source, opened_by=opened_by,
                 statuses=statuses, offset=start, length=limit,
                 order_by=order_by, order_dir=order_dir)
             pull_requests_total_count = PullRequestModel().count_all(
                 repo_name, source=source, statuses=statuses,
                 opened_by=opened_by)
 
         data = []
         comments_model = CommentsModel()
         for pr in pull_requests:
             comments = comments_model.get_all_comments(
                 self.db_repo.repo_id, pull_request=pr)
 
             data.append({
                 'name': _render('pullrequest_name',
                                 pr.pull_request_id, pr.target_repo.repo_name),
                 'name_raw': pr.pull_request_id,
                 'status': _render('pullrequest_status',
                                   pr.calculated_review_status()),
                 'title': _render(
                     'pullrequest_title', pr.title, pr.description),
                 'description': h.escape(pr.description),
                 'updated_on': _render('pullrequest_updated_on',
                                       h.datetime_to_time(pr.updated_on)),
                 'updated_on_raw': h.datetime_to_time(pr.updated_on),
                 'created_on': _render('pullrequest_updated_on',
                                       h.datetime_to_time(pr.created_on)),
                 'created_on_raw': h.datetime_to_time(pr.created_on),
                 'author': _render('pullrequest_author',
                                   pr.author.full_contact, ),
                 'author_raw': pr.author.full_name,
                 'comments': _render('pullrequest_comments', len(comments)),
                 'comments_raw': len(comments),
                 'closed': pr.is_closed(),
             })
 
         data = ({
             'draw': draw,
             'data': data,
             'recordsTotal': pull_requests_total_count,
             'recordsFiltered': pull_requests_total_count,
         })
         return data
 
-    def get_recache_flag(self):
-        for flag_name in ['force_recache', 'force-recache', 'no-cache']:
-            flag_val = self.request.GET.get(flag_name)
-            if str2bool(flag_val):
-                return True
-        return False
-
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='pullrequest_show_all', request_method='GET',
         renderer='rhodecode:templates/pullrequests/pullrequests.mako')
     def pull_request_list(self):
         c = self.load_default_context()
 
         req_get = self.request.GET
         c.source = str2bool(req_get.get('source'))
         c.closed = str2bool(req_get.get('closed'))
         c.my = str2bool(req_get.get('my'))
         c.awaiting_review = str2bool(req_get.get('awaiting_review'))
         c.awaiting_my_review = str2bool(req_get.get('awaiting_my_review'))
 
         c.active = 'open'
         if c.my:
             c.active = 'my'
         if c.closed:
             c.active = 'closed'
         if c.awaiting_review and not c.source:
             c.active = 'awaiting'
         if c.source and not c.awaiting_review:
             c.active = 'source'
         if c.awaiting_my_review:
             c.active = 'awaiting_my'
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='pullrequest_show_all_data', request_method='GET',
         renderer='json_ext', xhr=True)
     def pull_request_list_data(self):
         self.load_default_context()
 
         # additional filters
         req_get = self.request.GET
         source = str2bool(req_get.get('source'))
         closed = str2bool(req_get.get('closed'))
         my = str2bool(req_get.get('my'))
         awaiting_review = str2bool(req_get.get('awaiting_review'))
         awaiting_my_review = str2bool(req_get.get('awaiting_my_review'))
 
         filter_type = 'awaiting_review' if awaiting_review \
             else 'awaiting_my_review' if awaiting_my_review \
             else None
 
         opened_by = None
         if my:
             opened_by = [self._rhodecode_user.user_id]
 
         statuses = [PullRequest.STATUS_NEW, PullRequest.STATUS_OPEN]
         if closed:
             statuses = [PullRequest.STATUS_CLOSED]
 
         data = self._get_pull_requests_list(
             repo_name=self.db_repo_name, source=source,
             filter_type=filter_type, opened_by=opened_by, statuses=statuses)
 
         return data
 
     def _is_diff_cache_enabled(self, target_repo):
         caching_enabled = self._get_general_setting(
             target_repo, 'rhodecode_diff_cache')
         log.debug('Diff caching enabled: %s', caching_enabled)
         return caching_enabled
 
     def _get_diffset(self, source_repo_name, source_repo,
                      source_ref_id, target_ref_id,
                      target_commit, source_commit, diff_limit, file_limit,
                      fulldiff, hide_whitespace_changes, diff_context):
 
         vcs_diff = PullRequestModel().get_diff(
             source_repo, source_ref_id, target_ref_id,
             hide_whitespace_changes, diff_context)
 
         diff_processor = diffs.DiffProcessor(
             vcs_diff, format='newdiff', diff_limit=diff_limit,
             file_limit=file_limit, show_full_diff=fulldiff)
 
         _parsed = diff_processor.prepare()
 
         diffset = codeblocks.DiffSet(
             repo_name=self.db_repo_name,
             source_repo_name=source_repo_name,
             source_node_getter=codeblocks.diffset_node_getter(target_commit),
             target_node_getter=codeblocks.diffset_node_getter(source_commit),
         )
         diffset = self.path_filter.render_patchset_filtered(
             diffset, _parsed, target_commit.raw_id, source_commit.raw_id)
 
         return diffset
 
     def _get_range_diffset(self, source_scm, source_repo,
                            commit1, commit2, diff_limit, file_limit,
                            fulldiff, hide_whitespace_changes, diff_context):
         vcs_diff = source_scm.get_diff(
             commit1, commit2,
             ignore_whitespace=hide_whitespace_changes,
             context=diff_context)
 
         diff_processor = diffs.DiffProcessor(
             vcs_diff, format='newdiff', diff_limit=diff_limit,
             file_limit=file_limit, show_full_diff=fulldiff)
 
         _parsed = diff_processor.prepare()
 
         diffset = codeblocks.DiffSet(
             repo_name=source_repo.repo_name,
             source_node_getter=codeblocks.diffset_node_getter(commit1),
             target_node_getter=codeblocks.diffset_node_getter(commit2))
 
         diffset = self.path_filter.render_patchset_filtered(
             diffset, _parsed, commit1.raw_id, commit2.raw_id)
 
         return diffset
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='pullrequest_show', request_method='GET',
         renderer='rhodecode:templates/pullrequests/pullrequest_show.mako')
     def pull_request_show(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         pull_request_id = pull_request.pull_request_id
 
         if pull_request.pull_request_state != PullRequest.STATE_CREATED:
             log.debug('show: forbidden because pull request is in state %s',
                       pull_request.pull_request_state)
             msg = _(u'Cannot show pull requests in state other than `{}`. '
                     u'Current state is: `{}`').format(PullRequest.STATE_CREATED,
                                                       pull_request.pull_request_state)
             h.flash(msg, category='error')
             raise HTTPFound(h.route_path('pullrequest_show_all',
                                          repo_name=self.db_repo_name))
 
         version = self.request.GET.get('version')
         from_version = self.request.GET.get('from_version') or version
         merge_checks = self.request.GET.get('merge_checks')
         c.fulldiff = str2bool(self.request.GET.get('fulldiff'))
 
         # fetch global flags of ignore ws or context lines
         diff_context = diffs.get_diff_context(self.request)
         hide_whitespace_changes = diffs.get_diff_whitespace_flag(self.request)
 
         force_refresh = str2bool(self.request.GET.get('force_refresh'))
 
         (pull_request_latest,
          pull_request_at_ver,
          pull_request_display_obj,
          at_version) = PullRequestModel().get_pr_version(
             pull_request_id, version=version)
         pr_closed = pull_request_latest.is_closed()
 
         if pr_closed and (version or from_version):
             # not allow to browse versions
             raise HTTPFound(h.route_path(
                 'pullrequest_show', repo_name=self.db_repo_name,
                 pull_request_id=pull_request_id))
 
         versions = pull_request_display_obj.versions()
         # used to store per-commit range diffs
         c.changes = collections.OrderedDict()
         c.range_diff_on = self.request.GET.get('range-diff') == "1"
 
         c.at_version = at_version
         c.at_version_num = (at_version
                             if at_version and at_version != 'latest'
                             else None)
         c.at_version_pos = ChangesetComment.get_index_from_version(
             c.at_version_num, versions)
 
         (prev_pull_request_latest,
          prev_pull_request_at_ver,
          prev_pull_request_display_obj,
          prev_at_version) = PullRequestModel().get_pr_version(
             pull_request_id, version=from_version)
 
         c.from_version = prev_at_version
         c.from_version_num = (prev_at_version
                               if prev_at_version and prev_at_version != 'latest'
                               else None)
         c.from_version_pos = ChangesetComment.get_index_from_version(
             c.from_version_num, versions)
 
         # define if we're in COMPARE mode or VIEW at version mode
         compare = at_version != prev_at_version
 
         # pull_requests repo_name we opened it against
         # ie. target_repo must match
         if self.db_repo_name != pull_request_at_ver.target_repo.repo_name:
             raise HTTPNotFound()
 
         c.shadow_clone_url = PullRequestModel().get_shadow_clone_url(
             pull_request_at_ver)
 
         c.pull_request = pull_request_display_obj
         c.renderer = pull_request_at_ver.description_renderer or c.renderer
         c.pull_request_latest = pull_request_latest
 
         if compare or (at_version and not at_version == 'latest'):
             c.allowed_to_change_status = False
             c.allowed_to_update = False
             c.allowed_to_merge = False
             c.allowed_to_delete = False
             c.allowed_to_comment = False
             c.allowed_to_close = False
         else:
             can_change_status = PullRequestModel().check_user_change_status(
                 pull_request_at_ver, self._rhodecode_user)
             c.allowed_to_change_status = can_change_status and not pr_closed
 
             c.allowed_to_update = PullRequestModel().check_user_update(
                 pull_request_latest, self._rhodecode_user) and not pr_closed
             c.allowed_to_merge = PullRequestModel().check_user_merge(
                 pull_request_latest, self._rhodecode_user) and not pr_closed
             c.allowed_to_delete = PullRequestModel().check_user_delete(
                 pull_request_latest, self._rhodecode_user) and not pr_closed
             c.allowed_to_comment = not pr_closed
             c.allowed_to_close = c.allowed_to_merge and not pr_closed
 
         c.forbid_adding_reviewers = False
         c.forbid_author_to_review = False
         c.forbid_commit_author_to_review = False
 
         if pull_request_latest.reviewer_data and \
                         'rules' in pull_request_latest.reviewer_data:
             rules = pull_request_latest.reviewer_data['rules'] or {}
             try:
                 c.forbid_adding_reviewers = rules.get(
                     'forbid_adding_reviewers')
                 c.forbid_author_to_review = rules.get(
                     'forbid_author_to_review')
                 c.forbid_commit_author_to_review = rules.get(
                     'forbid_commit_author_to_review')
             except Exception:
                 pass
 
         # check merge capabilities
         _merge_check = MergeCheck.validate(
             pull_request_latest, auth_user=self._rhodecode_user,
             translator=self.request.translate,
             force_shadow_repo_refresh=force_refresh)
         c.pr_merge_errors = _merge_check.error_details
         c.pr_merge_possible = not _merge_check.failed
         c.pr_merge_message = _merge_check.merge_msg
 
         c.pr_merge_info = MergeCheck.get_merge_conditions(
             pull_request_latest, translator=self.request.translate)
 
         c.pull_request_review_status = _merge_check.review_status
         if merge_checks:
             self.request.override_renderer = \
                 'rhodecode:templates/pullrequests/pullrequest_merge_checks.mako'
             return self._get_template_context(c)
 
         comments_model = CommentsModel()
 
         # reviewers and statuses
         c.pull_request_reviewers = pull_request_at_ver.reviewers_statuses()
         allowed_reviewers = [x[0].user_id for x in c.pull_request_reviewers]
 
         # GENERAL COMMENTS with versions #
         q = comments_model._all_general_comments_of_pull_request(pull_request_latest)
         q = q.order_by(ChangesetComment.comment_id.asc())
         general_comments = q
 
         # pick comments we want to render at current version
         c.comment_versions = comments_model.aggregate_comments(
             general_comments, versions, c.at_version_num)
         c.comments = c.comment_versions[c.at_version_num]['until']
 
         # INLINE COMMENTS with versions  #
         q = comments_model._all_inline_comments_of_pull_request(pull_request_latest)
         q = q.order_by(ChangesetComment.comment_id.asc())
         inline_comments = q
 
         c.inline_versions = comments_model.aggregate_comments(
             inline_comments, versions, c.at_version_num, inline=True)
 
         # inject latest version
         latest_ver = PullRequest.get_pr_display_object(
             pull_request_latest, pull_request_latest)
 
         c.versions = versions + [latest_ver]
 
         # if we use version, then do not show later comments
         # than current version
         display_inline_comments = collections.defaultdict(
             lambda: collections.defaultdict(list))
         for co in inline_comments:
             if c.at_version_num:
                 # pick comments that are at least UPTO given version, so we
                 # don't render comments for higher version
                 should_render = co.pull_request_version_id and \
                                 co.pull_request_version_id <= c.at_version_num
             else:
                 # showing all, for 'latest'
                 should_render = True
 
             if should_render:
                 display_inline_comments[co.f_path][co.line_no].append(co)
 
         # load diff data into template context, if we use compare mode then
         # diff is calculated based on changes between versions of PR
 
         source_repo = pull_request_at_ver.source_repo
         source_ref_id = pull_request_at_ver.source_ref_parts.commit_id
 
         target_repo = pull_request_at_ver.target_repo
         target_ref_id = pull_request_at_ver.target_ref_parts.commit_id
 
         if compare:
             # in compare switch the diff base to latest commit from prev version
             target_ref_id = prev_pull_request_display_obj.revisions[0]
 
         # despite opening commits for bookmarks/branches/tags, we always
         # convert this to rev to prevent changes after bookmark or branch change
         c.source_ref_type = 'rev'
         c.source_ref = source_ref_id
 
         c.target_ref_type = 'rev'
         c.target_ref = target_ref_id
 
         c.source_repo = source_repo
         c.target_repo = target_repo
 
         c.commit_ranges = []
         source_commit = EmptyCommit()
         target_commit = EmptyCommit()
         c.missing_requirements = False
 
         source_scm = source_repo.scm_instance()
         target_scm = target_repo.scm_instance()
 
         shadow_scm = None
         try:
             shadow_scm = pull_request_latest.get_shadow_repo()
         except Exception:
             log.debug('Failed to get shadow repo', exc_info=True)
         # try first the existing source_repo, and then shadow
         # repo if we can obtain one
         commits_source_repo = source_scm or shadow_scm
 
         c.commits_source_repo = commits_source_repo
         c.ancestor = None  # set it to None, to hide it from PR view
 
         # empty version means latest, so we keep this to prevent
         # double caching
         version_normalized = version or 'latest'
         from_version_normalized = from_version or 'latest'
 
         cache_path = self.rhodecode_vcs_repo.get_create_shadow_cache_pr_path(target_repo)
         cache_file_path = diff_cache_exist(
             cache_path, 'pull_request', pull_request_id, version_normalized,
             from_version_normalized, source_ref_id, target_ref_id,
             hide_whitespace_changes, diff_context, c.fulldiff)
 
         caching_enabled = self._is_diff_cache_enabled(c.target_repo)
         force_recache = self.get_recache_flag()
 
         cached_diff = None
         if caching_enabled:
             cached_diff = load_cached_diff(cache_file_path)
 
         has_proper_commit_cache = (
                 cached_diff and cached_diff.get('commits')
                 and len(cached_diff.get('commits', [])) == 5
                 and cached_diff.get('commits')[0]
                 and cached_diff.get('commits')[3])
 
         if not force_recache and not c.range_diff_on and has_proper_commit_cache:
             diff_commit_cache = \
                 (ancestor_commit, commit_cache, missing_requirements,
                  source_commit, target_commit) = cached_diff['commits']
         else:
             diff_commit_cache = \
                 (ancestor_commit, commit_cache, missing_requirements,
                  source_commit, target_commit) = self.get_commits(
                     commits_source_repo,
                     pull_request_at_ver,
                     source_commit,
                     source_ref_id,
                     source_scm,
                     target_commit,
                     target_ref_id,
                     target_scm)
 
         # register our commit range
         for comm in commit_cache.values():
             c.commit_ranges.append(comm)
 
         c.missing_requirements = missing_requirements
         c.ancestor_commit = ancestor_commit
         c.statuses = source_repo.statuses(
             [x.raw_id for x in c.commit_ranges])
 
         # auto collapse if we have more than limit
         collapse_limit = diffs.DiffProcessor._collapse_commits_over
         c.collapse_all_commits = len(c.commit_ranges) > collapse_limit
         c.compare_mode = compare
 
         # diff_limit is the old behavior, will cut off the whole diff
         # if the limit is applied  otherwise will just hide the
         # big files from the front-end
         diff_limit = c.visual.cut_off_limit_diff
         file_limit = c.visual.cut_off_limit_file
 
         c.missing_commits = False
         if (c.missing_requirements
             or isinstance(source_commit, EmptyCommit)
             or source_commit == target_commit):
 
             c.missing_commits = True
         else:
             c.inline_comments = display_inline_comments
 
             has_proper_diff_cache = cached_diff and cached_diff.get('commits')
             if not force_recache and has_proper_diff_cache:
                 c.diffset = cached_diff['diff']
                 (ancestor_commit, commit_cache, missing_requirements,
                  source_commit, target_commit) = cached_diff['commits']
             else:
                 c.diffset = self._get_diffset(
                     c.source_repo.repo_name, commits_source_repo,
                     source_ref_id, target_ref_id,
                     target_commit, source_commit,
                     diff_limit, file_limit, c.fulldiff,
                     hide_whitespace_changes, diff_context)
 
                 # save cached diff
                 if caching_enabled:
                     cache_diff(cache_file_path, c.diffset, diff_commit_cache)
 
             c.limited_diff = c.diffset.limited_diff
 
             # calculate removed files that are bound to comments
             comment_deleted_files = [
                 fname for fname in display_inline_comments
                 if fname not in c.diffset.file_stats]
 
             c.deleted_files_comments = collections.defaultdict(dict)
             for fname, per_line_comments in display_inline_comments.items():
                 if fname in comment_deleted_files:
                     c.deleted_files_comments[fname]['stats'] = 0
                     c.deleted_files_comments[fname]['comments'] = list()
                     for lno, comments in per_line_comments.items():
                         c.deleted_files_comments[fname]['comments'].extend(comments)
 
             # maybe calculate the range diff
             if c.range_diff_on:
                 # TODO(marcink): set whitespace/context
                 context_lcl = 3
                 ign_whitespace_lcl = False
 
                 for commit in c.commit_ranges:
                     commit2 = commit
                     commit1 = commit.first_parent
 
                     range_diff_cache_file_path = diff_cache_exist(
                         cache_path, 'diff', commit.raw_id,
                         ign_whitespace_lcl, context_lcl, c.fulldiff)
 
                     cached_diff = None
                     if caching_enabled:
                         cached_diff = load_cached_diff(range_diff_cache_file_path)
 
                     has_proper_diff_cache = cached_diff and cached_diff.get('diff')
                     if not force_recache and has_proper_diff_cache:
                         diffset = cached_diff['diff']
                     else:
                         diffset = self._get_range_diffset(
                             source_scm, source_repo,
                             commit1, commit2, diff_limit, file_limit,
                             c.fulldiff, ign_whitespace_lcl, context_lcl
                         )
 
                     # save cached diff
                     if caching_enabled:
                         cache_diff(range_diff_cache_file_path, diffset, None)
 
                     c.changes[commit.raw_id] = diffset
 
         # this is a hack to properly display links, when creating PR, the
         # compare view and others uses different notation, and
         # compare_commits.mako renders links based on the target_repo.
         # We need to swap that here to generate it properly on the html side
         c.target_repo = c.source_repo
 
         c.commit_statuses = ChangesetStatus.STATUSES
 
         c.show_version_changes = not pr_closed
         if c.show_version_changes:
             cur_obj = pull_request_at_ver
             prev_obj = prev_pull_request_at_ver
 
             old_commit_ids = prev_obj.revisions
             new_commit_ids = cur_obj.revisions
             commit_changes = PullRequestModel()._calculate_commit_id_changes(
                 old_commit_ids, new_commit_ids)
             c.commit_changes_summary = commit_changes
 
             # calculate the diff for commits between versions
             c.commit_changes = []
             mark = lambda cs, fw: list(
                 h.itertools.izip_longest([], cs, fillvalue=fw))
             for c_type, raw_id in mark(commit_changes.added, 'a') \
                                 + mark(commit_changes.removed, 'r') \
                                 + mark(commit_changes.common, 'c'):
 
                 if raw_id in commit_cache:
                     commit = commit_cache[raw_id]
                 else:
                     try:
                         commit = commits_source_repo.get_commit(raw_id)
                     except CommitDoesNotExistError:
                         # in case we fail extracting still use "dummy" commit
                         # for display in commit diff
                         commit = h.AttributeDict(
                             {'raw_id': raw_id,
                              'message': 'EMPTY or MISSING COMMIT'})
                 c.commit_changes.append([c_type, commit])
 
             # current user review statuses for each version
             c.review_versions = {}
             if self._rhodecode_user.user_id in allowed_reviewers:
                 for co in general_comments:
                     if co.author.user_id == self._rhodecode_user.user_id:
                         status = co.status_change
                         if status:
                             _ver_pr = status[0].comment.pull_request_version_id
                             c.review_versions[_ver_pr] = status[0]
 
         return self._get_template_context(c)
 
     def get_commits(
             self, commits_source_repo, pull_request_at_ver, source_commit,
             source_ref_id, source_scm, target_commit, target_ref_id, target_scm):
         commit_cache = collections.OrderedDict()
         missing_requirements = False
         try:
             pre_load = ["author", "branch", "date", "message", "parents"]
             show_revs = pull_request_at_ver.revisions
             for rev in show_revs:
                 comm = commits_source_repo.get_commit(
                     commit_id=rev, pre_load=pre_load)
                 commit_cache[comm.raw_id] = comm
 
             # Order here matters, we first need to get target, and then
             # the source
             target_commit = commits_source_repo.get_commit(
                 commit_id=safe_str(target_ref_id))
 
             source_commit = commits_source_repo.get_commit(
                 commit_id=safe_str(source_ref_id))
         except CommitDoesNotExistError:
             log.warning(
                 'Failed to get commit from `{}` repo'.format(
                     commits_source_repo), exc_info=True)
         except RepositoryRequirementError:
             log.warning(
                 'Failed to get all required data from repo', exc_info=True)
             missing_requirements = True
         ancestor_commit = None
         try:
             ancestor_id = source_scm.get_common_ancestor(
                 source_commit.raw_id, target_commit.raw_id, target_scm)
             ancestor_commit = source_scm.get_commit(ancestor_id)
         except Exception:
             ancestor_commit = None
         return ancestor_commit, commit_cache, missing_requirements, source_commit, target_commit
 
     def assure_not_empty_repo(self):
         _ = self.request.translate
 
         try:
             self.db_repo.scm_instance().get_commit()
         except EmptyRepositoryError:
             h.flash(h.literal(_('There are no commits yet')),
                     category='warning')
             raise HTTPFound(
                 h.route_path('repo_summary', repo_name=self.db_repo.repo_name))
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='pullrequest_new', request_method='GET',
         renderer='rhodecode:templates/pullrequests/pullrequest.mako')
     def pull_request_new(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         self.assure_not_empty_repo()
         source_repo = self.db_repo
 
         commit_id = self.request.GET.get('commit')
         branch_ref = self.request.GET.get('branch')
         bookmark_ref = self.request.GET.get('bookmark')
 
         try:
             source_repo_data = PullRequestModel().generate_repo_data(
                 source_repo, commit_id=commit_id,
                 branch=branch_ref, bookmark=bookmark_ref,
                 translator=self.request.translate)
         except CommitDoesNotExistError as e:
             log.exception(e)
             h.flash(_('Commit does not exist'), 'error')
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=source_repo.repo_name))
 
         default_target_repo = source_repo
 
         if source_repo.parent and c.has_origin_repo_read_perm:
             parent_vcs_obj = source_repo.parent.scm_instance()
             if parent_vcs_obj and not parent_vcs_obj.is_empty():
                 # change default if we have a parent repo
                 default_target_repo = source_repo.parent
 
         target_repo_data = PullRequestModel().generate_repo_data(
             default_target_repo, translator=self.request.translate)
 
         selected_source_ref = source_repo_data['refs']['selected_ref']
         title_source_ref = ''
         if selected_source_ref:
             title_source_ref = selected_source_ref.split(':', 2)[1]
         c.default_title = PullRequestModel().generate_pullrequest_title(
             source=source_repo.repo_name,
             source_ref=title_source_ref,
             target=default_target_repo.repo_name
         )
 
         c.default_repo_data = {
             'source_repo_name': source_repo.repo_name,
             'source_refs_json': json.dumps(source_repo_data),
             'target_repo_name': default_target_repo.repo_name,
             'target_refs_json': json.dumps(target_repo_data),
         }
         c.default_source_ref = selected_source_ref
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='pullrequest_repo_refs', request_method='GET',
         renderer='json_ext', xhr=True)
     def pull_request_repo_refs(self):
         self.load_default_context()
         target_repo_name = self.request.matchdict['target_repo_name']
         repo = Repository.get_by_repo_name(target_repo_name)
         if not repo:
             raise HTTPNotFound()
 
         target_perm = HasRepoPermissionAny(
             'repository.read', 'repository.write', 'repository.admin')(
             target_repo_name)
         if not target_perm:
             raise HTTPNotFound()
 
         return PullRequestModel().generate_repo_data(
             repo, translator=self.request.translate)
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='pullrequest_repo_targets', request_method='GET',
         renderer='json_ext', xhr=True)
     def pullrequest_repo_targets(self):
         _ = self.request.translate
         filter_query = self.request.GET.get('query')
 
         # get the parents
         parent_target_repos = []
         if self.db_repo.parent:
             parents_query = Repository.query() \
                 .order_by(func.length(Repository.repo_name)) \
                 .filter(Repository.fork_id == self.db_repo.parent.repo_id)
 
             if filter_query:
                 ilike_expression = u'%{}%'.format(safe_unicode(filter_query))
                 parents_query = parents_query.filter(
                     Repository.repo_name.ilike(ilike_expression))
             parents = parents_query.limit(20).all()
 
             for parent in parents:
                 parent_vcs_obj = parent.scm_instance()
                 if parent_vcs_obj and not parent_vcs_obj.is_empty():
                     parent_target_repos.append(parent)
 
         # get other forks, and repo itself
         query = Repository.query() \
             .order_by(func.length(Repository.repo_name)) \
             .filter(
                 or_(Repository.repo_id == self.db_repo.repo_id,  # repo itself
                     Repository.fork_id == self.db_repo.repo_id)  # forks of this repo
             )  \
             .filter(~Repository.repo_id.in_([x.repo_id for x in parent_target_repos]))
 
         if filter_query:
             ilike_expression = u'%{}%'.format(safe_unicode(filter_query))
             query = query.filter(Repository.repo_name.ilike(ilike_expression))
 
         limit = max(20 - len(parent_target_repos), 5)  # not less then 5
         target_repos = query.limit(limit).all()
 
         all_target_repos = target_repos + parent_target_repos
 
         repos = []
         # This checks permissions to the repositories
         for obj in ScmModel().get_repos(all_target_repos):
             repos.append({
                 'id': obj['name'],
                 'text': obj['name'],
                 'type': 'repo',
                 'repo_id': obj['dbrepo']['repo_id'],
                 'repo_type': obj['dbrepo']['repo_type'],
                 'private': obj['dbrepo']['private'],
 
             })
 
         data = {
             'more': False,
             'results': [{
                 'text': _('Repositories'),
                 'children': repos
             }] if repos else []
         }
         return data
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='pullrequest_create', request_method='POST',
         renderer=None)
     def pull_request_create(self):
         _ = self.request.translate
         self.assure_not_empty_repo()
         self.load_default_context()
 
         controls = peppercorn.parse(self.request.POST.items())
 
         try:
             form = PullRequestForm(
                 self.request.translate, self.db_repo.repo_id)()
             _form = form.to_python(controls)
         except formencode.Invalid as errors:
             if errors.error_dict.get('revisions'):
                 msg = 'Revisions: %s' % errors.error_dict['revisions']
             elif errors.error_dict.get('pullrequest_title'):
                 msg = errors.error_dict.get('pullrequest_title')
             else:
                 msg = _('Error creating pull request: {}').format(errors)
             log.exception(msg)
             h.flash(msg, 'error')
 
             # would rather just go back to form ...
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=self.db_repo_name))
 
         source_repo = _form['source_repo']
         source_ref = _form['source_ref']
         target_repo = _form['target_repo']
         target_ref = _form['target_ref']
         commit_ids = _form['revisions'][::-1]
 
         # find the ancestor for this pr
         source_db_repo = Repository.get_by_repo_name(_form['source_repo'])
         target_db_repo = Repository.get_by_repo_name(_form['target_repo'])
 
         if not (source_db_repo or target_db_repo):
             h.flash(_('source_repo or target repo not found'), category='error')
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=self.db_repo_name))
 
         # re-check permissions again here
         # source_repo we must have read permissions
 
         source_perm = HasRepoPermissionAny(
             'repository.read', 'repository.write', 'repository.admin')(
             source_db_repo.repo_name)
         if not source_perm:
             msg = _('Not Enough permissions to source repo `{}`.'.format(
                 source_db_repo.repo_name))
             h.flash(msg, category='error')
             # copy the args back to redirect
             org_query = self.request.GET.mixed()
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=self.db_repo_name,
                              _query=org_query))
 
         # target repo we must have read permissions, and also later on
         # we want to check branch permissions here
         target_perm = HasRepoPermissionAny(
             'repository.read', 'repository.write', 'repository.admin')(
             target_db_repo.repo_name)
         if not target_perm:
             msg = _('Not Enough permissions to target repo `{}`.'.format(
                 target_db_repo.repo_name))
             h.flash(msg, category='error')
             # copy the args back to redirect
             org_query = self.request.GET.mixed()
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=self.db_repo_name,
                              _query=org_query))
 
         source_scm = source_db_repo.scm_instance()
         target_scm = target_db_repo.scm_instance()
 
         source_commit = source_scm.get_commit(source_ref.split(':')[-1])
         target_commit = target_scm.get_commit(target_ref.split(':')[-1])
 
         ancestor = source_scm.get_common_ancestor(
             source_commit.raw_id, target_commit.raw_id, target_scm)
 
         # recalculate target ref based on ancestor
         target_ref_type, target_ref_name, __ = _form['target_ref'].split(':')
         target_ref = ':'.join((target_ref_type, target_ref_name, ancestor))
 
         get_default_reviewers_data, validate_default_reviewers = \
             PullRequestModel().get_reviewer_functions()
 
         # recalculate reviewers logic, to make sure we can validate this
         reviewer_rules = get_default_reviewers_data(
             self._rhodecode_db_user, source_db_repo,
             source_commit, target_db_repo, target_commit)
 
         given_reviewers = _form['review_members']
         reviewers = validate_default_reviewers(
             given_reviewers, reviewer_rules)
 
         pullrequest_title = _form['pullrequest_title']
         title_source_ref = source_ref.split(':', 2)[1]
         if not pullrequest_title:
             pullrequest_title = PullRequestModel().generate_pullrequest_title(
                 source=source_repo,
                 source_ref=title_source_ref,
                 target=target_repo
             )
 
         description = _form['pullrequest_desc']
         description_renderer = _form['description_renderer']
 
         try:
             pull_request = PullRequestModel().create(
                 created_by=self._rhodecode_user.user_id,
                 source_repo=source_repo,
                 source_ref=source_ref,
                 target_repo=target_repo,
                 target_ref=target_ref,
                 revisions=commit_ids,
                 reviewers=reviewers,
                 title=pullrequest_title,
                 description=description,
                 description_renderer=description_renderer,
                 reviewer_data=reviewer_rules,
                 auth_user=self._rhodecode_user
             )
             Session().commit()
 
             h.flash(_('Successfully opened new pull request'),
                     category='success')
         except Exception:
             msg = _('Error occurred during creation of this pull request.')
             log.exception(msg)
             h.flash(msg, category='error')
 
             # copy the args back to redirect
             org_query = self.request.GET.mixed()
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=self.db_repo_name,
                              _query=org_query))
 
         raise HTTPFound(
             h.route_path('pullrequest_show', repo_name=target_repo,
                          pull_request_id=pull_request.pull_request_id))
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='pullrequest_update', request_method='POST',
         renderer='json_ext')
     def pull_request_update(self):
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         _ = self.request.translate
 
         self.load_default_context()
 
         if pull_request.is_closed():
             log.debug('update: forbidden because pull request is closed')
             msg = _(u'Cannot update closed pull requests.')
             h.flash(msg, category='error')
             return True
 
         if pull_request.pull_request_state != PullRequest.STATE_CREATED:
             log.debug('update: forbidden because pull request is in state %s',
                       pull_request.pull_request_state)
             msg = _(u'Cannot update pull requests in state other than `{}`. '
                     u'Current state is: `{}`').format(PullRequest.STATE_CREATED,
                                                       pull_request.pull_request_state)
             h.flash(msg, category='error')
             return True
 
         # only owner or admin can update it
         allowed_to_update = PullRequestModel().check_user_update(
             pull_request, self._rhodecode_user)
         if allowed_to_update:
             controls = peppercorn.parse(self.request.POST.items())
 
             if 'review_members' in controls:
                 self._update_reviewers(
                     pull_request, controls['review_members'],
                     pull_request.reviewer_data)
             elif str2bool(self.request.POST.get('update_commits', 'false')):
                 self._update_commits(pull_request)
             elif str2bool(self.request.POST.get('edit_pull_request', 'false')):
                 self._edit_pull_request(pull_request)
             else:
                 raise HTTPBadRequest()
             return True
         raise HTTPForbidden()
 
     def _edit_pull_request(self, pull_request):
         _ = self.request.translate
 
         try:
             PullRequestModel().edit(
                 pull_request,
                 self.request.POST.get('title'),
                 self.request.POST.get('description'),
                 self.request.POST.get('description_renderer'),
                 self._rhodecode_user)
         except ValueError:
             msg = _(u'Cannot update closed pull requests.')
             h.flash(msg, category='error')
             return
         else:
             Session().commit()
 
         msg = _(u'Pull request title & description updated.')
         h.flash(msg, category='success')
         return
 
     def _update_commits(self, pull_request):
         _ = self.request.translate
 
         with pull_request.set_state(PullRequest.STATE_UPDATING):
             resp = PullRequestModel().update_commits(pull_request)
 
         if resp.executed:
 
             if resp.target_changed and resp.source_changed:
                 changed = 'target and source repositories'
             elif resp.target_changed and not resp.source_changed:
                 changed = 'target repository'
             elif not resp.target_changed and resp.source_changed:
                 changed = 'source repository'
             else:
                 changed = 'nothing'
 
             msg = _(u'Pull request updated to "{source_commit_id}" with '
                     u'{count_added} added, {count_removed} removed commits. '
                     u'Source of changes: {change_source}')
             msg = msg.format(
                 source_commit_id=pull_request.source_ref_parts.commit_id,
                 count_added=len(resp.changes.added),
                 count_removed=len(resp.changes.removed),
                 change_source=changed)
             h.flash(msg, category='success')
 
             channel = '/repo${}$/pr/{}'.format(
                 pull_request.target_repo.repo_name, pull_request.pull_request_id)
             message = msg + (
                 ' - <a onclick="window.location.reload()">'
                 '<strong>{}</strong></a>'.format(_('Reload page')))
             channelstream.post_message(
                 channel, message, self._rhodecode_user.username,
                 registry=self.request.registry)
         else:
             msg = PullRequestModel.UPDATE_STATUS_MESSAGES[resp.reason]
             warning_reasons = [
                 UpdateFailureReason.NO_CHANGE,
                 UpdateFailureReason.WRONG_REF_TYPE,
             ]
             category = 'warning' if resp.reason in warning_reasons else 'error'
             h.flash(msg, category=category)
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='pullrequest_merge', request_method='POST',
         renderer='json_ext')
     def pull_request_merge(self):
         """
         Merge will perform a server-side merge of the specified
         pull request, if the pull request is approved and mergeable.
         After successful merging, the pull request is automatically
         closed, with a relevant comment.
         """
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         _ = self.request.translate
 
         if pull_request.pull_request_state != PullRequest.STATE_CREATED:
             log.debug('show: forbidden because pull request is in state %s',
                       pull_request.pull_request_state)
             msg = _(u'Cannot merge pull requests in state other than `{}`. '
                     u'Current state is: `{}`').format(PullRequest.STATE_CREATED,
                                                       pull_request.pull_request_state)
             h.flash(msg, category='error')
             raise HTTPFound(
                 h.route_path('pullrequest_show',
                              repo_name=pull_request.target_repo.repo_name,
                              pull_request_id=pull_request.pull_request_id))
 
         self.load_default_context()
 
         with pull_request.set_state(PullRequest.STATE_UPDATING):
             check = MergeCheck.validate(
                 pull_request, auth_user=self._rhodecode_user,
                 translator=self.request.translate)
         merge_possible = not check.failed
 
         for err_type, error_msg in check.errors:
             h.flash(error_msg, category=err_type)
 
         if merge_possible:
             log.debug("Pre-conditions checked, trying to merge.")
             extras = vcs_operation_context(
                 self.request.environ, repo_name=pull_request.target_repo.repo_name,
                 username=self._rhodecode_db_user.username, action='push',
                 scm=pull_request.target_repo.repo_type)
             with pull_request.set_state(PullRequest.STATE_UPDATING):
                 self._merge_pull_request(
                     pull_request, self._rhodecode_db_user, extras)
         else:
             log.debug("Pre-conditions failed, NOT merging.")
 
         raise HTTPFound(
             h.route_path('pullrequest_show',
                          repo_name=pull_request.target_repo.repo_name,
                          pull_request_id=pull_request.pull_request_id))
 
     def _merge_pull_request(self, pull_request, user, extras):
         _ = self.request.translate
         merge_resp = PullRequestModel().merge_repo(pull_request, user, extras=extras)
 
         if merge_resp.executed:
             log.debug("The merge was successful, closing the pull request.")
             PullRequestModel().close_pull_request(
                 pull_request.pull_request_id, user)
             Session().commit()
             msg = _('Pull request was successfully merged and closed.')
             h.flash(msg, category='success')
         else:
             log.debug(
                 "The merge was not successful. Merge response: %s", merge_resp)
             msg = merge_resp.merge_status_message
             h.flash(msg, category='error')
 
     def _update_reviewers(self, pull_request, review_members, reviewer_rules):
         _ = self.request.translate
 
         get_default_reviewers_data, validate_default_reviewers = \
             PullRequestModel().get_reviewer_functions()
 
         try:
             reviewers = validate_default_reviewers(review_members, reviewer_rules)
         except ValueError as e:
             log.error('Reviewers Validation: {}'.format(e))
             h.flash(e, category='error')
             return
 
         old_calculated_status = pull_request.calculated_review_status()
         PullRequestModel().update_reviewers(
             pull_request, reviewers, self._rhodecode_user)
         h.flash(_('Pull request reviewers updated.'), category='success')
         Session().commit()
 
         # trigger status changed if change in reviewers changes the status
         calculated_status = pull_request.calculated_review_status()
         if old_calculated_status != calculated_status:
             PullRequestModel().trigger_pull_request_hook(
                 pull_request, self._rhodecode_user, 'review_status_change',
                 data={'status': calculated_status})
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='pullrequest_delete', request_method='POST',
         renderer='json_ext')
     def pull_request_delete(self):
         _ = self.request.translate
 
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         self.load_default_context()
 
         pr_closed = pull_request.is_closed()
         allowed_to_delete = PullRequestModel().check_user_delete(
             pull_request, self._rhodecode_user) and not pr_closed
 
         # only owner can delete it !
         if allowed_to_delete:
             PullRequestModel().delete(pull_request, self._rhodecode_user)
             Session().commit()
             h.flash(_('Successfully deleted pull request'),
                     category='success')
             raise HTTPFound(h.route_path('pullrequest_show_all',
                                          repo_name=self.db_repo_name))
 
         log.warning('user %s tried to delete pull request without access',
                     self._rhodecode_user)
         raise HTTPNotFound()
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='pullrequest_comment_create', request_method='POST',
         renderer='json_ext')
     def pull_request_comment_create(self):
         _ = self.request.translate
 
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         pull_request_id = pull_request.pull_request_id
 
         if pull_request.is_closed():
             log.debug('comment: forbidden because pull request is closed')
             raise HTTPForbidden()
 
         allowed_to_comment = PullRequestModel().check_user_comment(
             pull_request, self._rhodecode_user)
         if not allowed_to_comment:
             log.debug(
                 'comment: forbidden because pull request is from forbidden repo')
             raise HTTPForbidden()
 
         c = self.load_default_context()
 
         status = self.request.POST.get('changeset_status', None)
         text = self.request.POST.get('text')
         comment_type = self.request.POST.get('comment_type')
         resolves_comment_id = self.request.POST.get('resolves_comment_id', None)
         close_pull_request = self.request.POST.get('close_pull_request')
 
         # the logic here should work like following, if we submit close
         # pr comment, use `close_pull_request_with_comment` function
         # else handle regular comment logic
 
         if close_pull_request:
             # only owner or admin or person with write permissions
             allowed_to_close = PullRequestModel().check_user_update(
                 pull_request, self._rhodecode_user)
             if not allowed_to_close:
                 log.debug('comment: forbidden because not allowed to close '
                           'pull request %s', pull_request_id)
                 raise HTTPForbidden()
 
             # This also triggers `review_status_change`
             comment, status = PullRequestModel().close_pull_request_with_comment(
                 pull_request, self._rhodecode_user, self.db_repo, message=text,
                 auth_user=self._rhodecode_user)
             Session().flush()
 
             PullRequestModel().trigger_pull_request_hook(
                 pull_request, self._rhodecode_user, 'comment',
                 data={'comment': comment})
 
         else:
             # regular comment case, could be inline, or one with status.
             # for that one we check also permissions
 
             allowed_to_change_status = PullRequestModel().check_user_change_status(
                 pull_request, self._rhodecode_user)
 
             if status and allowed_to_change_status:
                 message = (_('Status change %(transition_icon)s %(status)s')
                            % {'transition_icon': '>',
                               'status': ChangesetStatus.get_status_lbl(status)})
                 text = text or message
 
             comment = CommentsModel().create(
                 text=text,
                 repo=self.db_repo.repo_id,
                 user=self._rhodecode_user.user_id,
                 pull_request=pull_request,
                 f_path=self.request.POST.get('f_path'),
                 line_no=self.request.POST.get('line'),
                 status_change=(ChangesetStatus.get_status_lbl(status)
                                if status and allowed_to_change_status else None),
                 status_change_type=(status
                                     if status and allowed_to_change_status else None),
                 comment_type=comment_type,
                 resolves_comment_id=resolves_comment_id,
                 auth_user=self._rhodecode_user
             )
 
             if allowed_to_change_status:
                 # calculate old status before we change it
                 old_calculated_status = pull_request.calculated_review_status()
 
                 # get status if set !
                 if status:
                     ChangesetStatusModel().set_status(
                         self.db_repo.repo_id,
                         status,
                         self._rhodecode_user.user_id,
                         comment,
                         pull_request=pull_request
                     )
 
                 Session().flush()
                 # this is somehow required to get access to some relationship
                 # loaded on comment
                 Session().refresh(comment)
 
                 PullRequestModel().trigger_pull_request_hook(
                     pull_request, self._rhodecode_user, 'comment',
                     data={'comment': comment})
 
                 # we now calculate the status of pull request, and based on that
                 # calculation we set the commits status
                 calculated_status = pull_request.calculated_review_status()
                 if old_calculated_status != calculated_status:
                     PullRequestModel().trigger_pull_request_hook(
                         pull_request, self._rhodecode_user, 'review_status_change',
                         data={'status': calculated_status})
 
         Session().commit()
 
         data = {
             'target_id': h.safeid(h.safe_unicode(
                 self.request.POST.get('f_path'))),
         }
         if comment:
             c.co = comment
             rendered_comment = render(
                 'rhodecode:templates/changeset/changeset_comment_block.mako',
                 self._get_template_context(c), self.request)
 
             data.update(comment.get_dict())
             data.update({'rendered_text': rendered_comment})
 
         return data
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='pullrequest_comment_delete', request_method='POST',
         renderer='json_ext')
     def pull_request_comment_delete(self):
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
 
         comment = ChangesetComment.get_or_404(
             self.request.matchdict['comment_id'])
         comment_id = comment.comment_id
 
         if pull_request.is_closed():
             log.debug('comment: forbidden because pull request is closed')
             raise HTTPForbidden()
 
         if not comment:
             log.debug('Comment with id:%s not found, skipping', comment_id)
             # comment already deleted in another call probably
             return True
 
         if comment.pull_request.is_closed():
             # don't allow deleting comments on closed pull request
             raise HTTPForbidden()
 
         is_repo_admin = h.HasRepoPermissionAny('repository.admin')(self.db_repo_name)
         super_admin = h.HasPermissionAny('hg.admin')()
         comment_owner = comment.author.user_id == self._rhodecode_user.user_id
         is_repo_comment = comment.repo.repo_name == self.db_repo_name
         comment_repo_admin = is_repo_admin and is_repo_comment
 
         if super_admin or comment_owner or comment_repo_admin:
             old_calculated_status = comment.pull_request.calculated_review_status()
             CommentsModel().delete(comment=comment, auth_user=self._rhodecode_user)
             Session().commit()
             calculated_status = comment.pull_request.calculated_review_status()
             if old_calculated_status != calculated_status:
                 PullRequestModel().trigger_pull_request_hook(
                     comment.pull_request, self._rhodecode_user, 'review_status_change',
                     data={'status': calculated_status})
             return True
         else:
             log.warning('No permissions for user %s to delete comment_id: %s',
                         self._rhodecode_db_user, comment_id)
             raise HTTPNotFound()