diff --git a/rhodecode/apps/_base/__init__.py b/rhodecode/apps/_base/__init__.py --- a/rhodecode/apps/_base/__init__.py +++ b/rhodecode/apps/_base/__init__.py @@ -33,6 +33,7 @@ from rhodecode.model import user_group from rhodecode.model import user from rhodecode.model.db import User from rhodecode.model.scm import ScmModel +from rhodecode.model.settings import VcsSettingsModel log = logging.getLogger(__name__) @@ -256,6 +257,11 @@ class RepoAppView(BaseAppView): f_path_match = self._get_f_path_unchecked(matchdict, default) return self.path_filter.assert_path_permissions(f_path_match) + def _get_general_setting(self, target_repo, settings_key, default=False): + settings_model = VcsSettingsModel(repo=target_repo) + settings = settings_model.get_general_settings() + return settings.get(settings_key, default) + class PathFilter(object): diff --git a/rhodecode/apps/repository/views/repo_commits.py b/rhodecode/apps/repository/views/repo_commits.py --- a/rhodecode/apps/repository/views/repo_commits.py +++ b/rhodecode/apps/repository/views/repo_commits.py @@ -34,17 +34,18 @@ from rhodecode.lib.auth import ( LoginRequired, HasRepoPermissionAnyDecorator, NotAnonymous, CSRFRequired) from rhodecode.lib.compat import OrderedDict +from rhodecode.lib.diffs import cache_diff, load_cached_diff, diff_cache_exist from rhodecode.lib.exceptions import StatusChangeOnClosedPullRequestError import rhodecode.lib.helpers as h -from rhodecode.lib.utils2 import safe_unicode +from rhodecode.lib.utils2 import safe_unicode, str2bool from rhodecode.lib.vcs.backends.base import EmptyCommit from rhodecode.lib.vcs.exceptions import ( - RepositoryError, CommitDoesNotExistError, NodeDoesNotExistError) + RepositoryError, CommitDoesNotExistError) from rhodecode.model.db import ChangesetComment, ChangesetStatus from rhodecode.model.changeset_status import ChangesetStatusModel from rhodecode.model.comment import CommentsModel from rhodecode.model.meta import Session - +from rhodecode.model.settings import VcsSettingsModel log = logging.getLogger(__name__) @@ -152,6 +153,12 @@ class RepoCommitsView(RepoAppView): return c + def _is_diff_cache_enabled(self, target_repo): + caching_enabled = self._get_general_setting( + target_repo, 'rhodecode_diff_cache') + log.debug('Diff caching enabled: %s', caching_enabled) + return caching_enabled + def _commit(self, commit_id_range, method): _ = self.request.translate c = self.load_default_context() @@ -240,43 +247,63 @@ class RepoCommitsView(RepoAppView): commit2 = commit commit1 = commit.parents[0] if commit.parents else EmptyCommit() - _diff = self.rhodecode_vcs_repo.get_diff( - commit1, commit2, - ignore_whitespace=ign_whitespace_lcl, context=context_lcl) - diff_processor = diffs.DiffProcessor( - _diff, format='newdiff', diff_limit=diff_limit, - file_limit=file_limit, show_full_diff=c.fulldiff) - - commit_changes = OrderedDict() if method == 'show': - _parsed = diff_processor.prepare() - c.limited_diff = isinstance(_parsed, diffs.LimitedDiffContainer) - - _parsed = diff_processor.prepare() - - def _node_getter(commit): - def get_node(fname): - try: - return commit.get_node(fname) - except NodeDoesNotExistError: - return None - return get_node - inline_comments = CommentsModel().get_inline_comments( self.db_repo.repo_id, revision=commit.raw_id) c.inline_cnt = CommentsModel().get_inline_comments_count( inline_comments) + c.inline_comments = inline_comments - diffset = codeblocks.DiffSet( - repo_name=self.db_repo_name, - source_node_getter=_node_getter(commit1), - target_node_getter=_node_getter(commit2), - comments=inline_comments) - diffset = self.path_filter.render_patchset_filtered( - diffset, _parsed, commit1.raw_id, commit2.raw_id) + cache_path = self.rhodecode_vcs_repo.get_create_shadow_cache_pr_path( + self.db_repo) + cache_file_path = diff_cache_exist( + cache_path, 'diff', commit.raw_id, + ign_whitespace_lcl, context_lcl, c.fulldiff) + + caching_enabled = self._is_diff_cache_enabled(self.db_repo) + force_recache = str2bool(self.request.GET.get('force_recache')) + + cached_diff = None + if caching_enabled: + cached_diff = load_cached_diff(cache_file_path) + has_proper_diff_cache = cached_diff and cached_diff.get('diff') + if not force_recache and has_proper_diff_cache: + diffset = cached_diff['diff'] + else: + vcs_diff = self.rhodecode_vcs_repo.get_diff( + commit1, commit2, + ignore_whitespace=ign_whitespace_lcl, + context=context_lcl) + + diff_processor = diffs.DiffProcessor( + vcs_diff, format='newdiff', diff_limit=diff_limit, + file_limit=file_limit, show_full_diff=c.fulldiff) + + _parsed = diff_processor.prepare() + + diffset = codeblocks.DiffSet( + repo_name=self.db_repo_name, + source_node_getter=codeblocks.diffset_node_getter(commit1), + target_node_getter=codeblocks.diffset_node_getter(commit2)) + + diffset = self.path_filter.render_patchset_filtered( + diffset, _parsed, commit1.raw_id, commit2.raw_id) + + # save cached diff + if caching_enabled: + cache_diff(cache_file_path, diffset, None) + + c.limited_diff = diffset.limited_diff c.changes[commit.raw_id] = diffset else: + # TODO(marcink): no cache usage here... + _diff = self.rhodecode_vcs_repo.get_diff( + commit1, commit2, + ignore_whitespace=ign_whitespace_lcl, context=context_lcl) + diff_processor = diffs.DiffProcessor( + _diff, format='newdiff', diff_limit=diff_limit, + file_limit=file_limit, show_full_diff=c.fulldiff) # downloads/raw we only need RAW diff nothing else diff = self.path_filter.get_raw_patch(diff_processor) c.changes[commit.raw_id] = [None, None, None, None, diff, None, None] diff --git a/rhodecode/apps/repository/views/repo_compare.py b/rhodecode/apps/repository/views/repo_compare.py --- a/rhodecode/apps/repository/views/repo_compare.py +++ b/rhodecode/apps/repository/views/repo_compare.py @@ -295,19 +295,10 @@ class RepoCompareView(RepoAppView): file_limit=file_limit, show_full_diff=c.fulldiff) _parsed = diff_processor.prepare() - def _node_getter(commit): - """ Returns a function that returns a node for a commit or None """ - def get_node(fname): - try: - return commit.get_node(fname) - except NodeDoesNotExistError: - return None - return get_node - diffset = codeblocks.DiffSet( repo_name=source_repo.repo_name, - source_node_getter=_node_getter(source_commit), - target_node_getter=_node_getter(target_commit), + source_node_getter=codeblocks.diffset_node_getter(source_commit), + target_node_getter=codeblocks.diffset_node_getter(target_commit), ) c.diffset = self.path_filter.render_patchset_filtered( diffset, _parsed, source_ref, target_ref) diff --git a/rhodecode/apps/repository/views/repo_pull_requests.py b/rhodecode/apps/repository/views/repo_pull_requests.py --- a/rhodecode/apps/repository/views/repo_pull_requests.py +++ b/rhodecode/apps/repository/views/repo_pull_requests.py @@ -34,6 +34,7 @@ from rhodecode.apps._base import RepoApp from rhodecode.lib import helpers as h, diffs, codeblocks, channelstream from rhodecode.lib.base import vcs_operation_context +from rhodecode.lib.diffs import load_cached_diff, cache_diff, diff_cache_exist from rhodecode.lib.ext_json import json from rhodecode.lib.auth import ( LoginRequired, HasRepoPermissionAny, HasRepoPermissionAnyDecorator, @@ -41,7 +42,7 @@ from rhodecode.lib.auth import ( from rhodecode.lib.utils2 import str2bool, safe_str, safe_unicode from rhodecode.lib.vcs.backends.base import EmptyCommit, UpdateFailureReason from rhodecode.lib.vcs.exceptions import (CommitDoesNotExistError, - RepositoryRequirementError, NodeDoesNotExistError, EmptyRepositoryError) + RepositoryRequirementError, EmptyRepositoryError) from rhodecode.model.changeset_status import ChangesetStatusModel from rhodecode.model.comment import CommentsModel from rhodecode.model.db import (func, or_, PullRequest, PullRequestVersion, @@ -201,10 +202,16 @@ class RepoPullRequestsView(RepoAppView, return data + def _is_diff_cache_enabled(self, target_repo): + caching_enabled = self._get_general_setting( + target_repo, 'rhodecode_diff_cache') + log.debug('Diff caching enabled: %s', caching_enabled) + return caching_enabled + def _get_diffset(self, source_repo_name, source_repo, source_ref_id, target_ref_id, - target_commit, source_commit, diff_limit, fulldiff, - file_limit, display_inline_comments): + target_commit, source_commit, diff_limit, file_limit, + fulldiff): vcs_diff = PullRequestModel().get_diff( source_repo, source_ref_id, target_ref_id) @@ -215,21 +222,11 @@ class RepoPullRequestsView(RepoAppView, _parsed = diff_processor.prepare() - def _node_getter(commit): - def get_node(fname): - try: - return commit.get_node(fname) - except NodeDoesNotExistError: - return None - - return get_node - diffset = codeblocks.DiffSet( repo_name=self.db_repo_name, source_repo_name=source_repo_name, - source_node_getter=_node_getter(target_commit), - target_node_getter=_node_getter(source_commit), - comments=display_inline_comments + source_node_getter=codeblocks.diffset_node_getter(target_commit), + target_node_getter=codeblocks.diffset_node_getter(source_commit), ) diffset = self.path_filter.render_patchset_filtered( diffset, _parsed, target_commit.raw_id, source_commit.raw_id) @@ -443,42 +440,54 @@ class RepoPullRequestsView(RepoAppView, commits_source_repo = source_scm c.commits_source_repo = commits_source_repo - commit_cache = {} - try: - pre_load = ["author", "branch", "date", "message"] - show_revs = pull_request_at_ver.revisions - for rev in show_revs: - comm = commits_source_repo.get_commit( - commit_id=rev, pre_load=pre_load) - c.commit_ranges.append(comm) - commit_cache[comm.raw_id] = comm - - # Order here matters, we first need to get target, and then - # the source - target_commit = commits_source_repo.get_commit( - commit_id=safe_str(target_ref_id)) - - source_commit = commits_source_repo.get_commit( - commit_id=safe_str(source_ref_id)) - - except CommitDoesNotExistError: - log.warning( - 'Failed to get commit from `{}` repo'.format( - commits_source_repo), exc_info=True) - except RepositoryRequirementError: - log.warning( - 'Failed to get all required data from repo', exc_info=True) - c.missing_requirements = True - c.ancestor = None # set it to None, to hide it from PR view - try: - ancestor_id = source_scm.get_common_ancestor( - source_commit.raw_id, target_commit.raw_id, target_scm) - c.ancestor_commit = source_scm.get_commit(ancestor_id) - except Exception: - c.ancestor_commit = None + # empty version means latest, so we keep this to prevent + # double caching + version_normalized = version or 'latest' + from_version_normalized = from_version or 'latest' + + cache_path = self.rhodecode_vcs_repo.get_create_shadow_cache_pr_path( + target_repo) + cache_file_path = diff_cache_exist( + cache_path, 'pull_request', pull_request_id, version_normalized, + from_version_normalized, source_ref_id, target_ref_id, c.fulldiff) + + caching_enabled = self._is_diff_cache_enabled(c.target_repo) + force_recache = str2bool(self.request.GET.get('force_recache')) + + cached_diff = None + if caching_enabled: + cached_diff = load_cached_diff(cache_file_path) + has_proper_commit_cache = ( + cached_diff and cached_diff.get('commits') + and len(cached_diff.get('commits', [])) == 5 + and cached_diff.get('commits')[0] + and cached_diff.get('commits')[3]) + if not force_recache and has_proper_commit_cache: + diff_commit_cache = \ + (ancestor_commit, commit_cache, missing_requirements, + source_commit, target_commit) = cached_diff['commits'] + else: + diff_commit_cache = \ + (ancestor_commit, commit_cache, missing_requirements, + source_commit, target_commit) = self.get_commits( + commits_source_repo, + pull_request_at_ver, + source_commit, + source_ref_id, + source_scm, + target_commit, + target_ref_id, + target_scm) + + # register our commit range + for comm in commit_cache.values(): + c.commit_ranges.append(comm) + + c.missing_requirements = missing_requirements + c.ancestor_commit = ancestor_commit c.statuses = source_repo.statuses( [x.raw_id for x in c.commit_ranges]) @@ -500,12 +509,23 @@ class RepoPullRequestsView(RepoAppView, c.missing_commits = True else: + c.inline_comments = display_inline_comments - c.diffset = self._get_diffset( - c.source_repo.repo_name, commits_source_repo, - source_ref_id, target_ref_id, - target_commit, source_commit, - diff_limit, c.fulldiff, file_limit, display_inline_comments) + has_proper_diff_cache = cached_diff and cached_diff.get('commits') + if not force_recache and has_proper_diff_cache: + c.diffset = cached_diff['diff'] + (ancestor_commit, commit_cache, missing_requirements, + source_commit, target_commit) = cached_diff['commits'] + else: + c.diffset = self._get_diffset( + c.source_repo.repo_name, commits_source_repo, + source_ref_id, target_ref_id, + target_commit, source_commit, + diff_limit, file_limit, c.fulldiff) + + # save cached diff + if caching_enabled: + cache_diff(cache_file_path, c.diffset, diff_commit_cache) c.limited_diff = c.diffset.limited_diff @@ -568,7 +588,6 @@ class RepoPullRequestsView(RepoAppView, if self._rhodecode_user.user_id in allowed_reviewers: for co in general_comments: if co.author.user_id == self._rhodecode_user.user_id: - # each comment has a status change status = co.status_change if status: _ver_pr = status[0].comment.pull_request_version_id @@ -576,6 +595,43 @@ class RepoPullRequestsView(RepoAppView, return self._get_template_context(c) + def get_commits( + self, commits_source_repo, pull_request_at_ver, source_commit, + source_ref_id, source_scm, target_commit, target_ref_id, target_scm): + commit_cache = collections.OrderedDict() + missing_requirements = False + try: + pre_load = ["author", "branch", "date", "message"] + show_revs = pull_request_at_ver.revisions + for rev in show_revs: + comm = commits_source_repo.get_commit( + commit_id=rev, pre_load=pre_load) + commit_cache[comm.raw_id] = comm + + # Order here matters, we first need to get target, and then + # the source + target_commit = commits_source_repo.get_commit( + commit_id=safe_str(target_ref_id)) + + source_commit = commits_source_repo.get_commit( + commit_id=safe_str(source_ref_id)) + except CommitDoesNotExistError: + log.warning( + 'Failed to get commit from `{}` repo'.format( + commits_source_repo), exc_info=True) + except RepositoryRequirementError: + log.warning( + 'Failed to get all required data from repo', exc_info=True) + missing_requirements = True + ancestor_commit = None + try: + ancestor_id = source_scm.get_common_ancestor( + source_commit.raw_id, target_commit.raw_id, target_scm) + ancestor_commit = source_scm.get_commit(ancestor_id) + except Exception: + ancestor_commit = None + return ancestor_commit, commit_cache, missing_requirements, source_commit, target_commit + def assure_not_empty_repo(self): _ = self.request.translate diff --git a/rhodecode/lib/codeblocks.py b/rhodecode/lib/codeblocks.py --- a/rhodecode/lib/codeblocks.py +++ b/rhodecode/lib/codeblocks.py @@ -30,6 +30,7 @@ from rhodecode.lib.helpers import ( get_lexer_for_filenode, html_escape, get_custom_lexer) from rhodecode.lib.utils2 import AttributeDict, StrictAttributeDict from rhodecode.lib.vcs.nodes import FileNode +from rhodecode.lib.vcs.exceptions import VCSError, NodeDoesNotExistError from rhodecode.lib.diff_match_patch import diff_match_patch from rhodecode.lib.diffs import LimitedDiffContainer from pygments.lexers import get_lexer_by_name @@ -351,6 +352,16 @@ def tokens_diff(old_tokens, new_tokens, return old_tokens_result, new_tokens_result, similarity +def diffset_node_getter(commit): + def get_node(fname): + try: + return commit.get_node(fname) + except NodeDoesNotExistError: + return None + + return get_node + + class DiffSet(object): """ An object for parsing the diff result from diffs.DiffProcessor and @@ -515,6 +526,7 @@ class DiffSet(object): if target_file_path in self.comments_store: for lineno, comments in self.comments_store[target_file_path].items(): left_comments[lineno] = comments + # left comments are one that we couldn't place in diff lines. # could be outdated, or the diff changed and this line is no # longer available @@ -551,7 +563,7 @@ class DiffSet(object): result.lines.extend( self.parse_lines(before, after, source_file, target_file)) - result.unified = self.as_unified(result.lines) + result.unified = list(self.as_unified(result.lines)) result.sideside = result.lines return result @@ -606,8 +618,9 @@ class DiffSet(object): original.lineno = before['old_lineno'] original.content = before['line'] original.action = self.action_to_op(before['action']) - original.comments = self.get_comments_for('old', - source_file, before['old_lineno']) + + original.get_comment_args = ( + source_file, 'o', before['old_lineno']) if after: if after['action'] == 'new-no-nl': @@ -619,8 +632,9 @@ class DiffSet(object): modified.lineno = after['new_lineno'] modified.content = after['line'] modified.action = self.action_to_op(after['action']) - modified.comments = self.get_comments_for('new', - target_file, after['new_lineno']) + + modified.get_comment_args = ( + target_file, 'n', after['new_lineno']) # diff the lines if before_tokens and after_tokens: @@ -649,23 +663,6 @@ class DiffSet(object): return lines - def get_comments_for(self, version, filename, line_number): - if hasattr(filename, 'unicode_path'): - filename = filename.unicode_path - - if not isinstance(filename, basestring): - return None - - line_key = { - 'old': 'o', - 'new': 'n', - }[version] + str(line_number) - - if filename in self.comments_store: - file_comments = self.comments_store[filename] - if line_key in file_comments: - return file_comments.pop(line_key) - def get_line_tokens(self, line_text, line_number, file=None): filenode = None filename = None @@ -722,25 +719,25 @@ class DiffSet(object): if line.original.action == ' ': yield (line.original.lineno, line.modified.lineno, line.original.action, line.original.content, - line.original.comments) + line.original.get_comment_args) continue if line.original.action == '-': yield (line.original.lineno, None, line.original.action, line.original.content, - line.original.comments) + line.original.get_comment_args) if line.modified.action == '+': buf.append(( None, line.modified.lineno, line.modified.action, line.modified.content, - line.modified.comments)) + line.modified.get_comment_args)) continue if line.modified: yield (None, line.modified.lineno, line.modified.action, line.modified.content, - line.modified.comments) + line.modified.get_comment_args) for b in buf: yield b diff --git a/rhodecode/lib/diffs.py b/rhodecode/lib/diffs.py --- a/rhodecode/lib/diffs.py +++ b/rhodecode/lib/diffs.py @@ -23,16 +23,18 @@ Set of diffing helpers, previously part of vcs """ +import os import re import collections import difflib import logging +import cPickle as pickle from itertools import tee, imap from rhodecode.lib.vcs.exceptions import VCSError from rhodecode.lib.vcs.nodes import FileNode, SubModuleNode -from rhodecode.lib.utils2 import safe_unicode +from rhodecode.lib.utils2 import safe_unicode, safe_str log = logging.getLogger(__name__) @@ -1129,3 +1131,82 @@ class LineNotInDiffException(Exception): class DiffLimitExceeded(Exception): pass + + +def cache_diff(cached_diff_file, diff, commits): + + struct = { + 'version': 'v1', + 'diff': diff, + 'commits': commits + } + + try: + with open(cached_diff_file, 'wb') as f: + pickle.dump(struct, f) + log.debug('Saved diff cache under %s', cached_diff_file) + except Exception: + log.warn('Failed to save cache', exc_info=True) + # cleanup file to not store it "damaged" + try: + os.remove(cached_diff_file) + except Exception: + log.exception('Failed to cleanup path %s', cached_diff_file) + + +def load_cached_diff(cached_diff_file): + + default_struct = { + 'version': 'v1', + 'diff': None, + 'commits': None + } + + has_cache = os.path.isfile(cached_diff_file) + if not has_cache: + return default_struct + + data = None + try: + with open(cached_diff_file, 'rb') as f: + data = pickle.load(f) + log.debug('Loaded diff cache from %s', cached_diff_file) + except Exception: + log.warn('Failed to read diff cache file', exc_info=True) + + if not data: + data = default_struct + + if not isinstance(data, dict): + # old version of data ? + data = default_struct + + return data + + +def generate_diff_cache_key(*args): + """ + Helper to generate a cache key using arguments + """ + def arg_mapper(input_param): + input_param = safe_str(input_param) + # we cannot allow '/' in arguments since it would allow + # subdirectory usage + input_param.replace('/', '_') + return input_param or None # prevent empty string arguments + + return '_'.join([ + '{}' for i in range(len(args))]).format(*map(arg_mapper, args)) + + +def diff_cache_exist(cache_storage, *args): + """ + Based on all generated arguments check and return a cache path + """ + cache_key = generate_diff_cache_key(*args) + cache_file_path = os.path.join(cache_storage, cache_key) + # prevent path traversal attacks using some param that have e.g '../../' + if not os.path.abspath(cache_file_path).startswith(cache_storage): + raise ValueError('Final path must be within {}'.format(cache_storage)) + + return cache_file_path diff --git a/rhodecode/lib/utils2.py b/rhodecode/lib/utils2.py --- a/rhodecode/lib/utils2.py +++ b/rhodecode/lib/utils2.py @@ -709,7 +709,19 @@ def extract_mentioned_users(s): return sorted(list(usrs), key=lambda k: k.lower()) -class StrictAttributeDict(dict): +class AttributeDictBase(dict): + def __getstate__(self): + odict = self.__dict__ # get attribute dictionary + return odict + + def __setstate__(self, dict): + self.__dict__ = dict + + __setattr__ = dict.__setitem__ + __delattr__ = dict.__delitem__ + + +class StrictAttributeDict(AttributeDictBase): """ Strict Version of Attribute dict which raises an Attribute error when requested attribute is not set @@ -720,15 +732,12 @@ class StrictAttributeDict(dict): except KeyError: raise AttributeError('%s object has no attribute %s' % ( self.__class__, attr)) - __setattr__ = dict.__setitem__ - __delattr__ = dict.__delitem__ -class AttributeDict(dict): +class AttributeDict(AttributeDictBase): def __getattr__(self, attr): return self.get(attr, None) - __setattr__ = dict.__setitem__ - __delattr__ = dict.__delitem__ + def fix_PATH(os_=None): diff --git a/rhodecode/lib/vcs/backends/base.py b/rhodecode/lib/vcs/backends/base.py --- a/rhodecode/lib/vcs/backends/base.py +++ b/rhodecode/lib/vcs/backends/base.py @@ -206,6 +206,14 @@ class BaseRepository(object): def __ne__(self, other): return not self.__eq__(other) + def get_create_shadow_cache_pr_path(self, repo): + path = os.path.join( + os.path.dirname(self.path), + '.__shadow_diff_cache_repo_{}/'.format(repo.repo_id)) + if not os.path.exists(path): + os.makedirs(path, 0755) + return path + @classmethod def get_default_config(cls, default=None): config = Config() @@ -745,6 +753,12 @@ class BaseCommit(object): 'branch': self.branch } + def __getstate__(self): + d = self.__dict__.copy() + d.pop('_remote', None) + d.pop('repository', None) + return d + def _get_refs(self): return { 'branches': [self.branch] if self.branch else [], diff --git a/rhodecode/model/forms.py b/rhodecode/model/forms.py --- a/rhodecode/model/forms.py +++ b/rhodecode/model/forms.py @@ -430,6 +430,9 @@ class _BaseVcsSettingsForm(formencode.Sc vcs_svn_proxy_http_requests_enabled = v.StringBoolean(if_missing=False) vcs_svn_proxy_http_server_url = v.UnicodeString(strip=True, if_missing=None) + # cache + rhodecode_diff_cache = v.StringBoolean(if_missing=False) + def ApplicationUiSettingsForm(localizer): _ = localizer diff --git a/rhodecode/model/settings.py b/rhodecode/model/settings.py --- a/rhodecode/model/settings.py +++ b/rhodecode/model/settings.py @@ -417,7 +417,9 @@ class VcsSettingsModel(object): 'hg_use_rebase_for_merging', 'hg_close_branch_before_merging', 'git_use_rebase_for_merging', - 'git_close_branch_before_merging') + 'git_close_branch_before_merging', + 'diff_cache', + ) HOOKS_SETTINGS = ( ('hooks', 'changegroup.repo_size'), @@ -438,6 +440,7 @@ class VcsSettingsModel(object): GLOBAL_GIT_SETTINGS = ( ('vcs_git_lfs', 'enabled'), ('vcs_git_lfs', 'store_location')) + GLOBAL_SVN_SETTINGS = ( ('vcs_svn_proxy', 'http_requests_enabled'), ('vcs_svn_proxy', 'http_server_url')) @@ -571,6 +574,7 @@ class VcsSettingsModel(object): self._create_or_update_ui( self.repo_settings, *phases, value=safe_str(data[phases_key])) + def create_or_update_global_hg_settings(self, data): largefiles, largefiles_store, phases, hgsubversion, evolve \ = self.GLOBAL_HG_SETTINGS diff --git a/rhodecode/templates/base/vcs_settings.mako b/rhodecode/templates/base/vcs_settings.mako --- a/rhodecode/templates/base/vcs_settings.mako +++ b/rhodecode/templates/base/vcs_settings.mako @@ -312,6 +312,20 @@ % endif + % if display_globals or repo_type in ['hg', 'git', 'svn']: +
+
+

${_('Diff cache')}

+
+
+
+ ${h.checkbox('rhodecode_diff_cache' + suffix, 'True', **kwargs)} + +
+
+
+ % endif + % if display_globals or repo_type in ['hg',]:
diff --git a/rhodecode/templates/changeset/changeset.mako b/rhodecode/templates/changeset/changeset.mako --- a/rhodecode/templates/changeset/changeset.mako +++ b/rhodecode/templates/changeset/changeset.mako @@ -213,7 +213,7 @@ <%namespace name="cbdiffs" file="/codeblocks/diffs.mako"/> ${cbdiffs.render_diffset_menu()} ${cbdiffs.render_diffset( - c.changes[c.commit.raw_id], commit=c.commit, use_comments=True)} + c.changes[c.commit.raw_id], commit=c.commit, use_comments=True,inline_comments=c.inline_comments )}
## template for inline comment form diff --git a/rhodecode/templates/codeblocks/diffs.mako b/rhodecode/templates/codeblocks/diffs.mako --- a/rhodecode/templates/codeblocks/diffs.mako +++ b/rhodecode/templates/codeblocks/diffs.mako @@ -44,13 +44,15 @@ return '%s_%s_%i' % (h.safeid(filename), # special file-comments that were deleted in previous versions # it's used for showing outdated comments for deleted files in a PR - deleted_files_comments=None + deleted_files_comments=None, + + # for cache purpose + inline_comments=None )"> - %if use_comments:
- ${inline_comments_container([])} + ${inline_comments_container([], inline_comments)}
@@ -211,9 +213,9 @@ collapse_all = len(diffset.files) > coll %if c.diffmode == 'unified': - ${render_hunk_lines_unified(hunk, use_comments=use_comments)} + ${render_hunk_lines_unified(hunk, use_comments=use_comments, inline_comments=inline_comments)} %elif c.diffmode == 'sideside': - ${render_hunk_lines_sideside(hunk, use_comments=use_comments)} + ${render_hunk_lines_sideside(hunk, use_comments=use_comments, inline_comments=inline_comments)} %else: unknown diff mode @@ -230,7 +232,7 @@ collapse_all = len(diffset.files) > coll - ${inline_comments_container(comments)} + ${inline_comments_container(comments, inline_comments)} %elif c.diffmode == 'sideside': @@ -239,7 +241,7 @@ collapse_all = len(diffset.files) > coll % if lineno.startswith('o'): - ${inline_comments_container(comments)} + ${inline_comments_container(comments, inline_comments)} % endif @@ -247,7 +249,7 @@ collapse_all = len(diffset.files) > coll % if lineno.startswith('n'): - ${inline_comments_container(comments)} + ${inline_comments_container(comments, inline_comments)} % endif @@ -298,7 +300,7 @@ collapse_all = len(diffset.files) > coll - ${inline_comments_container(comments_dict['comments'])} + ${inline_comments_container(comments_dict['comments'], inline_comments)} %elif c.diffmode == 'sideside': @@ -310,7 +312,7 @@ collapse_all = len(diffset.files) > coll - ${inline_comments_container(comments_dict['comments'])} + ${inline_comments_container(comments_dict['comments'], inline_comments)} %endif @@ -484,12 +486,11 @@ from rhodecode.lib.diffs import NEW_FILE -<%def name="inline_comments_container(comments)"> +<%def name="inline_comments_container(comments, inline_comments)">
%for comment in comments: - ${commentblock.comment_block(comment, inline=True)} + ${commentblock.comment_block(comment, inline=True)} %endfor - % if comments and comments[-1].outdated:
+<%! +def get_comments_for(comments, filename, line_version, line_number): + if hasattr(filename, 'unicode_path'): + filename = filename.unicode_path -<%def name="render_hunk_lines_sideside(hunk, use_comments=False)"> + if not isinstance(filename, basestring): + return None + + line_key = '{}{}'.format(line_version, line_number) + if comments and filename in comments: + file_comments = comments[filename] + if line_key in file_comments: + return file_comments[line_key] +%> + +<%def name="render_hunk_lines_sideside(hunk, use_comments=False, inline_comments=None)"> + %for i, line in enumerate(hunk.sideside): <% old_line_anchor, new_line_anchor = None, None @@ -521,12 +537,16 @@ from rhodecode.lib.diffs import NEW_FILE data-line-no="${line.original.lineno}" >
- %if line.original.comments: - <% has_outdated = any([x.outdated for x in line.original.comments]) %> + <% loc = None %> + %if line.original.get_comment_args: + <% loc = get_comments_for(inline_comments, *line.original.get_comment_args) %> + %endif + %if loc: + <% has_outdated = any([x.outdated for x in loc]) %> % if has_outdated: - + % else: - + % endif %endif
@@ -548,20 +568,28 @@ from rhodecode.lib.diffs import NEW_FILE ${render_add_comment_button()} %endif ${line.original.action} ${line.original.content or '' | n} - %if use_comments and line.original.lineno and line.original.comments: - ${inline_comments_container(line.original.comments)} + + %if use_comments and line.original.lineno and loc: + ${inline_comments_container(loc, inline_comments)} %endif +
- %if line.modified.comments: - <% has_outdated = any([x.outdated for x in line.modified.comments]) %> + + %if line.modified.get_comment_args: + <% lmc = get_comments_for(inline_comments, *line.modified.get_comment_args) %> + %else: + <% lmc = None%> + %endif + %if lmc: + <% has_outdated = any([x.outdated for x in lmc]) %> % if has_outdated: - + % else: - + % endif %endif
@@ -583,8 +611,8 @@ from rhodecode.lib.diffs import NEW_FILE ${render_add_comment_button()} %endif ${line.modified.action} ${line.modified.content or '' | n} - %if use_comments and line.modified.lineno and line.modified.comments: - ${inline_comments_container(line.modified.comments)} + %if use_comments and line.modified.lineno and lmc: + ${inline_comments_container(lmc, inline_comments)} %endif @@ -592,8 +620,8 @@ from rhodecode.lib.diffs import NEW_FILE -<%def name="render_hunk_lines_unified(hunk, use_comments=False)"> - %for old_line_no, new_line_no, action, content, comments in hunk.unified: +<%def name="render_hunk_lines_unified(hunk, use_comments=False, inline_comments=None)"> + %for old_line_no, new_line_no, action, content, comments_args in hunk.unified: <% old_line_anchor, new_line_anchor = None, None if old_line_no: @@ -604,6 +632,13 @@ from rhodecode.lib.diffs import NEW_FILE
+ + %if comments_args: + <% comments = get_comments_for(inline_comments, *comments_args) %> + %else: + <% comments = None%> + %endif + % if comments: <% has_outdated = any([x.outdated for x in comments]) %> % if has_outdated: @@ -642,7 +677,7 @@ from rhodecode.lib.diffs import NEW_FILE %endif ${action} ${content or '' | n} %if use_comments and comments: - ${inline_comments_container(comments)} + ${inline_comments_container(comments, inline_comments)} %endif diff --git a/rhodecode/templates/pullrequests/pullrequest_show.mako b/rhodecode/templates/pullrequests/pullrequest_show.mako --- a/rhodecode/templates/pullrequests/pullrequest_show.mako +++ b/rhodecode/templates/pullrequests/pullrequest_show.mako @@ -570,7 +570,8 @@ c.diffset, use_comments=True, collapse_when_files_over=30, disable_new_comments=not c.allowed_to_comment, - deleted_files_comments=c.deleted_files_comments)} + deleted_files_comments=c.deleted_files_comments, + inline_comments=c.inline_comments)}
% else: ## skipping commits we need to clear the view for missing commits diff --git a/rhodecode/tests/models/settings/test_vcs_settings.py b/rhodecode/tests/models/settings/test_vcs_settings.py --- a/rhodecode/tests/models/settings/test_vcs_settings.py +++ b/rhodecode/tests/models/settings/test_vcs_settings.py @@ -44,6 +44,7 @@ GENERAL_FORM_DATA = { 'rhodecode_hg_close_branch_before_merging': True, 'rhodecode_git_use_rebase_for_merging': True, 'rhodecode_git_close_branch_before_merging': True, + 'rhodecode_diff_cache': True, }