caching: add option to cache diffs for commits and pull requests.

- new option for vcs settings to enable caching of diffs and commits
- enables auditing of PR after closing it, and removing commits from source repo
- speed up for large PR upto 20x
- reduced memory usage

rhodecode/apps/_base/__init__.py

@@ -33,6 +33,7 b' from rhodecode.model import user_group'
 from rhodecode.model import user
 from rhodecode.model.db import User
 from rhodecode.model.scm import ScmModel
+from rhodecode.model.settings import VcsSettingsModel
 
 log = logging.getLogger(__name__)
 
@@ -256,6 +257,11 b' class RepoAppView(BaseAppView):'
         f_path_match = self._get_f_path_unchecked(matchdict, default)
         return self.path_filter.assert_path_permissions(f_path_match)
 
+    def _get_general_setting(self, target_repo, settings_key, default=False):
+        settings_model = VcsSettingsModel(repo=target_repo)
+        settings = settings_model.get_general_settings()
+        return settings.get(settings_key, default)
+
 
 class PathFilter(object):
 

rhodecode/apps/repository/views/repo_commits.py

@@ -34,17 +34,18 b' from rhodecode.lib.auth import ('
     LoginRequired, HasRepoPermissionAnyDecorator, NotAnonymous, CSRFRequired)
 
 from rhodecode.lib.compat import OrderedDict
+from rhodecode.lib.diffs import cache_diff, load_cached_diff, diff_cache_exist
 from rhodecode.lib.exceptions import StatusChangeOnClosedPullRequestError
 import rhodecode.lib.helpers as h
-from rhodecode.lib.utils2 import safe_unicode
+from rhodecode.lib.utils2 import safe_unicode, str2bool
 from rhodecode.lib.vcs.backends.base import EmptyCommit
 from rhodecode.lib.vcs.exceptions import (
-    RepositoryError, CommitDoesNotExistError, NodeDoesNotExistError)
+    RepositoryError, CommitDoesNotExistError)
 from rhodecode.model.db import ChangesetComment, ChangesetStatus
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.comment import CommentsModel
 from rhodecode.model.meta import Session
-
+from rhodecode.model.settings import VcsSettingsModel
 
 log = logging.getLogger(__name__)
 
@@ -152,6 +153,12 b' class RepoCommitsView(RepoAppView):'
 
         return c
 
+    def _is_diff_cache_enabled(self, target_repo):
+        caching_enabled = self._get_general_setting(
+            target_repo, 'rhodecode_diff_cache')
+        log.debug('Diff caching enabled: %s', caching_enabled)
+        return caching_enabled
+
     def _commit(self, commit_id_range, method):
         _ = self.request.translate
         c = self.load_default_context()
@@ -240,43 +247,63 b' class RepoCommitsView(RepoAppView):'
             commit2 = commit
             commit1 = commit.parents[0] if commit.parents else EmptyCommit()
 
-            _diff = self.rhodecode_vcs_repo.get_diff(
-                commit1, commit2,
-                ignore_whitespace=ign_whitespace_lcl, context=context_lcl)
-            diff_processor = diffs.DiffProcessor(
-                _diff, format='newdiff', diff_limit=diff_limit,
-                file_limit=file_limit, show_full_diff=c.fulldiff)
-
-            commit_changes = OrderedDict()
             if method == 'show':
-                _parsed = diff_processor.prepare()
-                c.limited_diff = isinstance(_parsed, diffs.LimitedDiffContainer)
-
-                _parsed = diff_processor.prepare()
-
-                def _node_getter(commit):
-                    def get_node(fname):
-                        try:
-                            return commit.get_node(fname)
-                        except NodeDoesNotExistError:
-                            return None
-                    return get_node
-
                 inline_comments = CommentsModel().get_inline_comments(
                     self.db_repo.repo_id, revision=commit.raw_id)
                 c.inline_cnt = CommentsModel().get_inline_comments_count(
                     inline_comments)
+                c.inline_comments = inline_comments
 
-                diffset = codeblocks.DiffSet(
-                    repo_name=self.db_repo_name,
-                    source_node_getter=_node_getter(commit1),
-                    target_node_getter=_node_getter(commit2),
-                    comments=inline_comments)
-                diffset = self.path_filter.render_patchset_filtered(
-                    diffset, _parsed, commit1.raw_id, commit2.raw_id)
+                cache_path = self.rhodecode_vcs_repo.get_create_shadow_cache_pr_path(
+                    self.db_repo)
+                cache_file_path = diff_cache_exist(
+                    cache_path, 'diff', commit.raw_id,
+                    ign_whitespace_lcl, context_lcl, c.fulldiff)
+
+                caching_enabled = self._is_diff_cache_enabled(self.db_repo)
+                force_recache = str2bool(self.request.GET.get('force_recache'))
+
+                cached_diff = None
+                if caching_enabled:
+                    cached_diff = load_cached_diff(cache_file_path)
 
+                has_proper_diff_cache = cached_diff and cached_diff.get('diff')
+                if not force_recache and has_proper_diff_cache:
+                    diffset = cached_diff['diff']
+                else:
+                    vcs_diff = self.rhodecode_vcs_repo.get_diff(
+                        commit1, commit2,
+                        ignore_whitespace=ign_whitespace_lcl,
+                        context=context_lcl)
+
+                    diff_processor = diffs.DiffProcessor(
+                        vcs_diff, format='newdiff', diff_limit=diff_limit,
+                        file_limit=file_limit, show_full_diff=c.fulldiff)
+
+                    _parsed = diff_processor.prepare()
+
+                    diffset = codeblocks.DiffSet(
+                        repo_name=self.db_repo_name,
+                        source_node_getter=codeblocks.diffset_node_getter(commit1),
+                        target_node_getter=codeblocks.diffset_node_getter(commit2))
+
+                    diffset = self.path_filter.render_patchset_filtered(
+                        diffset, _parsed, commit1.raw_id, commit2.raw_id)
+
+                    # save cached diff
+                    if caching_enabled:
+                        cache_diff(cache_file_path, diffset, None)
+
+                c.limited_diff = diffset.limited_diff
                 c.changes[commit.raw_id] = diffset
             else:
+                # TODO(marcink): no cache usage here...
+                _diff = self.rhodecode_vcs_repo.get_diff(
+                    commit1, commit2,
+                    ignore_whitespace=ign_whitespace_lcl, context=context_lcl)
+                diff_processor = diffs.DiffProcessor(
+                    _diff, format='newdiff', diff_limit=diff_limit,
+                    file_limit=file_limit, show_full_diff=c.fulldiff)
                 # downloads/raw we only need RAW diff nothing else
                 diff = self.path_filter.get_raw_patch(diff_processor)
                 c.changes[commit.raw_id] = [None, None, None, None, diff, None, None]

rhodecode/apps/repository/views/repo_compare.py

@@ -295,19 +295,10 b' class RepoCompareView(RepoAppView):'
             file_limit=file_limit, show_full_diff=c.fulldiff)
         _parsed = diff_processor.prepare()
 
-        def _node_getter(commit):
-            """ Returns a function that returns a node for a commit or None """
-            def get_node(fname):
-                try:
-                    return commit.get_node(fname)
-                except NodeDoesNotExistError:
-                    return None
-            return get_node
-
         diffset = codeblocks.DiffSet(
             repo_name=source_repo.repo_name,
-            source_node_getter=_node_getter(source_commit),
-            target_node_getter=_node_getter(target_commit),
+            source_node_getter=codeblocks.diffset_node_getter(source_commit),
+            target_node_getter=codeblocks.diffset_node_getter(target_commit),
         )
         c.diffset = self.path_filter.render_patchset_filtered(
             diffset, _parsed, source_ref, target_ref)

rhodecode/apps/repository/views/repo_pull_requests.py

@@ -34,6 +34,7 b' from rhodecode.apps._base import RepoApp'
 
 from rhodecode.lib import helpers as h, diffs, codeblocks, channelstream
 from rhodecode.lib.base import vcs_operation_context
+from rhodecode.lib.diffs import load_cached_diff, cache_diff, diff_cache_exist
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.auth import (
     LoginRequired, HasRepoPermissionAny, HasRepoPermissionAnyDecorator,
@@ -41,7 +42,7 b' from rhodecode.lib.auth import ('
 from rhodecode.lib.utils2 import str2bool, safe_str, safe_unicode
 from rhodecode.lib.vcs.backends.base import EmptyCommit, UpdateFailureReason
 from rhodecode.lib.vcs.exceptions import (CommitDoesNotExistError,
-    RepositoryRequirementError, NodeDoesNotExistError, EmptyRepositoryError)
+    RepositoryRequirementError, EmptyRepositoryError)
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.comment import CommentsModel
 from rhodecode.model.db import (func, or_, PullRequest, PullRequestVersion,
@@ -201,10 +202,16 b' class RepoPullRequestsView(RepoAppView, '
 
         return data
 
+    def _is_diff_cache_enabled(self, target_repo):
+        caching_enabled = self._get_general_setting(
+            target_repo, 'rhodecode_diff_cache')
+        log.debug('Diff caching enabled: %s', caching_enabled)
+        return caching_enabled
+
     def _get_diffset(self, source_repo_name, source_repo,
                      source_ref_id, target_ref_id,
-                     target_commit, source_commit, diff_limit, fulldiff,
-                     file_limit, display_inline_comments):
+                     target_commit, source_commit, diff_limit, file_limit,
+                     fulldiff):
 
         vcs_diff = PullRequestModel().get_diff(
             source_repo, source_ref_id, target_ref_id)
@@ -215,21 +222,11 b' class RepoPullRequestsView(RepoAppView, '
 
         _parsed = diff_processor.prepare()
 
-        def _node_getter(commit):
-            def get_node(fname):
-                try:
-                    return commit.get_node(fname)
-                except NodeDoesNotExistError:
-                    return None
-
-            return get_node
-
         diffset = codeblocks.DiffSet(
             repo_name=self.db_repo_name,
             source_repo_name=source_repo_name,
-            source_node_getter=_node_getter(target_commit),
-            target_node_getter=_node_getter(source_commit),
-            comments=display_inline_comments
+            source_node_getter=codeblocks.diffset_node_getter(target_commit),
+            target_node_getter=codeblocks.diffset_node_getter(source_commit),
         )
         diffset = self.path_filter.render_patchset_filtered(
             diffset, _parsed, target_commit.raw_id, source_commit.raw_id)
@@ -443,42 +440,54 b' class RepoPullRequestsView(RepoAppView, '
             commits_source_repo = source_scm
 
         c.commits_source_repo = commits_source_repo
-        commit_cache = {}
-        try:
-            pre_load = ["author", "branch", "date", "message"]
-            show_revs = pull_request_at_ver.revisions
-            for rev in show_revs:
-                comm = commits_source_repo.get_commit(
-                    commit_id=rev, pre_load=pre_load)
-                c.commit_ranges.append(comm)
-                commit_cache[comm.raw_id] = comm
-
-            # Order here matters, we first need to get target, and then
-            # the source
-            target_commit = commits_source_repo.get_commit(
-                commit_id=safe_str(target_ref_id))
-
-            source_commit = commits_source_repo.get_commit(
-                commit_id=safe_str(source_ref_id))
-
-        except CommitDoesNotExistError:
-            log.warning(
-                'Failed to get commit from `{}` repo'.format(
-                    commits_source_repo), exc_info=True)
-        except RepositoryRequirementError:
-            log.warning(
-                'Failed to get all required data from repo', exc_info=True)
-            c.missing_requirements = True
-
         c.ancestor = None  # set it to None, to hide it from PR view
 
-        try:
-            ancestor_id = source_scm.get_common_ancestor(
-                source_commit.raw_id, target_commit.raw_id, target_scm)
-            c.ancestor_commit = source_scm.get_commit(ancestor_id)
-        except Exception:
-            c.ancestor_commit = None
+        # empty version means latest, so we keep this to prevent
+        # double caching
+        version_normalized = version or 'latest'
+        from_version_normalized = from_version or 'latest'
+
+        cache_path = self.rhodecode_vcs_repo.get_create_shadow_cache_pr_path(
+            target_repo)
+        cache_file_path = diff_cache_exist(
+            cache_path, 'pull_request', pull_request_id, version_normalized,
+            from_version_normalized, source_ref_id, target_ref_id, c.fulldiff)
+
+        caching_enabled = self._is_diff_cache_enabled(c.target_repo)
+        force_recache = str2bool(self.request.GET.get('force_recache'))
+
+        cached_diff = None
+        if caching_enabled:
+            cached_diff = load_cached_diff(cache_file_path)
 
+        has_proper_commit_cache = (
+                cached_diff and cached_diff.get('commits')
+                and len(cached_diff.get('commits', [])) == 5
+                and cached_diff.get('commits')[0]
+                and cached_diff.get('commits')[3])
+        if not force_recache and has_proper_commit_cache:
+            diff_commit_cache = \
+                (ancestor_commit, commit_cache, missing_requirements,
+                 source_commit, target_commit) = cached_diff['commits']
+        else:
+            diff_commit_cache = \
+                (ancestor_commit, commit_cache, missing_requirements,
+                 source_commit, target_commit) = self.get_commits(
+                    commits_source_repo,
+                    pull_request_at_ver,
+                    source_commit,
+                    source_ref_id,
+                    source_scm,
+                    target_commit,
+                    target_ref_id,
+                    target_scm)
+
+        # register our commit range
+        for comm in commit_cache.values():
+            c.commit_ranges.append(comm)
+
+        c.missing_requirements = missing_requirements
+        c.ancestor_commit = ancestor_commit
         c.statuses = source_repo.statuses(
             [x.raw_id for x in c.commit_ranges])
 
@@ -500,12 +509,23 b' class RepoPullRequestsView(RepoAppView, '
 
             c.missing_commits = True
         else:
+            c.inline_comments = display_inline_comments
 
-            c.diffset = self._get_diffset(
-                c.source_repo.repo_name, commits_source_repo,
-                source_ref_id, target_ref_id,
-                target_commit, source_commit,
-                diff_limit, c.fulldiff, file_limit, display_inline_comments)
+            has_proper_diff_cache = cached_diff and cached_diff.get('commits')
+            if not force_recache and has_proper_diff_cache:
+                c.diffset = cached_diff['diff']
+                (ancestor_commit, commit_cache, missing_requirements,
+                 source_commit, target_commit) = cached_diff['commits']
+            else:
+                c.diffset = self._get_diffset(
+                    c.source_repo.repo_name, commits_source_repo,
+                    source_ref_id, target_ref_id,
+                    target_commit, source_commit,
+                    diff_limit, file_limit, c.fulldiff)
+
+                # save cached diff
+                if caching_enabled:
+                    cache_diff(cache_file_path, c.diffset, diff_commit_cache)
 
             c.limited_diff = c.diffset.limited_diff
 
@@ -568,7 +588,6 b' class RepoPullRequestsView(RepoAppView, '
             if self._rhodecode_user.user_id in allowed_reviewers:
                 for co in general_comments:
                     if co.author.user_id == self._rhodecode_user.user_id:
-                        # each comment has a status change
                         status = co.status_change
                         if status:
                             _ver_pr = status[0].comment.pull_request_version_id
@@ -576,6 +595,43 b' class RepoPullRequestsView(RepoAppView, '
 
         return self._get_template_context(c)
 
+    def get_commits(
+            self, commits_source_repo, pull_request_at_ver, source_commit,
+            source_ref_id, source_scm, target_commit, target_ref_id, target_scm):
+        commit_cache = collections.OrderedDict()
+        missing_requirements = False
+        try:
+            pre_load = ["author", "branch", "date", "message"]
+            show_revs = pull_request_at_ver.revisions
+            for rev in show_revs:
+                comm = commits_source_repo.get_commit(
+                    commit_id=rev, pre_load=pre_load)
+                commit_cache[comm.raw_id] = comm
+
+            # Order here matters, we first need to get target, and then
+            # the source
+            target_commit = commits_source_repo.get_commit(
+                commit_id=safe_str(target_ref_id))
+
+            source_commit = commits_source_repo.get_commit(
+                commit_id=safe_str(source_ref_id))
+        except CommitDoesNotExistError:
+            log.warning(
+                'Failed to get commit from `{}` repo'.format(
+                    commits_source_repo), exc_info=True)
+        except RepositoryRequirementError:
+            log.warning(
+                'Failed to get all required data from repo', exc_info=True)
+            missing_requirements = True
+        ancestor_commit = None
+        try:
+            ancestor_id = source_scm.get_common_ancestor(
+                source_commit.raw_id, target_commit.raw_id, target_scm)
+            ancestor_commit = source_scm.get_commit(ancestor_id)
+        except Exception:
+            ancestor_commit = None
+        return ancestor_commit, commit_cache, missing_requirements, source_commit, target_commit
+
     def assure_not_empty_repo(self):
         _ = self.request.translate
 

rhodecode/lib/codeblocks.py

@@ -30,6 +30,7 b' from rhodecode.lib.helpers import ('
     get_lexer_for_filenode, html_escape, get_custom_lexer)
 from rhodecode.lib.utils2 import AttributeDict, StrictAttributeDict
 from rhodecode.lib.vcs.nodes import FileNode
+from rhodecode.lib.vcs.exceptions import VCSError, NodeDoesNotExistError
 from rhodecode.lib.diff_match_patch import diff_match_patch
 from rhodecode.lib.diffs import LimitedDiffContainer
 from pygments.lexers import get_lexer_by_name
@@ -351,6 +352,16 b' def tokens_diff(old_tokens, new_tokens, '
     return old_tokens_result, new_tokens_result, similarity
 
 
+def diffset_node_getter(commit):
+    def get_node(fname):
+        try:
+            return commit.get_node(fname)
+        except NodeDoesNotExistError:
+            return None
+
+    return get_node
+
+
 class DiffSet(object):
     """
     An object for parsing the diff result from diffs.DiffProcessor and
@@ -515,6 +526,7 b' class DiffSet(object):'
         if target_file_path in self.comments_store:
             for lineno, comments in self.comments_store[target_file_path].items():
                 left_comments[lineno] = comments
+
         # left comments are one that we couldn't place in diff lines.
         # could be outdated, or the diff changed and this line is no
         # longer available
@@ -551,7 +563,7 b' class DiffSet(object):'
 
         result.lines.extend(
             self.parse_lines(before, after, source_file, target_file))
-        result.unified = self.as_unified(result.lines)
+        result.unified = list(self.as_unified(result.lines))
         result.sideside = result.lines
 
         return result
@@ -606,8 +618,9 b' class DiffSet(object):'
                 original.lineno = before['old_lineno']
                 original.content = before['line']
                 original.action = self.action_to_op(before['action'])
-                original.comments = self.get_comments_for('old',
-                    source_file, before['old_lineno'])
+
+                original.get_comment_args = (
+                    source_file, 'o', before['old_lineno'])
 
             if after:
                 if after['action'] == 'new-no-nl':
@@ -619,8 +632,9 b' class DiffSet(object):'
                 modified.lineno = after['new_lineno']
                 modified.content = after['line']
                 modified.action = self.action_to_op(after['action'])
-                modified.comments = self.get_comments_for('new',
-                    target_file, after['new_lineno'])
+
+                modified.get_comment_args = (
+                    target_file, 'n', after['new_lineno'])
 
             # diff the lines
             if before_tokens and after_tokens:
@@ -649,23 +663,6 b' class DiffSet(object):'
 
         return lines
 
-    def get_comments_for(self, version, filename, line_number):
-        if hasattr(filename, 'unicode_path'):
-            filename = filename.unicode_path
-
-        if not isinstance(filename, basestring):
-            return None
-
-        line_key = {
-            'old': 'o',
-            'new': 'n',
-        }[version] + str(line_number)
-
-        if filename in self.comments_store:
-            file_comments = self.comments_store[filename]
-            if line_key in file_comments:
-                return file_comments.pop(line_key)
-
     def get_line_tokens(self, line_text, line_number, file=None):
         filenode = None
         filename = None
@@ -722,25 +719,25 b' class DiffSet(object):'
                     if line.original.action == ' ':
                         yield (line.original.lineno, line.modified.lineno,
                                line.original.action, line.original.content,
-                               line.original.comments)
+                               line.original.get_comment_args)
                         continue
 
                     if line.original.action == '-':
                         yield (line.original.lineno, None,
                                line.original.action, line.original.content,
-                               line.original.comments)
+                               line.original.get_comment_args)
 
                     if line.modified.action == '+':
                         buf.append((
                             None, line.modified.lineno,
                             line.modified.action, line.modified.content,
-                            line.modified.comments))
+                            line.modified.get_comment_args))
                         continue
 
                 if line.modified:
                     yield (None, line.modified.lineno,
                            line.modified.action, line.modified.content,
-                           line.modified.comments)
+                           line.modified.get_comment_args)
 
             for b in buf:
                 yield b

rhodecode/lib/diffs.py

@@ -23,16 +23,18 b''
 Set of diffing helpers, previously part of vcs
 """
 
+import os
 import re
 import collections
 import difflib
 import logging
+import cPickle as pickle
 
 from itertools import tee, imap
 
 from rhodecode.lib.vcs.exceptions import VCSError
 from rhodecode.lib.vcs.nodes import FileNode, SubModuleNode
-from rhodecode.lib.utils2 import safe_unicode
+from rhodecode.lib.utils2 import safe_unicode, safe_str
 
 log = logging.getLogger(__name__)
 
@@ -1129,3 +1131,82 b' class LineNotInDiffException(Exception):'
 
 class DiffLimitExceeded(Exception):
     pass
+
+
+def cache_diff(cached_diff_file, diff, commits):
+
+    struct = {
+        'version': 'v1',
+        'diff': diff,
+        'commits': commits
+    }
+
+    try:
+        with open(cached_diff_file, 'wb') as f:
+            pickle.dump(struct, f)
+        log.debug('Saved diff cache under %s', cached_diff_file)
+    except Exception:
+        log.warn('Failed to save cache', exc_info=True)
+        # cleanup file to not store it "damaged"
+        try:
+            os.remove(cached_diff_file)
+        except Exception:
+            log.exception('Failed to cleanup path %s', cached_diff_file)
+
+
+def load_cached_diff(cached_diff_file):
+
+    default_struct = {
+        'version': 'v1',
+        'diff': None,
+        'commits': None
+    }
+
+    has_cache = os.path.isfile(cached_diff_file)
+    if not has_cache:
+        return default_struct
+
+    data = None
+    try:
+        with open(cached_diff_file, 'rb') as f:
+            data = pickle.load(f)
+        log.debug('Loaded diff cache from %s', cached_diff_file)
+    except Exception:
+        log.warn('Failed to read diff cache file', exc_info=True)
+
+    if not data:
+        data = default_struct
+
+    if not isinstance(data, dict):
+        # old version of data ?
+        data = default_struct
+
+    return data
+
+
+def generate_diff_cache_key(*args):
+    """
+    Helper to generate a cache key using arguments
+    """
+    def arg_mapper(input_param):
+        input_param = safe_str(input_param)
+        # we cannot allow '/' in arguments since it would allow
+        # subdirectory usage
+        input_param.replace('/', '_')
+        return input_param or None  # prevent empty string arguments
+
+    return '_'.join([
+        '{}' for i in range(len(args))]).format(*map(arg_mapper, args))
+
+
+def diff_cache_exist(cache_storage, *args):
+    """
+    Based on all generated arguments check and return a cache path
+    """
+    cache_key = generate_diff_cache_key(*args)
+    cache_file_path = os.path.join(cache_storage, cache_key)
+    # prevent path traversal attacks using some param that have e.g '../../'
+    if not os.path.abspath(cache_file_path).startswith(cache_storage):
+        raise ValueError('Final path must be within {}'.format(cache_storage))
+
+    return cache_file_path

rhodecode/lib/utils2.py

@@ -709,7 +709,19 b' def extract_mentioned_users(s):'
     return sorted(list(usrs), key=lambda k: k.lower())
 
 
-class StrictAttributeDict(dict):
+class AttributeDictBase(dict):
+    def __getstate__(self):
+        odict = self.__dict__  # get attribute dictionary
+        return odict
+
+    def __setstate__(self, dict):
+        self.__dict__ = dict
+
+    __setattr__ = dict.__setitem__
+    __delattr__ = dict.__delitem__
+
+
+class StrictAttributeDict(AttributeDictBase):
     """
     Strict Version of Attribute dict which raises an Attribute error when
     requested attribute is not set
@@ -720,15 +732,12 b' class StrictAttributeDict(dict):'
         except KeyError:
             raise AttributeError('%s object has no attribute %s' % (
                 self.__class__, attr))
-    __setattr__ = dict.__setitem__
-    __delattr__ = dict.__delitem__
 
 
-class AttributeDict(dict):
+class AttributeDict(AttributeDictBase):
     def __getattr__(self, attr):
         return self.get(attr, None)
-    __setattr__ = dict.__setitem__
-    __delattr__ = dict.__delitem__
+
 
 
 def fix_PATH(os_=None):

rhodecode/lib/vcs/backends/base.py

@@ -206,6 +206,14 b' class BaseRepository(object):'
     def __ne__(self, other):
         return not self.__eq__(other)
 
+    def get_create_shadow_cache_pr_path(self, repo):
+        path = os.path.join(
+            os.path.dirname(self.path),
+            '.__shadow_diff_cache_repo_{}/'.format(repo.repo_id))
+        if not os.path.exists(path):
+            os.makedirs(path, 0755)
+        return path
+
     @classmethod
     def get_default_config(cls, default=None):
         config = Config()
@@ -745,6 +753,12 b' class BaseCommit(object):'
             'branch': self.branch
         }
 
+    def __getstate__(self):
+        d = self.__dict__.copy()
+        d.pop('_remote', None)
+        d.pop('repository', None)
+        return d
+
     def _get_refs(self):
         return {
             'branches': [self.branch] if self.branch else [],

rhodecode/model/forms.py

@@ -430,6 +430,9 b' class _BaseVcsSettingsForm(formencode.Sc'
     vcs_svn_proxy_http_requests_enabled = v.StringBoolean(if_missing=False)
     vcs_svn_proxy_http_server_url = v.UnicodeString(strip=True, if_missing=None)
 
+    # cache
+    rhodecode_diff_cache = v.StringBoolean(if_missing=False)
+
 
 def ApplicationUiSettingsForm(localizer):
     _ = localizer

rhodecode/model/settings.py

@@ -417,7 +417,9 b' class VcsSettingsModel(object):'
         'hg_use_rebase_for_merging',
         'hg_close_branch_before_merging',
         'git_use_rebase_for_merging',
-        'git_close_branch_before_merging')
+        'git_close_branch_before_merging',
+        'diff_cache',
+    )
 
     HOOKS_SETTINGS = (
         ('hooks', 'changegroup.repo_size'),
@@ -438,6 +440,7 b' class VcsSettingsModel(object):'
     GLOBAL_GIT_SETTINGS = (
         ('vcs_git_lfs', 'enabled'),
         ('vcs_git_lfs', 'store_location'))
+
     GLOBAL_SVN_SETTINGS = (
         ('vcs_svn_proxy', 'http_requests_enabled'),
         ('vcs_svn_proxy', 'http_server_url'))
@@ -571,6 +574,7 b' class VcsSettingsModel(object):'
         self._create_or_update_ui(
             self.repo_settings, *phases, value=safe_str(data[phases_key]))
 
+
     def create_or_update_global_hg_settings(self, data):
         largefiles, largefiles_store, phases, hgsubversion, evolve \
             = self.GLOBAL_HG_SETTINGS

rhodecode/templates/base/vcs_settings.mako

@@ -312,6 +312,20 b''
         </div>
     % endif
 
+    % if display_globals or repo_type in ['hg', 'git', 'svn']:
+        <div class="panel panel-default">
+            <div class="panel-heading" id="vcs-pull-requests-options">
+                <h3 class="panel-title">${_('Diff cache')}<a class="permalink" href="#vcs-pull-requests-options"> ¶</a></h3>
+            </div>
+            <div class="panel-body">
+                <div class="checkbox">
+                    ${h.checkbox('rhodecode_diff_cache' + suffix, 'True', **kwargs)}
+                    <label for="rhodecode_diff_cache${suffix}">${_('Enable caching diffs for pull requests cache and commits')}</label>
+                </div>
+            </div>
+        </div>
+    % endif
+
     % if display_globals or repo_type in ['hg',]:
         <div class="panel panel-default">
             <div class="panel-heading" id="vcs-pull-requests-options">

rhodecode/templates/changeset/changeset.mako

@@ -213,7 +213,7 b''
     <%namespace name="cbdiffs" file="/codeblocks/diffs.mako"/>
     ${cbdiffs.render_diffset_menu()}
     ${cbdiffs.render_diffset(
-      c.changes[c.commit.raw_id], commit=c.commit, use_comments=True)}
+      c.changes[c.commit.raw_id], commit=c.commit, use_comments=True,inline_comments=c.inline_comments )}
   </div>
 
     ## template for inline comment form

rhodecode/templates/codeblocks/diffs.mako

@@ -44,13 +44,15 b" return '%s_%s_%i' % (h.safeid(filename),"
 
     # special file-comments that were deleted in previous versions
     # it's used for showing outdated comments for deleted files in a PR
-    deleted_files_comments=None
+    deleted_files_comments=None,
+
+    # for cache purpose
+    inline_comments=None
 
 )">
-
 %if use_comments:
 <div id="cb-comments-inline-container-template" class="js-template">
-  ${inline_comments_container([])}
+  ${inline_comments_container([], inline_comments)}
 </div>
 <div class="js-template" id="cb-comment-inline-form-template">
     <div class="comment-inline-form ac">
@@ -211,9 +213,9 b' collapse_all = len(diffset.files) > coll'
                     </td>
                 </tr>
             %if c.diffmode == 'unified':
-                    ${render_hunk_lines_unified(hunk, use_comments=use_comments)}
+                    ${render_hunk_lines_unified(hunk, use_comments=use_comments, inline_comments=inline_comments)}
             %elif c.diffmode == 'sideside':
-                    ${render_hunk_lines_sideside(hunk, use_comments=use_comments)}
+                    ${render_hunk_lines_sideside(hunk, use_comments=use_comments, inline_comments=inline_comments)}
             %else:
                 <tr class="cb-line">
                     <td>unknown diff mode</td>
@@ -230,7 +232,7 b' collapse_all = len(diffset.files) > coll'
                 <td class="cb-lineno cb-context"></td>
                 <td class="cb-lineno cb-context"></td>
                 <td class="cb-content cb-context">
-                    ${inline_comments_container(comments)}
+                    ${inline_comments_container(comments, inline_comments)}
                 </td>
             </tr>
         %elif c.diffmode == 'sideside':
@@ -239,7 +241,7 b' collapse_all = len(diffset.files) > coll'
                 <td class="cb-lineno cb-context"></td>
                 <td class="cb-content cb-context">
                     % if lineno.startswith('o'):
-                        ${inline_comments_container(comments)}
+                        ${inline_comments_container(comments, inline_comments)}
                     % endif
                 </td>
 
@@ -247,7 +249,7 b' collapse_all = len(diffset.files) > coll'
                 <td class="cb-lineno cb-context"></td>
                 <td class="cb-content cb-context">
                     % if lineno.startswith('n'):
-                        ${inline_comments_container(comments)}
+                        ${inline_comments_container(comments, inline_comments)}
                     % endif
                 </td>
             </tr>
@@ -298,7 +300,7 b' collapse_all = len(diffset.files) > coll'
                         <td class="cb-lineno cb-context"></td>
                         <td class="cb-lineno cb-context"></td>
                         <td class="cb-content cb-context">
-                            ${inline_comments_container(comments_dict['comments'])}
+                            ${inline_comments_container(comments_dict['comments'], inline_comments)}
                         </td>
                     </tr>
                     %elif c.diffmode == 'sideside':
@@ -310,7 +312,7 b' collapse_all = len(diffset.files) > coll'
                         <td class="cb-data cb-context"></td>
                         <td class="cb-lineno cb-context"></td>
                         <td class="cb-content cb-context">
-                            ${inline_comments_container(comments_dict['comments'])}
+                            ${inline_comments_container(comments_dict['comments'], inline_comments)}
                         </td>
                     </tr>
                     %endif
@@ -484,12 +486,11 b' from rhodecode.lib.diffs import NEW_FILE'
 </%def>
 
 
-<%def name="inline_comments_container(comments)">
+<%def name="inline_comments_container(comments, inline_comments)">
 <div class="inline-comments">
     %for comment in comments:
-    ${commentblock.comment_block(comment, inline=True)}
+        ${commentblock.comment_block(comment, inline=True)}
     %endfor
-
     % if comments and comments[-1].outdated:
     <span class="btn btn-secondary cb-comment-add-button comment-outdated}"
           style="display: none;}">
@@ -505,8 +506,23 b' from rhodecode.lib.diffs import NEW_FILE'
 </div>
 </%def>
 
+<%!
+def get_comments_for(comments, filename, line_version, line_number):
+    if hasattr(filename, 'unicode_path'):
+        filename = filename.unicode_path
 
-<%def name="render_hunk_lines_sideside(hunk, use_comments=False)">
+    if not isinstance(filename, basestring):
+        return None
+
+    line_key = '{}{}'.format(line_version, line_number)
+    if comments and filename in comments:
+        file_comments = comments[filename]
+        if line_key in file_comments:
+            return file_comments[line_key]
+%>
+
+<%def name="render_hunk_lines_sideside(hunk, use_comments=False, inline_comments=None)">
+
     %for i, line in enumerate(hunk.sideside):
     <%
     old_line_anchor, new_line_anchor = None, None
@@ -521,12 +537,16 b' from rhodecode.lib.diffs import NEW_FILE'
             data-line-no="${line.original.lineno}"
             >
             <div>
-            %if line.original.comments:
-                <% has_outdated = any([x.outdated for x in line.original.comments]) %>
+            <% loc = None %>
+            %if line.original.get_comment_args:
+                <% loc = get_comments_for(inline_comments, *line.original.get_comment_args) %>
+            %endif
+            %if loc:
+                <% has_outdated = any([x.outdated for x in loc]) %>
                 % if has_outdated:
-                    <i title="${_('comments including outdated')}:${len(line.original.comments)}" class="icon-comment_toggle" onclick="return Rhodecode.comments.toggleLineComments(this)"></i>
+                    <i title="${_('comments including outdated')}:${len(loc)}" class="icon-comment_toggle" onclick="return Rhodecode.comments.toggleLineComments(this)"></i>
                 % else:
-                    <i title="${_('comments')}: ${len(line.original.comments)}" class="icon-comment" onclick="return Rhodecode.comments.toggleLineComments(this)"></i>
+                    <i title="${_('comments')}: ${len(loc)}" class="icon-comment" onclick="return Rhodecode.comments.toggleLineComments(this)"></i>
                 % endif
             %endif
             </div>
@@ -548,20 +568,28 b' from rhodecode.lib.diffs import NEW_FILE'
             ${render_add_comment_button()}
             %endif
             <span class="cb-code">${line.original.action} ${line.original.content or '' | n}</span>
-            %if use_comments and line.original.lineno and line.original.comments:
-            ${inline_comments_container(line.original.comments)}
+
+            %if use_comments and line.original.lineno and loc:
+                ${inline_comments_container(loc, inline_comments)}
             %endif
+
         </td>
         <td class="cb-data ${action_class(line.modified.action)}"
             data-line-no="${line.modified.lineno}"
             >
             <div>
-            %if line.modified.comments:
-                <% has_outdated = any([x.outdated for x in line.modified.comments]) %>
+
+            %if line.modified.get_comment_args:
+                <% lmc = get_comments_for(inline_comments, *line.modified.get_comment_args) %>
+            %else:
+                <% lmc = None%>
+            %endif
+            %if lmc:
+                <% has_outdated = any([x.outdated for x in lmc]) %>
                 % if has_outdated:
-                    <i title="${_('comments including outdated')}:${len(line.modified.comments)}" class="icon-comment_toggle" onclick="return Rhodecode.comments.toggleLineComments(this)"></i>
+                    <i title="${_('comments including outdated')}:${len(lmc)}" class="icon-comment_toggle" onclick="return Rhodecode.comments.toggleLineComments(this)"></i>
                 % else:
-                    <i title="${_('comments')}: ${len(line.modified.comments)}" class="icon-comment" onclick="return Rhodecode.comments.toggleLineComments(this)"></i>
+                    <i title="${_('comments')}: ${len(lmc)}" class="icon-comment" onclick="return Rhodecode.comments.toggleLineComments(this)"></i>
                 % endif
             %endif
             </div>
@@ -583,8 +611,8 b' from rhodecode.lib.diffs import NEW_FILE'
             ${render_add_comment_button()}
             %endif
             <span class="cb-code">${line.modified.action} ${line.modified.content or '' | n}</span>
-            %if use_comments and line.modified.lineno and line.modified.comments:
-            ${inline_comments_container(line.modified.comments)}
+            %if use_comments and line.modified.lineno and lmc:
+            ${inline_comments_container(lmc, inline_comments)}
             %endif
         </td>
     </tr>
@@ -592,8 +620,8 b' from rhodecode.lib.diffs import NEW_FILE'
 </%def>
 
 
-<%def name="render_hunk_lines_unified(hunk, use_comments=False)">
-    %for old_line_no, new_line_no, action, content, comments in hunk.unified:
+<%def name="render_hunk_lines_unified(hunk, use_comments=False,  inline_comments=None)">
+    %for old_line_no, new_line_no, action, content, comments_args in hunk.unified:
     <%
     old_line_anchor, new_line_anchor = None, None
     if old_line_no:
@@ -604,6 +632,13 b' from rhodecode.lib.diffs import NEW_FILE'
     <tr class="cb-line">
         <td class="cb-data ${action_class(action)}">
             <div>
+
+            %if comments_args:
+                <% comments = get_comments_for(inline_comments, *comments_args) %>
+            %else:
+                <% comments = None%>
+            %endif
+
             % if comments:
                 <% has_outdated = any([x.outdated for x in comments]) %>
                 % if has_outdated:
@@ -642,7 +677,7 b' from rhodecode.lib.diffs import NEW_FILE'
             %endif
             <span class="cb-code">${action} ${content or '' | n}</span>
             %if use_comments and comments:
-            ${inline_comments_container(comments)}
+            ${inline_comments_container(comments, inline_comments)}
             %endif
         </td>
     </tr>

rhodecode/templates/pullrequests/pullrequest_show.mako

@@ -570,7 +570,8 b''
                       c.diffset, use_comments=True,
                       collapse_when_files_over=30,
                       disable_new_comments=not c.allowed_to_comment,
-                      deleted_files_comments=c.deleted_files_comments)}
+                      deleted_files_comments=c.deleted_files_comments,
+                      inline_comments=c.inline_comments)}
                 </div>
               % else:
                   ## skipping commits we need to clear the view for missing commits

rhodecode/tests/models/settings/test_vcs_settings.py

@@ -44,6 +44,7 b' GENERAL_FORM_DATA = {'
     'rhodecode_hg_close_branch_before_merging': True,
     'rhodecode_git_use_rebase_for_merging': True,
     'rhodecode_git_close_branch_before_merging': True,
+    'rhodecode_diff_cache': True,
 }