Fixed issue with inproper handling of diff parsing that could lead to infinit loops.

This was an edge case when diff contained diff data inside. Regresion test was
added

rhodecode/tests/fixtures/diff_with_diff_data.diff

@@ -0,0 +1,416 b''
+diff --git a/vcs/backends/base.py b/vcs/backends/base.py
+index 212267ca23949807b8d89fa8ca495827dcfab3b1..ad17f16634da602503ed4ddd7cdd2e1ccdf4bed4 100644
+--- a/vcs/backends/base.py
++++ b/vcs/backends/base.py
+@@ -54,6 +54,7 @@ class BaseRepository(object):
+     """
+     scm = None
+     DEFAULT_BRANCH_NAME = None
++    EMPTY_CHANGESET = '0' * 40
+ 
+     def __init__(self, repo_path, create=False, **kwargs):
+         """
+@@ -204,6 +205,23 @@ class BaseRepository(object):
+         """
+         raise NotImplementedError
+ 
++    def get_diff(self, rev1, rev2, path=None, ignore_whitespace=False,
++            context=3):
++        """
++        Returns (git like) *diff*, as plain text. Shows changes introduced by
++        ``rev2`` since ``rev1``.
++
++        :param rev1: Entry point from which diff is shown. Can be
++          ``self.EMPTY_CHANGESET`` - in this case, patch showing all
++          the changes since empty state of the repository until ``rev2``
++        :param rev2: Until which revision changes should be shown.
++        :param ignore_whitespace: If set to ``True``, would not show whitespace
++          changes. Defaults to ``False``.
++        :param context: How many lines before/after changed lines should be
++          shown. Defaults to ``3``.
++        """
++        raise NotImplementedError
++
+     # ========== #
+     # COMMIT API #
+     # ========== #
+@@ -341,7 +359,6 @@ class BaseChangeset(object):
+             otherwise; trying to access this attribute while there is no
+             changesets would raise ``EmptyRepositoryError``
+     """
+-
+     def __str__(self):
+         return '<%s at %s:%s>' % (self.__class__.__name__, self.revision,
+             self.short_id)
+@@ -591,7 +608,6 @@ class BaseChangeset(object):
+         return data
+ 
+ 
+-
+ class BaseWorkdir(object):
+     """
+     Working directory representation of single repository.
+diff --git a/vcs/backends/git/repository.py b/vcs/backends/git/repository.py
+index 8b9d1247fdee44e7a021b80e4965d8609cfd5720..e9f04e74dedd2f57417eb91dd2f4f7c61ec7e097 100644
+--- a/vcs/backends/git/repository.py
++++ b/vcs/backends/git/repository.py
+@@ -12,6 +12,7 @@
+ import os
+ import re
+ import time
++import inspect
+ import posixpath
+ from dulwich.repo import Repo, NotGitRepository
+ #from dulwich.config import ConfigFile
+@@ -101,21 +102,6 @@ class GitRepository(BaseRepository):
+                 "stderr:\n%s" % (cmd, se))
+         return so, se
+ 
+-    def _get_diff(self, rev1, rev2, path=None, ignore_whitespace=False,
+-            context=3):
+-        rev1 = self._get_revision(rev1)
+-        rev2 = self._get_revision(rev2)
+-        
+-        if ignore_whitespace:
+-            cmd = 'diff -U%s -w %s %s' % (context, rev1, rev2)
+-        else:
+-            cmd = 'diff -U%s %s %s' % (context, rev1, rev2)
+-        if path:
+-            cmd += ' -- "%s"' % path
+-        so, se = self.run_git_command(cmd)
+-
+-        return so
+-
+     def _check_url(self, url):
+         """
+         Functon will check given url and try to verify if it's a valid
+@@ -322,6 +308,8 @@ class GitRepository(BaseRepository):
+         Returns ``GitChangeset`` object representing commit from git repository
+         at the given revision or head (most recent commit) if None given.
+         """
++        if isinstance(revision, GitChangeset):
++            return revision
+         revision = self._get_revision(revision)
+         changeset = GitChangeset(repository=self, revision=revision)
+         return changeset
+@@ -398,6 +386,49 @@ class GitRepository(BaseRepository):
+         for rev in revs:
+             yield self.get_changeset(rev)
+ 
++    def get_diff(self, rev1, rev2, path=None, ignore_whitespace=False,
++            context=3):
++        """
++        Returns (git like) *diff*, as plain text. Shows changes introduced by
++        ``rev2`` since ``rev1``.
++
++        :param rev1: Entry point from which diff is shown. Can be
++          ``self.EMPTY_CHANGESET`` - in this case, patch showing all
++          the changes since empty state of the repository until ``rev2``
++        :param rev2: Until which revision changes should be shown.
++        :param ignore_whitespace: If set to ``True``, would not show whitespace
++          changes. Defaults to ``False``.
++        :param context: How many lines before/after changed lines should be
++          shown. Defaults to ``3``.
++        """
++        flags = ['-U%s' % context]
++        if ignore_whitespace:
++            flags.append('-w')
++
++        if rev1 == self.EMPTY_CHANGESET:
++            rev2 = self.get_changeset(rev2).raw_id
++            cmd = ' '.join(['show'] + flags + [rev2])
++        else:
++            rev1 = self.get_changeset(rev1).raw_id
++            rev2 = self.get_changeset(rev2).raw_id
++            cmd = ' '.join(['diff'] + flags + [rev1, rev2])
++
++        if path:
++            cmd += ' -- "%s"' % path
++        stdout, stderr = self.run_git_command(cmd)
++        # If we used 'show' command, strip first few lines (until actual diff
++        # starts)
++        if rev1 == self.EMPTY_CHANGESET:
++            lines = stdout.splitlines()
++            x = 0
++            for line in lines:
++                if line.startswith('diff'):
++                    break
++                x += 1
++            # Append new line just like 'diff' command do
++            stdout = '\n'.join(lines[x:]) + '\n'
++        return stdout
++
+     @LazyProperty
+     def in_memory_changeset(self):
+         """
+diff --git a/vcs/backends/hg.py b/vcs/backends/hg.py
+index f1f9f95e4d476ab01d8e7b02a8b59034c0740a3b..b7d63c552c39b2f8aaec17ef46551369c8b8e793 100644
+--- a/vcs/backends/hg.py
++++ b/vcs/backends/hg.py
+@@ -256,13 +256,32 @@ class MercurialRepository(BaseRepository):
+ 
+         return map(lambda x: hex(x[7]), self._repo.changelog.index)[:-1]
+ 
+-    def _get_diff(self, rev1, rev2, path=None, ignore_whitespace=False,
++    def get_diff(self, rev1, rev2, path='', ignore_whitespace=False,
+                   context=3):
++        """
++        Returns (git like) *diff*, as plain text. Shows changes introduced by
++        ``rev2`` since ``rev1``.
++
++        :param rev1: Entry point from which diff is shown. Can be
++          ``self.EMPTY_CHANGESET`` - in this case, patch showing all
++          the changes since empty state of the repository until ``rev2``
++        :param rev2: Until which revision changes should be shown.
++        :param ignore_whitespace: If set to ``True``, would not show whitespace
++          changes. Defaults to ``False``.
++        :param context: How many lines before/after changed lines should be
++          shown. Defaults to ``3``.
++        """
++        # Check if given revisions are present at repository (may raise
++        # ChangesetDoesNotExistError)
++        if rev1 != self.EMPTY_CHANGESET:
++            self.get_changeset(rev1)
++        self.get_changeset(rev2)
++
+         file_filter = match(self.path, '', [path])
+-        return patch.diff(self._repo, rev1, rev2, match=file_filter,
++        return ''.join(patch.diff(self._repo, rev1, rev2, match=file_filter,
+                           opts=diffopts(git=True,
+                                         ignorews=ignore_whitespace,
+-                                        context=context))
++                                        context=context)))
+ 
+     def _check_url(self, url):
+         """
+diff --git a/vcs/tests/test_git.py b/vcs/tests/test_git.py
+index 30da035a2a35c3dca14064778e97188b6d4ce5d6..d4b82b9e612af8bb5bf490a827377c7c2567735a 100644
+--- a/vcs/tests/test_git.py
++++ b/vcs/tests/test_git.py
+@@ -639,19 +639,19 @@ class GitSpecificWithRepoTest(BackendTestMixin, unittest.TestCase):
+ 
+     def test_get_diff_runs_git_command_with_hashes(self):
+         self.repo.run_git_command = mock.Mock(return_value=['', ''])
+-        self.repo._get_diff(0, 1)
++        self.repo.get_diff(0, 1)
+         self.repo.run_git_command.assert_called_once_with('diff -U%s %s %s' %
+             (3, self.repo._get_revision(0), self.repo._get_revision(1)))
+ 
+     def test_get_diff_runs_git_command_with_str_hashes(self):
+         self.repo.run_git_command = mock.Mock(return_value=['', ''])
+-        self.repo._get_diff('0' * 40, 1)
+-        self.repo.run_git_command.assert_called_once_with('diff -U%s %s %s' %
+-            (3, self.repo._get_revision(0), self.repo._get_revision(1)))
++        self.repo.get_diff(self.repo.EMPTY_CHANGESET, 1)
++        self.repo.run_git_command.assert_called_once_with('show -U%s %s' %
++            (3, self.repo._get_revision(1)))
+ 
+     def test_get_diff_runs_git_command_with_path_if_its_given(self):
+         self.repo.run_git_command = mock.Mock(return_value=['', ''])
+-        self.repo._get_diff(0, 1, 'foo')
++        self.repo.get_diff(0, 1, 'foo')
+         self.repo.run_git_command.assert_called_once_with('diff -U%s %s %s -- "foo"'
+             % (3, self.repo._get_revision(0), self.repo._get_revision(1)))
+ 
+diff --git a/vcs/tests/test_repository.py b/vcs/tests/test_repository.py
+index e34033e29fa9b3d3366b723beab129cee73869b9..b6e3f419778d6009229e9108824acaf83eea1784 100644
+--- a/vcs/tests/test_repository.py
++++ b/vcs/tests/test_repository.py
+@@ -1,9 +1,12 @@
+ from __future__ import with_statement
++import datetime
+ from base import BackendTestMixin
+ from conf import SCM_TESTS
++from conf import TEST_USER_CONFIG_FILE
++from vcs.nodes import FileNode
+ from vcs.utils.compat import unittest
++from vcs.exceptions import ChangesetDoesNotExistError
+ 
+-from conf import TEST_USER_CONFIG_FILE
+ 
+ class RepositoryBaseTest(BackendTestMixin):
+     recreate_repo_per_test = False
+@@ -29,6 +32,176 @@ class RepositoryBaseTest(BackendTestMixin):
+             '[e-mail]')
+ 
+ 
++
++class RepositoryGetDiffTest(BackendTestMixin):
++
++    @classmethod
++    def _get_commits(cls):
++        commits = [
++            {
++                'message': 'Initial commit',
++                'author': 'Joe Doe <[e-mail]>',
++                'date': datetime.datetime(2010, 1, 1, 20),
++                'added': [
++                    FileNode('foobar', content='foobar'),
++                    FileNode('foobar2', content='foobar2'),
++                ],
++            },
++            {
++                'message': 'Changed foobar, added foobar3',
++                'author': 'Jane Doe <[e-mail]>',
++                'date': datetime.datetime(2010, 1, 1, 21),
++                'added': [
++                    FileNode('foobar3', content='foobar3'),
++                ],
++                'changed': [
++                    FileNode('foobar', 'FOOBAR'),
++                ],
++            },
++            {
++                'message': 'Removed foobar, changed foobar3',
++                'author': 'Jane Doe <[e-mail]>',
++                'date': datetime.datetime(2010, 1, 1, 22),
++                'changed': [
++                    FileNode('foobar3', content='FOOBAR\nFOOBAR\nFOOBAR\n'),
++                ],
++                'removed': [FileNode('foobar')],
++            },
++        ]
++        return commits
++
++    def test_raise_for_wrong(self):
++        with self.assertRaises(ChangesetDoesNotExistError):
++            self.repo.get_diff('a' * 40, 'b' * 40)
++
++class GitRepositoryGetDiffTest(RepositoryGetDiffTest, unittest.TestCase):
++    backend_alias = 'git'
++
++    def test_initial_commit_diff(self):
++        initial_rev = self.repo.revisions[0]
++        self.assertEqual(self.repo.get_diff(self.repo.EMPTY_CHANGESET, initial_rev), '''diff --git a/foobar b/foobar
++new file mode 100644
++index 0000000..f6ea049
++--- /dev/null
+++++ b/foobar
++@@ -0,0 +1 @@
+++foobar
++\ No newline at end of file
++diff --git a/foobar2 b/foobar2
++new file mode 100644
++index 0000000..e8c9d6b
++--- /dev/null
+++++ b/foobar2
++@@ -0,0 +1 @@
+++foobar2
++\ No newline at end of file
++''')
++
++    def test_second_changeset_diff(self):
++        revs = self.repo.revisions
++        self.assertEqual(self.repo.get_diff(revs[0], revs[1]), '''diff --git a/foobar b/foobar
++index f6ea049..389865b 100644
++--- a/foobar
+++++ b/foobar
++@@ -1 +1 @@
++-foobar
++\ No newline at end of file
+++FOOBAR
++\ No newline at end of file
++diff --git a/foobar3 b/foobar3
++new file mode 100644
++index 0000000..c11c37d
++--- /dev/null
+++++ b/foobar3
++@@ -0,0 +1 @@
+++foobar3
++\ No newline at end of file
++''')
++
++    def test_third_changeset_diff(self):
++        revs = self.repo.revisions
++        self.assertEqual(self.repo.get_diff(revs[1], revs[2]), '''diff --git a/foobar b/foobar
++deleted file mode 100644
++index 389865b..0000000
++--- a/foobar
+++++ /dev/null
++@@ -1 +0,0 @@
++-FOOBAR
++\ No newline at end of file
++diff --git a/foobar3 b/foobar3
++index c11c37d..f932447 100644
++--- a/foobar3
+++++ b/foobar3
++@@ -1 +1,3 @@
++-foobar3
++\ No newline at end of file
+++FOOBAR
+++FOOBAR
+++FOOBAR
++''')
++
++
++class HgRepositoryGetDiffTest(RepositoryGetDiffTest, unittest.TestCase):
++    backend_alias = 'hg'
++
++    def test_initial_commit_diff(self):
++        initial_rev = self.repo.revisions[0]
++        self.assertEqual(self.repo.get_diff(self.repo.EMPTY_CHANGESET, initial_rev), '''diff --git a/foobar b/foobar
++new file mode 100755
++--- /dev/null
+++++ b/foobar
++@@ -0,0 +1,1 @@
+++foobar
++\ No newline at end of file
++diff --git a/foobar2 b/foobar2
++new file mode 100755
++--- /dev/null
+++++ b/foobar2
++@@ -0,0 +1,1 @@
+++foobar2
++\ No newline at end of file
++''')
++
++    def test_second_changeset_diff(self):
++        revs = self.repo.revisions
++        self.assertEqual(self.repo.get_diff(revs[0], revs[1]), '''diff --git a/foobar b/foobar
++--- a/foobar
+++++ b/foobar
++@@ -1,1 +1,1 @@
++-foobar
++\ No newline at end of file
+++FOOBAR
++\ No newline at end of file
++diff --git a/foobar3 b/foobar3
++new file mode 100755
++--- /dev/null
+++++ b/foobar3
++@@ -0,0 +1,1 @@
+++foobar3
++\ No newline at end of file
++''')
++
++    def test_third_changeset_diff(self):
++        revs = self.repo.revisions
++        self.assertEqual(self.repo.get_diff(revs[1], revs[2]), '''diff --git a/foobar b/foobar
++deleted file mode 100755
++--- a/foobar
+++++ /dev/null
++@@ -1,1 +0,0 @@
++-FOOBAR
++\ No newline at end of file
++diff --git a/foobar3 b/foobar3
++--- a/foobar3
+++++ b/foobar3
++@@ -1,1 +1,3 @@
++-foobar3
++\ No newline at end of file
+++FOOBAR
+++FOOBAR
+++FOOBAR
++''')
++
++
+ # For each backend create test case class
+ for alias in SCM_TESTS:
+     attrs = {
+@@ -38,7 +211,6 @@ for alias in SCM_TESTS:
+     bases = (RepositoryBaseTest, unittest.TestCase)
+     globals()[cls_name] = type(cls_name, bases, attrs)
+ 
+-
+ if __name__ == '__main__':
+     unittest.main()

rhodecode/lib/diffs.py

@@ -1,768 +1,769 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.lib.diffs
     ~~~~~~~~~~~~~~~~~~~
 
     Set of diffing helpers, previously part of vcs
 
 
     :created_on: Dec 4, 2011
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :original copyright: 2007-2008 by Armin Ronacher
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import re
 import difflib
 import logging
 import traceback
 
 from itertools import tee, imap
 
 from mercurial import patch
 from mercurial.mdiff import diffopts
 from mercurial.bundlerepo import bundlerepository
 
 from pylons.i18n.translation import _
 
 from rhodecode.lib.compat import BytesIO
 from rhodecode.lib.vcs.utils.hgcompat import localrepo
 from rhodecode.lib.vcs.exceptions import VCSError
 from rhodecode.lib.vcs.nodes import FileNode, SubModuleNode
 from rhodecode.lib.vcs.backends.base import EmptyChangeset
 from rhodecode.lib.helpers import escape
 from rhodecode.lib.utils import make_ui
 from rhodecode.lib.utils2 import safe_unicode
 
 log = logging.getLogger(__name__)
 
 
 def wrap_to_table(str_):
     return '''<table class="code-difftable">
                 <tr class="line no-comment">
                 <td class="lineno new"></td>
                 <td class="code no-comment"><pre>%s</pre></td>
                 </tr>
               </table>''' % str_
 
 
 def wrapped_diff(filenode_old, filenode_new, cut_off_limit=None,
                 ignore_whitespace=True, line_context=3,
                 enable_comments=False):
     """
     returns a wrapped diff into a table, checks for cut_off_limit and presents
     proper message
     """
 
     if filenode_old is None:
         filenode_old = FileNode(filenode_new.path, '', EmptyChangeset())
 
     if filenode_old.is_binary or filenode_new.is_binary:
         diff = wrap_to_table(_('binary file'))
         stats = (0, 0)
         size = 0
 
     elif cut_off_limit != -1 and (cut_off_limit is None or
     (filenode_old.size < cut_off_limit and filenode_new.size < cut_off_limit)):
 
         f_gitdiff = get_gitdiff(filenode_old, filenode_new,
                                 ignore_whitespace=ignore_whitespace,
                                 context=line_context)
         diff_processor = DiffProcessor(f_gitdiff, format='gitdiff')
 
         diff = diff_processor.as_html(enable_comments=enable_comments)
         stats = diff_processor.stat()
         size = len(diff or '')
     else:
         diff = wrap_to_table(_('Changeset was too big and was cut off, use '
                                'diff menu to display this diff'))
         stats = (0, 0)
         size = 0
     if not diff:
         submodules = filter(lambda o: isinstance(o, SubModuleNode),
                             [filenode_new, filenode_old])
         if submodules:
             diff = wrap_to_table(escape('Submodule %r' % submodules[0]))
         else:
             diff = wrap_to_table(_('No changes detected'))
 
     cs1 = filenode_old.changeset.raw_id
     cs2 = filenode_new.changeset.raw_id
 
     return size, cs1, cs2, diff, stats
 
 
 def get_gitdiff(filenode_old, filenode_new, ignore_whitespace=True, context=3):
     """
     Returns git style diff between given ``filenode_old`` and ``filenode_new``.
 
     :param ignore_whitespace: ignore whitespaces in diff
     """
     # make sure we pass in default context
     context = context or 3
     submodules = filter(lambda o: isinstance(o, SubModuleNode),
                         [filenode_new, filenode_old])
     if submodules:
         return ''
 
     for filenode in (filenode_old, filenode_new):
         if not isinstance(filenode, FileNode):
             raise VCSError("Given object should be FileNode object, not %s"
                 % filenode.__class__)
 
     repo = filenode_new.changeset.repository
     old_raw_id = getattr(filenode_old.changeset, 'raw_id', repo.EMPTY_CHANGESET)
     new_raw_id = getattr(filenode_new.changeset, 'raw_id', repo.EMPTY_CHANGESET)
 
     vcs_gitdiff = repo.get_diff(old_raw_id, new_raw_id, filenode_new.path,
                                 ignore_whitespace, context)
     return vcs_gitdiff
 
 NEW_FILENODE = 1
 DEL_FILENODE = 2
 MOD_FILENODE = 3
 RENAMED_FILENODE = 4
 CHMOD_FILENODE = 5
 
 
 class DiffLimitExceeded(Exception):
     pass
 
 
 class LimitedDiffContainer(object):
 
     def __init__(self, diff_limit, cur_diff_size, diff):
         self.diff = diff
         self.diff_limit = diff_limit
         self.cur_diff_size = cur_diff_size
 
     def __iter__(self):
         for l in self.diff:
             yield l
 
 
 class DiffProcessor(object):
     """
     Give it a unified or git diff and it returns a list of the files that were
     mentioned in the diff together with a dict of meta information that
     can be used to render it in a HTML template.
     """
-    _chunk_re = re.compile(r'@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(.*)')
-    _newline_marker = '\\ No newline at end of file\n'
+    _chunk_re = re.compile(r'^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(.*)')
+    _newline_marker = re.compile(r'^\\ No newline at end of file')
     _git_header_re = re.compile(r"""
         #^diff[ ]--git
             [ ]a/(?P<a_path>.+?)[ ]b/(?P<b_path>.+?)\n
         (?:^similarity[ ]index[ ](?P<similarity_index>\d+)%\n
            ^rename[ ]from[ ](?P<rename_from>\S+)\n
            ^rename[ ]to[ ](?P<rename_to>\S+)(?:\n|$))?
         (?:^old[ ]mode[ ](?P<old_mode>\d+)\n
            ^new[ ]mode[ ](?P<new_mode>\d+)(?:\n|$))?
         (?:^new[ ]file[ ]mode[ ](?P<new_file_mode>.+)(?:\n|$))?
         (?:^deleted[ ]file[ ]mode[ ](?P<deleted_file_mode>.+)(?:\n|$))?
         (?:^index[ ](?P<a_blob_id>[0-9A-Fa-f]+)
             \.\.(?P<b_blob_id>[0-9A-Fa-f]+)[ ]?(?P<b_mode>.+)?(?:\n|$))?
         (?:^---[ ](a/(?P<a_file>.+)|/dev/null)(?:\n|$))?
         (?:^\+\+\+[ ](b/(?P<b_file>.+)|/dev/null)(?:\n|$))?
     """, re.VERBOSE | re.MULTILINE)
     _hg_header_re = re.compile(r"""
         #^diff[ ]--git
             [ ]a/(?P<a_path>.+?)[ ]b/(?P<b_path>.+?)\n
         (?:^similarity[ ]index[ ](?P<similarity_index>\d+)%(?:\n|$))?
         (?:^rename[ ]from[ ](?P<rename_from>\S+)\n
            ^rename[ ]to[ ](?P<rename_to>\S+)(?:\n|$))?
         (?:^old[ ]mode[ ](?P<old_mode>\d+)\n
            ^new[ ]mode[ ](?P<new_mode>\d+)(?:\n|$))?
         (?:^new[ ]file[ ]mode[ ](?P<new_file_mode>.+)(?:\n|$))?
         (?:^deleted[ ]file[ ]mode[ ](?P<deleted_file_mode>.+)(?:\n|$))?
         (?:^index[ ](?P<a_blob_id>[0-9A-Fa-f]+)
             \.\.(?P<b_blob_id>[0-9A-Fa-f]+)[ ]?(?P<b_mode>.+)?(?:\n|$))?
         (?:^---[ ](a/(?P<a_file>.+)|/dev/null)(?:\n|$))?
         (?:^\+\+\+[ ](b/(?P<b_file>.+)|/dev/null)(?:\n|$))?
     """, re.VERBOSE | re.MULTILINE)
 
     def __init__(self, diff, vcs='hg', format='gitdiff', diff_limit=None):
         """
         :param diff:   a text in diff format
         :param vcs: type of version controll hg or git
         :param format: format of diff passed, `udiff` or `gitdiff`
         :param diff_limit: define the size of diff that is considered "big"
             based on that parameter cut off will be triggered, set to None
             to show full diff
         """
         if not isinstance(diff, basestring):
             raise Exception('Diff must be a basestring got %s instead' % type(diff))
 
         self._diff = diff
         self._format = format
         self.adds = 0
         self.removes = 0
         # calculate diff size
         self.diff_size = len(diff)
         self.diff_limit = diff_limit
         self.cur_diff_size = 0
         self.parsed = False
         self.parsed_diff = []
         self.vcs = vcs
 
         if format == 'gitdiff':
             self.differ = self._highlight_line_difflib
             self._parser = self._parse_gitdiff
         else:
             self.differ = self._highlight_line_udiff
             self._parser = self._parse_udiff
 
     def _copy_iterator(self):
         """
         make a fresh copy of generator, we should not iterate thru
         an original as it's needed for repeating operations on
         this instance of DiffProcessor
         """
         self.__udiff, iterator_copy = tee(self.__udiff)
         return iterator_copy
 
     def _escaper(self, string):
         """
         Escaper for diff escapes special chars and checks the diff limit
 
         :param string:
         :type string:
         """
 
         self.cur_diff_size += len(string)
 
         # escaper get's iterated on each .next() call and it checks if each
         # parsed line doesn't exceed the diff limit
         if self.diff_limit is not None and self.cur_diff_size > self.diff_limit:
             raise DiffLimitExceeded('Diff Limit Exceeded')
 
         return safe_unicode(string).replace('&', '&amp;')\
                 .replace('<', '&lt;')\
                 .replace('>', '&gt;')
 
     def _line_counter(self, l):
         """
         Checks each line and bumps total adds/removes for this diff
 
         :param l:
         """
         if l.startswith('+') and not l.startswith('+++'):
             self.adds += 1
         elif l.startswith('-') and not l.startswith('---'):
             self.removes += 1
         return safe_unicode(l)
 
     def _highlight_line_difflib(self, line, next_):
         """
         Highlight inline changes in both lines.
         """
 
         if line['action'] == 'del':
             old, new = line, next_
         else:
             old, new = next_, line
 
         oldwords = re.split(r'(\W)', old['line'])
         newwords = re.split(r'(\W)', new['line'])
 
         sequence = difflib.SequenceMatcher(None, oldwords, newwords)
 
         oldfragments, newfragments = [], []
         for tag, i1, i2, j1, j2 in sequence.get_opcodes():
             oldfrag = ''.join(oldwords[i1:i2])
             newfrag = ''.join(newwords[j1:j2])
             if tag != 'equal':
                 if oldfrag:
                     oldfrag = '<del>%s</del>' % oldfrag
                 if newfrag:
                     newfrag = '<ins>%s</ins>' % newfrag
             oldfragments.append(oldfrag)
             newfragments.append(newfrag)
 
         old['line'] = "".join(oldfragments)
         new['line'] = "".join(newfragments)
 
     def _highlight_line_udiff(self, line, next_):
         """
         Highlight inline changes in both lines.
         """
         start = 0
         limit = min(len(line['line']), len(next_['line']))
         while start < limit and line['line'][start] == next_['line'][start]:
             start += 1
         end = -1
         limit -= start
         while -end <= limit and line['line'][end] == next_['line'][end]:
             end -= 1
         end += 1
         if start or end:
             def do(l):
                 last = end + len(l['line'])
                 if l['action'] == 'add':
                     tag = 'ins'
                 else:
                     tag = 'del'
                 l['line'] = '%s<%s>%s</%s>%s' % (
                     l['line'][:start],
                     tag,
                     l['line'][start:last],
                     tag,
                     l['line'][last:]
                 )
             do(line)
             do(next_)
 
     def _get_header(self, diff_chunk):
         """
         parses the diff header, and returns parts, and leftover diff
         parts consists of 14 elements::
 
             a_path, b_path, similarity_index, rename_from, rename_to,
             old_mode, new_mode, new_file_mode, deleted_file_mode,
             a_blob_id, b_blob_id, b_mode, a_file, b_file
 
         :param diff_chunk:
         :type diff_chunk:
         """
 
         if self.vcs == 'git':
             match = self._git_header_re.match(diff_chunk)
             diff = diff_chunk[match.end():]
             return match.groupdict(), imap(self._escaper, diff.splitlines(1))
         elif self.vcs == 'hg':
             match = self._hg_header_re.match(diff_chunk)
             diff = diff_chunk[match.end():]
             return match.groupdict(), imap(self._escaper, diff.splitlines(1))
         else:
             raise Exception('VCS type %s is not supported' % self.vcs)
 
+    def _clean_line(self, line, command):
+        if command in ['+', '-', ' ']:
+            #only modify the line if it's actually a diff thing
+            line = line[1:]
+        return line
+
     def _parse_gitdiff(self, inline_diff=True):
         _files = []
         diff_container = lambda arg: arg
 
         ##split the diff in chunks of separate --git a/file b/file chunks
         for raw_diff in ('\n' + self._diff).split('\ndiff --git')[1:]:
             binary = False
             binary_msg = 'unknown binary'
             head, diff = self._get_header(raw_diff)
 
             if not head['a_file'] and head['b_file']:
                 op = 'A'
             elif head['a_file'] and head['b_file']:
                 op = 'M'
             elif head['a_file'] and not head['b_file']:
                 op = 'D'
             else:
                 #probably we're dealing with a binary file 1
                 binary = True
                 if head['deleted_file_mode']:
                     op = 'D'
                     stats = ['b', DEL_FILENODE]
                     binary_msg = 'deleted binary file'
                 elif head['new_file_mode']:
                     op = 'A'
                     stats = ['b', NEW_FILENODE]
                     binary_msg = 'new binary file %s' % head['new_file_mode']
                 else:
                     if head['new_mode'] and head['old_mode']:
                         stats = ['b', CHMOD_FILENODE]
                         op = 'M'
                         binary_msg = ('modified binary file chmod %s => %s'
                                       % (head['old_mode'], head['new_mode']))
                     elif (head['rename_from'] and head['rename_to']
                           and head['rename_from'] != head['rename_to']):
                         stats = ['b', RENAMED_FILENODE]
                         op = 'M'
                         binary_msg = ('file renamed from %s to %s'
                                       % (head['rename_from'], head['rename_to']))
                     else:
                         stats = ['b', MOD_FILENODE]
                         op = 'M'
                         binary_msg = 'modified binary file'
 
             if not binary:
                 try:
                     chunks, stats = self._parse_lines(diff)
                 except DiffLimitExceeded:
                     diff_container = lambda _diff: LimitedDiffContainer(
                                                 self.diff_limit,
                                                 self.cur_diff_size,
                                                 _diff)
                     break
             else:
                 chunks = []
                 chunks.append([{
                     'old_lineno': '',
                     'new_lineno': '',
                     'action':     'binary',
                     'line':       binary_msg,
                 }])
 
             _files.append({
                 'filename':         head['b_path'],
                 'old_revision':     head['a_blob_id'],
                 'new_revision':     head['b_blob_id'],
                 'chunks':           chunks,
                 'operation':        op,
                 'stats':            stats,
             })
 
         sorter = lambda info: {'A': 0, 'M': 1, 'D': 2}.get(info['operation'])
 
         if inline_diff is False:
             return diff_container(sorted(_files, key=sorter))
 
         # highlight inline changes
         for diff_data in _files:
             for chunk in diff_data['chunks']:
                 lineiter = iter(chunk)
                 try:
                     while 1:
                         line = lineiter.next()
                         if line['action'] not in ['unmod', 'context']:
                             nextline = lineiter.next()
                             if nextline['action'] in ['unmod', 'context'] or \
                                nextline['action'] == line['action']:
                                 continue
                             self.differ(line, nextline)
                 except StopIteration:
                     pass
 
         return diff_container(sorted(_files, key=sorter))
 
     def _parse_udiff(self, inline_diff=True):
         raise NotImplementedError()
 
     def _parse_lines(self, diff):
         """
         Parse the diff an return data for the template.
         """
 
         lineiter = iter(diff)
         stats = [0, 0]
 
         try:
             chunks = []
             line = lineiter.next()
 
             while line:
                 lines = []
                 chunks.append(lines)
 
                 match = self._chunk_re.match(line)
 
                 if not match:
                     break
 
                 gr = match.groups()
                 (old_line, old_end,
                  new_line, new_end) = [int(x or 1) for x in gr[:-1]]
                 old_line -= 1
                 new_line -= 1
 
                 context = len(gr) == 5
                 old_end += old_line
                 new_end += new_line
 
                 if context:
                     # skip context only if it's first line
                     if int(gr[0]) > 1:
                         lines.append({
                             'old_lineno': '...',
                             'new_lineno': '...',
                             'action':     'context',
                             'line':       line,
                         })
 
                 line = lineiter.next()
 
                 while old_line < old_end or new_line < new_end:
+                    command = ' '
                     if line:
                         command = line[0]
-                        if command in ['+', '-', ' ']:
-                            #only modify the line if it's actually a diff
-                            # thing
-                            line = line[1:]
-                    else:
-                        command = ' '
 
                     affects_old = affects_new = False
 
                     # ignore those if we don't expect them
                     if command in '#@':
                         continue
                     elif command == '+':
                         affects_new = True
                         action = 'add'
                         stats[0] += 1
                     elif command == '-':
                         affects_old = True
                         action = 'del'
                         stats[1] += 1
                     else:
                         affects_old = affects_new = True
                         action = 'unmod'
 
-                    if line != self._newline_marker:
+                    if not self._newline_marker.match(line):
                         old_line += affects_old
                         new_line += affects_new
                         lines.append({
                             'old_lineno':   affects_old and old_line or '',
                             'new_lineno':   affects_new and new_line or '',
                             'action':       action,
-                            'line':         line
+                            'line':         self._clean_line(line, command)
                         })
 
                     line = lineiter.next()
 
-                    if line == self._newline_marker:
+                    if self._newline_marker.match(line):
                         # we need to append to lines, since this is not
                         # counted in the line specs of diff
                         lines.append({
                             'old_lineno':   '...',
                             'new_lineno':   '...',
                             'action':       'context',
-                            'line':         line
+                            'line':         self._clean_line(line, command)
                         })
 
         except StopIteration:
             pass
         return chunks, stats
 
     def _safe_id(self, idstring):
         """Make a string safe for including in an id attribute.
 
         The HTML spec says that id attributes 'must begin with
         a letter ([A-Za-z]) and may be followed by any number
         of letters, digits ([0-9]), hyphens ("-"), underscores
         ("_"), colons (":"), and periods (".")'. These regexps
         are slightly over-zealous, in that they remove colons
         and periods unnecessarily.
 
         Whitespace is transformed into underscores, and then
         anything which is not a hyphen or a character that
         matches \w (alphanumerics and underscore) is removed.
 
         """
         # Transform all whitespace to underscore
         idstring = re.sub(r'\s', "_", '%s' % idstring)
         # Remove everything that is not a hyphen or a member of \w
         idstring = re.sub(r'(?!-)\W', "", idstring).lower()
         return idstring
 
     def prepare(self, inline_diff=True):
         """
         Prepare the passed udiff for HTML rendering. It'l return a list
         of dicts with diff information
         """
         parsed = self._parser(inline_diff=inline_diff)
         self.parsed = True
         self.parsed_diff = parsed
         return parsed
 
     def as_raw(self, diff_lines=None):
         """
         Returns raw string diff
         """
         return self._diff
         #return u''.join(imap(self._line_counter, self._diff.splitlines(1)))
 
     def as_html(self, table_class='code-difftable', line_class='line',
                 new_lineno_class='lineno old', old_lineno_class='lineno new',
                 code_class='code', enable_comments=False, parsed_lines=None):
         """
         Return given diff as html table with customized css classes
         """
         def _link_to_if(condition, label, url):
             """
             Generates a link if condition is meet or just the label if not.
             """
 
             if condition:
                 return '''<a href="%(url)s">%(label)s</a>''' % {
                     'url': url,
                     'label': label
                 }
             else:
                 return label
         if not self.parsed:
             self.prepare()
 
         diff_lines = self.parsed_diff
         if parsed_lines:
             diff_lines = parsed_lines
 
         _html_empty = True
         _html = []
         _html.append('''<table class="%(table_class)s">\n''' % {
             'table_class': table_class
         })
 
         for diff in diff_lines:
             for line in diff['chunks']:
                 _html_empty = False
                 for change in line:
                     _html.append('''<tr class="%(lc)s %(action)s">\n''' % {
                         'lc': line_class,
                         'action': change['action']
                     })
                     anchor_old_id = ''
                     anchor_new_id = ''
                     anchor_old = "%(filename)s_o%(oldline_no)s" % {
                         'filename': self._safe_id(diff['filename']),
                         'oldline_no': change['old_lineno']
                     }
                     anchor_new = "%(filename)s_n%(oldline_no)s" % {
                         'filename': self._safe_id(diff['filename']),
                         'oldline_no': change['new_lineno']
                     }
                     cond_old = (change['old_lineno'] != '...' and
                                 change['old_lineno'])
                     cond_new = (change['new_lineno'] != '...' and
                                 change['new_lineno'])
                     if cond_old:
                         anchor_old_id = 'id="%s"' % anchor_old
                     if cond_new:
                         anchor_new_id = 'id="%s"' % anchor_new
                     ###########################################################
                     # OLD LINE NUMBER
                     ###########################################################
                     _html.append('''\t<td %(a_id)s class="%(olc)s">''' % {
                         'a_id': anchor_old_id,
                         'olc': old_lineno_class
                     })
 
                     _html.append('''%(link)s''' % {
                         'link': _link_to_if(True, change['old_lineno'],
                                             '#%s' % anchor_old)
                     })
                     _html.append('''</td>\n''')
                     ###########################################################
                     # NEW LINE NUMBER
                     ###########################################################
 
                     _html.append('''\t<td %(a_id)s class="%(nlc)s">''' % {
                         'a_id': anchor_new_id,
                         'nlc': new_lineno_class
                     })
 
                     _html.append('''%(link)s''' % {
                         'link': _link_to_if(True, change['new_lineno'],
                                             '#%s' % anchor_new)
                     })
                     _html.append('''</td>\n''')
                     ###########################################################
                     # CODE
                     ###########################################################
                     comments = '' if enable_comments else 'no-comment'
                     _html.append('''\t<td class="%(cc)s %(inc)s">''' % {
                         'cc': code_class,
                         'inc': comments
                     })
                     _html.append('''\n\t\t<pre>%(code)s</pre>\n''' % {
                         'code': change['line']
                     })
 
                     _html.append('''\t</td>''')
                     _html.append('''\n</tr>\n''')
         _html.append('''</table>''')
         if _html_empty:
             return None
         return ''.join(_html)
 
     def stat(self):
         """
         Returns tuple of added, and removed lines for this instance
         """
         return self.adds, self.removes
 
 
 class InMemoryBundleRepo(bundlerepository):
     def __init__(self, ui, path, bundlestream):
         self._tempparent = None
         localrepo.localrepository.__init__(self, ui, path)
         self.ui.setconfig('phases', 'publish', False)
 
         self.bundle = bundlestream
 
         # dict with the mapping 'filename' -> position in the bundle
         self.bundlefilespos = {}
 
 
 def differ(org_repo, org_ref, other_repo, other_ref, discovery_data=None,
            remote_compare=False, context=3, ignore_whitespace=False):
     """
     General differ between branches, bookmarks, revisions of two remote or
     local but related repositories
 
     :param org_repo:
     :param org_ref:
     :param other_repo:
     :type other_repo:
     :type other_ref:
     """
 
     org_repo_scm = org_repo.scm_instance
     other_repo_scm = other_repo.scm_instance
 
     org_repo = org_repo_scm._repo
     other_repo = other_repo_scm._repo
 
     org_ref = org_ref[1]
     other_ref = other_ref[1]
 
     if org_repo == other_repo:
         log.debug('running diff between %s@%s and %s@%s'
                   % (org_repo, org_ref, other_repo, other_ref))
         _diff = org_repo_scm.get_diff(rev1=org_ref, rev2=other_ref,
             ignore_whitespace=ignore_whitespace, context=context)
         return _diff
 
     elif remote_compare:
         opts = diffopts(git=True, ignorews=ignore_whitespace, context=context)
         common, incoming, rheads = discovery_data
         org_repo_peer = localrepo.locallegacypeer(org_repo.local())
         # create a bundle (uncompressed if other repo is not local)
         if org_repo_peer.capable('getbundle'):
             # disable repo hooks here since it's just bundle !
             # patch and reset hooks section of UI config to not run any
             # hooks on fetching archives with subrepos
             for k, _ in org_repo.ui.configitems('hooks'):
                 org_repo.ui.setconfig('hooks', k, None)
 
             unbundle = org_repo.getbundle('incoming', common=common,
                                           heads=None)
 
             buf = BytesIO()
             while True:
                 chunk = unbundle._stream.read(1024 * 4)
                 if not chunk:
                     break
                 buf.write(chunk)
 
             buf.seek(0)
             # replace chunked _stream with data that can do tell() and seek()
             unbundle._stream = buf
 
             ui = make_ui('db')
             bundlerepo = InMemoryBundleRepo(ui, path=org_repo.root,
                                             bundlestream=unbundle)
 
             return ''.join(patch.diff(bundlerepo,
                                       node1=other_repo[other_ref].node(),
                                       node2=org_repo[org_ref].node(),
                                       opts=opts))
 
     return ''
  No newline at end of file

rhodecode/tests/models/test_diff_parsers.py

@@ -1,78 +1,85 b''
 import os
 import unittest
 from rhodecode.tests import *
 from rhodecode.lib.diffs import DiffProcessor, NEW_FILENODE, DEL_FILENODE, \
     MOD_FILENODE, RENAMED_FILENODE, CHMOD_FILENODE
 
 dn = os.path.dirname
 FIXTURES = os.path.join(dn(dn(os.path.abspath(__file__))), 'fixtures')
 
 DIFF_FIXTURES = {
     'hg_diff_add_single_binary_file.diff': [
         (u'US Warszawa.jpg', 'A', ['b', NEW_FILENODE]),
     ],
     'hg_diff_mod_single_binary_file.diff': [
         (u'US Warszawa.jpg', 'M', ['b', MOD_FILENODE]),
     ],
     'hg_diff_del_single_binary_file.diff': [
         (u'US Warszawa.jpg', 'D', ['b', DEL_FILENODE]),
     ],
     'hg_diff_binary_and_normal.diff': [
         (u'img/baseline-10px.png', 'A', ['b', NEW_FILENODE]),
         (u'js/jquery/hashgrid.js', 'A', [340, 0]),
         (u'index.html',            'M', [3, 2]),
         (u'less/docs.less',        'M', [34, 0]),
         (u'less/scaffolding.less', 'M', [1, 3]),
         (u'readme.markdown',       'M', [1, 10]),
         (u'img/baseline-20px.png', 'D', ['b', DEL_FILENODE]),
         (u'js/global.js',          'D', [0, 75])
     ],
     'hg_diff_chmod.diff': [
         (u'file', 'M', ['b', CHMOD_FILENODE]),
     ],
     'hg_diff_rename_file.diff': [
         (u'file_renamed', 'M', ['b', RENAMED_FILENODE]),
     ],
     'git_diff_chmod.diff': [
         (u'work-horus.xls', 'M', ['b', CHMOD_FILENODE]),
     ],
     'git_diff_rename_file.diff': [
         (u'file.xls', 'M', ['b', RENAMED_FILENODE]),
     ],
     'git_diff_mod_single_binary_file.diff': [
         ('US Warszawa.jpg', 'M', ['b', MOD_FILENODE])
 
     ],
     'git_diff_binary_and_normal.diff': [
         (u'img/baseline-10px.png', 'A', ['b', NEW_FILENODE]),
         (u'js/jquery/hashgrid.js', 'A', [340, 0]),
         (u'index.html',            'M', [3, 2]),
         (u'less/docs.less',        'M', [34, 0]),
         (u'less/scaffolding.less', 'M', [1, 3]),
         (u'readme.markdown',       'M', [1, 10]),
         (u'img/baseline-20px.png', 'D', ['b', DEL_FILENODE]),
         (u'js/global.js',          'D', [0, 75])
     ],
+    'diff_with_diff_data.diff': [
+        (u'vcs/backends/base.py', 'M', [18, 2]),
+        (u'vcs/backends/git/repository.py', 'M', [46, 15]),
+        (u'vcs/backends/hg.py', 'M', [22, 3]),
+        (u'vcs/tests/test_git.py', 'M', [5, 5]),
+        (u'vcs/tests/test_repository.py', 'M', [174, 2])
+    ],
 #    'large_diff.diff': [
 #
 #    ],
 
 
 }
 
 
 def _diff_checker(fixture):
     with open(os.path.join(FIXTURES, fixture)) as f:
         diff = f.read()
 
     diff_proc = DiffProcessor(diff)
     diff_proc_d = diff_proc.prepare()
     data = [(x['filename'], x['operation'], x['stats']) for x in diff_proc_d]
     expected_data = DIFF_FIXTURES[fixture]
 
     assert expected_data == data
 
 
 def test_parse_diff():
     for fixture in DIFF_FIXTURES:
         yield _diff_checker, fixture