diffs: fixed problem with rendering no newline at the end of file markers.

- fixes  #5402

rhodecode/tests/fixtures/hg_diff_no_newline.diff

@@ -0,0 +1,10
+diff --git a/server.properties b/server.properties
+--- a/server.properties
++++ b/server.properties
+@@ -1,2 +1,3 @@
+property=value
+-anotherProperty=value
+\ No newline at end of file
++anotherProperty=value
++newProperty=super_important_value
+\ No newline at end of file
  No newline at end of file

rhodecode/lib/codeblocks.py

@@ -1,711 +1,735
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import difflib
 from itertools import groupby
 
 from pygments import lex
 from pygments.formatters.html import _get_ttype_class as pygment_token_class
 from rhodecode.lib.helpers import (
     get_lexer_for_filenode, html_escape, get_custom_lexer)
 from rhodecode.lib.utils2 import AttributeDict
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.lib.diff_match_patch import diff_match_patch
 from rhodecode.lib.diffs import LimitedDiffContainer
 from pygments.lexers import get_lexer_by_name
 
 plain_text_lexer = get_lexer_by_name(
     'text', stripall=False, stripnl=False, ensurenl=False)
 
 
 log = logging.getLogger()
 
 
 def filenode_as_lines_tokens(filenode, lexer=None):
     org_lexer = lexer
     lexer = lexer or get_lexer_for_filenode(filenode)
     log.debug('Generating file node pygment tokens for %s, %s, org_lexer:%s',
               lexer, filenode, org_lexer)
     tokens = tokenize_string(filenode.content, lexer)
     lines = split_token_stream(tokens, split_string='\n')
     rv = list(lines)
     return rv
 
 
 def tokenize_string(content, lexer):
     """
     Use pygments to tokenize some content based on a lexer
     ensuring all original new lines and whitespace is preserved
     """
 
     lexer.stripall = False
     lexer.stripnl = False
     lexer.ensurenl = False
     for token_type, token_text in lex(content, lexer):
         yield pygment_token_class(token_type), token_text
 
 
 def split_token_stream(tokens, split_string=u'\n'):
     """
     Take a list of (TokenType, text) tuples and split them by a string
 
     >>> split_token_stream([(TEXT, 'some\ntext'), (TEXT, 'more\n')])
     [(TEXT, 'some'), (TEXT, 'text'),
      (TEXT, 'more'), (TEXT, 'text')]
     """
 
     buffer = []
     for token_class, token_text in tokens:
         parts = token_text.split(split_string)
         for part in parts[:-1]:
             buffer.append((token_class, part))
             yield buffer
             buffer = []
 
         buffer.append((token_class, parts[-1]))
 
     if buffer:
         yield buffer
 
 
 def filenode_as_annotated_lines_tokens(filenode):
     """
     Take a file node and return a list of annotations => lines, if no annotation
     is found, it will be None.
 
     eg:
 
     [
         (annotation1, [
             (1, line1_tokens_list),
             (2, line2_tokens_list),
         ]),
         (annotation2, [
             (3, line1_tokens_list),
         ]),
         (None, [
             (4, line1_tokens_list),
         ]),
         (annotation1, [
             (5, line1_tokens_list),
             (6, line2_tokens_list),
         ])
     ]
     """
 
     commit_cache = {}  # cache commit_getter lookups
 
     def _get_annotation(commit_id, commit_getter):
         if commit_id not in commit_cache:
             commit_cache[commit_id] = commit_getter()
         return commit_cache[commit_id]
 
     annotation_lookup = {
         line_no: _get_annotation(commit_id, commit_getter)
         for line_no, commit_id, commit_getter, line_content
         in filenode.annotate
     }
 
     annotations_lines = ((annotation_lookup.get(line_no), line_no, tokens)
                           for line_no, tokens
                           in enumerate(filenode_as_lines_tokens(filenode), 1))
 
     grouped_annotations_lines = groupby(annotations_lines, lambda x: x[0])
 
     for annotation, group in grouped_annotations_lines:
         yield (
             annotation, [(line_no, tokens)
                           for (_, line_no, tokens) in group]
         )
 
 
 def render_tokenstream(tokenstream):
     result = []
     for token_class, token_ops_texts in rollup_tokenstream(tokenstream):
 
         if token_class:
             result.append(u'<span class="%s">' % token_class)
         else:
             result.append(u'<span>')
 
         for op_tag, token_text in token_ops_texts:
 
             if op_tag:
                 result.append(u'<%s>' % op_tag)
 
             escaped_text = html_escape(token_text)
 
             # TODO: dan: investigate showing hidden characters like space/nl/tab
             # escaped_text = escaped_text.replace(' ', '<sp> </sp>')
             # escaped_text = escaped_text.replace('\n', '<nl>\n</nl>')
             # escaped_text = escaped_text.replace('\t', '<tab>\t</tab>')
 
             result.append(escaped_text)
 
             if op_tag:
                 result.append(u'</%s>' % op_tag)
 
         result.append(u'</span>')
 
     html = ''.join(result)
     return html
 
 
 def rollup_tokenstream(tokenstream):
     """
     Group a token stream of the format:
 
         ('class', 'op', 'text')
     or
         ('class', 'text')
 
     into
 
         [('class1',
             [('op1', 'text'),
              ('op2', 'text')]),
          ('class2',
             [('op3', 'text')])]
 
     This is used to get the minimal tags necessary when
     rendering to html eg for a token stream ie.
 
     <span class="A"><ins>he</ins>llo</span>
     vs
     <span class="A"><ins>he</ins></span><span class="A">llo</span>
 
     If a 2 tuple is passed in, the output op will be an empty string.
 
     eg:
 
     >>> rollup_tokenstream([('classA', '',      'h'),
                             ('classA', 'del',   'ell'),
                             ('classA', '',      'o'),
                             ('classB', '',      ' '),
                             ('classA', '',      'the'),
                             ('classA', '',      're'),
                             ])
 
     [('classA', [('', 'h'), ('del', 'ell'), ('', 'o')],
      ('classB', [('', ' ')],
      ('classA', [('', 'there')]]
 
     """
     if tokenstream and len(tokenstream[0]) == 2:
         tokenstream = ((t[0], '', t[1]) for t in tokenstream)
 
     result = []
     for token_class, op_list in groupby(tokenstream, lambda t: t[0]):
         ops = []
         for token_op, token_text_list in groupby(op_list, lambda o: o[1]):
             text_buffer = []
             for t_class, t_op, t_text in token_text_list:
                 text_buffer.append(t_text)
             ops.append((token_op, ''.join(text_buffer)))
         result.append((token_class, ops))
     return result
 
 
 def tokens_diff(old_tokens, new_tokens, use_diff_match_patch=True):
     """
     Converts a list of (token_class, token_text) tuples to a list of
     (token_class, token_op, token_text) tuples where token_op is one of
     ('ins', 'del', '')
 
     :param old_tokens: list of (token_class, token_text) tuples of old line
     :param new_tokens: list of (token_class, token_text) tuples of new line
     :param use_diff_match_patch: boolean, will use google's diff match patch
         library which has options to 'smooth' out the character by character
         differences making nicer ins/del blocks
     """
 
     old_tokens_result = []
     new_tokens_result = []
 
     similarity = difflib.SequenceMatcher(None,
         ''.join(token_text for token_class, token_text in old_tokens),
         ''.join(token_text for token_class, token_text in new_tokens)
     ).ratio()
 
     if similarity < 0.6: # return, the blocks are too different
         for token_class, token_text in old_tokens:
             old_tokens_result.append((token_class, '', token_text))
         for token_class, token_text in new_tokens:
             new_tokens_result.append((token_class, '', token_text))
         return old_tokens_result, new_tokens_result, similarity
 
     token_sequence_matcher = difflib.SequenceMatcher(None,
         [x[1] for x in old_tokens],
         [x[1] for x in new_tokens])
 
     for tag, o1, o2, n1, n2 in token_sequence_matcher.get_opcodes():
         # check the differences by token block types first to give a more
         # nicer "block" level replacement vs character diffs
 
         if tag == 'equal':
             for token_class, token_text in old_tokens[o1:o2]:
                 old_tokens_result.append((token_class, '', token_text))
             for token_class, token_text in new_tokens[n1:n2]:
                 new_tokens_result.append((token_class, '', token_text))
         elif tag == 'delete':
             for token_class, token_text in old_tokens[o1:o2]:
                 old_tokens_result.append((token_class, 'del', token_text))
         elif tag == 'insert':
             for token_class, token_text in new_tokens[n1:n2]:
                 new_tokens_result.append((token_class, 'ins', token_text))
         elif tag == 'replace':
             # if same type token blocks must be replaced, do a diff on the
             # characters in the token blocks to show individual changes
 
             old_char_tokens = []
             new_char_tokens = []
             for token_class, token_text in old_tokens[o1:o2]:
                 for char in token_text:
                     old_char_tokens.append((token_class, char))
 
             for token_class, token_text in new_tokens[n1:n2]:
                 for char in token_text:
                     new_char_tokens.append((token_class, char))
 
             old_string = ''.join([token_text for
                 token_class, token_text in old_char_tokens])
             new_string = ''.join([token_text for
                 token_class, token_text in new_char_tokens])
 
             char_sequence = difflib.SequenceMatcher(
                 None, old_string, new_string)
             copcodes = char_sequence.get_opcodes()
             obuffer, nbuffer = [], []
 
             if use_diff_match_patch:
                 dmp = diff_match_patch()
                 dmp.Diff_EditCost = 11 # TODO: dan: extract this to a setting
                 reps = dmp.diff_main(old_string, new_string)
                 dmp.diff_cleanupEfficiency(reps)
 
                 a, b = 0, 0
                 for op, rep in reps:
                     l = len(rep)
                     if op == 0:
                         for i, c in enumerate(rep):
                             obuffer.append((old_char_tokens[a+i][0], '', c))
                             nbuffer.append((new_char_tokens[b+i][0], '', c))
                         a += l
                         b += l
                     elif op == -1:
                         for i, c in enumerate(rep):
                             obuffer.append((old_char_tokens[a+i][0], 'del', c))
                         a += l
                     elif op == 1:
                         for i, c in enumerate(rep):
                             nbuffer.append((new_char_tokens[b+i][0], 'ins', c))
                         b += l
             else:
                 for ctag, co1, co2, cn1, cn2 in copcodes:
                     if ctag == 'equal':
                         for token_class, token_text in old_char_tokens[co1:co2]:
                             obuffer.append((token_class, '', token_text))
                         for token_class, token_text in new_char_tokens[cn1:cn2]:
                             nbuffer.append((token_class, '', token_text))
                     elif ctag == 'delete':
                         for token_class, token_text in old_char_tokens[co1:co2]:
                             obuffer.append((token_class, 'del', token_text))
                     elif ctag == 'insert':
                         for token_class, token_text in new_char_tokens[cn1:cn2]:
                             nbuffer.append((token_class, 'ins', token_text))
                     elif ctag == 'replace':
                         for token_class, token_text in old_char_tokens[co1:co2]:
                             obuffer.append((token_class, 'del', token_text))
                         for token_class, token_text in new_char_tokens[cn1:cn2]:
                             nbuffer.append((token_class, 'ins', token_text))
 
             old_tokens_result.extend(obuffer)
             new_tokens_result.extend(nbuffer)
 
     return old_tokens_result, new_tokens_result, similarity
 
 
 class DiffSet(object):
     """
     An object for parsing the diff result from diffs.DiffProcessor and
     adding highlighting, side by side/unified renderings and line diffs
     """
 
     HL_REAL = 'REAL' # highlights using original file, slow
     HL_FAST = 'FAST' # highlights using just the line, fast but not correct
                      # in the case of multiline code
     HL_NONE = 'NONE' # no highlighting, fastest
 
     def __init__(self, highlight_mode=HL_REAL, repo_name=None,
                  source_repo_name=None,
                  source_node_getter=lambda filename: None,
                  target_node_getter=lambda filename: None,
                  source_nodes=None, target_nodes=None,
                  max_file_size_limit=150 * 1024, # files over this size will
                                                  # use fast highlighting
                  comments=None,
                  ):
 
         self.highlight_mode = highlight_mode
         self.highlighted_filenodes = {}
         self.source_node_getter = source_node_getter
         self.target_node_getter = target_node_getter
         self.source_nodes = source_nodes or {}
         self.target_nodes = target_nodes or {}
         self.repo_name = repo_name
         self.source_repo_name = source_repo_name or repo_name
         self.comments = comments or {}
         self.comments_store = self.comments.copy()
         self.max_file_size_limit = max_file_size_limit
 
     def render_patchset(self, patchset, source_ref=None, target_ref=None):
         diffset = AttributeDict(dict(
             lines_added=0,
             lines_deleted=0,
             changed_files=0,
             files=[],
             file_stats={},
             limited_diff=isinstance(patchset, LimitedDiffContainer),
             repo_name=self.repo_name,
             source_repo_name=self.source_repo_name,
             source_ref=source_ref,
             target_ref=target_ref,
         ))
         for patch in patchset:
             diffset.file_stats[patch['filename']] = patch['stats']
             filediff = self.render_patch(patch)
             filediff.diffset = diffset
             diffset.files.append(filediff)
             diffset.changed_files += 1
             if not patch['stats']['binary']:
                 diffset.lines_added += patch['stats']['added']
                 diffset.lines_deleted += patch['stats']['deleted']
 
         return diffset
 
     _lexer_cache = {}
 
     def _get_lexer_for_filename(self, filename, filenode=None):
         # cached because we might need to call it twice for source/target
         if filename not in self._lexer_cache:
             if filenode:
                 lexer = filenode.lexer
                 extension = filenode.extension
             else:
                 lexer = FileNode.get_lexer(filename=filename)
                 extension = filename.split('.')[-1]
 
             lexer = get_custom_lexer(extension) or lexer
             self._lexer_cache[filename] = lexer
         return self._lexer_cache[filename]
 
     def render_patch(self, patch):
         log.debug('rendering diff for %r' % patch['filename'])
 
         source_filename = patch['original_filename']
         target_filename = patch['filename']
 
         source_lexer = plain_text_lexer
         target_lexer = plain_text_lexer
 
         if not patch['stats']['binary']:
             if self.highlight_mode == self.HL_REAL:
                 if (source_filename and patch['operation'] in ('D', 'M')
                     and source_filename not in self.source_nodes):
                         self.source_nodes[source_filename] = (
                             self.source_node_getter(source_filename))
 
                 if (target_filename and patch['operation'] in ('A', 'M')
                     and target_filename not in self.target_nodes):
                         self.target_nodes[target_filename] = (
                             self.target_node_getter(target_filename))
 
             elif self.highlight_mode == self.HL_FAST:
                 source_lexer = self._get_lexer_for_filename(source_filename)
                 target_lexer = self._get_lexer_for_filename(target_filename)
 
         source_file = self.source_nodes.get(source_filename, source_filename)
         target_file = self.target_nodes.get(target_filename, target_filename)
 
         source_filenode, target_filenode = None, None
 
         # TODO: dan: FileNode.lexer works on the content of the file - which
         # can be slow - issue #4289 explains a lexer clean up - which once
         # done can allow caching a lexer for a filenode to avoid the file lookup
         if isinstance(source_file, FileNode):
             source_filenode = source_file
             #source_lexer = source_file.lexer
             source_lexer = self._get_lexer_for_filename(source_filename)
             source_file.lexer = source_lexer
 
         if isinstance(target_file, FileNode):
             target_filenode = target_file
             #target_lexer = target_file.lexer
             target_lexer = self._get_lexer_for_filename(target_filename)
             target_file.lexer = target_lexer
 
         source_file_path, target_file_path = None, None
 
         if source_filename != '/dev/null':
             source_file_path = source_filename
         if target_filename != '/dev/null':
             target_file_path = target_filename
 
         source_file_type = source_lexer.name
         target_file_type = target_lexer.name
 
         filediff = AttributeDict({
             'source_file_path': source_file_path,
             'target_file_path': target_file_path,
             'source_filenode': source_filenode,
             'target_filenode': target_filenode,
             'source_file_type': target_file_type,
             'target_file_type': source_file_type,
             'patch': {'filename': patch['filename'], 'stats': patch['stats']},
             'operation': patch['operation'],
             'source_mode': patch['stats']['old_mode'],
             'target_mode': patch['stats']['new_mode'],
             'limited_diff': isinstance(patch, LimitedDiffContainer),
             'hunks': [],
             'diffset': self,
         })
 
         for hunk in patch['chunks'][1:]:
             hunkbit = self.parse_hunk(hunk, source_file, target_file)
             hunkbit.source_file_path = source_file_path
             hunkbit.target_file_path = target_file_path
             filediff.hunks.append(hunkbit)
 
         left_comments = {}
         if source_file_path in self.comments_store:
             for lineno, comments in self.comments_store[source_file_path].items():
                 left_comments[lineno] = comments
 
         if target_file_path in self.comments_store:
             for lineno, comments in self.comments_store[target_file_path].items():
                 left_comments[lineno] = comments
         # left comments are one that we couldn't place in diff lines.
         # could be outdated, or the diff changed and this line is no
         # longer available
         filediff.left_comments = left_comments
 
         return filediff
 
     def parse_hunk(self, hunk, source_file, target_file):
         result = AttributeDict(dict(
             source_start=hunk['source_start'],
             source_length=hunk['source_length'],
             target_start=hunk['target_start'],
             target_length=hunk['target_length'],
             section_header=hunk['section_header'],
             lines=[],
         ))
         before, after = [], []
 
         for line in hunk['lines']:
 
             if line['action'] == 'unmod':
                 result.lines.extend(
                     self.parse_lines(before, after, source_file, target_file))
                 after.append(line)
                 before.append(line)
             elif line['action'] == 'add':
                 after.append(line)
             elif line['action'] == 'del':
                 before.append(line)
             elif line['action'] == 'old-no-nl':
                 before.append(line)
             elif line['action'] == 'new-no-nl':
                 after.append(line)
 
         result.lines.extend(
             self.parse_lines(before, after, source_file, target_file))
         result.unified = self.as_unified(result.lines)
         result.sideside = result.lines
 
         return result
 
     def parse_lines(self, before_lines, after_lines, source_file, target_file):
         # TODO: dan: investigate doing the diff comparison and fast highlighting
         # on the entire before and after buffered block lines rather than by
         # line, this means we can get better 'fast' highlighting if the context
         # allows it - eg.
         # line 4: """
         # line 5: this gets highlighted as a string
         # line 6: """
 
         lines = []
+
+        before_newline = AttributeDict()
+        after_newline = AttributeDict()
+        if before_lines and before_lines[-1]['action'] == 'old-no-nl':
+            before_newline_line = before_lines.pop(-1)
+            before_newline.content = '\n {}'.format(
+                render_tokenstream(
+                    [(x[0], '', x[1])
+                     for x in [('nonl', before_newline_line['line'])]]))
+
+        if after_lines and after_lines[-1]['action'] == 'new-no-nl':
+            after_newline_line = after_lines.pop(-1)
+            after_newline.content = '\n {}'.format(
+                render_tokenstream(
+                    [(x[0], '', x[1])
+                     for x in [('nonl', after_newline_line['line'])]]))
+
         while before_lines or after_lines:
             before, after = None, None
             before_tokens, after_tokens = None, None
 
             if before_lines:
                 before = before_lines.pop(0)
             if after_lines:
                 after = after_lines.pop(0)
 
             original = AttributeDict()
             modified = AttributeDict()
 
             if before:
                 if before['action'] == 'old-no-nl':
                     before_tokens = [('nonl', before['line'])]
                 else:
                     before_tokens = self.get_line_tokens(
                         line_text=before['line'],
                         line_number=before['old_lineno'],
                         file=source_file)
                 original.lineno = before['old_lineno']
                 original.content = before['line']
                 original.action = self.action_to_op(before['action'])
                 original.comments = self.get_comments_for('old',
                     source_file, before['old_lineno'])
 
             if after:
                 if after['action'] == 'new-no-nl':
                     after_tokens = [('nonl', after['line'])]
                 else:
                     after_tokens = self.get_line_tokens(
                         line_text=after['line'], line_number=after['new_lineno'],
                         file=target_file)
                 modified.lineno = after['new_lineno']
                 modified.content = after['line']
                 modified.action = self.action_to_op(after['action'])
                 modified.comments = self.get_comments_for('new',
                     target_file, after['new_lineno'])
 
             # diff the lines
             if before_tokens and after_tokens:
                 o_tokens, m_tokens, similarity = tokens_diff(
                     before_tokens, after_tokens)
                 original.content = render_tokenstream(o_tokens)
                 modified.content = render_tokenstream(m_tokens)
             elif before_tokens:
                 original.content = render_tokenstream(
                     [(x[0], '', x[1]) for x in before_tokens])
             elif after_tokens:
                 modified.content = render_tokenstream(
                     [(x[0], '', x[1]) for x in after_tokens])
 
+            if not before_lines and before_newline:
+                original.content += before_newline.content
+                before_newline = None
+            if not after_lines and after_newline:
+                modified.content += after_newline.content
+                after_newline = None
+
             lines.append(AttributeDict({
                 'original': original,
                 'modified': modified,
             }))
 
         return lines
 
     def get_comments_for(self, version, filename, line_number):
         if hasattr(filename, 'unicode_path'):
             filename = filename.unicode_path
 
         if not isinstance(filename, basestring):
             return None
 
         line_key = {
             'old': 'o',
             'new': 'n',
         }[version] + str(line_number)
 
         if filename in self.comments_store:
             file_comments = self.comments_store[filename]
             if line_key in file_comments:
                 return file_comments.pop(line_key)
 
     def get_line_tokens(self, line_text, line_number, file=None):
         filenode = None
         filename = None
 
         if isinstance(file, basestring):
             filename = file
         elif isinstance(file, FileNode):
             filenode = file
             filename = file.unicode_path
 
         if self.highlight_mode == self.HL_REAL and filenode:
             lexer = self._get_lexer_for_filename(filename)
             file_size_allowed = file.size < self.max_file_size_limit
             if line_number and file_size_allowed:
                 return self.get_tokenized_filenode_line(
                     file, line_number, lexer)
 
         if self.highlight_mode in (self.HL_REAL, self.HL_FAST) and filename:
             lexer = self._get_lexer_for_filename(filename)
             return list(tokenize_string(line_text, lexer))
 
         return list(tokenize_string(line_text, plain_text_lexer))
 
     def get_tokenized_filenode_line(self, filenode, line_number, lexer=None):
 
         if filenode not in self.highlighted_filenodes:
             tokenized_lines = filenode_as_lines_tokens(filenode, lexer)
             self.highlighted_filenodes[filenode] = tokenized_lines
         return self.highlighted_filenodes[filenode][line_number - 1]
 
     def action_to_op(self, action):
         return {
             'add': '+',
             'del': '-',
             'unmod': ' ',
             'old-no-nl': ' ',
             'new-no-nl': ' ',
         }.get(action, action)
 
     def as_unified(self, lines):
         """
         Return a generator that yields the lines of a diff in unified order
         """
         def generator():
             buf = []
             for line in lines:
 
                 if buf and not line.original or line.original.action == ' ':
                     for b in buf:
                         yield b
                     buf = []
 
                 if line.original:
                     if line.original.action == ' ':
                         yield (line.original.lineno, line.modified.lineno,
                                line.original.action, line.original.content,
                                line.original.comments)
                         continue
 
                     if line.original.action == '-':
                         yield (line.original.lineno, None,
                                line.original.action, line.original.content,
                                line.original.comments)
 
                     if line.modified.action == '+':
                         buf.append((
                             None, line.modified.lineno,
                             line.modified.action, line.modified.content,
                             line.modified.comments))
                         continue
 
                 if line.modified:
                     yield (None, line.modified.lineno,
                            line.modified.action, line.modified.content,
                            line.modified.comments)
 
             for b in buf:
                 yield b
 
         return generator()

rhodecode/tests/lib/test_diffs.py

@@ -1,805 +1,813
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import textwrap
 
 import pytest
 
 from rhodecode.lib.diffs import (
     DiffProcessor,
     NEW_FILENODE, DEL_FILENODE, MOD_FILENODE, RENAMED_FILENODE,
     CHMOD_FILENODE, BIN_FILENODE, COPIED_FILENODE)
 from rhodecode.tests.fixture import Fixture
 from rhodecode.lib.vcs.backends.git.repository import GitDiff
 from rhodecode.lib.vcs.backends.hg.repository import MercurialDiff
 from rhodecode.lib.vcs.backends.svn.repository import SubversionDiff
 
 fixture = Fixture()
 
 
 def test_diffprocessor_as_html_with_comments():
     raw_diff = textwrap.dedent('''
         diff --git a/setup.py b/setup.py
         index 5b36422..cfd698e 100755
         --- a/setup.py
         +++ b/setup.py
         @@ -2,7 +2,7 @@
          #!/usr/bin/python
          # Setup file for X
          # Copyright (C) No one
         -
         +x
          try:
              from setuptools import setup, Extension
          except ImportError:
     ''')
     diff = GitDiff(raw_diff)
     processor = DiffProcessor(diff)
     processor.prepare()
 
     # Note that the cell with the context in line 5 (in the html) has the
     # no-comment class, which will prevent the add comment icon to be displayed.
     expected_html = textwrap.dedent('''
         <table class="code-difftable">
         <tr class="line context">
             <td class="add-comment-line"><span class="add-comment-content"></span></td><td class="comment-toggle tooltip" title="Toggle Comment Thread"><i class="icon-comment"></i></td>
             <td  class="lineno old">...</td>
             <td  class="lineno new">...</td>
             <td class="code no-comment">
                 <pre>@@ -2,7 +2,7 @@
         </pre>
             </td>
         </tr>
         <tr class="line unmod">
             <td class="add-comment-line"><span class="add-comment-content"><a href="#"><span class="icon-comment-add"></span></a></span></td><td class="comment-toggle tooltip" title="Toggle Comment Thread"><i class="icon-comment"></i></td>
             <td id="setuppy_o2" class="lineno old"><a href="#setuppy_o2" class="tooltip"
                         title="Click to select line">2</a></td>
             <td id="setuppy_n2" class="lineno new"><a href="#setuppy_n2" class="tooltip"
                         title="Click to select line">2</a></td>
             <td class="code">
                 <pre>#!/usr/bin/python
         </pre>
             </td>
         </tr>
         <tr class="line unmod">
             <td class="add-comment-line"><span class="add-comment-content"><a href="#"><span class="icon-comment-add"></span></a></span></td><td class="comment-toggle tooltip" title="Toggle Comment Thread"><i class="icon-comment"></i></td>
             <td id="setuppy_o3" class="lineno old"><a href="#setuppy_o3" class="tooltip"
                         title="Click to select line">3</a></td>
             <td id="setuppy_n3" class="lineno new"><a href="#setuppy_n3" class="tooltip"
                         title="Click to select line">3</a></td>
             <td class="code">
                 <pre># Setup file for X
         </pre>
             </td>
         </tr>
         <tr class="line unmod">
             <td class="add-comment-line"><span class="add-comment-content"><a href="#"><span class="icon-comment-add"></span></a></span></td><td class="comment-toggle tooltip" title="Toggle Comment Thread"><i class="icon-comment"></i></td>
             <td id="setuppy_o4" class="lineno old"><a href="#setuppy_o4" class="tooltip"
                         title="Click to select line">4</a></td>
             <td id="setuppy_n4" class="lineno new"><a href="#setuppy_n4" class="tooltip"
                         title="Click to select line">4</a></td>
             <td class="code">
                 <pre># Copyright (C) No one
         </pre>
             </td>
         </tr>
         <tr class="line del">
             <td class="add-comment-line"><span class="add-comment-content"><a href="#"><span class="icon-comment-add"></span></a></span></td><td class="comment-toggle tooltip" title="Toggle Comment Thread"><i class="icon-comment"></i></td>
             <td id="setuppy_o5" class="lineno old"><a href="#setuppy_o5" class="tooltip"
                         title="Click to select line">5</a></td>
             <td  class="lineno new"><a href="#setuppy_n" class="tooltip"
                         title="Click to select line"></a></td>
             <td class="code">
                 <pre>
         </pre>
             </td>
         </tr>
         <tr class="line add">
             <td class="add-comment-line"><span class="add-comment-content"><a href="#"><span class="icon-comment-add"></span></a></span></td><td class="comment-toggle tooltip" title="Toggle Comment Thread"><i class="icon-comment"></i></td>
             <td  class="lineno old"><a href="#setuppy_o" class="tooltip"
                         title="Click to select line"></a></td>
             <td id="setuppy_n5" class="lineno new"><a href="#setuppy_n5" class="tooltip"
                         title="Click to select line">5</a></td>
             <td class="code">
                 <pre><ins>x</ins>
         </pre>
             </td>
         </tr>
         <tr class="line unmod">
             <td class="add-comment-line"><span class="add-comment-content"><a href="#"><span class="icon-comment-add"></span></a></span></td><td class="comment-toggle tooltip" title="Toggle Comment Thread"><i class="icon-comment"></i></td>
             <td id="setuppy_o6" class="lineno old"><a href="#setuppy_o6" class="tooltip"
                         title="Click to select line">6</a></td>
             <td id="setuppy_n6" class="lineno new"><a href="#setuppy_n6" class="tooltip"
                         title="Click to select line">6</a></td>
             <td class="code">
                 <pre>try:
         </pre>
             </td>
         </tr>
         <tr class="line unmod">
             <td class="add-comment-line"><span class="add-comment-content"><a href="#"><span class="icon-comment-add"></span></a></span></td><td class="comment-toggle tooltip" title="Toggle Comment Thread"><i class="icon-comment"></i></td>
             <td id="setuppy_o7" class="lineno old"><a href="#setuppy_o7" class="tooltip"
                         title="Click to select line">7</a></td>
             <td id="setuppy_n7" class="lineno new"><a href="#setuppy_n7" class="tooltip"
                         title="Click to select line">7</a></td>
             <td class="code">
                 <pre>    from setuptools import setup, Extension
         </pre>
             </td>
         </tr>
         <tr class="line unmod">
             <td class="add-comment-line"><span class="add-comment-content"><a href="#"><span class="icon-comment-add"></span></a></span></td><td class="comment-toggle tooltip" title="Toggle Comment Thread"><i class="icon-comment"></i></td>
             <td id="setuppy_o8" class="lineno old"><a href="#setuppy_o8" class="tooltip"
                         title="Click to select line">8</a></td>
             <td id="setuppy_n8" class="lineno new"><a href="#setuppy_n8" class="tooltip"
                         title="Click to select line">8</a></td>
             <td class="code">
                 <pre>except ImportError:
         </pre>
             </td>
         </tr>
         </table>
     ''').strip()
     html = processor.as_html(enable_comments=True).replace('\t', '    ')
 
     assert html == expected_html
 
 
 class TestMixedFilenameEncodings:
 
     @pytest.fixture(scope="class")
     def raw_diff(self):
         return fixture.load_resource(
             'hg_diff_mixed_filename_encodings.diff')
 
     @pytest.fixture
     def processor(self, raw_diff):
         diff = MercurialDiff(raw_diff)
         processor = DiffProcessor(diff)
         return processor
 
     def test_filenames_are_decoded_to_unicode(self, processor):
         diff_data = processor.prepare()
         filenames = [item['filename'] for item in diff_data]
         assert filenames == [
             u'späcial-utf8.txt', u'sp�cial-cp1252.txt', u'sp�cial-latin1.txt']
 
     def test_raw_diff_is_decoded_to_unicode(self, processor):
         diff_data = processor.prepare()
         raw_diffs = [item['raw_diff'] for item in diff_data]
         new_file_message = u'\nnew file mode 100644\n'
         expected_raw_diffs = [
             u' a/späcial-utf8.txt b/späcial-utf8.txt' + new_file_message,
             u' a/sp�cial-cp1252.txt b/sp�cial-cp1252.txt' + new_file_message,
             u' a/sp�cial-latin1.txt b/sp�cial-latin1.txt' + new_file_message]
         assert raw_diffs == expected_raw_diffs
 
     def test_as_raw_preserves_the_encoding(self, processor, raw_diff):
         assert processor.as_raw() == raw_diff
 
 
 # TODO: mikhail: format the following data structure properly
 DIFF_FIXTURES = [
     ('hg',
      'hg_diff_add_single_binary_file.diff',
      [('US Warszawa.jpg', 'A',
        {'added': 0,
         'deleted': 0,
         'binary': True,
         'ops': {NEW_FILENODE: 'new file 100755',
                 BIN_FILENODE: 'binary diff hidden'}}),
       ]),
     ('hg',
      'hg_diff_mod_single_binary_file.diff',
      [('US Warszawa.jpg', 'M',
        {'added': 0,
         'deleted': 0,
         'binary': True,
         'ops': {MOD_FILENODE: 'modified file',
                 BIN_FILENODE: 'binary diff hidden'}}),
       ]),
     ('hg',
      'hg_diff_mod_single_file_and_rename_and_chmod.diff',
      [('README', 'M',
        {'added': 3,
         'deleted': 0,
         'binary': False,
         'ops': {MOD_FILENODE: 'modified file',
                 RENAMED_FILENODE: 'file renamed from README.rst to README',
                 CHMOD_FILENODE: 'modified file chmod 100755 => 100644'}}),
       ]),
     ('hg',
+     'hg_diff_no_newline.diff',
+     [('server.properties', 'M',
+       {'added': 2,
+        'deleted': 1,
+        'binary': False,
+        'ops': {MOD_FILENODE: 'modified file'}}),
+      ]),
+    ('hg',
      'hg_diff_mod_file_and_rename.diff',
      [('README.rst', 'M',
        {'added': 3,
         'deleted': 0,
         'binary': False,
         'ops': {MOD_FILENODE: 'modified file',
                 RENAMED_FILENODE: 'file renamed from README to README.rst'}}),
       ]),
     ('hg',
      'hg_diff_del_single_binary_file.diff',
      [('US Warszawa.jpg', 'D',
        {'added': 0,
         'deleted': 0,
         'binary': True,
         'ops': {DEL_FILENODE: 'deleted file',
                 BIN_FILENODE: 'binary diff hidden'}}),
       ]),
     ('hg',
      'hg_diff_chmod_and_mod_single_binary_file.diff',
      [('gravatar.png', 'M',
        {'added': 0,
         'deleted': 0,
         'binary': True,
         'ops': {CHMOD_FILENODE: 'modified file chmod 100644 => 100755',
                 BIN_FILENODE: 'binary diff hidden'}}),
       ]),
     ('hg',
      'hg_diff_chmod.diff',
      [('file', 'M',
        {'added': 0,
         'deleted': 0,
         'binary': True,
         'ops': {CHMOD_FILENODE: 'modified file chmod 100755 => 100644'}}),
       ]),
     ('hg',
      'hg_diff_rename_file.diff',
      [('file_renamed', 'M',
        {'added': 0,
         'deleted': 0,
         'binary': True,
         'ops': {RENAMED_FILENODE: 'file renamed from file to file_renamed'}}),
       ]),
     ('hg',
      'hg_diff_rename_and_chmod_file.diff',
      [('README', 'M',
        {'added': 0,
         'deleted': 0,
         'binary': True,
         'ops': {CHMOD_FILENODE: 'modified file chmod 100644 => 100755',
                 RENAMED_FILENODE: 'file renamed from README.rst to README'}}),
       ]),
     ('hg',
      'hg_diff_binary_and_normal.diff',
      [('img/baseline-10px.png', 'A',
        {'added': 0,
         'deleted': 0,
         'binary': True,
         'ops': {NEW_FILENODE: 'new file 100644',
                 BIN_FILENODE: 'binary diff hidden'}}),
       ('js/jquery/hashgrid.js', 'A',
        {'added': 340,
         'deleted': 0,
         'binary': False,
         'ops': {NEW_FILENODE: 'new file 100755'}}),
       ('index.html', 'M',
        {'added': 3,
         'deleted': 2,
         'binary': False,
         'ops': {MOD_FILENODE: 'modified file'}}),
       ('less/docs.less', 'M',
        {'added': 34,
         'deleted': 0,
         'binary': False,
         'ops': {MOD_FILENODE: 'modified file'}}),
       ('less/scaffolding.less', 'M',
        {'added': 1,
         'deleted': 3,
         'binary': False,
         'ops': {MOD_FILENODE: 'modified file'}}),
       ('readme.markdown', 'M',
        {'added': 1,
         'deleted': 10,
         'binary': False,
         'ops': {MOD_FILENODE: 'modified file'}}),
       ('img/baseline-20px.png', 'D',
        {'added': 0,
         'deleted': 0,
         'binary': True,
         'ops': {DEL_FILENODE: 'deleted file',
                 BIN_FILENODE: 'binary diff hidden'}}),
       ('js/global.js', 'D',
        {'added': 0,
         'deleted': 75,
         'binary': False,
         'ops': {DEL_FILENODE: 'deleted file'}})
       ]),
     ('git',
      'git_diff_chmod.diff',
      [('work-horus.xls', 'M',
        {'added': 0,
         'deleted': 0,
         'binary': True,
         'ops': {CHMOD_FILENODE: 'modified file chmod 100644 => 100755'}})
       ]),
     ('git',
      'git_diff_rename_file.diff',
      [('file.xls', 'M',
        {'added': 0,
         'deleted': 0,
         'binary': True,
         'ops': {
             RENAMED_FILENODE: 'file renamed from work-horus.xls to file.xls'}})
       ]),
     ('git',
      'git_diff_mod_single_binary_file.diff',
      [('US Warszawa.jpg', 'M',
        {'added': 0,
         'deleted': 0,
         'binary': True,
         'ops': {MOD_FILENODE: 'modified file',
                 BIN_FILENODE: 'binary diff hidden'}})
       ]),
     ('git',
      'git_diff_binary_and_normal.diff',
      [('img/baseline-10px.png', 'A',
        {'added': 0,
         'deleted': 0,
         'binary': True,
         'ops': {NEW_FILENODE: 'new file 100644',
                 BIN_FILENODE: 'binary diff hidden'}}),
       ('js/jquery/hashgrid.js', 'A',
        {'added': 340,
         'deleted': 0,
         'binary': False,
         'ops': {NEW_FILENODE: 'new file 100755'}}),
       ('index.html', 'M',
        {'added': 3,
         'deleted': 2,
         'binary': False,
         'ops': {MOD_FILENODE: 'modified file'}}),
       ('less/docs.less', 'M',
        {'added': 34,
         'deleted': 0,
         'binary': False,
         'ops': {MOD_FILENODE: 'modified file'}}),
       ('less/scaffolding.less', 'M',
        {'added': 1,
         'deleted': 3,
         'binary': False,
         'ops': {MOD_FILENODE: 'modified file'}}),
       ('readme.markdown', 'M',
        {'added': 1,
         'deleted': 10,
         'binary': False,
         'ops': {MOD_FILENODE: 'modified file'}}),
       ('img/baseline-20px.png', 'D',
        {'added': 0,
         'deleted': 0,
         'binary': True,
         'ops': {DEL_FILENODE: 'deleted file',
                 BIN_FILENODE: 'binary diff hidden'}}),
       ('js/global.js', 'D',
        {'added': 0,
         'deleted': 75,
         'binary': False,
         'ops': {DEL_FILENODE: 'deleted file'}}),
       ]),
     ('hg',
      'diff_with_diff_data.diff',
      [('vcs/backends/base.py', 'M',
        {'added': 18,
         'deleted': 2,
         'binary': False,
         'ops': {MOD_FILENODE: 'modified file'}}),
       ('vcs/backends/git/repository.py', 'M',
        {'added': 46,
         'deleted': 15,
         'binary': False,
         'ops': {MOD_FILENODE: 'modified file'}}),
       ('vcs/backends/hg.py', 'M',
        {'added': 22,
         'deleted': 3,
         'binary': False,
         'ops': {MOD_FILENODE: 'modified file'}}),
       ('vcs/tests/test_git.py', 'M',
        {'added': 5,
         'deleted': 5,
         'binary': False,
         'ops': {MOD_FILENODE: 'modified file'}}),
       ('vcs/tests/test_repository.py', 'M',
        {'added': 174,
         'deleted': 2,
         'binary': False,
         'ops': {MOD_FILENODE: 'modified file'}}),
       ]),
     ('hg',
      'hg_diff_copy_file.diff',
      [('file2', 'M',
        {'added': 0,
         'deleted': 0,
         'binary': True,
         'ops': {COPIED_FILENODE: 'file copied from file1 to file2'}}),
       ]),
     ('hg',
      'hg_diff_copy_and_modify_file.diff',
      [('file3', 'M',
        {'added': 1,
         'deleted': 0,
         'binary': False,
         'ops': {COPIED_FILENODE: 'file copied from file2 to file3',
                 MOD_FILENODE: 'modified file'}}),
       ]),
     ('hg',
      'hg_diff_copy_and_chmod_file.diff',
      [('file4', 'M',
        {'added': 0,
         'deleted': 0,
         'binary': True,
         'ops': {COPIED_FILENODE: 'file copied from file3 to file4',
                 CHMOD_FILENODE: 'modified file chmod 100644 => 100755'}}),
       ]),
     ('hg',
      'hg_diff_copy_chmod_and_edit_file.diff',
      [('file5', 'M',
        {'added': 2,
         'deleted': 1,
         'binary': False,
         'ops': {COPIED_FILENODE: 'file copied from file4 to file5',
                 CHMOD_FILENODE: 'modified file chmod 100755 => 100644',
                 MOD_FILENODE: 'modified file'}})]),
 
     # Diffs to validate rename and copy file with space in its name
     ('git',
      'git_diff_rename_file_with_spaces.diff',
      [('file_with_  two spaces.txt', 'M',
          {'added': 0,
           'deleted': 0,
           'binary': True,
           'ops': {
               RENAMED_FILENODE: (
                   'file renamed from file_with_ spaces.txt to file_with_ '
                   ' two spaces.txt')}
           }), ]),
     ('hg',
      'hg_diff_rename_file_with_spaces.diff',
      [('file_changed _.txt', 'M',
          {'added': 0,
           'deleted': 0,
           'binary': True,
           'ops': {
               RENAMED_FILENODE: (
                   'file renamed from file_ with update.txt to file_changed'
                   ' _.txt')}
           }), ]),
     ('hg',
      'hg_diff_copy_file_with_spaces.diff',
      [('file_copied_ with  spaces.txt', 'M',
          {'added': 0,
           'deleted': 0,
           'binary': True,
           'ops': {
               COPIED_FILENODE: (
                   'file copied from file_changed_without_spaces.txt to'
                   ' file_copied_ with  spaces.txt')}
           }),
       ]),
 
     # special signs from git
     ('git',
      'git_diff_binary_special_files.diff',
      [('css/_Icon\\r', 'A',
          {'added': 0,
           'deleted': 0,
           'binary': True,
           'ops': {NEW_FILENODE: 'new file 100644',
                   BIN_FILENODE: 'binary diff hidden'}
           }),
       ]),
     ('git',
      'git_diff_binary_special_files_2.diff',
      [('css/Icon\\r', 'A',
          {'added': 0,
           'deleted': 0,
           'binary': True,
           'ops': {NEW_FILENODE: 'new file 100644', }
           }),
       ]),
 
     ('svn',
      'svn_diff_binary_add_file.diff',
      [('intl.dll', 'A',
        {'added': 0,
         'deleted': 0,
         'binary': False,
         'ops': {NEW_FILENODE: 'new file 10644',
                 #TODO(Marcink): depends on binary detection on svn patches
                 # BIN_FILENODE: 'binary diff hidden'
                 }
         }),
       ]),
 
     ('svn',
      'svn_diff_multiple_changes.diff',
      [('trunk/doc/images/SettingsOverlay.png', 'M',
        {'added': 0,
         'deleted': 0,
         'binary': False,
         'ops': {MOD_FILENODE: 'modified file',
                 #TODO(Marcink): depends on binary detection on svn patches
                 # BIN_FILENODE: 'binary diff hidden'
                 }
         }),
       ('trunk/doc/source/de/tsvn_ch04.xml', 'M',
        {'added': 89,
         'deleted': 34,
         'binary': False,
         'ops': {MOD_FILENODE: 'modified file'}
         }),
       ('trunk/doc/source/en/tsvn_ch04.xml', 'M',
        {'added': 66,
         'deleted': 21,
         'binary': False,
         'ops': {MOD_FILENODE: 'modified file'}
         }),
       ('trunk/src/Changelog.txt', 'M',
        {'added': 2,
         'deleted': 0,
         'binary': False,
         'ops': {MOD_FILENODE: 'modified file'}
         }),
       ('trunk/src/Resources/TortoiseProcENG.rc', 'M',
        {'added': 19,
         'deleted': 13,
         'binary': False,
         'ops': {MOD_FILENODE: 'modified file'}
         }),
       ('trunk/src/TortoiseProc/SetOverlayPage.cpp', 'M',
        {'added': 16,
         'deleted': 1,
         'binary': False,
         'ops': {MOD_FILENODE: 'modified file'}
         }),
       ('trunk/src/TortoiseProc/SetOverlayPage.h', 'M',
        {'added': 3,
         'deleted': 0,
         'binary': False,
         'ops': {MOD_FILENODE: 'modified file'}
         }),
       ('trunk/src/TortoiseProc/resource.h', 'M',
        {'added': 2,
         'deleted': 0,
         'binary': False,
         'ops': {MOD_FILENODE: 'modified file'}
         }),
       ('trunk/src/TortoiseShell/ShellCache.h', 'M',
        {'added': 50,
         'deleted': 1,
         'binary': False,
         'ops': {MOD_FILENODE: 'modified file'}
         }),
       ]),
 
 
     # TODO: mikhail: do we still need this?
     # (
     #     'hg',
     #     'large_diff.diff',
     #     [
     #         ('.hgignore', 'A', {
     #             'deleted': 0, 'binary': False, 'added': 3, 'ops': {
     #                 1: 'new file 100644'}}),
     #         (
     #             'MANIFEST.in', 'A',
     #             {'deleted': 0, 'binary': False, 'added': 3, 'ops': {
     #                 1: 'new file 100644'}}),
     #         (
     #             'README.txt', 'A',
     #             {'deleted': 0, 'binary': False, 'added': 19, 'ops': {
     #                 1: 'new file 100644'}}),
     #         (
     #             'development.ini', 'A', {
     #                 'deleted': 0, 'binary': False, 'added': 116, 'ops': {
     #                     1: 'new file 100644'}}),
     #         (
     #             'docs/index.txt', 'A', {
     #                 'deleted': 0, 'binary': False, 'added': 19, 'ops': {
     #                     1: 'new file 100644'}}),
     #         (
     #             'ez_setup.py', 'A', {
     #                 'deleted': 0, 'binary': False, 'added': 276, 'ops': {
     #                     1: 'new file 100644'}}),
     #         (
     #             'hgapp.py', 'A', {
     #                 'deleted': 0, 'binary': False, 'added': 26, 'ops': {
     #                     1: 'new file 100644'}}),
     #         (
     #             'hgwebdir.config', 'A', {
     #                 'deleted': 0, 'binary': False, 'added': 21, 'ops': {
     #                     1: 'new file 100644'}}),
     #         (
     #             'pylons_app.egg-info/PKG-INFO', 'A', {
     #                 'deleted': 0, 'binary': False, 'added': 10, 'ops': {
     #                     1: 'new file 100644'}}),
     #         (
     #             'pylons_app.egg-info/SOURCES.txt', 'A', {
     #                 'deleted': 0, 'binary': False, 'added': 33, 'ops': {
     #                     1: 'new file 100644'}}),
     #         (
     #             'pylons_app.egg-info/dependency_links.txt', 'A', {
     #                 'deleted': 0, 'binary': False, 'added': 1, 'ops': {
     #                     1: 'new file 100644'}}),
     #     ]
     # ),
 ]
 
 DIFF_FIXTURES_WITH_CONTENT = [
     (
         'hg', 'hg_diff_single_file_change_newline.diff',
         [
             (
                 'file_b',  # filename
                 'A',  # change
                 {  # stats
                    'added': 1,
                    'deleted': 0,
                    'binary': False,
                    'ops': {NEW_FILENODE: 'new file 100644', }
                 },
                 '@@ -0,0 +1 @@\n+test_content b\n'  # diff
             ),
         ],
     ),
     (
         'hg', 'hg_diff_double_file_change_newline.diff',
         [
             (
                 'file_b',  # filename
                 'A',  # change
                 {  # stats
                    'added': 1,
                    'deleted': 0,
                    'binary': False,
                    'ops': {NEW_FILENODE: 'new file 100644', }
                 },
                 '@@ -0,0 +1 @@\n+test_content b\n'  # diff
             ),
             (
                 'file_c',  # filename
                 'A',  # change
                 {  # stats
                    'added': 1,
                    'deleted': 0,
                    'binary': False,
                    'ops': {NEW_FILENODE: 'new file 100644', }
                 },
                 '@@ -0,0 +1 @@\n+test_content c\n'  # diff
             ),
         ],
     ),
     (
         'hg', 'hg_diff_double_file_change_double_newline.diff',
         [
             (
                 'file_b',  # filename
                 'A',  # change
                 {  # stats
                    'added': 1,
                    'deleted': 0,
                    'binary': False,
                    'ops': {NEW_FILENODE: 'new file 100644', }
                 },
                 '@@ -0,0 +1 @@\n+test_content b\n\n'  # diff
             ),
             (
                 'file_c',  # filename
                 'A',  # change
                 {  # stats
                    'added': 1,
                    'deleted': 0,
                    'binary': False,
                    'ops': {NEW_FILENODE: 'new file 100644', }
                 },
                 '@@ -0,0 +1 @@\n+test_content c\n'  # diff
             ),
         ],
     ),
     (
         'hg', 'hg_diff_four_file_change_newline.diff',
         [
             (
                 'file',  # filename
                 'A',  # change
                 {  # stats
                    'added': 1,
                    'deleted': 0,
                    'binary': False,
                    'ops': {NEW_FILENODE: 'new file 100644', }
                 },
                 '@@ -0,0 +1,1 @@\n+file\n'  # diff
             ),
             (
                 'file2',  # filename
                 'A',  # change
                 {  # stats
                    'added': 1,
                    'deleted': 0,
                    'binary': False,
                    'ops': {NEW_FILENODE: 'new file 100644', }
                 },
                 '@@ -0,0 +1,1 @@\n+another line\n'  # diff
             ),
             (
                 'file3',  # filename
                 'A',  # change
                 {  # stats
                    'added': 1,
                    'deleted': 0,
                    'binary': False,
                    'ops': {NEW_FILENODE: 'new file 100644', }
                 },
                 '@@ -0,0 +1,1 @@\n+newline\n'  # diff
             ),
             (
                 'file4',  # filename
                 'A',  # change
                 {  # stats
                    'added': 1,
                    'deleted': 0,
                    'binary': False,
                    'ops': {NEW_FILENODE: 'new file 100644', }
                 },
                 '@@ -0,0 +1,1 @@\n+fil4\n\\ No newline at end of file'  # diff
             ),
         ],
     ),
 
 ]
 
 
 diff_class = {
     'git': GitDiff,
     'hg': MercurialDiff,
     'svn': SubversionDiff,
 }
 
 
 @pytest.fixture(params=DIFF_FIXTURES)
 def diff_fixture(request):
     vcs, diff_fixture, expected = request.param
     diff_txt = fixture.load_resource(diff_fixture)
     diff = diff_class[vcs](diff_txt)
     return diff, expected
 
 
 def test_diff_lib(diff_fixture):
     diff, expected_data = diff_fixture
     diff_proc = DiffProcessor(diff)
     diff_proc_d = diff_proc.prepare()
     data = [(x['filename'], x['operation'], x['stats']) for x in diff_proc_d]
     assert expected_data == data
 
 
 @pytest.fixture(params=DIFF_FIXTURES_WITH_CONTENT)
 def diff_fixture_w_content(request):
     vcs, diff_fixture, expected = request.param
     diff_txt = fixture.load_resource(diff_fixture)
     diff = diff_class[vcs](diff_txt)
     return diff, expected
 
 
 def test_diff_lib_newlines(diff_fixture_w_content):
     diff, expected_data = diff_fixture_w_content
     diff_proc = DiffProcessor(diff)
     diff_proc_d = diff_proc.prepare()
     data = [(x['filename'], x['operation'], x['stats'], x['raw_diff'])
             for x in diff_proc_d]
     assert expected_data == data