codeblocks: add new code token rendering function that

supports diff and normal tokens.

rhodecode/tests/lib/test_codeblocks.py

@@ -0,0 +1,330 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2016-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import pytest
+
+from rhodecode.lib.codeblocks import (
+    tokenize_string, split_token_stream, rollup_tokenstream,
+    render_tokenstream)
+from pygments.lexers import get_lexer_by_name
+
+
+class TestTokenizeString(object):
+
+    python_code = '''
+    import this
+
+    var = 6
+    print "this"
+
+    '''
+
+    def test_tokenize_as_python(self):
+        lexer = get_lexer_by_name('python')
+        tokens = list(tokenize_string(self.python_code, lexer))
+
+        assert tokens == [
+            ('',    u'\n'),
+            ('',    u'    '),
+            ('kn',  u'import'),
+            ('',    u' '),
+            ('nn',  u'this'),
+            ('',    u'\n'),
+            ('',    u'\n'),
+            ('',    u'    '),
+            ('n',   u'var'),
+            ('',    u' '),
+            ('o',   u'='),
+            ('',    u' '),
+            ('mi',  u'6'),
+            ('',    u'\n'),
+            ('',    u'    '),
+            ('k',   u'print'),
+            ('',    u' '),
+            ('s2',  u'"'),
+            ('s2',  u'this'),
+            ('s2',  u'"'),
+            ('',    u'\n'),
+            ('',    u'\n'),
+            ('',    u'    ')
+        ]
+
+    def test_tokenize_as_text(self):
+        lexer = get_lexer_by_name('text')
+        tokens = list(tokenize_string(self.python_code, lexer))
+
+        assert tokens == [
+            ('',
+            u'\n    import this\n\n    var = 6\n    print "this"\n\n    ')
+        ]
+
+
+class TestSplitTokenStream(object):
+
+    def test_split_token_stream(self):
+        lines = list(split_token_stream(
+            [('type1', 'some\ntext'), ('type2', 'more\n')]))
+
+        assert lines == [
+            [('type1', u'some')],
+            [('type1', u'text'), ('type2', u'more')],
+            [('type2', u'')],
+        ]
+
+    def test_split_token_stream_other_char(self):
+        lines = list(split_token_stream(
+            [('type1', 'some\ntext'), ('type2', 'more\n')],
+            split_string='m'))
+
+        assert lines == [
+            [('type1', 'so')],
+            [('type1', 'e\ntext'), ('type2', '')],
+            [('type2', 'ore\n')],
+        ]
+
+    def test_split_token_stream_without_char(self):
+        lines = list(split_token_stream(
+            [('type1', 'some\ntext'), ('type2', 'more\n')],
+            split_string='z'))
+
+        assert lines == [
+            [('type1', 'some\ntext'), ('type2', 'more\n')]
+        ]
+
+    def test_split_token_stream_single(self):
+        lines = list(split_token_stream(
+            [('type1', '\n')], split_string='\n'))
+
+        assert lines == [
+            [('type1', '')],
+            [('type1', '')],
+        ]
+
+    def test_split_token_stream_single_repeat(self):
+        lines = list(split_token_stream(
+            [('type1', '\n\n\n')], split_string='\n'))
+
+        assert lines == [
+            [('type1', '')],
+            [('type1', '')],
+            [('type1', '')],
+            [('type1', '')],
+        ]
+
+    def test_split_token_stream_multiple_repeat(self):
+        lines = list(split_token_stream(
+            [('type1', '\n\n'), ('type2', '\n\n')], split_string='\n'))
+
+        assert lines == [
+            [('type1', '')],
+            [('type1', '')],
+            [('type1', ''), ('type2', '')],
+            [('type2', '')],
+            [('type2', '')],
+        ]
+
+
+class TestRollupTokens(object):
+
+    @pytest.mark.parametrize('tokenstream,output', [
+        ([],
+            []),
+        ([('A', 'hell'), ('A', 'o')], [
+            ('A', [
+                ('', 'hello')]),
+        ]),
+        ([('A', 'hell'), ('B', 'o')], [
+            ('A', [
+                ('', 'hell')]),
+            ('B', [
+                ('', 'o')]),
+        ]),
+        ([('A', 'hel'), ('A', 'lo'), ('B', ' '), ('A', 'there')], [
+            ('A', [
+                ('', 'hello')]),
+            ('B', [
+                ('', ' ')]),
+            ('A', [
+                ('', 'there')]),
+        ]),
+    ])
+    def test_rollup_tokenstream_without_ops(self, tokenstream, output):
+        assert list(rollup_tokenstream(tokenstream)) == output
+
+    @pytest.mark.parametrize('tokenstream,output', [
+        ([],
+            []),
+        ([('A', '', 'hell'), ('A', '', 'o')], [
+            ('A', [
+                ('', 'hello')]),
+        ]),
+        ([('A', '', 'hell'), ('B', '', 'o')], [
+            ('A', [
+                ('', 'hell')]),
+            ('B', [
+                ('', 'o')]),
+        ]),
+        ([('A', '', 'h'), ('B', '', 'e'), ('C', '', 'y')], [
+            ('A', [
+                ('', 'h')]),
+            ('B', [
+                ('', 'e')]),
+            ('C', [
+                ('', 'y')]),
+        ]),
+        ([('A', '', 'h'), ('A', '', 'e'), ('C', '', 'y')], [
+            ('A', [
+                ('', 'he')]),
+            ('C', [
+                ('', 'y')]),
+        ]),
+        ([('A', 'ins', 'h'), ('A', 'ins', 'e')], [
+            ('A', [
+                ('ins', 'he')
+            ]),
+        ]),
+        ([('A', 'ins', 'h'), ('A', 'del', 'e')], [
+            ('A', [
+                ('ins', 'h'),
+                ('del', 'e')
+            ]),
+        ]),
+        ([('A', 'ins', 'h'), ('B', 'del', 'e'), ('B', 'del', 'y')], [
+            ('A', [
+                ('ins', 'h'),
+            ]),
+            ('B', [
+                ('del', 'ey'),
+            ]),
+        ]),
+        ([('A', 'ins', 'h'), ('A', 'del', 'e'), ('B', 'del', 'y')], [
+            ('A', [
+                ('ins', 'h'),
+                ('del', 'e'),
+            ]),
+            ('B', [
+                ('del', 'y'),
+            ]),
+        ]),
+        ([('A', '', 'some'), ('A', 'ins', 'new'), ('A', '', 'name')], [
+            ('A', [
+                ('', 'some'),
+                ('ins', 'new'),
+                ('', 'name'),
+            ]),
+        ]),
+    ])
+    def test_rollup_tokenstream_with_ops(self, tokenstream, output):
+        assert list(rollup_tokenstream(tokenstream)) == output
+
+
+class TestRenderTokenStream(object):
+
+    @pytest.mark.parametrize('tokenstream,output', [
+        (
+            [],
+            '',
+        ),
+        (
+            [('', '', u'')],
+            '<span></span>',
+        ),
+        (
+            [('', '', u'text')],
+            '<span>text</span>',
+        ),
+        (
+            [('A', '', u'')],
+            '<span class="A"></span>',
+        ),
+        (
+            [('A', '', u'hello')],
+            '<span class="A">hello</span>',
+        ),
+        (
+            [('A', '', u'hel'), ('A', '', u'lo')],
+            '<span class="A">hello</span>',
+        ),
+        (
+            [('A', '', u'two\n'), ('A', '', u'lines')],
+            '<span class="A">two<nl>\n</nl>lines</span>',
+        ),
+        (
+            [('A', '', u'\nthree\n'), ('A', '', u'lines')],
+            '<span class="A"><nl>\n</nl>three<nl>\n</nl>lines</span>',
+        ),
+        (
+            [('', '', u'\n'), ('A', '', u'line')],
+            '<span><nl>\n</nl></span><span class="A">line</span>',
+        ),
+        (
+            [('', 'ins', u'\n'), ('A', '', u'line')],
+            '<span><ins><nl>\n</nl></ins></span><span class="A">line</span>',
+        ),
+        (
+            [('A', '', u'hel'), ('A', 'ins', u'lo')],
+            '<span class="A">hel<ins>lo</ins></span>',
+        ),
+        (
+            [('A', '', u'hel'), ('A', 'ins', u'l'), ('A', 'ins', u'o')],
+            '<span class="A">hel<ins>lo</ins></span>',
+        ),
+        (
+            [('A', '', u'hel'), ('A', 'ins', u'l'), ('A', 'del', u'o')],
+            '<span class="A">hel<ins>l</ins><del>o</del></span>',
+        ),
+        (
+            [('A', '', u'hel'), ('B', '', u'lo')],
+            '<span class="A">hel</span><span class="B">lo</span>',
+        ),
+        (
+            [('A', '', u'hel'), ('B', 'ins', u'lo')],
+            '<span class="A">hel</span><span class="B"><ins>lo</ins></span>',
+        ),
+    ])
+    def test_render_tokenstream_with_ops(self, tokenstream, output):
+        html = render_tokenstream(tokenstream)
+        assert html == output
+
+    @pytest.mark.parametrize('tokenstream,output', [
+        (
+            [('A', u'hel'), ('A', u'lo')],
+            '<span class="A">hello</span>',
+        ),
+        (
+            [('A', u'hel'), ('A', u'l'), ('A', u'o')],
+            '<span class="A">hello</span>',
+        ),
+        (
+            [('A', u'hel'), ('A', u'l'), ('A', u'o')],
+            '<span class="A">hello</span>',
+        ),
+        (
+            [('A', u'hel'), ('B', u'lo')],
+            '<span class="A">hel</span><span class="B">lo</span>',
+        ),
+        (
+            [('A', u'hel'), ('B', u'lo')],
+            '<span class="A">hel</span><span class="B">lo</span>',
+        ),
+    ])
+    def test_render_tokenstream_without_ops(self, tokenstream, output):
+        html = render_tokenstream(tokenstream)
+        assert html == output

rhodecode/lib/codeblocks.py

@@ -1,147 +1,214 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
-
+import logging
 from itertools import groupby
 
 from pygments import lex
-# PYGMENTS_TOKEN_TYPES is used in a hot loop keep attribute lookups to a minimum
-from pygments.token import STANDARD_TYPES as PYGMENTS_TOKEN_TYPES
+from pygments.formatters.html import _get_ttype_class as pygment_token_class
+from rhodecode.lib.helpers import get_lexer_for_filenode, html_escape
+from rhodecode.lib.utils2 import AttributeDict
+from rhodecode.lib.vcs.nodes import FileNode
+from pygments.lexers import get_lexer_by_name
+
+plain_text_lexer = get_lexer_by_name(
+    'text', stripall=False, stripnl=False, ensurenl=False)
+
+
+log = logging.getLogger()
 
-from rhodecode.lib.helpers import get_lexer_for_filenode
 
-def tokenize_file(content, lexer):
+def filenode_as_lines_tokens(filenode, lexer=None):
+    lexer = lexer or get_lexer_for_filenode(filenode)
+    log.debug('Generating file node pygment tokens for %s, %s', lexer, filenode)
+    tokens = tokenize_string(filenode.content, get_lexer_for_filenode(filenode))
+    lines = split_token_stream(tokens, split_string='\n')
+    rv = list(lines)
+    return rv
+
+
+def tokenize_string(content, lexer):
     """
     Use pygments to tokenize some content based on a lexer
     ensuring all original new lines and whitespace is preserved
     """
 
     lexer.stripall = False
     lexer.stripnl = False
     lexer.ensurenl = False
-    return lex(content, lexer)
+    for token_type, token_text in lex(content, lexer):
+        yield pygment_token_class(token_type), token_text
 
 
-def pygment_token_class(token_type):
-    """ Convert a pygments token type to html class name """
-
-    fname = PYGMENTS_TOKEN_TYPES.get(token_type)
-    if fname:
-        return fname
-
-    aname = ''
-    while fname is None:
-        aname = '-' + token_type[-1] + aname
-        token_type = token_type.parent
-        fname = PYGMENTS_TOKEN_TYPES.get(token_type)
-
-    return fname + aname
-
-
-def tokens_as_lines(tokens, split_string=u'\n'):
+def split_token_stream(tokens, split_string=u'\n'):
     """
     Take a list of (TokenType, text) tuples and split them by a string
 
-    eg. [(TEXT, 'some\ntext')] => [(TEXT, 'some'), (TEXT, 'text')]
+    >>> split_token_stream([(TEXT, 'some\ntext'), (TEXT, 'more\n')])
+    [(TEXT, 'some'), (TEXT, 'text'),
+     (TEXT, 'more'), (TEXT, 'text')]
     """
 
     buffer = []
-    for token_type, token_text in tokens:
+    for token_class, token_text in tokens:
         parts = token_text.split(split_string)
         for part in parts[:-1]:
-            buffer.append((token_type, part))
+            buffer.append((token_class, part))
             yield buffer
             buffer = []
 
-        buffer.append((token_type, parts[-1]))
+        buffer.append((token_class, parts[-1]))
 
     if buffer:
         yield buffer
 
 
-def filenode_as_lines_tokens(filenode):
-    """
-    Return a generator of lines with pygment tokens for a filenode eg:
-
-    [
-        (1, line1_tokens_list),
-        (2, line1_tokens_list]),
-    ]
-    """
-
-    return enumerate(
-      tokens_as_lines(
-        tokenize_file(
-          filenode.content, get_lexer_for_filenode(filenode)
-        )
-      ),
-    1)
-
-
 def filenode_as_annotated_lines_tokens(filenode):
     """
     Take a file node and return a list of annotations => lines, if no annotation
     is found, it will be None.
 
     eg:
 
     [
         (annotation1, [
             (1, line1_tokens_list),
             (2, line2_tokens_list),
         ]),
         (annotation2, [
             (3, line1_tokens_list),
         ]),
         (None, [
             (4, line1_tokens_list),
         ]),
         (annotation1, [
             (5, line1_tokens_list),
             (6, line2_tokens_list),
         ])
     ]
     """
 
+    commit_cache = {} # cache commit_getter lookups
 
-    # cache commit_getter lookups
-    commit_cache = {}
     def _get_annotation(commit_id, commit_getter):
         if commit_id not in commit_cache:
             commit_cache[commit_id] = commit_getter()
         return commit_cache[commit_id]
 
     annotation_lookup = {
         line_no: _get_annotation(commit_id, commit_getter)
         for line_no, commit_id, commit_getter, line_content
         in filenode.annotate
     }
 
     annotations_lines = ((annotation_lookup.get(line_no), line_no, tokens)
                           for line_no, tokens
-                          in filenode_as_lines_tokens(filenode))
+                          in enumerate(filenode_as_lines_tokens(filenode), 1))
 
     grouped_annotations_lines = groupby(annotations_lines, lambda x: x[0])
 
     for annotation, group in grouped_annotations_lines:
         yield (
             annotation, [(line_no, tokens)
                           for (_, line_no, tokens) in group]
         )
+
+
+def render_tokenstream(tokenstream):
+    result = []
+    for token_class, token_ops_texts in rollup_tokenstream(tokenstream):
+
+        if token_class:
+            result.append(u'<span class="%s">' % token_class)
+        else:
+            result.append(u'<span>')
+
+        for op_tag, token_text in token_ops_texts:
+
+            if op_tag:
+                result.append(u'<%s>' % op_tag)
+
+            escaped_text = html_escape(token_text)
+            escaped_text = escaped_text.replace('\n', '<nl>\n</nl>')
+
+            result.append(escaped_text)
+
+            if op_tag:
+                result.append(u'</%s>' % op_tag)
+
+        result.append(u'</span>')
+
+    html = ''.join(result)
+    return html
+
+
+def rollup_tokenstream(tokenstream):
+    """
+    Group a token stream of the format:
+
+        ('class', 'op', 'text')
+    or
+        ('class', 'text')
+
+    into
+
+        [('class1',
+            [('op1', 'text'),
+             ('op2', 'text')]),
+         ('class2',
+            [('op3', 'text')])]
+
+    This is used to get the minimal tags necessary when
+    rendering to html eg for a token stream ie.
+
+    <span class="A"><ins>he</ins>llo</span>
+    vs
+    <span class="A"><ins>he</ins></span><span class="A">llo</span>
+
+    If a 2 tuple is passed in, the output op will be an empty string.
+
+    eg:
+
+    >>> rollup_tokenstream([('classA', '',      'h'),
+                            ('classA', 'del',   'ell'),
+                            ('classA', '',      'o'),
+                            ('classB', '',      ' '),
+                            ('classA', '',      'the'),
+                            ('classA', '',      're'),
+                            ])
+
+    [('classA', [('', 'h'), ('del', 'ell'), ('', 'o')],
+     ('classB', [('', ' ')],
+     ('classA', [('', 'there')]]
+
+    """
+    if tokenstream and len(tokenstream[0]) == 2:
+        tokenstream = ((t[0], '', t[1]) for t in tokenstream)
+
+    result = []
+    for token_class, op_list in groupby(tokenstream, lambda t: t[0]):
+        ops = []
+        for token_op, token_text_list in groupby(op_list, lambda o: o[1]):
+            text_buffer = []
+            for t_class, t_op, t_text in token_text_list:
+                text_buffer.append(t_text)
+            ops.append((token_op, ''.join(text_buffer)))
+        result.append((token_class, ops))
+    return result

rhodecode/public/css/code-block.less

@@ -1,749 +1,753 b''
 // Default styles
 
 .diff-collapse {
     margin: @padding 0;
     text-align: right;
 }
 
 .diff-container {
     margin-bottom: @space;
 
     .diffblock {
         margin-bottom: @space;
     }
 
     &.hidden {
         display: none;
         overflow: hidden;
     }
 }
 
 .compare_view_files {
 
     .diff-container {
 
         .diffblock {
             margin-bottom: 0;
         }
     }
 }
 
 div.diffblock .sidebyside {
  background: #ffffff;
 }
 
 div.diffblock {
     overflow-x: auto;
     overflow-y: hidden;
     clear: both;
     padding: 0px;
     background: @grey6;
     border: @border-thickness solid @grey5;
     -webkit-border-radius: @border-radius @border-radius 0px 0px;
     border-radius: @border-radius @border-radius 0px 0px;
 
 
     .comments-number {
         float: right;
     }
 
     // BEGIN CODE-HEADER STYLES
 
     .code-header {
         background: @grey6;
         padding: 10px 0 10px 0;
         height: auto;
         width: 100%;
 
         .hash {
             float: left;
             padding: 2px 0 0 2px;
         }
 
         .date {
             float: left;
             text-transform: uppercase;
             padding: 4px 0px 0px 2px;
         }
 
         div {
             margin-left: 4px;
         }
 
         div.compare_header {
             min-height: 40px;
             margin: 0;
             padding: 0 @padding;
 
             .drop-menu {
                 float:left;
                 display: block;
                 margin:0 0 @padding 0;
             }
 
             .compare-label {
                 float: left;
                 clear: both;
                 display: inline-block;
                 min-width: 5em;
                 margin: 0;
                 padding: @button-padding @button-padding @button-padding 0;
                 font-family: @text-semibold;
             }
 
             .compare-buttons {
                 float: left;
                 margin: 0;
                 padding: 0 0 @padding;
 
                 .btn {
                     margin: 0 @padding 0 0;
                 }
             }
         }
 
     }
 
     .parents {
         float: left;
         width: 100px;
         font-weight: 400;
         vertical-align: middle;
         padding: 0px 2px 0px 2px;
         background-color: @grey6;
 
         #parent_link {
            margin: 00px 2px;
 
            &.double {
                margin: 0px 2px;
             }
 
             &.disabled{
                 margin-right: @padding;
             }
         }
     }
 
     .children {
         float: right;
         width: 100px;
         font-weight: 400;
         vertical-align: middle;
         text-align: right;
         padding: 0px 2px 0px 2px;
         background-color: @grey6;
 
         #child_link {
            margin: 0px 2px;
 
            &.double {
                margin: 0px 2px;
             }
 
             &.disabled{
                 margin-right: @padding;
             }
         }
     }
 
     .changeset_header {
         height: 16px;
 
         & > div{
             margin-right: @padding;
        }
     }
 
     .changeset_file {
         text-align: left;
         float: left;
         padding: 0;
 
         a{
             display: inline-block;
             margin-right: 0.5em;
         }
 
        #selected_mode{
             margin-left: 0;
        }
     }
 
     .diff-menu-wrapper {
         float: left;
     }
 
     .diff-menu {
         position: absolute;
         background: none repeat scroll 0 0 #FFFFFF;
         border-color: #003367 @grey3 @grey3;
         border-right: 1px solid @grey3;
         border-style: solid solid solid;
         border-width: @border-thickness;
         box-shadow: 2px 8px 4px rgba(0, 0, 0, 0.2);
         margin-top: 5px;
         margin-left: 1px;
     }
 
     .diff-actions, .editor-actions {
         float: left;
 
         input{
             margin: 0 0.5em 0 0;
         }
     }
 
     // END CODE-HEADER STYLES
 
     // BEGIN CODE-BODY STYLES
 
     .code-body {
         background: white;
         padding: 0;
         background-color: #ffffff;
         position: relative;
         max-width: none;
         box-sizing: border-box;
         // TODO: johbo: Parent has overflow: auto, this forces the child here
         // to have the intended size and to scroll. Should be simplified.
         width: 100%;
         overflow-x: auto;
     }
 
     pre.raw {
         background: white;
         color: @grey1;
     }
     // END CODE-BODY STYLES
 
 }
 
 
 table.code-difftable {
     border-collapse: collapse;
     width: 99%;
     border-radius: 0px !important;
 
      td {
         padding: 0 !important;
         background: none !important;
         border: 0 !important;
     }
 
     .context {
         background: none repeat scroll 0 0 #DDE7EF;
     }
 
     .add {
         background: none repeat scroll 0 0 #DDFFDD;
 
         ins {
             background: none repeat scroll 0 0 #AAFFAA;
             text-decoration: none;
         }
     }
 
     .del {
         background: none repeat scroll 0 0 #FFDDDD;
 
         del {
             background: none repeat scroll 0 0 #FFAAAA;
             text-decoration: none;
         }
     }
 
     /** LINE NUMBERS **/
     .lineno {
         padding-left: 2px !important;
         padding-right: 2px;
         text-align: right;
         width: 32px;
         -moz-user-select: none;
         -webkit-user-select: none;
         border-right: @border-thickness solid @grey5 !important;
         border-left: 0px solid #CCC !important;
         border-top: 0px solid #CCC !important;
         border-bottom: none !important;
 
         a {
             &:extend(pre);
             text-align: right;
             padding-right: 2px;
             cursor: pointer;
             display: block;
             width: 32px;
         }
     }
 
     .context {
         cursor: auto;
         &:extend(pre);
     }
 
     .lineno-inline {
         background: none repeat scroll 0 0 #FFF !important;
         padding-left: 2px;
         padding-right: 2px;
         text-align: right;
         width: 30px;
         -moz-user-select: none;
         -webkit-user-select: none;
     }
 
     /** CODE **/
     .code {
         display: block;
         width: 100%;
 
         td {
             margin: 0;
             padding: 0;
         }
 
         pre {
             margin: 0;
             padding: 0;
             margin-left: .5em;
         }
     }
 }
 
 
 // Comments
 
 div.comment:target {
     border-left: 6px solid @comment-highlight-color;
     padding-left: 3px;
     margin-left: -9px;
 }
 
 //TODO: anderson: can't get an absolute number out of anything, so had to put the
 //current values that might change. But to make it clear I put as a calculation
 @comment-max-width: 1065px;
 @pr-extra-margin: 34px;
 @pr-border-spacing: 4px;
 @pr-comment-width: @comment-max-width - @pr-extra-margin - @pr-border-spacing;
 
 // Pull Request
 .cs_files .code-difftable {
     border: @border-thickness solid @grey5; //borders only on PRs
 
     .comment-inline-form,
     div.comment {
         width: @pr-comment-width;
     }
 }
 
 // Changeset
 .code-difftable {
     .comment-inline-form,
     div.comment {
         width: @comment-max-width;
     }
 }
 
 //Style page
 @style-extra-margin: @sidebar-width + (@sidebarpadding * 3) + @padding;
 #style-page .code-difftable{
     .comment-inline-form,
     div.comment {
         width: @comment-max-width - @style-extra-margin;
     }
 }
 
 #context-bar > h2 {
     font-size: 20px;
 }
 
 #context-bar > h2> a {
     font-size: 20px;
 }
 // end of defaults
 
 .file_diff_buttons {
     padding: 0 0 @padding;
 
     .drop-menu {
         float: left;
         margin: 0 @padding 0 0;
     }
     .btn {
         margin: 0 @padding 0 0;
     }
 }
 
 .code-body.textarea.editor {
     max-width: none;
     padding: 15px;
 }
 
 td.injected_diff{
     max-width: 1178px;
     overflow-x: auto;
     overflow-y: hidden;
 
     div.diff-container,
     div.diffblock{
         max-width: 100%;
     }
 
     div.code-body {
         max-width: 1124px;
         overflow-x: auto;
         overflow-y: hidden;
         padding: 0;
     }
     div.diffblock {
         border: none;
     }
 
     &.inline-form {
         width: 99%
     }
  }
 
 
 table.code-difftable {
     width: 100%;
 }
 
 /** PYGMENTS COLORING **/
 div.codeblock {
 
     // TODO: johbo: Added interim to get rid of the margin around
     // Select2 widgets. This needs further cleanup.
     margin-top: @padding;
 
     overflow: auto;
     padding: 0px;
     border: @border-thickness solid @grey5;
     background: @grey6;
     .border-radius(@border-radius);
 
     #remove_gist {
         float: right;
     }
 
     .author {
         clear: both;
         vertical-align: middle;
         font-family: @text-bold;
     }
 
     .btn-mini {
         float: left;
         margin: 0 5px 0 0;
     }
 
     .code-header {
         padding: @padding;
         border-bottom: @border-thickness solid @grey5;
 
         .rc-user {
             min-width: 0;
             margin-right: .5em;
         }
 
         .stats {
             clear: both;
             margin: 0 0 @padding 0;
             padding: 0;
             .left {
                 float: left;
                 clear: left;
                 max-width: 75%;
                 margin: 0 0 @padding 0;
 
                 &.item {
                     margin-right: @padding;
                     &.last { border-right: none; }
                 }
             }
             .buttons { float: right; }
             .author {
                 height: 25px; margin-left: 15px; font-weight: bold;
             }
         }
 
         .commit {
             margin: 5px 0 0 26px;
             font-weight: normal;
             white-space: pre-wrap;
         }
     }
 
     .message {
         position: relative;
         margin: @padding;
 
         .codeblock-label {
             margin: 0 0 1em 0;
         }
     }
 
     .code-body {
         padding: @padding;
         background-color: #ffffff;
         min-width: 100%;
         box-sizing: border-box;
         // TODO: johbo: Parent has overflow: auto, this forces the child here
         // to have the intended size and to scroll. Should be simplified.
         width: 100%;
         overflow-x: auto;
     }
 }
 
 .code-highlighttable,
 div.codeblock {
 
     &.readme {
         background-color: white;
     }
 
     .markdown-block table {
         border-collapse: collapse;
 
         th,
         td {
             padding: .5em;
             border: @border-thickness solid @border-default-color;
         }
     }
 
     table {
         border: 0px;
         margin: 0;
         letter-spacing: normal;
 
 
         td {
             border: 0px;
             vertical-align: top;
         }
     }
 }
 
 div.codeblock .code-header .search-path { padding: 0 0 0 10px; }
 div.search-code-body {
     background-color: #ffffff; padding: 5px 0 5px 10px;
     pre {
         .match { background-color: #faffa6;}
         .break { display: block; width: 100%; background-color: #DDE7EF; color: #747474; }
     }
     .code-highlighttable {
         border-collapse: collapse;
 
         tr:hover {
             background: #fafafa;
         }
         td.code {
           padding-left: 10px;
         }
         td.line {
             border-right: 1px solid #ccc !important;
             padding-right: 10px;
             text-align: right;
             font-family: "Lucida Console",Monaco,monospace;
             span {
                 white-space: pre-wrap;
                 color: #666666;
             }
         }
     }
 }
 
 div.annotatediv { margin-left: 2px; margin-right: 4px; }
 .code-highlight {
     margin: 0; padding: 0; border-left: @border-thickness solid @grey5;
     pre, .linenodiv pre { padding: 0 5px; margin: 0; }
     pre div:target {background-color: @comment-highlight-color !important;}
 }
 
 .linenos a { text-decoration: none; }
 
 .CodeMirror-selected { background: @rchighlightblue; }
 .CodeMirror-focused .CodeMirror-selected { background: @rchighlightblue; }
 .CodeMirror ::selection { background: @rchighlightblue; }
 .CodeMirror ::-moz-selection { background: @rchighlightblue; }
 
 .code { display: block; border:0px !important; }
 .code-highlight, /* TODO: dan: merge codehilite into code-highlight */
 .codehilite {
     .hll { background-color: #ffffcc }
     .c { color: #408080; font-style: italic } /* Comment */
     .err, .codehilite .err { border: @border-thickness solid #FF0000 } /* Error */
     .k { color: #008000; font-weight: bold } /* Keyword */
     .o { color: #666666 } /* Operator */
     .cm { color: #408080; font-style: italic } /* Comment.Multiline */
     .cp { color: #BC7A00 } /* Comment.Preproc */
     .c1 { color: #408080; font-style: italic } /* Comment.Single */
     .cs { color: #408080; font-style: italic } /* Comment.Special */
     .gd { color: #A00000 } /* Generic.Deleted */
     .ge { font-style: italic } /* Generic.Emph */
     .gr { color: #FF0000 } /* Generic.Error */
     .gh { color: #000080; font-weight: bold } /* Generic.Heading */
     .gi { color: #00A000 } /* Generic.Inserted */
     .go { color: #808080 } /* Generic.Output */
     .gp { color: #000080; font-weight: bold } /* Generic.Prompt */
     .gs { font-weight: bold } /* Generic.Strong */
     .gu { color: #800080; font-weight: bold } /* Generic.Subheading */
     .gt { color: #0040D0 } /* Generic.Traceback */
     .kc { color: #008000; font-weight: bold } /* Keyword.Constant */
     .kd { color: #008000; font-weight: bold } /* Keyword.Declaration */
     .kn { color: #008000; font-weight: bold } /* Keyword.Namespace */
     .kp { color: #008000 } /* Keyword.Pseudo */
     .kr { color: #008000; font-weight: bold } /* Keyword.Reserved */
     .kt { color: #B00040 } /* Keyword.Type */
     .m { color: #666666 } /* Literal.Number */
     .s { color: #BA2121 } /* Literal.String */
     .na { color: #7D9029 } /* Name.Attribute */
     .nb { color: #008000 } /* Name.Builtin */
     .nc { color: #0000FF; font-weight: bold } /* Name.Class */
     .no { color: #880000 } /* Name.Constant */
     .nd { color: #AA22FF } /* Name.Decorator */
     .ni { color: #999999; font-weight: bold } /* Name.Entity */
     .ne { color: #D2413A; font-weight: bold } /* Name.Exception */
     .nf { color: #0000FF } /* Name.Function */
     .nl { color: #A0A000 } /* Name.Label */
     .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */
     .nt { color: #008000; font-weight: bold } /* Name.Tag */
     .nv { color: #19177C } /* Name.Variable */
     .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */
     .w { color: #bbbbbb } /* Text.Whitespace */
     .mf { color: #666666 } /* Literal.Number.Float */
     .mh { color: #666666 } /* Literal.Number.Hex */
     .mi { color: #666666 } /* Literal.Number.Integer */
     .mo { color: #666666 } /* Literal.Number.Oct */
     .sb { color: #BA2121 } /* Literal.String.Backtick */
     .sc { color: #BA2121 } /* Literal.String.Char */
     .sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */
     .s2 { color: #BA2121 } /* Literal.String.Double */
     .se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */
     .sh { color: #BA2121 } /* Literal.String.Heredoc */
     .si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */
     .sx { color: #008000 } /* Literal.String.Other */
     .sr { color: #BB6688 } /* Literal.String.Regex */
     .s1 { color: #BA2121 } /* Literal.String.Single */
     .ss { color: #19177C } /* Literal.String.Symbol */
     .bp { color: #008000 } /* Name.Builtin.Pseudo */
     .vc { color: #19177C } /* Name.Variable.Class */
     .vg { color: #19177C } /* Name.Variable.Global */
     .vi { color: #19177C } /* Name.Variable.Instance */
     .il { color: #666666 } /* Literal.Number.Integer.Long */
 }
 
 /* customized pre blocks for markdown/rst */
 pre.literal-block, .codehilite pre{
     padding: @padding;
     border: 1px solid @grey6;
     .border-radius(@border-radius);
     background-color: @grey7;
 }
 
 
 /* START NEW CODE BLOCK CSS */
 
+@cb-line-height: 18px;
+@cb-line-code-padding: 10px;
+
 table.cb {
     width: 100%;
     border-collapse: collapse;
     margin-bottom: 10px;
 
     * {
         box-sizing: border-box;
     }
 
     /* intentionally general selector since .cb-line-selected must override it
        and they both use !important since the td itself may have a random color
        generated by annotation blocks. TLDR: if you change it, make sure
        annotated block selection and line selection in file view still work */
     .cb-line-fresh .cb-content {
         background: white !important;
     }
 
     tr.cb-annotate {
         border-top: 1px solid #eee;
 
         &+ .cb-line {
             border-top: 1px solid #eee;
         }
 
         &:first-child {
             border-top: none;
             &+ .cb-line {
                 border-top: none;
             }
         }
     }
 
     td {
         vertical-align: top;
-        padding: 2px 10px;
+        padding: 0;
 
         &.cb-content {
+            font-size: 12.35px;
+
+            span.cb-code {
+                line-height: @cb-line-height;
+                padding-left: @cb-line-code-padding;
+                display: block;
             white-space: pre-wrap;
             font-family: @font-family-monospace;
-            font-size: 12.35px;
-
-            span {
                 word-break: break-word;
             }
         }
 
         &.cb-lineno {
             padding: 0;
-            height: 1px; /* this allows the <a> link to fill to 100% height of the td */
             width: 50px;
             color: rgba(0, 0, 0, 0.3);
             text-align: right;
             border-right: 1px solid #eee;
             font-family: @font-family-monospace;
 
             a::before {
                 content: attr(data-line-no);
             }
-            &.cb-line-selected {
+            &.cb-line-selected a {
                 background: @comment-highlight-color !important;
             }
 
             a {
                 display: block;
-                height: 100%;
+                padding-right: @cb-line-code-padding;
+                line-height: @cb-line-height;
                 color: rgba(0, 0, 0, 0.3);
-                padding: 0 10px; /* vertical padding is 0 so that height: 100% works */
-                line-height: 18px; /* use this instead of vertical padding */
             }
         }
 
         &.cb-content {
-            &.cb-line-selected {
+            &.cb-line-selected .cb-code {
                 background: @comment-highlight-color !important;
             }
         }
 
         &.cb-annotate-info {
             width: 320px;
             min-width: 320px;
             max-width: 320px;
             padding: 5px 2px;
             font-size: 13px;
 
             strong.cb-annotate-message {
                 padding: 5px 0;
                 white-space: pre-line;
                 display: inline-block;
             }
             .rc-user {
                 float: none;
                 padding: 0 6px 0 17px;
                 min-width: auto;
                 min-height: auto;
             }
         }
 
         &.cb-annotate-revision {
             cursor: pointer;
             text-align: right;
         }
     }
 }

rhodecode/templates/codeblocks/source.html

@@ -1,70 +1,68 b''
 <%def name="render_line(line_num, tokens,
                         annotation=None,
                         bgcolor=None)">
     <%
-    # avoid module lookups for performance
-    from rhodecode.lib.codeblocks import pygment_token_class
-    from rhodecode.lib.helpers import html_escape
+    from rhodecode.lib.codeblocks import render_tokenstream
+    # avoid module lookup for performance
+    html_escape = h.html_escape
     %>
     <tr class="cb-line cb-line-fresh"
     %if annotation:
     data-revision="${annotation.revision}"
     %endif
     >
     <td class="cb-lineno" id="L${line_num}">
       <a data-line-no="${line_num}" href="#L${line_num}"></a>
     </td>
     <td class="cb-content cb-content-fresh"
     %if bgcolor:
     style="background: ${bgcolor}"
     %endif
-    >${
-      ''.join(
-       '<span class="%s">%s</span>' %
-        (pygment_token_class(token_type), html_escape(token_text))
-        for token_type, token_text in tokens) + '\n' | n
-    }</td>
-    ## this ugly list comp is necessary for performance
+    >
+    ## newline at end is necessary for highlight to work when line is empty
+    ## and for copy pasting code to work as expected
+      <span class="cb-code">${render_tokenstream(tokens)|n}${'\n'}</span>
+    </td>
   </tr>
 </%def>
 
 <%def name="render_annotation_lines(annotation, lines, color_hasher)">
   <%
   rowspan = len(lines) + 1 # span the line's <tr> and annotation <tr>
   %>
   %if not annotation:
   <tr class="cb-annotate">
     <td class="cb-annotate-message" rowspan="${rowspan}"></td>
     <td class="cb-annotate-revision" rowspan="${rowspan}"></td>
   </tr>
   %else:
   <tr class="cb-annotate">
     <td class="cb-annotate-info tooltip"
         rowspan="${rowspan}"
         title="Author: ${annotation.author | entity}<br>Date: ${annotation.date}<br>Message: ${annotation.message | entity}"
     >
       ${h.gravatar_with_user(annotation.author, 16) | n}
       <strong class="cb-annotate-message">${
           h.truncate(annotation.message, len(lines) * 30)
       }</strong>
     </td>
     <td
       class="cb-annotate-revision"
       rowspan="${rowspan}"
       data-revision="${annotation.revision}"
       onclick="$('[data-revision=${annotation.revision}]').toggleClass('cb-line-fresh')"
       style="background: ${color_hasher(annotation.raw_id)}">
     <a href="${h.url('changeset_home',repo_name=c.repo_name,revision=annotation.raw_id)}">
       r${annotation.revision}
     </a>
     </td>
   </tr>
   %endif
 
   %for line_num, tokens in lines:
     ${render_line(line_num, tokens,
       bgcolor=color_hasher(annotation and annotation.raw_id or ''),
       annotation=annotation,
       )}
   %endfor
 </%def>

rhodecode/templates/files/files_source.html

@@ -1,82 +1,82 b''
 <%namespace name="sourceblock" file="/codeblocks/source.html"/>
 
 <div id="codeblock" class="codeblock">
     <div class="codeblock-header">
       <div class="stats">
             <span> <strong>${c.file}</strong></span>
             <span> | ${c.file.lines()[0]} ${ungettext('line', 'lines', c.file.lines()[0])}</span>
             <span> | ${h.format_byte_size_binary(c.file.size)}</span>
             <span> | ${c.file.mimetype} </span>
             <span class="item last"> | ${h.get_lexer_for_filenode(c.file).__class__.__name__}</span>
       </div>
       <div class="buttons">
         <a id="file_history_overview" href="#">
             ${_('History')}
         </a>
         <a id="file_history_overview_full" style="display: none" href="${h.url('changelog_file_home',repo_name=c.repo_name, revision=c.commit.raw_id, f_path=c.f_path)}">
            ${_('Show Full History')}
         </a> |
         %if c.annotate:
           ${h.link_to(_('Source'), h.url('files_home', repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path))}
         %else:
           ${h.link_to(_('Annotation'), h.url('files_annotate_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path))}
         %endif
          | ${h.link_to(_('Raw'), h.url('files_raw_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path))}
          | <a href="${h.url('files_rawfile_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path)}">
           ${_('Download')}
         </a>
 
         %if h.HasRepoPermissionAny('repository.write','repository.admin')(c.repo_name):
            |
          %if c.on_branch_head and c.branch_or_raw_id and not c.file.is_binary:
             <a href="${h.url('files_edit_home',repo_name=c.repo_name,revision=c.branch_or_raw_id,f_path=c.f_path, anchor='edit')}">
               ${_('Edit on Branch:%s') % c.branch_or_raw_id}
             </a>
             | <a class="btn-danger btn-link" href="${h.url('files_delete_home',repo_name=c.repo_name,revision=c.branch_or_raw_id,f_path=c.f_path, anchor='edit')}">${_('Delete')}
             </a>
          %elif c.on_branch_head and c.branch_or_raw_id and c.file.is_binary:
           ${h.link_to(_('Edit'), '#', class_="btn btn-link disabled tooltip", title=_('Editing binary files not allowed'))}
            | ${h.link_to(_('Delete'), h.url('files_delete_home',repo_name=c.repo_name,revision=c.branch_or_raw_id,f_path=c.f_path, anchor='edit'),class_="btn-danger btn-link")}
          %else:
           ${h.link_to(_('Edit'), '#', class_="btn btn-link disabled tooltip", title=_('Editing files allowed only when on branch head commit'))}
            | ${h.link_to(_('Delete'), '#', class_="btn btn-danger btn-link disabled tooltip", title=_('Deleting files allowed only when on branch head commit'))}
          %endif
         %endif
       </div>
     </div>
     <div id="file_history_container"></div>
     <div class="code-body">
        %if c.file.is_binary:
            <div>
            ${_('Binary file (%s)') % c.file.mimetype}
            </div>
        %else:
         % if c.file.size < c.cut_off_limit:
             %if c.renderer and not c.annotate:
                 ${h.render(c.file.content, renderer=c.renderer)}
             %else:
                 <table class="cb codehilite">
                 %if c.annotate:
                   <% color_hasher = h.color_hasher() %>
                   %for annotation, lines in c.annotated_lines:
                     ${sourceblock.render_annotation_lines(annotation, lines, color_hasher)}
                   %endfor
                 %else:
-                  %for line_num, tokens in c.lines:
+                  %for line_num, tokens in enumerate(c.lines, 1):
                     ${sourceblock.render_line(line_num, tokens)}
                   %endfor
                 %endif
                 </table>
               </div>
             %endif
         %else:
             ${_('File is too big to display')} ${h.link_to(_('Show as raw'),
             h.url('files_raw_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path))}
         %endif
      %endif
     </div>
 </div>
 
 <script>
     var source_page = true;
 </script>