search: add syntax highlighting, line numbers and line context

to file content search results

rhodecode/public/js/src/plugins/jquery.mark.js

@@ -0,0 +1,490 b''
+/*!***************************************************
+ * mark.js v6.1.0
+ * https://github.com/julmot/mark.js
+ * Copyright (c) 2014–2016, Julian Motz
+ * Released under the MIT license https://git.io/vwTVl
+ *****************************************************/
+
+"use strict";
+
+var _extends = Object.assign || function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; };
+
+var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }();
+
+var _typeof = typeof Symbol === "function" && typeof Symbol.iterator === "symbol" ? function (obj) { return typeof obj; } : function (obj) { return obj && typeof Symbol === "function" && obj.constructor === Symbol ? "symbol" : typeof obj; };
+
+function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } }
+
+(function (factory, window, document) {
+    if (typeof define === "function" && define.amd) {
+        define(["jquery"], function (jQuery) {
+            return factory(window, document, jQuery);
+        });
+    } else if ((typeof exports === "undefined" ? "undefined" : _typeof(exports)) === "object") {
+        factory(window, document, require("jquery"));
+    } else {
+        factory(window, document, jQuery);
+    }
+})(function (window, document, $) {
+    var Mark = function () {
+        function Mark(ctx) {
+            _classCallCheck(this, Mark);
+
+            this.ctx = ctx;
+        }
+
+        _createClass(Mark, [{
+            key: "log",
+            value: function log(msg) {
+                var level = arguments.length <= 1 || arguments[1] === undefined ? "debug" : arguments[1];
+
+                var log = this.opt.log;
+                if (!this.opt.debug) {
+                    return;
+                }
+                if ((typeof log === "undefined" ? "undefined" : _typeof(log)) === "object" && typeof log[level] === "function") {
+                    log[level]("mark.js: " + msg);
+                }
+            }
+        }, {
+            key: "escapeStr",
+            value: function escapeStr(str) {
+                return str.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&");
+            }
+        }, {
+            key: "createRegExp",
+            value: function createRegExp(str) {
+                str = this.escapeStr(str);
+                if (Object.keys(this.opt.synonyms).length) {
+                    str = this.createSynonymsRegExp(str);
+                }
+                if (this.opt.diacritics) {
+                    str = this.createDiacriticsRegExp(str);
+                }
+                str = this.createAccuracyRegExp(str);
+                return str;
+            }
+        }, {
+            key: "createSynonymsRegExp",
+            value: function createSynonymsRegExp(str) {
+                var syn = this.opt.synonyms;
+                for (var index in syn) {
+                    if (syn.hasOwnProperty(index)) {
+                        var value = syn[index],
+                            k1 = this.escapeStr(index),
+                            k2 = this.escapeStr(value);
+                        str = str.replace(new RegExp("(" + k1 + "|" + k2 + ")", "gmi"), "(" + k1 + "|" + k2 + ")");
+                    }
+                }
+                return str;
+            }
+        }, {
+            key: "createDiacriticsRegExp",
+            value: function createDiacriticsRegExp(str) {
+                var dct = ["aÀÁÂÃÄÅàáâãäåĀāąĄ", "cÇçćĆčČ", "dđĐďĎ", "eÈÉÊËèéêëěĚĒēęĘ", "iÌÍÎÏìíîïĪī", "lłŁ", "nÑñňŇńŃ", "oÒÓÔÕÕÖØòóôõöøŌō", "rřŘ", "sŠšśŚ", "tťŤ", "uÙÚÛÜùúûüůŮŪū", "yŸÿýÝ", "zŽžżŻźŹ"];
+                var handled = [];
+                str.split("").forEach(function (ch) {
+                    dct.every(function (dct) {
+                        if (dct.indexOf(ch) !== -1) {
+                            if (handled.indexOf(dct) > -1) {
+                                return false;
+                            }
+
+                            str = str.replace(new RegExp("[" + dct + "]", "gmi"), "[" + dct + "]");
+                            handled.push(dct);
+                        }
+                        return true;
+                    });
+                });
+                return str;
+            }
+        }, {
+            key: "createAccuracyRegExp",
+            value: function createAccuracyRegExp(str) {
+                switch (this.opt.accuracy) {
+                    case "partially":
+                        return "()(" + str + ")";
+                    case "complementary":
+                        return "()(\\S*" + str + "\\S*)";
+                    case "exactly":
+                        return "(^|\\s)(" + str + ")(?=\\s|$)";
+                }
+            }
+        }, {
+            key: "getSeparatedKeywords",
+            value: function getSeparatedKeywords(sv) {
+                var _this = this;
+
+                var stack = [];
+                sv.forEach(function (kw) {
+                    if (!_this.opt.separateWordSearch) {
+                        if (kw.trim()) {
+                            stack.push(kw);
+                        }
+                    } else {
+                        kw.split(" ").forEach(function (kwSplitted) {
+                            if (kwSplitted.trim()) {
+                                stack.push(kwSplitted);
+                            }
+                        });
+                    }
+                });
+                return {
+                    "keywords": stack,
+                    "length": stack.length
+                };
+            }
+        }, {
+            key: "getElements",
+            value: function getElements() {
+                var ctx = void 0,
+                    stack = [];
+                if (typeof this.ctx === "undefined") {
+                    ctx = [];
+                } else if (this.ctx instanceof HTMLElement) {
+                    ctx = [this.ctx];
+                } else if (Array.isArray(this.ctx)) {
+                    ctx = this.ctx;
+                } else {
+                    ctx = Array.prototype.slice.call(this.ctx);
+                }
+                ctx.forEach(function (ctx) {
+                    stack.push(ctx);
+                    var childs = ctx.querySelectorAll("*");
+                    if (childs.length) {
+                        stack = stack.concat(Array.prototype.slice.call(childs));
+                    }
+                });
+                if (!ctx.length) {
+                    this.log("Empty context", "warn");
+                }
+                return {
+                    "elements": stack,
+                    "length": stack.length
+                };
+            }
+        }, {
+            key: "matches",
+            value: function matches(el, selector) {
+                return (el.matches || el.matchesSelector || el.msMatchesSelector || el.mozMatchesSelector || el.webkitMatchesSelector || el.oMatchesSelector).call(el, selector);
+            }
+        }, {
+            key: "matchesFilter",
+            value: function matchesFilter(el, exclM) {
+                var _this2 = this;
+
+                var remain = true;
+                var fltr = this.opt.filter.concat(["script", "style", "title"]);
+                if (!this.opt.iframes) {
+                    fltr = fltr.concat(["iframe"]);
+                }
+                if (exclM) {
+                    fltr = fltr.concat(["*[data-markjs='true']"]);
+                }
+                fltr.every(function (filter) {
+                    if (_this2.matches(el, filter)) {
+                        return remain = false;
+                    }
+                    return true;
+                });
+                return !remain;
+            }
+        }, {
+            key: "onIframeReady",
+            value: function onIframeReady(ifr, successFn, errorFn) {
+                try {
+                    (function () {
+                        var ifrWin = ifr.contentWindow,
+                            bl = "about:blank",
+                            compl = "complete";
+                        var callCallback = function callCallback() {
+                            try {
+                                if (ifrWin.document === null) {
+                                    throw new Error("iframe inaccessible");
+                                }
+                                successFn(ifrWin.document);
+                            } catch (e) {
+                                errorFn();
+                            }
+                        };
+                        var isBlank = function isBlank() {
+                            var src = ifr.getAttribute("src").trim(),
+                                href = ifrWin.location.href;
+                            return href === bl && src !== bl && src;
+                        };
+                        var observeOnload = function observeOnload() {
+                            var listener = function listener() {
+                                try {
+                                    if (!isBlank()) {
+                                        ifr.removeEventListener("load", listener);
+                                        callCallback();
+                                    }
+                                } catch (e) {
+                                    errorFn();
+                                }
+                            };
+                            ifr.addEventListener("load", listener);
+                        };
+                        if (ifrWin.document.readyState === compl) {
+                            if (isBlank()) {
+                                observeOnload();
+                            } else {
+                                callCallback();
+                            }
+                        } else {
+                            observeOnload();
+                        }
+                    })();
+                } catch (e) {
+                    errorFn();
+                }
+            }
+        }, {
+            key: "forEachElementInIframe",
+            value: function forEachElementInIframe(ifr, cb) {
+                var _this3 = this;
+
+                var end = arguments.length <= 2 || arguments[2] === undefined ? function () {} : arguments[2];
+
+                var open = 0;
+                var checkEnd = function checkEnd() {
+                    if (--open < 1) {
+                        end();
+                    }
+                };
+                this.onIframeReady(ifr, function (con) {
+                    var stack = Array.prototype.slice.call(con.querySelectorAll("*"));
+                    if ((open = stack.length) === 0) {
+                        checkEnd();
+                    }
+                    stack.forEach(function (el) {
+                        if (el.tagName.toLowerCase() === "iframe") {
+                            (function () {
+                                var j = 0;
+                                _this3.forEachElementInIframe(el, function (iel, len) {
+                                    cb(iel, len);
+                                    if (len - 1 === j) {
+                                        checkEnd();
+                                    }
+                                    j++;
+                                }, checkEnd);
+                            })();
+                        } else {
+                            cb(el, stack.length);
+                            checkEnd();
+                        }
+                    });
+                }, function () {
+                    var src = ifr.getAttribute("src");
+                    _this3.log("iframe '" + src + "' could not be accessed", "warn");
+                    checkEnd();
+                });
+            }
+        }, {
+            key: "forEachElement",
+            value: function forEachElement(cb) {
+                var _this4 = this;
+
+                var end = arguments.length <= 1 || arguments[1] === undefined ? function () {} : arguments[1];
+                var exclM = arguments.length <= 2 || arguments[2] === undefined ? true : arguments[2];
+
+                var _getElements = this.getElements();
+
+                var stack = _getElements.elements;
+                var open = _getElements.length;
+
+                var checkEnd = function checkEnd() {
+                    if (--open === 0) {
+                        end();
+                    }
+                };
+                checkEnd(++open);
+                stack.forEach(function (el) {
+                    if (!_this4.matchesFilter(el, exclM)) {
+                        if (el.tagName.toLowerCase() === "iframe") {
+                            _this4.forEachElementInIframe(el, function (iel) {
+                                if (!_this4.matchesFilter(iel, exclM)) {
+                                    cb(iel);
+                                }
+                            }, checkEnd);
+                            return;
+                        } else {
+                                cb(el);
+                            }
+                    }
+                    checkEnd();
+                });
+            }
+        }, {
+            key: "forEachNode",
+            value: function forEachNode(cb) {
+                var end = arguments.length <= 1 || arguments[1] === undefined ? function () {} : arguments[1];
+
+                this.forEachElement(function (n) {
+                    for (n = n.firstChild; n; n = n.nextSibling) {
+                        if (n.nodeType === 3 && n.textContent.trim()) {
+                            cb(n);
+                        }
+                    }
+                }, end);
+            }
+        }, {
+            key: "wrapMatches",
+            value: function wrapMatches(node, regex, custom, cb) {
+                var hEl = !this.opt.element ? "mark" : this.opt.element,
+                    index = custom ? 0 : 2;
+                var match = void 0;
+                while ((match = regex.exec(node.textContent)) !== null) {
+                    var pos = match.index;
+                    if (!custom) {
+                        pos += match[index - 1].length;
+                    }
+                    var startNode = node.splitText(pos);
+
+                    node = startNode.splitText(match[index].length);
+                    if (startNode.parentNode !== null) {
+                        var repl = document.createElement(hEl);
+                        repl.setAttribute("data-markjs", "true");
+                        if (this.opt.className) {
+                            repl.setAttribute("class", this.opt.className);
+                        }
+                        repl.textContent = match[index];
+                        startNode.parentNode.replaceChild(repl, startNode);
+                        cb(repl);
+                    }
+                    regex.lastIndex = 0;
+                }
+            }
+        }, {
+            key: "unwrapMatches",
+            value: function unwrapMatches(node) {
+                var parent = node.parentNode;
+                var docFrag = document.createDocumentFragment();
+                while (node.firstChild) {
+                    docFrag.appendChild(node.removeChild(node.firstChild));
+                }
+                parent.replaceChild(docFrag, node);
+                parent.normalize();
+            }
+        }, {
+            key: "markRegExp",
+            value: function markRegExp(regexp, opt) {
+                var _this5 = this;
+
+                this.opt = opt;
+                this.log("Searching with expression \"" + regexp + "\"");
+                var found = false;
+                var eachCb = function eachCb(element) {
+                    found = true;
+                    _this5.opt.each(element);
+                };
+                this.forEachNode(function (node) {
+                    _this5.wrapMatches(node, regexp, true, eachCb);
+                }, function () {
+                    if (!found) {
+                        _this5.opt.noMatch(regexp);
+                    }
+                    _this5.opt.complete();
+                    _this5.opt.done();
+                });
+            }
+        }, {
+            key: "mark",
+            value: function mark(sv, opt) {
+                var _this6 = this;
+
+                this.opt = opt;
+                sv = typeof sv === "string" ? [sv] : sv;
+
+                var _getSeparatedKeywords = this.getSeparatedKeywords(sv);
+
+                var kwArr = _getSeparatedKeywords.keywords;
+                var kwArrLen = _getSeparatedKeywords.length;
+
+                if (kwArrLen === 0) {
+                    this.opt.complete();
+                    this.opt.done();
+                }
+                kwArr.forEach(function (kw) {
+                    var regex = new RegExp(_this6.createRegExp(kw), "gmi"),
+                        found = false;
+                    var eachCb = function eachCb(element) {
+                        found = true;
+                        _this6.opt.each(element);
+                    };
+                    _this6.log("Searching with expression \"" + regex + "\"");
+                    _this6.forEachNode(function (node) {
+                        _this6.wrapMatches(node, regex, false, eachCb);
+                    }, function () {
+                        if (!found) {
+                            _this6.opt.noMatch(kw);
+                        }
+                        if (kwArr[kwArrLen - 1] === kw) {
+                            _this6.opt.complete();
+                            _this6.opt.done();
+                        }
+                    });
+                });
+            }
+        }, {
+            key: "unmark",
+            value: function unmark(opt) {
+                var _this7 = this;
+
+                this.opt = opt;
+                var sel = this.opt.element ? this.opt.element : "*";
+                sel += "[data-markjs]";
+                if (this.opt.className) {
+                    sel += "." + this.opt.className;
+                }
+                this.log("Removal selector \"" + sel + "\"");
+                this.forEachElement(function (el) {
+                    if (_this7.matches(el, sel)) {
+                        _this7.unwrapMatches(el);
+                    }
+                }, function () {
+                    _this7.opt.complete();
+                    _this7.opt.done();
+                }, false);
+            }
+        }, {
+            key: "opt",
+            set: function set(val) {
+                this._opt = _extends({}, {
+                    "element": "",
+                    "className": "",
+                    "filter": [],
+                    "iframes": false,
+                    "separateWordSearch": true,
+                    "diacritics": true,
+                    "synonyms": {},
+                    "accuracy": "partially",
+                    "each": function each() {},
+                    "noMatch": function noMatch() {},
+                    "done": function done() {},
+                    "complete": function complete() {},
+                    "debug": false,
+                    "log": window.console
+                }, val);
+            },
+            get: function get() {
+                return this._opt;
+            }
+        }]);
+
+        return Mark;
+    }();
+
+    $.fn.mark = function (sv, opt) {
+        new Mark(this).mark(sv, opt);
+        return this;
+    };
+    $.fn.markRegExp = function (regexp, opt) {
+        new Mark(this).markRegExp(regexp, opt);
+        return this;
+    };
+    $.fn.unmark = function (opt) {
+        new Mark(this).unmark(opt);
+        return this;
+    };
+}, window, document);

Gruntfile.js

@@ -1,137 +1,139 b''
 module.exports = function(grunt) {
   grunt.initConfig({
 
     dirs: {
       css: "rhodecode/public/css",
       js: {
         "src": "rhodecode/public/js/src",
         "dest": "rhodecode/public/js"
       }
     },
 
     concat: {
       dist: {
         src: [
           // Base libraries
           '<%= dirs.js.src %>/jquery-1.11.1.min.js',
           '<%= dirs.js.src %>/logging.js',
           '<%= dirs.js.src %>/bootstrap.js',
           '<%= dirs.js.src %>/mousetrap.js',
           '<%= dirs.js.src %>/moment.js',
+          '<%= dirs.js.src %>/moment.js',
           '<%= dirs.js.src %>/appenlight-client-0.4.1.min.js',
 
           // Plugins
           '<%= dirs.js.src %>/plugins/jquery.pjax.js',
           '<%= dirs.js.src %>/plugins/jquery.dataTables.js',
           '<%= dirs.js.src %>/plugins/flavoured_checkbox.js',
           '<%= dirs.js.src %>/plugins/jquery.auto-grow-input.js',
           '<%= dirs.js.src %>/plugins/jquery.autocomplete.js',
           '<%= dirs.js.src %>/plugins/jquery.debounce.js',
+          '<%= dirs.js.src %>/plugins/jquery.mark.js',
           '<%= dirs.js.src %>/plugins/jquery.timeago.js',
           '<%= dirs.js.src %>/plugins/jquery.timeago-extension.js',
 
           // Select2
           '<%= dirs.js.src %>/select2/select2.js',
-          
+
           // Code-mirror
           '<%= dirs.js.src %>/codemirror/codemirror.js',
           '<%= dirs.js.src %>/codemirror/codemirror_loadmode.js',
           '<%= dirs.js.src %>/codemirror/codemirror_hint.js',
           '<%= dirs.js.src %>/codemirror/codemirror_overlay.js',
           '<%= dirs.js.src %>/codemirror/codemirror_placeholder.js',
           // TODO: mikhail: this is an exception. Since the code mirror modes
           // are loaded "on the fly", we need to keep them in a public folder
           '<%= dirs.js.dest %>/mode/meta.js',
           '<%= dirs.js.dest %>/mode/meta_ext.js',
           '<%= dirs.js.dest %>/rhodecode/i18n/select2/translations.js',
 
           // Rhodecode utilities
           '<%= dirs.js.src %>/rhodecode/utils/array.js',
           '<%= dirs.js.src %>/rhodecode/utils/string.js',
           '<%= dirs.js.src %>/rhodecode/utils/pyroutes.js',
           '<%= dirs.js.src %>/rhodecode/utils/ajax.js',
           '<%= dirs.js.src %>/rhodecode/utils/autocomplete.js',
           '<%= dirs.js.src %>/rhodecode/utils/colorgenerator.js',
           '<%= dirs.js.src %>/rhodecode/utils/ie.js',
           '<%= dirs.js.src %>/rhodecode/utils/os.js',
 
           // Rhodecode widgets
           '<%= dirs.js.src %>/rhodecode/widgets/multiselect.js',
 
           // Rhodecode components
           '<%= dirs.js.src %>/rhodecode/pyroutes.js',
           '<%= dirs.js.src %>/rhodecode/codemirror.js',
           '<%= dirs.js.src %>/rhodecode/comments.js',
           '<%= dirs.js.src %>/rhodecode/constants.js',
           '<%= dirs.js.src %>/rhodecode/files.js',
           '<%= dirs.js.src %>/rhodecode/followers.js',
           '<%= dirs.js.src %>/rhodecode/menus.js',
           '<%= dirs.js.src %>/rhodecode/notifications.js',
           '<%= dirs.js.src %>/rhodecode/permissions.js',
           '<%= dirs.js.src %>/rhodecode/pjax.js',
           '<%= dirs.js.src %>/rhodecode/pullrequests.js',
           '<%= dirs.js.src %>/rhodecode/settings.js',
           '<%= dirs.js.src %>/rhodecode/select2_widgets.js',
           '<%= dirs.js.src %>/rhodecode/tooltips.js',
           '<%= dirs.js.src %>/rhodecode/users.js',
           '<%= dirs.js.src %>/rhodecode/appenlight.js',
 
           // Rhodecode main module
           '<%= dirs.js.src %>/rhodecode.js'
         ],
         dest: '<%= dirs.js.dest %>/scripts.js',
         nonull: true
       }
     },
 
     less: {
       development: {
         options: {
           compress: false,
           yuicompress: false,
           optimization: 0
         },
         files: {
           "<%= dirs.css %>/style.css": "<%= dirs.css %>/main.less"
         }
       },
       production: {
         options: {
           compress: true,
           yuicompress: true,
           optimization: 2
         },
         files: {
           "<%= dirs.css %>/style.css": "<%= dirs.css %>/main.less"
         }
       }
     },
 
     watch: {
       less: {
         files: ["<%= dirs.css %>/*.less"],
         tasks: ["less:production"]
       },
       js: {
         files: ["<%= dirs.js.src %>/**/*.js"],
         tasks: ["concat:dist"]
       }
     },
 
     jshint: {
       rhodecode: {
         src: '<%= dirs.js.src %>/rhodecode/**/*.js',
         options: {
           jshintrc: '.jshintrc'
         }
       }
     }
   });
 
   grunt.loadNpmTasks('grunt-contrib-less');
   grunt.loadNpmTasks('grunt-contrib-concat');
   grunt.loadNpmTasks('grunt-contrib-watch');
   grunt.loadNpmTasks('grunt-contrib-jshint');
 
   grunt.registerTask('default', ['less:production', 'concat:dist']);
 };

rhodecode/controllers/search.py

@@ -1,106 +1,107 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Search controller for RhodeCode
 """
 
 import logging
 import urllib
 
 from pylons import request, config, tmpl_context as c
 
 from webhelpers.util import update_params
 
 from rhodecode.lib.auth import LoginRequired, AuthUser
 from rhodecode.lib.base import BaseRepoController, render
 from rhodecode.lib.helpers import Page
 from rhodecode.lib.utils2 import safe_str, safe_int
 from rhodecode.lib.index import searcher_from_config
 from rhodecode.model import validation_schema
 
 log = logging.getLogger(__name__)
 
 
 class SearchController(BaseRepoController):
 
     @LoginRequired()
     def index(self, repo_name=None):
 
         searcher = searcher_from_config(config)
         formatted_results = []
         execution_time = ''
 
         schema = validation_schema.SearchParamsSchema()
 
         search_params = {}
         errors = []
         try:
             search_params = schema.deserialize(
                 dict(search_query=request.GET.get('q'),
                      search_type=request.GET.get('type'),
                      page_limit=request.GET.get('page_limit'),
                      requested_page=request.GET.get('page'))
             )
         except validation_schema.Invalid as e:
             errors = e.children
 
         search_query = search_params.get('search_query')
         search_type = search_params.get('search_type')
 
         if search_params.get('search_query'):
             page_limit = search_params['page_limit']
             requested_page = search_params['requested_page']
 
             def url_generator(**kw):
                 q = urllib.quote(safe_str(search_query))
                 return update_params(
                     "?q=%s&type=%s" % (q, safe_str(search_type)), **kw)
 
             c.perm_user = AuthUser(user_id=c.rhodecode_user.user_id,
                                    ip_addr=self.ip_addr)
 
             try:
                 search_result = searcher.search(
-                    search_query, search_type, c.perm_user, repo_name)
+                    search_query, search_type, c.perm_user, repo_name,
+                    requested_page, page_limit)
 
                 formatted_results = Page(
                     search_result['results'], page=requested_page,
                     item_count=search_result['count'],
                     items_per_page=page_limit, url=url_generator)
             finally:
                 searcher.cleanup()
 
             if not search_result['error']:
                 execution_time = '%s results (%.3f seconds)' % (
                     search_result['count'],
                     search_result['runtime'])
             elif not errors:
                 node = schema['search_query']
                 errors = [
                     validation_schema.Invalid(node, search_result['error'])]
 
         c.errors = errors
         c.formatted_results = formatted_results
         c.runtime = execution_time
         c.cur_query = search_query
         c.search_type = search_type
         # Return a rendered template
         return render('/search/search.html')

rhodecode/lib/helpers.py

@@ -1,1712 +1,1885 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Helper functions
 
 Consists of functions to typically be used within templates, but also
 available to Controllers. This module is available to both as 'h'.
 """
 
 import random
 import hashlib
 import StringIO
 import urllib
 import math
 import logging
 import re
 import urlparse
 import time
 import string
 import hashlib
+import pygments
 
 from datetime import datetime
 from functools import partial
 from pygments.formatters.html import HtmlFormatter
 from pygments import highlight as code_highlight
+from pygments.lexers import (
+    get_lexer_by_name, get_lexer_for_filename, get_lexer_for_mimetype)
 from pylons import url
 from pylons.i18n.translation import _, ungettext
 from pyramid.threadlocal import get_current_request
 
 from webhelpers.html import literal, HTML, escape
 from webhelpers.html.tools import *
 from webhelpers.html.builder import make_tag
 from webhelpers.html.tags import auto_discovery_link, checkbox, css_classes, \
     end_form, file, form as wh_form, hidden, image, javascript_link, link_to, \
     link_to_if, link_to_unless, ol, required_legend, select, stylesheet_link, \
     submit, text, password, textarea, title, ul, xml_declaration, radio
 from webhelpers.html.tools import auto_link, button_to, highlight, \
     js_obfuscate, mail_to, strip_links, strip_tags, tag_re
 from webhelpers.pylonslib import Flash as _Flash
 from webhelpers.text import chop_at, collapse, convert_accented_entities, \
     convert_misc_entities, lchop, plural, rchop, remove_formatting, \
     replace_whitespace, urlify, truncate, wrap_paragraphs
 from webhelpers.date import time_ago_in_words
 from webhelpers.paginate import Page as _Page
 from webhelpers.html.tags import _set_input_attrs, _set_id_attr, \
     convert_boolean_attrs, NotGiven, _make_safe_id_component
 from webhelpers2.number import format_byte_size
 
 from rhodecode.lib.annotate import annotate_highlight
 from rhodecode.lib.action_parser import action_parser
 from rhodecode.lib.utils import repo_name_slug, get_custom_lexer
 from rhodecode.lib.utils2 import str2bool, safe_unicode, safe_str, \
     get_commit_safe, datetime_to_time, time_to_datetime, AttributeDict, \
     safe_int, md5, md5_safe
 from rhodecode.lib.markup_renderer import MarkupRenderer
 from rhodecode.lib.vcs.exceptions import CommitDoesNotExistError
 from rhodecode.lib.vcs.backends.base import BaseChangeset, EmptyCommit
 from rhodecode.config.conf import DATE_FORMAT, DATETIME_FORMAT
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.db import Permission, User, Repository
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.settings import IssueTrackerSettingsModel
 
 log = logging.getLogger(__name__)
 
 DEFAULT_USER = User.DEFAULT_USER
 DEFAULT_USER_EMAIL = User.DEFAULT_USER_EMAIL
 
 
 def html_escape(text, html_escape_table=None):
     """Produce entities within text."""
     if not html_escape_table:
         html_escape_table = {
             "&": "&amp;",
             '"': "&quot;",
             "'": "&apos;",
             ">": "&gt;",
             "<": "&lt;",
         }
     return "".join(html_escape_table.get(c, c) for c in text)
 
 
 def chop_at_smart(s, sub, inclusive=False, suffix_if_chopped=None):
     """
     Truncate string ``s`` at the first occurrence of ``sub``.
 
     If ``inclusive`` is true, truncate just after ``sub`` rather than at it.
     """
     suffix_if_chopped = suffix_if_chopped or ''
     pos = s.find(sub)
     if pos == -1:
         return s
 
     if inclusive:
         pos += len(sub)
 
     chopped = s[:pos]
     left = s[pos:].strip()
 
     if left and suffix_if_chopped:
         chopped += suffix_if_chopped
 
     return chopped
 
 
 def shorter(text, size=20):
     postfix = '...'
     if len(text) > size:
         return text[:size - len(postfix)] + postfix
     return text
 
 
 def _reset(name, value=None, id=NotGiven, type="reset", **attrs):
     """
     Reset button
     """
     _set_input_attrs(attrs, type, name, value)
     _set_id_attr(attrs, id, name)
     convert_boolean_attrs(attrs, ["disabled"])
     return HTML.input(**attrs)
 
 reset = _reset
 safeid = _make_safe_id_component
 
 
 def branding(name, length=40):
     return truncate(name, length, indicator="")
 
 
 def FID(raw_id, path):
     """
     Creates a unique ID for filenode based on it's hash of path and commit
     it's safe to use in urls
 
     :param raw_id:
     :param path:
     """
 
     return 'c-%s-%s' % (short_id(raw_id), md5_safe(path)[:12])
 
 
 class _GetError(object):
     """Get error from form_errors, and represent it as span wrapped error
     message
 
     :param field_name: field to fetch errors for
     :param form_errors: form errors dict
     """
 
     def __call__(self, field_name, form_errors):
         tmpl = """<span class="error_msg">%s</span>"""
         if form_errors and field_name in form_errors:
             return literal(tmpl % form_errors.get(field_name))
 
 get_error = _GetError()
 
 
 class _ToolTip(object):
 
     def __call__(self, tooltip_title, trim_at=50):
         """
         Special function just to wrap our text into nice formatted
         autowrapped text
 
         :param tooltip_title:
         """
         tooltip_title = escape(tooltip_title)
         tooltip_title = tooltip_title.replace('<', '&lt;').replace('>', '&gt;')
         return tooltip_title
 tooltip = _ToolTip()
 
 
 def files_breadcrumbs(repo_name, commit_id, file_path):
     if isinstance(file_path, str):
         file_path = safe_unicode(file_path)
 
     # TODO: johbo: Is this always a url like path, or is this operating
     # system dependent?
     path_segments = file_path.split('/')
 
     repo_name_html = escape(repo_name)
     if len(path_segments) == 1 and path_segments[0] == '':
         url_segments = [repo_name_html]
     else:
         url_segments = [
             link_to(
                 repo_name_html,
                 url('files_home',
                     repo_name=repo_name,
                     revision=commit_id,
                     f_path=''),
                 class_='pjax-link')]
 
     last_cnt = len(path_segments) - 1
     for cnt, segment in enumerate(path_segments):
         if not segment:
             continue
         segment_html = escape(segment)
 
         if cnt != last_cnt:
             url_segments.append(
                 link_to(
                     segment_html,
                     url('files_home',
                         repo_name=repo_name,
                         revision=commit_id,
                         f_path='/'.join(path_segments[:cnt + 1])),
                     class_='pjax-link'))
         else:
             url_segments.append(segment_html)
 
     return literal('/'.join(url_segments))
 
 
 class CodeHtmlFormatter(HtmlFormatter):
     """
     My code Html Formatter for source codes
     """
 
     def wrap(self, source, outfile):
         return self._wrap_div(self._wrap_pre(self._wrap_code(source)))
 
     def _wrap_code(self, source):
         for cnt, it in enumerate(source):
             i, t = it
             t = '<div id="L%s">%s</div>' % (cnt + 1, t)
             yield i, t
 
     def _wrap_tablelinenos(self, inner):
         dummyoutfile = StringIO.StringIO()
         lncount = 0
         for t, line in inner:
             if t:
                 lncount += 1
             dummyoutfile.write(line)
 
         fl = self.linenostart
         mw = len(str(lncount + fl - 1))
         sp = self.linenospecial
         st = self.linenostep
         la = self.lineanchors
         aln = self.anchorlinenos
         nocls = self.noclasses
         if sp:
             lines = []
 
             for i in range(fl, fl + lncount):
                 if i % st == 0:
                     if i % sp == 0:
                         if aln:
                             lines.append('<a href="#%s%d" class="special">%*d</a>' %
                                          (la, i, mw, i))
                         else:
                             lines.append('<span class="special">%*d</span>' % (mw, i))
                     else:
                         if aln:
                             lines.append('<a href="#%s%d">%*d</a>' % (la, i, mw, i))
                         else:
                             lines.append('%*d' % (mw, i))
                 else:
                     lines.append('')
             ls = '\n'.join(lines)
         else:
             lines = []
             for i in range(fl, fl + lncount):
                 if i % st == 0:
                     if aln:
                         lines.append('<a href="#%s%d">%*d</a>' % (la, i, mw, i))
                     else:
                         lines.append('%*d' % (mw, i))
                 else:
                     lines.append('')
             ls = '\n'.join(lines)
 
         # in case you wonder about the seemingly redundant <div> here: since the
         # content in the other cell also is wrapped in a div, some browsers in
         # some configurations seem to mess up the formatting...
         if nocls:
             yield 0, ('<table class="%stable">' % self.cssclass +
                       '<tr><td><div class="linenodiv" '
                       'style="background-color: #f0f0f0; padding-right: 10px">'
                       '<pre style="line-height: 125%">' +
                       ls + '</pre></div></td><td id="hlcode" class="code">')
         else:
             yield 0, ('<table class="%stable">' % self.cssclass +
                       '<tr><td class="linenos"><div class="linenodiv"><pre>' +
                       ls + '</pre></div></td><td id="hlcode" class="code">')
         yield 0, dummyoutfile.getvalue()
         yield 0, '</td></tr></table>'
 
 
+class SearchContentCodeHtmlFormatter(CodeHtmlFormatter):
+    def __init__(self, **kw):
+        # only show these line numbers if set
+        self.only_lines = kw.pop('only_line_numbers', [])
+        self.query_terms = kw.pop('query_terms', [])
+        self.max_lines = kw.pop('max_lines', 5)
+        self.line_context = kw.pop('line_context', 3)
+        self.url = kw.pop('url', None)
+
+        super(CodeHtmlFormatter, self).__init__(**kw)
+
+    def _wrap_code(self, source):
+        for cnt, it in enumerate(source):
+            i, t = it
+            t = '<pre>%s</pre>' % t
+            yield i, t
+
+    def _wrap_tablelinenos(self, inner):
+        yield 0, '<table class="code-highlight %stable">' % self.cssclass
+
+        last_shown_line_number = 0
+        current_line_number = 1
+
+        for t, line in inner:
+            if not t:
+                yield t, line
+                continue
+
+            if current_line_number in self.only_lines:
+                if last_shown_line_number + 1 != current_line_number:
+                    yield 0, '<tr>'
+                    yield 0, '<td class="line">...</td>'
+                    yield 0, '<td id="hlcode" class="code"></td>'
+                    yield 0, '</tr>'
+
+                yield 0, '<tr>'
+                if self.url:
+                    yield 0, '<td class="line"><a href="%s#L%i">%i</a></td>' % (
+                        self.url, current_line_number, current_line_number)
+                else:
+                    yield 0, '<td class="line"><a href="">%i</a></td>' % (
+                        current_line_number)
+                yield 0, '<td id="hlcode" class="code">' + line + '</td>'
+                yield 0, '</tr>'
+
+                last_shown_line_number = current_line_number
+
+            current_line_number += 1
+
+
+        yield 0, '</table>'
+
+
+def extract_phrases(text_query):
+    """
+    Extracts phrases from search term string making sure phrases
+    contained in double quotes are kept together - and discarding empty values
+    or fully whitespace values eg.
+
+    'some   text "a phrase" more' => ['some', 'text', 'a phrase', 'more']
+
+    """
+
+    in_phrase = False
+    buf = ''
+    phrases = []
+    for char in text_query:
+        if in_phrase:
+            if char == '"': # end phrase
+                phrases.append(buf)
+                buf = ''
+                in_phrase = False
+                continue
+            else:
+                buf += char
+                continue
+        else:
+            if char == '"': # start phrase
+                in_phrase = True
+                phrases.append(buf)
+                buf = ''
+                continue
+            elif char == ' ':
+                phrases.append(buf)
+                buf = ''
+                continue
+            else:
+                buf += char
+
+    phrases.append(buf)
+    phrases = [phrase.strip() for phrase in phrases if phrase.strip()]
+    return phrases
+
+
+def get_matching_offsets(text, phrases):
+    """
+    Returns a list of string offsets in `text` that the list of `terms` match
+
+    >>> get_matching_offsets('some text here', ['some', 'here'])
+    [(0, 4), (10, 14)]
+
+    """
+    offsets = []
+    for phrase in phrases:
+        for match in re.finditer(phrase, text):
+            offsets.append((match.start(), match.end()))
+
+    return offsets
+
+
+def normalize_text_for_matching(x):
+    """
+    Replaces all non alnum characters to spaces and lower cases the string,
+    useful for comparing two text strings without punctuation
+    """
+    return re.sub(r'[^\w]', ' ', x.lower())
+
+
+def get_matching_line_offsets(lines, terms):
+    """ Return a set of `lines` indices (starting from 1) matching a
+    text search query, along with `context` lines above/below matching lines
+
+    :param lines: list of strings representing lines
+    :param terms: search term string to match in lines eg. 'some text'
+    :param context: number of lines above/below a matching line to add to result
+    :param max_lines: cut off for lines of interest
+     eg.
+
+    >>> get_matching_line_offsets('''
+words words words
+words words words
+some text some
+words words words
+words words words
+text here what
+''', 'text', context=1)
+    {3: [(5, 9)], 6: [(0, 4)]]
+    """
+    matching_lines = {}
+    phrases = [normalize_text_for_matching(phrase)
+               for phrase in extract_phrases(terms)]
+
+    for line_index, line in enumerate(lines, start=1):
+        match_offsets = get_matching_offsets(
+            normalize_text_for_matching(line), phrases)
+        if match_offsets:
+            matching_lines[line_index] = match_offsets
+
+    return matching_lines
+
+def get_lexer_safe(mimetype=None, filepath=None):
+    """
+    Tries to return a relevant pygments lexer using mimetype/filepath name,
+    defaulting to plain text if none could be found
+    """
+    lexer = None
+    try:
+        if mimetype:
+            lexer = get_lexer_for_mimetype(mimetype)
+        if not lexer:
+            lexer = get_lexer_for_filename(path)
+    except pygments.util.ClassNotFound:
+        pass
+
+    if not lexer:
+        lexer = get_lexer_by_name('text')
+
+    return lexer
+
+
 def pygmentize(filenode, **kwargs):
     """
     pygmentize function using pygments
 
     :param filenode:
     """
     lexer = get_custom_lexer(filenode.extension) or filenode.lexer
     return literal(code_highlight(filenode.content, lexer,
                                   CodeHtmlFormatter(**kwargs)))
 
 
 def pygmentize_annotation(repo_name, filenode, **kwargs):
     """
     pygmentize function for annotation
 
     :param filenode:
     """
 
     color_dict = {}
 
     def gen_color(n=10000):
         """generator for getting n of evenly distributed colors using
         hsv color and golden ratio. It always return same order of colors
 
         :returns: RGB tuple
         """
 
         def hsv_to_rgb(h, s, v):
             if s == 0.0:
                 return v, v, v
             i = int(h * 6.0)  # XXX assume int() truncates!
             f = (h * 6.0) - i
             p = v * (1.0 - s)
             q = v * (1.0 - s * f)
             t = v * (1.0 - s * (1.0 - f))
             i = i % 6
             if i == 0:
                 return v, t, p
             if i == 1:
                 return q, v, p
             if i == 2:
                 return p, v, t
             if i == 3:
                 return p, q, v
             if i == 4:
                 return t, p, v
             if i == 5:
                 return v, p, q
 
         golden_ratio = 0.618033988749895
         h = 0.22717784590367374
 
         for _ in xrange(n):
             h += golden_ratio
             h %= 1
             HSV_tuple = [h, 0.95, 0.95]
             RGB_tuple = hsv_to_rgb(*HSV_tuple)
             yield map(lambda x: str(int(x * 256)), RGB_tuple)
 
     cgenerator = gen_color()
 
     def get_color_string(commit_id):
         if commit_id in color_dict:
             col = color_dict[commit_id]
         else:
             col = color_dict[commit_id] = cgenerator.next()
         return "color: rgb(%s)! important;" % (', '.join(col))
 
     def url_func(repo_name):
 
         def _url_func(commit):
             author = commit.author
             date = commit.date
             message = tooltip(commit.message)
 
             tooltip_html = ("<div style='font-size:0.8em'><b>Author:</b>"
                             " %s<br/><b>Date:</b> %s</b><br/><b>Message:"
                             "</b> %s<br/></div>")
 
             tooltip_html = tooltip_html % (author, date, message)
             lnk_format = '%5s:%s' % ('r%s' % commit.idx, commit.short_id)
             uri = link_to(
                 lnk_format,
                 url('changeset_home', repo_name=repo_name,
                     revision=commit.raw_id),
                 style=get_color_string(commit.raw_id),
                 class_='tooltip',
                 title=tooltip_html
             )
 
             uri += '\n'
             return uri
         return _url_func
 
     return literal(annotate_highlight(filenode, url_func(repo_name), **kwargs))
 
 
 def is_following_repo(repo_name, user_id):
     from rhodecode.model.scm import ScmModel
     return ScmModel().is_following_repo(repo_name, user_id)
 
 
 class _Message(object):
     """A message returned by ``Flash.pop_messages()``.
 
     Converting the message to a string returns the message text. Instances
     also have the following attributes:
 
     * ``message``: the message text.
     * ``category``: the category specified when the message was created.
     """
 
     def __init__(self, category, message):
         self.category = category
         self.message = message
 
     def __str__(self):
         return self.message
 
     __unicode__ = __str__
 
     def __html__(self):
         return escape(safe_unicode(self.message))
 
 
 class Flash(_Flash):
 
     def pop_messages(self):
         """Return all accumulated messages and delete them from the session.
 
         The return value is a list of ``Message`` objects.
         """
         from pylons import session
 
         messages = []
 
         # Pop the 'old' pylons flash messages. They are tuples of the form
         # (category, message)
         for cat, msg in session.pop(self.session_key, []):
             messages.append(_Message(cat, msg))
 
         # Pop the 'new' pyramid flash messages for each category as list
         # of strings.
         for cat in self.categories:
             for msg in session.pop_flash(queue=cat):
                 messages.append(_Message(cat, msg))
         # Map messages from the default queue to the 'notice' category.
         for msg in session.pop_flash():
             messages.append(_Message('notice', msg))
 
         session.save()
         return messages
 
 flash = Flash()
 
 #==============================================================================
 # SCM FILTERS available via h.
 #==============================================================================
 from rhodecode.lib.vcs.utils import author_name, author_email
 from rhodecode.lib.utils2 import credentials_filter, age as _age
 from rhodecode.model.db import User, ChangesetStatus
 
 age = _age
 capitalize = lambda x: x.capitalize()
 email = author_email
 short_id = lambda x: x[:12]
 hide_credentials = lambda x: ''.join(credentials_filter(x))
 
 
 def age_component(datetime_iso, value=None):
     title = value or format_date(datetime_iso)
 
     # detect if we have a timezone info, if not assume UTC
     if isinstance(datetime_iso, datetime) and not datetime_iso.tzinfo:
         tzinfo = '+00:00'
 
     return literal(
         '<time class="timeago tooltip" '
         'title="{1}" datetime="{0}{2}">{1}</time>'.format(
             datetime_iso, title, tzinfo))
 
 
 def _shorten_commit_id(commit_id):
     from rhodecode import CONFIG
     def_len = safe_int(CONFIG.get('rhodecode_show_sha_length', 12))
     return commit_id[:def_len]
 
 
 def get_repo_id_from_name(repo_name):
     repo = get_by_repo_name(repo_name)
     return repo.repo_id
 
 
 def show_id(commit):
     """
     Configurable function that shows ID
     by default it's r123:fffeeefffeee
 
     :param commit: commit instance
     """
     from rhodecode import CONFIG
     show_idx = str2bool(CONFIG.get('rhodecode_show_revision_number', True))
 
     raw_id = _shorten_commit_id(commit.raw_id)
     if show_idx:
         return 'r%s:%s' % (commit.idx, raw_id)
     else:
         return '%s' % (raw_id, )
 
 
 def format_date(date):
     """
     use a standardized formatting for dates used in RhodeCode
 
     :param date: date/datetime object
     :return: formatted date
     """
 
     if date:
         _fmt = "%a, %d %b %Y %H:%M:%S"
         return safe_unicode(date.strftime(_fmt))
 
     return u""
 
 
 class _RepoChecker(object):
 
     def __init__(self, backend_alias):
         self._backend_alias = backend_alias
 
     def __call__(self, repository):
         if hasattr(repository, 'alias'):
             _type = repository.alias
         elif hasattr(repository, 'repo_type'):
             _type = repository.repo_type
         else:
             _type = repository
         return _type == self._backend_alias
 
 is_git = _RepoChecker('git')
 is_hg = _RepoChecker('hg')
 is_svn = _RepoChecker('svn')
 
 
 def get_repo_type_by_name(repo_name):
     repo = Repository.get_by_repo_name(repo_name)
     return repo.repo_type
 
 
 def is_svn_without_proxy(repository):
     from rhodecode import CONFIG
     if is_svn(repository):
         if not CONFIG.get('rhodecode_proxy_subversion_http_requests', False):
             return True
     return False
 
 
 def email_or_none(author):
     # extract email from the commit string
     _email = author_email(author)
     if _email != '':
         # check it against RhodeCode database, and use the MAIN email for this
         # user
         user = User.get_by_email(_email, case_insensitive=True, cache=True)
         if user is not None:
             return user.email
         return _email
 
     # See if it contains a username we can get an email from
     user = User.get_by_username(author_name(author), case_insensitive=True,
                                 cache=True)
     if user is not None:
         return user.email
 
     # No valid email, not a valid user in the system, none!
     return None
 
 
 def discover_user(author):
     # if author is already an instance use it for extraction
     if isinstance(author, User):
         return author
 
     # Valid email in the attribute passed, see if they're in the system
     _email = email(author)
     if _email != '':
         user = User.get_by_email(_email, case_insensitive=True, cache=True)
         if user is not None:
             return user
 
     # Maybe it's a username?
     _author = author_name(author)
     user = User.get_by_username(_author, case_insensitive=True,
                                 cache=True)
     if user is not None:
         return user
 
     return None
 
 
 def link_to_user(author, length=0, **kwargs):
     user = discover_user(author)
     display_person = person(author, 'username_or_name_or_email')
     if length:
         display_person = shorter(display_person, length)
 
     if user:
         return link_to(
             escape(display_person),
             url('user_profile', username=user.username),
             **kwargs)
     else:
         return escape(display_person)
 
 
 def person(author, show_attr="username_and_name"):
     # attr to return from fetched user
     person_getter = lambda usr: getattr(usr, show_attr)
     user = discover_user(author)
     if user:
         return person_getter(user)
     else:
         _author = author_name(author)
         _email = email(author)
         return _author or _email
 
 
 def person_by_id(id_, show_attr="username_and_name"):
     # attr to return from fetched user
     person_getter = lambda usr: getattr(usr, show_attr)
 
     #maybe it's an ID ?
     if str(id_).isdigit() or isinstance(id_, int):
         id_ = int(id_)
         user = User.get(id_)
         if user is not None:
             return person_getter(user)
     return id_
 
 
 def gravatar_with_user(author):
     from rhodecode.lib.utils import PartialRenderer
     _render = PartialRenderer('base/base.html')
     return _render('gravatar_with_user', author)
 
 
 def desc_stylize(value):
     """
     converts tags from value into html equivalent
 
     :param value:
     """
     if not value:
         return ''
 
     value = re.sub(r'\[see\ \=\>\ *([a-zA-Z0-9\/\=\?\&\ \:\/\.\-]*)\]',
                    '<div class="metatag" tag="see">see =&gt; \\1 </div>', value)
     value = re.sub(r'\[license\ \=\>\ *([a-zA-Z0-9\/\=\?\&\ \:\/\.\-]*)\]',
                    '<div class="metatag" tag="license"><a href="http:\/\/www.opensource.org/licenses/\\1">\\1</a></div>', value)
     value = re.sub(r'\[(requires|recommends|conflicts|base)\ \=\>\ *([a-zA-Z0-9\-\/]*)\]',
                    '<div class="metatag" tag="\\1">\\1 =&gt; <a href="/\\2">\\2</a></div>', value)
     value = re.sub(r'\[(lang|language)\ \=\>\ *([a-zA-Z\-\/\#\+]*)\]',
                    '<div class="metatag" tag="lang">\\2</div>', value)
     value = re.sub(r'\[([a-z]+)\]',
                   '<div class="metatag" tag="\\1">\\1</div>', value)
 
     return value
 
 
 def escaped_stylize(value):
     """
     converts tags from value into html equivalent, but escaping its value first
     """
     if not value:
         return ''
 
     # Using default webhelper escape method, but has to force it as a
     # plain unicode instead of a markup tag to be used in regex expressions
     value = unicode(escape(safe_unicode(value)))
 
     value = re.sub(r'\[see\ \=\&gt;\ *([a-zA-Z0-9\/\=\?\&amp;\ \:\/\.\-]*)\]',
                    '<div class="metatag" tag="see">see =&gt; \\1 </div>', value)
     value = re.sub(r'\[license\ \=\&gt;\ *([a-zA-Z0-9\/\=\?\&amp;\ \:\/\.\-]*)\]',
                    '<div class="metatag" tag="license"><a href="http:\/\/www.opensource.org/licenses/\\1">\\1</a></div>', value)
     value = re.sub(r'\[(requires|recommends|conflicts|base)\ \=\&gt;\ *([a-zA-Z0-9\-\/]*)\]',
                    '<div class="metatag" tag="\\1">\\1 =&gt; <a href="/\\2">\\2</a></div>', value)
     value = re.sub(r'\[(lang|language)\ \=\&gt;\ *([a-zA-Z\-\/\#\+]*)\]',
                    '<div class="metatag" tag="lang">\\2</div>', value)
     value = re.sub(r'\[([a-z]+)\]',
                   '<div class="metatag" tag="\\1">\\1</div>', value)
 
     return value
 
 
 def bool2icon(value):
     """
     Returns boolean value of a given value, represented as html element with
     classes that will represent icons
 
     :param value: given value to convert to html node
     """
 
     if value:  # does bool conversion
         return HTML.tag('i', class_="icon-true")
     else:  # not true as bool
         return HTML.tag('i', class_="icon-false")
 
 
 #==============================================================================
 # PERMS
 #==============================================================================
 from rhodecode.lib.auth import HasPermissionAny, HasPermissionAll, \
 HasRepoPermissionAny, HasRepoPermissionAll, HasRepoGroupPermissionAll, \
 HasRepoGroupPermissionAny, HasRepoPermissionAnyApi, get_csrf_token
 
 
 #==============================================================================
 # GRAVATAR URL
 #==============================================================================
 class InitialsGravatar(object):
     def __init__(self, email_address, first_name, last_name, size=30,
                  background=None, text_color='#fff'):
         self.size = size
         self.first_name = first_name
         self.last_name = last_name
         self.email_address = email_address
         self.background = background or self.str2color(email_address)
         self.text_color = text_color
 
     def get_color_bank(self):
         """
         returns a predefined list of colors that gravatars can use.
         Those are randomized distinct colors that guarantee readability and
         uniqueness.
 
         generated with: http://phrogz.net/css/distinct-colors.html
         """
         return [
             '#bf3030', '#a67f53', '#00ff00', '#5989b3', '#392040', '#d90000',
             '#402910', '#204020', '#79baf2', '#a700b3', '#bf6060', '#7f5320',
             '#008000', '#003059', '#ee00ff', '#ff0000', '#8c4b00', '#007300',
             '#005fb3', '#de73e6', '#ff4040', '#ffaa00', '#3df255', '#203140',
             '#47004d', '#591616', '#664400', '#59b365', '#0d2133', '#83008c',
             '#592d2d', '#bf9f60', '#73e682', '#1d3f73', '#73006b', '#402020',
             '#b2862d', '#397341', '#597db3', '#e600d6', '#a60000', '#736039',
             '#00b318', '#79aaf2', '#330d30', '#ff8080', '#403010', '#16591f',
             '#002459', '#8c4688', '#e50000', '#ffbf40', '#00732e', '#102340',
             '#bf60ac', '#8c4646', '#cc8800', '#00a642', '#1d3473', '#b32d98',
             '#660e00', '#ffd580', '#80ffb2', '#7391e6', '#733967', '#d97b6c',
             '#8c5e00', '#59b389', '#3967e6', '#590047', '#73281d', '#665200',
             '#00e67a', '#2d50b3', '#8c2377', '#734139', '#b2982d', '#16593a',
             '#001859', '#ff00aa', '#a65e53', '#ffcc00', '#0d3321', '#2d3959',
             '#731d56', '#401610', '#4c3d00', '#468c6c', '#002ca6', '#d936a3',
             '#d94c36', '#403920', '#36d9a3', '#0d1733', '#592d4a', '#993626',
             '#cca300', '#00734d', '#46598c', '#8c005e', '#7f1100', '#8c7000',
             '#00a66f', '#7382e6', '#b32d74', '#d9896c', '#ffe680', '#1d7362',
             '#364cd9', '#73003d', '#d93a00', '#998a4d', '#59b3a1', '#5965b3',
             '#e5007a', '#73341d', '#665f00', '#00b38f', '#0018b3', '#59163a',
             '#b2502d', '#bfb960', '#00ffcc', '#23318c', '#a6537f', '#734939',
             '#b2a700', '#104036', '#3d3df2', '#402031', '#e56739', '#736f39',
             '#79f2ea', '#000059', '#401029', '#4c1400', '#ffee00', '#005953',
             '#101040', '#990052', '#402820', '#403d10', '#00ffee', '#0000d9',
             '#ff80c4', '#a66953', '#eeff00', '#00ccbe', '#8080ff', '#e673a1',
             '#a62c00', '#474d00', '#1a3331', '#46468c', '#733950', '#662900',
             '#858c23', '#238c85', '#0f0073', '#b20047', '#d9986c', '#becc00',
             '#396f73', '#281d73', '#ff0066', '#ff6600', '#dee673', '#59adb3',
             '#6559b3', '#590024', '#b2622d', '#98b32d', '#36ced9', '#332d59',
             '#40001a', '#733f1d', '#526600', '#005359', '#242040', '#bf6079',
             '#735039', '#cef23d', '#007780', '#5630bf', '#66001b', '#b24700',
             '#acbf60', '#1d6273', '#25008c', '#731d34', '#a67453', '#50592d',
             '#00ccff', '#6600ff', '#ff0044', '#4c1f00', '#8a994d', '#79daf2',
             '#a173e6', '#d93662', '#402310', '#aaff00', '#2d98b3', '#8c40ff',
             '#592d39', '#ff8c40', '#354020', '#103640', '#1a0040', '#331a20',
             '#331400', '#334d00', '#1d5673', '#583973', '#7f0022', '#4c3626',
             '#88cc00', '#36a3d9', '#3d0073', '#d9364c', '#33241a', '#698c23',
             '#5995b3', '#300059', '#e57382', '#7f3300', '#366600', '#00aaff',
             '#3a1659', '#733941', '#663600', '#74b32d', '#003c59', '#7f53a6',
             '#73000f', '#ff8800', '#baf279', '#79caf2', '#291040', '#a6293a',
             '#b2742d', '#587339', '#0077b3', '#632699', '#400009', '#d9a66c',
             '#294010', '#2d4a59', '#aa00ff', '#4c131b', '#b25f00', '#5ce600',
             '#267399', '#a336d9', '#990014', '#664e33', '#86bf60', '#0088ff',
             '#7700b3', '#593a16', '#073300', '#1d4b73', '#ac60bf', '#e59539',
             '#4f8c46', '#368dd9', '#5c0073'
         ]
 
     def rgb_to_hex_color(self, rgb_tuple):
         """
         Converts an rgb_tuple passed to an hex color.
 
         :param rgb_tuple: tuple with 3 ints represents rgb color space
         """
         return '#' + ("".join(map(chr, rgb_tuple)).encode('hex'))
 
     def email_to_int_list(self, email_str):
         """
         Get every byte of the hex digest value of email and turn it to integer.
         It's going to be always between 0-255
         """
         digest = md5_safe(email_str.lower())
         return [int(digest[i * 2:i * 2 + 2], 16) for i in range(16)]
 
     def pick_color_bank_index(self, email_str, color_bank):
         return self.email_to_int_list(email_str)[0] % len(color_bank)
 
     def str2color(self, email_str):
         """
         Tries to map in a stable algorithm an email to color
 
         :param email_str:
         """
         color_bank = self.get_color_bank()
         # pick position (module it's length so we always find it in the
         # bank even if it's smaller than 256 values
         pos = self.pick_color_bank_index(email_str, color_bank)
         return color_bank[pos]
 
     def normalize_email(self, email_address):
         import unicodedata
         # default host used to fill in the fake/missing email
         default_host = u'localhost'
 
         if not email_address:
             email_address = u'%s@%s' % (User.DEFAULT_USER, default_host)
 
         email_address = safe_unicode(email_address)
 
         if u'@' not in email_address:
             email_address = u'%s@%s' % (email_address, default_host)
 
         if email_address.endswith(u'@'):
             email_address = u'%s%s' % (email_address, default_host)
 
         email_address = unicodedata.normalize('NFKD', email_address)\
             .encode('ascii', 'ignore')
         return email_address
 
     def get_initials(self):
         """
         Returns 2 letter initials calculated based on the input.
         The algorithm picks first given email address, and takes first letter
         of part before @, and then the first letter of server name. In case
         the part before @ is in a format of `somestring.somestring2` it replaces
         the server letter with first letter of somestring2
 
         In case function was initialized with both first and lastname, this
         overrides the extraction from email by first letter of the first and
         last name. We add special logic to that functionality, In case Full name
         is compound, like Guido Von Rossum, we use last part of the last name
         (Von Rossum) picking `R`.
 
         Function also normalizes the non-ascii characters to they ascii
         representation, eg Ą => A
         """
         import unicodedata
         # replace non-ascii to ascii
         first_name = unicodedata.normalize(
             'NFKD', safe_unicode(self.first_name)).encode('ascii', 'ignore')
         last_name = unicodedata.normalize(
             'NFKD', safe_unicode(self.last_name)).encode('ascii', 'ignore')
 
         # do NFKD encoding, and also make sure email has proper format
         email_address = self.normalize_email(self.email_address)
 
         # first push the email initials
         prefix, server = email_address.split('@', 1)
 
         # check if prefix is maybe a 'firstname.lastname' syntax
         _dot_split = prefix.rsplit('.', 1)
         if len(_dot_split) == 2:
             initials = [_dot_split[0][0], _dot_split[1][0]]
         else:
             initials = [prefix[0], server[0]]
 
         # then try to replace either firtname or lastname
         fn_letter = (first_name or " ")[0].strip()
         ln_letter = (last_name.split(' ', 1)[-1] or " ")[0].strip()
 
         if fn_letter:
             initials[0] = fn_letter
 
         if ln_letter:
             initials[1] = ln_letter
 
         return ''.join(initials).upper()
 
     def get_img_data_by_type(self, font_family, img_type):
         default_user = """
         <svg xmlns="http://www.w3.org/2000/svg"
         version="1.1" x="0px" y="0px" width="{size}" height="{size}"
         viewBox="-15 -10 439.165 429.164"
 
         xml:space="preserve"
         style="background:{background};" >
 
         <path d="M204.583,216.671c50.664,0,91.74-48.075,
                  91.74-107.378c0-82.237-41.074-107.377-91.74-107.377
                  c-50.668,0-91.74,25.14-91.74,107.377C112.844,
                  168.596,153.916,216.671,
                  204.583,216.671z" fill="{text_color}"/>
         <path d="M407.164,374.717L360.88,
                  270.454c-2.117-4.771-5.836-8.728-10.465-11.138l-71.83-37.392
                  c-1.584-0.823-3.502-0.663-4.926,0.415c-20.316,
                  15.366-44.203,23.488-69.076,23.488c-24.877,
                  0-48.762-8.122-69.078-23.488
                  c-1.428-1.078-3.346-1.238-4.93-0.415L58.75,
                  259.316c-4.631,2.41-8.346,6.365-10.465,11.138L2.001,374.717
                  c-3.191,7.188-2.537,15.412,1.75,22.005c4.285,
                  6.592,11.537,10.526,19.4,10.526h362.861c7.863,0,15.117-3.936,
                  19.402-10.527 C409.699,390.129,
                  410.355,381.902,407.164,374.717z" fill="{text_color}"/>
         </svg>""".format(
             size=self.size,
             background='#979797',  # @grey4
             text_color=self.text_color,
             font_family=font_family)
 
         return {
             "default_user": default_user
         }[img_type]
 
     def get_img_data(self, svg_type=None):
         """
         generates the svg metadata for image
         """
 
         font_family = ','.join([
             'proximanovaregular',
             'Proxima Nova Regular',
             'Proxima Nova',
             'Arial',
             'Lucida Grande',
             'sans-serif'
             ])
         if svg_type:
             return self.get_img_data_by_type(font_family, svg_type)
 
         initials = self.get_initials()
         img_data = """
         <svg xmlns="http://www.w3.org/2000/svg" pointer-events="none"
              width="{size}" height="{size}"
              style="width: 100%; height: 100%; background-color: {background}"
              viewBox="0 0 {size} {size}">
             <text text-anchor="middle" y="50%" x="50%" dy="0.35em"
                   pointer-events="auto" fill="{text_color}"
                   font-family="{font_family}"
                   style="font-weight: 400; font-size: {f_size}px;">{text}
             </text>
         </svg>""".format(
             size=self.size,
             f_size=self.size/1.85,  # scale the text inside the box nicely
             background=self.background,
             text_color=self.text_color,
             text=initials.upper(),
             font_family=font_family)
 
         return img_data
 
     def generate_svg(self, svg_type=None):
         img_data = self.get_img_data(svg_type)
         return "data:image/svg+xml;base64,%s" % img_data.encode('base64')
 
 
 def initials_gravatar(email_address, first_name, last_name, size=30):
     svg_type = None
     if email_address == User.DEFAULT_USER_EMAIL:
         svg_type = 'default_user'
     klass = InitialsGravatar(email_address, first_name, last_name, size)
     return klass.generate_svg(svg_type=svg_type)
 
 
 def gravatar_url(email_address, size=30):
     # doh, we need to re-import those to mock it later
     from pylons import tmpl_context as c
 
     _use_gravatar = c.visual.use_gravatar
     _gravatar_url = c.visual.gravatar_url or User.DEFAULT_GRAVATAR_URL
 
     email_address = email_address or User.DEFAULT_USER_EMAIL
     if isinstance(email_address, unicode):
         # hashlib crashes on unicode items
         email_address = safe_str(email_address)
 
     # empty email or default user
     if not email_address or email_address == User.DEFAULT_USER_EMAIL:
         return initials_gravatar(User.DEFAULT_USER_EMAIL, '', '', size=size)
 
     if _use_gravatar:
         # TODO: Disuse pyramid thread locals. Think about another solution to
         # get the host and schema here.
         request = get_current_request()
         tmpl = safe_str(_gravatar_url)
         tmpl = tmpl.replace('{email}', email_address)\
                    .replace('{md5email}', md5_safe(email_address.lower())) \
                    .replace('{netloc}', request.host)\
                    .replace('{scheme}', request.scheme)\
                    .replace('{size}', safe_str(size))
         return tmpl
     else:
         return initials_gravatar(email_address, '', '', size=size)
 
 
 class Page(_Page):
     """
     Custom pager to match rendering style with paginator
     """
 
     def _get_pos(self, cur_page, max_page, items):
         edge = (items / 2) + 1
         if (cur_page <= edge):
             radius = max(items / 2, items - cur_page)
         elif (max_page - cur_page) < edge:
             radius = (items - 1) - (max_page - cur_page)
         else:
             radius = items / 2
 
         left = max(1, (cur_page - (radius)))
         right = min(max_page, cur_page + (radius))
         return left, cur_page, right
 
     def _range(self, regexp_match):
         """
         Return range of linked pages (e.g. '1 2 [3] 4 5 6 7 8').
 
         Arguments:
 
         regexp_match
             A "re" (regular expressions) match object containing the
             radius of linked pages around the current page in
             regexp_match.group(1) as a string
 
         This function is supposed to be called as a callable in
         re.sub.
 
         """
         radius = int(regexp_match.group(1))
 
         # Compute the first and last page number within the radius
         # e.g. '1 .. 5 6 [7] 8 9 .. 12'
         # -> leftmost_page  = 5
         # -> rightmost_page = 9
         leftmost_page, _cur, rightmost_page = self._get_pos(self.page,
                                                             self.last_page,
                                                             (radius * 2) + 1)
         nav_items = []
 
         # Create a link to the first page (unless we are on the first page
         # or there would be no need to insert '..' spacers)
         if self.page != self.first_page and self.first_page < leftmost_page:
             nav_items.append(self._pagerlink(self.first_page, self.first_page))
 
         # Insert dots if there are pages between the first page
         # and the currently displayed page range
         if leftmost_page - self.first_page > 1:
             # Wrap in a SPAN tag if nolink_attr is set
             text = '..'
             if self.dotdot_attr:
                 text = HTML.span(c=text, **self.dotdot_attr)
             nav_items.append(text)
 
         for thispage in xrange(leftmost_page, rightmost_page + 1):
             # Hilight the current page number and do not use a link
             if thispage == self.page:
                 text = '%s' % (thispage,)
                 # Wrap in a SPAN tag if nolink_attr is set
                 if self.curpage_attr:
                     text = HTML.span(c=text, **self.curpage_attr)
                 nav_items.append(text)
             # Otherwise create just a link to that page
             else:
                 text = '%s' % (thispage,)
                 nav_items.append(self._pagerlink(thispage, text))
 
         # Insert dots if there are pages between the displayed
         # page numbers and the end of the page range
         if self.last_page - rightmost_page > 1:
             text = '..'
             # Wrap in a SPAN tag if nolink_attr is set
             if self.dotdot_attr:
                 text = HTML.span(c=text, **self.dotdot_attr)
             nav_items.append(text)
 
         # Create a link to the very last page (unless we are on the last
         # page or there would be no need to insert '..' spacers)
         if self.page != self.last_page and rightmost_page < self.last_page:
             nav_items.append(self._pagerlink(self.last_page, self.last_page))
 
         ## prerender links
         #_page_link = url.current()
         #nav_items.append(literal('<link rel="prerender" href="%s?page=%s">' % (_page_link, str(int(self.page)+1))))
         #nav_items.append(literal('<link rel="prefetch" href="%s?page=%s">' % (_page_link, str(int(self.page)+1))))
         return self.separator.join(nav_items)
 
     def pager(self, format='~2~', page_param='page', partial_param='partial',
         show_if_single_page=False, separator=' ', onclick=None,
         symbol_first='<<', symbol_last='>>',
         symbol_previous='<', symbol_next='>',
         link_attr={'class': 'pager_link', 'rel': 'prerender'},
         curpage_attr={'class': 'pager_curpage'},
         dotdot_attr={'class': 'pager_dotdot'}, **kwargs):
 
         self.curpage_attr = curpage_attr
         self.separator = separator
         self.pager_kwargs = kwargs
         self.page_param = page_param
         self.partial_param = partial_param
         self.onclick = onclick
         self.link_attr = link_attr
         self.dotdot_attr = dotdot_attr
 
         # Don't show navigator if there is no more than one page
         if self.page_count == 0 or (self.page_count == 1 and not show_if_single_page):
             return ''
 
         from string import Template
         # Replace ~...~ in token format by range of pages
         result = re.sub(r'~(\d+)~', self._range, format)
 
         # Interpolate '%' variables
         result = Template(result).safe_substitute({
             'first_page': self.first_page,
             'last_page': self.last_page,
             'page': self.page,
             'page_count': self.page_count,
             'items_per_page': self.items_per_page,
             'first_item': self.first_item,
             'last_item': self.last_item,
             'item_count': self.item_count,
             'link_first': self.page > self.first_page and \
                     self._pagerlink(self.first_page, symbol_first) or '',
             'link_last': self.page < self.last_page and \
                     self._pagerlink(self.last_page, symbol_last) or '',
             'link_previous': self.previous_page and \
                     self._pagerlink(self.previous_page, symbol_previous) \
                     or HTML.span(symbol_previous, class_="pg-previous disabled"),
             'link_next': self.next_page and \
                     self._pagerlink(self.next_page, symbol_next) \
                     or HTML.span(symbol_next, class_="pg-next disabled")
         })
 
         return literal(result)
 
 
 #==============================================================================
 # REPO PAGER, PAGER FOR REPOSITORY
 #==============================================================================
 class RepoPage(Page):
 
     def __init__(self, collection, page=1, items_per_page=20,
                  item_count=None, url=None, **kwargs):
 
         """Create a "RepoPage" instance. special pager for paging
         repository
         """
         self._url_generator = url
 
         # Safe the kwargs class-wide so they can be used in the pager() method
         self.kwargs = kwargs
 
         # Save a reference to the collection
         self.original_collection = collection
 
         self.collection = collection
 
         # The self.page is the number of the current page.
         # The first page has the number 1!
         try:
             self.page = int(page)  # make it int() if we get it as a string
         except (ValueError, TypeError):
             self.page = 1
 
         self.items_per_page = items_per_page
 
         # Unless the user tells us how many items the collections has
         # we calculate that ourselves.
         if item_count is not None:
             self.item_count = item_count
         else:
             self.item_count = len(self.collection)
 
         # Compute the number of the first and last available page
         if self.item_count > 0:
             self.first_page = 1
             self.page_count = int(math.ceil(float(self.item_count) /
                                             self.items_per_page))
             self.last_page = self.first_page + self.page_count - 1
 
             # Make sure that the requested page number is the range of
             # valid pages
             if self.page > self.last_page:
                 self.page = self.last_page
             elif self.page < self.first_page:
                 self.page = self.first_page
 
             # Note: the number of items on this page can be less than
             #       items_per_page if the last page is not full
             self.first_item = max(0, (self.item_count) - (self.page *
                                                           items_per_page))
             self.last_item = ((self.item_count - 1) - items_per_page *
                               (self.page - 1))
 
             self.items = list(self.collection[self.first_item:self.last_item + 1])
 
             # Links to previous and next page
             if self.page > self.first_page:
                 self.previous_page = self.page - 1
             else:
                 self.previous_page = None
 
             if self.page < self.last_page:
                 self.next_page = self.page + 1
             else:
                 self.next_page = None
 
         # No items available
         else:
             self.first_page = None
             self.page_count = 0
             self.last_page = None
             self.first_item = None
             self.last_item = None
             self.previous_page = None
             self.next_page = None
             self.items = []
 
         # This is a subclass of the 'list' type. Initialise the list now.
         list.__init__(self, reversed(self.items))
 
 
 def changed_tooltip(nodes):
     """
     Generates a html string for changed nodes in commit page.
     It limits the output to 30 entries
 
     :param nodes: LazyNodesGenerator
     """
     if nodes:
         pref = ': <br/> '
         suf = ''
         if len(nodes) > 30:
             suf = '<br/>' + _(' and %s more') % (len(nodes) - 30)
         return literal(pref + '<br/> '.join([safe_unicode(x.path)
                                              for x in nodes[:30]]) + suf)
     else:
         return ': ' + _('No Files')
 
 
 def breadcrumb_repo_link(repo):
     """
     Makes a breadcrumbs path link to repo
 
     ex::
         group >> subgroup >> repo
 
     :param repo: a Repository instance
     """
 
     path = [
         link_to(group.name, url('repo_group_home', group_name=group.group_name))
         for group in repo.groups_with_parents
     ] + [
         link_to(repo.just_name, url('summary_home', repo_name=repo.repo_name))
     ]
 
     return literal(' &raquo; '.join(path))
 
 
 def format_byte_size_binary(file_size):
     """
     Formats file/folder sizes to standard.
     """
     formatted_size = format_byte_size(file_size, binary=True)
     return formatted_size
 
 
 def fancy_file_stats(stats):
     """
     Displays a fancy two colored bar for number of added/deleted
     lines of code on file
 
     :param stats: two element list of added/deleted lines of code
     """
     from rhodecode.lib.diffs import NEW_FILENODE, DEL_FILENODE, \
         MOD_FILENODE, RENAMED_FILENODE, CHMOD_FILENODE, BIN_FILENODE
 
     def cgen(l_type, a_v, d_v):
         mapping = {'tr': 'top-right-rounded-corner-mid',
                    'tl': 'top-left-rounded-corner-mid',
                    'br': 'bottom-right-rounded-corner-mid',
                    'bl': 'bottom-left-rounded-corner-mid'}
         map_getter = lambda x: mapping[x]
 
         if l_type == 'a' and d_v:
             #case when added and deleted are present
             return ' '.join(map(map_getter, ['tl', 'bl']))
 
         if l_type == 'a' and not d_v:
             return ' '.join(map(map_getter, ['tr', 'br', 'tl', 'bl']))
 
         if l_type == 'd' and a_v:
             return ' '.join(map(map_getter, ['tr', 'br']))
 
         if l_type == 'd' and not a_v:
             return ' '.join(map(map_getter, ['tr', 'br', 'tl', 'bl']))
 
     a, d = stats['added'], stats['deleted']
     width = 100
 
     if stats['binary']:  # binary operations like chmod/rename etc
         lbl = []
         bin_op = 0  # undefined
 
         # prefix with bin for binary files
         if BIN_FILENODE in stats['ops']:
             lbl += ['bin']
 
         if NEW_FILENODE in stats['ops']:
             lbl += [_('new file')]
             bin_op = NEW_FILENODE
         elif MOD_FILENODE in stats['ops']:
             lbl += [_('mod')]
             bin_op = MOD_FILENODE
         elif DEL_FILENODE in stats['ops']:
             lbl += [_('del')]
             bin_op = DEL_FILENODE
         elif RENAMED_FILENODE in stats['ops']:
             lbl += [_('rename')]
             bin_op = RENAMED_FILENODE
 
         # chmod can go with other operations, so we add a + to lbl if needed
         if CHMOD_FILENODE in stats['ops']:
             lbl += [_('chmod')]
             if bin_op == 0:
                 bin_op = CHMOD_FILENODE
 
         lbl = '+'.join(lbl)
         b_a = '<div class="bin bin%s %s" style="width:100%%">%s</div>' \
               % (bin_op, cgen('a', a_v='', d_v=0), lbl)
         b_d = '<div class="bin bin1" style="width:0%%"></div>'
         return literal('<div style="width:%spx">%s%s</div>' % (width, b_a, b_d))
 
     t = stats['added'] + stats['deleted']
     unit = float(width) / (t or 1)
 
     # needs > 9% of width to be visible or 0 to be hidden
     a_p = max(9, unit * a) if a > 0 else 0
     d_p = max(9, unit * d) if d > 0 else 0
     p_sum = a_p + d_p
 
     if p_sum > width:
         #adjust the percentage to be == 100% since we adjusted to 9
         if a_p > d_p:
             a_p = a_p - (p_sum - width)
         else:
             d_p = d_p - (p_sum - width)
 
     a_v = a if a > 0 else ''
     d_v = d if d > 0 else ''
 
     d_a = '<div class="added %s" style="width:%s%%">%s</div>' % (
         cgen('a', a_v, d_v), a_p, a_v
     )
     d_d = '<div class="deleted %s" style="width:%s%%">%s</div>' % (
         cgen('d', a_v, d_v), d_p, d_v
     )
     return literal('<div style="width:%spx">%s%s</div>' % (width, d_a, d_d))
 
 
 def urlify_text(text_, safe=True):
     """
     Extrac urls from text and make html links out of them
 
     :param text_:
     """
 
     url_pat = re.compile(r'''(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@#.&+]'''
                          '''|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)''')
 
     def url_func(match_obj):
         url_full = match_obj.groups()[0]
         return '<a href="%(url)s">%(url)s</a>' % ({'url': url_full})
     _newtext = url_pat.sub(url_func, text_)
     if safe:
         return literal(_newtext)
     return _newtext
 
 
 def urlify_commits(text_, repository):
     """
     Extract commit ids from text and make link from them
 
     :param text_:
     :param repository: repo name to build the URL with
     """
     from pylons import url  # doh, we need to re-import url to mock it later
     URL_PAT = re.compile(r'(^|\s)([0-9a-fA-F]{12,40})($|\s)')
 
     def url_func(match_obj):
         commit_id = match_obj.groups()[1]
         pref = match_obj.groups()[0]
         suf = match_obj.groups()[2]
 
         tmpl = (
             '%(pref)s<a class="%(cls)s" href="%(url)s">'
             '%(commit_id)s</a>%(suf)s'
         )
         return tmpl % {
             'pref': pref,
             'cls': 'revision-link',
             'url': url('changeset_home', repo_name=repository,
                        revision=commit_id),
             'commit_id': commit_id,
             'suf': suf
         }
 
     newtext = URL_PAT.sub(url_func, text_)
 
     return newtext
 
 
 def _process_url_func(match_obj, repo_name, uid, entry):
     pref = ''
     if match_obj.group().startswith(' '):
         pref = ' '
 
     issue_id = ''.join(match_obj.groups())
     tmpl = (
         '%(pref)s<a class="%(cls)s" href="%(url)s">'
         '%(issue-prefix)s%(id-repr)s'
         '</a>')
 
     (repo_name_cleaned,
      parent_group_name) = RepoGroupModel().\
         _get_group_name_and_parent(repo_name)
 
     # variables replacement
     named_vars = {
         'id': issue_id,
         'repo': repo_name,
         'repo_name': repo_name_cleaned,
         'group_name': parent_group_name
     }
     # named regex variables
     named_vars.update(match_obj.groupdict())
     _url = string.Template(entry['url']).safe_substitute(**named_vars)
 
     return tmpl % {
         'pref': pref,
         'cls': 'issue-tracker-link',
         'url': _url,
         'id-repr': issue_id,
         'issue-prefix': entry['pref'],
         'serv': entry['url'],
     }
 
 
 def process_patterns(text_string, repo_name, config):
     repo = None
     if repo_name:
         # Retrieving repo_name to avoid invalid repo_name to explode on
         # IssueTrackerSettingsModel but still passing invalid name further down
         repo = Repository.get_by_repo_name(repo_name)
 
     settings_model = IssueTrackerSettingsModel(repo=repo)
     active_entries = settings_model.get_settings()
 
     newtext = text_string
     for uid, entry in active_entries.items():
         url_func = partial(
             _process_url_func, repo_name=repo_name, entry=entry, uid=uid)
 
         log.debug('found issue tracker entry with uid %s' % (uid,))
 
         if not (entry['pat'] and entry['url']):
             log.debug('skipping due to missing data')
             continue
 
         log.debug('issue tracker entry: uid: `%s` PAT:%s URL:%s PREFIX:%s'
                   % (uid, entry['pat'], entry['url'], entry['pref']))
 
         try:
             pattern = re.compile(r'%s' % entry['pat'])
         except re.error:
             log.exception(
                 'issue tracker pattern: `%s` failed to compile',
                 entry['pat'])
             continue
 
         newtext = pattern.sub(url_func, newtext)
         log.debug('processed prefix:uid `%s`' % (uid,))
 
     return newtext
 
 
 def urlify_commit_message(commit_text, repository=None):
     """
     Parses given text message and makes proper links.
     issues are linked to given issue-server, and rest is a commit link
 
     :param commit_text:
     :param repository:
     """
     from pylons import url  # doh, we need to re-import url to mock it later
     from rhodecode import CONFIG
 
     def escaper(string):
         return string.replace('<', '&lt;').replace('>', '&gt;')
 
     newtext = escaper(commit_text)
     # urlify commits - extract commit ids and make link out of them, if we have
     # the scope of repository present.
     if repository:
         newtext = urlify_commits(newtext, repository)
 
     # extract http/https links and make them real urls
     newtext = urlify_text(newtext, safe=False)
 
     # process issue tracker patterns
     newtext = process_patterns(newtext, repository or '', CONFIG)
 
     return literal(newtext)
 
 
 def rst(source, mentions=False):
     return literal('<div class="rst-block">%s</div>' %
                    MarkupRenderer.rst(source, mentions=mentions))
 
 
 def markdown(source, mentions=False):
     return literal('<div class="markdown-block">%s</div>' %
                    MarkupRenderer.markdown(source, flavored=False,
                                            mentions=mentions))
 
 def renderer_from_filename(filename, exclude=None):
     from rhodecode.config.conf import MARKDOWN_EXTS, RST_EXTS
 
     def _filter(elements):
         if isinstance(exclude, (list, tuple)):
             return [x for x in elements if x not in exclude]
         return elements
 
     if filename.endswith(tuple(_filter([x[0] for x in MARKDOWN_EXTS if x[0]]))):
         return 'markdown'
     if filename.endswith(tuple(_filter([x[0] for x in RST_EXTS if x[0]]))):
         return 'rst'
 
 
 def render(source, renderer='rst', mentions=False):
     if renderer == 'rst':
         return rst(source, mentions=mentions)
     if renderer == 'markdown':
         return markdown(source, mentions=mentions)
 
 
 def commit_status(repo, commit_id):
     return ChangesetStatusModel().get_status(repo, commit_id)
 
 
 def commit_status_lbl(commit_status):
     return dict(ChangesetStatus.STATUSES).get(commit_status)
 
 
 def commit_time(repo_name, commit_id):
     repo = Repository.get_by_repo_name(repo_name)
     commit = repo.get_commit(commit_id=commit_id)
     return commit.date
 
 
 def get_permission_name(key):
     return dict(Permission.PERMS).get(key)
 
 
 def journal_filter_help():
     return _(
         'Example filter terms:\n' +
         '     repository:vcs\n' +
         '     username:marcin\n' +
         '     action:*push*\n' +
         '     ip:127.0.0.1\n' +
         '     date:20120101\n' +
         '     date:[20120101100000 TO 20120102]\n' +
         '\n' +
         'Generate wildcards using \'*\' character:\n' +
         '     "repository:vcs*" - search everything starting with \'vcs\'\n' +
         '     "repository:*vcs*" - search for repository containing \'vcs\'\n' +
         '\n' +
         'Optional AND / OR operators in queries\n' +
         '     "repository:vcs OR repository:test"\n' +
         '     "username:test AND repository:test*"\n'
     )
 
 
 def not_mapped_error(repo_name):
     flash(_('%s repository is not mapped to db perhaps'
             ' it was created or renamed from the filesystem'
             ' please run the application again'
             ' in order to rescan repositories') % repo_name, category='error')
 
 
 def ip_range(ip_addr):
     from rhodecode.model.db import UserIpMap
     s, e = UserIpMap._get_ip_range(ip_addr)
     return '%s - %s' % (s, e)
 
 
 def form(url, method='post', needs_csrf_token=True, **attrs):
     """Wrapper around webhelpers.tags.form to prevent CSRF attacks."""
     if method.lower() != 'get' and needs_csrf_token:
         raise Exception(
             'Forms to POST/PUT/DELETE endpoints should have (in general) a ' +
             'CSRF token. If the endpoint does not require such token you can ' +
             'explicitly set the parameter needs_csrf_token to false.')
 
     return wh_form(url, method=method, **attrs)
 
 
 def secure_form(url, method="POST", multipart=False, **attrs):
     """Start a form tag that points the action to an url. This
     form tag will also include the hidden field containing
     the auth token.
 
     The url options should be given either as a string, or as a
     ``url()`` function. The method for the form defaults to POST.
 
     Options:
 
     ``multipart``
         If set to True, the enctype is set to "multipart/form-data".
     ``method``
         The method to use when submitting the form, usually either
         "GET" or "POST". If "PUT", "DELETE", or another verb is used, a
         hidden input with name _method is added to simulate the verb
         over POST.
 
     """
     from webhelpers.pylonslib.secure_form import insecure_form
     from rhodecode.lib.auth import get_csrf_token, csrf_token_key
     form = insecure_form(url, method, multipart, **attrs)
     token = HTML.div(hidden(csrf_token_key, get_csrf_token()), style="display: none;")
     return literal("%s\n%s" % (form, token))
 
 def dropdownmenu(name, selected, options, enable_filter=False, **attrs):
     select_html = select(name, selected, options, **attrs)
     select2 = """
         <script>
             $(document).ready(function() {
                   $('#%s').select2({
                       containerCssClass: 'drop-menu',
                       dropdownCssClass: 'drop-menu-dropdown',
                       dropdownAutoWidth: true%s
                   });
             });
         </script>
     """
     filter_option = """,
                         minimumResultsForSearch: -1
     """
     input_id = attrs.get('id') or name
     filter_enabled = "" if enable_filter else filter_option
     select_script = literal(select2 % (input_id, filter_enabled))
 
     return literal(select_html+select_script)
 
 
 def get_visual_attr(tmpl_context_var, attr_name):
     """
     A safe way to get a variable from visual variable of template context
 
     :param tmpl_context_var: instance of tmpl_context, usually present as `c`
     :param attr_name: name of the attribute we fetch from the c.visual
     """
     visual = getattr(tmpl_context_var, 'visual', None)
     if not visual:
         return
     else:
         return getattr(visual, attr_name, None)
 
 
 def get_last_path_part(file_node):
     if not file_node.path:
         return u''
 
     path = safe_unicode(file_node.path.split('/')[-1])
     return u'../' + path
 
 
 def route_path(*args, **kwds):
     """
     Wrapper around pyramids `route_path` function. It is used to generate
     URLs from within pylons views or templates. This will be removed when
     pyramid migration if finished.
     """
     req = get_current_request()
     return req.route_path(*args, **kwds)
 
 
 def resource_path(*args, **kwds):
     """
     Wrapper around pyramids `route_path` function. It is used to generate
     URLs from within pylons views or templates. This will be removed when
     pyramid migration if finished.
     """
     req = get_current_request()
     return req.resource_path(*args, **kwds)

rhodecode/lib/index/whoosh.py

@@ -1,260 +1,261 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Index schema for RhodeCode
 """
 
 from __future__ import absolute_import
 import logging
 import os
 
 from pylons.i18n.translation import _
 
 from whoosh import query as query_lib, sorting
 from whoosh.highlight import HtmlFormatter, ContextFragmenter
 from whoosh.index import create_in, open_dir, exists_in, EmptyIndexError
 from whoosh.qparser import QueryParser, QueryParserError
 
 import rhodecode.lib.helpers as h
 from rhodecode.lib.index import BaseSearch
 
 log = logging.getLogger(__name__)
 
 
 try:
     # we first try to import from rhodecode tools, fallback to copies if
     # we're unable to
     from rhodecode_tools.lib.fts_index.whoosh_schema import (
         ANALYZER, FILE_INDEX_NAME, FILE_SCHEMA, COMMIT_INDEX_NAME,
         COMMIT_SCHEMA)
 except ImportError:
     log.warning('rhodecode_tools schema not available, doing a fallback '
                 'import from `rhodecode.lib.index.whoosh_fallback_schema`')
     from rhodecode.lib.index.whoosh_fallback_schema import (
         ANALYZER, FILE_INDEX_NAME, FILE_SCHEMA, COMMIT_INDEX_NAME,
         COMMIT_SCHEMA)
 
 
 FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n')
 FRAGMENTER = ContextFragmenter(200)
 
 log = logging.getLogger(__name__)
 
 
 class Search(BaseSearch):
 
     name = 'whoosh'
 
     def __init__(self, config):
         self.config = config
         if not os.path.isdir(self.config['location']):
             os.makedirs(self.config['location'])
 
         opener = create_in
         if exists_in(self.config['location'], indexname=FILE_INDEX_NAME):
             opener = open_dir
         file_index = opener(self.config['location'], schema=FILE_SCHEMA,
                             indexname=FILE_INDEX_NAME)
 
         opener = create_in
         if exists_in(self.config['location'], indexname=COMMIT_INDEX_NAME):
             opener = open_dir
         changeset_index = opener(self.config['location'], schema=COMMIT_SCHEMA,
                                  indexname=COMMIT_INDEX_NAME)
 
         self.commit_schema = COMMIT_SCHEMA
         self.commit_index = changeset_index
         self.file_schema = FILE_SCHEMA
         self.file_index = file_index
         self.searcher = None
 
     def cleanup(self):
         if self.searcher:
             self.searcher.close()
 
-    def search(self, query, document_type, search_user, repo_name=None):
+    def search(self, query, document_type, search_user, repo_name=None,
+        requested_page=1, page_limit=10):
         log.debug(u'QUERY: %s on %s', query, document_type)
         result = {
             'results': [],
             'count': 0,
             'error': None,
             'runtime': 0
         }
         search_type, index_name, schema_defn = self._prepare_for_search(
             document_type)
         self._init_searcher(index_name)
         try:
             qp = QueryParser(search_type, schema=schema_defn)
             allowed_repos_filter = self._get_repo_filter(
                 search_user, repo_name)
             try:
                 query = qp.parse(unicode(query))
                 log.debug('query: %s (%s)' % (query, repr(query)))
 
                 sortedby = None
                 if search_type == 'message':
                     sortedby = sorting.FieldFacet('commit_idx', reverse=True)
 
                 whoosh_results = self.searcher.search(
                     query, filter=allowed_repos_filter, limit=None,
                     sortedby=sortedby,)
 
                 # fixes for 32k limit that whoosh uses for highlight
                 whoosh_results.fragmenter.charlimit = None
                 res_ln = whoosh_results.scored_length()
                 result['runtime'] = whoosh_results.runtime
                 result['count'] = res_ln
                 result['results'] = WhooshResultWrapper(
                     search_type, res_ln, whoosh_results)
 
             except QueryParserError:
                 result['error'] = _('Invalid search query. Try quoting it.')
         except (EmptyIndexError, IOError, OSError):
             msg = _('There is no index to search in. '
                     'Please run whoosh indexer')
             log.exception(msg)
             result['error'] = msg
         except Exception:
             msg = _('An error occurred during this search operation')
             log.exception(msg)
             result['error'] = msg
 
         return result
 
     def statistics(self):
         stats = [
             {'key': _('Index Type'), 'value': 'Whoosh'},
             {'key': _('File Index'), 'value': str(self.file_index)},
             {'key': _('Indexed documents'),
              'value': self.file_index.doc_count()},
             {'key': _('Last update'),
              'value': h.time_to_datetime(self.file_index.last_modified())},
             {'key': _('Commit index'), 'value': str(self.commit_index)},
             {'key': _('Indexed documents'),
              'value': str(self.commit_index.doc_count())},
             {'key': _('Last update'),
              'value': h.time_to_datetime(self.commit_index.last_modified())}
         ]
         return stats
 
     def _get_repo_filter(self, auth_user, repo_name):
 
         allowed_to_search = [
             repo for repo, perm in
             auth_user.permissions['repositories'].items()
             if perm != 'repository.none']
 
         if repo_name:
             repo_filter = [query_lib.Term('repository', repo_name)]
 
         elif 'hg.admin' in auth_user.permissions.get('global', []):
             return None
 
         else:
             repo_filter = [query_lib.Term('repository', _rn)
                            for _rn in allowed_to_search]
             # in case we're not allowed to search anywhere, it's a trick
             # to tell whoosh we're filtering, on ALL results
             repo_filter = repo_filter or [query_lib.Term('repository', '')]
 
         return query_lib.Or(repo_filter)
 
     def _prepare_for_search(self, cur_type):
         search_type = {
             'content': 'content',
             'commit': 'message',
             'path': 'path',
             'repository': 'repository'
         }.get(cur_type, 'content')
 
         index_name = {
             'content': FILE_INDEX_NAME,
             'commit': COMMIT_INDEX_NAME,
             'path': FILE_INDEX_NAME
         }.get(cur_type, FILE_INDEX_NAME)
 
         schema_defn = {
             'content': self.file_schema,
             'commit': self.commit_schema,
             'path': self.file_schema
         }.get(cur_type, self.file_schema)
 
         log.debug('IDX: %s' % index_name)
         log.debug('SCHEMA: %s' % schema_defn)
         return search_type, index_name, schema_defn
 
     def _init_searcher(self, index_name):
         idx = open_dir(self.config['location'], indexname=index_name)
         self.searcher = idx.searcher()
         return self.searcher
 
 
 class WhooshResultWrapper(object):
     def __init__(self, search_type, total_hits, results):
         self.search_type = search_type
         self.results = results
         self.total_hits = total_hits
 
     def __str__(self):
         return '<%s at %s>' % (self.__class__.__name__, len(self))
 
     def __repr__(self):
         return self.__str__()
 
     def __len__(self):
         return self.total_hits
 
     def __iter__(self):
         """
         Allows Iteration over results,and lazy generate content
 
         *Requires* implementation of ``__getitem__`` method.
         """
         for hit in self.results:
             yield self.get_full_content(hit)
 
     def __getitem__(self, key):
         """
         Slicing of resultWrapper
         """
         i, j = key.start, key.stop
         for hit in self.results[i:j]:
             yield self.get_full_content(hit)
 
     def get_full_content(self, hit):
         # TODO: marcink: this feels like an overkill, there's a lot of data
         # inside hit object, and we don't need all
         res = dict(hit)
 
         f_path = ''  # noqa
         if self.search_type in ['content', 'path']:
             f_path = res['path'].split(res['repository'])[-1]
             f_path = f_path.lstrip(os.sep)
 
         if self.search_type == 'content':
             res.update({'content_short_hl': hit.highlights('content'),
                         'f_path': f_path})
         elif self.search_type == 'path':
             res.update({'f_path': f_path})
         elif self.search_type == 'message':
             res.update({'message_hl': hit.highlights('message')})
 
         return res

rhodecode/public/css/code-block.less

@@ -1,605 +1,625 b''
 // Default styles
 
 .diff-collapse {
     margin: @padding 0;
     text-align: right;
 }
 
 .diff-container {
     margin-bottom: @space;
 
     .diffblock {
         margin-bottom: @space;
     }
 
     &.hidden {
         display: none;
         overflow: hidden;
     }
 }
 
 .compare_view_files {
 
     .diff-container {
 
         .diffblock {
             margin-bottom: 0;
         }
     }
 }
 
 div.diffblock .sidebyside {
  background: #ffffff;
 }
 
 div.diffblock {
     overflow-x: auto;
     overflow-y: hidden;
     clear: both;
     padding: 0px;
     background: @grey6;
     border: @border-thickness solid @grey5;
     -webkit-border-radius: @border-radius @border-radius 0px 0px;
     border-radius: @border-radius @border-radius 0px 0px;
 
 
     .comments-number {
         float: right;
     }
 
     // BEGIN CODE-HEADER STYLES
 
     .code-header {
         background: @grey6;
         padding: 10px 0 10px 0;
         height: auto;
         width: 100%;
 
         .hash {
             float: left;
             padding: 2px 0 0 2px;
         }
 
         .date {
             float: left;
             text-transform: uppercase;
             padding: 4px 0px 0px 2px;
         }
 
         div {
             margin-left: 4px;
         }
 
         div.compare_header {
             min-height: 40px;
             margin: 0;
             padding: 0 @padding;
 
             .drop-menu {
                 float:left;
                 display: block;
                 margin:0 0 @padding 0;
             }
 
             .compare-label {
                 float: left;
                 clear: both;
                 display: inline-block;
                 min-width: 5em;
                 margin: 0;
                 padding: @button-padding @button-padding @button-padding 0;
                 font-family: @text-semibold;
             }
 
             .compare-buttons {
                 float: left;
                 margin: 0;
                 padding: 0 0 @padding;
 
                 .btn {
                     margin: 0 @padding 0 0;
                 }
             }
         }
 
     }
 
     .parents {
         float: left;
         width: 100px;
         font-weight: 400;
         vertical-align: middle;
         padding: 0px 2px 0px 2px;
         background-color: @grey6;
 
         #parent_link {
            margin: 00px 2px;
 
            &.double {
                margin: 0px 2px;
             }
 
             &.disabled{
                 margin-right: @padding;
             }
         }
     }
 
     .children {
         float: right;
         width: 100px;
         font-weight: 400;
         vertical-align: middle;
         text-align: right;
         padding: 0px 2px 0px 2px;
         background-color: @grey6;
 
         #child_link {
            margin: 0px 2px;
 
            &.double {
                margin: 0px 2px;
             }
 
             &.disabled{
                 margin-right: @padding;
             }
         }
     }
 
     .changeset_header {
         height: 16px;
 
         & > div{
             margin-right: @padding;
        }
     }
 
     .changeset_file {
         text-align: left;
         float: left;
         padding: 0;
 
         a{
             display: inline-block;
             margin-right: 0.5em;
         }
 
        #selected_mode{
             margin-left: 0;
        }
     }
 
     .diff-menu-wrapper {
         float: left;
     }
 
     .diff-menu {
         position: absolute;
         background: none repeat scroll 0 0 #FFFFFF;
         border-color: #003367 @grey3 @grey3;
         border-right: 1px solid @grey3;
         border-style: solid solid solid;
         border-width: @border-thickness;
         box-shadow: 2px 8px 4px rgba(0, 0, 0, 0.2);
         margin-top: 5px;
         margin-left: 1px;
     }
 
     .diff-actions, .editor-actions {
         float: left;
 
         input{
             margin: 0 0.5em 0 0;
         }
     }
 
     // END CODE-HEADER STYLES
 
     // BEGIN CODE-BODY STYLES
 
     .code-body {
         background: white;
         padding: 0;
         background-color: #ffffff;
         position: relative;
         max-width: none;
         box-sizing: border-box;
         // TODO: johbo: Parent has overflow: auto, this forces the child here
         // to have the intended size and to scroll. Should be simplified.
         width: 100%;
         overflow-x: auto;
     }
 
     pre.raw {
         background: white;
         color: @grey1;
     }
     // END CODE-BODY STYLES
 
 }
 
 
 table.code-difftable {
     border-collapse: collapse;
     width: 99%;
     border-radius: 0px !important;
 
      td {
         padding: 0 !important;
         background: none !important;
         border: 0 !important;
     }
 
     .context {
         background: none repeat scroll 0 0 #DDE7EF;
     }
 
     .add {
         background: none repeat scroll 0 0 #DDFFDD;
 
         ins {
             background: none repeat scroll 0 0 #AAFFAA;
             text-decoration: none;
         }
     }
 
     .del {
         background: none repeat scroll 0 0 #FFDDDD;
 
         del {
             background: none repeat scroll 0 0 #FFAAAA;
             text-decoration: none;
         }
     }
 
     /** LINE NUMBERS **/
     .lineno {
         padding-left: 2px;
         padding-right: 2px;
         text-align: right;
         width: 32px;
         -moz-user-select: none;
         -webkit-user-select: none;
         border-right: @border-thickness solid @grey5 !important;
         border-left: 0px solid #CCC !important;
         border-top: 0px solid #CCC !important;
         border-bottom: none !important;
 
         a {
             &:extend(pre);
             text-align: right;
             padding-right: 2px;
             cursor: pointer;
             display: block;
             width: 32px;
         }
     }
 
     .context {
         cursor: auto;
         &:extend(pre);
     }
 
     .lineno-inline {
         background: none repeat scroll 0 0 #FFF !important;
         padding-left: 2px;
         padding-right: 2px;
         text-align: right;
         width: 30px;
         -moz-user-select: none;
         -webkit-user-select: none;
     }
 
     /** CODE **/
     .code {
         display: block;
         width: 100%;
 
         td {
             margin: 0;
             padding: 0;
         }
 
         pre {
             margin: 0;
             padding: 0;
             margin-left: .5em;
         }
     }
 }
 
 
 // Comments
 
 div.comment:target {
     border-left: 6px solid @comment-highlight-color;
     padding-left: 3px;
     margin-left: -9px;
 }
 
 //TODO: anderson: can't get an absolute number out of anything, so had to put the
 //current values that might change. But to make it clear I put as a calculation
 @comment-max-width: 1065px;
 @pr-extra-margin: 34px;
 @pr-border-spacing: 4px;
 @pr-comment-width: @comment-max-width - @pr-extra-margin - @pr-border-spacing;
 
 // Pull Request
 .cs_files .code-difftable {
     border: @border-thickness solid @grey5; //borders only on PRs
 
     .comment-inline-form,
     div.comment {
         width: @pr-comment-width;
     }
 }
 
 // Changeset
 .code-difftable {
     .comment-inline-form,
     div.comment {
         width: @comment-max-width;
     }
 }
 
 //Style page
 @style-extra-margin: @sidebar-width + (@sidebarpadding * 3) + @padding;
 #style-page .code-difftable{
     .comment-inline-form,
     div.comment {
         width: @comment-max-width - @style-extra-margin;
     }
 }
 
 #context-bar > h2 {
     font-size: 20px;
 }
 
 #context-bar > h2> a {
     font-size: 20px;
 }
 // end of defaults
 
 .file_diff_buttons {
     padding: 0 0 @padding;
 
     .drop-menu {
         float: left;
         margin: 0 @padding 0 0;
     }
     .btn {
         margin: 0 @padding 0 0;
     }
 }
 
 .code-body.textarea.editor {
     max-width: none;
     padding: 15px;
 }
 
 td.injected_diff{
     max-width: 1178px;
     overflow-x: auto;
     overflow-y: hidden;
 
     div.diff-container,
     div.diffblock{
         max-width: 100%;
     }
 
     div.code-body {
         max-width: 1124px;
         overflow-x: auto;
         padding: 0;
     }
     div.diffblock {
         border: none;
     }
 
     &.inline-form {
         width: 99%
     }
  }
 
 
 table.code-difftable {
     width: 100%;
 }
 
 /** PYGMENTS COLORING **/
 div.codeblock {
 
     // TODO: johbo: Added interim to get rid of the margin around
     // Select2 widgets. This needs further cleanup.
     margin-top: @padding;
 
     overflow: auto;
     padding: 0px;
     border: @border-thickness solid @grey5;
     background: @grey6;
     .border-radius(@border-radius);
 
     #remove_gist {
         float: right;
     }
 
     .author {
         clear: both;
         vertical-align: middle;
         font-family: @text-bold;
     }
 
     .btn-mini {
         float: left;
         margin: 0 5px 0 0;
     }
 
     .code-header {
         padding: @padding;
         border-bottom: @border-thickness solid @grey5;
 
         .rc-user {
             min-width: 0;
             margin-right: .5em;
         }
 
         .stats {
             clear: both;
             margin: 0 0 @padding 0;
             padding: 0;
             .left {
                 float: left;
                 clear: left;
                 max-width: 75%;
                 margin: 0 0 @padding 0;
 
                 &.item {
                     margin-right: @padding;
                     &.last { border-right: none; }
                 }
             }
             .buttons { float: right; }
             .author {
                 height: 25px; margin-left: 15px; font-weight: bold;
             }
         }
 
         .commit {
             margin: 5px 0 0 26px;
             font-weight: normal;
             white-space: pre-wrap;
         }
     }
 
     .message {
         position: relative;
         margin: @padding;
 
         .codeblock-label {
             margin: 0 0 1em 0;
         }
     }
 
     .code-body {
         padding: @padding;
         background-color: #ffffff;
         min-width: 100%;
         box-sizing: border-box;
         // TODO: johbo: Parent has overflow: auto, this forces the child here
         // to have the intended size and to scroll. Should be simplified.
         width: 100%;
         overflow-x: auto;
     }
 }
 
 .code-highlighttable,
 div.codeblock .code-body table {
     width: 0 !important;
     border: 0px !important;
     margin: 0;
     letter-spacing: normal;
 
 
     td {
         border: 0px !important;
         vertical-align: top;
     }
 }
 
 div.codeblock .code-header .search-path { padding: 0 0 0 10px; }
 div.search-code-body {
     background-color: #ffffff; padding: 5px 0 5px 10px;
     pre {
         .match { background-color: #faffa6;}
         .break { display: block; width: 100%; background-color: #DDE7EF; color: #747474; }
     }
+    .code-highlighttable {
+        border-collapse: collapse;
+
+        tr:hover {
+            background: #fafafa;
+        }
+        td.code {
+          padding-left: 10px;
+        }
+        td.line {
+            border-right: 1px solid #ccc !important;
+            padding-right: 10px;
+            text-align: right;
+            font-family: "Lucida Console",Monaco,monospace;
+            span {
+                white-space: pre-wrap;
+                color: #666666;
+            }
+        }
+    }
 }
 
 div.annotatediv { margin-left: 2px; margin-right: 4px; }
 .code-highlight {
     margin: 0; padding: 0; border-left: @border-thickness solid @grey5;
     pre, .linenodiv pre { padding: 0 5px; margin: 0; }
     pre div:target {background-color: @comment-highlight-color !important;}
 }
 
 .linenos a { text-decoration: none; }
 
 .CodeMirror-selected { background: @rchighlightblue; }
 .CodeMirror-focused .CodeMirror-selected { background: @rchighlightblue; }
 .CodeMirror ::selection { background: @rchighlightblue; }
 .CodeMirror ::-moz-selection { background: @rchighlightblue; }
 
 .code { display: block; border:0px !important; }
 .code-highlight,
 .codehilite {
     .hll { background-color: #ffffcc }
     .c { color: #408080; font-style: italic } /* Comment */
     .err, .codehilite .err { border: @border-thickness solid #FF0000 } /* Error */
     .k { color: #008000; font-weight: bold } /* Keyword */
     .o { color: #666666 } /* Operator */
     .cm { color: #408080; font-style: italic } /* Comment.Multiline */
     .cp { color: #BC7A00 } /* Comment.Preproc */
     .c1 { color: #408080; font-style: italic } /* Comment.Single */
     .cs { color: #408080; font-style: italic } /* Comment.Special */
     .gd { color: #A00000 } /* Generic.Deleted */
     .ge { font-style: italic } /* Generic.Emph */
     .gr { color: #FF0000 } /* Generic.Error */
     .gh { color: #000080; font-weight: bold } /* Generic.Heading */
     .gi { color: #00A000 } /* Generic.Inserted */
     .go { color: #808080 } /* Generic.Output */
     .gp { color: #000080; font-weight: bold } /* Generic.Prompt */
     .gs { font-weight: bold } /* Generic.Strong */
     .gu { color: #800080; font-weight: bold } /* Generic.Subheading */
     .gt { color: #0040D0 } /* Generic.Traceback */
     .kc { color: #008000; font-weight: bold } /* Keyword.Constant */
     .kd { color: #008000; font-weight: bold } /* Keyword.Declaration */
     .kn { color: #008000; font-weight: bold } /* Keyword.Namespace */
     .kp { color: #008000 } /* Keyword.Pseudo */
     .kr { color: #008000; font-weight: bold } /* Keyword.Reserved */
     .kt { color: #B00040 } /* Keyword.Type */
     .m { color: #666666 } /* Literal.Number */
     .s { color: #BA2121 } /* Literal.String */
     .na { color: #7D9029 } /* Name.Attribute */
     .nb { color: #008000 } /* Name.Builtin */
     .nc { color: #0000FF; font-weight: bold } /* Name.Class */
     .no { color: #880000 } /* Name.Constant */
     .nd { color: #AA22FF } /* Name.Decorator */
     .ni { color: #999999; font-weight: bold } /* Name.Entity */
     .ne { color: #D2413A; font-weight: bold } /* Name.Exception */
     .nf { color: #0000FF } /* Name.Function */
     .nl { color: #A0A000 } /* Name.Label */
     .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */
     .nt { color: #008000; font-weight: bold } /* Name.Tag */
     .nv { color: #19177C } /* Name.Variable */
     .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */
     .w { color: #bbbbbb } /* Text.Whitespace */
     .mf { color: #666666 } /* Literal.Number.Float */
     .mh { color: #666666 } /* Literal.Number.Hex */
     .mi { color: #666666 } /* Literal.Number.Integer */
     .mo { color: #666666 } /* Literal.Number.Oct */
     .sb { color: #BA2121 } /* Literal.String.Backtick */
     .sc { color: #BA2121 } /* Literal.String.Char */
     .sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */
     .s2 { color: #BA2121 } /* Literal.String.Double */
     .se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */
     .sh { color: #BA2121 } /* Literal.String.Heredoc */
     .si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */
     .sx { color: #008000 } /* Literal.String.Other */
     .sr { color: #BB6688 } /* Literal.String.Regex */
     .s1 { color: #BA2121 } /* Literal.String.Single */
     .ss { color: #19177C } /* Literal.String.Symbol */
     .bp { color: #008000 } /* Name.Builtin.Pseudo */
     .vc { color: #19177C } /* Name.Variable.Class */
     .vg { color: #19177C } /* Name.Variable.Global */
     .vi { color: #19177C } /* Name.Variable.Instance */
     .il { color: #666666 } /* Literal.Number.Integer.Long */
 }
 
 /* customized pre blocks for markdown/rst */
 pre.literal-block, .codehilite pre{
     padding: @padding;
     border: 1px solid @grey6;
     .border-radius(@border-radius);
     background-color: @grey7;
 }

rhodecode/templates/search/search_commit.html

@@ -1,76 +1,76 b''
 <%namespace name="base" file="/base/base.html"/>
 
 <table class="rctable search-results">
     <tr>
         <th>${_('Repository')}</th>
         <th>${_('Commit')}</th>
         <th></th>
         <th>${_('Commit message')}</th>
         <th>${_('Age')}</th>
         <th>${_('Author')}</th>
     </tr>
     %for entry in c.formatted_results:
         ## search results are additionally filtered, and this check is just a safe gate
         % if h.HasRepoPermissionAny('repository.write','repository.read','repository.admin')(entry['repository'], 'search results commit check'):
             <tr class="body">
                 <td class="td-componentname">
                     %if h.get_repo_type_by_name(entry.get('repository')) == 'hg':
                         <i class="icon-hg"></i>
                     %elif h.get_repo_type_by_name(entry.get('repository')) == 'git':
                         <i class="icon-git"></i>
                     %elif h.get_repo_type_by_name(entry.get('repository')) == 'svn':
                         <i class="icon-svn"></i>
                     %endif
                     ${h.link_to(entry['repository'], h.url('summary_home',repo_name=entry['repository']))}
                 </td>
                 <td class="td-commit">
                     ${h.link_to(h._shorten_commit_id(entry['commit_id']),
                       h.url('changeset_home',repo_name=entry['repository'],revision=entry['commit_id']))}
                 </td>
                 <td class="td-message expand_commit search open" data-commit-id="${h.md5_safe(entry['repository'])+entry['commit_id']}" id="t-${h.md5_safe(entry['repository'])+entry['commit_id']}" title="${_('Expand commit message')}">
                     <div class="show_more_col">
                     <i class="show_more"></i>&nbsp;
                     </div>
                 </td>
                 <td data-commit-id="${h.md5_safe(entry['repository'])+entry['commit_id']}" id="c-${h.md5_safe(entry['repository'])+entry['commit_id']}" class="message td-description open">
-                    %if entry['message_hl']:
+                    %if entry.get('message_hl'):
                         ${h.literal(entry['message_hl'])}
                     %else:
                         ${h.urlify_commit_message(entry['message'], entry['repository'])}
                     %endif
                 </td>
                 <td class="td-time">
                     ${h.age_component(h.time_to_datetime(entry['date']))}
                 </td>
 
                 <td class="td-user author">
                     ${base.gravatar_with_user(entry['author'])}
                 </td>
             </tr>
         % endif
     %endfor
 </table>
 
 %if c.cur_query and c.formatted_results:
 <div class="pagination-wh pagination-left">
     ${c.formatted_results.pager('$link_previous ~2~ $link_next')}
 </div>
 %endif
 
 <script>
     $('.expand_commit').on('click',function(e){
       var target_expand = $(this);
       var cid = target_expand.data('commit-id');
 
       if (target_expand.hasClass('open')){
         $('#c-'+cid).css({'height': '1.5em', 'white-space': 'nowrap', 'text-overflow': 'ellipsis', 'overflow':'hidden'})
         $('#t-'+cid).css({'height': 'auto', 'line-height': '.9em', 'text-overflow': 'ellipsis', 'overflow':'hidden'})
         target_expand.removeClass('open');
       }
       else {
         $('#c-'+cid).css({'height': 'auto', 'white-space': 'normal', 'text-overflow': 'initial', 'overflow':'visible'})
         $('#t-'+cid).css({'height': 'auto', 'max-height': 'none', 'text-overflow': 'initial', 'overflow':'visible'})
         target_expand.addClass('open');
       }
     });
 </script>

rhodecode/templates/search/search_content.html

@@ -1,51 +1,101 b''
+<%def name="highlight_text_file(terms, text, url, line_context=3,
+                                max_lines=10,
+                                mimetype=None, filepath=None)">
+<%
+lines = text.split('\n')
+lines_of_interest = set()
+matching_lines = h.get_matching_line_offsets(lines, terms)
+shown_matching_lines = 0
+
+for line_number in matching_lines:
+    if len(lines_of_interest) < max_lines:
+        lines_of_interest |= set(range(
+            max(line_number - line_context, 0),
+            min(line_number + line_context, len(lines))))
+        shown_matching_lines += 1
+
+%>
+${h.code_highlight(
+    text,
+    h.get_lexer_safe(
+        mimetype=mimetype,
+        filepath=filepath,
+    ),
+    h.SearchContentCodeHtmlFormatter(
+        linenos=True,
+        cssclass="code-highlight",
+        url=url,
+        query_terms=terms,
+        only_line_numbers=lines_of_interest
+))|n}
+%if len(matching_lines) > shown_matching_lines:
+<a href="${url}">
+  ${len(matching_lines) - shown_matching_lines} ${_('more matches in this file')}
+</p>
+%endif
+</%def>
+
 <div class="search-results">
 %for entry in c.formatted_results:
   ## search results are additionally filtered, and this check is just a safe gate
   % if h.HasRepoPermissionAny('repository.write','repository.read','repository.admin')(entry['repository'], 'search results content check'):
       <div id="codeblock" class="codeblock">
         <div class="codeblock-header">
           <h2>
             %if h.get_repo_type_by_name(entry.get('repository')) == 'hg':
                 <i class="icon-hg"></i>
             %elif h.get_repo_type_by_name(entry.get('repository')) == 'git':
                 <i class="icon-git"></i>
             %elif h.get_repo_type_by_name(entry.get('repository')) == 'svn':
                 <i class="icon-svn"></i>
             %endif
             ${h.link_to(entry['repository'], h.url('summary_home',repo_name=entry['repository']))}
           </h2>
           <div class="stats">
             ${h.link_to(h.literal(entry['f_path']), h.url('files_home',repo_name=entry['repository'],revision=entry.get('commit_id', 'tip'),f_path=entry['f_path']))}
             %if entry.get('lines'):
               | ${entry.get('lines', 0.)} ${ungettext('line', 'lines', entry.get('lines', 0.))}
             %endif
             %if entry.get('size'):
               | ${h.format_byte_size_binary(entry['size'])}
             %endif
             %if entry.get('mimetype'):
               | ${entry.get('mimetype', "unknown mimetype")}
             %endif
           </div>
           <div class="buttons">
             <a id="file_history_overview_full" href="${h.url('changelog_file_home',repo_name=entry.get('repository',''),revision=entry.get('commit_id', 'tip'),f_path=entry.get('f_path',''))}">
                ${_('Show Full History')}
-            </a> | 
+            </a> |
               ${h.link_to(_('Annotation'), h.url('files_annotate_home', repo_name=entry.get('repository',''),revision=entry.get('commit_id', 'tip'),f_path=entry.get('f_path','')))}
              | ${h.link_to(_('Raw'), h.url('files_raw_home', repo_name=entry.get('repository',''),revision=entry.get('commit_id', 'tip'),f_path=entry.get('f_path','')))}
              | <a href="${h.url('files_rawfile_home',repo_name=entry.get('repository',''),revision=entry.get('commit_id', 'tip'),f_path=entry.get('f_path',''))}">
                 ${_('Download')}
                </a>
           </div>
         </div>
         <div class="code-body search-code-body">
-            <pre>${h.literal(entry['content_short_hl'])}</pre>
-       </div>
+            ${highlight_text_file(c.cur_query, entry['content'],
+            url=h.url('files_home',repo_name=entry['repository'],revision=entry.get('commit_id', 'tip'),f_path=entry['f_path']),
+            mimetype=entry.get('mimetype'), filepath=entry.get('path'))}
+        </div>
       </div>
     % endif
 %endfor
 </div>
 %if c.cur_query and c.formatted_results:
 <div class="pagination-wh pagination-left" >
     ${c.formatted_results.pager('$link_previous ~2~ $link_next')}
 </div>
 %endif
+
+%if c.cur_query:
+<script type="text/javascript">
+$(function(){
+  $(".code").mark(
+    '${' '.join(h.normalize_text_for_matching(c.cur_query).split())}',
+    {"className": 'match',
+  });
+})
+</script>
+%endif
  No newline at end of file

rhodecode/tests/lib/test_helpers.py

@@ -1,157 +1,196 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import copy
 import mock
 import pytest
 
 from pylons.util import ContextObj
 
 from rhodecode.lib import helpers
 from rhodecode.lib.utils2 import AttributeDict
 from rhodecode.model.settings import IssueTrackerSettingsModel
 
 
 @pytest.mark.parametrize('url, expected_url', [
     ('http://rc.rc/test', '<a href="http://rc.rc/test">http://rc.rc/test</a>'),
     ('http://rc.rc/@foo', '<a href="http://rc.rc/@foo">http://rc.rc/@foo</a>'),
     ('http://rc.rc/!foo', '<a href="http://rc.rc/!foo">http://rc.rc/!foo</a>'),
     ('http://rc.rc/&foo', '<a href="http://rc.rc/&foo">http://rc.rc/&foo</a>'),
     ('http://rc.rc/#foo', '<a href="http://rc.rc/#foo">http://rc.rc/#foo</a>'),
 ])
 def test_urlify_text(url, expected_url):
     assert helpers.urlify_text(url) == expected_url
 
 
 @pytest.mark.parametrize('repo_name, commit_id, path, expected_result', [
     ('rX<X', 'cX<X', 'pX<X/aX<X/bX<X',
      '<a class="pjax-link" href="/rX%3CX/files/cX%3CX/">rX&lt;X</a>/'
      '<a class="pjax-link" href="/rX%3CX/files/cX%3CX/pX%3CX">pX&lt;X</a>/'
      '<a class="pjax-link" href="/rX%3CX/files/cX%3CX/pX%3CX/aX%3CX">aX&lt;X'
      '</a>/bX&lt;X'),
     # Path with only one segment
     ('rX<X', 'cX<X', 'pX<X',
      '<a class="pjax-link" href="/rX%3CX/files/cX%3CX/">rX&lt;X</a>/pX&lt;X'),
     # Empty path
     ('rX<X', 'cX<X', '', 'rX&lt;X'),
     ('rX"X', 'cX"X', 'pX"X/aX"X/bX"X',
      '<a class="pjax-link" href="/rX%22X/files/cX%22X/">rX&#34;X</a>/'
      '<a class="pjax-link" href="/rX%22X/files/cX%22X/pX%22X">pX&#34;X</a>/'
      '<a class="pjax-link" href="/rX%22X/files/cX%22X/pX%22X/aX%22X">aX&#34;X'
      '</a>/bX&#34;X'),
 ], ids=['simple', 'one_segment', 'empty_path', 'simple_quote'])
 def test_files_breadcrumbs_xss(
         repo_name, commit_id, path, pylonsapp, expected_result):
     result = helpers.files_breadcrumbs(repo_name, commit_id, path)
     # Expect it to encode all path fragments properly. This is important
     # because it returns an instance of `literal`.
     assert result == expected_result
 
 
 def test_format_binary():
     assert helpers.format_byte_size_binary(298489462784) == '278.0 GiB'
 
 
 @pytest.mark.parametrize('text_string, pattern, expected_text', [
     ('Fix #42', '(?:#)(?P<issue_id>\d+)',
      'Fix <a class="issue-tracker-link" href="http://r.io/{repo}/i/42">#42</a>'
      ),
     ('Fix #42', '(?:#)?<issue_id>\d+)', 'Fix #42'),  # Broken regex
 ])
 def test_process_patterns_repo(backend, text_string, pattern, expected_text):
     repo = backend.create_repo()
     config = {'123': {
         'uid': '123',
         'pat': pattern,
         'url': 'http://r.io/${repo}/i/${issue_id}',
         'pref': '#',
         }
     }
     with mock.patch.object(IssueTrackerSettingsModel,
                            'get_settings', lambda s: config):
         processed_text = helpers.process_patterns(
             text_string, repo.repo_name, config)
 
     assert processed_text == expected_text.format(repo=repo.repo_name)
 
 
 @pytest.mark.parametrize('text_string, pattern, expected_text', [
     ('Fix #42', '(?:#)(?P<issue_id>\d+)',
      'Fix <a class="issue-tracker-link" href="http://r.io/i/42">#42</a>'
      ),
     ('Fix #42', '(?:#)?<issue_id>\d+)', 'Fix #42'),  # Broken regex
 ])
 def test_process_patterns_no_repo(text_string, pattern, expected_text):
     config = {'123': {
         'uid': '123',
         'pat': pattern,
         'url': 'http://r.io/i/${issue_id}',
         'pref': '#',
         }
     }
     with mock.patch.object(IssueTrackerSettingsModel,
                            'get_global_settings', lambda s, cache: config):
         processed_text = helpers.process_patterns(
             text_string, '', config)
 
     assert processed_text == expected_text
 
 
 def test_process_patterns_non_existent_repo_name(backend):
     text_string = 'Fix #42'
     pattern = '(?:#)(?P<issue_id>\d+)'
     expected_text = ('Fix <a class="issue-tracker-link" '
                      'href="http://r.io/do-not-exist/i/42">#42</a>')
     config = {'123': {
         'uid': '123',
         'pat': pattern,
         'url': 'http://r.io/${repo}/i/${issue_id}',
         'pref': '#',
         }
     }
     with mock.patch.object(IssueTrackerSettingsModel,
                            'get_global_settings', lambda s, cache: config):
         processed_text = helpers.process_patterns(
             text_string, 'do-not-exist', config)
 
     assert processed_text == expected_text
 
 
 def test_get_visual_attr(pylonsapp):
     c = ContextObj()
     assert None is helpers.get_visual_attr(c, 'fakse')
 
     # emulate the c.visual behaviour
     c.visual = AttributeDict({})
     assert None is helpers.get_visual_attr(c, 'some_var')
 
     c.visual.some_var = 'foobar'
     assert 'foobar' == helpers.get_visual_attr(c, 'some_var')
 
 
 @pytest.mark.parametrize('test_text, inclusive, expected_text', [
     ('just a string', False, 'just a string'),
     ('just a string\n', False, 'just a string'),
     ('just a string\n next line', False, 'just a string...'),
     ('just a string\n next line', True, 'just a string\n...'),
 ])
 def test_chop_at(test_text, inclusive, expected_text):
     assert helpers.chop_at_smart(
         test_text, '\n', inclusive, '...') == expected_text
+
+
[e-mail]('test_text, expected_output', [
+    ('some text', ['some', 'text']),
+    ('some    text', ['some', 'text']),
+    ('some text "with  a phrase"', ['some', 'text', 'with  a phrase']),
+    ('"a phrase" "another phrase"', ['a phrase', 'another phrase']),
+    ('"justphrase"', ['justphrase']),
+    ('""', []),
+    ('', []),
+    ('  ', []),
+    ('"   "', []),
+])
+def test_extract_phrases(test_text, expected_output):
+    assert helpers.extract_phrases(test_text) == expected_output
+
+
[e-mail]('test_text, text_phrases, expected_output', [
+    ('some text here', ['some', 'here'], [(0, 4), (10, 14)]),
+    ('here here there', ['here'], [(0, 4), (5, 9), (11, 15)]),
+    ('irrelevant', ['not found'], []),
+    ('irrelevant', ['not found'], []),
+])
+def test_get_matching_offsets(test_text, text_phrases, expected_output):
+    assert helpers.get_matching_offsets(
+        test_text, text_phrases) == expected_output
+
+def test_normalize_text_for_matching():
+    assert helpers.normalize_text_for_matching(
+        'OJjfe)*#$*@)$JF*)3r2f80h') == 'ojjfe        jf  3r2f80h'
+
+def test_get_matching_line_offsets():
+    assert helpers.get_matching_line_offsets([
+        'words words words',
+        'words words words',
+        'some text some',
+        'words words words',
+        'words words words',
+        'text here what'], 'text') == {3: [(5, 9)], 6: [(0, 4)]}
  No newline at end of file