search: add support for elastic search 6

- elasticsearch new query lang and searcher
- much more advanced query lang
- added context aware search in repository groups, repository, commits, files view
- optimized quick search bar speed when using full text search
- added option to increase hits per file number from query URL using max_lines
- search results can be now marked inside a text file using ?mark=HIGHLIGHT_TEXT added to the url

rhodecode/lib/index/search_utils.py

@@ -0,0 +1,257 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2012-2018 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import re
+
+import pygments.filter
+import pygments.filters
+from pygments.token import Comment
+
+HL_BEG_MARKER = '__RCSearchHLMarkBEG__'
+HL_END_MARKER = '__RCSearchHLMarkEND__'
+HL_MARKER_RE = '{}(.*?){}'.format(HL_BEG_MARKER, HL_END_MARKER)
+
+
+class ElasticSearchHLFilter(pygments.filters.Filter):
+    _names = [HL_BEG_MARKER, HL_END_MARKER]
+
+    def __init__(self, **options):
+        pygments.filters.Filter.__init__(self, **options)
+
+    def filter(self, lexer, stream):
+        def tokenize(_value):
+            for token in re.split('({}|{})'.format(
+                    self._names[0], self._names[1]), _value):
+                if token:
+                    yield token
+
+        hl = False
+        for ttype, value in stream:
+
+            if self._names[0] in value or self._names[1] in value:
+                for item in tokenize(value):
+                    if item == self._names[0]:
+                        # skip marker, but start HL
+                        hl = True
+                        continue
+                    elif item == self._names[1]:
+                        hl = False
+                        continue
+
+                    if hl:
+                        yield Comment.ElasticMatch, item
+                    else:
+                        yield ttype, item
+            else:
+                if hl:
+                    yield Comment.ElasticMatch, value
+                else:
+                    yield ttype, value
+
+
+def extract_phrases(text_query):
+    """
+    Extracts phrases from search term string making sure phrases
+    contained in double quotes are kept together - and discarding empty values
+    or fully whitespace values eg.
+
+    'some   text "a phrase" more' => ['some', 'text', 'a phrase', 'more']
+
+    """
+
+    in_phrase = False
+    buf = ''
+    phrases = []
+    for char in text_query:
+        if in_phrase:
+            if char == '"':  # end phrase
+                phrases.append(buf)
+                buf = ''
+                in_phrase = False
+                continue
+            else:
+                buf += char
+                continue
+        else:
+            if char == '"':  # start phrase
+                in_phrase = True
+                phrases.append(buf)
+                buf = ''
+                continue
+            elif char == ' ':
+                phrases.append(buf)
+                buf = ''
+                continue
+            else:
+                buf += char
+
+    phrases.append(buf)
+    phrases = [phrase.strip() for phrase in phrases if phrase.strip()]
+    return phrases
+
+
+def get_matching_phrase_offsets(text, phrases):
+    """
+    Returns a list of string offsets in `text` that the list of `terms` match
+
+    >>> get_matching_phrase_offsets('some text here', ['some', 'here'])
+    [(0, 4), (10, 14)]
+
+    """
+    phrases = phrases or []
+    offsets = []
+
+    for phrase in phrases:
+        for match in re.finditer(phrase, text):
+            offsets.append((match.start(), match.end()))
+
+    return offsets
+
+
+def get_matching_markers_offsets(text, markers=None):
+    """
+    Returns a list of string offsets in `text` that the are between matching markers
+
+    >>> get_matching_markers_offsets('$1some$2 text $1here$2 marked', ['\$1(.*?)\$2'])
+    [(0, 5), (16, 22)]
+
+    """
+    markers = markers or [HL_MARKER_RE]
+    offsets = []
+
+    if markers:
+        for mark in markers:
+            for match in re.finditer(mark, text):
+                offsets.append((match.start(), match.end()))
+
+    return offsets
+
+
+def normalize_text_for_matching(x):
+    """
+    Replaces all non alfanum characters to spaces and lower cases the string,
+    useful for comparing two text strings without punctuation
+    """
+    return re.sub(r'[^\w]', ' ', x.lower())
+
+
+def get_matching_line_offsets(lines, terms=None, markers=None):
+    """ Return a set of `lines` indices (starting from 1) matching a
+    text search query, along with `context` lines above/below matching lines
+
+    :param lines: list of strings representing lines
+    :param terms: search term string to match in lines eg. 'some text'
+    :param markers: instead of terms, use highlight markers instead that
+        mark beginning and end for matched item. eg. ['START(.*?)END']
+
+     eg.
+
+    text = '''
+    words words words
+    words words words
+    some text some
+    words words words
+    words words words
+    text here what
+    '''
+    get_matching_line_offsets(text, 'text', context=1)
+    6, {3: [(5, 9)], 6: [(0, 4)]]
+
+    """
+    matching_lines = {}
+    line_index = 0
+
+    if terms:
+        phrases = [normalize_text_for_matching(phrase)
+                   for phrase in extract_phrases(terms)]
+
+        for line_index, line in enumerate(lines.splitlines(), start=1):
+            normalized_line = normalize_text_for_matching(line)
+            match_offsets = get_matching_phrase_offsets(normalized_line, phrases)
+            if match_offsets:
+                matching_lines[line_index] = match_offsets
+
+    else:
+        markers = markers or [HL_MARKER_RE]
+        for line_index, line in enumerate(lines.splitlines(), start=1):
+            match_offsets = get_matching_markers_offsets(line, markers=markers)
+            if match_offsets:
+                matching_lines[line_index] = match_offsets
+
+    return line_index, matching_lines
+
+
+def lucene_query_parser():
+    # from pyparsing lucene_grammar
+    from pyparsing import (
+        Literal, CaselessKeyword, Forward, Regex, QuotedString, Suppress,
+        Optional, Group, infixNotation, opAssoc, ParserElement, pyparsing_common)
+
+    ParserElement.enablePackrat()
+
+    COLON, LBRACK, RBRACK, LBRACE, RBRACE, TILDE, CARAT = map(Literal, ":[]{}~^")
+    LPAR, RPAR = map(Suppress, "()")
+    and_, or_, not_, to_ = map(CaselessKeyword, "AND OR NOT TO".split())
+    keyword = and_ | or_ | not_ | to_
+
+    expression = Forward()
+
+    valid_word = Regex(r'([a-zA-Z0-9*_+.-]|\\[!(){}\[\]^"~*?\\:])+').setName("word")
+    valid_word.setParseAction(
+        lambda t: t[0]
+            .replace('\\\\', chr(127))
+            .replace('\\', '')
+            .replace(chr(127), '\\')
+    )
+
+    string = QuotedString('"')
+
+    required_modifier = Literal("+")("required")
+    prohibit_modifier = Literal("-")("prohibit")
+    integer = Regex(r"\d+").setParseAction(lambda t: int(t[0]))
+    proximity_modifier = Group(TILDE + integer("proximity"))
+    number = pyparsing_common.fnumber()
+    fuzzy_modifier = TILDE + Optional(number, default=0.5)("fuzzy")
+
+    term = Forward()
+    field_name = valid_word().setName("fieldname")
+    incl_range_search = Group(LBRACK + term("lower") + to_ + term("upper") + RBRACK)
+    excl_range_search = Group(LBRACE + term("lower") + to_ + term("upper") + RBRACE)
+    range_search = incl_range_search("incl_range") | excl_range_search("excl_range")
+    boost = (CARAT + number("boost"))
+
+    string_expr = Group(string + proximity_modifier) | string
+    word_expr = Group(valid_word + fuzzy_modifier) | valid_word
+    term << (Optional(field_name("field") + COLON) +
+             (word_expr | string_expr | range_search | Group(
+                 LPAR + expression + RPAR)) +
+             Optional(boost))
+    term.setParseAction(lambda t: [t] if 'field' in t or 'boost' in t else None)
+
+    expression << infixNotation(
+        term,
+        [
+            (required_modifier | prohibit_modifier, 1, opAssoc.RIGHT),
+            ((not_ | '!').setParseAction(lambda: "NOT"), 1, opAssoc.RIGHT),
+            ((and_ | '&&').setParseAction(lambda: "AND"), 2, opAssoc.LEFT),
+            (Optional(or_ | '||').setParseAction(lambda: "OR"), 2, opAssoc.LEFT),
+        ]
+    )
+
+    return expression

rhodecode/tests/lib/test_search_utils.py

@@ -0,0 +1,100 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2018 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import copy
+import mock
+import pytest
+
+from rhodecode.lib.index import search_utils
+
+
[e-mail]('test_text, expected_output', [
+    ('some text', ['some', 'text']),
+    ('some    text', ['some', 'text']),
+    ('some text "with  a phrase"', ['some', 'text', 'with  a phrase']),
+    ('"a phrase" "another phrase"', ['a phrase', 'another phrase']),
+    ('"justphrase"', ['justphrase']),
+    ('""', []),
+    ('', []),
+    ('  ', []),
+    ('"   "', []),
+])
+def test_extract_phrases(test_text, expected_output):
+    assert search_utils.extract_phrases(test_text) == expected_output
+
+
[e-mail]('test_text, text_phrases, expected_output', [
+    ('some text here', ['some', 'here'], [(0, 4), (10, 14)]),
+    ('here here there', ['here'], [(0, 4), (5, 9), (11, 15)]),
+    ('irrelevant', ['not found'], []),
+    ('irrelevant', ['not found'], []),
+])
+def test_get_matching_phrase_offsets(test_text, text_phrases, expected_output):
+    assert search_utils.get_matching_phrase_offsets(
+        test_text, text_phrases) == expected_output
+
+
[e-mail]('test_text, text_phrases, expected_output', [
+    ('__RCSearchHLMarkBEG__some__RCSearchHLMarkEND__ text __RCSearchHLMarkBEG__here__RCSearchHLMarkEND__', [], [(0, 46), (52, 98)]),
+    ('__RCSearchHLMarkBEG__here__RCSearchHLMarkEND__ __RCSearchHLMarkBEG__here__RCSearchHLMarkEND__ there', [], [(0, 46), (47, 93)]),
+    ('some text __RCSearchHLMarkBEG__here__RCSearchHLMarkEND__', [], [(10, 56)]),
+    ('__RCSearchHLMarkBEG__here__RCSearchHLMarkEND__ __RCSearchHLMarkBEG__here__RCSearchHLMarkEND__ __RCSearchHLMarkBEG__there__RCSearchHLMarkEND__', [], [(0, 46), (47, 93), (94, 141)]),
+    ('irrelevant', ['not found'], []),
+    ('irrelevant', ['not found'], []),
+])
+def test_get_matching_marker_offsets(test_text, text_phrases, expected_output):
+
+    assert search_utils.get_matching_markers_offsets(test_text) == expected_output
+
+
+def test_normalize_text_for_matching():
+    assert search_utils.normalize_text_for_matching(
+        'OJjfe)*#$*@)$JF*)3r2f80h') == 'ojjfe        jf  3r2f80h'
+
+
+def test_get_matching_line_offsets():
+    words = '\n'.join([
+        'words words words',
+        'words words words',
+        'some text some',
+        'words words words',
+        'words words words',
+        'text here what'
+    ])
+    total_lines, matched_offsets = \
+        search_utils.get_matching_line_offsets(words, terms='text')
+    assert total_lines == 6
+    assert matched_offsets == {3: [(5, 9)], 6: [(0, 4)]}
+
+
+def test_get_matching_line_offsets_using_markers():
+    words = '\n'.join([
+        'words words words',
+        'words words words',
+        'some __1__text__2__ some',
+        'words words words',
+        'words words words',
+        '__1__text__2__ here what'
+    ])
+    total_lines, matched_offsets = \
+        search_utils.get_matching_line_offsets(words, terms=None,
+                                               markers=['__1__(.*?)__2__'])
+    assert total_lines == 6
+    assert matched_offsets == {3: [(5, 19)], 6: [(0, 14)]}

pkgs/python-packages.nix

@@ -407,30 +407,75 b' self: super: {'
     };
   };
   "elasticsearch" = super.buildPythonPackage {
-    name = "elasticsearch-2.3.0";
+    name = "elasticsearch-6.3.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."urllib3"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/10/35/5fd52c5f0b0ee405ed4b5195e8bce44c5e041787680dc7b94b8071cac600/elasticsearch-2.3.0.tar.gz";
-      sha256 = "10ad2dk73xsys9vajwsncibs69asa63w1hgwz6lz1prjpyi80c5y";
+      url = "https://files.pythonhosted.org/packages/9d/ce/c4664e8380e379a9402ecfbaf158e56396da90d520daba21cfa840e0eb71/elasticsearch-6.3.1.tar.gz";
+      sha256 = "12y93v0yn7a4xmf969239g8gb3l4cdkclfpbk1qc8hx5qkymrnma";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   "elasticsearch-dsl" = super.buildPythonPackage {
-    name = "elasticsearch-dsl-2.2.0";
+    name = "elasticsearch-dsl-6.3.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."python-dateutil"
       self."elasticsearch"
+      self."ipaddress"
+    ];
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/4c/0d/1549f50c591db6bb4e66cbcc8d34a6e537c3d89aa426b167c244fd46420a/elasticsearch-dsl-6.3.1.tar.gz";
+      sha256 = "1gh8a0shqi105k325hgwb9avrpdjh0mc6mxwfg9ba7g6lssb702z";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.asl20 ];
+    };
+  };
+  "elasticsearch1" = super.buildPythonPackage {
+    name = "elasticsearch1-1.10.0";
+    doCheck = false;
+    propagatedBuildInputs = [
+      self."urllib3"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/66/2f/52a086968788e58461641570f45c3207a52d46ebbe9b77dc22b6a8ffda66/elasticsearch-dsl-2.2.0.tar.gz";
-      sha256 = "1g4kxzxsdwlsl2a9kscmx11pafgimhj7y8wrfksv8pgvpkfb9fwr";
+      url = "https://files.pythonhosted.org/packages/a6/eb/73e75f9681fa71e3157b8ee878534235d57f24ee64f0e77f8d995fb57076/elasticsearch1-1.10.0.tar.gz";
+      sha256 = "0g89444kd5zwql4vbvyrmi2m6l6dcj6ga98j4hqxyyyz6z20aki2";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.asl20 ];
+    };
+  };
+  "elasticsearch1-dsl" = super.buildPythonPackage {
+    name = "elasticsearch1-dsl-0.0.12";
+    doCheck = false;
+    propagatedBuildInputs = [
+      self."six"
+      self."python-dateutil"
+      self."elasticsearch1"
+    ];
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/eb/9d/785342775cb10eddc9b8d7457d618a423b4f0b89d8b2b2d1bc27190d71db/elasticsearch1-dsl-0.0.12.tar.gz";
+      sha256 = "0ig1ly39v93hba0z975wnhbmzwj28w6w1sqlr2g7cn5spp732bhk";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.asl20 ];
+    };
+  };
+  "elasticsearch2" = super.buildPythonPackage {
+    name = "elasticsearch2-2.5.0";
+    doCheck = false;
+    propagatedBuildInputs = [
+      self."urllib3"
+    ];
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/84/77/63cf63d4ba11d913b5278406f2a37b0712bec6fc85edfb6151a33eaeba25/elasticsearch2-2.5.0.tar.gz";
+      sha256 = "0ky0q16lbvz022yv6q3pix7aamf026p1y994537ccjf0p0dxnbxr";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
@@ -818,11 +863,11 b' self: super: {'
     };
   };
   "markupsafe" = super.buildPythonPackage {
-    name = "markupsafe-1.0";
+    name = "markupsafe-1.1.0";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/4d/de/32d741db316d8fdb7680822dd37001ef7a448255de9699ab4bfcbdf4172b/MarkupSafe-1.0.tar.gz";
-      sha256 = "0rdn1s8x9ni7ss8rfiacj7x1085lx8mh2zdwqslnw8xc3l4nkgm6";
+      url = "https://files.pythonhosted.org/packages/ac/7e/1b4c2e05809a4414ebce0892fe1e32c14ace86ca7d50c70f00979ca9b3a3/MarkupSafe-1.1.0.tar.gz";
+      sha256 = "1lxirjypbdd3l9jl4vliilhfnhy7c7f2vlldqg1b0i74khn375sf";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
@@ -1271,11 +1316,11 b' self: super: {'
     };
   };
   "pyparsing" = super.buildPythonPackage {
-    name = "pyparsing-1.5.7";
+    name = "pyparsing-2.3.0";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/6f/2c/47457771c02a8ff0f302b695e094ec309e30452232bd79198ee94fda689f/pyparsing-1.5.7.tar.gz";
-      sha256 = "17z7ws076z977sclj628fvwrp8y9j2rvdjcsq42v129n1gwi8vk4";
+      url = "https://files.pythonhosted.org/packages/d0/09/3e6a5eeb6e04467b737d55f8bba15247ac0876f98fae659e58cd744430c6/pyparsing-2.3.0.tar.gz";
+      sha256 = "14k5v7n3xqw8kzf42x06bzp184spnlkya2dpjyflax6l3yrallzk";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
@@ -1642,7 +1687,7 b' self: super: {'
     };
   };
   "rhodecode-enterprise-ce" = super.buildPythonPackage {
-    name = "rhodecode-enterprise-ce-4.15.0";
+    name = "rhodecode-enterprise-ce-4.16.0";
     buildInputs = [
       self."pytest"
       self."py"
@@ -1788,7 +1833,7 b' self: super: {'
     };
   };
   "rhodecode-tools" = super.buildPythonPackage {
-    name = "rhodecode-tools-1.0.1";
+    name = "rhodecode-tools-1.1.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."click"
@@ -1797,14 +1842,16 b' self: super: {'
       self."mako"
       self."markupsafe"
       self."requests"
-      self."elasticsearch"
-      self."elasticsearch-dsl"
       self."urllib3"
       self."whoosh"
+      self."elasticsearch"
+      self."elasticsearch-dsl"
+      self."elasticsearch2"
+      self."elasticsearch1-dsl"
     ];
     src = fetchurl {
-      url = "https://code.rhodecode.com/rhodecode-tools-ce/archive/v1.0.1.tar.gz?md5=ffb5d6bcb855305b93cfe23ad42e500b";
-      sha256 = "0nr300s4sg685qs4wgbwlplwriawrwi6jq79z37frcnpyc89gpvm";
+      url = "https://code.rhodecode.com/rhodecode-tools-ce/archive/v1.1.0.tar.gz?md5=cc320c277cb2add546220290ac9be626";
+      sha256 = "1wbnnfrzyp0d4ys55vj5vnfrzfhwlqgdhc8yv8i6kwinizf8hfrn";
     };
     meta = {
       license = [ { fullName = "Apache 2.0 and Proprietary"; } ];
@@ -1848,11 +1895,11 b' self: super: {'
     };
   };
   "setuptools" = super.buildPythonPackage {
-    name = "setuptools-40.6.2";
+    name = "setuptools-40.6.3";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/b0/d1/8acb42f391cba52e35b131e442e80deffbb8d0676b93261d761b1f0ef8fb/setuptools-40.6.2.zip";
-      sha256 = "0r2c5hapirlzm34h7pl1lgkm6gk7bcrlrdj28qgsvaqg3f74vfw6";
+      url = "https://files.pythonhosted.org/packages/37/1b/b25507861991beeade31473868463dad0e58b1978c209de27384ae541b0b/setuptools-40.6.3.zip";
+      sha256 = "1y085dnk574sxw9aymdng9gijvrsbw86hsv9hqnhv7y4d6nlsirv";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
@@ -2043,11 +2090,11 b' self: super: {'
     };
   };
   "urllib3" = super.buildPythonPackage {
-    name = "urllib3-1.21";
+    name = "urllib3-1.24.1";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/34/95/7b28259d0006ed681c424cd71a668363265eac92b67dddd018eb9a22bff8/urllib3-1.21.tar.gz";
-      sha256 = "0irnj4wvh2y36s4q3l2vas9qr9m766w6w418nb490j3mf8a8zw6h";
+      url = "https://files.pythonhosted.org/packages/b1/53/37d82ab391393565f2f831b8eedbffd57db5a718216f82f1a8b4d381a1c1/urllib3-1.24.1.tar.gz";
+      sha256 = "08lwd9f3hqznyf32vnzwvp87pchx062nkbgyrf67rwlkgj0jk5fy";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];

requirements.txt

@@ -36,7 +36,7 b' kombu==4.2.0'
 lxml==4.2.5
 mako==1.0.7
 markdown==2.6.11
-markupsafe==1.0.0
+markupsafe==1.1.0
 msgpack-python==0.5.6
 pyotp==2.2.7
 packaging==15.2
@@ -51,7 +51,7 b' pycrypto==2.6.1'
 pycurl==7.43.0.2
 pyflakes==0.8.1
 pygments==2.3.0
-pyparsing==1.5.7
+pyparsing==2.3.0
 pyramid-beaker==0.8
 pyramid-debugtoolbar==4.4.0
 pyramid-jinja2==2.7
@@ -79,7 +79,7 b' subprocess32==3.5.2'
 supervisor==3.3.4
 tempita==0.5.2
 translationstring==1.3
-urllib3==1.21
+urllib3==1.24.1
 urlobject==2.4.3
 venusian==1.1.0
 weberror==0.10.3
@@ -123,7 +123,7 b' ipdb==0.11.0'
 ipython==5.1.0
 
 ## rhodecode-tools, special case
-https://code.rhodecode.com/rhodecode-tools-ce/archive/v1.0.1.tar.gz?md5=ffb5d6bcb855305b93cfe23ad42e500b#egg=rhodecode-tools==1.0.1
+https://code.rhodecode.com/rhodecode-tools-ce/archive/v1.1.0.tar.gz?md5=cc320c277cb2add546220290ac9be626#egg=rhodecode-tools==1.1.0
 
 ## appenlight
 appenlight-client==0.6.26

rhodecode/apps/admin/views/settings.py

@@ -666,8 +666,8 b' class AdminSettingsView(BaseAppView):'
         c = self.load_default_context()
         c.active = 'search'
 
-        searcher = searcher_from_config(self.request.registry.settings)
-        c.statistics = searcher.statistics(self.request.translate)
+        c.searcher = searcher_from_config(self.request.registry.settings)
+        c.statistics = c.searcher.statistics(self.request.translate)
 
         return self._get_template_context(c)
 

rhodecode/apps/home/views.py

@@ -246,9 +246,9 b' class HomeView(BaseAppView):'
             }
             for obj in acl_iter]
 
-    def _get_hash_commit_list(self, auth_user, query):
+    def _get_hash_commit_list(self, auth_user, searcher, query):
         org_query = query
-        if not query or len(query) < 3:
+        if not query or len(query) < 3 or not searcher:
             return []
 
         commit_hashes = re.compile('(?:commit:)([0-9a-f]{2,40})').findall(query)
@@ -257,9 +257,8 b' class HomeView(BaseAppView):'
             return []
         commit_hash = commit_hashes[0]
 
-        searcher = searcher_from_config(self.request.registry.settings)
         result = searcher.search(
-            'commit_id:%s*' % commit_hash, 'commit', auth_user,
+            'commit_id:{}*'.format(commit_hash), 'commit', auth_user,
             raise_on_exc=False)
 
         return [
@@ -303,6 +302,84 b' class HomeView(BaseAppView):'
         }
         return data
 
+    def _get_default_search_queries(self, search_context, searcher, query):
+        if not searcher:
+            return []
+        is_es_6 = searcher.is_es_6
+
+        queries = []
+        repo_group_name, repo_name, repo_context = None, None, None
+
+        # repo group context
+        if search_context.get('search_context[repo_group_name]'):
+            repo_group_name = search_context.get('search_context[repo_group_name]')
+        if search_context.get('search_context[repo_name]'):
+            repo_name = search_context.get('search_context[repo_name]')
+            repo_context = search_context.get('search_context[repo_view_type]')
+
+        if is_es_6 and repo_name:
+            def query_modifier():
+                qry = '{} repo_name.raw:{} '.format(
+                    query, searcher.escape_specials(repo_name))
+                return {'q': qry, 'type': 'content'}
+            label = u'Search for `{}` through files in this repository.'.format(query)
+            queries.append(
+                {
+                    'id': -10,
+                    'value': query,
+                    'value_display': label,
+                    'type': 'search',
+                    'url': h.route_path(
+                        'search_repo', repo_name=repo_name, _query=query_modifier())
+                }
+            )
+
+            def query_modifier():
+                qry = '{} repo_name.raw:{} '.format(
+                    query, searcher.escape_specials(repo_name))
+                return {'q': qry, 'type': 'commit'}
+            label = u'Search for `{}` through commits in this repository.'.format(query)
+            queries.append(
+                {
+                    'id': -10,
+                    'value': query,
+                    'value_display': label,
+                    'type': 'search',
+                    'url': h.route_path(
+                        'search_repo', repo_name=repo_name, _query=query_modifier())
+                }
+            )
+
+        elif is_es_6 and repo_group_name:
+            def query_modifier():
+                qry = '{} repo_name.raw:{} '.format(
+                    query, searcher.escape_specials(repo_group_name + '/*'))
+                return {'q': qry, 'type': 'content'}
+            label = u'Search for `{}` through files in this repository group'.format(query)
+            queries.append(
+                {
+                    'id': -20,
+                    'value': query,
+                    'value_display': label,
+                    'type': 'search',
+                    'url': h.route_path('search', _query=query_modifier())
+                }
+            )
+
+        if not queries:
+            queries.append(
+                {
+                    'id': -1,
+                    'value': query,
+                    'value_display': u'Search for: `{}`'.format(query),
+                    'type': 'search',
+                    'url': h.route_path('search',
+                                        _query={'q': query, 'type': 'content'})
+                }
+            )
+
+        return queries
+
     @LoginRequired()
     @view_config(
         route_name='goto_switcher_data', request_method='GET',
@@ -315,26 +392,21 b' class HomeView(BaseAppView):'
         query = self.request.GET.get('query')
         log.debug('generating main filter data, query %s', query)
 
-        default_search_val = u'Full text search for: `{}`'.format(query)
         res = []
         if not query:
             return {'suggestions': res}
 
-        res.append({
-            'id': -1,
-            'value': query,
-            'value_display': default_search_val,
-            'type': 'search',
-            'url': h.route_path(
-                'search', _query={'q': query})
-        })
-        repo_group_id = safe_int(self.request.GET.get('repo_group_id'))
+        searcher = searcher_from_config(self.request.registry.settings)
+        for _q in self._get_default_search_queries(self.request.GET, searcher, query):
+            res.append(_q)
+
+        repo_group_id = safe_int(self.request.GET.get('search_context[repo_group_id]'))
         if repo_group_id:
             repo_group = RepoGroup.get(repo_group_id)
             composed_hint = '{}/{}'.format(repo_group.group_name, query)
             show_hint = not query.startswith(repo_group.group_name)
             if repo_group and show_hint:
-                hint = u'Group search: `{}`'.format(composed_hint)
+                hint = u'Repository search inside: `{}`'.format(composed_hint)
                 res.append({
                     'id': -1,
                     'value': composed_hint,
@@ -351,7 +423,7 b' class HomeView(BaseAppView):'
         for serialized_repo in repos:
             res.append(serialized_repo)
 
-        # TODO(marcink): permissions for that ?
+        # TODO(marcink): should all logged in users be allowed to search others?
         allowed_user_search = self._rhodecode_user.username != User.DEFAULT_USER
         if allowed_user_search:
             users = self._get_user_list(query)
@@ -362,7 +434,7 b' class HomeView(BaseAppView):'
             for serialized_user_group in user_groups:
                 res.append(serialized_user_group)
 
-        commits = self._get_hash_commit_list(c.auth_user, query)
+        commits = self._get_hash_commit_list(c.auth_user, searcher, query)
         if commits:
             unique_repos = collections.OrderedDict()
             for commit in commits:

rhodecode/apps/search/views.py

@@ -45,11 +45,14 b' def search(request, tmpl_context, repo_n'
     errors = []
     try:
         search_params = schema.deserialize(
-            dict(search_query=request.GET.get('q'),
-                 search_type=request.GET.get('type'),
-                 search_sort=request.GET.get('sort'),
-                 page_limit=request.GET.get('page_limit'),
-                 requested_page=request.GET.get('page'))
+            dict(
+                search_query=request.GET.get('q'),
+                search_type=request.GET.get('type'),
+                search_sort=request.GET.get('sort'),
+                search_max_lines=request.GET.get('max_lines'),
+                page_limit=request.GET.get('page_limit'),
+                requested_page=request.GET.get('page'),
+             )
         )
     except validation_schema.Invalid as e:
         errors = e.children
@@ -57,12 +60,13 b' def search(request, tmpl_context, repo_n'
     def url_generator(**kw):
         q = urllib.quote(safe_str(search_query))
         return update_params(
-            "?q=%s&type=%s" % (q, safe_str(search_type)), **kw)
+            "?q=%s&type=%s&max_lines=%s" % (q, safe_str(search_type), search_max_lines), **kw)
 
     c = tmpl_context
     search_query = search_params.get('search_query')
     search_type = search_params.get('search_type')
     search_sort = search_params.get('search_sort')
+    search_max_lines = search_params.get('search_max_lines')
     if search_params.get('search_query'):
         page_limit = search_params['page_limit']
         requested_page = search_params['requested_page']

rhodecode/lib/helpers.py

@@ -48,7 +48,6 b' import bleach'
 from datetime import datetime
 from functools import partial
 from pygments.formatters.html import HtmlFormatter
-from pygments import highlight as code_highlight
 from pygments.lexers import (
     get_lexer_by_name, get_lexer_for_filename, get_lexer_for_mimetype)
 
@@ -81,12 +80,14 b' from rhodecode.lib.utils2 import str2boo'
 from rhodecode.lib.markup_renderer import MarkupRenderer, relative_links
 from rhodecode.lib.vcs.exceptions import CommitDoesNotExistError
 from rhodecode.lib.vcs.backends.base import BaseChangeset, EmptyCommit
+from rhodecode.lib.index.search_utils import get_matching_line_offsets
 from rhodecode.config.conf import DATE_FORMAT, DATETIME_FORMAT
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.db import Permission, User, Repository
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.settings import IssueTrackerSettingsModel
 
+
 log = logging.getLogger(__name__)
 
 
@@ -260,6 +261,21 b' def files_breadcrumbs(repo_name, commit_'
     return literal('/'.join(url_segments))
 
 
+def code_highlight(code, lexer, formatter, use_hl_filter=False):
+    """
+    Lex ``code`` with ``lexer`` and format it with the formatter ``formatter``.
+
+    If ``outfile`` is given and a valid file object (an object
+    with a ``write`` method), the result will be written to it, otherwise
+    it is returned as a string.
+    """
+    if use_hl_filter:
+        # add HL filter
+        from rhodecode.lib.index import search_utils
+        lexer.add_filter(search_utils.ElasticSearchHLFilter())
+    return pygments.format(pygments.lex(code, lexer), formatter)
+
+
 class CodeHtmlFormatter(HtmlFormatter):
     """
     My code Html Formatter for source codes
@@ -386,110 +402,9 b' class SearchContentCodeHtmlFormatter(Cod'
 
             current_line_number += 1
 
-
         yield 0, '</table>'
 
 
-def extract_phrases(text_query):
-    """
-    Extracts phrases from search term string making sure phrases
-    contained in double quotes are kept together - and discarding empty values
-    or fully whitespace values eg.
-
-    'some   text "a phrase" more' => ['some', 'text', 'a phrase', 'more']
-
-    """
-
-    in_phrase = False
-    buf = ''
-    phrases = []
-    for char in text_query:
-        if in_phrase:
-            if char == '"': # end phrase
-                phrases.append(buf)
-                buf = ''
-                in_phrase = False
-                continue
-            else:
-                buf += char
-                continue
-        else:
-            if char == '"': # start phrase
-                in_phrase = True
-                phrases.append(buf)
-                buf = ''
-                continue
-            elif char == ' ':
-                phrases.append(buf)
-                buf = ''
-                continue
-            else:
-                buf += char
-
-    phrases.append(buf)
-    phrases = [phrase.strip() for phrase in phrases if phrase.strip()]
-    return phrases
-
-
-def get_matching_offsets(text, phrases):
-    """
-    Returns a list of string offsets in `text` that the list of `terms` match
-
-    >>> get_matching_offsets('some text here', ['some', 'here'])
-    [(0, 4), (10, 14)]
-
-    """
-    offsets = []
-    for phrase in phrases:
-        for match in re.finditer(phrase, text):
-            offsets.append((match.start(), match.end()))
-
-    return offsets
-
-
-def normalize_text_for_matching(x):
-    """
-    Replaces all non alnum characters to spaces and lower cases the string,
-    useful for comparing two text strings without punctuation
-    """
-    return re.sub(r'[^\w]', ' ', x.lower())
-
-
-def get_matching_line_offsets(lines, terms):
-    """ Return a set of `lines` indices (starting from 1) matching a
-    text search query, along with `context` lines above/below matching lines
-
-    :param lines: list of strings representing lines
-    :param terms: search term string to match in lines eg. 'some text'
-    :param context: number of lines above/below a matching line to add to result
-    :param max_lines: cut off for lines of interest
-     eg.
-
-    text = '''
-    words words words
-    words words words
-    some text some
-    words words words
-    words words words
-    text here what
-    '''
-    get_matching_line_offsets(text, 'text', context=1)
-    {3: [(5, 9)], 6: [(0, 4)]]
-
-    """
-    matching_lines = {}
-    phrases = [normalize_text_for_matching(phrase)
-               for phrase in extract_phrases(terms)]
-
-    for line_index, line in enumerate(lines, start=1):
-        match_offsets = get_matching_offsets(
-            normalize_text_for_matching(line), phrases)
-        if match_offsets:
-            matching_lines[line_index] = match_offsets
-
-    return matching_lines
-
-
 def hsv_to_rgb(h, s, v):
     """ Convert hsv color values to rgb """
 
@@ -1904,25 +1819,6 b' def journal_filter_help(request):'
     ).format(actions=actions)
 
 
-def search_filter_help(searcher, request):
-    _ = request.translate
-
-    terms = ''
-    return _(
-        'Example filter terms for `{searcher}` search:\n' +
-        '{terms}\n' +
-        'Generate wildcards using \'*\' character:\n' +
-        '     "repo_name:vcs*" - search everything starting with \'vcs\'\n' +
-        '     "repo_name:*vcs*" - search for repository containing \'vcs\'\n' +
-        '\n' +
-        'Optional AND / OR operators in queries\n' +
-        '     "repo_name:vcs OR repo_name:test"\n' +
-        '     "owner:test AND repo_name:test*"\n' +
-        'More: {search_doc}'
-    ).format(searcher=searcher.name,
-             terms=terms, search_doc=searcher.query_lang_doc)
-
-
 def not_mapped_error(repo_name):
     from rhodecode.translation import _
     flash(_('%s repository is not mapped to db perhaps'
@@ -2107,3 +2003,15 b' def go_import_header(request, db_repo=No'
 def reviewer_as_json(*args, **kwargs):
     from rhodecode.apps.repository.utils import reviewer_as_json as _reviewer_as_json
     return _reviewer_as_json(*args, **kwargs)
+
+
+def get_repo_view_type(request):
+    route_name = request.matched_route.name
+    route_to_view_type = {
+        'repo_changelog': 'changelog',
+        'repo_files': 'files',
+        'repo_summary': 'summary',
+        'repo_commit': 'commit'
+
+    }
+    return route_to_view_type.get(route_name)

rhodecode/lib/index/__init__.py

@@ -25,15 +25,27 b' Index schema for RhodeCode'
 import importlib
 import logging
 
+from rhodecode.lib.index.search_utils import normalize_text_for_matching
+
 log = logging.getLogger(__name__)
 
 # leave defaults for backward compat
 default_searcher = 'rhodecode.lib.index.whoosh'
 default_location = '%(here)s/data/index'
 
+ES_VERSION_2 = '2'
+ES_VERSION_6 = '6'
+# for legacy reasons we keep 2 compat as default
+DEFAULT_ES_VERSION = ES_VERSION_2
 
-class BaseSearch(object):
+from rhodecode_tools.lib.fts_index.elasticsearch_engine_6 import \
+    ES_CONFIG  # pragma: no cover
+
+
+class BaseSearcher(object):
     query_lang_doc = ''
+    es_version = None
+    name = None
 
     def __init__(self):
         pass
@@ -45,15 +57,42 b' class BaseSearch(object):'
                raise_on_exc=True):
         raise Exception('NotImplemented')
 
+    @staticmethod
+    def query_to_mark(query, default_field=None):
+        """
+        Formats the query to mark token for jquery.mark.js highlighting. ES could
+        have a different format optionally.
 
-def searcher_from_config(config, prefix='search.'):
+        :param default_field:
+        :param query:
+        """
+        return ' '.join(normalize_text_for_matching(query).split())
+
+    @property
+    def is_es_6(self):
+        return self.es_version == ES_VERSION_6
+
+    def get_handlers(self):
+        return {}
+
+
+def search_config(config, prefix='search.'):
     _config = {}
     for key in config.keys():
         if key.startswith(prefix):
             _config[key[len(prefix):]] = config[key]
+    return _config
+
+
+def searcher_from_config(config, prefix='search.'):
+    _config = search_config(config, prefix)
 
     if 'location' not in _config:
         _config['location'] = default_location
+    if 'es_version' not in _config:
+        # use old legacy ES version set to 2
+        _config['es_version'] = '2'
+
     imported = importlib.import_module(_config.get('module', default_searcher))
-    searcher = imported.Search(config=_config)
+    searcher = imported.Searcher(config=_config)
     return searcher

rhodecode/lib/index/whoosh.py

@@ -33,7 +33,7 b' from whoosh.index import create_in, open'
 from whoosh.qparser import QueryParser, QueryParserError
 
 import rhodecode.lib.helpers as h
-from rhodecode.lib.index import BaseSearch
+from rhodecode.lib.index import BaseSearcher
 from rhodecode.lib.utils2 import safe_unicode
 
 log = logging.getLogger(__name__)
@@ -59,13 +59,13 b' FRAGMENTER = ContextFragmenter(200)'
 log = logging.getLogger(__name__)
 
 
-class Search(BaseSearch):
+class WhooshSearcher(BaseSearcher):
     # this also shows in UI
     query_lang_doc = 'http://whoosh.readthedocs.io/en/latest/querylang.html'
     name = 'whoosh'
 
     def __init__(self, config):
-        super(Search, self).__init__()
+        super(Searcher, self).__init__()
         self.config = config
         if not os.path.isdir(self.config['location']):
             os.makedirs(self.config['location'])
@@ -162,16 +162,17 b' class Search(BaseSearch):'
         _ = translator
         stats = [
             {'key': _('Index Type'), 'value': 'Whoosh'},
+            {'sep': True},
+
             {'key': _('File Index'), 'value': str(self.file_index)},
-            {'key': _('Indexed documents'),
-             'value': self.file_index.doc_count()},
-            {'key': _('Last update'),
-             'value': h.time_to_datetime(self.file_index.last_modified())},
+            {'key': _('Indexed documents'), 'value': self.file_index.doc_count()},
+            {'key': _('Last update'), 'value': h.time_to_datetime(self.file_index.last_modified())},
+
+            {'sep': True},
+
             {'key': _('Commit index'), 'value': str(self.commit_index)},
-            {'key': _('Indexed documents'),
-             'value': str(self.commit_index.doc_count())},
-            {'key': _('Last update'),
-             'value': h.time_to_datetime(self.commit_index.last_modified())}
+            {'key': _('Indexed documents'), 'value': str(self.commit_index.doc_count())},
+            {'key': _('Last update'), 'value': h.time_to_datetime(self.commit_index.last_modified())}
         ]
         return stats
 
@@ -227,6 +228,9 b' class Search(BaseSearch):'
         return self.searcher
 
 
+Searcher = WhooshSearcher
+
+
 class WhooshResultWrapper(object):
     def __init__(self, search_type, total_hits, results):
         self.search_type = search_type
@@ -263,6 +267,8 b' class WhooshResultWrapper(object):'
         # TODO: marcink: this feels like an overkill, there's a lot of data
         # inside hit object, and we don't need all
         res = dict(hit)
+        # elastic search uses that, we set it empty so it fallbacks to regular HL logic
+        res['content_highlight'] = ''
 
         f_path = ''  # pragma: no cover
         if self.search_type in ['content', 'path']:

rhodecode/lib/utils2.py

@@ -1009,3 +1009,14 b' def glob2re(pat):'
         else:
             res = res + re.escape(c)
     return res + '\Z(?ms)'
+
+
+def parse_byte_string(size_str):
+    match = re.match(r'(\d+)(MB|KB)', size_str, re.IGNORECASE)
+    if not match:
+        raise ValueError('Given size:%s is invalid, please make sure '
+                         'to use format of <num>(MB|KB)' % size_str)
+
+    _parts = match.groups()
+    num, type_ = _parts
+    return long(num) * {'mb': 1024*1024, 'kb': 1024}[type_.lower()]

rhodecode/lib/vcs/utils/__init__.py

@@ -58,7 +58,7 b' def author_name(author):'
     to get the username
     """
 
-    if not author or not '@' in author:
+    if not author or '@' not in author:
         return author
     else:
         return author.replace(author_email(author), '').replace('<', '')\

rhodecode/model/validation_schema/schemas/search_schema.py

@@ -34,6 +34,9 b' class SearchParamsSchema(colander.Mappin'
         colander.String(),
         missing='newfirst',
         validator=colander.OneOf(['oldfirst', 'newfirst']))
+    search_max_lines = colander.SchemaNode(
+        colander.Integer(),
+        missing=10)
     page_limit = colander.SchemaNode(
         colander.Integer(),
         missing=10,

rhodecode/public/css/code-block.less

@@ -572,6 +572,7 b' div.annotatediv { margin-left: 2px; marg'
 .code-highlight, /* TODO: dan: merge codehilite into code-highlight */
 /* This can be generated with `pygmentize -S default -f html` */
 .codehilite {
+    .c-ElasticMatch { background-color: #faffa6; padding: 0.2em;}
     .hll { background-color: #ffffcc }
     .c { color: #408080; font-style: italic } /* Comment */
     .err, .codehilite .err { border: none } /* Error */
@@ -640,6 +641,7 b' div.annotatediv { margin-left: 2px; marg'
     .vi { color: #19177C } /* Name.Variable.Instance */
     .vm { color: #19177C } /* Name.Variable.Magic */
     .il { color: #666666 } /* Literal.Number.Integer.Long */
+
 }
 
 /* customized pre blocks for markdown/rst */

rhodecode/public/css/type.less

@@ -166,7 +166,6 b' small,'
 
 mark,
 .mark {
-  background-color: @rclightblue;
   padding: .2em;
 }
 

rhodecode/templates/admin/settings/settings_search.mako

@@ -5,8 +5,13 b''
     <div class="panel-body">
         <dl class="dl-horizontal">
           % for stat in c.statistics:
-          <dt>${stat['key']}</dt>
-          <dd>${stat['value']}</dd>
+              % if stat.get('sep'):
+                  <dt></dt>
+                  <dd>--</dd>
+              % else:
+                  <dt>${stat['key']}</dt>
+                  <dd>${stat['value']}</dd>
+              % endif
           % endfor
         </dl>
     </div>

rhodecode/templates/base/root.mako

@@ -7,9 +7,12 b" go_import_header = ''"
 if hasattr(c, 'rhodecode_db_repo'):
     c.template_context['repo_type'] = c.rhodecode_db_repo.repo_type
     c.template_context['repo_landing_commit'] = c.rhodecode_db_repo.landing_rev[1]
+    ## check repo context
+    c.template_context['repo_view_type'] = h.get_repo_view_type(request)
 
 if getattr(c, 'repo_group', None):
     c.template_context['repo_group_id'] = c.repo_group.group_id
+    c.template_context['repo_group_name'] = c.repo_group.group_name
 
 if getattr(c, 'rhodecode_user', None) and c.rhodecode_user.user_id:
     c.template_context['rhodecode_user']['username'] = c.rhodecode_user.username
@@ -23,6 +26,12 b" c.template_context['default_user'] = {"
     'username': h.DEFAULT_USER,
     'user_id': 1
 }
+c.template_context['search_context'] = {
+    'repo_group_id': c.template_context.get('repo_group_id'),
+    'repo_group_name': c.template_context.get('repo_group_name'),
+    'repo_name': c.template_context.get('repo_name'),
+    'repo_view_type': c.template_context.get('repo_view_type'),
+}
 
 %>
 <html xmlns="http://www.w3.org/1999/xhtml">

rhodecode/templates/search/search.mako

@@ -18,10 +18,7 b''
   %else:
     ${_('Search inside all accessible repositories')}
   %endif
-  %if c.cur_query:
-    »
-    ${c.cur_query}
-  %endif
+
 </%def>
 
 <%def name="menu_bar_nav()">
@@ -59,7 +56,8 b''
         <div class="fields">
             ${h.text('q', c.cur_query, placeholder="Enter query...")}
 
-            ${h.select('type',c.search_type,[('content',_('File contents')), ('commit',_('Commit messages')), ('path',_('File names')),],id='id_search_type')}
+            ${h.select('type',c.search_type,[('content',_('Files')), ('path',_('File path')),('commit',_('Commits'))],id='id_search_type')}
+            ${h.hidden('max_lines', '10')}
             <input type="submit" value="${_('Search')}" class="btn"/>
             <br/>
 
@@ -72,8 +70,54 b''
               </span>
             % endfor
             <div class="field">
-                <p class="filterexample" style="position: inherit" onclick="$('#search-help').toggle()">${_('Example Queries')}</p>
-                <pre id="search-help" style="display: none">${h.tooltip(h.search_filter_help(c.searcher, request))}</pre>
+                <p class="filterexample" style="position: inherit" onclick="$('#search-help').toggle()">${_('Query Langague examples')}</p>
+<pre id="search-help" style="display: none">\
+
+% if c.searcher.name == 'whoosh':
+Example filter terms for `Whoosh` search:
+query lang: <a href="${c.searcher.query_lang_doc}">Whoosh Query Language</a>
+Whoosh has limited query capabilities. For advanced search use ElasticSearch 6 from RhodeCode EE edition.
+
+Generate wildcards using '*' character:
+  "repo_name:vcs*" - search everything starting with 'vcs'
+  "repo_name:*vcs*" - search for repository containing 'vcs'
+
+Optional AND / OR operators in queries
+  "repo_name:vcs OR repo_name:test"
+  "owner:test AND repo_name:test*" AND extension:py
+
+Move advanced search is available via ElasticSearch6 backend in EE edition.
+% elif c.searcher.name == 'elasticsearch' and c.searcher.es_version == '2':
+Example filter terms for `ElasticSearch-${c.searcher.es_version}`search:
+ElasticSearch-2 has limited query capabilities. For advanced search use ElasticSearch 6 from RhodeCode EE edition.
+
+search type: content (File Content)
+  indexed fields: content
+
+  # search for `fix` string in all files
+  fix
+
+search type: commit (Commit message)
+  indexed fields: message
+
+search type: path (File name)
+  indexed fields: path
+
+% else:
+Example filter terms for `ElasticSearch-${c.searcher.es_version}`search:
+query lang: <a href="${c.searcher.query_lang_doc}">ES 6 Query Language</a>
+The reserved characters needed espace by `\`: + - = && || > < ! ( ) { } [ ] ^ " ~ * ? : \ /
+% for handler in c.searcher.get_handlers().values():
+
+search type: ${handler.search_type_label}
+  *indexed fields*: ${', '.join( [('\n    ' if x[0]%4==0 else '')+x[1] for x in enumerate(handler.es_6_field_names)])}
+  % for entry in handler.es_6_example_queries:
+  ${entry.rstrip()}
+  % endfor
+% endfor
+
+% endif
+</pre>
             </div>
 
             <div class="field">${c.runtime}</div>
@@ -96,6 +140,7 b''
 </div>
 <script>
     $(document).ready(function(){
+        $('#q').autoGrowInput();
         $("#id_search_type").select2({
             'containerCssClass': "drop-menu",
             'dropdownCssClass': "drop-menu-dropdown",

rhodecode/templates/search/search_commit.mako

@@ -1,5 +1,7 b''
 <%namespace name="base" file="/base/base.mako"/>
 
+% if c.formatted_results:
+
 <table class="rctable search-results">
     <tr>
         <th>${_('Repository')}</th>
@@ -50,14 +52,20 b''
                 </td>
 
                 <td class="td-user author">
-                    ${base.gravatar_with_user(entry['author'])}
+                    <%
+                    ## es6 stores this as object
+                    author = entry['author']
+                    if isinstance(author, dict):
+                        author = author['email']
+                    %>
+                    ${base.gravatar_with_user(author)}
                 </td>
             </tr>
         % endif
     %endfor
 </table>
 
-%if c.cur_query and c.formatted_results:
+%if c.cur_query:
 <div class="pagination-wh pagination-left">
     ${c.formatted_results.pager('$link_previous ~2~ $link_next')}
 </div>
@@ -79,4 +87,16 b''
         target_expand.addClass('open');
       }
     });
+
+    $(".message.td-description").mark(
+        "${c.searcher.query_to_mark(c.cur_query, 'message')}",
+        {
+            "className": 'match',
+            "accuracy": "complementary",
+            "ignorePunctuation": ":._(){}[]!'+=".split("")
+        }
+    );
+
 </script>
+
+% endif

rhodecode/templates/search/search_content.mako

@@ -1,33 +1,10 b''
-<%def name="highlight_text_file(terms, text, url, line_context=3,
-                                max_lines=10,
-                                mimetype=None, filepath=None)">
-<%
-lines = text.split('\n')
-lines_of_interest = set()
-matching_lines = h.get_matching_line_offsets(lines, terms)
-shown_matching_lines = 0
 
-for line_number in matching_lines:
-    if len(lines_of_interest) < max_lines:
-        lines_of_interest |= set(range(
-            max(line_number - line_context, 0),
-            min(line_number + line_context, len(lines) + 1)))
-        shown_matching_lines += 1
-
-%>
-${h.code_highlight(
-    text,
-    h.get_lexer_safe(
-        mimetype=mimetype,
-        filepath=filepath,
-    ),
-    h.SearchContentCodeHtmlFormatter(
-        linenos=True,
-        cssclass="code-highlight",
-        url=url,
-        query_terms=terms,
-        only_line_numbers=lines_of_interest
-))|n}
+<%def name="highlight_text_file(has_matched_content, file_content, lexer, html_formatter, matching_lines, shown_matching_lines, url, use_hl_filter)">
+% if has_matched_content:
+    ${h.code_highlight(file_content, lexer, html_formatter, use_hl_filter=use_hl_filter)|n}
+% else:
+    ${_('No content matched')} <br/>
+% endif
 
 %if len(matching_lines) > shown_matching_lines:
 <a href="${url}">
@@ -37,12 +14,52 b' for line_number in matching_lines:'
 </%def>
 
 <div class="search-results">
+<% query_mark = c.searcher.query_to_mark(c.cur_query, 'content') %>
+
 %for entry in c.formatted_results:
+
+  <%
+    file_content = entry['content_highlight'] or entry['content']
+    mimetype = entry.get('mimetype')
+    filepath = entry.get('path')
+    max_lines = h.safe_int(request.GET.get('max_lines', '10'))
+    line_context = h.safe_int(request.GET.get('line_contenxt', '3'))
+
+    match_file_url=h.route_path('repo_files',repo_name=entry['repository'], commit_id=entry.get('commit_id', 'tip'),f_path=entry['f_path'], _query={"mark": query_mark})
+    terms = c.cur_query
+
+    if c.searcher.is_es_6:
+        # use empty terms so we default to markers usage
+        total_lines, matching_lines = h.get_matching_line_offsets(file_content, terms=None)
+    else:
+        total_lines, matching_lines = h.get_matching_line_offsets(file_content, terms)
+
+    shown_matching_lines = 0
+    lines_of_interest = set()
+    for line_number in matching_lines:
+        if len(lines_of_interest) < max_lines:
+            lines_of_interest |= set(range(
+                max(line_number - line_context, 0),
+                min(line_number + line_context, total_lines + 1)))
+            shown_matching_lines += 1
+    lexer = h.get_lexer_safe(mimetype=mimetype, filepath=filepath)
+
+    html_formatter = h.SearchContentCodeHtmlFormatter(
+        linenos=True,
+        cssclass="code-highlight",
+        url=match_file_url,
+        query_terms=terms,
+        only_line_numbers=lines_of_interest
+    )
+
+    has_matched_content = len(lines_of_interest) >= 1
+
+  %>
   ## search results are additionally filtered, and this check is just a safe gate
   % if h.HasRepoPermissionAny('repository.write','repository.read','repository.admin')(entry['repository'], 'search results content check'):
       <div id="codeblock" class="codeblock">
         <div class="codeblock-header">
-          <h2>
+          <h1>
             %if h.get_repo_type_by_name(entry.get('repository')) == 'hg':
                 <i class="icon-hg"></i>
             %elif h.get_repo_type_by_name(entry.get('repository')) == 'git':
@@ -51,18 +68,39 b' for line_number in matching_lines:'
                 <i class="icon-svn"></i>
             %endif
             ${h.link_to(entry['repository'], h.route_path('repo_summary',repo_name=entry['repository']))}
-          </h2>
+          </h1>
+
           <div class="stats">
-            ${h.link_to(h.literal(entry['f_path']), h.route_path('repo_files',repo_name=entry['repository'],commit_id=entry.get('commit_id', 'tip'),f_path=entry['f_path']))}
-            %if entry.get('lines'):
-              | ${entry.get('lines', 0.)} ${_ungettext('line', 'lines', entry.get('lines', 0.))}
-            %endif
-            %if entry.get('size'):
-              | ${h.format_byte_size_binary(entry['size'])}
-            %endif
-            %if entry.get('mimetype'):
-              | ${entry.get('mimetype', "unknown mimetype")}
-            %endif
+            <span class="stats-filename">
+                <strong>
+                    <i class="icon-file-text"></i>
+                    ${h.link_to(h.literal(entry['f_path']), h.route_path('repo_files',repo_name=entry['repository'],commit_id=entry.get('commit_id', 'tip'),f_path=entry['f_path']))}
+                </strong>
+            </span>
+            <span class="item last"><i class="tooltip icon-clipboard clipboard-action" data-clipboard-text="${entry['f_path']}" title="${_('Copy the full path')}"></i></span>
+            <br/>
+            <span class="stats-first-item">
+                ${len(matching_lines)} ${_ungettext('search match', 'search matches', len(matching_lines))}
+            </span>
+
+            <span >
+                %if entry.get('lines'):
+                  | ${entry.get('lines', 0.)} ${_ungettext('line', 'lines', entry.get('lines', 0.))}
+                %endif
+            </span>
+
+            <span>
+                %if entry.get('size'):
+                  | ${h.format_byte_size_binary(entry['size'])}
+                %endif
+            </span>
+
+            <span>
+                %if entry.get('mimetype'):
+                  | ${entry.get('mimetype', "unknown mimetype")}
+                %endif
+            </span>
+
           </div>
           <div class="buttons">
             <a id="file_history_overview_full" href="${h.route_path('repo_changelog_file',repo_name=entry.get('repository',''),commit_id=entry.get('commit_id', 'tip'),f_path=entry.get('f_path',''))}">
@@ -74,10 +112,19 b' for line_number in matching_lines:'
           </div>
         </div>
         <div class="code-body search-code-body">
-            ${highlight_text_file(c.cur_query, entry['content'],
-            url=h.route_path('repo_files',repo_name=entry['repository'],commit_id=entry.get('commit_id', 'tip'),f_path=entry['f_path']),
-            mimetype=entry.get('mimetype'), filepath=entry.get('path'))}
+
+            ${highlight_text_file(
+                has_matched_content=has_matched_content,
+                file_content=file_content,
+                lexer=lexer,
+                html_formatter=html_formatter,
+                matching_lines=matching_lines,
+                shown_matching_lines=shown_matching_lines,
+                url=match_file_url,
+                use_hl_filter=c.searcher.is_es_6
+            )}
         </div>
+
       </div>
     % endif
 %endfor
@@ -91,10 +138,14 b' for line_number in matching_lines:'
 %if c.cur_query:
 <script type="text/javascript">
 $(function(){
-  $(".code").mark(
-    '${' '.join(h.normalize_text_for_matching(c.cur_query).split())}',
-    {"className": 'match',
-  });
+  $(".search-code-body").mark(
+      "${query_mark}",
+          {
+              "className": 'match',
+              "accuracy": "complementary",
+              "ignorePunctuation": ":._(){}[]!'+=".split("")
+          }
+  );
 })
 </script>
-%endif
  No newline at end of file
+%endif

rhodecode/templates/search/search_path.mako

@@ -1,3 +1,5 b''
+% if c.formatted_results:
+
 <table class="rctable search-results">
     <tr>
         <th>${_('Repository')}</th>
@@ -27,8 +29,10 b''
     %endfor
 </table>
 
-%if c.cur_query and c.formatted_results:
+%if c.cur_query:
 <div class="pagination-wh pagination-left">
     ${c.formatted_results.pager('$link_previous ~2~ $link_next')}
 </div>
-%endif
  No newline at end of file
+%endif
+
+% endif

rhodecode/tests/lib/test_helpers.py

@@ -208,44 +208,3 b' def test_get_visual_attr(baseapp):'
 def test_chop_at(test_text, inclusive, expected_text):
     assert helpers.chop_at_smart(
         test_text, '\n', inclusive, '...') == expected_text
-
-
[e-mail]('test_text, expected_output', [
-    ('some text', ['some', 'text']),
-    ('some    text', ['some', 'text']),
-    ('some text "with  a phrase"', ['some', 'text', 'with  a phrase']),
-    ('"a phrase" "another phrase"', ['a phrase', 'another phrase']),
-    ('"justphrase"', ['justphrase']),
-    ('""', []),
-    ('', []),
-    ('  ', []),
-    ('"   "', []),
-])
-def test_extract_phrases(test_text, expected_output):
-    assert helpers.extract_phrases(test_text) == expected_output
-
-
[e-mail]('test_text, text_phrases, expected_output', [
-    ('some text here', ['some', 'here'], [(0, 4), (10, 14)]),
-    ('here here there', ['here'], [(0, 4), (5, 9), (11, 15)]),
-    ('irrelevant', ['not found'], []),
-    ('irrelevant', ['not found'], []),
-])
-def test_get_matching_offsets(test_text, text_phrases, expected_output):
-    assert helpers.get_matching_offsets(
-        test_text, text_phrases) == expected_output
-
-
-def test_normalize_text_for_matching():
-    assert helpers.normalize_text_for_matching(
-        'OJjfe)*#$*@)$JF*)3r2f80h') == 'ojjfe        jf  3r2f80h'
-
-
-def test_get_matching_line_offsets():
-    assert helpers.get_matching_line_offsets([
-        'words words words',
-        'words words words',
-        'some text some',
-        'words words words',
-        'words words words',
-        'text here what'], 'text') == {3: [(5, 9)], 6: [(0, 4)]}