##// END OF EJS Templates
search: add support for elastic search 6...
dan -
r3319:b8fd1d7a default
parent child
Show More
@@ -0,0 +1,257
1 # -*- coding: utf-8 -*-
2
3 # Copyright (C) 2012-2018 RhodeCode GmbH
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 import re
21
22 import pygments.filter
23 import pygments.filters
24 from pygments.token import Comment
25
26 HL_BEG_MARKER = '__RCSearchHLMarkBEG__'
27 HL_END_MARKER = '__RCSearchHLMarkEND__'
28 HL_MARKER_RE = '{}(.*?){}'.format(HL_BEG_MARKER, HL_END_MARKER)
29
30
31 class ElasticSearchHLFilter(pygments.filters.Filter):
32 _names = [HL_BEG_MARKER, HL_END_MARKER]
33
34 def __init__(self, **options):
35 pygments.filters.Filter.__init__(self, **options)
36
37 def filter(self, lexer, stream):
38 def tokenize(_value):
39 for token in re.split('({}|{})'.format(
40 self._names[0], self._names[1]), _value):
41 if token:
42 yield token
43
44 hl = False
45 for ttype, value in stream:
46
47 if self._names[0] in value or self._names[1] in value:
48 for item in tokenize(value):
49 if item == self._names[0]:
50 # skip marker, but start HL
51 hl = True
52 continue
53 elif item == self._names[1]:
54 hl = False
55 continue
56
57 if hl:
58 yield Comment.ElasticMatch, item
59 else:
60 yield ttype, item
61 else:
62 if hl:
63 yield Comment.ElasticMatch, value
64 else:
65 yield ttype, value
66
67
68 def extract_phrases(text_query):
69 """
70 Extracts phrases from search term string making sure phrases
71 contained in double quotes are kept together - and discarding empty values
72 or fully whitespace values eg.
73
74 'some text "a phrase" more' => ['some', 'text', 'a phrase', 'more']
75
76 """
77
78 in_phrase = False
79 buf = ''
80 phrases = []
81 for char in text_query:
82 if in_phrase:
83 if char == '"': # end phrase
84 phrases.append(buf)
85 buf = ''
86 in_phrase = False
87 continue
88 else:
89 buf += char
90 continue
91 else:
92 if char == '"': # start phrase
93 in_phrase = True
94 phrases.append(buf)
95 buf = ''
96 continue
97 elif char == ' ':
98 phrases.append(buf)
99 buf = ''
100 continue
101 else:
102 buf += char
103
104 phrases.append(buf)
105 phrases = [phrase.strip() for phrase in phrases if phrase.strip()]
106 return phrases
107
108
109 def get_matching_phrase_offsets(text, phrases):
110 """
111 Returns a list of string offsets in `text` that the list of `terms` match
112
113 >>> get_matching_phrase_offsets('some text here', ['some', 'here'])
114 [(0, 4), (10, 14)]
115
116 """
117 phrases = phrases or []
118 offsets = []
119
120 for phrase in phrases:
121 for match in re.finditer(phrase, text):
122 offsets.append((match.start(), match.end()))
123
124 return offsets
125
126
127 def get_matching_markers_offsets(text, markers=None):
128 """
129 Returns a list of string offsets in `text` that the are between matching markers
130
131 >>> get_matching_markers_offsets('$1some$2 text $1here$2 marked', ['\$1(.*?)\$2'])
132 [(0, 5), (16, 22)]
133
134 """
135 markers = markers or [HL_MARKER_RE]
136 offsets = []
137
138 if markers:
139 for mark in markers:
140 for match in re.finditer(mark, text):
141 offsets.append((match.start(), match.end()))
142
143 return offsets
144
145
146 def normalize_text_for_matching(x):
147 """
148 Replaces all non alfanum characters to spaces and lower cases the string,
149 useful for comparing two text strings without punctuation
150 """
151 return re.sub(r'[^\w]', ' ', x.lower())
152
153
154 def get_matching_line_offsets(lines, terms=None, markers=None):
155 """ Return a set of `lines` indices (starting from 1) matching a
156 text search query, along with `context` lines above/below matching lines
157
158 :param lines: list of strings representing lines
159 :param terms: search term string to match in lines eg. 'some text'
160 :param markers: instead of terms, use highlight markers instead that
161 mark beginning and end for matched item. eg. ['START(.*?)END']
162
163 eg.
164
165 text = '''
166 words words words
167 words words words
168 some text some
169 words words words
170 words words words
171 text here what
172 '''
173 get_matching_line_offsets(text, 'text', context=1)
174 6, {3: [(5, 9)], 6: [(0, 4)]]
175
176 """
177 matching_lines = {}
178 line_index = 0
179
180 if terms:
181 phrases = [normalize_text_for_matching(phrase)
182 for phrase in extract_phrases(terms)]
183
184 for line_index, line in enumerate(lines.splitlines(), start=1):
185 normalized_line = normalize_text_for_matching(line)
186 match_offsets = get_matching_phrase_offsets(normalized_line, phrases)
187 if match_offsets:
188 matching_lines[line_index] = match_offsets
189
190 else:
191 markers = markers or [HL_MARKER_RE]
192 for line_index, line in enumerate(lines.splitlines(), start=1):
193 match_offsets = get_matching_markers_offsets(line, markers=markers)
194 if match_offsets:
195 matching_lines[line_index] = match_offsets
196
197 return line_index, matching_lines
198
199
200 def lucene_query_parser():
201 # from pyparsing lucene_grammar
202 from pyparsing import (
203 Literal, CaselessKeyword, Forward, Regex, QuotedString, Suppress,
204 Optional, Group, infixNotation, opAssoc, ParserElement, pyparsing_common)
205
206 ParserElement.enablePackrat()
207
208 COLON, LBRACK, RBRACK, LBRACE, RBRACE, TILDE, CARAT = map(Literal, ":[]{}~^")
209 LPAR, RPAR = map(Suppress, "()")
210 and_, or_, not_, to_ = map(CaselessKeyword, "AND OR NOT TO".split())
211 keyword = and_ | or_ | not_ | to_
212
213 expression = Forward()
214
215 valid_word = Regex(r'([a-zA-Z0-9*_+.-]|\\[!(){}\[\]^"~*?\\:])+').setName("word")
216 valid_word.setParseAction(
217 lambda t: t[0]
218 .replace('\\\\', chr(127))
219 .replace('\\', '')
220 .replace(chr(127), '\\')
221 )
222
223 string = QuotedString('"')
224
225 required_modifier = Literal("+")("required")
226 prohibit_modifier = Literal("-")("prohibit")
227 integer = Regex(r"\d+").setParseAction(lambda t: int(t[0]))
228 proximity_modifier = Group(TILDE + integer("proximity"))
229 number = pyparsing_common.fnumber()
230 fuzzy_modifier = TILDE + Optional(number, default=0.5)("fuzzy")
231
232 term = Forward()
233 field_name = valid_word().setName("fieldname")
234 incl_range_search = Group(LBRACK + term("lower") + to_ + term("upper") + RBRACK)
235 excl_range_search = Group(LBRACE + term("lower") + to_ + term("upper") + RBRACE)
236 range_search = incl_range_search("incl_range") | excl_range_search("excl_range")
237 boost = (CARAT + number("boost"))
238
239 string_expr = Group(string + proximity_modifier) | string
240 word_expr = Group(valid_word + fuzzy_modifier) | valid_word
241 term << (Optional(field_name("field") + COLON) +
242 (word_expr | string_expr | range_search | Group(
243 LPAR + expression + RPAR)) +
244 Optional(boost))
245 term.setParseAction(lambda t: [t] if 'field' in t or 'boost' in t else None)
246
247 expression << infixNotation(
248 term,
249 [
250 (required_modifier | prohibit_modifier, 1, opAssoc.RIGHT),
251 ((not_ | '!').setParseAction(lambda: "NOT"), 1, opAssoc.RIGHT),
252 ((and_ | '&&').setParseAction(lambda: "AND"), 2, opAssoc.LEFT),
253 (Optional(or_ | '||').setParseAction(lambda: "OR"), 2, opAssoc.LEFT),
254 ]
255 )
256
257 return expression
@@ -0,0 +1,100
1 # -*- coding: utf-8 -*-
2
3 # Copyright (C) 2010-2018 RhodeCode GmbH
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
21 import copy
22 import mock
23 import pytest
24
25 from rhodecode.lib.index import search_utils
26
27
28 @pytest.mark.parametrize('test_text, expected_output', [
29 ('some text', ['some', 'text']),
30 ('some text', ['some', 'text']),
31 ('some text "with a phrase"', ['some', 'text', 'with a phrase']),
32 ('"a phrase" "another phrase"', ['a phrase', 'another phrase']),
33 ('"justphrase"', ['justphrase']),
34 ('""', []),
35 ('', []),
36 (' ', []),
37 ('" "', []),
38 ])
39 def test_extract_phrases(test_text, expected_output):
40 assert search_utils.extract_phrases(test_text) == expected_output
41
42
43 @pytest.mark.parametrize('test_text, text_phrases, expected_output', [
44 ('some text here', ['some', 'here'], [(0, 4), (10, 14)]),
45 ('here here there', ['here'], [(0, 4), (5, 9), (11, 15)]),
46 ('irrelevant', ['not found'], []),
47 ('irrelevant', ['not found'], []),
48 ])
49 def test_get_matching_phrase_offsets(test_text, text_phrases, expected_output):
50 assert search_utils.get_matching_phrase_offsets(
51 test_text, text_phrases) == expected_output
52
53
54 @pytest.mark.parametrize('test_text, text_phrases, expected_output', [
55 ('__RCSearchHLMarkBEG__some__RCSearchHLMarkEND__ text __RCSearchHLMarkBEG__here__RCSearchHLMarkEND__', [], [(0, 46), (52, 98)]),
56 ('__RCSearchHLMarkBEG__here__RCSearchHLMarkEND__ __RCSearchHLMarkBEG__here__RCSearchHLMarkEND__ there', [], [(0, 46), (47, 93)]),
57 ('some text __RCSearchHLMarkBEG__here__RCSearchHLMarkEND__', [], [(10, 56)]),
58 ('__RCSearchHLMarkBEG__here__RCSearchHLMarkEND__ __RCSearchHLMarkBEG__here__RCSearchHLMarkEND__ __RCSearchHLMarkBEG__there__RCSearchHLMarkEND__', [], [(0, 46), (47, 93), (94, 141)]),
59 ('irrelevant', ['not found'], []),
60 ('irrelevant', ['not found'], []),
61 ])
62 def test_get_matching_marker_offsets(test_text, text_phrases, expected_output):
63
64 assert search_utils.get_matching_markers_offsets(test_text) == expected_output
65
66
67 def test_normalize_text_for_matching():
68 assert search_utils.normalize_text_for_matching(
69 'OJjfe)*#$*@)$JF*)3r2f80h') == 'ojjfe jf 3r2f80h'
70
71
72 def test_get_matching_line_offsets():
73 words = '\n'.join([
74 'words words words',
75 'words words words',
76 'some text some',
77 'words words words',
78 'words words words',
79 'text here what'
80 ])
81 total_lines, matched_offsets = \
82 search_utils.get_matching_line_offsets(words, terms='text')
83 assert total_lines == 6
84 assert matched_offsets == {3: [(5, 9)], 6: [(0, 4)]}
85
86
87 def test_get_matching_line_offsets_using_markers():
88 words = '\n'.join([
89 'words words words',
90 'words words words',
91 'some __1__text__2__ some',
92 'words words words',
93 'words words words',
94 '__1__text__2__ here what'
95 ])
96 total_lines, matched_offsets = \
97 search_utils.get_matching_line_offsets(words, terms=None,
98 markers=['__1__(.*?)__2__'])
99 assert total_lines == 6
100 assert matched_offsets == {3: [(5, 19)], 6: [(0, 14)]}
@@ -407,30 +407,75 self: super: {
407 };
407 };
408 };
408 };
409 "elasticsearch" = super.buildPythonPackage {
409 "elasticsearch" = super.buildPythonPackage {
410 name = "elasticsearch-2.3.0";
410 name = "elasticsearch-6.3.1";
411 doCheck = false;
411 doCheck = false;
412 propagatedBuildInputs = [
412 propagatedBuildInputs = [
413 self."urllib3"
413 self."urllib3"
414 ];
414 ];
415 src = fetchurl {
415 src = fetchurl {
416 url = "https://files.pythonhosted.org/packages/10/35/5fd52c5f0b0ee405ed4b5195e8bce44c5e041787680dc7b94b8071cac600/elasticsearch-2.3.0.tar.gz";
416 url = "https://files.pythonhosted.org/packages/9d/ce/c4664e8380e379a9402ecfbaf158e56396da90d520daba21cfa840e0eb71/elasticsearch-6.3.1.tar.gz";
417 sha256 = "10ad2dk73xsys9vajwsncibs69asa63w1hgwz6lz1prjpyi80c5y";
417 sha256 = "12y93v0yn7a4xmf969239g8gb3l4cdkclfpbk1qc8hx5qkymrnma";
418 };
418 };
419 meta = {
419 meta = {
420 license = [ pkgs.lib.licenses.asl20 ];
420 license = [ pkgs.lib.licenses.asl20 ];
421 };
421 };
422 };
422 };
423 "elasticsearch-dsl" = super.buildPythonPackage {
423 "elasticsearch-dsl" = super.buildPythonPackage {
424 name = "elasticsearch-dsl-2.2.0";
424 name = "elasticsearch-dsl-6.3.1";
425 doCheck = false;
425 doCheck = false;
426 propagatedBuildInputs = [
426 propagatedBuildInputs = [
427 self."six"
427 self."six"
428 self."python-dateutil"
428 self."python-dateutil"
429 self."elasticsearch"
429 self."elasticsearch"
430 self."ipaddress"
431 ];
432 src = fetchurl {
433 url = "https://files.pythonhosted.org/packages/4c/0d/1549f50c591db6bb4e66cbcc8d34a6e537c3d89aa426b167c244fd46420a/elasticsearch-dsl-6.3.1.tar.gz";
434 sha256 = "1gh8a0shqi105k325hgwb9avrpdjh0mc6mxwfg9ba7g6lssb702z";
435 };
436 meta = {
437 license = [ pkgs.lib.licenses.asl20 ];
438 };
439 };
440 "elasticsearch1" = super.buildPythonPackage {
441 name = "elasticsearch1-1.10.0";
442 doCheck = false;
443 propagatedBuildInputs = [
444 self."urllib3"
430 ];
445 ];
431 src = fetchurl {
446 src = fetchurl {
432 url = "https://files.pythonhosted.org/packages/66/2f/52a086968788e58461641570f45c3207a52d46ebbe9b77dc22b6a8ffda66/elasticsearch-dsl-2.2.0.tar.gz";
447 url = "https://files.pythonhosted.org/packages/a6/eb/73e75f9681fa71e3157b8ee878534235d57f24ee64f0e77f8d995fb57076/elasticsearch1-1.10.0.tar.gz";
433 sha256 = "1g4kxzxsdwlsl2a9kscmx11pafgimhj7y8wrfksv8pgvpkfb9fwr";
448 sha256 = "0g89444kd5zwql4vbvyrmi2m6l6dcj6ga98j4hqxyyyz6z20aki2";
449 };
450 meta = {
451 license = [ pkgs.lib.licenses.asl20 ];
452 };
453 };
454 "elasticsearch1-dsl" = super.buildPythonPackage {
455 name = "elasticsearch1-dsl-0.0.12";
456 doCheck = false;
457 propagatedBuildInputs = [
458 self."six"
459 self."python-dateutil"
460 self."elasticsearch1"
461 ];
462 src = fetchurl {
463 url = "https://files.pythonhosted.org/packages/eb/9d/785342775cb10eddc9b8d7457d618a423b4f0b89d8b2b2d1bc27190d71db/elasticsearch1-dsl-0.0.12.tar.gz";
464 sha256 = "0ig1ly39v93hba0z975wnhbmzwj28w6w1sqlr2g7cn5spp732bhk";
465 };
466 meta = {
467 license = [ pkgs.lib.licenses.asl20 ];
468 };
469 };
470 "elasticsearch2" = super.buildPythonPackage {
471 name = "elasticsearch2-2.5.0";
472 doCheck = false;
473 propagatedBuildInputs = [
474 self."urllib3"
475 ];
476 src = fetchurl {
477 url = "https://files.pythonhosted.org/packages/84/77/63cf63d4ba11d913b5278406f2a37b0712bec6fc85edfb6151a33eaeba25/elasticsearch2-2.5.0.tar.gz";
478 sha256 = "0ky0q16lbvz022yv6q3pix7aamf026p1y994537ccjf0p0dxnbxr";
434 };
479 };
435 meta = {
480 meta = {
436 license = [ pkgs.lib.licenses.asl20 ];
481 license = [ pkgs.lib.licenses.asl20 ];
@@ -818,11 +863,11 self: super: {
818 };
863 };
819 };
864 };
820 "markupsafe" = super.buildPythonPackage {
865 "markupsafe" = super.buildPythonPackage {
821 name = "markupsafe-1.0";
866 name = "markupsafe-1.1.0";
822 doCheck = false;
867 doCheck = false;
823 src = fetchurl {
868 src = fetchurl {
824 url = "https://files.pythonhosted.org/packages/4d/de/32d741db316d8fdb7680822dd37001ef7a448255de9699ab4bfcbdf4172b/MarkupSafe-1.0.tar.gz";
869 url = "https://files.pythonhosted.org/packages/ac/7e/1b4c2e05809a4414ebce0892fe1e32c14ace86ca7d50c70f00979ca9b3a3/MarkupSafe-1.1.0.tar.gz";
825 sha256 = "0rdn1s8x9ni7ss8rfiacj7x1085lx8mh2zdwqslnw8xc3l4nkgm6";
870 sha256 = "1lxirjypbdd3l9jl4vliilhfnhy7c7f2vlldqg1b0i74khn375sf";
826 };
871 };
827 meta = {
872 meta = {
828 license = [ pkgs.lib.licenses.bsdOriginal ];
873 license = [ pkgs.lib.licenses.bsdOriginal ];
@@ -1271,11 +1316,11 self: super: {
1271 };
1316 };
1272 };
1317 };
1273 "pyparsing" = super.buildPythonPackage {
1318 "pyparsing" = super.buildPythonPackage {
1274 name = "pyparsing-1.5.7";
1319 name = "pyparsing-2.3.0";
1275 doCheck = false;
1320 doCheck = false;
1276 src = fetchurl {
1321 src = fetchurl {
1277 url = "https://files.pythonhosted.org/packages/6f/2c/47457771c02a8ff0f302b695e094ec309e30452232bd79198ee94fda689f/pyparsing-1.5.7.tar.gz";
1322 url = "https://files.pythonhosted.org/packages/d0/09/3e6a5eeb6e04467b737d55f8bba15247ac0876f98fae659e58cd744430c6/pyparsing-2.3.0.tar.gz";
1278 sha256 = "17z7ws076z977sclj628fvwrp8y9j2rvdjcsq42v129n1gwi8vk4";
1323 sha256 = "14k5v7n3xqw8kzf42x06bzp184spnlkya2dpjyflax6l3yrallzk";
1279 };
1324 };
1280 meta = {
1325 meta = {
1281 license = [ pkgs.lib.licenses.mit ];
1326 license = [ pkgs.lib.licenses.mit ];
@@ -1642,7 +1687,7 self: super: {
1642 };
1687 };
1643 };
1688 };
1644 "rhodecode-enterprise-ce" = super.buildPythonPackage {
1689 "rhodecode-enterprise-ce" = super.buildPythonPackage {
1645 name = "rhodecode-enterprise-ce-4.15.0";
1690 name = "rhodecode-enterprise-ce-4.16.0";
1646 buildInputs = [
1691 buildInputs = [
1647 self."pytest"
1692 self."pytest"
1648 self."py"
1693 self."py"
@@ -1788,7 +1833,7 self: super: {
1788 };
1833 };
1789 };
1834 };
1790 "rhodecode-tools" = super.buildPythonPackage {
1835 "rhodecode-tools" = super.buildPythonPackage {
1791 name = "rhodecode-tools-1.0.1";
1836 name = "rhodecode-tools-1.1.0";
1792 doCheck = false;
1837 doCheck = false;
1793 propagatedBuildInputs = [
1838 propagatedBuildInputs = [
1794 self."click"
1839 self."click"
@@ -1797,14 +1842,16 self: super: {
1797 self."mako"
1842 self."mako"
1798 self."markupsafe"
1843 self."markupsafe"
1799 self."requests"
1844 self."requests"
1800 self."elasticsearch"
1801 self."elasticsearch-dsl"
1802 self."urllib3"
1845 self."urllib3"
1803 self."whoosh"
1846 self."whoosh"
1847 self."elasticsearch"
1848 self."elasticsearch-dsl"
1849 self."elasticsearch2"
1850 self."elasticsearch1-dsl"
1804 ];
1851 ];
1805 src = fetchurl {
1852 src = fetchurl {
1806 url = "https://code.rhodecode.com/rhodecode-tools-ce/archive/v1.0.1.tar.gz?md5=ffb5d6bcb855305b93cfe23ad42e500b";
1853 url = "https://code.rhodecode.com/rhodecode-tools-ce/archive/v1.1.0.tar.gz?md5=cc320c277cb2add546220290ac9be626";
1807 sha256 = "0nr300s4sg685qs4wgbwlplwriawrwi6jq79z37frcnpyc89gpvm";
1854 sha256 = "1wbnnfrzyp0d4ys55vj5vnfrzfhwlqgdhc8yv8i6kwinizf8hfrn";
1808 };
1855 };
1809 meta = {
1856 meta = {
1810 license = [ { fullName = "Apache 2.0 and Proprietary"; } ];
1857 license = [ { fullName = "Apache 2.0 and Proprietary"; } ];
@@ -1848,11 +1895,11 self: super: {
1848 };
1895 };
1849 };
1896 };
1850 "setuptools" = super.buildPythonPackage {
1897 "setuptools" = super.buildPythonPackage {
1851 name = "setuptools-40.6.2";
1898 name = "setuptools-40.6.3";
1852 doCheck = false;
1899 doCheck = false;
1853 src = fetchurl {
1900 src = fetchurl {
1854 url = "https://files.pythonhosted.org/packages/b0/d1/8acb42f391cba52e35b131e442e80deffbb8d0676b93261d761b1f0ef8fb/setuptools-40.6.2.zip";
1901 url = "https://files.pythonhosted.org/packages/37/1b/b25507861991beeade31473868463dad0e58b1978c209de27384ae541b0b/setuptools-40.6.3.zip";
1855 sha256 = "0r2c5hapirlzm34h7pl1lgkm6gk7bcrlrdj28qgsvaqg3f74vfw6";
1902 sha256 = "1y085dnk574sxw9aymdng9gijvrsbw86hsv9hqnhv7y4d6nlsirv";
1856 };
1903 };
1857 meta = {
1904 meta = {
1858 license = [ pkgs.lib.licenses.mit ];
1905 license = [ pkgs.lib.licenses.mit ];
@@ -2043,11 +2090,11 self: super: {
2043 };
2090 };
2044 };
2091 };
2045 "urllib3" = super.buildPythonPackage {
2092 "urllib3" = super.buildPythonPackage {
2046 name = "urllib3-1.21";
2093 name = "urllib3-1.24.1";
2047 doCheck = false;
2094 doCheck = false;
2048 src = fetchurl {
2095 src = fetchurl {
2049 url = "https://files.pythonhosted.org/packages/34/95/7b28259d0006ed681c424cd71a668363265eac92b67dddd018eb9a22bff8/urllib3-1.21.tar.gz";
2096 url = "https://files.pythonhosted.org/packages/b1/53/37d82ab391393565f2f831b8eedbffd57db5a718216f82f1a8b4d381a1c1/urllib3-1.24.1.tar.gz";
2050 sha256 = "0irnj4wvh2y36s4q3l2vas9qr9m766w6w418nb490j3mf8a8zw6h";
2097 sha256 = "08lwd9f3hqznyf32vnzwvp87pchx062nkbgyrf67rwlkgj0jk5fy";
2051 };
2098 };
2052 meta = {
2099 meta = {
2053 license = [ pkgs.lib.licenses.mit ];
2100 license = [ pkgs.lib.licenses.mit ];
@@ -36,7 +36,7 kombu==4.2.0
36 lxml==4.2.5
36 lxml==4.2.5
37 mako==1.0.7
37 mako==1.0.7
38 markdown==2.6.11
38 markdown==2.6.11
39 markupsafe==1.0.0
39 markupsafe==1.1.0
40 msgpack-python==0.5.6
40 msgpack-python==0.5.6
41 pyotp==2.2.7
41 pyotp==2.2.7
42 packaging==15.2
42 packaging==15.2
@@ -51,7 +51,7 pycrypto==2.6.1
51 pycurl==7.43.0.2
51 pycurl==7.43.0.2
52 pyflakes==0.8.1
52 pyflakes==0.8.1
53 pygments==2.3.0
53 pygments==2.3.0
54 pyparsing==1.5.7
54 pyparsing==2.3.0
55 pyramid-beaker==0.8
55 pyramid-beaker==0.8
56 pyramid-debugtoolbar==4.4.0
56 pyramid-debugtoolbar==4.4.0
57 pyramid-jinja2==2.7
57 pyramid-jinja2==2.7
@@ -79,7 +79,7 subprocess32==3.5.2
79 supervisor==3.3.4
79 supervisor==3.3.4
80 tempita==0.5.2
80 tempita==0.5.2
81 translationstring==1.3
81 translationstring==1.3
82 urllib3==1.21
82 urllib3==1.24.1
83 urlobject==2.4.3
83 urlobject==2.4.3
84 venusian==1.1.0
84 venusian==1.1.0
85 weberror==0.10.3
85 weberror==0.10.3
@@ -123,7 +123,7 ipdb==0.11.0
123 ipython==5.1.0
123 ipython==5.1.0
124
124
125 ## rhodecode-tools, special case
125 ## rhodecode-tools, special case
126 https://code.rhodecode.com/rhodecode-tools-ce/archive/v1.0.1.tar.gz?md5=ffb5d6bcb855305b93cfe23ad42e500b#egg=rhodecode-tools==1.0.1
126 https://code.rhodecode.com/rhodecode-tools-ce/archive/v1.1.0.tar.gz?md5=cc320c277cb2add546220290ac9be626#egg=rhodecode-tools==1.1.0
127
127
128 ## appenlight
128 ## appenlight
129 appenlight-client==0.6.26
129 appenlight-client==0.6.26
@@ -666,8 +666,8 class AdminSettingsView(BaseAppView):
666 c = self.load_default_context()
666 c = self.load_default_context()
667 c.active = 'search'
667 c.active = 'search'
668
668
669 searcher = searcher_from_config(self.request.registry.settings)
669 c.searcher = searcher_from_config(self.request.registry.settings)
670 c.statistics = searcher.statistics(self.request.translate)
670 c.statistics = c.searcher.statistics(self.request.translate)
671
671
672 return self._get_template_context(c)
672 return self._get_template_context(c)
673
673
@@ -246,9 +246,9 class HomeView(BaseAppView):
246 }
246 }
247 for obj in acl_iter]
247 for obj in acl_iter]
248
248
249 def _get_hash_commit_list(self, auth_user, query):
249 def _get_hash_commit_list(self, auth_user, searcher, query):
250 org_query = query
250 org_query = query
251 if not query or len(query) < 3:
251 if not query or len(query) < 3 or not searcher:
252 return []
252 return []
253
253
254 commit_hashes = re.compile('(?:commit:)([0-9a-f]{2,40})').findall(query)
254 commit_hashes = re.compile('(?:commit:)([0-9a-f]{2,40})').findall(query)
@@ -257,9 +257,8 class HomeView(BaseAppView):
257 return []
257 return []
258 commit_hash = commit_hashes[0]
258 commit_hash = commit_hashes[0]
259
259
260 searcher = searcher_from_config(self.request.registry.settings)
261 result = searcher.search(
260 result = searcher.search(
262 'commit_id:%s*' % commit_hash, 'commit', auth_user,
261 'commit_id:{}*'.format(commit_hash), 'commit', auth_user,
263 raise_on_exc=False)
262 raise_on_exc=False)
264
263
265 return [
264 return [
@@ -303,6 +302,84 class HomeView(BaseAppView):
303 }
302 }
304 return data
303 return data
305
304
305 def _get_default_search_queries(self, search_context, searcher, query):
306 if not searcher:
307 return []
308 is_es_6 = searcher.is_es_6
309
310 queries = []
311 repo_group_name, repo_name, repo_context = None, None, None
312
313 # repo group context
314 if search_context.get('search_context[repo_group_name]'):
315 repo_group_name = search_context.get('search_context[repo_group_name]')
316 if search_context.get('search_context[repo_name]'):
317 repo_name = search_context.get('search_context[repo_name]')
318 repo_context = search_context.get('search_context[repo_view_type]')
319
320 if is_es_6 and repo_name:
321 def query_modifier():
322 qry = '{} repo_name.raw:{} '.format(
323 query, searcher.escape_specials(repo_name))
324 return {'q': qry, 'type': 'content'}
325 label = u'Search for `{}` through files in this repository.'.format(query)
326 queries.append(
327 {
328 'id': -10,
329 'value': query,
330 'value_display': label,
331 'type': 'search',
332 'url': h.route_path(
333 'search_repo', repo_name=repo_name, _query=query_modifier())
334 }
335 )
336
337 def query_modifier():
338 qry = '{} repo_name.raw:{} '.format(
339 query, searcher.escape_specials(repo_name))
340 return {'q': qry, 'type': 'commit'}
341 label = u'Search for `{}` through commits in this repository.'.format(query)
342 queries.append(
343 {
344 'id': -10,
345 'value': query,
346 'value_display': label,
347 'type': 'search',
348 'url': h.route_path(
349 'search_repo', repo_name=repo_name, _query=query_modifier())
350 }
351 )
352
353 elif is_es_6 and repo_group_name:
354 def query_modifier():
355 qry = '{} repo_name.raw:{} '.format(
356 query, searcher.escape_specials(repo_group_name + '/*'))
357 return {'q': qry, 'type': 'content'}
358 label = u'Search for `{}` through files in this repository group'.format(query)
359 queries.append(
360 {
361 'id': -20,
362 'value': query,
363 'value_display': label,
364 'type': 'search',
365 'url': h.route_path('search', _query=query_modifier())
366 }
367 )
368
369 if not queries:
370 queries.append(
371 {
372 'id': -1,
373 'value': query,
374 'value_display': u'Search for: `{}`'.format(query),
375 'type': 'search',
376 'url': h.route_path('search',
377 _query={'q': query, 'type': 'content'})
378 }
379 )
380
381 return queries
382
306 @LoginRequired()
383 @LoginRequired()
307 @view_config(
384 @view_config(
308 route_name='goto_switcher_data', request_method='GET',
385 route_name='goto_switcher_data', request_method='GET',
@@ -315,26 +392,21 class HomeView(BaseAppView):
315 query = self.request.GET.get('query')
392 query = self.request.GET.get('query')
316 log.debug('generating main filter data, query %s', query)
393 log.debug('generating main filter data, query %s', query)
317
394
318 default_search_val = u'Full text search for: `{}`'.format(query)
319 res = []
395 res = []
320 if not query:
396 if not query:
321 return {'suggestions': res}
397 return {'suggestions': res}
322
398
323 res.append({
399 searcher = searcher_from_config(self.request.registry.settings)
324 'id': -1,
400 for _q in self._get_default_search_queries(self.request.GET, searcher, query):
325 'value': query,
401 res.append(_q)
326 'value_display': default_search_val,
402
327 'type': 'search',
403 repo_group_id = safe_int(self.request.GET.get('search_context[repo_group_id]'))
328 'url': h.route_path(
329 'search', _query={'q': query})
330 })
331 repo_group_id = safe_int(self.request.GET.get('repo_group_id'))
332 if repo_group_id:
404 if repo_group_id:
333 repo_group = RepoGroup.get(repo_group_id)
405 repo_group = RepoGroup.get(repo_group_id)
334 composed_hint = '{}/{}'.format(repo_group.group_name, query)
406 composed_hint = '{}/{}'.format(repo_group.group_name, query)
335 show_hint = not query.startswith(repo_group.group_name)
407 show_hint = not query.startswith(repo_group.group_name)
336 if repo_group and show_hint:
408 if repo_group and show_hint:
337 hint = u'Group search: `{}`'.format(composed_hint)
409 hint = u'Repository search inside: `{}`'.format(composed_hint)
338 res.append({
410 res.append({
339 'id': -1,
411 'id': -1,
340 'value': composed_hint,
412 'value': composed_hint,
@@ -351,7 +423,7 class HomeView(BaseAppView):
351 for serialized_repo in repos:
423 for serialized_repo in repos:
352 res.append(serialized_repo)
424 res.append(serialized_repo)
353
425
354 # TODO(marcink): permissions for that ?
426 # TODO(marcink): should all logged in users be allowed to search others?
355 allowed_user_search = self._rhodecode_user.username != User.DEFAULT_USER
427 allowed_user_search = self._rhodecode_user.username != User.DEFAULT_USER
356 if allowed_user_search:
428 if allowed_user_search:
357 users = self._get_user_list(query)
429 users = self._get_user_list(query)
@@ -362,7 +434,7 class HomeView(BaseAppView):
362 for serialized_user_group in user_groups:
434 for serialized_user_group in user_groups:
363 res.append(serialized_user_group)
435 res.append(serialized_user_group)
364
436
365 commits = self._get_hash_commit_list(c.auth_user, query)
437 commits = self._get_hash_commit_list(c.auth_user, searcher, query)
366 if commits:
438 if commits:
367 unique_repos = collections.OrderedDict()
439 unique_repos = collections.OrderedDict()
368 for commit in commits:
440 for commit in commits:
@@ -45,11 +45,14 def search(request, tmpl_context, repo_n
45 errors = []
45 errors = []
46 try:
46 try:
47 search_params = schema.deserialize(
47 search_params = schema.deserialize(
48 dict(search_query=request.GET.get('q'),
48 dict(
49 search_type=request.GET.get('type'),
49 search_query=request.GET.get('q'),
50 search_sort=request.GET.get('sort'),
50 search_type=request.GET.get('type'),
51 page_limit=request.GET.get('page_limit'),
51 search_sort=request.GET.get('sort'),