search: added per-backend sorting fields.

- unified naming to be more user friendly
- only sort on supported fields that each backend exposes
- fixed few cases of sort errors and added additional tests

rhodecode/api/tests/test_fts_search.py

@@ -1,93 +1,115 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import pytest
 from rhodecode.tests import HG_REPO
 from rhodecode.api.tests.utils import (
     build_data, api_call, assert_error, assert_ok)
 
 
 @pytest.mark.usefixtures("testuser_api", "app")
 class TestApiSearch(object):
 
+    @pytest.mark.parametrize("sort_dir", [
+        "asc",
+        "desc",
+    ])
+    @pytest.mark.parametrize("sort", [
+        "xxx",
+        "author_email",
+        "date",
+        "message",
+    ])
     @pytest.mark.parametrize("query, expected_hits, expected_paths", [
         ('todo', 23, [
             'vcs/backends/hg/inmemory.py',
             'vcs/tests/test_git.py']),
         ('extension:rst installation', 6, [
             'docs/index.rst',
             'docs/installation.rst']),
         ('def repo', 87, [
             'vcs/tests/test_git.py',
             'vcs/tests/test_changesets.py']),
         ('repository:%s def test' % HG_REPO, 18, [
             'vcs/tests/test_git.py',
             'vcs/tests/test_changesets.py']),
         ('"def main"', 9, [
             'vcs/__init__.py',
             'vcs/tests/__init__.py',
             'vcs/utils/progressbar.py']),
         ('owner:test_admin', 358, [
             'vcs/tests/base.py',
             'MANIFEST.in',
             'vcs/utils/termcolors.py',
             'docs/theme/ADC/static/documentation.png']),
         ('owner:test_admin def main', 72, [
             'vcs/__init__.py',
             'vcs/tests/test_utils_filesize.py',
             'vcs/tests/test_cli.py']),
         ('owner:michał test', 0, []),
     ])
-    def test_search_content_results(self, query, expected_hits, expected_paths):
+    def test_search_content_results(self, sort_dir, sort, query, expected_hits, expected_paths):
         id_, params = build_data(
             self.apikey_regular, 'search',
             search_query=query,
+            search_sort='{}:{}'.format(sort_dir, sort),
             search_type='content')
 
         response = api_call(self.app, params)
         json_response = response.json
 
         assert json_response['result']['item_count'] == expected_hits
         paths = [x['f_path'] for x in json_response['result']['results']]
 
         for expected_path in expected_paths:
             assert expected_path in paths
 
+    @pytest.mark.parametrize("sort_dir", [
+        "asc",
+        "desc",
+    ])
+    @pytest.mark.parametrize("sort", [
+        "xxx",
+        "date",
+        "file",
+        "size",
+    ])
     @pytest.mark.parametrize("query, expected_hits, expected_paths", [
         ('readme.rst', 3, []),
         ('test*', 75, []),
         ('*model*', 1, []),
         ('extension:rst', 48, []),
         ('extension:rst api', 24, []),
     ])
-    def test_search_file_paths(self, query, expected_hits, expected_paths):
+    def test_search_file_paths(self, sort_dir, sort, query, expected_hits, expected_paths):
         id_, params = build_data(
             self.apikey_regular, 'search',
             search_query=query,
+            search_sort='{}:{}'.format(sort_dir, sort),
             search_type='path')
 
         response = api_call(self.app, params)
         json_response = response.json
 
         assert json_response['result']['item_count'] == expected_hits
         paths = [x['f_path'] for x in json_response['result']['results']]
 
         for expected_path in expected_paths:
             assert expected_path in paths

rhodecode/apps/search/views.py

@@ -1,169 +1,173 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import urllib
 from pyramid.view import view_config
 from webhelpers.util import update_params
 
 from rhodecode.apps._base import BaseAppView, RepoAppView, RepoGroupAppView
 from rhodecode.lib.auth import (
     LoginRequired, HasRepoPermissionAnyDecorator, HasRepoGroupPermissionAnyDecorator)
 from rhodecode.lib.helpers import Page
 from rhodecode.lib.utils2 import safe_str
 from rhodecode.lib.index import searcher_from_config
 from rhodecode.model import validation_schema
 from rhodecode.model.validation_schema.schemas import search_schema
 
 log = logging.getLogger(__name__)
 
 
 def perform_search(request, tmpl_context, repo_name=None, repo_group_name=None):
     searcher = searcher_from_config(request.registry.settings)
     formatted_results = []
     execution_time = ''
 
     schema = search_schema.SearchParamsSchema()
     search_tags = []
     search_params = {}
     errors = []
 
     try:
         search_params = schema.deserialize(
             dict(
                 search_query=request.GET.get('q'),
                 search_type=request.GET.get('type'),
                 search_sort=request.GET.get('sort'),
                 search_max_lines=request.GET.get('max_lines'),
                 page_limit=request.GET.get('page_limit'),
                 requested_page=request.GET.get('page'),
              )
         )
     except validation_schema.Invalid as e:
         errors = e.children
 
     def url_generator(**kw):
         q = urllib.quote(safe_str(search_query))
         return update_params(
             "?q=%s&type=%s&max_lines=%s&sort=%s" % (
                 q, safe_str(search_type), search_max_lines, search_sort), **kw)
 
     c = tmpl_context
     search_query = search_params.get('search_query')
     search_type = search_params.get('search_type')
     search_sort = search_params.get('search_sort')
     search_max_lines = search_params.get('search_max_lines')
     if search_params.get('search_query'):
         page_limit = search_params['page_limit']
         requested_page = search_params['requested_page']
 
         try:
             search_result = searcher.search(
                 search_query, search_type, c.auth_user, repo_name, repo_group_name,
                 requested_page=requested_page, page_limit=page_limit, sort=search_sort)
 
             formatted_results = Page(
                 search_result['results'], page=requested_page,
                 item_count=search_result['count'],
                 items_per_page=page_limit, url=url_generator)
         finally:
             searcher.cleanup()
 
         search_tags = searcher.extract_search_tags(search_query)
 
         if not search_result['error']:
             execution_time = '%s results (%.4f seconds)' % (
                 search_result['count'],
                 search_result['runtime'])
         elif not errors:
             node = schema['search_query']
             errors = [
                 validation_schema.Invalid(node, search_result['error'])]
 
     c.perm_user = c.auth_user
     c.repo_name = repo_name
     c.repo_group_name = repo_group_name
     c.url_generator = url_generator
     c.errors = errors
     c.formatted_results = formatted_results
     c.runtime = execution_time
     c.cur_query = search_query
     c.search_type = search_type
     c.searcher = searcher
     c.search_tags = search_tags
 
     direction, sort_field = searcher.get_sort(search_type, search_sort)
-    c.sort = '{}:{}'.format(direction, sort_field)
-    c.sort_tag = sort_field
+    sort_definition = searcher.sort_def(search_type, direction, sort_field)
+    c.sort = ''
+    c.sort_tag = None
     c.sort_tag_dir = direction
+    if sort_definition:
+        c.sort = '{}:{}'.format(direction, sort_field)
+        c.sort_tag = sort_field
 
 
 class SearchView(BaseAppView):
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         return c
 
     @LoginRequired()
     @view_config(
         route_name='search', request_method='GET',
         renderer='rhodecode:templates/search/search.mako')
     def search(self):
         c = self.load_default_context()
         perform_search(self.request, c)
         return self._get_template_context(c)
 
 
 class SearchRepoView(RepoAppView):
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         c.active = 'search'
         return c
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='search_repo', request_method='GET',
         renderer='rhodecode:templates/search/search.mako')
     @view_config(
         route_name='search_repo_alt', request_method='GET',
         renderer='rhodecode:templates/search/search.mako')
     def search_repo(self):
         c = self.load_default_context()
         perform_search(self.request, c, repo_name=self.db_repo_name)
         return self._get_template_context(c)
 
 
 class SearchRepoGroupView(RepoGroupAppView):
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         c.active = 'search'
         return c
 
     @LoginRequired()
     @HasRepoGroupPermissionAnyDecorator(
         'group.read', 'group.write', 'group.admin')
     @view_config(
         route_name='search_repo_group', request_method='GET',
         renderer='rhodecode:templates/search/search.mako')
     def search_repo_group(self):
         c = self.load_default_context()
         perform_search(self.request, c, repo_group_name=self.db_repo_group_name)
         return self._get_template_context(c)

rhodecode/lib/index/__init__.py

@@ -1,143 +1,152 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Index schema for RhodeCode
 """
 
 import importlib
 import logging
 
 from rhodecode.lib.index.search_utils import normalize_text_for_matching
 
 log = logging.getLogger(__name__)
 
 # leave defaults for backward compat
 default_searcher = 'rhodecode.lib.index.whoosh'
 default_location = '%(here)s/data/index'
 
 ES_VERSION_2 = '2'
 ES_VERSION_6 = '6'
 # for legacy reasons we keep 2 compat as default
 DEFAULT_ES_VERSION = ES_VERSION_2
 
 from rhodecode_tools.lib.fts_index.elasticsearch_engine_6 import \
     ES_CONFIG  # pragma: no cover
 
 
 class BaseSearcher(object):
     query_lang_doc = ''
     es_version = None
     name = None
     DIRECTION_ASC = 'asc'
     DIRECTION_DESC = 'desc'
 
     def __init__(self):
         pass
 
     def cleanup(self):
         pass
 
     def search(self, query, document_type, search_user,
                repo_name=None, repo_group_name=None,
                raise_on_exc=True):
         raise Exception('NotImplemented')
 
     @staticmethod
     def query_to_mark(query, default_field=None):
         """
         Formats the query to mark token for jquery.mark.js highlighting. ES could
         have a different format optionally.
 
         :param default_field:
         :param query:
         """
         return ' '.join(normalize_text_for_matching(query).split())
 
     @property
     def is_es_6(self):
         return self.es_version == ES_VERSION_6
 
     def get_handlers(self):
         return {}
 
     @staticmethod
     def extract_search_tags(query):
         return []
 
     @staticmethod
     def escape_specials(val):
         """
         Handle and escape reserved chars for search
         """
         return val
 
+    def sort_def(self, search_type, direction, sort_field):
+        """
+        Defines sorting for search. This function should decide if for given
+        search_type, sorting can be done with sort_field.
+
+        It also should translate common sort fields into backend specific. e.g elasticsearch
+        """
+        raise NotImplementedError()
+
     @staticmethod
     def get_sort(search_type, search_val):
         """
         Method used to parse the GET search sort value to a field and direction.
         e.g asc:lines == asc, lines
 
         There's also a legacy support for newfirst/oldfirst which defines commit
         sorting only
         """
 
         direction = BaseSearcher.DIRECTION_ASC
         sort_field = None
 
         if not search_val:
             return direction, sort_field
 
         if search_val.startswith('asc:'):
             sort_field = search_val[4:]
             direction = BaseSearcher.DIRECTION_ASC
         elif search_val.startswith('desc:'):
             sort_field = search_val[5:]
             direction = BaseSearcher.DIRECTION_DESC
         elif search_val == 'newfirst' and search_type == 'commit':
             sort_field = 'date'
             direction = BaseSearcher.DIRECTION_DESC
         elif search_val == 'oldfirst' and search_type == 'commit':
             sort_field = 'date'
             direction = BaseSearcher.DIRECTION_ASC
 
         return direction, sort_field
 
 
 def search_config(config, prefix='search.'):
     _config = {}
     for key in config.keys():
         if key.startswith(prefix):
             _config[key[len(prefix):]] = config[key]
     return _config
 
 
 def searcher_from_config(config, prefix='search.'):
     _config = search_config(config, prefix)
 
     if 'location' not in _config:
         _config['location'] = default_location
     if 'es_version' not in _config:
         # use old legacy ES version set to 2
         _config['es_version'] = '2'
 
     imported = importlib.import_module(_config.get('module', default_searcher))
     searcher = imported.Searcher(config=_config)
     return searcher

rhodecode/lib/index/whoosh.py

@@ -1,293 +1,311 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Index schema for RhodeCode
 """
 
 from __future__ import absolute_import
 import os
 import re
 import logging
 
 from whoosh import query as query_lib
 from whoosh.highlight import HtmlFormatter, ContextFragmenter
 from whoosh.index import create_in, open_dir, exists_in, EmptyIndexError
 from whoosh.qparser import QueryParser, QueryParserError
 
 import rhodecode.lib.helpers as h
 from rhodecode.lib.index import BaseSearcher
 from rhodecode.lib.utils2 import safe_unicode
 
 log = logging.getLogger(__name__)
 
 
 try:
     # we first try to import from rhodecode tools, fallback to copies if
     # we're unable to
     from rhodecode_tools.lib.fts_index.whoosh_schema import (
         ANALYZER, FILE_INDEX_NAME, FILE_SCHEMA, COMMIT_INDEX_NAME,
         COMMIT_SCHEMA)
 except ImportError:
     log.warning('rhodecode_tools schema not available, doing a fallback '
                 'import from `rhodecode.lib.index.whoosh_fallback_schema`')
     from rhodecode.lib.index.whoosh_fallback_schema import (
         ANALYZER, FILE_INDEX_NAME, FILE_SCHEMA, COMMIT_INDEX_NAME,
         COMMIT_SCHEMA)
 
 
 FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n')
 FRAGMENTER = ContextFragmenter(200)
 
 log = logging.getLogger(__name__)
 
 
 class WhooshSearcher(BaseSearcher):
     # this also shows in UI
     query_lang_doc = 'http://whoosh.readthedocs.io/en/latest/querylang.html'
     name = 'whoosh'
 
     def __init__(self, config):
         super(Searcher, self).__init__()
         self.config = config
         if not os.path.isdir(self.config['location']):
             os.makedirs(self.config['location'])
 
         opener = create_in
         if exists_in(self.config['location'], indexname=FILE_INDEX_NAME):
             opener = open_dir
         file_index = opener(self.config['location'], schema=FILE_SCHEMA,
                             indexname=FILE_INDEX_NAME)
 
         opener = create_in
         if exists_in(self.config['location'], indexname=COMMIT_INDEX_NAME):
             opener = open_dir
         changeset_index = opener(self.config['location'], schema=COMMIT_SCHEMA,
                                  indexname=COMMIT_INDEX_NAME)
 
         self.commit_schema = COMMIT_SCHEMA
         self.commit_index = changeset_index
         self.file_schema = FILE_SCHEMA
         self.file_index = file_index
         self.searcher = None
 
     def cleanup(self):
         if self.searcher:
             self.searcher.close()
 
     def _extend_query(self, query):
         hashes = re.compile('([0-9a-f]{5,40})').findall(query)
         if hashes:
             hashes_or_query = ' OR '.join('commit_id:%s*' % h for h in hashes)
             query = u'(%s) OR %s' % (query, hashes_or_query)
         return query
 
+    def sort_def(self, search_type, direction, sort_field):
+
+        if search_type == 'commit':
+            field_defs = {
+                'message': 'message',
+                'date': 'date',
+                'author_email': 'author',
+            }
+        elif search_type == 'path':
+            field_defs = {
+                'file': 'path',
+                'size': 'size',
+                'lines': 'lines',
+            }
+        elif search_type == 'content':
+            # NOTE(dan): content doesn't support any sorting
+            field_defs = {}
+        else:
+            return ''
+
+        if sort_field in field_defs:
+            return field_defs[sort_field]
+
     def search(self, query, document_type, search_user,
                repo_name=None, repo_group_name=None,
                requested_page=1, page_limit=10, sort=None, raise_on_exc=True):
 
         original_query = query
         query = self._extend_query(query)
 
         log.debug(u'QUERY: %s on %s', query, document_type)
         result = {
             'results': [],
             'count': 0,
             'error': None,
             'runtime': 0
         }
         search_type, index_name, schema_defn = self._prepare_for_search(
             document_type)
         self._init_searcher(index_name)
         try:
             qp = QueryParser(search_type, schema=schema_defn)
             allowed_repos_filter = self._get_repo_filter(
                 search_user, repo_name)
             try:
                 query = qp.parse(safe_unicode(query))
                 log.debug('query: %s (%s)', query, repr(query))
 
-                def sort_def(_direction, _sort_field):
-                    field2whoosh = {
-                        'message.raw': 'message',
-                        'author.email.raw': 'author',
-                    }
-                    return field2whoosh.get(_sort_field) or _sort_field
-
                 reverse, sorted_by = False, None
                 direction, sort_field = self.get_sort(search_type, sort)
                 if sort_field:
-                    if direction == Searcher.DIRECTION_DESC:
-                        reverse = True
-                    if direction == Searcher.DIRECTION_ASC:
-                        reverse = False
-                    sorted_by = sort_def(direction, sort_field)
+                    sort_definition = self.sort_def(search_type, direction, sort_field)
+                    if sort_definition:
+                        sorted_by = sort_definition
+                        if direction == Searcher.DIRECTION_DESC:
+                            reverse = True
+                        if direction == Searcher.DIRECTION_ASC:
+                            reverse = False
 
                 whoosh_results = self.searcher.search(
                     query, filter=allowed_repos_filter, limit=None,
                     sortedby=sorted_by, reverse=reverse)
 
                 # fixes for 32k limit that whoosh uses for highlight
                 whoosh_results.fragmenter.charlimit = None
                 res_ln = whoosh_results.scored_length()
                 result['runtime'] = whoosh_results.runtime
                 result['count'] = res_ln
                 result['results'] = WhooshResultWrapper(
                     search_type, res_ln, whoosh_results)
 
             except QueryParserError:
                 result['error'] = 'Invalid search query. Try quoting it.'
         except (EmptyIndexError, IOError, OSError):
             msg = 'There is no index to search in. Please run whoosh indexer'
             log.exception(msg)
             result['error'] = msg
         except Exception:
             msg = 'An error occurred during this search operation'
             log.exception(msg)
             result['error'] = msg
 
         return result
 
     def statistics(self, translator):
         _ = translator
         stats = [
             {'key': _('Index Type'), 'value': 'Whoosh'},
             {'sep': True},
 
             {'key': _('File Index'), 'value': str(self.file_index)},
             {'key': _('Indexed documents'), 'value': self.file_index.doc_count()},
             {'key': _('Last update'), 'value': h.time_to_datetime(self.file_index.last_modified())},
 
             {'sep': True},
 
             {'key': _('Commit index'), 'value': str(self.commit_index)},
             {'key': _('Indexed documents'), 'value': str(self.commit_index.doc_count())},
             {'key': _('Last update'), 'value': h.time_to_datetime(self.commit_index.last_modified())}
         ]
         return stats
 
     def _get_repo_filter(self, auth_user, repo_name):
 
         allowed_to_search = [
             repo for repo, perm in
             auth_user.permissions['repositories'].items()
             if perm != 'repository.none']
 
         if repo_name:
             repo_filter = [query_lib.Term('repository', repo_name)]
 
         elif 'hg.admin' in auth_user.permissions.get('global', []):
             return None
 
         else:
             repo_filter = [query_lib.Term('repository', _rn)
                            for _rn in allowed_to_search]
             # in case we're not allowed to search anywhere, it's a trick
             # to tell whoosh we're filtering, on ALL results
             repo_filter = repo_filter or [query_lib.Term('repository', '')]
 
         return query_lib.Or(repo_filter)
 
     def _prepare_for_search(self, cur_type):
         search_type = {
             'content': 'content',
             'commit': 'message',
             'path': 'path',
             'repository': 'repository'
         }.get(cur_type, 'content')
 
         index_name = {
             'content': FILE_INDEX_NAME,
             'commit': COMMIT_INDEX_NAME,
             'path': FILE_INDEX_NAME
         }.get(cur_type, FILE_INDEX_NAME)
 
         schema_defn = {
             'content': self.file_schema,
             'commit': self.commit_schema,
             'path': self.file_schema
         }.get(cur_type, self.file_schema)
 
         log.debug('IDX: %s', index_name)
         log.debug('SCHEMA: %s', schema_defn)
         return search_type, index_name, schema_defn
 
     def _init_searcher(self, index_name):
         idx = open_dir(self.config['location'], indexname=index_name)
         self.searcher = idx.searcher()
         return self.searcher
 
 
 Searcher = WhooshSearcher
 
 
 class WhooshResultWrapper(object):
     def __init__(self, search_type, total_hits, results):
         self.search_type = search_type
         self.results = results
         self.total_hits = total_hits
 
     def __str__(self):
         return '<%s at %s>' % (self.__class__.__name__, len(self))
 
     def __repr__(self):
         return self.__str__()
 
     def __len__(self):
         return self.total_hits
 
     def __iter__(self):
         """
         Allows Iteration over results,and lazy generate content
 
         *Requires* implementation of ``__getitem__`` method.
         """
         for hit in self.results:
             yield self.get_full_content(hit)
 
     def __getitem__(self, key):
         """
         Slicing of resultWrapper
         """
         i, j = key.start, key.stop
         for hit in self.results[i:j]:
             yield self.get_full_content(hit)
 
     def get_full_content(self, hit):
         # TODO: marcink: this feels like an overkill, there's a lot of data
         # inside hit object, and we don't need all
         res = dict(hit)
         # elastic search uses that, we set it empty so it fallbacks to regular HL logic
         res['content_highlight'] = ''
 
         f_path = ''  # pragma: no cover
         if self.search_type in ['content', 'path']:
             f_path = res['path'][len(res['repository']):]
             f_path = f_path.lstrip(os.sep)
 
         if self.search_type == 'content':
             res.update({'content_short_hl': hit.highlights('content'),
                         'f_path': f_path})
         elif self.search_type == 'path':
             res.update({'f_path': f_path})
         elif self.search_type == 'message':
             res.update({'message_hl': hit.highlights('message')})
 
         return res

rhodecode/templates/search/search.mako

@@ -1,241 +1,241 b''
 ## -*- coding: utf-8 -*-
 <%inherit file="/base/base.mako"/>
 
 <%def name="title()">
     %if c.repo_name:
         ${_('Search inside repository {repo_name}').format(repo_name=c.repo_name)}
     %elif c.repo_group_name:
         ${_('Search inside repository group {repo_group_name}').format(repo_group_name=c.repo_group_name)}
     %else:
         ${_('Search inside all accessible repositories')}
     %endif
     %if c.rhodecode_name:
         &middot; ${h.branding(c.rhodecode_name)}
     %endif
 </%def>
 
 <%def name="breadcrumbs_links()">
   %if c.repo_name:
     ${_('Search inside repository {repo_name}').format(repo_name=c.repo_name)}
   %elif c.repo_group_name:
     ${_('Search inside repository group {repo_group_name}').format(repo_group_name=c.repo_group_name)}
   %else:
     ${_('Search inside all accessible repositories')}
   %endif
 
 </%def>
 
 <%def name="menu_bar_nav()">
     %if c.repo_name:
     ${self.menu_items(active='search')}
     %elif c.repo_group_name:
     ${self.menu_items(active='search')}
     %else:
     ${self.menu_items(active='search')}
     %endif
 </%def>
 
 <%def name="menu_bar_subnav()">
     %if c.repo_name:
         <% active_entry = {'content':'files', 'path':'files', 'commit':'commits'}.get(c.search_type, 'summary')%>
         ${self.repo_menu(active=active_entry)}
     %elif c.repo_group_name:
         ${self.repo_group_menu(active='home')}
     %endif
 </%def>
 
 <%def name="repo_icon(db_repo)">
     %if h.is_hg(db_repo):
         <i class="icon-hg"></i>
     %endif
     %if h.is_git(db_repo):
         <i class="icon-git"></i>
     %endif
     %if h.is_svn(db_repo):
         <i class="icon-svn"></i>
     %endif
 </%def>
 
 <%def name="repo_group_icon()">
     <i class="icon-repo-group"></i>
 </%def>
 
 
 <%def name="field_sort(field_name)">
 
     <%
     if c.sort.startswith('asc:'):
         return c.url_generator(sort='desc:{}'.format(field_name))
     elif c.sort.startswith('desc:'):
         return c.url_generator(sort='asc:{}'.format(field_name))
 
-    return 'asc:{}'.format(field_name)
+    return c.url_generator(sort='asc:{}'.format(field_name))
     %>
 </%def>
 
 
 <%def name="main()">
 <div class="box">
     %if c.repo_name:
         <!-- box / title -->
         ${h.form(h.route_path('search_repo',repo_name=c.repo_name),method='get')}
     %elif c.repo_group_name:
         <!-- box / title -->
         ${h.form(h.route_path('search_repo_group',repo_group_name=c.repo_group_name),method='get')}
     %else:
         <!-- box / title -->
         <div class="title">
             ${self.breadcrumbs()}
             <ul class="links">&nbsp;</ul>
         </div>
         <!-- end box / title -->
         ${h.form(h.route_path('search'), method='get')}
     %endif
     <div class="form search-form">
         <div class="fields">
 
             ${h.text('q', c.cur_query, placeholder="Enter query...")}
 
             ${h.select('type',c.search_type,[('content',_('Files')), ('path',_('File path')),('commit',_('Commits'))],id='id_search_type')}
             ${h.hidden('max_lines', '10')}
 
             <input type="submit" value="${_('Search')}" class="btn"/>
             <br/>
 
             <div class="search-tags">
                 <span class="tag tag8">
                     %if c.repo_name:
                         <a href="${h.route_path('search', _query={'q': c.cur_query, 'type': request.GET.get('type', 'content')})}">${_('Global Search')}</a>
                     %elif c.repo_group_name:
                         <a href="${h.route_path('search', _query={'q': c.cur_query, 'type': request.GET.get('type', 'content')})}">${_('Global Search')}</a>
                     % else:
                         ${_('Global Search')}
                     %endif
                 </span>
 
             %if c.repo_name:
                 »
                 <span class="tag tag8">
                     ${repo_icon(c.rhodecode_db_repo)}
                     ${c.repo_name}
                 </span>
 
             %elif c.repo_group_name:
                 »
                 <span class="tag tag8">
                     ${repo_group_icon()}
                     ${c.repo_group_name}
                 </span>
             %endif
 
             % if c.sort_tag:
                 <span class="tag tag8">
                     % if c.sort_tag_dir == 'asc':
                         <i class="icon-angle-down"></i>
                     % elif c.sort_tag_dir == 'desc':
                         <i class="icon-angle-up"></i>
                     % endif
                     ${_('sort')}:${c.sort_tag}
                 </span>
             % endif
 
             % for search_tag in c.search_tags:
                 <br/><span class="tag disabled" style="margin-top: 3px">${search_tag}</span>
             % endfor
 
             </div>
 
             <div class="search-feedback-items">
             % for error in c.errors:
               <span class="error-message">
                   % for k,v in error.asdict().items():
                     ${k} - ${v}
                   % endfor
               </span>
             % endfor
             <div class="field">
                 <p class="filterexample" style="position: inherit" onclick="$('#search-help').toggle()">${_('Query Language examples')}</p>
 <pre id="search-help" style="display: none">\
 
 % if c.searcher.name == 'whoosh':
 Example filter terms for `Whoosh` search:
 query lang: <a href="${c.searcher.query_lang_doc}">Whoosh Query Language</a>
 Whoosh has limited query capabilities. For advanced search use ElasticSearch 6 from RhodeCode EE edition.
 
 Generate wildcards using '*' character:
   "repo_name:vcs*" - search everything starting with 'vcs'
   "repo_name:*vcs*" - search for repository containing 'vcs'
 
 Optional AND / OR operators in queries
   "repo_name:vcs OR repo_name:test"
   "owner:test AND repo_name:test*" AND extension:py
 
 Move advanced search is available via ElasticSearch6 backend in EE edition.
 % elif c.searcher.name == 'elasticsearch' and c.searcher.es_version == '2':
 Example filter terms for `ElasticSearch-${c.searcher.es_version}`search:
 ElasticSearch-2 has limited query capabilities. For advanced search use ElasticSearch 6 from RhodeCode EE edition.
 
 search type: content (File Content)
   indexed fields: content
 
   # search for `fix` string in all files
   fix
 
 search type: commit (Commit message)
   indexed fields: message
 
 search type: path (File name)
   indexed fields: path
 
 % else:
 Example filter terms for `ElasticSearch-${c.searcher.es_version}`search:
 query lang: <a href="${c.searcher.query_lang_doc}">ES 6 Query Language</a>
 The reserved characters needed espace by `\`: + - = && || > < ! ( ) { } [ ] ^ " ~ * ? : \ /
 % for handler in c.searcher.get_handlers().values():
 
 search type: ${handler.search_type_label}
   *indexed fields*: ${', '.join( [('\n    ' if x[0]%4==0 else '')+x[1] for x in enumerate(handler.es_6_field_names)])}
   % for entry in handler.es_6_example_queries:
   ${entry.rstrip()}
   % endfor
 % endfor
 
 % endif
 </pre>
             </div>
 
             <div class="field">${c.runtime}</div>
             </div>
         </div>
     </div>
 
     ${h.end_form()}
     <div class="search">
     % if c.search_type == 'content':
         <%include file='search_content.mako'/>
     % elif c.search_type == 'path':
         <%include file='search_path.mako'/>
     % elif c.search_type == 'commit':
         <%include file='search_commit.mako'/>
     % elif c.search_type == 'repository':
         <%include file='search_repository.mako'/>
     % endif
     </div>
 </div>
 <script>
     $(document).ready(function(){
         $("#id_search_type").select2({
             'containerCssClass': "drop-menu",
             'dropdownCssClass': "drop-menu-dropdown",
             'dropdownAutoWidth': true,
             'minimumResultsForSearch': -1
         });
 
         $('#q').autoGrowInput({maxWidth: 920});
 
         setTimeout(function() {
             $('#q').keyup()
         }, 1);
     })
 </script>
 </%def>

rhodecode/templates/search/search_commit.mako

@@ -1,98 +1,98 b''
 <%namespace name="base" file="/base/base.mako"/>
 <%namespace name="search" file="/search/search.mako"/>
 
 % if c.formatted_results:
 
 <table class="rctable search-results">
     <tr>
         <th>${_('Repository')}</th>
         <th>${_('Commit')}</th>
         <th></th>
         <th>
-            <a href="${search.field_sort('message.raw')}">${_('Commit message')}</a>
+            <a href="${search.field_sort('message')}">${_('Commit message')}</a>
         </th>
         <th>
-            <a href="${search.field_sort('date')}">${_('Age')}</a>
+            <a href="${search.field_sort('date')}">${_('Commit date')}</a>
         </th>
         <th>
-            <a href="${search.field_sort('author.email.raw')}">${_('Author')}</a>
+            <a href="${search.field_sort('author_email')}">${_('Author')}</a>
         </th>
     </tr>
     %for entry in c.formatted_results:
         ## search results are additionally filtered, and this check is just a safe gate
         % if c.rhodecode_user.is_admin or h.HasRepoPermissionAny('repository.write','repository.read','repository.admin')(entry['repository'], 'search results commit check'):
             <tr class="body">
                 <td class="td-componentname">
                     <% repo_type = entry.get('repo_type') or h.get_repo_type_by_name(entry.get('repository')) %>
                     ${search.repo_icon(repo_type)}
                     ${h.link_to(entry['repository'], h.route_path('repo_summary',repo_name=entry['repository']))}
                 </td>
                 <td class="td-hash">
                     ${h.link_to(h._shorten_commit_id(entry['commit_id']),
                       h.route_path('repo_commit',repo_name=entry['repository'],commit_id=entry['commit_id']))}
                 </td>
                 <td class="td-message expand_commit search open" data-commit-id="${h.md5_safe(entry['repository'])+entry['commit_id']}" id="t-${h.md5_safe(entry['repository'])+entry['commit_id']}" title="${_('Expand commit message')}">
                     <div>
                     <i class="icon-expand-linked"></i>&nbsp;
                     </div>
                 </td>
                 <td data-commit-id="${h.md5_safe(entry['repository'])+entry['commit_id']}" id="c-${h.md5_safe(entry['repository'])+entry['commit_id']}" class="message td-description open">
                     %if entry.get('message_hl'):
                         ${h.literal(entry['message_hl'])}
                     %else:
                         ${h.urlify_commit_message(entry['message'], entry['repository'])}
                     %endif
                 </td>
                 <td class="td-time">
                     ${h.age_component(h.time_to_utcdatetime(entry['date']))}
                 </td>
 
                 <td class="td-user author">
                     <%
                     ## es6 stores this as object
                     author = entry['author']
                     if isinstance(author, dict):
                         author = author['email']
                     %>
                     ${base.gravatar_with_user(author)}
                 </td>
             </tr>
         % endif
     %endfor
 </table>
 
 %if c.cur_query:
 <div class="pagination-wh pagination-left">
     ${c.formatted_results.pager('$link_previous ~2~ $link_next')}
 </div>
 %endif
 
 <script>
     $('.expand_commit').on('click',function(e){
       var target_expand = $(this);
       var cid = target_expand.data('commit-id');
 
       if (target_expand.hasClass('open')){
         $('#c-'+cid).css({'height': '1.5em', 'white-space': 'nowrap', 'text-overflow': 'ellipsis', 'overflow':'hidden'});
         $('#t-'+cid).css({'height': 'auto', 'line-height': '.9em', 'text-overflow': 'ellipsis', 'overflow':'hidden'});
         target_expand.removeClass('open');
       }
       else {
         $('#c-'+cid).css({'height': 'auto', 'white-space': 'normal', 'text-overflow': 'initial', 'overflow':'visible'});
         $('#t-'+cid).css({'height': 'auto', 'max-height': 'none', 'text-overflow': 'initial', 'overflow':'visible'});
         target_expand.addClass('open');
       }
     });
 
     $(".message.td-description").mark(
         "${c.searcher.query_to_mark(c.cur_query, 'message')}",
         {
             "className": 'match',
             "accuracy": "complementary",
             "ignorePunctuation": ":._(){}[]!'+=".split("")
         }
     );
 
 </script>
 
 % endif

rhodecode/templates/search/search_path.mako

@@ -1,53 +1,53 b''
 <%namespace name="search" file="/search/search.mako"/>
 
 % if c.formatted_results:
 
 <table class="rctable search-results">
     <tr>
         <th>${_('Repository')}</th>
         <th>
-            <a href="${search.field_sort('file.raw')}">${_('File')}</a>
+            <a href="${search.field_sort('file')}">${_('File')}</a>
         </th>
         <th>
             <a href="${search.field_sort('size')}">${_('Size')}</a>
         </th>
         <th>
             <a href="${search.field_sort('lines')}">${_('Lines')}</a>
         </th>
     </tr>
     %for entry in c.formatted_results:
         ## search results are additionally filtered, and this check is just a safe gate
         % if c.rhodecode_user.is_admin or h.HasRepoPermissionAny('repository.write','repository.read','repository.admin')(entry['repository'], 'search results path check'):
             <tr class="body">
                 <td class="td-componentname">
                     <% repo_type = entry.get('repo_type') or h.get_repo_type_by_name(entry.get('repository')) %>
                     ${search.repo_icon(repo_type)}
                     ${h.link_to(entry['repository'], h.route_path('repo_summary',repo_name=entry['repository']))}
                 </td>
                 <td class="td-componentname">
                     <i class="icon-file"></i>
                     ${h.link_to(h.literal(entry['f_path']),
                         h.route_path('repo_files',repo_name=entry['repository'],commit_id='tip',f_path=entry['f_path']))}
                 </td>
                 <td>
                     %if entry.get('size'):
                       ${h.format_byte_size_binary(entry['size'])}
                     %endif
                 </td>
                 <td>
                     %if entry.get('lines'):
                       ${entry.get('lines', 0.)}
                     %endif
                 </td>
             </tr>
         % endif
     %endfor
 </table>
 
 %if c.cur_query:
 <div class="pagination-wh pagination-left">
     ${c.formatted_results.pager('$link_previous ~2~ $link_next')}
 </div>
 %endif
 
 % endif