search: sort commit messages by date rather than revision

rhodecode/controllers/search.py

@@ -1,107 +1,111 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Search controller for RhodeCode
 """
 
 import logging
 import urllib
 
 from pylons import request, config, tmpl_context as c
 
 from webhelpers.util import update_params
 
 from rhodecode.lib.auth import LoginRequired, AuthUser
 from rhodecode.lib.base import BaseRepoController, render
 from rhodecode.lib.helpers import Page
 from rhodecode.lib.utils2 import safe_str, safe_int
 from rhodecode.lib.index import searcher_from_config
 from rhodecode.model import validation_schema
 
 log = logging.getLogger(__name__)
 
 
 class SearchController(BaseRepoController):
 
     @LoginRequired()
     def index(self, repo_name=None):
 
         searcher = searcher_from_config(config)
         formatted_results = []
         execution_time = ''
 
         schema = validation_schema.SearchParamsSchema()
 
         search_params = {}
         errors = []
         try:
             search_params = schema.deserialize(
                 dict(search_query=request.GET.get('q'),
                      search_type=request.GET.get('type'),
+                     search_sort=request.GET.get('sort'),
                      page_limit=request.GET.get('page_limit'),
                      requested_page=request.GET.get('page'))
             )
         except validation_schema.Invalid as e:
             errors = e.children
 
+        def url_generator(**kw):
+            q = urllib.quote(safe_str(search_query))
+            return update_params(
+                "?q=%s&type=%s" % (q, safe_str(search_type)), **kw)
+
         search_query = search_params.get('search_query')
         search_type = search_params.get('search_type')
-
+        search_sort = search_params.get('search_sort')
         if search_params.get('search_query'):
             page_limit = search_params['page_limit']
             requested_page = search_params['requested_page']
 
-            def url_generator(**kw):
-                q = urllib.quote(safe_str(search_query))
-                return update_params(
-                    "?q=%s&type=%s" % (q, safe_str(search_type)), **kw)
 
             c.perm_user = AuthUser(user_id=c.rhodecode_user.user_id,
                                    ip_addr=self.ip_addr)
 
             try:
                 search_result = searcher.search(
                     search_query, search_type, c.perm_user, repo_name,
-                    requested_page, page_limit)
+                    requested_page, page_limit, search_sort)
 
                 formatted_results = Page(
                     search_result['results'], page=requested_page,
                     item_count=search_result['count'],
                     items_per_page=page_limit, url=url_generator)
             finally:
                 searcher.cleanup()
 
             if not search_result['error']:
                 execution_time = '%s results (%.3f seconds)' % (
                     search_result['count'],
                     search_result['runtime'])
             elif not errors:
                 node = schema['search_query']
                 errors = [
                     validation_schema.Invalid(node, search_result['error'])]
 
+        c.sort = search_sort
+        c.url_generator = url_generator
         c.errors = errors
         c.formatted_results = formatted_results
         c.runtime = execution_time
         c.cur_query = search_query
         c.search_type = search_type
         # Return a rendered template
         return render('/search/search.html')

rhodecode/lib/index/whoosh.py

@@ -1,274 +1,279 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Index schema for RhodeCode
 """
 
 from __future__ import absolute_import
 import logging
 import os
 import re
 
 from pylons.i18n.translation import _
 
 from whoosh import query as query_lib, sorting
 from whoosh.highlight import HtmlFormatter, ContextFragmenter
 from whoosh.index import create_in, open_dir, exists_in, EmptyIndexError
 from whoosh.qparser import QueryParser, QueryParserError
 
 import rhodecode.lib.helpers as h
 from rhodecode.lib.index import BaseSearch
 
 log = logging.getLogger(__name__)
 
 
 try:
     # we first try to import from rhodecode tools, fallback to copies if
     # we're unable to
     from rhodecode_tools.lib.fts_index.whoosh_schema import (
         ANALYZER, FILE_INDEX_NAME, FILE_SCHEMA, COMMIT_INDEX_NAME,
         COMMIT_SCHEMA)
 except ImportError:
     log.warning('rhodecode_tools schema not available, doing a fallback '
                 'import from `rhodecode.lib.index.whoosh_fallback_schema`')
     from rhodecode.lib.index.whoosh_fallback_schema import (
         ANALYZER, FILE_INDEX_NAME, FILE_SCHEMA, COMMIT_INDEX_NAME,
         COMMIT_SCHEMA)
 
 
 FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n')
 FRAGMENTER = ContextFragmenter(200)
 
 log = logging.getLogger(__name__)
 
 
 
 class Search(BaseSearch):
 
     name = 'whoosh'
 
     def __init__(self, config):
         self.config = config
         if not os.path.isdir(self.config['location']):
             os.makedirs(self.config['location'])
 
         opener = create_in
         if exists_in(self.config['location'], indexname=FILE_INDEX_NAME):
             opener = open_dir
         file_index = opener(self.config['location'], schema=FILE_SCHEMA,
                             indexname=FILE_INDEX_NAME)
 
         opener = create_in
         if exists_in(self.config['location'], indexname=COMMIT_INDEX_NAME):
             opener = open_dir
         changeset_index = opener(self.config['location'], schema=COMMIT_SCHEMA,
                                  indexname=COMMIT_INDEX_NAME)
 
         self.commit_schema = COMMIT_SCHEMA
         self.commit_index = changeset_index
         self.file_schema = FILE_SCHEMA
         self.file_index = file_index
         self.searcher = None
 
     def cleanup(self):
         if self.searcher:
             self.searcher.close()
 
     def _extend_query(self, query):
         hashes = re.compile('([0-9a-f]{5,40})').findall(query)
         if hashes:
             hashes_or_query = ' OR '.join('commit_id:%s*' % h for h in hashes)
             query = u'(%s) OR %s' % (query, hashes_or_query)
         return query
 
     def search(self, query, document_type, search_user, repo_name=None,
-        requested_page=1, page_limit=10):
+        requested_page=1, page_limit=10, sort=None):
 
         original_query = query
         query = self._extend_query(query)
 
         log.debug(u'QUERY: %s on %s', query, document_type)
         result = {
             'results': [],
             'count': 0,
             'error': None,
             'runtime': 0
         }
         search_type, index_name, schema_defn = self._prepare_for_search(
             document_type)
         self._init_searcher(index_name)
         try:
             qp = QueryParser(search_type, schema=schema_defn)
             allowed_repos_filter = self._get_repo_filter(
                 search_user, repo_name)
             try:
                 query = qp.parse(unicode(query))
                 log.debug('query: %s (%s)' % (query, repr(query)))
 
-                sortedby = None
+                reverse, sortedby = False, None
                 if search_type == 'message':
-                    sortedby = sorting.FieldFacet('commit_idx', reverse=True)
+                    if sort == 'oldfirst':
+                        sortedby = 'date'
+                        reverse = False
+                    elif sort == 'newfirst':
+                        sortedby = 'date'
+                        reverse = True
 
                 whoosh_results = self.searcher.search(
                     query, filter=allowed_repos_filter, limit=None,
-                    sortedby=sortedby,)
+                    sortedby=sortedby, reverse=reverse)
 
                 # fixes for 32k limit that whoosh uses for highlight
                 whoosh_results.fragmenter.charlimit = None
                 res_ln = whoosh_results.scored_length()
                 result['runtime'] = whoosh_results.runtime
                 result['count'] = res_ln
                 result['results'] = WhooshResultWrapper(
                     search_type, res_ln, whoosh_results)
 
             except QueryParserError:
                 result['error'] = _('Invalid search query. Try quoting it.')
         except (EmptyIndexError, IOError, OSError):
             msg = _('There is no index to search in. '
                     'Please run whoosh indexer')
             log.exception(msg)
             result['error'] = msg
         except Exception:
             msg = _('An error occurred during this search operation')
             log.exception(msg)
             result['error'] = msg
 
         return result
 
     def statistics(self):
         stats = [
             {'key': _('Index Type'), 'value': 'Whoosh'},
             {'key': _('File Index'), 'value': str(self.file_index)},
             {'key': _('Indexed documents'),
              'value': self.file_index.doc_count()},
             {'key': _('Last update'),
              'value': h.time_to_datetime(self.file_index.last_modified())},
             {'key': _('Commit index'), 'value': str(self.commit_index)},
             {'key': _('Indexed documents'),
              'value': str(self.commit_index.doc_count())},
             {'key': _('Last update'),
              'value': h.time_to_datetime(self.commit_index.last_modified())}
         ]
         return stats
 
     def _get_repo_filter(self, auth_user, repo_name):
 
         allowed_to_search = [
             repo for repo, perm in
             auth_user.permissions['repositories'].items()
             if perm != 'repository.none']
 
         if repo_name:
             repo_filter = [query_lib.Term('repository', repo_name)]
 
         elif 'hg.admin' in auth_user.permissions.get('global', []):
             return None
 
         else:
             repo_filter = [query_lib.Term('repository', _rn)
                            for _rn in allowed_to_search]
             # in case we're not allowed to search anywhere, it's a trick
             # to tell whoosh we're filtering, on ALL results
             repo_filter = repo_filter or [query_lib.Term('repository', '')]
 
         return query_lib.Or(repo_filter)
 
     def _prepare_for_search(self, cur_type):
         search_type = {
             'content': 'content',
             'commit': 'message',
             'path': 'path',
             'repository': 'repository'
         }.get(cur_type, 'content')
 
         index_name = {
             'content': FILE_INDEX_NAME,
             'commit': COMMIT_INDEX_NAME,
             'path': FILE_INDEX_NAME
         }.get(cur_type, FILE_INDEX_NAME)
 
         schema_defn = {
             'content': self.file_schema,
             'commit': self.commit_schema,
             'path': self.file_schema
         }.get(cur_type, self.file_schema)
 
         log.debug('IDX: %s' % index_name)
         log.debug('SCHEMA: %s' % schema_defn)
         return search_type, index_name, schema_defn
 
     def _init_searcher(self, index_name):
         idx = open_dir(self.config['location'], indexname=index_name)
         self.searcher = idx.searcher()
         return self.searcher
 
 
 class WhooshResultWrapper(object):
     def __init__(self, search_type, total_hits, results):
         self.search_type = search_type
         self.results = results
         self.total_hits = total_hits
 
     def __str__(self):
         return '<%s at %s>' % (self.__class__.__name__, len(self))
 
     def __repr__(self):
         return self.__str__()
 
     def __len__(self):
         return self.total_hits
 
     def __iter__(self):
         """
         Allows Iteration over results,and lazy generate content
 
         *Requires* implementation of ``__getitem__`` method.
         """
         for hit in self.results:
             yield self.get_full_content(hit)
 
     def __getitem__(self, key):
         """
         Slicing of resultWrapper
         """
         i, j = key.start, key.stop
         for hit in self.results[i:j]:
             yield self.get_full_content(hit)
 
     def get_full_content(self, hit):
         # TODO: marcink: this feels like an overkill, there's a lot of data
         # inside hit object, and we don't need all
         res = dict(hit)
 
         f_path = ''  # noqa
         if self.search_type in ['content', 'path']:
             f_path = res['path'].split(res['repository'])[-1]
             f_path = f_path.lstrip(os.sep)
 
         if self.search_type == 'content':
             res.update({'content_short_hl': hit.highlights('content'),
                         'f_path': f_path})
         elif self.search_type == 'path':
             res.update({'f_path': f_path})
         elif self.search_type == 'message':
             res.update({'message_hl': hit.highlights('message')})
 
         return res

rhodecode/model/validation_schema.py

@@ -1,61 +1,66 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import colander
 from colander import Invalid  # noqa
 
 
 class GroupNameType(colander.String):
     SEPARATOR = '/'
 
     def deserialize(self, node, cstruct):
         result = super(GroupNameType, self).deserialize(node, cstruct)
         return self._replace_extra_slashes(result)
 
     def _replace_extra_slashes(self, path):
         path = path.split(self.SEPARATOR)
         path = [item for item in path if item]
         return self.SEPARATOR.join(path)
 
 
 class RepoGroupSchema(colander.Schema):
     group_name = colander.SchemaNode(GroupNameType())
 
 
 class RepoSchema(colander.Schema):
     repo_name = colander.SchemaNode(GroupNameType())
 
 
 class SearchParamsSchema(colander.MappingSchema):
     search_query = colander.SchemaNode(
         colander.String(),
         missing='')
     search_type = colander.SchemaNode(
         colander.String(),
         missing='content',
         validator=colander.OneOf(['content', 'path', 'commit', 'repository']))
+    search_sort = colander.SchemaNode(
+        colander.String(),
+        missing='newfirst',
+        validator=colander.OneOf(
+            ['oldfirst', 'newfirst']))
     page_limit = colander.SchemaNode(
         colander.Integer(),
         missing=10,
         validator=colander.Range(1, 500))
     requested_page = colander.SchemaNode(
         colander.Integer(),
         missing=1)
 

rhodecode/templates/search/search_commit.html

@@ -1,76 +1,82 b''
 <%namespace name="base" file="/base/base.html"/>
 
 <table class="rctable search-results">
     <tr>
         <th>${_('Repository')}</th>
         <th>${_('Commit')}</th>
         <th></th>
         <th>${_('Commit message')}</th>
-        <th>${_('Age')}</th>
+        <th>
+        %if c.sort == 'newfirst':
+            <a href="${c.url_generator(sort='oldfirst')}">${_('Age (new first)')}</a>
+        %else:
+            <a href="${c.url_generator(sort='newfirst')}">${_('Age (old first)')}</a>
+        %endif
+        </th>
         <th>${_('Author')}</th>
     </tr>
     %for entry in c.formatted_results:
         ## search results are additionally filtered, and this check is just a safe gate
         % if h.HasRepoPermissionAny('repository.write','repository.read','repository.admin')(entry['repository'], 'search results commit check'):
             <tr class="body">
                 <td class="td-componentname">
                     %if h.get_repo_type_by_name(entry.get('repository')) == 'hg':
                         <i class="icon-hg"></i>
                     %elif h.get_repo_type_by_name(entry.get('repository')) == 'git':
                         <i class="icon-git"></i>
                     %elif h.get_repo_type_by_name(entry.get('repository')) == 'svn':
                         <i class="icon-svn"></i>
                     %endif
                     ${h.link_to(entry['repository'], h.url('summary_home',repo_name=entry['repository']))}
                 </td>
                 <td class="td-commit">
                     ${h.link_to(h._shorten_commit_id(entry['commit_id']),
                       h.url('changeset_home',repo_name=entry['repository'],revision=entry['commit_id']))}
                 </td>
                 <td class="td-message expand_commit search open" data-commit-id="${h.md5_safe(entry['repository'])+entry['commit_id']}" id="t-${h.md5_safe(entry['repository'])+entry['commit_id']}" title="${_('Expand commit message')}">
                     <div class="show_more_col">
                     <i class="show_more"></i>&nbsp;
                     </div>
                 </td>
                 <td data-commit-id="${h.md5_safe(entry['repository'])+entry['commit_id']}" id="c-${h.md5_safe(entry['repository'])+entry['commit_id']}" class="message td-description open">
                     %if entry.get('message_hl'):
                         ${h.literal(entry['message_hl'])}
                     %else:
                         ${h.urlify_commit_message(entry['message'], entry['repository'])}
                     %endif
                 </td>
                 <td class="td-time">
                     ${h.age_component(h.time_to_datetime(entry['date']))}
                 </td>
 
                 <td class="td-user author">
                     ${base.gravatar_with_user(entry['author'])}
                 </td>
             </tr>
         % endif
     %endfor
 </table>
 
 %if c.cur_query and c.formatted_results:
 <div class="pagination-wh pagination-left">
     ${c.formatted_results.pager('$link_previous ~2~ $link_next')}
 </div>
 %endif
 
 <script>
     $('.expand_commit').on('click',function(e){
       var target_expand = $(this);
       var cid = target_expand.data('commit-id');
 
       if (target_expand.hasClass('open')){
         $('#c-'+cid).css({'height': '1.5em', 'white-space': 'nowrap', 'text-overflow': 'ellipsis', 'overflow':'hidden'})
         $('#t-'+cid).css({'height': 'auto', 'line-height': '.9em', 'text-overflow': 'ellipsis', 'overflow':'hidden'})
         target_expand.removeClass('open');
       }
       else {
         $('#c-'+cid).css({'height': 'auto', 'white-space': 'normal', 'text-overflow': 'initial', 'overflow':'visible'})
         $('#t-'+cid).css({'height': 'auto', 'max-height': 'none', 'text-overflow': 'initial', 'overflow':'visible'})
         target_expand.addClass('open');
       }
     });
 </script>