search: add option to search within a repository group.

rhodecode/apps/home/views.py

@@ -1,563 +1,587 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import re
 import logging
 import collections
 
 from pyramid.view import view_config
 
 from rhodecode.apps._base import BaseAppView
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import (
     LoginRequired, NotAnonymous, HasRepoGroupPermissionAnyDecorator,
     CSRFRequired)
 from rhodecode.lib.index import searcher_from_config
 from rhodecode.lib.utils2 import safe_unicode, str2bool, safe_int
 from rhodecode.lib.ext_json import json
 from rhodecode.model.db import (
     func, true, or_, in_filter_generator, Repository, RepoGroup, User, UserGroup)
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.scm import RepoGroupList, RepoList
 from rhodecode.model.user import UserModel
 from rhodecode.model.user_group import UserGroupModel
 
 log = logging.getLogger(__name__)
 
 
 class HomeView(BaseAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         c.user = c.auth_user.get_instance()
 
         return c
 
     @LoginRequired()
     @view_config(
         route_name='user_autocomplete_data', request_method='GET',
         renderer='json_ext', xhr=True)
     def user_autocomplete_data(self):
         self.load_default_context()
         query = self.request.GET.get('query')
         active = str2bool(self.request.GET.get('active') or True)
         include_groups = str2bool(self.request.GET.get('user_groups'))
         expand_groups = str2bool(self.request.GET.get('user_groups_expand'))
         skip_default_user = str2bool(self.request.GET.get('skip_default_user'))
 
         log.debug('generating user list, query:%s, active:%s, with_groups:%s',
                   query, active, include_groups)
 
         _users = UserModel().get_users(
             name_contains=query, only_active=active)
 
         def maybe_skip_default_user(usr):
             if skip_default_user and usr['username'] == UserModel.cls.DEFAULT_USER:
                 return False
             return True
         _users = filter(maybe_skip_default_user, _users)
 
         if include_groups:
             # extend with user groups
             _user_groups = UserGroupModel().get_user_groups(
                 name_contains=query, only_active=active,
                 expand_groups=expand_groups)
             _users = _users + _user_groups
 
         return {'suggestions': _users}
 
     @LoginRequired()
     @NotAnonymous()
     @view_config(
         route_name='user_group_autocomplete_data', request_method='GET',
         renderer='json_ext', xhr=True)
     def user_group_autocomplete_data(self):
         self.load_default_context()
         query = self.request.GET.get('query')
         active = str2bool(self.request.GET.get('active') or True)
         expand_groups = str2bool(self.request.GET.get('user_groups_expand'))
 
         log.debug('generating user group list, query:%s, active:%s',
                   query, active)
 
         _user_groups = UserGroupModel().get_user_groups(
             name_contains=query, only_active=active,
             expand_groups=expand_groups)
         _user_groups = _user_groups
 
         return {'suggestions': _user_groups}
 
     def _get_repo_list(self, name_contains=None, repo_type=None, limit=20):
         org_query = name_contains
         allowed_ids = self._rhodecode_user.repo_acl_ids(
             ['repository.read', 'repository.write', 'repository.admin'],
             cache=False, name_filter=name_contains) or [-1]
 
         query = Repository.query()\
             .order_by(func.length(Repository.repo_name))\
             .order_by(Repository.repo_name)\
             .filter(Repository.archived.isnot(true()))\
             .filter(or_(
                 # generate multiple IN to fix limitation problems
                 *in_filter_generator(Repository.repo_id, allowed_ids)
             ))
 
         if repo_type:
             query = query.filter(Repository.repo_type == repo_type)
 
         if name_contains:
             ilike_expression = u'%{}%'.format(safe_unicode(name_contains))
             query = query.filter(
                 Repository.repo_name.ilike(ilike_expression))
             query = query.limit(limit)
 
         acl_iter = query
 
         return [
             {
                 'id': obj.repo_name,
                 'value': org_query,
                 'value_display': obj.repo_name,
                 'text': obj.repo_name,
                 'type': 'repo',
                 'repo_id': obj.repo_id,
                 'repo_type': obj.repo_type,
                 'private': obj.private,
                 'url': h.route_path('repo_summary', repo_name=obj.repo_name)
             }
             for obj in acl_iter]
 
     def _get_repo_group_list(self, name_contains=None, limit=20):
         org_query = name_contains
         allowed_ids = self._rhodecode_user.repo_group_acl_ids(
             ['group.read', 'group.write', 'group.admin'],
             cache=False, name_filter=name_contains) or [-1]
 
         query = RepoGroup.query()\
             .order_by(func.length(RepoGroup.group_name))\
             .order_by(RepoGroup.group_name) \
             .filter(or_(
                 # generate multiple IN to fix limitation problems
                 *in_filter_generator(RepoGroup.group_id, allowed_ids)
             ))
 
         if name_contains:
             ilike_expression = u'%{}%'.format(safe_unicode(name_contains))
             query = query.filter(
                 RepoGroup.group_name.ilike(ilike_expression))
             query = query.limit(limit)
 
         acl_iter = query
 
         return [
             {
                 'id': obj.group_name,
                 'value': org_query,
                 'value_display': obj.group_name,
                 'text': obj.group_name,
                 'type': 'repo_group',
                 'repo_group_id': obj.group_id,
                 'url': h.route_path(
                     'repo_group_home', repo_group_name=obj.group_name)
             }
             for obj in acl_iter]
 
     def _get_user_list(self, name_contains=None, limit=20):
         org_query = name_contains
         if not name_contains:
             return []
 
         name_contains = re.compile('(?:user:)(.+)').findall(name_contains)
         if len(name_contains) != 1:
             return []
         name_contains = name_contains[0]
 
         query = User.query()\
             .order_by(func.length(User.username))\
             .order_by(User.username) \
             .filter(User.username != User.DEFAULT_USER)
 
         if name_contains:
             ilike_expression = u'%{}%'.format(safe_unicode(name_contains))
             query = query.filter(
                 User.username.ilike(ilike_expression))
             query = query.limit(limit)
 
         acl_iter = query
 
         return [
             {
                 'id': obj.user_id,
                 'value': org_query,
                 'value_display': obj.username,
                 'type': 'user',
                 'icon_link': h.gravatar_url(obj.email, 30),
                 'url': h.route_path(
                     'user_profile', username=obj.username)
             }
             for obj in acl_iter]
 
     def _get_user_groups_list(self, name_contains=None, limit=20):
         org_query = name_contains
         if not name_contains:
             return []
 
         name_contains = re.compile('(?:user_group:)(.+)').findall(name_contains)
         if len(name_contains) != 1:
             return []
         name_contains = name_contains[0]
 
         query = UserGroup.query()\
             .order_by(func.length(UserGroup.users_group_name))\
             .order_by(UserGroup.users_group_name)
 
         if name_contains:
             ilike_expression = u'%{}%'.format(safe_unicode(name_contains))
             query = query.filter(
                 UserGroup.users_group_name.ilike(ilike_expression))
             query = query.limit(limit)
 
         acl_iter = query
 
         return [
             {
                 'id': obj.users_group_id,
                 'value': org_query,
                 'value_display': obj.users_group_name,
                 'type': 'user_group',
                 'url': h.route_path(
                     'user_group_profile', user_group_name=obj.users_group_name)
             }
             for obj in acl_iter]
 
     def _get_hash_commit_list(self, auth_user, searcher, query):
         org_query = query
         if not query or len(query) < 3 or not searcher:
             return []
 
         commit_hashes = re.compile('(?:commit:)([0-9a-f]{2,40})').findall(query)
 
         if len(commit_hashes) != 1:
             return []
         commit_hash = commit_hashes[0]
 
         result = searcher.search(
             'commit_id:{}*'.format(commit_hash), 'commit', auth_user,
             raise_on_exc=False)
 
         return [
             {
                 'id': entry['commit_id'],
                 'value': org_query,
                 'value_display': 'repo `{}` commit: {}'.format(
                     entry['repository'], entry['commit_id']),
                 'type': 'commit',
                 'repo': entry['repository'],
                 'url': h.route_path(
                     'repo_commit',
                     repo_name=entry['repository'], commit_id=entry['commit_id'])
             }
             for entry in result['results']]
 
     @LoginRequired()
     @view_config(
         route_name='repo_list_data', request_method='GET',
         renderer='json_ext', xhr=True)
     def repo_list_data(self):
         _ = self.request.translate
         self.load_default_context()
 
         query = self.request.GET.get('query')
         repo_type = self.request.GET.get('repo_type')
         log.debug('generating repo list, query:%s, repo_type:%s',
                   query, repo_type)
 
         res = []
         repos = self._get_repo_list(query, repo_type=repo_type)
         if repos:
             res.append({
                 'text': _('Repositories'),
                 'children': repos
             })
 
         data = {
             'more': False,
             'results': res
         }
         return data
 
     @LoginRequired()
     @view_config(
         route_name='repo_group_list_data', request_method='GET',
         renderer='json_ext', xhr=True)
     def repo_group_list_data(self):
         _ = self.request.translate
         self.load_default_context()
 
         query = self.request.GET.get('query')
 
         log.debug('generating repo group list, query:%s',
                   query)
 
         res = []
         repo_groups = self._get_repo_group_list(query)
         if repo_groups:
             res.append({
                 'text': _('Repository Groups'),
                 'children': repo_groups
             })
 
         data = {
             'more': False,
             'results': res
         }
         return data
 
     def _get_default_search_queries(self, search_context, searcher, query):
         if not searcher:
             return []
         is_es_6 = searcher.is_es_6
 
         queries = []
         repo_group_name, repo_name, repo_context = None, None, None
 
         # repo group context
         if search_context.get('search_context[repo_group_name]'):
             repo_group_name = search_context.get('search_context[repo_group_name]')
         if search_context.get('search_context[repo_name]'):
             repo_name = search_context.get('search_context[repo_name]')
             repo_context = search_context.get('search_context[repo_view_type]')
 
         if is_es_6 and repo_name:
+            # files
             def query_modifier():
-                qry = '{} repo_name.raw:{} '.format(
-                    query, searcher.escape_specials(repo_name))
+                qry = query
                 return {'q': qry, 'type': 'content'}
-            label = u'Search for `{}` through files in this repository.'.format(query)
+            label = u'File search for `{}` in this repository.'.format(query)
+            queries.append(
+                {
+                    'id': -10,
+                    'value': query,
+                    'value_display': label,
+                    'type': 'search',
+                    'url': h.route_path('search_repo',
+                                        repo_name=repo_name,
+                                        _query=query_modifier())
+                }
+            )
+
+            # commits
+            def query_modifier():
+                qry = query
+                return {'q': qry, 'type': 'commit'}
+
+            label = u'Commit search for `{}` in this repository.'.format(query)
             queries.append(
                 {
                     'id': -10,
                     'value': query,
                     'value_display': label,
                     'type': 'search',
-                    'url': h.route_path(
-                        'search_repo', repo_name=repo_name, _query=query_modifier())
-                }
-            )
-
-            def query_modifier():
-                qry = '{} repo_name.raw:{} '.format(
-                    query, searcher.escape_specials(repo_name))
-                return {'q': qry, 'type': 'commit'}
-            label = u'Search for `{}` through commits in this repository.'.format(query)
-            queries.append(
-                {
-                    'id': -10,
-                    'value': query,
-                    'value_display': label,
-                    'type': 'search',
-                    'url': h.route_path(
-                        'search_repo', repo_name=repo_name, _query=query_modifier())
+                    'url': h.route_path('search_repo',
+                                        repo_name=repo_name,
+                                        _query=query_modifier())
                 }
             )
 
         elif is_es_6 and repo_group_name:
+            # files
             def query_modifier():
-                qry = '{} repo_name.raw:{} '.format(
-                    query, searcher.escape_specials(repo_group_name + '/*'))
+                qry = query
                 return {'q': qry, 'type': 'content'}
-            label = u'Search for `{}` through files in this repository group'.format(query)
+
+            label = u'File search for `{}` in this repository group'.format(query)
             queries.append(
                 {
                     'id': -20,
                     'value': query,
                     'value_display': label,
                     'type': 'search',
-                    'url': h.route_path('search', _query=query_modifier())
+                    'url': h.route_path('search_repo_group',
+                                        repo_group_name=repo_group_name,
+                                        _query=query_modifier())
+                }
+            )
+
+            # commits
+            def query_modifier():
+                qry = query
+                return {'q': qry, 'type': 'commit'}
+
+            label = u'Commit search for `{}` in this repository group'.format(query)
+            queries.append(
+                {
+                    'id': -20,
+                    'value': query,
+                    'value_display': label,
+                    'type': 'search',
+                    'url': h.route_path('search_repo_group',
+                                        repo_group_name=repo_group_name,
+                                        _query=query_modifier())
                 }
             )
 
         if not queries:
             queries.append(
                 {
                     'id': -1,
                     'value': query,
                     'value_display': u'Search for: `{}`'.format(query),
                     'type': 'search',
                     'url': h.route_path('search',
                                         _query={'q': query, 'type': 'content'})
                 }
             )
 
         return queries
 
     @LoginRequired()
     @view_config(
         route_name='goto_switcher_data', request_method='GET',
         renderer='json_ext', xhr=True)
     def goto_switcher_data(self):
         c = self.load_default_context()
 
         _ = self.request.translate
 
         query = self.request.GET.get('query')
         log.debug('generating main filter data, query %s', query)
 
         res = []
         if not query:
             return {'suggestions': res}
 
         searcher = searcher_from_config(self.request.registry.settings)
         for _q in self._get_default_search_queries(self.request.GET, searcher, query):
             res.append(_q)
 
         repo_group_id = safe_int(self.request.GET.get('search_context[repo_group_id]'))
         if repo_group_id:
             repo_group = RepoGroup.get(repo_group_id)
             composed_hint = '{}/{}'.format(repo_group.group_name, query)
             show_hint = not query.startswith(repo_group.group_name)
             if repo_group and show_hint:
                 hint = u'Repository search inside: `{}`'.format(composed_hint)
                 res.append({
                     'id': -1,
                     'value': composed_hint,
                     'value_display': hint,
                     'type': 'hint',
                     'url': ""
                 })
 
         repo_groups = self._get_repo_group_list(query)
         for serialized_repo_group in repo_groups:
             res.append(serialized_repo_group)
 
         repos = self._get_repo_list(query)
         for serialized_repo in repos:
             res.append(serialized_repo)
 
         # TODO(marcink): should all logged in users be allowed to search others?
         allowed_user_search = self._rhodecode_user.username != User.DEFAULT_USER
         if allowed_user_search:
             users = self._get_user_list(query)
             for serialized_user in users:
                 res.append(serialized_user)
 
             user_groups = self._get_user_groups_list(query)
             for serialized_user_group in user_groups:
                 res.append(serialized_user_group)
 
         commits = self._get_hash_commit_list(c.auth_user, searcher, query)
         if commits:
             unique_repos = collections.OrderedDict()
             for commit in commits:
                 repo_name = commit['repo']
                 unique_repos.setdefault(repo_name, []).append(commit)
 
             for repo, commits in unique_repos.items():
                 for commit in commits:
                     res.append(commit)
 
         return {'suggestions': res}
 
     def _get_groups_and_repos(self, repo_group_id=None):
         # repo groups groups
         repo_group_list = RepoGroup.get_all_repo_groups(group_id=repo_group_id)
         _perms = ['group.read', 'group.write', 'group.admin']
         repo_group_list_acl = RepoGroupList(repo_group_list, perm_set=_perms)
         repo_group_data = RepoGroupModel().get_repo_groups_as_dict(
             repo_group_list=repo_group_list_acl, admin=False)
 
         # repositories
         repo_list = Repository.get_all_repos(group_id=repo_group_id)
         _perms = ['repository.read', 'repository.write', 'repository.admin']
         repo_list_acl = RepoList(repo_list, perm_set=_perms)
         repo_data = RepoModel().get_repos_as_dict(
             repo_list=repo_list_acl, admin=False)
 
         return repo_data, repo_group_data
 
     @LoginRequired()
     @view_config(
         route_name='home', request_method='GET',
         renderer='rhodecode:templates/index.mako')
     def main_page(self):
         c = self.load_default_context()
         c.repo_group = None
 
         repo_data, repo_group_data = self._get_groups_and_repos()
         # json used to render the grids
         c.repos_data = json.dumps(repo_data)
         c.repo_groups_data = json.dumps(repo_group_data)
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasRepoGroupPermissionAnyDecorator(
         'group.read', 'group.write', 'group.admin')
     @view_config(
         route_name='repo_group_home', request_method='GET',
         renderer='rhodecode:templates/index_repo_group.mako')
     @view_config(
         route_name='repo_group_home_slash', request_method='GET',
         renderer='rhodecode:templates/index_repo_group.mako')
     def repo_group_main_page(self):
         c = self.load_default_context()
         c.repo_group = self.request.db_repo_group
         repo_data, repo_group_data = self._get_groups_and_repos(
             c.repo_group.group_id)
 
         # json used to render the grids
         c.repos_data = json.dumps(repo_data)
         c.repo_groups_data = json.dumps(repo_group_data)
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @CSRFRequired()
     @view_config(
         route_name='markup_preview', request_method='POST',
         renderer='string', xhr=True)
     def markup_preview(self):
         # Technically a CSRF token is not needed as no state changes with this
         # call. However, as this is a POST is better to have it, so automated
         # tools don't flag it as potential CSRF.
         # Post is required because the payload could be bigger than the maximum
         # allowed by GET.
 
         text = self.request.POST.get('text')
         renderer = self.request.POST.get('renderer') or 'rst'
         if text:
             return h.render(text, renderer=renderer, mentions=True)
         return ''
 
     @LoginRequired()
     @CSRFRequired()
     @view_config(
         route_name='store_user_session_value', request_method='POST',
         renderer='string', xhr=True)
     def store_user_session_attr(self):
         key = self.request.POST.get('key')
         val = self.request.POST.get('val')
 
         existing_value = self.request.session.get(key)
         if existing_value != val:
             self.request.session[key] = val
 
         return 'stored:{}'.format(key)

rhodecode/apps/search/__init__.py

@@ -1,34 +1,43 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 from rhodecode.apps._base import ADMIN_PREFIX
 
 
 def includeme(config):
 
     config.add_route(
         name='search',
         pattern=ADMIN_PREFIX + '/search')
 
     config.add_route(
         name='search_repo',
+        pattern='/{repo_name:.*?[^/]}/_search', repo_route=True)
+
+    config.add_route(
+        name='search_repo_alt',
         pattern='/{repo_name:.*?[^/]}/search', repo_route=True)
 
+    config.add_route(
+        name='search_repo_group',
+        pattern='/{repo_group_name:.*?[^/]}/_search',
+        repo_group_route=True)
+
     # Scan module for configuration decorators.
     config.scan('.views', ignore='.tests')

rhodecode/apps/search/views.py

@@ -1,138 +1,161 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import urllib
 from pyramid.view import view_config
 from webhelpers.util import update_params
 
-from rhodecode.apps._base import BaseAppView, RepoAppView
-from rhodecode.lib.auth import (LoginRequired, HasRepoPermissionAnyDecorator)
+from rhodecode.apps._base import BaseAppView, RepoAppView, RepoGroupAppView
+from rhodecode.lib.auth import (
+    LoginRequired, HasRepoPermissionAnyDecorator, HasRepoGroupPermissionAnyDecorator)
 from rhodecode.lib.helpers import Page
 from rhodecode.lib.utils2 import safe_str
 from rhodecode.lib.index import searcher_from_config
 from rhodecode.model import validation_schema
 from rhodecode.model.validation_schema.schemas import search_schema
 
 log = logging.getLogger(__name__)
 
 
-def search(request, tmpl_context, repo_name):
+def perform_search(request, tmpl_context, repo_name=None, repo_group_name=None):
     searcher = searcher_from_config(request.registry.settings)
     formatted_results = []
     execution_time = ''
 
     schema = search_schema.SearchParamsSchema()
 
     search_params = {}
     errors = []
     try:
         search_params = schema.deserialize(
             dict(
                 search_query=request.GET.get('q'),
                 search_type=request.GET.get('type'),
                 search_sort=request.GET.get('sort'),
                 search_max_lines=request.GET.get('max_lines'),
                 page_limit=request.GET.get('page_limit'),
                 requested_page=request.GET.get('page'),
              )
         )
     except validation_schema.Invalid as e:
         errors = e.children
 
     def url_generator(**kw):
         q = urllib.quote(safe_str(search_query))
         return update_params(
-            "?q=%s&type=%s&max_lines=%s" % (q, safe_str(search_type), search_max_lines), **kw)
+            "?q=%s&type=%s&max_lines=%s" % (
+                q, safe_str(search_type), search_max_lines), **kw)
 
     c = tmpl_context
     search_query = search_params.get('search_query')
     search_type = search_params.get('search_type')
     search_sort = search_params.get('search_sort')
     search_max_lines = search_params.get('search_max_lines')
     if search_params.get('search_query'):
         page_limit = search_params['page_limit']
         requested_page = search_params['requested_page']
 
         try:
             search_result = searcher.search(
-                search_query, search_type, c.auth_user, repo_name,
-                requested_page, page_limit, search_sort)
+                search_query, search_type, c.auth_user, repo_name, repo_group_name,
+                requested_page=requested_page, page_limit=page_limit, sort=search_sort)
 
             formatted_results = Page(
                 search_result['results'], page=requested_page,
                 item_count=search_result['count'],
                 items_per_page=page_limit, url=url_generator)
         finally:
             searcher.cleanup()
 
         if not search_result['error']:
             execution_time = '%s results (%.3f seconds)' % (
                 search_result['count'],
                 search_result['runtime'])
         elif not errors:
             node = schema['search_query']
             errors = [
                 validation_schema.Invalid(node, search_result['error'])]
 
     c.perm_user = c.auth_user
     c.repo_name = repo_name
+    c.repo_group_name = repo_group_name
     c.sort = search_sort
     c.url_generator = url_generator
     c.errors = errors
     c.formatted_results = formatted_results
     c.runtime = execution_time
     c.cur_query = search_query
     c.search_type = search_type
     c.searcher = searcher
 
 
 class SearchView(BaseAppView):
     def load_default_context(self):
         c = self._get_local_tmpl_context()
-
         return c
 
     @LoginRequired()
     @view_config(
         route_name='search', request_method='GET',
         renderer='rhodecode:templates/search/search.mako')
     def search(self):
         c = self.load_default_context()
-        search(self.request, c, repo_name=None)
+        perform_search(self.request, c)
         return self._get_template_context(c)
 
 
 class SearchRepoView(RepoAppView):
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         c.active = 'search'
         return c
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='search_repo', request_method='GET',
         renderer='rhodecode:templates/search/search.mako')
+    @view_config(
+        route_name='search_repo_alt', request_method='GET',
+        renderer='rhodecode:templates/search/search.mako')
     def search_repo(self):
         c = self.load_default_context()
-        search(self.request, c, repo_name=self.db_repo_name)
+        perform_search(self.request, c, repo_name=self.db_repo_name)
         return self._get_template_context(c)
+
+
+class SearchRepoGroupView(RepoGroupAppView):
+    def load_default_context(self):
+        c = self._get_local_tmpl_context()
+        c.active = 'search'
+        return c
+
+    @LoginRequired()
+    @HasRepoGroupPermissionAnyDecorator(
+        'group.read', 'group.write', 'group.admin')
+    @view_config(
+        route_name='search_repo_group', request_method='GET',
+        renderer='rhodecode:templates/search/search.mako')
+    def search_repo_group(self):
+        c = self.load_default_context()
+        perform_search(self.request, c, repo_group_name=self.db_repo_group_name)
+        return self._get_template_context(c)

rhodecode/lib/index/__init__.py

@@ -1,98 +1,99 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Index schema for RhodeCode
 """
 
 import importlib
 import logging
 
 from rhodecode.lib.index.search_utils import normalize_text_for_matching
 
 log = logging.getLogger(__name__)
 
 # leave defaults for backward compat
 default_searcher = 'rhodecode.lib.index.whoosh'
 default_location = '%(here)s/data/index'
 
 ES_VERSION_2 = '2'
 ES_VERSION_6 = '6'
 # for legacy reasons we keep 2 compat as default
 DEFAULT_ES_VERSION = ES_VERSION_2
 
 from rhodecode_tools.lib.fts_index.elasticsearch_engine_6 import \
     ES_CONFIG  # pragma: no cover
 
 
 class BaseSearcher(object):
     query_lang_doc = ''
     es_version = None
     name = None
 
     def __init__(self):
         pass
 
     def cleanup(self):
         pass
 
-    def search(self, query, document_type, search_user, repo_name=None,
+    def search(self, query, document_type, search_user,
+               repo_name=None, repo_group_name=None,
                raise_on_exc=True):
         raise Exception('NotImplemented')
 
     @staticmethod
     def query_to_mark(query, default_field=None):
         """
         Formats the query to mark token for jquery.mark.js highlighting. ES could
         have a different format optionally.
 
         :param default_field:
         :param query:
         """
         return ' '.join(normalize_text_for_matching(query).split())
 
     @property
     def is_es_6(self):
         return self.es_version == ES_VERSION_6
 
     def get_handlers(self):
         return {}
 
 
 def search_config(config, prefix='search.'):
     _config = {}
     for key in config.keys():
         if key.startswith(prefix):
             _config[key[len(prefix):]] = config[key]
     return _config
 
 
 def searcher_from_config(config, prefix='search.'):
     _config = search_config(config, prefix)
 
     if 'location' not in _config:
         _config['location'] = default_location
     if 'es_version' not in _config:
         # use old legacy ES version set to 2
         _config['es_version'] = '2'
 
     imported = importlib.import_module(_config.get('module', default_searcher))
     searcher = imported.Searcher(config=_config)
     return searcher

rhodecode/lib/index/whoosh.py

@@ -1,286 +1,286 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Index schema for RhodeCode
 """
 
 from __future__ import absolute_import
 import os
 import re
 import logging
 
 from whoosh import query as query_lib
 from whoosh.highlight import HtmlFormatter, ContextFragmenter
 from whoosh.index import create_in, open_dir, exists_in, EmptyIndexError
 from whoosh.qparser import QueryParser, QueryParserError
 
 import rhodecode.lib.helpers as h
 from rhodecode.lib.index import BaseSearcher
 from rhodecode.lib.utils2 import safe_unicode
 
 log = logging.getLogger(__name__)
 
 
 try:
     # we first try to import from rhodecode tools, fallback to copies if
     # we're unable to
     from rhodecode_tools.lib.fts_index.whoosh_schema import (
         ANALYZER, FILE_INDEX_NAME, FILE_SCHEMA, COMMIT_INDEX_NAME,
         COMMIT_SCHEMA)
 except ImportError:
     log.warning('rhodecode_tools schema not available, doing a fallback '
                 'import from `rhodecode.lib.index.whoosh_fallback_schema`')
     from rhodecode.lib.index.whoosh_fallback_schema import (
         ANALYZER, FILE_INDEX_NAME, FILE_SCHEMA, COMMIT_INDEX_NAME,
         COMMIT_SCHEMA)
 
 
 FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n')
 FRAGMENTER = ContextFragmenter(200)
 
 log = logging.getLogger(__name__)
 
 
 class WhooshSearcher(BaseSearcher):
     # this also shows in UI
     query_lang_doc = 'http://whoosh.readthedocs.io/en/latest/querylang.html'
     name = 'whoosh'
 
     def __init__(self, config):
         super(Searcher, self).__init__()
         self.config = config
         if not os.path.isdir(self.config['location']):
             os.makedirs(self.config['location'])
 
         opener = create_in
         if exists_in(self.config['location'], indexname=FILE_INDEX_NAME):
             opener = open_dir
         file_index = opener(self.config['location'], schema=FILE_SCHEMA,
                             indexname=FILE_INDEX_NAME)
 
         opener = create_in
         if exists_in(self.config['location'], indexname=COMMIT_INDEX_NAME):
             opener = open_dir
         changeset_index = opener(self.config['location'], schema=COMMIT_SCHEMA,
                                  indexname=COMMIT_INDEX_NAME)
 
         self.commit_schema = COMMIT_SCHEMA
         self.commit_index = changeset_index
         self.file_schema = FILE_SCHEMA
         self.file_index = file_index
         self.searcher = None
 
     def cleanup(self):
         if self.searcher:
             self.searcher.close()
 
     def _extend_query(self, query):
         hashes = re.compile('([0-9a-f]{5,40})').findall(query)
         if hashes:
             hashes_or_query = ' OR '.join('commit_id:%s*' % h for h in hashes)
             query = u'(%s) OR %s' % (query, hashes_or_query)
         return query
 
     def search(self, query, document_type, search_user,
-               repo_name=None, requested_page=1, page_limit=10, sort=None,
-               raise_on_exc=True):
+               repo_name=None, repo_group_name=None,
+               requested_page=1, page_limit=10, sort=None, raise_on_exc=True):
 
         original_query = query
         query = self._extend_query(query)
 
         log.debug(u'QUERY: %s on %s', query, document_type)
         result = {
             'results': [],
             'count': 0,
             'error': None,
             'runtime': 0
         }
         search_type, index_name, schema_defn = self._prepare_for_search(
             document_type)
         self._init_searcher(index_name)
         try:
             qp = QueryParser(search_type, schema=schema_defn)
             allowed_repos_filter = self._get_repo_filter(
                 search_user, repo_name)
             try:
                 query = qp.parse(safe_unicode(query))
                 log.debug('query: %s (%s)', query, repr(query))
 
                 reverse, sortedby = False, None
                 if search_type == 'message':
                     if sort == 'oldfirst':
                         sortedby = 'date'
                         reverse = False
                     elif sort == 'newfirst':
                         sortedby = 'date'
                         reverse = True
 
                 whoosh_results = self.searcher.search(
                     query, filter=allowed_repos_filter, limit=None,
                     sortedby=sortedby, reverse=reverse)
 
                 # fixes for 32k limit that whoosh uses for highlight
                 whoosh_results.fragmenter.charlimit = None
                 res_ln = whoosh_results.scored_length()
                 result['runtime'] = whoosh_results.runtime
                 result['count'] = res_ln
                 result['results'] = WhooshResultWrapper(
                     search_type, res_ln, whoosh_results)
 
             except QueryParserError:
                 result['error'] = 'Invalid search query. Try quoting it.'
         except (EmptyIndexError, IOError, OSError):
             msg = 'There is no index to search in. Please run whoosh indexer'
             log.exception(msg)
             result['error'] = msg
         except Exception:
             msg = 'An error occurred during this search operation'
             log.exception(msg)
             result['error'] = msg
 
         return result
 
     def statistics(self, translator):
         _ = translator
         stats = [
             {'key': _('Index Type'), 'value': 'Whoosh'},
             {'sep': True},
 
             {'key': _('File Index'), 'value': str(self.file_index)},
             {'key': _('Indexed documents'), 'value': self.file_index.doc_count()},
             {'key': _('Last update'), 'value': h.time_to_datetime(self.file_index.last_modified())},
 
             {'sep': True},
 
             {'key': _('Commit index'), 'value': str(self.commit_index)},
             {'key': _('Indexed documents'), 'value': str(self.commit_index.doc_count())},
             {'key': _('Last update'), 'value': h.time_to_datetime(self.commit_index.last_modified())}
         ]
         return stats
 
     def _get_repo_filter(self, auth_user, repo_name):
 
         allowed_to_search = [
             repo for repo, perm in
             auth_user.permissions['repositories'].items()
             if perm != 'repository.none']
 
         if repo_name:
             repo_filter = [query_lib.Term('repository', repo_name)]
 
         elif 'hg.admin' in auth_user.permissions.get('global', []):
             return None
 
         else:
             repo_filter = [query_lib.Term('repository', _rn)
                            for _rn in allowed_to_search]
             # in case we're not allowed to search anywhere, it's a trick
             # to tell whoosh we're filtering, on ALL results
             repo_filter = repo_filter or [query_lib.Term('repository', '')]
 
         return query_lib.Or(repo_filter)
 
     def _prepare_for_search(self, cur_type):
         search_type = {
             'content': 'content',
             'commit': 'message',
             'path': 'path',
             'repository': 'repository'
         }.get(cur_type, 'content')
 
         index_name = {
             'content': FILE_INDEX_NAME,
             'commit': COMMIT_INDEX_NAME,
             'path': FILE_INDEX_NAME
         }.get(cur_type, FILE_INDEX_NAME)
 
         schema_defn = {
             'content': self.file_schema,
             'commit': self.commit_schema,
             'path': self.file_schema
         }.get(cur_type, self.file_schema)
 
         log.debug('IDX: %s', index_name)
         log.debug('SCHEMA: %s', schema_defn)
         return search_type, index_name, schema_defn
 
     def _init_searcher(self, index_name):
         idx = open_dir(self.config['location'], indexname=index_name)
         self.searcher = idx.searcher()
         return self.searcher
 
 
 Searcher = WhooshSearcher
 
 
 class WhooshResultWrapper(object):
     def __init__(self, search_type, total_hits, results):
         self.search_type = search_type
         self.results = results
         self.total_hits = total_hits
 
     def __str__(self):
         return '<%s at %s>' % (self.__class__.__name__, len(self))
 
     def __repr__(self):
         return self.__str__()
 
     def __len__(self):
         return self.total_hits
 
     def __iter__(self):
         """
         Allows Iteration over results,and lazy generate content
 
         *Requires* implementation of ``__getitem__`` method.
         """
         for hit in self.results:
             yield self.get_full_content(hit)
 
     def __getitem__(self, key):
         """
         Slicing of resultWrapper
         """
         i, j = key.start, key.stop
         for hit in self.results[i:j]:
             yield self.get_full_content(hit)
 
     def get_full_content(self, hit):
         # TODO: marcink: this feels like an overkill, there's a lot of data
         # inside hit object, and we don't need all
         res = dict(hit)
         # elastic search uses that, we set it empty so it fallbacks to regular HL logic
         res['content_highlight'] = ''
 
         f_path = ''  # pragma: no cover
         if self.search_type in ['content', 'path']:
             f_path = res['path'][len(res['repository']):]
             f_path = f_path.lstrip(os.sep)
 
         if self.search_type == 'content':
             res.update({'content_short_hl': hit.highlights('content'),
                         'f_path': f_path})
         elif self.search_type == 'path':
             res.update({'f_path': f_path})
         elif self.search_type == 'message':
             res.update({'message_hl': hit.highlights('message')})
 
         return res