search: moved search into pyramid views.

rhodecode/apps/search/__init__.py

@@ -0,0 +1,44 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2016-2017 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+from rhodecode.apps._base import ADMIN_PREFIX
+
+
+def includeme(config):
+
+    config.add_route(
+        name='search',
+        pattern=ADMIN_PREFIX + '/search')
+
+    config.add_route(
+        name='search_repo',
+        pattern='/{repo_name:.*?[^/]}/search', repo_route=True)
+
+    # Scan module for configuration decorators.
+    config.scan()
+
+
+    # # FULL TEXT SEARCH
+    # rmap.connect('search', '%s/search' % (ADMIN_PREFIX,),
+    #              controller='search')
+    # rmap.connect('search_repo_home', '/{repo_name}/search',
+    #              controller='search',
+    #              action='index',
+    #              conditions={'function': check_repo},
+    #              requirements=URL_NAME_REQUIREMENTS)
  No newline at end of file

rhodecode/apps/search/tests/test_search.py

@@ -0,0 +1,202 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2017 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import os
+
+import mock
+import pytest
+from whoosh import query
+
+from rhodecode.tests import (
+    TestController, SkipTest, HG_REPO,
+    TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS)
+from rhodecode.tests.utils import AssertResponse
+
+
+def route_path(name, **kwargs):
+    from rhodecode.apps._base import ADMIN_PREFIX
+    return {
+        'search':
+            ADMIN_PREFIX + '/search',
+        'search_repo':
+            '/{repo_name}/search',
+
+    }[name].format(**kwargs)
+
+
+class TestSearchController(TestController):
+
+    def test_index(self):
+        self.log_user()
+        response = self.app.get(route_path('search'))
+        assert_response = AssertResponse(response)
+        assert_response.one_element_exists('input#q')
+
+    def test_search_files_empty_search(self):
+        if os.path.isdir(self.index_location):
+            raise SkipTest('skipped due to existing index')
+        else:
+            self.log_user()
+            response = self.app.get(route_path('search'),
+                                    {'q': HG_REPO})
+            response.mustcontain('There is no index to search in. '
+                                 'Please run whoosh indexer')
+
+    def test_search_validation(self):
+        self.log_user()
+        response = self.app.get(route_path('search'),
+            {'q': query, 'type': 'content', 'page_limit': 1000})
+
+        response.mustcontain(
+            'page_limit - 1000 is greater than maximum value 500')
+
+    @pytest.mark.parametrize("query, expected_hits, expected_paths", [
+        ('todo', 23, [
+            'vcs/backends/hg/inmemory.py',
+            'vcs/tests/test_git.py']),
+        ('extension:rst installation', 6, [
+            'docs/index.rst',
+            'docs/installation.rst']),
+        ('def repo', 87, [
+            'vcs/tests/test_git.py',
+            'vcs/tests/test_changesets.py']),
+        ('repository:%s def test' % HG_REPO, 18, [
+            'vcs/tests/test_git.py',
+            'vcs/tests/test_changesets.py']),
+        ('"def main"', 9, [
+            'vcs/__init__.py',
+            'vcs/tests/__init__.py',
+            'vcs/utils/progressbar.py']),
+        ('owner:test_admin', 358, [
+            'vcs/tests/base.py',
+            'MANIFEST.in',
+            'vcs/utils/termcolors.py',
+            'docs/theme/ADC/static/documentation.png']),
+        ('owner:test_admin def main', 72, [
+            'vcs/__init__.py',
+            'vcs/tests/test_utils_filesize.py',
+            'vcs/tests/test_cli.py']),
+        ('owner:michał test', 0, []),
+    ])
+    def test_search_files(self, query, expected_hits, expected_paths):
+        self.log_user()
+        response = self.app.get(route_path('search'),
+            {'q': query, 'type': 'content', 'page_limit': 500})
+
+        response.mustcontain('%s results' % expected_hits)
+        for path in expected_paths:
+            response.mustcontain(path)
+
+    @pytest.mark.parametrize("query, expected_hits, expected_commits", [
+        ('bother to ask where to fetch repo during tests', 3, [
+            ('hg', 'a00c1b6f5d7a6ae678fd553a8b81d92367f7ecf1'),
+            ('git', 'c6eb379775c578a95dad8ddab53f963b80894850'),
+            ('svn', '98')]),
+        ('michał', 0, []),
+        ('changed:tests/utils.py', 36, [
+            ('hg', 'a00c1b6f5d7a6ae678fd553a8b81d92367f7ecf1')]),
+        ('changed:vcs/utils/archivers.py', 11, [
+            ('hg', '25213a5fbb048dff8ba65d21e466a835536e5b70'),
+            ('hg', '47aedd538bf616eedcb0e7d630ea476df0e159c7'),
+            ('hg', 'f5d23247fad4856a1dabd5838afade1e0eed24fb'),
+            ('hg', '04ad456aefd6461aea24f90b63954b6b1ce07b3e'),
+            ('git', 'c994f0de03b2a0aa848a04fc2c0d7e737dba31fc'),
+            ('git', 'd1f898326327e20524fe22417c22d71064fe54a1'),
+            ('git', 'fe568b4081755c12abf6ba673ba777fc02a415f3'),
+            ('git', 'bafe786f0d8c2ff7da5c1dcfcfa577de0b5e92f1')]),
+        ('added:README.rst', 3, [
+            ('hg', '3803844fdbd3b711175fc3da9bdacfcd6d29a6fb'),
+            ('git', 'ff7ca51e58c505fec0dd2491de52c622bb7a806b'),
+            ('svn', '8')]),
+        ('changed:lazy.py', 15, [
+            ('hg', 'eaa291c5e6ae6126a203059de9854ccf7b5baa12'),
+            ('git', '17438a11f72b93f56d0e08e7d1fa79a378578a82'),
+            ('svn', '82'),
+            ('svn', '262'),
+            ('hg', 'f5d23247fad4856a1dabd5838afade1e0eed24fb'),
+            ('git', '33fa3223355104431402a888fa77a4e9956feb3e')
+            ]),
+        ('author:[e-mail] '
+         'commit_id:b986218ba1c9b0d6a259fac9b050b1724ed8e545', 1, [
+             ('hg', 'b986218ba1c9b0d6a259fac9b050b1724ed8e545')]),
+        ('b986218ba1c9b0d6a259fac9b050b1724ed8e545', 1, [
+             ('hg', 'b986218ba1c9b0d6a259fac9b050b1724ed8e545')]),
+        ('b986218b', 1, [
+             ('hg', 'b986218ba1c9b0d6a259fac9b050b1724ed8e545')]),
+    ])
+    def test_search_commit_messages(
+            self, query, expected_hits, expected_commits, enabled_backends):
+        self.log_user()
+        response = self.app.get(route_path('search'),
+            {'q': query, 'type': 'commit', 'page_limit': 500})
+
+        response.mustcontain('%s results' % expected_hits)
+        for backend, commit_id in expected_commits:
+            if backend in enabled_backends:
+                response.mustcontain(commit_id)
+
+    @pytest.mark.parametrize("query, expected_hits, expected_paths", [
+        ('readme.rst', 3, []),
+        ('test*', 75, []),
+        ('*model*', 1, []),
+        ('extension:rst', 48, []),
+        ('extension:rst api', 24, []),
+    ])
+    def test_search_file_paths(self, query, expected_hits, expected_paths):
+        self.log_user()
+        response = self.app.get(route_path('search'),
+            {'q': query, 'type': 'path', 'page_limit': 500})
+
+        response.mustcontain('%s results' % expected_hits)
+        for path in expected_paths:
+            response.mustcontain(path)
+
+    def test_search_commit_message_specific_repo(self, backend):
+        self.log_user()
+        response = self.app.get(
+            route_path('search_repo',repo_name=backend.repo_name),
+            {'q': 'bother to ask where to fetch repo during tests',
+             'type': 'commit'})
+
+        response.mustcontain('1 results')
+
+    def test_filters_are_not_applied_for_admin_user(self):
+        self.log_user()
+        with mock.patch('whoosh.searching.Searcher.search') as search_mock:
+            self.app.get(route_path('search'),
+                {'q': 'test query', 'type': 'commit'})
+        assert search_mock.call_count == 1
+        _, kwargs = search_mock.call_args
+        assert kwargs['filter'] is None
+
+    def test_filters_are_applied_for_normal_user(self, enabled_backends):
+        self.log_user(TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS)
+        with mock.patch('whoosh.searching.Searcher.search') as search_mock:
+            self.app.get(route_path('search'),
+                {'q': 'test query', 'type': 'commit'})
+        assert search_mock.call_count == 1
+        _, kwargs = search_mock.call_args
+        assert isinstance(kwargs['filter'], query.Or)
+        expected_repositories = [
+            'vcs_test_{}'.format(b) for b in enabled_backends]
+        queried_repositories = [
+            name for type_, name in kwargs['filter'].all_terms()]
+        for repository in expected_repositories:
+            assert repository in queried_repositories

rhodecode/apps/search/views.py

@@ -0,0 +1,133 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2011-2017 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import logging
+import urllib
+from pyramid.view import view_config
+from webhelpers.util import update_params
+
+from rhodecode.apps._base import BaseAppView, RepoAppView
+from rhodecode.lib.auth import (LoginRequired, HasRepoPermissionAnyDecorator)
+from rhodecode.lib.helpers import Page
+from rhodecode.lib.utils2 import safe_str, safe_int
+from rhodecode.lib.index import searcher_from_config
+from rhodecode.model import validation_schema
+from rhodecode.model.validation_schema.schemas import search_schema
+
+log = logging.getLogger(__name__)
+
+
+def search(request, tmpl_context, repo_name):
+    searcher = searcher_from_config(request.registry.settings)
+    formatted_results = []
+    execution_time = ''
+
+    schema = search_schema.SearchParamsSchema()
+
+    search_params = {}
+    errors = []
+    try:
+        search_params = schema.deserialize(
+            dict(search_query=request.GET.get('q'),
+                 search_type=request.GET.get('type'),
+                 search_sort=request.GET.get('sort'),
+                 page_limit=request.GET.get('page_limit'),
+                 requested_page=request.GET.get('page'))
+        )
+    except validation_schema.Invalid as e:
+        errors = e.children
+
+    def url_generator(**kw):
+        q = urllib.quote(safe_str(search_query))
+        return update_params(
+            "?q=%s&type=%s" % (q, safe_str(search_type)), **kw)
+
+    c = tmpl_context
+    search_query = search_params.get('search_query')
+    search_type = search_params.get('search_type')
+    search_sort = search_params.get('search_sort')
+    if search_params.get('search_query'):
+        page_limit = search_params['page_limit']
+        requested_page = search_params['requested_page']
+
+        try:
+            search_result = searcher.search(
+                search_query, search_type, c.auth_user, repo_name,
+                requested_page, page_limit, search_sort)
+
+            formatted_results = Page(
+                search_result['results'], page=requested_page,
+                item_count=search_result['count'],
+                items_per_page=page_limit, url=url_generator)
+        finally:
+            searcher.cleanup()
+
+        if not search_result['error']:
+            execution_time = '%s results (%.3f seconds)' % (
+                search_result['count'],
+                search_result['runtime'])
+        elif not errors:
+            node = schema['search_query']
+            errors = [
+                validation_schema.Invalid(node, search_result['error'])]
+
+    c.perm_user = c.auth_user
+    c.repo_name = repo_name
+    c.sort = search_sort
+    c.url_generator = url_generator
+    c.errors = errors
+    c.formatted_results = formatted_results
+    c.runtime = execution_time
+    c.cur_query = search_query
+    c.search_type = search_type
+    c.searcher = searcher
+
+
+class SearchView(BaseAppView):
+    def load_default_context(self):
+        c = self._get_local_tmpl_context()
+        self._register_global_c(c)
+        return c
+
+    @LoginRequired()
+    @view_config(
+        route_name='search', request_method='GET',
+        renderer='rhodecode:templates/search/search.mako')
+    def search(self):
+        c = self.load_default_context()
+        search(self.request, c, repo_name=None)
+        return self._get_template_context(c)
+
+
+class SearchRepoView(RepoAppView):
+    def load_default_context(self):
+        c = self._get_local_tmpl_context()
+        self._register_global_c(c)
+        return c
+
+    @LoginRequired()
+    @HasRepoPermissionAnyDecorator('repository.admin')
+    @view_config(
+        route_name='search_repo', request_method='GET',
+        renderer='rhodecode:templates/search/search.mako')
+    def search_repo(self):
+        c = self.load_default_context()
+        search(self.request, c, repo_name=self.db_repo_name)
+        return self._get_template_context(c)

rhodecode/config/middleware.py

@@ -291,6 +291,7 b' def includeme(config):'
     config.include('rhodecode.apps.login')
     config.include('rhodecode.apps.home')
     config.include('rhodecode.apps.repository')
+    config.include('rhodecode.apps.search')
     config.include('rhodecode.apps.user_profile')
     config.include('rhodecode.apps.my_account')
     config.include('rhodecode.apps.svn_support')

rhodecode/config/routing.py

@@ -614,15 +614,6 b' def make_map(config):'
                  controller='journal', action='toggle_following', jsroute=True,
                  conditions={'method': ['POST']})
 
-    # FULL TEXT SEARCH
-    rmap.connect('search', '%s/search' % (ADMIN_PREFIX,),
-                 controller='search')
-    rmap.connect('search_repo_home', '/{repo_name}/search',
-                 controller='search',
-                 action='index',
-                 conditions={'function': check_repo},
-                 requirements=URL_NAME_REQUIREMENTS)
-
     # FEEDS
     rmap.connect('rss_feed_home', '/{repo_name}/feed/rss',
                  controller='feed', action='rss',

rhodecode/templates/base/base.mako

@@ -253,7 +253,7 b''
                    ${_('Compare fork')}</a></li>
               %endif
 
-              <li><a href="${h.url('search_repo_home',repo_name=c.repo_name)}">${_('Search')}</a></li>
+              <li><a href="${h.route_path('search_repo',repo_name=c.repo_name)}">${_('Search')}</a></li>
 
               %if h.HasRepoPermissionAny('repository.write','repository.admin')(c.repo_name) and c.rhodecode_db_repo.enable_locking:
                 %if c.rhodecode_db_repo.locked[0]:
@@ -399,7 +399,7 b''
           </a>
         </li>
       <li class="${is_active('search')}">
-          <a class="menulink" title="${_('Search in repositories you have access to')}" href="${h.url('search')}">
+          <a class="menulink" title="${_('Search in repositories you have access to')}" href="${h.route_path('search')}">
             <div class="menulabel">${_('Search')}</div>
           </a>
       </li>

rhodecode/templates/search/search.mako

@@ -45,7 +45,7 b''
         <div class="title">
             ${self.repo_page_title(c.rhodecode_db_repo)}
         </div>
-        ${h.form(h.url('search_repo_home',repo_name=c.repo_name),method='get')}
+        ${h.form(h.route_path('search_repo',repo_name=c.repo_name),method='get')}
     %else:
         <!-- box / title -->
         <div class="title">
@@ -53,7 +53,7 b''
             <ul class="links">&nbsp;</ul>
         </div>
         <!-- end box / title -->
-        ${h.form(h.url('search'),method='get')}
+        ${h.form(h.route_path('search'), method='get')}
     %endif
     <div class="form search-form">
         <div class="fields">

rhodecode/templates/search/search_content.mako

@@ -54,7 +54,7 b' for line_number in matching_lines:'
           <div class="stats">
             ${h.link_to(h.literal(entry['f_path']), h.url('files_home',repo_name=entry['repository'],revision=entry.get('commit_id', 'tip'),f_path=entry['f_path']))}
             %if entry.get('lines'):
-              | ${entry.get('lines', 0.)} ${ungettext('line', 'lines', entry.get('lines', 0.))}
+              | ${entry.get('lines', 0.)} ${_ungettext('line', 'lines', entry.get('lines', 0.))}
             %endif
             %if entry.get('size'):
               | ${h.format_byte_size_binary(entry['size'])}