merge with default

rhodecode/api/tests/test_fts_search.py

@@ -0,0 +1,93 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2019 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import pytest
+from rhodecode.tests import HG_REPO
+from rhodecode.api.tests.utils import (
+    build_data, api_call, assert_error, assert_ok)
+
+
[e-mail]("testuser_api", "app")
+class TestApiSearch(object):
+
+    @pytest.mark.parametrize("query, expected_hits, expected_paths", [
+        ('todo', 23, [
+            'vcs/backends/hg/inmemory.py',
+            'vcs/tests/test_git.py']),
+        ('extension:rst installation', 6, [
+            'docs/index.rst',
+            'docs/installation.rst']),
+        ('def repo', 87, [
+            'vcs/tests/test_git.py',
+            'vcs/tests/test_changesets.py']),
+        ('repository:%s def test' % HG_REPO, 18, [
+            'vcs/tests/test_git.py',
+            'vcs/tests/test_changesets.py']),
+        ('"def main"', 9, [
+            'vcs/__init__.py',
+            'vcs/tests/__init__.py',
+            'vcs/utils/progressbar.py']),
+        ('owner:test_admin', 358, [
+            'vcs/tests/base.py',
+            'MANIFEST.in',
+            'vcs/utils/termcolors.py',
+            'docs/theme/ADC/static/documentation.png']),
+        ('owner:test_admin def main', 72, [
+            'vcs/__init__.py',
+            'vcs/tests/test_utils_filesize.py',
+            'vcs/tests/test_cli.py']),
+        ('owner:michał test', 0, []),
+    ])
+    def test_search_content_results(self, query, expected_hits, expected_paths):
+        id_, params = build_data(
+            self.apikey_regular, 'search',
+            search_query=query,
+            search_type='content')
+
+        response = api_call(self.app, params)
+        json_response = response.json
+
+        assert json_response['result']['item_count'] == expected_hits
+        paths = [x['f_path'] for x in json_response['result']['results']]
+
+        for expected_path in expected_paths:
+            assert expected_path in paths
+
+    @pytest.mark.parametrize("query, expected_hits, expected_paths", [
+        ('readme.rst', 3, []),
+        ('test*', 75, []),
+        ('*model*', 1, []),
+        ('extension:rst', 48, []),
+        ('extension:rst api', 24, []),
+    ])
+    def test_search_file_paths(self, query, expected_hits, expected_paths):
+        id_, params = build_data(
+            self.apikey_regular, 'search',
+            search_query=query,
+            search_type='path')
+
+        response = api_call(self.app, params)
+        json_response = response.json
+
+        assert json_response['result']['item_count'] == expected_hits
+        paths = [x['f_path'] for x in json_response['result']['results']]
+
+        for expected_path in expected_paths:
+            assert expected_path in paths

rhodecode/api/views/search_api.py

@@ -0,0 +1,112 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2011-2019 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+
+import logging
+
+from rhodecode.api import jsonrpc_method
+from rhodecode.api.exc import JSONRPCValidationError
+from rhodecode.api.utils import Optional
+from rhodecode.lib.index import searcher_from_config
+from rhodecode.model import validation_schema
+from rhodecode.model.validation_schema.schemas import search_schema
+
+log = logging.getLogger(__name__)
+
+
+@jsonrpc_method()
+def search(request, apiuser, search_query, search_type, page_limit=Optional(10),
+           page=Optional(1), search_sort=Optional('newfirst'),
+           repo_name=Optional(None), repo_group_name=Optional(None)):
+    """
+    Fetch Full Text Search results using API.
+
+    :param apiuser: This is filled automatically from the |authtoken|.
+    :type apiuser: AuthUser
+    :param search_query: Search query.
+    :type search_query: str
+    :param search_type: Search type. The following are valid options:
+        * commit
+        * content
+        * path
+    :type search_type: str
+    :param page_limit: Page item limit, from 1 to 500. Default 10 items.
+    :type page_limit: Optional(int)
+    :param page: Page number. Default first page.
+    :type page: Optional(int)
+    :param search_sort: Search sort order. Default newfirst. The following are valid options:
+        * newfirst
+        * oldfirst
+    :type search_sort: Optional(str)
+    :param repo_name: Filter by one repo. Default is all.
+    :type repo_name: Optional(str)
+    :param repo_group_name: Filter by one repo group. Default is all.
+    :type repo_group_name: Optional(str)
+    """
+
+    data = {'execution_time': ''}
+    repo_name = Optional.extract(repo_name)
+    repo_group_name = Optional.extract(repo_group_name)
+
+    schema = search_schema.SearchParamsSchema()
+
+    try:
+        search_params = schema.deserialize(
+            dict(search_query=search_query,
+                 search_type=search_type,
+                 search_sort=Optional.extract(search_sort),
+                 page_limit=Optional.extract(page_limit),
+                 requested_page=Optional.extract(page))
+        )
+    except validation_schema.Invalid as err:
+        raise JSONRPCValidationError(colander_exc=err)
+
+    search_query = search_params.get('search_query')
+    search_type = search_params.get('search_type')
+    search_sort = search_params.get('search_sort')
+
+    if search_params.get('search_query'):
+        page_limit = search_params['page_limit']
+        requested_page = search_params['requested_page']
+
+        searcher = searcher_from_config(request.registry.settings)
+
+        try:
+            search_result = searcher.search(
+                search_query, search_type, apiuser, repo_name, repo_group_name,
+                requested_page=requested_page, page_limit=page_limit, sort=search_sort)
+
+            data.update(dict(
+                results=list(search_result['results']), page=requested_page,
+                item_count=search_result['count'],
+                items_per_page=page_limit))
+        finally:
+            searcher.cleanup()
+
+        if not search_result['error']:
+            data['execution_time'] = '%s results (%.3f seconds)' % (
+                search_result['count'],
+                search_result['runtime'])
+        else:
+            node = schema['search_query']
+            raise JSONRPCValidationError(
+                colander_exc=validation_schema.Invalid(node, search_result['error']))
+
+    return data

pkgs/python-packages-overrides.nix

@@ -164,12 +164,6 b' self: super: {'
     };
   });
 
-  "pytest-runner" = super."pytest-runner".override (attrs: {
-    propagatedBuildInputs = [
-      self."setuptools-scm"
-    ];
-  });
-
   "python-ldap" = super."python-ldap".override (attrs: {
     propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
       pkgs.openldap

pkgs/python-packages.nix

@@ -5,7 +5,7 b''
 
 self: super: {
   "alembic" = super.buildPythonPackage {
-    name = "alembic-1.0.5";
+    name = "alembic-1.0.8";
     doCheck = false;
     propagatedBuildInputs = [
       self."sqlalchemy"
@@ -14,8 +14,8 b' self: super: {'
       self."python-dateutil"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/1c/65/b8e4f5b2f345bb13b5e0a3fddd892b0b3f0e8ad4880e954fdc6a50d00d84/alembic-1.0.5.tar.gz";
-      sha256 = "0rpjqp2iq6p49x1nli18ivak1izz547nnjxi110mzrgc1v7dxzz9";
+      url = "https://files.pythonhosted.org/packages/d6/bb/ec1e21f2e303689ad2170eb47fc67df9ad4199ade6759a99474c4d3535c8/alembic-1.0.8.tar.gz";
+      sha256 = "1s34i1j0dsxbflxligwhnkf37a5hvcshsv8ibkcfdjf03ph42pah";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
@@ -1051,14 +1051,14 b' self: super: {'
     };
   };
   "paste" = super.buildPythonPackage {
-    name = "paste-3.0.5";
+    name = "paste-3.0.8";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/d4/41/91bde422400786b1b06357c1e6e3a5379f54dc3002aeb337cb767233304e/Paste-3.0.5.tar.gz";
-      sha256 = "1a6i8fh1fg8r4x800fvy9r82m15clwjim6yf2g9r4dff0y40dchv";
+      url = "https://files.pythonhosted.org/packages/66/65/e3acf1663438483c1f6ced0b6c6f3b90da9f0faacb0a6e2aa0f3f9f4b235/Paste-3.0.8.tar.gz";
+      sha256 = "05w1sh6ky4d7pmdb8nv82n13w22jcn3qsagg5ih3hjmbws9kkwf4";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
@@ -1076,7 +1076,7 b' self: super: {'
     };
   };
   "pastescript" = super.buildPythonPackage {
-    name = "pastescript-3.0.0";
+    name = "pastescript-3.1.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."paste"
@@ -1084,8 +1084,8 b' self: super: {'
       self."six"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/08/2a/3797377a884ab9a064ad4d564ed612e54d26d7997caa8229c9c9df4eac31/PasteScript-3.0.0.tar.gz";
-      sha256 = "1hvmyz1sbn7ws1syw567ph7km9fi0wi75r3vlyzx6sk0z26xkm6r";
+      url = "https://files.pythonhosted.org/packages/9e/1d/14db1c283eb21a5d36b6ba1114c13b709629711e64acab653d9994fe346f/PasteScript-3.1.0.tar.gz";
+      sha256 = "02qcxjjr32ks7a6d4f533wl34ysc7yhwlrfcyqwqbzr52250v4fs";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
@@ -1752,7 +1752,6 b' self: super: {'
     ];
     doCheck = true;
     propagatedBuildInputs = [
-      self."setuptools-scm"
       self."amqp"
       self."authomatic"
       self."babel"
@@ -1942,17 +1941,6 b' self: super: {'
       license = [ pkgs.lib.licenses.mit ];
     };
   };
-  "setuptools-scm" = super.buildPythonPackage {
-    name = "setuptools-scm-2.1.0";
-    doCheck = false;
-    src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/e5/62/f9e1ac314464eb5945c97542acb6bf6f3381dfa5d7a658de7730c36f31a1/setuptools_scm-2.1.0.tar.gz";
-      sha256 = "0yb364cgk15sfw3x8ln4ssh98z1dj6n8iiz4r2rw1cfsxhgi8rx7";
-    };
-    meta = {
-      license = [ pkgs.lib.licenses.mit ];
-    };
-  };
   "simplegeneric" = super.buildPythonPackage {
     name = "simplegeneric-0.8.1";
     doCheck = false;

requirements.txt

@@ -1,6 +1,5 b''
 ## dependencies
 
-setuptools-scm==2.1.0
 amqp==2.3.1
 # not released authomatic that has updated some oauth providers
 https://code.rhodecode.com/upstream/authomatic/archive/90a9ce60cc405ae8a2bf5c3713acd5d78579a04e.tar.gz?md5=3c68720a1322b25254009518d1ff6801#egg=authomatic==0.1.0.post1
@@ -35,9 +34,9 b' markupsafe==1.1.0'
 msgpack-python==0.5.6
 pyotp==2.2.7
 packaging==15.2
-paste==3.0.5
+paste==3.0.8
 pastedeploy==2.0.1
-pastescript==3.0.0
+pastescript==3.1.0
 peppercorn==0.6
 psutil==5.5.1
 py-bcrypt==0.4
@@ -98,7 +97,7 b' nbformat==4.4.0'
 jupyter_client==5.0.0
 
 ## cli tools
-alembic==1.0.5
+alembic==1.0.8
 invoke==0.13.0
 bumpversion==0.5.3
 

rhodecode/config/licenses.json

@@ -272,17 +272,6 b''
     {
         "license": [
             {
-                "fullName": "MIT License",
-                "shortName": "mit",
-                "spdxId": "MIT",
-                "url": "http://spdx.org/licenses/MIT.html"
-            }
-        ],
-        "name": "python2.7-setuptools-scm-2.1.0"
-    },
-    {
-        "license": [
-            {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",