api: expose new functions for FTS

- vcs: added special md5 calculation that doesn't use content cache for more memory efficient md5 calculation.

rhodecode/api/views/repo_api.py

@@ -428,8 +428,8 b' def get_repo_nodes(request, apiuser, rep'
         ``all`` (default), ``files`` and ``dirs``.
     :type ret_type: Optional(str)
     :param details: Returns extended information about nodes, such as
-        md5, binary, and or content.  The valid options are ``basic`` and
-        ``full``.
+        md5, binary, and or content.
+        The valid options are ``basic`` and ``full``.
     :type details: Optional(str)
     :param max_file_bytes: Only return file content under this file size bytes
     :type details: Optional(int)
@@ -440,12 +440,17 b' def get_repo_nodes(request, apiuser, rep'
 
         id : <id_given_in_input>
         result: [
-                  {
-                    "name" : "<name>"
-                    "type" : "<type>",
-                    "binary": "<true|false>" (only in extended mode)
-                    "md5"  : "<md5 of file content>" (only in extended mode)
-                  },
+                    {
+                      "binary": false,
+                      "content": "File line\nLine2\n",
+                      "extension": "md",
+                      "lines": 2,
+                      "md5": "059fa5d29b19c0657e384749480f6422",
+                      "mimetype": "text/x-minidsrc",
+                      "name": "file.md",
+                      "size": 580,
+                      "type": "file"
+                    },
                   ...
                 ]
         error:  null
@@ -453,16 +458,14 b' def get_repo_nodes(request, apiuser, rep'
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
-        _perms = (
-            'repository.admin', 'repository.write', 'repository.read',)
+        _perms = ('repository.admin', 'repository.write', 'repository.read',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     ret_type = Optional.extract(ret_type)
     details = Optional.extract(details)
     _extended_types = ['basic', 'full']
     if details not in _extended_types:
-        raise JSONRPCError(
-            'ret_type must be one of %s' % (','.join(_extended_types)))
+        raise JSONRPCError('ret_type must be one of %s' % (','.join(_extended_types)))
     extended_info = False
     content = False
     if details == 'basic':
@@ -499,6 +502,117 b' def get_repo_nodes(request, apiuser, rep'
 
 
 @jsonrpc_method()
+def get_repo_file(request, apiuser, repoid, commit_id, file_path,
+                  max_file_bytes=Optional(None), details=Optional('basic')):
+    """
+    Returns a single file from repository at given revision.
+
+    This command can only be run using an |authtoken| with admin rights,
+    or users with at least read rights to |repos|.
+
+    :param apiuser: This is filled automatically from the |authtoken|.
+    :type apiuser: AuthUser
+    :param repoid: The repository name or repository ID.
+    :type repoid: str or int
+    :param commit_id: The revision for which listing should be done.
+    :type commit_id: str
+    :param file_path: The path from which to start displaying.
+    :type file_path: str
+    :param details: Returns different set of information about nodes.
+        The valid options are ``minimal`` ``basic`` and ``full``.
+    :type details: Optional(str)
+    :param max_file_bytes: Only return file content under this file size bytes
+    :type details: Optional(int)
+
+    Example output:
+
+    .. code-block:: bash
+
+        id : <id_given_in_input>
+        result: {
+            "binary": false,
+            "extension": "py",
+            "lines": 35,
+            "content": "....",
+            "md5": "76318336366b0f17ee249e11b0c99c41",
+            "mimetype": "text/x-python",
+            "name": "python.py",
+            "size": 817,
+            "type": "file",
+        }
+        error:  null
+    """
+
+    repo = get_repo_or_error(repoid)
+    if not has_superadmin_permission(apiuser):
+        _perms = ('repository.admin', 'repository.write', 'repository.read',)
+        validate_repo_permissions(apiuser, repoid, repo, _perms)
+
+    details = Optional.extract(details)
+    _extended_types = ['minimal', 'minimal+search', 'basic', 'full']
+    if details not in _extended_types:
+        raise JSONRPCError(
+            'ret_type must be one of %s, got %s' % (','.join(_extended_types)), details)
+    extended_info = False
+    content = False
+
+    if details == 'minimal':
+        extended_info = False
+
+    elif details == 'basic':
+        extended_info = True
+
+    elif details == 'full':
+        extended_info = content = True
+
+    try:
+        # check if repo is not empty by any chance, skip quicker if it is.
+        _scm = repo.scm_instance()
+        if _scm.is_empty():
+            return None
+
+        node = ScmModel().get_node(
+            repo, commit_id, file_path, extended_info=extended_info,
+            content=content, max_file_bytes=max_file_bytes)
+
+    except Exception:
+        log.exception("Exception occurred while trying to get repo node")
+        raise JSONRPCError('failed to get repo: `%s` nodes' % repo.repo_name)
+
+    return node
+
+
+@jsonrpc_method()
+def get_repo_fts_tree(request, apiuser, repoid, commit_id, root_path):
+    """
+    Returns a list of tree nodes for path at given revision. This api is built
+    strictly for usage in full text search building, and shouldn't be consumed
+
+    This command can only be run using an |authtoken| with admin rights,
+    or users with at least read rights to |repos|.
+
+    """
+
+    repo = get_repo_or_error(repoid)
+    if not has_superadmin_permission(apiuser):
+        _perms = ('repository.admin', 'repository.write', 'repository.read',)
+        validate_repo_permissions(apiuser, repoid, repo, _perms)
+
+    try:
+        # check if repo is not empty by any chance, skip quicker if it is.
+        _scm = repo.scm_instance()
+        if _scm.is_empty():
+            return []
+
+        tree_files = ScmModel().get_fts_data(repo, commit_id, root_path)
+        return tree_files
+
+    except Exception:
+        log.exception("Exception occurred while trying to get repo nodes")
+        raise JSONRPCError('failed to get repo: `%s` nodes' % repo.repo_name)
+
+
+@jsonrpc_method()
 def get_repo_refs(request, apiuser, repoid):
     """
     Returns a dictionary of current references. It returns

rhodecode/lib/vcs/nodes.py

@@ -372,6 +372,22 b' class FileNode(Node):'
         """
         return md5(self.raw_bytes)
 
+    def metadata_uncached(self):
+        """
+        Returns md5, binary flag of the file node, without any cache usage.
+        """
+
+        if self.commit:
+            content = self.commit.get_file_content(self.path)
+        else:
+            content = self._content
+
+        is_binary = content and '\0' in content
+        size = 0
+        if content:
+            size = len(content)
+        return is_binary, md5(content), size
+
     @LazyProperty
     def content(self):
         """

rhodecode/model/scm.py

@@ -573,11 +573,93 b' class ScmModel(BaseModel):'
                         })
                     _dirs.append(_data)
         except RepositoryError:
-            log.debug("Exception in get_nodes", exc_info=True)
+            log.exception("Exception in get_nodes")
             raise
 
         return _dirs, _files
 
+    def get_node(self, repo_name, commit_id, file_path,
+                 extended_info=False, content=False, max_file_bytes=None):
+        """
+        retrieve single node from commit
+        """
+        try:
+
+            _repo = self._get_repo(repo_name)
+            commit = _repo.scm_instance().get_commit(commit_id=commit_id)
+
+            file_node = commit.get_node(file_path)
+            if file_node.is_dir():
+                raise RepositoryError('The given path is a directory')
+
+            _content = None
+            f_name = file_node.unicode_path
+
+            file_data = {
+                "name": h.escape(f_name),
+                "type": "file",
+            }
+
+            if extended_info:
+                file_data.update({
+                    "md5": file_node.md5,
+                    "binary": file_node.is_binary,
+                    "size": file_node.size,
+                    "extension": file_node.extension,
+                    "mimetype": file_node.mimetype,
+                    "lines": file_node.lines()[0]
+                })
+
+            if content:
+                over_size_limit = (max_file_bytes is not None
+                                   and file_node.size > max_file_bytes)
+                full_content = None
+                if not file_node.is_binary and not over_size_limit:
+                    full_content = safe_str(file_node.content)
+
+                file_data.update({
+                    "content": full_content,
+                })
+
+        except RepositoryError:
+            log.exception("Exception in get_node")
+            raise
+
+        return file_data
+
+    def get_fts_data(self, repo_name, commit_id, root_path='/'):
+        """
+        Fetch node tree for usage in full text search
+        """
+
+        tree_info = list()
+
+        try:
+            _repo = self._get_repo(repo_name)
+            commit = _repo.scm_instance().get_commit(commit_id=commit_id)
+            root_path = root_path.lstrip('/')
+            for __, dirs, files in commit.walk(root_path):
+
+                for f in files:
+                    _content = None
+                    _data = f_name = f.unicode_path
+                    is_binary, md5, size = f.metadata_uncached()
+                    _data = {
+                        "name": h.escape(f_name),
+                        "md5": md5,
+                        "extension": f.extension,
+                        "binary": is_binary,
+                        "size": size
+                    }
+
+                    tree_info.append(_data)
+
+        except RepositoryError:
+            log.exception("Exception in get_nodes")
+            raise
+
+        return tree_info
+
     def create_nodes(self, user, repo, message, nodes, parent_commit=None,
                      author=None, trigger_push_hook=True):
         """