diff --git a/rhodecode/api/views/repo_api.py b/rhodecode/api/views/repo_api.py --- a/rhodecode/api/views/repo_api.py +++ b/rhodecode/api/views/repo_api.py @@ -428,8 +428,8 @@ def get_repo_nodes(request, apiuser, rep ``all`` (default), ``files`` and ``dirs``. :type ret_type: Optional(str) :param details: Returns extended information about nodes, such as - md5, binary, and or content. The valid options are ``basic`` and - ``full``. + md5, binary, and or content. + The valid options are ``basic`` and ``full``. :type details: Optional(str) :param max_file_bytes: Only return file content under this file size bytes :type details: Optional(int) @@ -440,12 +440,17 @@ def get_repo_nodes(request, apiuser, rep id : result: [ - { - "name" : "" - "type" : "", - "binary": "" (only in extended mode) - "md5" : "" (only in extended mode) - }, + { + "binary": false, + "content": "File line\nLine2\n", + "extension": "md", + "lines": 2, + "md5": "059fa5d29b19c0657e384749480f6422", + "mimetype": "text/x-minidsrc", + "name": "file.md", + "size": 580, + "type": "file" + }, ... ] error: null @@ -453,16 +458,14 @@ def get_repo_nodes(request, apiuser, rep repo = get_repo_or_error(repoid) if not has_superadmin_permission(apiuser): - _perms = ( - 'repository.admin', 'repository.write', 'repository.read',) + _perms = ('repository.admin', 'repository.write', 'repository.read',) validate_repo_permissions(apiuser, repoid, repo, _perms) ret_type = Optional.extract(ret_type) details = Optional.extract(details) _extended_types = ['basic', 'full'] if details not in _extended_types: - raise JSONRPCError( - 'ret_type must be one of %s' % (','.join(_extended_types))) + raise JSONRPCError('ret_type must be one of %s' % (','.join(_extended_types))) extended_info = False content = False if details == 'basic': @@ -499,6 +502,117 @@ def get_repo_nodes(request, apiuser, rep @jsonrpc_method() +def get_repo_file(request, apiuser, repoid, commit_id, file_path, + max_file_bytes=Optional(None), details=Optional('basic')): + """ + Returns a single file from repository at given revision. + + This command can only be run using an |authtoken| with admin rights, + or users with at least read rights to |repos|. + + :param apiuser: This is filled automatically from the |authtoken|. + :type apiuser: AuthUser + :param repoid: The repository name or repository ID. + :type repoid: str or int + :param commit_id: The revision for which listing should be done. + :type commit_id: str + :param file_path: The path from which to start displaying. + :type file_path: str + :param details: Returns different set of information about nodes. + The valid options are ``minimal`` ``basic`` and ``full``. + :type details: Optional(str) + :param max_file_bytes: Only return file content under this file size bytes + :type details: Optional(int) + + Example output: + + .. code-block:: bash + + id : + result: { + "binary": false, + "extension": "py", + "lines": 35, + "content": "....", + "md5": "76318336366b0f17ee249e11b0c99c41", + "mimetype": "text/x-python", + "name": "python.py", + "size": 817, + "type": "file", + } + error: null + """ + + repo = get_repo_or_error(repoid) + if not has_superadmin_permission(apiuser): + _perms = ('repository.admin', 'repository.write', 'repository.read',) + validate_repo_permissions(apiuser, repoid, repo, _perms) + + details = Optional.extract(details) + _extended_types = ['minimal', 'minimal+search', 'basic', 'full'] + if details not in _extended_types: + raise JSONRPCError( + 'ret_type must be one of %s, got %s' % (','.join(_extended_types)), details) + extended_info = False + content = False + + if details == 'minimal': + extended_info = False + + elif details == 'basic': + extended_info = True + + elif details == 'full': + extended_info = content = True + + try: + # check if repo is not empty by any chance, skip quicker if it is. + _scm = repo.scm_instance() + if _scm.is_empty(): + return None + + node = ScmModel().get_node( + repo, commit_id, file_path, extended_info=extended_info, + content=content, max_file_bytes=max_file_bytes) + + except Exception: + log.exception("Exception occurred while trying to get repo node") + raise JSONRPCError('failed to get repo: `%s` nodes' % repo.repo_name) + + return node + + +@jsonrpc_method() +def get_repo_fts_tree(request, apiuser, repoid, commit_id, root_path): + """ + Returns a list of tree nodes for path at given revision. This api is built + strictly for usage in full text search building, and shouldn't be consumed + + This command can only be run using an |authtoken| with admin rights, + or users with at least read rights to |repos|. + + """ + + repo = get_repo_or_error(repoid) + if not has_superadmin_permission(apiuser): + _perms = ('repository.admin', 'repository.write', 'repository.read',) + validate_repo_permissions(apiuser, repoid, repo, _perms) + + try: + # check if repo is not empty by any chance, skip quicker if it is. + _scm = repo.scm_instance() + if _scm.is_empty(): + return [] + + tree_files = ScmModel().get_fts_data(repo, commit_id, root_path) + return tree_files + + except Exception: + log.exception("Exception occurred while trying to get repo nodes") + raise JSONRPCError('failed to get repo: `%s` nodes' % repo.repo_name) + + +@jsonrpc_method() def get_repo_refs(request, apiuser, repoid): """ Returns a dictionary of current references. It returns diff --git a/rhodecode/lib/vcs/nodes.py b/rhodecode/lib/vcs/nodes.py --- a/rhodecode/lib/vcs/nodes.py +++ b/rhodecode/lib/vcs/nodes.py @@ -372,6 +372,22 @@ class FileNode(Node): """ return md5(self.raw_bytes) + def metadata_uncached(self): + """ + Returns md5, binary flag of the file node, without any cache usage. + """ + + if self.commit: + content = self.commit.get_file_content(self.path) + else: + content = self._content + + is_binary = content and '\0' in content + size = 0 + if content: + size = len(content) + return is_binary, md5(content), size + @LazyProperty def content(self): """ diff --git a/rhodecode/model/scm.py b/rhodecode/model/scm.py --- a/rhodecode/model/scm.py +++ b/rhodecode/model/scm.py @@ -573,11 +573,93 @@ class ScmModel(BaseModel): }) _dirs.append(_data) except RepositoryError: - log.debug("Exception in get_nodes", exc_info=True) + log.exception("Exception in get_nodes") raise return _dirs, _files + def get_node(self, repo_name, commit_id, file_path, + extended_info=False, content=False, max_file_bytes=None): + """ + retrieve single node from commit + """ + try: + + _repo = self._get_repo(repo_name) + commit = _repo.scm_instance().get_commit(commit_id=commit_id) + + file_node = commit.get_node(file_path) + if file_node.is_dir(): + raise RepositoryError('The given path is a directory') + + _content = None + f_name = file_node.unicode_path + + file_data = { + "name": h.escape(f_name), + "type": "file", + } + + if extended_info: + file_data.update({ + "md5": file_node.md5, + "binary": file_node.is_binary, + "size": file_node.size, + "extension": file_node.extension, + "mimetype": file_node.mimetype, + "lines": file_node.lines()[0] + }) + + if content: + over_size_limit = (max_file_bytes is not None + and file_node.size > max_file_bytes) + full_content = None + if not file_node.is_binary and not over_size_limit: + full_content = safe_str(file_node.content) + + file_data.update({ + "content": full_content, + }) + + except RepositoryError: + log.exception("Exception in get_node") + raise + + return file_data + + def get_fts_data(self, repo_name, commit_id, root_path='/'): + """ + Fetch node tree for usage in full text search + """ + + tree_info = list() + + try: + _repo = self._get_repo(repo_name) + commit = _repo.scm_instance().get_commit(commit_id=commit_id) + root_path = root_path.lstrip('/') + for __, dirs, files in commit.walk(root_path): + + for f in files: + _content = None + _data = f_name = f.unicode_path + is_binary, md5, size = f.metadata_uncached() + _data = { + "name": h.escape(f_name), + "md5": md5, + "extension": f.extension, + "binary": is_binary, + "size": size + } + + tree_info.append(_data) + + except RepositoryError: + log.exception("Exception in get_nodes") + raise + + return tree_info + def create_nodes(self, user, repo, message, nodes, parent_commit=None, author=None, trigger_push_hook=True): """