##// END OF EJS Templates
api: expose new functions for FTS...
marcink -
r3460:866fba74 default
parent child Browse files
Show More
@@ -428,8 +428,8 b' def get_repo_nodes(request, apiuser, rep'
428 428 ``all`` (default), ``files`` and ``dirs``.
429 429 :type ret_type: Optional(str)
430 430 :param details: Returns extended information about nodes, such as
431 md5, binary, and or content. The valid options are ``basic`` and
432 ``full``.
431 md5, binary, and or content.
432 The valid options are ``basic`` and ``full``.
433 433 :type details: Optional(str)
434 434 :param max_file_bytes: Only return file content under this file size bytes
435 435 :type details: Optional(int)
@@ -441,10 +441,15 b' def get_repo_nodes(request, apiuser, rep'
441 441 id : <id_given_in_input>
442 442 result: [
443 443 {
444 "name" : "<name>"
445 "type" : "<type>",
446 "binary": "<true|false>" (only in extended mode)
447 "md5" : "<md5 of file content>" (only in extended mode)
444 "binary": false,
445 "content": "File line\nLine2\n",
446 "extension": "md",
447 "lines": 2,
448 "md5": "059fa5d29b19c0657e384749480f6422",
449 "mimetype": "text/x-minidsrc",
450 "name": "file.md",
451 "size": 580,
452 "type": "file"
448 453 },
449 454 ...
450 455 ]
@@ -453,16 +458,14 b' def get_repo_nodes(request, apiuser, rep'
453 458
454 459 repo = get_repo_or_error(repoid)
455 460 if not has_superadmin_permission(apiuser):
456 _perms = (
457 'repository.admin', 'repository.write', 'repository.read',)
461 _perms = ('repository.admin', 'repository.write', 'repository.read',)
458 462 validate_repo_permissions(apiuser, repoid, repo, _perms)
459 463
460 464 ret_type = Optional.extract(ret_type)
461 465 details = Optional.extract(details)
462 466 _extended_types = ['basic', 'full']
463 467 if details not in _extended_types:
464 raise JSONRPCError(
465 'ret_type must be one of %s' % (','.join(_extended_types)))
468 raise JSONRPCError('ret_type must be one of %s' % (','.join(_extended_types)))
466 469 extended_info = False
467 470 content = False
468 471 if details == 'basic':
@@ -499,6 +502,117 b' def get_repo_nodes(request, apiuser, rep'
499 502
500 503
501 504 @jsonrpc_method()
505 def get_repo_file(request, apiuser, repoid, commit_id, file_path,
506 max_file_bytes=Optional(None), details=Optional('basic')):
507 """
508 Returns a single file from repository at given revision.
509
510 This command can only be run using an |authtoken| with admin rights,
511 or users with at least read rights to |repos|.
512
513 :param apiuser: This is filled automatically from the |authtoken|.
514 :type apiuser: AuthUser
515 :param repoid: The repository name or repository ID.
516 :type repoid: str or int
517 :param commit_id: The revision for which listing should be done.
518 :type commit_id: str
519 :param file_path: The path from which to start displaying.
520 :type file_path: str
521 :param details: Returns different set of information about nodes.
522 The valid options are ``minimal`` ``basic`` and ``full``.
523 :type details: Optional(str)
524 :param max_file_bytes: Only return file content under this file size bytes
525 :type details: Optional(int)
526
527 Example output:
528
529 .. code-block:: bash
530
531 id : <id_given_in_input>
532 result: {
533 "binary": false,
534 "extension": "py",
535 "lines": 35,
536 "content": "....",
537 "md5": "76318336366b0f17ee249e11b0c99c41",
538 "mimetype": "text/x-python",
539 "name": "python.py",
540 "size": 817,
541 "type": "file",
542 }
543 error: null
544 """
545
546 repo = get_repo_or_error(repoid)
547 if not has_superadmin_permission(apiuser):
548 _perms = ('repository.admin', 'repository.write', 'repository.read',)
549 validate_repo_permissions(apiuser, repoid, repo, _perms)
550
551 details = Optional.extract(details)
552 _extended_types = ['minimal', 'minimal+search', 'basic', 'full']
553 if details not in _extended_types:
554 raise JSONRPCError(
555 'ret_type must be one of %s, got %s' % (','.join(_extended_types)), details)
556 extended_info = False
557 content = False
558
559 if details == 'minimal':
560 extended_info = False
561
562 elif details == 'basic':
563 extended_info = True
564
565 elif details == 'full':
566 extended_info = content = True
567
568 try:
569 # check if repo is not empty by any chance, skip quicker if it is.
570 _scm = repo.scm_instance()
571 if _scm.is_empty():
572 return None
573
574 node = ScmModel().get_node(
575 repo, commit_id, file_path, extended_info=extended_info,
576 content=content, max_file_bytes=max_file_bytes)
577
578 except Exception:
579 log.exception("Exception occurred while trying to get repo node")
580 raise JSONRPCError('failed to get repo: `%s` nodes' % repo.repo_name)
581
582 return node
583
584
585 @jsonrpc_method()
586 def get_repo_fts_tree(request, apiuser, repoid, commit_id, root_path):
587 """
588 Returns a list of tree nodes for path at given revision. This api is built
589 strictly for usage in full text search building, and shouldn't be consumed
590
591 This command can only be run using an |authtoken| with admin rights,
592 or users with at least read rights to |repos|.
593
594 """
595
596 repo = get_repo_or_error(repoid)
597 if not has_superadmin_permission(apiuser):
598 _perms = ('repository.admin', 'repository.write', 'repository.read',)
599 validate_repo_permissions(apiuser, repoid, repo, _perms)
600
601 try:
602 # check if repo is not empty by any chance, skip quicker if it is.
603 _scm = repo.scm_instance()
604 if _scm.is_empty():
605 return []
606
607 tree_files = ScmModel().get_fts_data(repo, commit_id, root_path)
608 return tree_files
609
610 except Exception:
611 log.exception("Exception occurred while trying to get repo nodes")
612 raise JSONRPCError('failed to get repo: `%s` nodes' % repo.repo_name)
613
614
615 @jsonrpc_method()
502 616 def get_repo_refs(request, apiuser, repoid):
503 617 """
504 618 Returns a dictionary of current references. It returns
@@ -372,6 +372,22 b' class FileNode(Node):'
372 372 """
373 373 return md5(self.raw_bytes)
374 374
375 def metadata_uncached(self):
376 """
377 Returns md5, binary flag of the file node, without any cache usage.
378 """
379
380 if self.commit:
381 content = self.commit.get_file_content(self.path)
382 else:
383 content = self._content
384
385 is_binary = content and '\0' in content
386 size = 0
387 if content:
388 size = len(content)
389 return is_binary, md5(content), size
390
375 391 @LazyProperty
376 392 def content(self):
377 393 """
@@ -573,11 +573,93 b' class ScmModel(BaseModel):'
573 573 })
574 574 _dirs.append(_data)
575 575 except RepositoryError:
576 log.debug("Exception in get_nodes", exc_info=True)
576 log.exception("Exception in get_nodes")
577 577 raise
578 578
579 579 return _dirs, _files
580 580
581 def get_node(self, repo_name, commit_id, file_path,
582 extended_info=False, content=False, max_file_bytes=None):
583 """
584 retrieve single node from commit
585 """
586 try:
587
588 _repo = self._get_repo(repo_name)
589 commit = _repo.scm_instance().get_commit(commit_id=commit_id)
590
591 file_node = commit.get_node(file_path)
592 if file_node.is_dir():
593 raise RepositoryError('The given path is a directory')
594
595 _content = None
596 f_name = file_node.unicode_path
597
598 file_data = {
599 "name": h.escape(f_name),
600 "type": "file",
601 }
602
603 if extended_info:
604 file_data.update({
605 "md5": file_node.md5,
606 "binary": file_node.is_binary,
607 "size": file_node.size,
608 "extension": file_node.extension,
609 "mimetype": file_node.mimetype,
610 "lines": file_node.lines()[0]
611 })
612
613 if content:
614 over_size_limit = (max_file_bytes is not None
615 and file_node.size > max_file_bytes)
616 full_content = None
617 if not file_node.is_binary and not over_size_limit:
618 full_content = safe_str(file_node.content)
619
620 file_data.update({
621 "content": full_content,
622 })
623
624 except RepositoryError:
625 log.exception("Exception in get_node")
626 raise
627
628 return file_data
629
630 def get_fts_data(self, repo_name, commit_id, root_path='/'):
631 """
632 Fetch node tree for usage in full text search
633 """
634
635 tree_info = list()
636
637 try:
638 _repo = self._get_repo(repo_name)
639 commit = _repo.scm_instance().get_commit(commit_id=commit_id)
640 root_path = root_path.lstrip('/')
641 for __, dirs, files in commit.walk(root_path):
642
643 for f in files:
644 _content = None
645 _data = f_name = f.unicode_path
646 is_binary, md5, size = f.metadata_uncached()
647 _data = {
648 "name": h.escape(f_name),
649 "md5": md5,
650 "extension": f.extension,
651 "binary": is_binary,
652 "size": size
653 }
654
655 tree_info.append(_data)
656
657 except RepositoryError:
658 log.exception("Exception in get_nodes")
659 raise
660
661 return tree_info
662
581 663 def create_nodes(self, user, repo, message, nodes, parent_commit=None,
582 664 author=None, trigger_push_hook=True):
583 665 """
General Comments 0
You need to be logged in to leave comments. Login now