##// END OF EJS Templates
feat(remap and rescan): added more relient remap and removal option, and also split the logic to either add or cleanup
feat(remap and rescan): added more relient remap and removal option, and also split the logic to either add or cleanup

File last commit:

r5608:6d33e504 default
r5619:c9e499e7 default
Show More
nodes.py
960 lines | 29.9 KiB | text/x-python | PythonLexer
core: updated copyright to 2024
r5608 # Copyright (C) 2014-2024 RhodeCode GmbH
project: added all source files and assets
r1 #
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
"""
Module holding everything related to vcs nodes, with vcs2 architecture.
"""
vcs-support: bulk of changes for python3
r5075 import functools
largefiles: enabled download of largefiles for git and mercurial from web interface....
r1577 import os
project: added all source files and assets
r1 import stat
from zope.cachedescriptors.property import Lazy as LazyProperty
lexers: added small extensions table to extend syntaxt hl for file sources....
r796 from rhodecode.config.conf import LANGUAGES_EXTENSIONS_MAP
vcs-support: bulk of changes for python3
r5075 from rhodecode.lib.str_utils import safe_str, safe_bytes
from rhodecode.lib.hash_utils import md5
project: added all source files and assets
r1 from rhodecode.lib.vcs import path as vcspath
from rhodecode.lib.vcs.backends.base import EmptyCommit, FILEMODE_DEFAULT
from rhodecode.lib.vcs.conf.mtypes import get_mimetypes_db
from rhodecode.lib.vcs.exceptions import NodeError, RemovedFileNodeError
LARGEFILE_PREFIX = '.hglf'
class NodeKind:
SUBMODULE = -1
DIR = 1
FILE = 2
LARGEFILE = 3
class NodeState:
FOLD: into unicode changes
r4959 ADDED = 'added'
CHANGED = 'changed'
NOT_CHANGED = 'not changed'
REMOVED = 'removed'
project: added all source files and assets
r1
vcs-support: bulk of changes for python3
r5075 #TODO: not sure if that should be bytes or str ?
# most probably bytes because content should be bytes and we check it
BIN_BYTE_MARKER = b'\0'
project: added all source files and assets
r1
class NodeGeneratorBase(object):
"""
Base class for removed added and changed filenodes, it's a lazy generator
class that will create filenodes only on iteration or call
The len method doesn't need to create filenodes at all
"""
def __init__(self, current_paths, cs):
self.cs = cs
self.current_paths = current_paths
def __call__(self):
return [n for n in self]
vcs-support: bulk of changes for python3
r5075 def __getitem__(self, key):
if isinstance(key, slice):
for p in self.current_paths[key.start:key.stop]:
yield self.cs.get_node(p)
project: added all source files and assets
r1
def __len__(self):
return len(self.current_paths)
def __iter__(self):
for p in self.current_paths:
yield self.cs.get_node(p)
class AddedFileNodesGenerator(NodeGeneratorBase):
"""
Class holding added files for current commit
"""
class ChangedFileNodesGenerator(NodeGeneratorBase):
"""
Class holding changed files for current commit
"""
class RemovedFileNodesGenerator(NodeGeneratorBase):
"""
Class holding removed files for current commit
"""
def __iter__(self):
for p in self.current_paths:
vcs-support: bulk of changes for python3
r5075 yield RemovedFileNode(path=safe_bytes(p))
project: added all source files and assets
r1
vcs-support: bulk of changes for python3
r5075 def __getitem__(self, key):
if isinstance(key, slice):
for p in self.current_paths[key.start:key.stop]:
yield RemovedFileNode(path=safe_bytes(p))
project: added all source files and assets
r1
vcs-support: bulk of changes for python3
r5075 @functools.total_ordering
project: added all source files and assets
r1 class Node(object):
"""
Simplest class representing file or directory on repository. SCM backends
should use ``FileNode`` and ``DirNode`` subclasses rather than ``Node``
directly.
Node's ``path`` cannot start with slash as we operate on *relative* paths
only. Moreover, every single node is identified by the ``path`` attribute,
so it cannot end with slash, too. Otherwise, path could lead to mistakes.
"""
vcs-support: bulk of changes for python3
r5075 # RTLO marker allows swapping text, and certain
# security attacks could be used with this
RTLO_MARKER = "\u202E"
project: added all source files and assets
r1 commit = None
vcs-support: bulk of changes for python3
r5075 def __init__(self, path: bytes, kind):
project: added all source files and assets
r1 self._validate_path(path) # can throw exception if path is invalid
vcs-support: bulk of changes for python3
r5075
self.bytes_path = path.rstrip(b'/') # store for __repr__
self.path = safe_str(self.bytes_path) # we store paths as str
if self.bytes_path == b'' and kind != NodeKind.DIR:
project: added all source files and assets
r1 raise NodeError("Only DirNode and its subclasses may be "
"initialized with empty path")
self.kind = kind
if self.is_root() and not self.is_dir():
raise NodeError("Root node cannot be FILE kind")
vcs-support: bulk of changes for python3
r5075 def __eq__(self, other):
if type(self) is not type(other):
return False
for attr in ['name', 'path', 'kind']:
if getattr(self, attr) != getattr(other, attr):
return False
if self.is_file():
# FileNode compare, we need to fallback to content compare
return None
else:
# For DirNode's check without entering each dir
self_nodes_paths = list(sorted(n.path for n in self.nodes))
other_nodes_paths = list(sorted(n.path for n in self.nodes))
if self_nodes_paths != other_nodes_paths:
return False
return True
def __lt__(self, other):
if self.kind < other.kind:
return True
if self.kind > other.kind:
return False
if self.path < other.path:
return True
if self.path > other.path:
return False
# def __cmp__(self, other):
# """
# Comparator using name of the node, needed for quick list sorting.
# """
#
# kind_cmp = cmp(self.kind, other.kind)
# if kind_cmp:
# if isinstance(self, SubModuleNode):
# # we make submodules equal to dirnode for "sorting" purposes
# return NodeKind.DIR
# return kind_cmp
# return cmp(self.name, other.name)
def __repr__(self):
maybe_path = getattr(self, 'path', 'UNKNOWN_PATH')
return f'<{self.__class__.__name__} {maybe_path!r}>'
def __str__(self):
return self.name
def _validate_path(self, path: bytes):
self._assert_bytes(path)
if path.startswith(b'/'):
project: added all source files and assets
r1 raise NodeError(
vcs-support: bulk of changes for python3
r5075 f"Cannot initialize Node objects with slash at "
f"the beginning as only relative paths are supported. "
f"Got {path}")
def _assert_bytes(self, value):
if not isinstance(value, bytes):
raise TypeError(f"Bytes required as input, got {type(value)} of {value}.")
project: added all source files and assets
r1
@LazyProperty
def parent(self):
parent_path = self.get_parent_path()
if parent_path:
if self.commit:
return self.commit.get_node(parent_path)
return DirNode(parent_path)
return None
@LazyProperty
vcs-support: bulk of changes for python3
r5075 def str_path(self) -> str:
return safe_str(self.path)
project: added all source files and assets
r1
@LazyProperty
files: remove rigth-to-left override character for display in files....
r2162 def has_rtlo(self):
"""Detects if a path has right-to-left-override marker"""
vcs-support: bulk of changes for python3
r5075 return self.RTLO_MARKER in self.str_path
files: remove rigth-to-left override character for display in files....
r2162
@LazyProperty
project: added all source files and assets
r1 def dir_path(self):
"""
Returns name of the directory from full path of this vcs node. Empty
string is returned if there's no directory in the path
"""
_parts = self.path.rstrip('/').rsplit('/', 1)
if len(_parts) == 2:
vcs-support: bulk of changes for python3
r5075 return _parts[0]
FOLD: into unicode changes
r4959 return ''
project: added all source files and assets
r1
@LazyProperty
def name(self):
"""
Returns name of the node so if its path
then only last part is returned.
"""
vcs-support: bulk of changes for python3
r5075 return self.path.rstrip('/').split('/')[-1]
project: added all source files and assets
r1
@property
def kind(self):
return self._kind
@kind.setter
def kind(self, kind):
if hasattr(self, '_kind'):
raise NodeError("Cannot change node's kind")
else:
self._kind = kind
# Post setter check (path's trailing slash)
if self.path.endswith('/'):
raise NodeError("Node's path cannot end with slash")
vcs-support: bulk of changes for python3
r5075 def get_parent_path(self) -> bytes:
project: added all source files and assets
r1 """
Returns node's parent path or empty string if node is root.
"""
if self.is_root():
vcs-support: bulk of changes for python3
r5075 return b''
str_path = vcspath.dirname(self.path.rstrip('/')) + '/'
return safe_bytes(str_path)
project: added all source files and assets
r1
def is_file(self):
"""
Returns ``True`` if node's kind is ``NodeKind.FILE``, ``False``
otherwise.
"""
return self.kind == NodeKind.FILE
def is_dir(self):
"""
Returns ``True`` if node's kind is ``NodeKind.DIR``, ``False``
otherwise.
"""
return self.kind == NodeKind.DIR
def is_root(self):
"""
Returns ``True`` if node is a root node and ``False`` otherwise.
"""
return self.kind == NodeKind.DIR and self.path == ''
def is_submodule(self):
"""
Returns ``True`` if node's kind is ``NodeKind.SUBMODULE``, ``False``
otherwise.
"""
return self.kind == NodeKind.SUBMODULE
def is_largefile(self):
"""
Returns ``True`` if node's kind is ``NodeKind.LARGEFILE``, ``False``
otherwise
"""
return self.kind == NodeKind.LARGEFILE
def is_link(self):
if self.commit:
return self.commit.is_link(self.path)
return False
@LazyProperty
def added(self):
return self.state is NodeState.ADDED
@LazyProperty
def changed(self):
return self.state is NodeState.CHANGED
@LazyProperty
def not_changed(self):
return self.state is NodeState.NOT_CHANGED
@LazyProperty
def removed(self):
return self.state is NodeState.REMOVED
class FileNode(Node):
"""
Class representing file nodes.
:attribute: path: path to the node, relative to repository's root
:attribute: content: if given arbitrary sets content of the file
:attribute: commit: if given, first time content is accessed, callback
:attribute: mode: stat mode for a node. Default is `FILEMODE_DEFAULT`.
"""
vcs: added possibility to pre-load attributes for FileNodes.
r1355 _filter_pre_load = []
project: added all source files and assets
r1
vcs-support: bulk of changes for python3
r5075 def __init__(self, path: bytes, content: bytes | None = None, commit=None, mode=None, pre_load=None):
project: added all source files and assets
r1 """
Only one of ``content`` and ``commit`` may be given. Passing both
would raise ``NodeError`` exception.
:param path: relative path to the node
:param content: content may be passed to constructor
:param commit: if given, will use it to lazily fetch content
:param mode: ST_MODE (i.e. 0100644)
"""
if content and commit:
raise NodeError("Cannot use both content and commit")
vcs-support: bulk of changes for python3
r5075
super().__init__(path, kind=NodeKind.FILE)
project: added all source files and assets
r1 self.commit = commit
vcs-support: bulk of changes for python3
r5075 if content and not isinstance(content, bytes):
# File content is one thing that inherently must be bytes
# we support passing str too, and convert the content
content = safe_bytes(content)
project: added all source files and assets
r1 self._content = content
self._mode = mode or FILEMODE_DEFAULT
vcs: added possibility to pre-load attributes for FileNodes.
r1355 self._set_bulk_properties(pre_load)
vcs-support: bulk of changes for python3
r5075 def __eq__(self, other):
libs: more python3 reformats
r5091 eq = super().__eq__(other)
vcs-support: bulk of changes for python3
r5075 if eq is not None:
return eq
return self.content == other.content
def __hash__(self):
raw_id = getattr(self.commit, 'raw_id', '')
return hash((self.path, raw_id))
def __lt__(self, other):
libs: more python3 reformats
r5091 lt = super().__lt__(other)
vcs-support: bulk of changes for python3
r5075 if lt is not None:
return lt
return self.content < other.content
def __repr__(self):
short_id = getattr(self.commit, 'short_id', '')
return f'<{self.__class__.__name__} path={self.path!r}, short_id={short_id}>'
vcs: added possibility to pre-load attributes for FileNodes.
r1355 def _set_bulk_properties(self, pre_load):
if not pre_load:
return
pre_load = [entry for entry in pre_load
if entry not in self._filter_pre_load]
if not pre_load:
return
vcs-support: bulk of changes for python3
r5075 remote = self.commit.get_remote()
result = remote.bulk_file_request(self.commit.raw_id, self.path, pre_load)
for attr, value in result.items():
if attr == "flags":
self.__dict__['mode'] = safe_str(value)
elif attr == "size":
self.__dict__['size'] = value
elif attr == "data":
self.__dict__['_content'] = value
elif attr == "is_binary":
self.__dict__['is_binary'] = value
elif attr == "md5":
self.__dict__['md5'] = value
else:
raise ValueError(f'Unsupported attr in bulk_property: {attr}')
vcs: added possibility to pre-load attributes for FileNodes.
r1355
project: added all source files and assets
r1 @LazyProperty
def mode(self):
"""
Returns lazily mode of the FileNode. If `commit` is not set, would
use value given at initialization or `FILEMODE_DEFAULT` (default).
"""
if self.commit:
mode = self.commit.get_file_mode(self.path)
else:
mode = self._mode
return mode
dan
scm: change ._get_content() to .raw_bytes attribute to file nodes to...
r501 @LazyProperty
vcs-support: bulk of changes for python3
r5075 def raw_bytes(self) -> bytes:
dan
scm: change ._get_content() to .raw_bytes attribute to file nodes to...
r501 """
Returns lazily the raw bytes of the FileNode.
"""
project: added all source files and assets
r1 if self.commit:
dan
scm: change ._get_content() to .raw_bytes attribute to file nodes to...
r501 if self._content is None:
self._content = self.commit.get_file_content(self.path)
content = self._content
project: added all source files and assets
r1 else:
content = self._content
return content
vcs-support: bulk of changes for python3
r5075 def content_uncached(self):
"""
Returns lazily content of the FileNode.
"""
if self.commit:
content = self.commit.get_file_content(self.path)
else:
content = self._content
return content
dan
file-nodes: added streaming remote attributes for vcsserver....
r3895 def stream_bytes(self):
"""
Returns an iterator that will stream the content of the file directly from
vcsserver without loading it to memory.
"""
if self.commit:
return self.commit.get_file_content_streamed(self.path)
dan
vcsserver: made binary content check be calculated on vcsserver...
r3896 raise NodeError("Cannot retrieve stream_bytes without related commit attribute")
dan
file-nodes: added streaming remote attributes for vcsserver....
r3895
api: expose new functions for FTS...
r3460 def metadata_uncached(self):
"""
Returns md5, binary flag of the file node, without any cache usage.
"""
api: allow uncached content fetching....
r3479 content = self.content_uncached()
api: expose new functions for FTS...
r3460
vcs-support: bulk of changes for python3
r5075 is_binary = bool(content and BIN_BYTE_MARKER in content)
api: expose new functions for FTS...
r3460 size = 0
if content:
size = len(content)
api: allow uncached content fetching....
r3479
return is_binary, md5(content), size, content
vcs-support: bulk of changes for python3
r5075 @LazyProperty
def content(self) -> bytes:
api: allow uncached content fetching....
r3479 """
vcs-support: bulk of changes for python3
r5075 Returns lazily content of the FileNode.
"""
content = self.raw_bytes
if content and not isinstance(content, bytes):
raise ValueError(f'Content is of type {type(content)} instead of bytes')
api: allow uncached content fetching....
r3479 return content
api: expose new functions for FTS...
r3460
dan
scm: change ._get_content() to .raw_bytes attribute to file nodes to...
r501 @LazyProperty
vcs-support: bulk of changes for python3
r5075 def str_content(self) -> str:
return safe_str(self.raw_bytes)
project: added all source files and assets
r1
@LazyProperty
def size(self):
if self.commit:
return self.commit.get_file_size(self.path)
raise NodeError(
"Cannot retrieve size of the file without related "
"commit attribute")
@LazyProperty
def message(self):
if self.commit:
return self.last_commit.message
raise NodeError(
"Cannot retrieve message of the file without related "
"commit attribute")
@LazyProperty
def last_commit(self):
if self.commit:
vcs: optimized pre-load attributes for better caching.
r3850 pre_load = ["author", "date", "message", "parents"]
vcs: rename get_file_history to get_path_history as it better reflects what it does.
r3275 return self.commit.get_path_commit(self.path, pre_load=pre_load)
project: added all source files and assets
r1 raise NodeError(
"Cannot retrieve last commit of the file without "
"related commit attribute")
def get_mimetype(self):
"""
Mimetype is calculated based on the file's content. If ``_mimetype``
attribute is available, it will be returned (backends which store
mimetypes or can easily recognize them, should set this private
attribute to indicate that type should *NOT* be calculated).
"""
if hasattr(self, '_mimetype'):
vcs-support: bulk of changes for python3
r5075 if (isinstance(self._mimetype, (tuple, list)) and
project: added all source files and assets
r1 len(self._mimetype) == 2):
return self._mimetype
else:
raise NodeError('given _mimetype attribute must be an 2 '
'element list or tuple')
db = get_mimetypes_db()
mtype, encoding = db.guess_type(self.name)
if mtype is None:
largefiles: added fix for downloading largefiles without an extension in name.
r4666 if not self.is_largefile() and self.is_binary:
project: added all source files and assets
r1 mtype = 'application/octet-stream'
encoding = None
else:
mtype = 'text/plain'
encoding = None
# try with pygments
try:
from pygments.lexers import get_lexer_for_filename
mt = get_lexer_for_filename(self.name).mimetypes
except Exception:
mt = None
if mt:
mtype = mt[0]
return mtype, encoding
@LazyProperty
def mimetype(self):
"""
Wrapper around full mimetype info. It returns only type of fetched
mimetype without the encoding part. use get_mimetype function to fetch
full set of (type,encoding)
"""
return self.get_mimetype()[0]
@LazyProperty
def mimetype_main(self):
return self.mimetype.split('/')[0]
vcs: refactor get_lexer for nodes so it can be used in external code.
r1357 @classmethod
def get_lexer(cls, filename, content=None):
project: added all source files and assets
r1 from pygments import lexers
lexers: added small extensions table to extend syntaxt hl for file sources....
r796
vcs: refactor get_lexer for nodes so it can be used in external code.
r1357 extension = filename.split('.')[-1]
lexers: added small extensions table to extend syntaxt hl for file sources....
r796 lexer = None
vcs: refactor get_lexer for nodes so it can be used in external code.
r1357
project: added all source files and assets
r1 try:
lexers: added small extensions table to extend syntaxt hl for file sources....
r796 lexer = lexers.guess_lexer_for_filename(
vcs: refactor get_lexer for nodes so it can be used in external code.
r1357 filename, content, stripnl=False)
project: added all source files and assets
r1 except lexers.ClassNotFound:
vcs-support: bulk of changes for python3
r5075 pass
lexers: added small extensions table to extend syntaxt hl for file sources....
r796
# try our EXTENSION_MAP
if not lexer:
try:
vcs: refactor get_lexer for nodes so it can be used in external code.
r1357 lexer_class = LANGUAGES_EXTENSIONS_MAP.get(extension)
lexers: added small extensions table to extend syntaxt hl for file sources....
r796 if lexer_class:
lexer = lexers.get_lexer_by_name(lexer_class[0])
except lexers.ClassNotFound:
vcs-support: bulk of changes for python3
r5075 pass
lexers: added small extensions table to extend syntaxt hl for file sources....
r796
if not lexer:
project: added all source files and assets
r1 lexer = lexers.TextLexer(stripnl=False)
lexers: added small extensions table to extend syntaxt hl for file sources....
r796
project: added all source files and assets
r1 return lexer
@LazyProperty
vcs: refactor get_lexer for nodes so it can be used in external code.
r1357 def lexer(self):
"""
Returns pygment's lexer class. Would try to guess lexer taking file's
content, name and mimetype.
"""
vcs-support: bulk of changes for python3
r5075 # TODO: this is more proper, but super heavy on investigating the type based on the content
#self.get_lexer(self.name, self.content)
return self.get_lexer(self.name)
vcs: refactor get_lexer for nodes so it can be used in external code.
r1357
@LazyProperty
project: added all source files and assets
r1 def lexer_alias(self):
"""
Returns first alias of the lexer guessed for this file.
"""
return self.lexer.aliases[0]
@LazyProperty
def history(self):
"""
Returns a list of commit for this file in which the file was changed
"""
if self.commit is None:
raise NodeError('Unable to get commit for this FileNode')
vcs: rename get_file_history to get_path_history as it better reflects what it does.
r3275 return self.commit.get_path_history(self.path)
project: added all source files and assets
r1
@LazyProperty
def annotate(self):
"""
Returns a list of three element tuples with lineno, commit and line
"""
if self.commit is None:
raise NodeError('Unable to get commit for this FileNode')
vcs: optimized pre-load attributes for better caching.
r3850 pre_load = ["author", "date", "message", "parents"]
project: added all source files and assets
r1 return self.commit.get_file_annotate(self.path, pre_load=pre_load)
@LazyProperty
def state(self):
if not self.commit:
raise NodeError(
"Cannot check state of the node if it's not "
"linked with commit")
elif self.path in (node.path for node in self.commit.added):
return NodeState.ADDED
elif self.path in (node.path for node in self.commit.changed):
return NodeState.CHANGED
else:
return NodeState.NOT_CHANGED
dan
scm: change ._get_content() to .raw_bytes attribute to file nodes to...
r501 @LazyProperty
project: added all source files and assets
r1 def is_binary(self):
"""
Returns True if file has binary content.
"""
dan
vcsserver: made binary content check be calculated on vcsserver...
r3896 if self.commit:
return self.commit.is_node_binary(self.path)
else:
raw_bytes = self._content
vcs-support: bulk of changes for python3
r5075 return bool(raw_bytes and BIN_BYTE_MARKER in raw_bytes)
@LazyProperty
def md5(self):
"""
Returns md5 of the file node.
"""
if self.commit:
return self.commit.node_md5_hash(self.path)
else:
raw_bytes = self._content
# TODO: this sucks, we're computing md5 on potentially super big stream data...
return md5(raw_bytes)
project: added all source files and assets
r1
@LazyProperty
def extension(self):
"""Returns filenode extension"""
return self.name.split('.')[-1]
@property
def is_executable(self):
"""
Returns ``True`` if file has executable flag turned on.
"""
return bool(self.mode & stat.S_IXUSR)
def get_largefile_node(self):
"""
Try to return a Mercurial FileNode from this node. It does internal
checks inside largefile store, if that file exist there it will
create special instance of LargeFileNode which can get content from
LF store.
"""
largefiles: enabled download of largefiles for git and mercurial from web interface....
r1577 if self.commit:
return self.commit.get_largefile_node(self.path)
project: added all source files and assets
r1
vcs-support: bulk of changes for python3
r5075 def count_lines(self, content: str | bytes, count_empty=False):
if isinstance(content, str):
newline_marker = '\n'
elif isinstance(content, bytes):
newline_marker = b'\n'
else:
raise ValueError('content must be bytes or str got {type(content)} instead')
nodes: expose line counts in node information. This would be used in full text search
r3962
if count_empty:
all_lines = 0
empty_lines = 0
for line in content.splitlines(True):
vcs-support: bulk of changes for python3
r5075 if line == newline_marker:
nodes: expose line counts in node information. This would be used in full text search
r3962 empty_lines += 1
all_lines += 1
return all_lines, all_lines - empty_lines
else:
# fast method
vcs-support: bulk of changes for python3
r5075 empty_lines = all_lines = content.count(newline_marker)
nodes: expose line counts in node information. This would be used in full text search
r3962 if all_lines == 0 and content:
# one-line without a newline
empty_lines = all_lines = 1
return all_lines, empty_lines
project: added all source files and assets
r1 def lines(self, count_empty=False):
all_lines, empty_lines = 0, 0
if not self.is_binary:
dan
scm: change ._get_content() to .raw_bytes attribute to file nodes to...
r501 content = self.content
nodes: expose line counts in node information. This would be used in full text search
r3962 all_lines, empty_lines = self.count_lines(content, count_empty=count_empty)
project: added all source files and assets
r1 return all_lines, empty_lines
class RemovedFileNode(FileNode):
"""
Dummy FileNode class - trying to access any public attribute except path,
name, kind or state (or methods/attributes checking those two) would raise
RemovedFileNodeError.
"""
ALLOWED_ATTRIBUTES = [
'name', 'path', 'state', 'is_root', 'is_file', 'is_dir', 'kind',
vcs-support: bulk of changes for python3
r5075 'added', 'changed', 'not_changed', 'removed', 'bytes_path'
project: added all source files and assets
r1 ]
def __init__(self, path):
"""
:param path: relative path to the node
"""
vcs-support: bulk of changes for python3
r5075 super().__init__(path=path)
project: added all source files and assets
r1
def __getattribute__(self, attr):
if attr.startswith('_') or attr in RemovedFileNode.ALLOWED_ATTRIBUTES:
vcs-support: bulk of changes for python3
r5075 return super().__getattribute__(attr)
raise RemovedFileNodeError(f"Cannot access attribute {attr} on RemovedFileNode. Not in allowed attributes")
project: added all source files and assets
r1
@LazyProperty
def state(self):
return NodeState.REMOVED
class DirNode(Node):
"""
DirNode stores list of files and directories within this node.
Nodes may be used standalone but within repository context they
quick-filter: use a dedicated method for fetching quick filter nodes....
r3925 lazily fetch data within same repository's commit.
project: added all source files and assets
r1 """
vcs-support: bulk of changes for python3
r5075 def __init__(self, path, nodes=(), commit=None, default_pre_load=None):
project: added all source files and assets
r1 """
Only one of ``nodes`` and ``commit`` may be given. Passing both
would raise ``NodeError`` exception.
:param path: relative path to the node
:param nodes: content may be passed to constructor
:param commit: if given, will use it to lazily fetch content
"""
if nodes and commit:
raise NodeError("Cannot use both nodes and commit")
libs: more python3 reformats
r5091 super().__init__(path, NodeKind.DIR)
project: added all source files and assets
r1 self.commit = commit
self._nodes = nodes
vcs-support: bulk of changes for python3
r5075 self.default_pre_load = default_pre_load or ['is_binary', 'size']
def __iter__(self):
libs: more python3 reformats
r5091 yield from self.nodes
vcs-support: bulk of changes for python3
r5075
def __eq__(self, other):
libs: more python3 reformats
r5091 eq = super().__eq__(other)
vcs-support: bulk of changes for python3
r5075 if eq is not None:
return eq
# check without entering each dir
self_nodes_paths = list(sorted(n.path for n in self.nodes))
other_nodes_paths = list(sorted(n.path for n in self.nodes))
return self_nodes_paths == other_nodes_paths
def __lt__(self, other):
libs: more python3 reformats
r5091 lt = super().__lt__(other)
vcs-support: bulk of changes for python3
r5075 if lt is not None:
return lt
# check without entering each dir
self_nodes_paths = list(sorted(n.path for n in self.nodes))
other_nodes_paths = list(sorted(n.path for n in self.nodes))
return self_nodes_paths < other_nodes_paths
project: added all source files and assets
r1
@LazyProperty
def content(self):
vcs-support: bulk of changes for python3
r5075 raise NodeError(f"{self} represents a dir and has no `content` attribute")
project: added all source files and assets
r1
@LazyProperty
def nodes(self):
if self.commit:
vcs-support: bulk of changes for python3
r5075 nodes = self.commit.get_nodes(self.path, pre_load=self.default_pre_load)
project: added all source files and assets
r1 else:
nodes = self._nodes
libs: more python3 reformats
r5091 self._nodes_dict = {node.path: node for node in nodes}
project: added all source files and assets
r1 return sorted(nodes)
@LazyProperty
def files(self):
libs: more python3 reformats
r5091 return sorted(node for node in self.nodes if node.is_file())
project: added all source files and assets
r1
@LazyProperty
def dirs(self):
libs: more python3 reformats
r5091 return sorted(node for node in self.nodes if node.is_dir())
project: added all source files and assets
r1
def get_node(self, path):
"""
Returns node from within this particular ``DirNode``, so it is now
allowed to fetch, i.e. node located at 'docs/api/index.rst' from node
'docs'. In order to access deeper nodes one must fetch nodes between
them first - this would work::
docs = root.get_node('docs')
docs.get_node('api').get_node('index.rst')
:param: path - relative to the current node
.. note::
To access lazily (as in example above) node have to be initialized
with related commit object - without it node is out of
context and may know nothing about anything else than nearest
(located at same level) nodes.
"""
try:
path = path.rstrip('/')
if path == '':
raise NodeError("Cannot retrieve node without path")
self.nodes # access nodes first in order to set _nodes_dict
paths = path.split('/')
if len(paths) == 1:
if not self.is_root():
path = '/'.join((self.path, paths[0]))
else:
path = paths[0]
return self._nodes_dict[path]
elif len(paths) > 1:
if self.commit is None:
dan
vcsserver: made binary content check be calculated on vcsserver...
r3896 raise NodeError("Cannot access deeper nodes without commit")
project: added all source files and assets
r1 else:
path1, path2 = paths[0], '/'.join(paths[1:])
return self.get_node(path1).get_node(path2)
else:
raise KeyError
except KeyError:
vcs-support: bulk of changes for python3
r5075 raise NodeError(f"Node does not exist at {path}")
project: added all source files and assets
r1
@LazyProperty
def state(self):
raise NodeError("Cannot access state of DirNode")
@LazyProperty
def size(self):
size = 0
for root, dirs, files in self.commit.walk(self.path):
for f in files:
size += f.size
return size
vcs: rename get_file_history to get_path_history as it better reflects what it does.
r3275 @LazyProperty
def last_commit(self):
if self.commit:
vcs: optimized pre-load attributes for better caching.
r3850 pre_load = ["author", "date", "message", "parents"]
vcs: rename get_file_history to get_path_history as it better reflects what it does.
r3275 return self.commit.get_path_commit(self.path, pre_load=pre_load)
raise NodeError(
"Cannot retrieve last commit of the file without "
"related commit attribute")
project: added all source files and assets
r1 def __repr__(self):
vcs-support: bulk of changes for python3
r5075 short_id = getattr(self.commit, 'short_id', '')
return f'<{self.__class__.__name__} {self.path!r} @ {short_id}>'
project: added all source files and assets
r1
class RootNode(DirNode):
"""
DirNode being the root node of the repository.
"""
def __init__(self, nodes=(), commit=None):
libs: more python3 reformats
r5091 super().__init__(path=b'', nodes=nodes, commit=commit)
project: added all source files and assets
r1
def __repr__(self):
vcs-support: bulk of changes for python3
r5075 return f'<{self.__class__.__name__}>'
project: added all source files and assets
r1
class SubModuleNode(Node):
"""
represents a SubModule of Git or SubRepo of Mercurial
"""
is_binary = False
size = 0
def __init__(self, name, url=None, commit=None, alias=None):
self.path = name
self.kind = NodeKind.SUBMODULE
self.alias = alias
# we have to use EmptyCommit here since this can point to svn/git/hg
# submodules we cannot get from repository
self.commit = EmptyCommit(str(commit), alias=alias)
self.url = url or self._extract_submodule_url()
def __repr__(self):
vcs-support: bulk of changes for python3
r5075 short_id = getattr(self.commit, 'short_id', '')
return f'<{self.__class__.__name__} {self.path!r} @ {short_id}>'
project: added all source files and assets
r1
def _extract_submodule_url(self):
# TODO: find a way to parse gits submodule file and extract the
# linking URL
return self.path
@LazyProperty
def name(self):
"""
Returns name of the node so if its path
then only last part is returned.
"""
vcs-support: bulk of changes for python3
r5075 org = safe_str(self.path.rstrip('/').split('/')[-1])
return f'{org} @ {self.commit.short_id}'
project: added all source files and assets
r1
class LargeFileNode(FileNode):
largefiles: enabled download of largefiles for git and mercurial from web interface....
r1577 def __init__(self, path, url=None, commit=None, alias=None, org_path=None):
vcs-support: bulk of changes for python3
r5075 self._validate_path(path) # can throw exception if path is invalid
self.org_path = org_path # as stored in VCS as LF pointer
self.bytes_path = path.rstrip(b'/') # store for __repr__
self.path = safe_str(self.bytes_path) # we store paths as str
largefiles: enabled download of largefiles for git and mercurial from web interface....
r1577 self.kind = NodeKind.LARGEFILE
self.alias = alias
vcs-support: bulk of changes for python3
r5075 self._content = b''
largefiles: enabled download of largefiles for git and mercurial from web interface....
r1577
vcs-support: bulk of changes for python3
r5075 def _validate_path(self, path: bytes):
project: added all source files and assets
r1 """
vcs-support: bulk of changes for python3
r5075 we override check since the LargeFileNode path is system absolute, but we check for bytes only
project: added all source files and assets
r1 """
vcs-support: bulk of changes for python3
r5075 self._assert_bytes(path)
project: added all source files and assets
r1
largefiles: enabled download of largefiles for git and mercurial from web interface....
r1577 def __repr__(self):
vcs-support: bulk of changes for python3
r5075 return f'<{self.__class__.__name__} {self.org_path} -> {self.path!r}>'
largefiles: enabled download of largefiles for git and mercurial from web interface....
r1577
@LazyProperty
def size(self):
return os.stat(self.path).st_size
@LazyProperty
dan
scm: change ._get_content() to .raw_bytes attribute to file nodes to...
r501 def raw_bytes(self):
nodes: fetching largefiles content is commit independent because the pointes is...
r1627 with open(self.path, 'rb') as f:
content = f.read()
largefiles: enabled download of largefiles for git and mercurial from web interface....
r1577 return content
@LazyProperty
def name(self):
"""
Overwrites name to be the org lf path
"""
return self.org_path
dan
file-nodes: added streaming remote attributes for vcsserver....
r3895
def stream_bytes(self):
with open(self.path, 'rb') as stream:
while True:
data = stream.read(16 * 1024)
if not data:
break
yield data