vcs: added possibility to pre-load attributes for FileNodes.

rhodecode/lib/vcs/backends/git/commit.py

@@ -1,527 +1,527 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2014-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 GIT commit module
 """
 
 import re
 import stat
 from ConfigParser import ConfigParser
 from itertools import chain
 from StringIO import StringIO
 
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 from rhodecode.lib.datelib import utcdate_fromtimestamp
 from rhodecode.lib.utils import safe_unicode, safe_str
 from rhodecode.lib.utils2 import safe_int
 from rhodecode.lib.vcs.conf import settings
 from rhodecode.lib.vcs.backends import base
 from rhodecode.lib.vcs.exceptions import CommitError, NodeDoesNotExistError
 from rhodecode.lib.vcs.nodes import (
     FileNode, DirNode, NodeKind, RootNode, SubModuleNode,
     ChangedFileNodesGenerator, AddedFileNodesGenerator,
     RemovedFileNodesGenerator)
 
 
 class GitCommit(base.BaseCommit):
     """
     Represents state of the repository at single commit id.
     """
     _author_property = 'author'
     _committer_property = 'committer'
     _date_property = 'commit_time'
     _date_tz_property = 'commit_timezone'
     _message_property = 'message'
     _parents_property = 'parents'
 
     _filter_pre_load = [
         # done through a more complex tree walk on parents
         "affected_files",
         # based on repository cached property
         "branch",
         # done through subprocess not remote call
         "children",
         # done through a more complex tree walk on parents
         "status",
         # mercurial specific property not supported here
         "_file_paths",
     ]
 
     def __init__(self, repository, raw_id, idx, pre_load=None):
         self.repository = repository
         self._remote = repository._remote
         # TODO: johbo: Tweak of raw_id should not be necessary
         self.raw_id = safe_str(raw_id)
         self.idx = idx
 
         self._set_bulk_properties(pre_load)
 
         # caches
         self._stat_modes = {}  # stat info for paths
         self._paths = {}  # path processed with parse_tree
         self.nodes = {}
         self._submodules = None
 
     def _set_bulk_properties(self, pre_load):
         if not pre_load:
             return
         pre_load = [entry for entry in pre_load
                     if entry not in self._filter_pre_load]
         if not pre_load:
             return
 
         result = self._remote.bulk_request(self.raw_id, pre_load)
         for attr, value in result.items():
             if attr in ["author", "message"]:
                 if value:
                     value = safe_unicode(value)
             elif attr == "date":
                 value = utcdate_fromtimestamp(*value)
             elif attr == "parents":
                 value = self._make_commits(value)
             self.__dict__[attr] = value
 
     @LazyProperty
     def _commit(self):
         return self._remote[self.raw_id]
 
     @LazyProperty
     def _tree_id(self):
         return self._remote[self._commit['tree']]['id']
 
     @LazyProperty
     def id(self):
         return self.raw_id
 
     @LazyProperty
     def short_id(self):
         return self.raw_id[:12]
 
     @LazyProperty
     def message(self):
         return safe_unicode(
             self._remote.commit_attribute(self.id, self._message_property))
 
     @LazyProperty
     def committer(self):
         return safe_unicode(
             self._remote.commit_attribute(self.id, self._committer_property))
 
     @LazyProperty
     def author(self):
         return safe_unicode(
             self._remote.commit_attribute(self.id, self._author_property))
 
     @LazyProperty
     def date(self):
         unix_ts, tz = self._remote.get_object_attrs(
             self.raw_id, self._date_property, self._date_tz_property)
         return utcdate_fromtimestamp(unix_ts, tz)
 
     @LazyProperty
     def status(self):
         """
         Returns modified, added, removed, deleted files for current commit
         """
         return self.changed, self.added, self.removed
 
     @LazyProperty
     def tags(self):
         tags = [safe_unicode(name) for name,
                 commit_id in self.repository.tags.iteritems()
                 if commit_id == self.raw_id]
         return tags
 
     @LazyProperty
     def branch(self):
         for name, commit_id in self.repository.branches.iteritems():
             if commit_id == self.raw_id:
                 return safe_unicode(name)
         return None
 
     def _get_id_for_path(self, path):
         path = safe_str(path)
         if path in self._paths:
             return self._paths[path]
 
         tree_id = self._tree_id
 
         path = path.strip('/')
         if path == '':
             data = [tree_id, "tree"]
             self._paths[''] = data
             return data
 
         parts = path.split('/')
         dirs, name = parts[:-1], parts[-1]
         cur_dir = ''
 
         # initially extract things from root dir
         tree_items = self._remote.tree_items(tree_id)
         self._process_tree_items(tree_items, cur_dir)
 
         for dir in dirs:
             if cur_dir:
                 cur_dir = '/'.join((cur_dir, dir))
             else:
                 cur_dir = dir
             dir_id = None
             for item, stat_, id_, type_ in tree_items:
                 if item == dir:
                     dir_id = id_
                     break
             if dir_id:
                 if type_ != "tree":
                     raise CommitError('%s is not a directory' % cur_dir)
                 # update tree
                 tree_items = self._remote.tree_items(dir_id)
             else:
                 raise CommitError('%s have not been found' % cur_dir)
 
             # cache all items from the given traversed tree
             self._process_tree_items(tree_items, cur_dir)
 
         if path not in self._paths:
             raise self.no_node_at_path(path)
 
         return self._paths[path]
 
     def _process_tree_items(self, items, cur_dir):
         for item, stat_, id_, type_ in items:
             if cur_dir:
                 name = '/'.join((cur_dir, item))
             else:
                 name = item
             self._paths[name] = [id_, type_]
             self._stat_modes[name] = stat_
 
     def _get_kind(self, path):
         path_id, type_ = self._get_id_for_path(path)
         if type_ == 'blob':
             return NodeKind.FILE
         elif type_ == 'tree':
             return NodeKind.DIR
         elif type == 'link':
             return NodeKind.SUBMODULE
         return None
 
     def _get_filectx(self, path):
         path = self._fix_path(path)
         if self._get_kind(path) != NodeKind.FILE:
             raise CommitError(
                 "File does not exist for commit %s at  '%s'" %
                 (self.raw_id, path))
         return path
 
     def _get_file_nodes(self):
         return chain(*(t[2] for t in self.walk()))
 
     @LazyProperty
     def parents(self):
         """
         Returns list of parent commits.
         """
         parent_ids = self._remote.commit_attribute(
             self.id, self._parents_property)
         return self._make_commits(parent_ids)
 
     @LazyProperty
     def children(self):
         """
         Returns list of child commits.
         """
         rev_filter = settings.GIT_REV_FILTER
         output, __ = self.repository.run_git_command(
             ['rev-list', '--children'] + rev_filter)
 
         child_ids = []
         pat = re.compile(r'^%s' % self.raw_id)
         for l in output.splitlines():
             if pat.match(l):
                 found_ids = l.split(' ')[1:]
                 child_ids.extend(found_ids)
         return self._make_commits(child_ids)
 
     def _make_commits(self, commit_ids):
         return [self.repository.get_commit(commit_id=commit_id)
                 for commit_id in commit_ids]
 
     def get_file_mode(self, path):
         """
         Returns stat mode of the file at the given `path`.
         """
         path = safe_str(path)
         # ensure path is traversed
         self._get_id_for_path(path)
         return self._stat_modes[path]
 
     def is_link(self, path):
         return stat.S_ISLNK(self.get_file_mode(path))
 
     def get_file_content(self, path):
         """
         Returns content of the file at given `path`.
         """
         id_, _ = self._get_id_for_path(path)
         return self._remote.blob_as_pretty_string(id_)
 
     def get_file_size(self, path):
         """
         Returns size of the file at given `path`.
         """
         id_, _ = self._get_id_for_path(path)
         return self._remote.blob_raw_length(id_)
 
     def get_file_history(self, path, limit=None, pre_load=None):
         """
         Returns history of file as reversed list of `GitCommit` objects for
         which file at given `path` has been modified.
 
         TODO: This function now uses an underlying 'git' command which works
         quickly but ideally we should replace with an algorithm.
         """
         self._get_filectx(path)
         f_path = safe_str(path)
 
         cmd = ['log']
         if limit:
             cmd.extend(['-n', str(safe_int(limit, 0))])
         cmd.extend(['--pretty=format: %H', '-s', self.raw_id, '--', f_path])
 
         output, __ = self.repository.run_git_command(cmd)
         commit_ids = re.findall(r'[0-9a-fA-F]{40}', output)
 
         return [
             self.repository.get_commit(commit_id=commit_id, pre_load=pre_load)
             for commit_id in commit_ids]
 
     # TODO: unused for now potential replacement for subprocess
     def get_file_history_2(self, path, limit=None, pre_load=None):
         """
         Returns history of file as reversed list of `Commit` objects for
         which file at given `path` has been modified.
         """
         self._get_filectx(path)
         f_path = safe_str(path)
 
         commit_ids = self._remote.get_file_history(f_path, self.id, limit)
 
         return [
             self.repository.get_commit(commit_id=commit_id, pre_load=pre_load)
             for commit_id in commit_ids]
 
     def get_file_annotate(self, path, pre_load=None):
         """
         Returns a generator of four element tuples with
             lineno, commit_id, commit lazy loader and line
 
         TODO: This function now uses os underlying 'git' command which is
         generally not good. Should be replaced with algorithm iterating
         commits.
         """
         cmd = ['blame', '-l', '--root', '-r', self.raw_id, '--', path]
         # -l     ==> outputs long shas (and we need all 40 characters)
         # --root ==> doesn't put '^' character for bounderies
         # -r commit_id ==> blames for the given commit
         output, __ = self.repository.run_git_command(cmd)
 
         for i, blame_line in enumerate(output.split('\n')[:-1]):
             line_no = i + 1
             commit_id, line = re.split(r' ', blame_line, 1)
             yield (
                 line_no, commit_id,
                 lambda: self.repository.get_commit(commit_id=commit_id,
                                                    pre_load=pre_load),
                 line)
 
     def get_nodes(self, path):
         if self._get_kind(path) != NodeKind.DIR:
             raise CommitError(
                 "Directory does not exist for commit %s at "
                 " '%s'" % (self.raw_id, path))
         path = self._fix_path(path)
         id_, _ = self._get_id_for_path(path)
         tree_id = self._remote[id_]['id']
         dirnodes = []
         filenodes = []
         alias = self.repository.alias
         for name, stat_, id_, type_ in self._remote.tree_items(tree_id):
             if type_ == 'link':
                 url = self._get_submodule_url('/'.join((path, name)))
                 dirnodes.append(SubModuleNode(
                     name, url=url, commit=id_, alias=alias))
                 continue
 
             if path != '':
                 obj_path = '/'.join((path, name))
             else:
                 obj_path = name
             if obj_path not in self._stat_modes:
                 self._stat_modes[obj_path] = stat_
 
             if type_ == 'tree':
                 dirnodes.append(DirNode(obj_path, commit=self))
             elif type_ == 'blob':
                 filenodes.append(FileNode(obj_path, commit=self, mode=stat_))
             else:
                 raise CommitError(
                     "Requested object should be Tree or Blob, is %s", type_)
 
         nodes = dirnodes + filenodes
         for node in nodes:
             if node.path not in self.nodes:
                 self.nodes[node.path] = node
         nodes.sort()
         return nodes
 
-    def get_node(self, path):
+    def get_node(self, path, pre_load=None):
         if isinstance(path, unicode):
             path = path.encode('utf-8')
         path = self._fix_path(path)
         if path not in self.nodes:
             try:
                 id_, type_ = self._get_id_for_path(path)
             except CommitError:
                 raise NodeDoesNotExistError(
                     "Cannot find one of parents' directories for a given "
                     "path: %s" % path)
 
             if type_ == 'link':
                 url = self._get_submodule_url(path)
                 node = SubModuleNode(path, url=url, commit=id_,
                                      alias=self.repository.alias)
             elif type_ == 'tree':
                 if path == '':
                     node = RootNode(commit=self)
                 else:
                     node = DirNode(path, commit=self)
             elif type_ == 'blob':
-                node = FileNode(path, commit=self)
+                node = FileNode(path, commit=self, pre_load=pre_load)
             else:
                 raise self.no_node_at_path(path)
 
             # cache node
             self.nodes[path] = node
         return self.nodes[path]
 
     @LazyProperty
     def affected_files(self):
         """
         Gets a fast accessible file changes for given commit
         """
         added, modified, deleted = self._changes_cache
         return list(added.union(modified).union(deleted))
 
     @LazyProperty
     def _changes_cache(self):
         added = set()
         modified = set()
         deleted = set()
         _r = self._remote
 
         parents = self.parents
         if not self.parents:
             parents = [base.EmptyCommit()]
         for parent in parents:
             if isinstance(parent, base.EmptyCommit):
                 oid = None
             else:
                 oid = parent.raw_id
             changes = _r.tree_changes(oid, self.raw_id)
             for (oldpath, newpath), (_, _), (_, _) in changes:
                 if newpath and oldpath:
                     modified.add(newpath)
                 elif newpath and not oldpath:
                     added.add(newpath)
                 elif not newpath and oldpath:
                     deleted.add(oldpath)
         return added, modified, deleted
 
     def _get_paths_for_status(self, status):
         """
         Returns sorted list of paths for given ``status``.
 
         :param status: one of: *added*, *modified* or *deleted*
         """
         added, modified, deleted = self._changes_cache
         return sorted({
             'added': list(added),
             'modified': list(modified),
             'deleted': list(deleted)}[status]
         )
 
     @LazyProperty
     def added(self):
         """
         Returns list of added ``FileNode`` objects.
         """
         if not self.parents:
             return list(self._get_file_nodes())
         return AddedFileNodesGenerator(
             [n for n in self._get_paths_for_status('added')], self)
 
     @LazyProperty
     def changed(self):
         """
         Returns list of modified ``FileNode`` objects.
         """
         if not self.parents:
             return []
         return ChangedFileNodesGenerator(
             [n for n in self._get_paths_for_status('modified')], self)
 
     @LazyProperty
     def removed(self):
         """
         Returns list of removed ``FileNode`` objects.
         """
         if not self.parents:
             return []
         return RemovedFileNodesGenerator(
             [n for n in self._get_paths_for_status('deleted')], self)
 
     def _get_submodule_url(self, submodule_path):
         git_modules_path = '.gitmodules'
 
         if self._submodules is None:
             self._submodules = {}
 
             try:
                 submodules_node = self.get_node(git_modules_path)
             except NodeDoesNotExistError:
                 return None
 
             content = submodules_node.content
 
             # ConfigParser fails if there are whitespaces
             content = '\n'.join(l.strip() for l in content.split('\n'))
 
             parser = ConfigParser()
             parser.readfp(StringIO(content))
 
             for section in parser.sections():
                 path = parser.get(section, 'path')
                 url = parser.get(section, 'url')
                 if path and url:
                     self._submodules[path.strip('/')] = url
 
         return self._submodules.get(submodule_path.strip('/'))

rhodecode/lib/vcs/backends/hg/commit.py

@@ -1,362 +1,362 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2014-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 HG commit module
 """
 
 import os
 
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 from rhodecode.lib.datelib import utcdate_fromtimestamp
 from rhodecode.lib.utils import safe_str, safe_unicode
 from rhodecode.lib.vcs import path as vcspath
 from rhodecode.lib.vcs.backends import base
 from rhodecode.lib.vcs.backends.hg.diff import MercurialDiff
 from rhodecode.lib.vcs.exceptions import CommitError
 from rhodecode.lib.vcs.nodes import (
     AddedFileNodesGenerator, ChangedFileNodesGenerator, DirNode, FileNode,
     NodeKind, RemovedFileNodesGenerator, RootNode, SubModuleNode,
     LargeFileNode, LARGEFILE_PREFIX)
 from rhodecode.lib.vcs.utils.paths import get_dirs_for_path
 
 
 class MercurialCommit(base.BaseCommit):
     """
     Represents state of the repository at the single commit.
     """
 
     _filter_pre_load = [
         # git specific property not supported here
         "_commit",
     ]
 
     def __init__(self, repository, raw_id, idx, pre_load=None):
         raw_id = safe_str(raw_id)
 
         self.repository = repository
         self._remote = repository._remote
 
         self.raw_id = raw_id
         self.idx = repository._sanitize_commit_idx(idx)
 
         self._set_bulk_properties(pre_load)
 
         # caches
         self.nodes = {}
 
     def _set_bulk_properties(self, pre_load):
         if not pre_load:
             return
         pre_load = [entry for entry in pre_load
                     if entry not in self._filter_pre_load]
         if not pre_load:
             return
 
         result = self._remote.bulk_request(self.idx, pre_load)
         for attr, value in result.items():
             if attr in ["author", "branch", "message"]:
                 value = safe_unicode(value)
             elif attr == "affected_files":
                 value = map(safe_unicode, value)
             elif attr == "date":
                 value = utcdate_fromtimestamp(*value)
             elif attr in ["children", "parents"]:
                 value = self._make_commits(value)
             self.__dict__[attr] = value
 
     @LazyProperty
     def tags(self):
         tags = [name for name, commit_id in self.repository.tags.iteritems()
                 if commit_id == self.raw_id]
         return tags
 
     @LazyProperty
     def branch(self):
         return safe_unicode(self._remote.ctx_branch(self.idx))
 
     @LazyProperty
     def bookmarks(self):
         bookmarks = [
             name for name, commit_id in self.repository.bookmarks.iteritems()
             if commit_id == self.raw_id]
         return bookmarks
 
     @LazyProperty
     def message(self):
         return safe_unicode(self._remote.ctx_description(self.idx))
 
     @LazyProperty
     def committer(self):
         return safe_unicode(self.author)
 
     @LazyProperty
     def author(self):
         return safe_unicode(self._remote.ctx_user(self.idx))
 
     @LazyProperty
     def date(self):
         return utcdate_fromtimestamp(*self._remote.ctx_date(self.idx))
 
     @LazyProperty
     def status(self):
         """
         Returns modified, added, removed, deleted files for current commit
         """
         return self._remote.ctx_status(self.idx)
 
     @LazyProperty
     def _file_paths(self):
         return self._remote.ctx_list(self.idx)
 
     @LazyProperty
     def _dir_paths(self):
         p = list(set(get_dirs_for_path(*self._file_paths)))
         p.insert(0, '')
         return p
 
     @LazyProperty
     def _paths(self):
         return self._dir_paths + self._file_paths
 
     @LazyProperty
     def id(self):
         if self.last:
             return u'tip'
         return self.short_id
 
     @LazyProperty
     def short_id(self):
         return self.raw_id[:12]
 
     def _make_commits(self, indexes):
         return [self.repository.get_commit(commit_idx=idx)
                 for idx in indexes if idx >= 0]
 
     @LazyProperty
     def parents(self):
         """
         Returns list of parent commits.
         """
         parents = self._remote.ctx_parents(self.idx)
         return self._make_commits(parents)
 
     @LazyProperty
     def children(self):
         """
         Returns list of child commits.
         """
         children = self._remote.ctx_children(self.idx)
         return self._make_commits(children)
 
     def diff(self, ignore_whitespace=True, context=3):
         result = self._remote.ctx_diff(
             self.idx,
             git=True, ignore_whitespace=ignore_whitespace, context=context)
         diff = ''.join(result)
         return MercurialDiff(diff)
 
     def _fix_path(self, path):
         """
         Mercurial keeps filenodes as str so we need to encode from unicode
         to str.
         """
         return safe_str(super(MercurialCommit, self)._fix_path(path))
 
     def _get_kind(self, path):
         path = self._fix_path(path)
         if path in self._file_paths:
             return NodeKind.FILE
         elif path in self._dir_paths:
             return NodeKind.DIR
         else:
             raise CommitError(
                 "Node does not exist at the given path '%s'" % (path, ))
 
     def _get_filectx(self, path):
         path = self._fix_path(path)
         if self._get_kind(path) != NodeKind.FILE:
             raise CommitError(
                 "File does not exist for idx %s at '%s'" % (self.raw_id, path))
         return path
 
     def get_file_mode(self, path):
         """
         Returns stat mode of the file at the given ``path``.
         """
         path = self._get_filectx(path)
         if 'x' in self._remote.fctx_flags(self.idx, path):
             return base.FILEMODE_EXECUTABLE
         else:
             return base.FILEMODE_DEFAULT
 
     def is_link(self, path):
         path = self._get_filectx(path)
         return 'l' in self._remote.fctx_flags(self.idx, path)
 
     def get_file_content(self, path):
         """
         Returns content of the file at given ``path``.
         """
         path = self._get_filectx(path)
         return self._remote.fctx_data(self.idx, path)
 
     def get_file_size(self, path):
         """
         Returns size of the file at given ``path``.
         """
         path = self._get_filectx(path)
         return self._remote.fctx_size(self.idx, path)
 
     def get_file_history(self, path, limit=None, pre_load=None):
         """
         Returns history of file as reversed list of `MercurialCommit` objects
         for which file at given ``path`` has been modified.
         """
         path = self._get_filectx(path)
         hist = self._remote.file_history(self.idx, path, limit)
         return [
             self.repository.get_commit(commit_id=commit_id, pre_load=pre_load)
             for commit_id in hist]
 
     def get_file_annotate(self, path, pre_load=None):
         """
         Returns a generator of four element tuples with
             lineno, commit_id, commit lazy loader and line
         """
         result = self._remote.fctx_annotate(self.idx, path)
 
         for ln_no, commit_id, content in result:
             yield (
                 ln_no, commit_id,
                 lambda: self.repository.get_commit(commit_id=commit_id,
                                                    pre_load=pre_load),
                 content)
 
     def get_nodes(self, path):
         """
         Returns combined ``DirNode`` and ``FileNode`` objects list representing
         state of commit at the given ``path``. If node at the given ``path``
         is not instance of ``DirNode``, CommitError would be raised.
         """
 
         if self._get_kind(path) != NodeKind.DIR:
             raise CommitError(
                 "Directory does not exist for idx %s at '%s'" %
                 (self.idx, path))
         path = self._fix_path(path)
 
         filenodes = [
             FileNode(f, commit=self) for f in self._file_paths
             if os.path.dirname(f) == path]
         # TODO: johbo: Check if this can be done in a more obvious way
         dirs = path == '' and '' or [
             d for d in self._dir_paths
             if d and vcspath.dirname(d) == path]
         dirnodes = [
             DirNode(d, commit=self) for d in dirs
             if os.path.dirname(d) == path]
 
         alias = self.repository.alias
         for k, vals in self._submodules.iteritems():
             loc = vals[0]
             commit = vals[1]
             dirnodes.append(
                 SubModuleNode(k, url=loc, commit=commit, alias=alias))
         nodes = dirnodes + filenodes
         # cache nodes
         for node in nodes:
             self.nodes[node.path] = node
         nodes.sort()
 
         return nodes
 
-    def get_node(self, path):
+    def get_node(self, path, pre_load=None):
         """
         Returns `Node` object from the given `path`. If there is no node at
         the given `path`, `NodeDoesNotExistError` would be raised.
         """
         path = self._fix_path(path)
 
         if path not in self.nodes:
             if path in self._file_paths:
-                node = FileNode(path, commit=self)
+                node = FileNode(path, commit=self, pre_load=pre_load)
             elif path in self._dir_paths:
                 if path == '':
                     node = RootNode(commit=self)
                 else:
                     node = DirNode(path, commit=self)
             else:
                 raise self.no_node_at_path(path)
 
             # cache node
             self.nodes[path] = node
         return self.nodes[path]
 
     def get_largefile_node(self, path):
         path = os.path.join(LARGEFILE_PREFIX, path)
 
         if self._remote.is_large_file(path):
             # content of that file regular FileNode is the hash of largefile
             file_id = self.get_file_content(path).strip()
             if self._remote.in_store(file_id):
                 path = self._remote.store_path(file_id)
                 return LargeFileNode(path, commit=self)
             elif self._remote.in_user_cache(file_id):
                 path = self._remote.store_path(file_id)
                 self._remote.link(file_id, path)
                 return LargeFileNode(path, commit=self)
 
     @LazyProperty
     def _submodules(self):
         """
         Returns a dictionary with submodule information from substate file
         of hg repository.
         """
         return self._remote.ctx_substate(self.idx)
 
     @LazyProperty
     def affected_files(self):
         """
         Gets a fast accessible file changes for given commit
         """
         return self._remote.ctx_files(self.idx)
 
     @property
     def added(self):
         """
         Returns list of added ``FileNode`` objects.
         """
         return AddedFileNodesGenerator([n for n in self.status[1]], self)
 
     @property
     def changed(self):
         """
         Returns list of modified ``FileNode`` objects.
         """
         return ChangedFileNodesGenerator([n for n in self.status[0]], self)
 
     @property
     def removed(self):
         """
         Returns list of removed ``FileNode`` objects.
         """
         return RemovedFileNodesGenerator([n for n in self.status[2]], self)

rhodecode/lib/vcs/backends/svn/commit.py

@@ -1,236 +1,236 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2014-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 SVN commit module
 """
 
 
 import dateutil.parser
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 from rhodecode.lib.utils import safe_str, safe_unicode
 from rhodecode.lib.vcs import nodes, path as vcspath
 from rhodecode.lib.vcs.backends import base
 from rhodecode.lib.vcs.exceptions import CommitError, NodeDoesNotExistError
 
 
 _SVN_PROP_TRUE = '*'
 
 
 class SubversionCommit(base.BaseCommit):
     """
     Subversion specific implementation of commits
 
     .. attribute:: branch
 
        The Subversion backend does not support to assign branches to
        specific commits. This attribute has always the value `None`.
 
     """
 
     def __init__(self, repository, commit_id):
         self.repository = repository
         self.idx = self.repository._get_commit_idx(commit_id)
         self._svn_rev = self.idx + 1
         self._remote = repository._remote
         # TODO: handling of raw_id should be a method on repository itself,
         # which knows how to translate commit index and commit id
         self.raw_id = commit_id
         self.short_id = commit_id
         self.id = 'r%s' % (commit_id, )
 
         # TODO: Implement the following placeholder attributes
         self.nodes = {}
         self.tags = []
 
     @property
     def author(self):
         return safe_unicode(self._properties.get('svn:author'))
 
     @property
     def date(self):
         return _date_from_svn_properties(self._properties)
 
     @property
     def message(self):
         return safe_unicode(self._properties.get('svn:log'))
 
     @LazyProperty
     def _properties(self):
         return self._remote.revision_properties(self._svn_rev)
 
     @LazyProperty
     def parents(self):
         parent_idx = self.idx - 1
         if parent_idx >= 0:
             parent = self.repository.get_commit(commit_idx=parent_idx)
             return [parent]
         return []
 
     @LazyProperty
     def children(self):
         child_idx = self.idx + 1
         if child_idx < len(self.repository.commit_ids):
             child = self.repository.get_commit(commit_idx=child_idx)
             return [child]
         return []
 
     def get_file_mode(self, path):
         # Note: Subversion flags files which are executable with a special
         # property `svn:executable` which is set to the value ``"*"``.
         if self._get_file_property(path, 'svn:executable') == _SVN_PROP_TRUE:
             return base.FILEMODE_EXECUTABLE
         else:
             return base.FILEMODE_DEFAULT
 
     def is_link(self, path):
         # Note: Subversion has a flag for special files, the content of the
         # file contains the type of that file.
         if self._get_file_property(path, 'svn:special') == _SVN_PROP_TRUE:
             return self.get_file_content(path).startswith('link')
         return False
 
     def _get_file_property(self, path, name):
         file_properties = self._remote.node_properties(
             safe_str(path), self._svn_rev)
         return file_properties.get(name)
 
     def get_file_content(self, path):
         path = self._fix_path(path)
         return self._remote.get_file_content(safe_str(path), self._svn_rev)
 
     def get_file_size(self, path):
         path = self._fix_path(path)
         return self._remote.get_file_size(safe_str(path), self._svn_rev)
 
     def get_file_history(self, path, limit=None, pre_load=None):
         path = safe_str(self._fix_path(path))
         history = self._remote.node_history(path, self._svn_rev, limit)
         return [
             self.repository.get_commit(commit_id=str(svn_rev))
             for svn_rev in history]
 
     def get_file_annotate(self, path, pre_load=None):
         result = self._remote.file_annotate(safe_str(path), self._svn_rev)
 
         for zero_based_line_no, svn_rev, content in result:
             commit_id = str(svn_rev)
             line_no = zero_based_line_no + 1
             yield (
                 line_no,
                 commit_id,
                 lambda: self.repository.get_commit(commit_id=commit_id),
                 content)
 
-    def get_node(self, path):
+    def get_node(self, path, pre_load=None):
         path = self._fix_path(path)
         if path not in self.nodes:
 
             if path == '':
                 node = nodes.RootNode(commit=self)
             else:
                 node_type = self._remote.get_node_type(
                     safe_str(path), self._svn_rev)
                 if node_type == 'dir':
                     node = nodes.DirNode(path, commit=self)
                 elif node_type == 'file':
-                    node = nodes.FileNode(path, commit=self)
+                    node = nodes.FileNode(path, commit=self, pre_load=pre_load)
                 else:
                     raise NodeDoesNotExistError(self.no_node_at_path(path))
 
             self.nodes[path] = node
         return self.nodes[path]
 
     def get_nodes(self, path):
         if self._get_kind(path) != nodes.NodeKind.DIR:
             raise CommitError(
                 "Directory does not exist for commit %s at "
                 " '%s'" % (self.raw_id, path))
         path = self._fix_path(path)
 
         path_nodes = []
         for name, kind in self._remote.get_nodes(
                 safe_str(path), revision=self._svn_rev):
             node_path = vcspath.join(path, name)
             if kind == 'dir':
                 node = nodes.DirNode(node_path, commit=self)
             elif kind == 'file':
                 node = nodes.FileNode(node_path, commit=self)
             else:
                 raise ValueError("Node kind %s not supported." % (kind, ))
             self.nodes[node_path] = node
             path_nodes.append(node)
 
         return path_nodes
 
     def _get_kind(self, path):
         path = self._fix_path(path)
         kind = self._remote.get_node_type(path, self._svn_rev)
         if kind == 'file':
             return nodes.NodeKind.FILE
         elif kind == 'dir':
             return nodes.NodeKind.DIR
         else:
             raise CommitError(
                 "Node does not exist at the given path '%s'" % (path, ))
 
     @LazyProperty
     def _changes_cache(self):
         return self._remote.revision_changes(self._svn_rev)
 
     @LazyProperty
     def affected_files(self):
         changed_files = set()
         for files in self._changes_cache.itervalues():
             changed_files.update(files)
         return list(changed_files)
 
     @LazyProperty
     def id(self):
         return self.raw_id
 
     @property
     def added(self):
         return nodes.AddedFileNodesGenerator(
             self._changes_cache['added'], self)
 
     @property
     def changed(self):
         return nodes.ChangedFileNodesGenerator(
             self._changes_cache['changed'], self)
 
     @property
     def removed(self):
         return nodes.RemovedFileNodesGenerator(
             self._changes_cache['removed'], self)
 
 
 def _date_from_svn_properties(properties):
     """
     Parses the date out of given svn properties.
 
     :return: :class:`datetime.datetime` instance. The object is naive.
     """
 
     aware_date = dateutil.parser.parse(properties.get('svn:date'))
     # final_date = aware_date.astimezone(dateutil.tz.tzlocal())
     final_date = aware_date
     return final_date.replace(tzinfo=None)

rhodecode/lib/vcs/nodes.py

@@ -1,756 +1,773 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2014-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Module holding everything related to vcs nodes, with vcs2 architecture.
 """
 
 
 import stat
 
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 from rhodecode.config.conf import LANGUAGES_EXTENSIONS_MAP
 from rhodecode.lib.utils import safe_unicode, safe_str
 from rhodecode.lib.utils2 import md5
 from rhodecode.lib.vcs import path as vcspath
 from rhodecode.lib.vcs.backends.base import EmptyCommit, FILEMODE_DEFAULT
 from rhodecode.lib.vcs.conf.mtypes import get_mimetypes_db
 from rhodecode.lib.vcs.exceptions import NodeError, RemovedFileNodeError
 
 LARGEFILE_PREFIX = '.hglf'
 
 
 class NodeKind:
     SUBMODULE = -1
     DIR = 1
     FILE = 2
     LARGEFILE = 3
 
 
 class NodeState:
     ADDED = u'added'
     CHANGED = u'changed'
     NOT_CHANGED = u'not changed'
     REMOVED = u'removed'
 
 
 class NodeGeneratorBase(object):
     """
     Base class for removed added and changed filenodes, it's a lazy generator
     class that will create filenodes only on iteration or call
 
     The len method doesn't need to create filenodes at all
     """
 
     def __init__(self, current_paths, cs):
         self.cs = cs
         self.current_paths = current_paths
 
     def __call__(self):
         return [n for n in self]
 
     def __getslice__(self, i, j):
         for p in self.current_paths[i:j]:
             yield self.cs.get_node(p)
 
     def __len__(self):
         return len(self.current_paths)
 
     def __iter__(self):
         for p in self.current_paths:
             yield self.cs.get_node(p)
 
 
 class AddedFileNodesGenerator(NodeGeneratorBase):
     """
     Class holding added files for current commit
     """
 
 
 class ChangedFileNodesGenerator(NodeGeneratorBase):
     """
     Class holding changed files for current commit
     """
 
 
 class RemovedFileNodesGenerator(NodeGeneratorBase):
     """
     Class holding removed files for current commit
     """
     def __iter__(self):
         for p in self.current_paths:
             yield RemovedFileNode(path=p)
 
     def __getslice__(self, i, j):
         for p in self.current_paths[i:j]:
             yield RemovedFileNode(path=p)
 
 
 class Node(object):
     """
     Simplest class representing file or directory on repository.  SCM backends
     should use ``FileNode`` and ``DirNode`` subclasses rather than ``Node``
     directly.
 
     Node's ``path`` cannot start with slash as we operate on *relative* paths
     only. Moreover, every single node is identified by the ``path`` attribute,
     so it cannot end with slash, too. Otherwise, path could lead to mistakes.
     """
 
     commit = None
 
     def __init__(self, path, kind):
         self._validate_path(path)  # can throw exception if path is invalid
         self.path = safe_str(path.rstrip('/'))  # we store paths as str
         if path == '' and kind != NodeKind.DIR:
             raise NodeError("Only DirNode and its subclasses may be "
                             "initialized with empty path")
         self.kind = kind
 
         if self.is_root() and not self.is_dir():
             raise NodeError("Root node cannot be FILE kind")
 
     def _validate_path(self, path):
         if path.startswith('/'):
             raise NodeError(
                 "Cannot initialize Node objects with slash at "
                 "the beginning as only relative paths are supported. "
                 "Got %s" % (path,))
 
     @LazyProperty
     def parent(self):
         parent_path = self.get_parent_path()
         if parent_path:
             if self.commit:
                 return self.commit.get_node(parent_path)
             return DirNode(parent_path)
         return None
 
     @LazyProperty
     def unicode_path(self):
         return safe_unicode(self.path)
 
     @LazyProperty
     def dir_path(self):
         """
         Returns name of the directory from full path of this vcs node. Empty
         string is returned if there's no directory in the path
         """
         _parts = self.path.rstrip('/').rsplit('/', 1)
         if len(_parts) == 2:
             return safe_unicode(_parts[0])
         return u''
 
     @LazyProperty
     def name(self):
         """
         Returns name of the node so if its path
         then only last part is returned.
         """
         return safe_unicode(self.path.rstrip('/').split('/')[-1])
 
     @property
     def kind(self):
         return self._kind
 
     @kind.setter
     def kind(self, kind):
         if hasattr(self, '_kind'):
             raise NodeError("Cannot change node's kind")
         else:
             self._kind = kind
             # Post setter check (path's trailing slash)
             if self.path.endswith('/'):
                 raise NodeError("Node's path cannot end with slash")
 
     def __cmp__(self, other):
         """
         Comparator using name of the node, needed for quick list sorting.
         """
         kind_cmp = cmp(self.kind, other.kind)
         if kind_cmp:
             return kind_cmp
         return cmp(self.name, other.name)
 
     def __eq__(self, other):
         for attr in ['name', 'path', 'kind']:
             if getattr(self, attr) != getattr(other, attr):
                 return False
         if self.is_file():
             if self.content != other.content:
                 return False
         else:
             # For DirNode's check without entering each dir
             self_nodes_paths = list(sorted(n.path for n in self.nodes))
             other_nodes_paths = list(sorted(n.path for n in self.nodes))
             if self_nodes_paths != other_nodes_paths:
                 return False
         return True
 
     def __ne__(self, other):
         return not self.__eq__(other)
 
     def __repr__(self):
         return '<%s %r>' % (self.__class__.__name__, self.path)
 
     def __str__(self):
         return self.__repr__()
 
     def __unicode__(self):
         return self.name
 
     def get_parent_path(self):
         """
         Returns node's parent path or empty string if node is root.
         """
         if self.is_root():
             return ''
         return vcspath.dirname(self.path.rstrip('/')) + '/'
 
     def is_file(self):
         """
         Returns ``True`` if node's kind is ``NodeKind.FILE``, ``False``
         otherwise.
         """
         return self.kind == NodeKind.FILE
 
     def is_dir(self):
         """
         Returns ``True`` if node's kind is ``NodeKind.DIR``, ``False``
         otherwise.
         """
         return self.kind == NodeKind.DIR
 
     def is_root(self):
         """
         Returns ``True`` if node is a root node and ``False`` otherwise.
         """
         return self.kind == NodeKind.DIR and self.path == ''
 
     def is_submodule(self):
         """
         Returns ``True`` if node's kind is ``NodeKind.SUBMODULE``, ``False``
         otherwise.
         """
         return self.kind == NodeKind.SUBMODULE
 
     def is_largefile(self):
         """
         Returns ``True`` if node's kind is ``NodeKind.LARGEFILE``, ``False``
         otherwise
         """
         return self.kind == NodeKind.LARGEFILE
 
     def is_link(self):
         if self.commit:
             return self.commit.is_link(self.path)
         return False
 
     @LazyProperty
     def added(self):
         return self.state is NodeState.ADDED
 
     @LazyProperty
     def changed(self):
         return self.state is NodeState.CHANGED
 
     @LazyProperty
     def not_changed(self):
         return self.state is NodeState.NOT_CHANGED
 
     @LazyProperty
     def removed(self):
         return self.state is NodeState.REMOVED
 
 
 class FileNode(Node):
     """
     Class representing file nodes.
 
     :attribute: path: path to the node, relative to repository's root
     :attribute: content: if given arbitrary sets content of the file
     :attribute: commit: if given, first time content is accessed, callback
     :attribute: mode: stat mode for a node. Default is `FILEMODE_DEFAULT`.
     """
+    _filter_pre_load = []
 
-    def __init__(self, path, content=None, commit=None, mode=None):
+    def __init__(self, path, content=None, commit=None, mode=None, pre_load=None):
         """
         Only one of ``content`` and ``commit`` may be given. Passing both
         would raise ``NodeError`` exception.
 
         :param path: relative path to the node
         :param content: content may be passed to constructor
         :param commit: if given, will use it to lazily fetch content
         :param mode: ST_MODE (i.e. 0100644)
         """
         if content and commit:
             raise NodeError("Cannot use both content and commit")
         super(FileNode, self).__init__(path, kind=NodeKind.FILE)
         self.commit = commit
         self._content = content
         self._mode = mode or FILEMODE_DEFAULT
 
+        self._set_bulk_properties(pre_load)
+
+    def _set_bulk_properties(self, pre_load):
+        if not pre_load:
+            return
+        pre_load = [entry for entry in pre_load
+                    if entry not in self._filter_pre_load]
+        if not pre_load:
+            return
+
+        for attr_name in pre_load:
+            result = getattr(self, attr_name)
+            if callable(result):
+                result = result()
+            self.__dict__[attr_name] = result
+
     @LazyProperty
     def mode(self):
         """
         Returns lazily mode of the FileNode. If `commit` is not set, would
         use value given at initialization or `FILEMODE_DEFAULT` (default).
         """
         if self.commit:
             mode = self.commit.get_file_mode(self.path)
         else:
             mode = self._mode
         return mode
 
     @LazyProperty
     def raw_bytes(self):
         """
         Returns lazily the raw bytes of the FileNode.
         """
         if self.commit:
             if self._content is None:
                 self._content = self.commit.get_file_content(self.path)
             content = self._content
         else:
             content = self._content
         return content
 
     @LazyProperty
     def md5(self):
         """
         Returns md5 of the file node.
         """
         return md5(self.raw_bytes)
 
     @LazyProperty
     def content(self):
         """
         Returns lazily content of the FileNode. If possible, would try to
         decode content from UTF-8.
         """
         content = self.raw_bytes
 
         if self.is_binary:
             return content
         return safe_unicode(content)
 
     @LazyProperty
     def size(self):
         if self.commit:
             return self.commit.get_file_size(self.path)
         raise NodeError(
             "Cannot retrieve size of the file without related "
             "commit attribute")
 
     @LazyProperty
     def message(self):
         if self.commit:
             return self.last_commit.message
         raise NodeError(
             "Cannot retrieve message of the file without related "
             "commit attribute")
 
     @LazyProperty
     def last_commit(self):
         if self.commit:
             pre_load = ["author", "date", "message"]
             return self.commit.get_file_commit(self.path, pre_load=pre_load)
         raise NodeError(
             "Cannot retrieve last commit of the file without "
             "related commit attribute")
 
     def get_mimetype(self):
         """
         Mimetype is calculated based on the file's content. If ``_mimetype``
         attribute is available, it will be returned (backends which store
         mimetypes or can easily recognize them, should set this private
         attribute to indicate that type should *NOT* be calculated).
         """
 
         if hasattr(self, '_mimetype'):
             if (isinstance(self._mimetype, (tuple, list,)) and
                     len(self._mimetype) == 2):
                 return self._mimetype
             else:
                 raise NodeError('given _mimetype attribute must be an 2 '
                                 'element list or tuple')
 
         db = get_mimetypes_db()
         mtype, encoding = db.guess_type(self.name)
 
         if mtype is None:
             if self.is_binary:
                 mtype = 'application/octet-stream'
                 encoding = None
             else:
                 mtype = 'text/plain'
                 encoding = None
 
                 # try with pygments
                 try:
                     from pygments.lexers import get_lexer_for_filename
                     mt = get_lexer_for_filename(self.name).mimetypes
                 except Exception:
                     mt = None
 
                 if mt:
                     mtype = mt[0]
 
         return mtype, encoding
 
     @LazyProperty
     def mimetype(self):
         """
         Wrapper around full mimetype info. It returns only type of fetched
         mimetype without the encoding part. use get_mimetype function to fetch
         full set of (type,encoding)
         """
         return self.get_mimetype()[0]
 
     @LazyProperty
     def mimetype_main(self):
         return self.mimetype.split('/')[0]
 
     @LazyProperty
     def lexer(self):
         """
         Returns pygment's lexer class. Would try to guess lexer taking file's
         content, name and mimetype.
         """
         from pygments import lexers
 
         lexer = None
         try:
             lexer = lexers.guess_lexer_for_filename(
                 self.name, self.content, stripnl=False)
         except lexers.ClassNotFound:
             lexer = None
 
         # try our EXTENSION_MAP
         if not lexer:
             try:
                 lexer_class = LANGUAGES_EXTENSIONS_MAP.get(self.extension)
                 if lexer_class:
                     lexer = lexers.get_lexer_by_name(lexer_class[0])
             except lexers.ClassNotFound:
                 lexer = None
 
         if not lexer:
             lexer = lexers.TextLexer(stripnl=False)
 
         return lexer
 
     @LazyProperty
     def lexer_alias(self):
         """
         Returns first alias of the lexer guessed for this file.
         """
         return self.lexer.aliases[0]
 
     @LazyProperty
     def history(self):
         """
         Returns a list of commit for this file in which the file was changed
         """
         if self.commit is None:
             raise NodeError('Unable to get commit for this FileNode')
         return self.commit.get_file_history(self.path)
 
     @LazyProperty
     def annotate(self):
         """
         Returns a list of three element tuples with lineno, commit and line
         """
         if self.commit is None:
             raise NodeError('Unable to get commit for this FileNode')
         pre_load = ["author", "date", "message"]
         return self.commit.get_file_annotate(self.path, pre_load=pre_load)
 
     @LazyProperty
     def state(self):
         if not self.commit:
             raise NodeError(
                 "Cannot check state of the node if it's not "
                 "linked with commit")
         elif self.path in (node.path for node in self.commit.added):
             return NodeState.ADDED
         elif self.path in (node.path for node in self.commit.changed):
             return NodeState.CHANGED
         else:
             return NodeState.NOT_CHANGED
 
     @LazyProperty
     def is_binary(self):
         """
         Returns True if file has binary content.
         """
         _bin = self.raw_bytes and '\0' in self.raw_bytes
         return _bin
 
     @LazyProperty
     def extension(self):
         """Returns filenode extension"""
         return self.name.split('.')[-1]
 
     @property
     def is_executable(self):
         """
         Returns ``True`` if file has executable flag turned on.
         """
         return bool(self.mode & stat.S_IXUSR)
 
     def get_largefile_node(self):
         """
         Try to return a Mercurial FileNode from this node. It does internal
         checks inside largefile store, if that file exist there it will
         create special instance of LargeFileNode which can get content from
         LF store.
         """
         if self.commit and self.path.startswith(LARGEFILE_PREFIX):
             largefile_path = self.path.split(LARGEFILE_PREFIX)[-1].lstrip('/')
             return self.commit.get_largefile_node(largefile_path)
 
     def lines(self, count_empty=False):
         all_lines, empty_lines = 0, 0
 
         if not self.is_binary:
             content = self.content
             if count_empty:
                 all_lines = 0
                 empty_lines = 0
                 for line in content.splitlines(True):
                     if line == '\n':
                         empty_lines += 1
                     all_lines += 1
 
                 return all_lines, all_lines - empty_lines
             else:
                 # fast method
                 empty_lines = all_lines = content.count('\n')
                 if all_lines == 0 and content:
                     # one-line without a newline
                     empty_lines = all_lines = 1
 
         return all_lines, empty_lines
 
     def __repr__(self):
         return '<%s %r @ %s>' % (self.__class__.__name__, self.path,
                                  getattr(self.commit, 'short_id', ''))
 
 
 class RemovedFileNode(FileNode):
     """
     Dummy FileNode class - trying to access any public attribute except path,
     name, kind or state (or methods/attributes checking those two) would raise
     RemovedFileNodeError.
     """
     ALLOWED_ATTRIBUTES = [
         'name', 'path', 'state', 'is_root', 'is_file', 'is_dir', 'kind',
         'added', 'changed', 'not_changed', 'removed'
     ]
 
     def __init__(self, path):
         """
         :param path: relative path to the node
         """
         super(RemovedFileNode, self).__init__(path=path)
 
     def __getattribute__(self, attr):
         if attr.startswith('_') or attr in RemovedFileNode.ALLOWED_ATTRIBUTES:
             return super(RemovedFileNode, self).__getattribute__(attr)
         raise RemovedFileNodeError(
             "Cannot access attribute %s on RemovedFileNode" % attr)
 
     @LazyProperty
     def state(self):
         return NodeState.REMOVED
 
 
 class DirNode(Node):
     """
     DirNode stores list of files and directories within this node.
     Nodes may be used standalone but within repository context they
     lazily fetch data within same repositorty's commit.
     """
 
     def __init__(self, path, nodes=(), commit=None):
         """
         Only one of ``nodes`` and ``commit`` may be given. Passing both
         would raise ``NodeError`` exception.
 
         :param path: relative path to the node
         :param nodes: content may be passed to constructor
         :param commit: if given, will use it to lazily fetch content
         """
         if nodes and commit:
             raise NodeError("Cannot use both nodes and commit")
         super(DirNode, self).__init__(path, NodeKind.DIR)
         self.commit = commit
         self._nodes = nodes
 
     @LazyProperty
     def content(self):
         raise NodeError(
             "%s represents a dir and has no `content` attribute" % self)
 
     @LazyProperty
     def nodes(self):
         if self.commit:
             nodes = self.commit.get_nodes(self.path)
         else:
             nodes = self._nodes
         self._nodes_dict = dict((node.path, node) for node in nodes)
         return sorted(nodes)
 
     @LazyProperty
     def files(self):
         return sorted((node for node in self.nodes if node.is_file()))
 
     @LazyProperty
     def dirs(self):
         return sorted((node for node in self.nodes if node.is_dir()))
 
     def __iter__(self):
         for node in self.nodes:
             yield node
 
     def get_node(self, path):
         """
         Returns node from within this particular ``DirNode``, so it is now
         allowed to fetch, i.e. node located at 'docs/api/index.rst' from node
         'docs'. In order to access deeper nodes one must fetch nodes between
         them first - this would work::
 
            docs = root.get_node('docs')
            docs.get_node('api').get_node('index.rst')
 
         :param: path - relative to the current node
 
         .. note::
            To access lazily (as in example above) node have to be initialized
            with related commit object - without it node is out of
            context and may know nothing about anything else than nearest
            (located at same level) nodes.
         """
         try:
             path = path.rstrip('/')
             if path == '':
                 raise NodeError("Cannot retrieve node without path")
             self.nodes  # access nodes first in order to set _nodes_dict
             paths = path.split('/')
             if len(paths) == 1:
                 if not self.is_root():
                     path = '/'.join((self.path, paths[0]))
                 else:
                     path = paths[0]
                 return self._nodes_dict[path]
             elif len(paths) > 1:
                 if self.commit is None:
                     raise NodeError(
                         "Cannot access deeper nodes without commit")
                 else:
                     path1, path2 = paths[0], '/'.join(paths[1:])
                     return self.get_node(path1).get_node(path2)
             else:
                 raise KeyError
         except KeyError:
             raise NodeError("Node does not exist at %s" % path)
 
     @LazyProperty
     def state(self):
         raise NodeError("Cannot access state of DirNode")
 
     @LazyProperty
     def size(self):
         size = 0
         for root, dirs, files in self.commit.walk(self.path):
             for f in files:
                 size += f.size
 
         return size
 
     def __repr__(self):
         return '<%s %r @ %s>' % (self.__class__.__name__, self.path,
                                  getattr(self.commit, 'short_id', ''))
 
 
 class RootNode(DirNode):
     """
     DirNode being the root node of the repository.
     """
 
     def __init__(self, nodes=(), commit=None):
         super(RootNode, self).__init__(path='', nodes=nodes, commit=commit)
 
     def __repr__(self):
         return '<%s>' % self.__class__.__name__
 
 
 class SubModuleNode(Node):
     """
     represents a SubModule of Git or SubRepo of Mercurial
     """
     is_binary = False
     size = 0
 
     def __init__(self, name, url=None, commit=None, alias=None):
         self.path = name
         self.kind = NodeKind.SUBMODULE
         self.alias = alias
 
         # we have to use EmptyCommit here since this can point to svn/git/hg
         # submodules we cannot get from repository
         self.commit = EmptyCommit(str(commit), alias=alias)
         self.url = url or self._extract_submodule_url()
 
     def __repr__(self):
         return '<%s %r @ %s>' % (self.__class__.__name__, self.path,
                                  getattr(self.commit, 'short_id', ''))
 
     def _extract_submodule_url(self):
         # TODO: find a way to parse gits submodule file and extract the
         # linking URL
         return self.path
 
     @LazyProperty
     def name(self):
         """
         Returns name of the node so if its path
         then only last part is returned.
         """
         org = safe_unicode(self.path.rstrip('/').split('/')[-1])
         return u'%s @ %s' % (org, self.commit.short_id)
 
 
 class LargeFileNode(FileNode):
 
     def _validate_path(self, path):
         """
         we override check since the LargeFileNode path is system absolute
         """
 
     def raw_bytes(self):
         if self.commit:
             with open(self.path, 'rb') as f:
                 content = f.read()
         else:
             content = self._content
         return content
  No newline at end of file