vcs-support: bulk of changes for python3

rhodecode/lib/vcs/client_http.py

@@ -1,418 +1,432 b''
 
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Client for the VCSServer implemented based on HTTP.
 """
 
 import copy
 import logging
 import threading
 import time
-import urllib.request, urllib.error, urllib.parse
+import urllib.request
+import urllib.error
+import urllib.parse
 import urllib.parse
 import uuid
 import traceback
 
 import pycurl
 import msgpack
 import requests
 from requests.packages.urllib3.util.retry import Retry
 
 import rhodecode
 from rhodecode.lib import rc_cache
 from rhodecode.lib.rc_cache.utils import compute_key_from_params
 from rhodecode.lib.system_info import get_cert_path
 from rhodecode.lib.vcs import exceptions, CurlSession
 from rhodecode.lib.utils2 import str2bool
 
 log = logging.getLogger(__name__)
 
 
 # TODO: mikhail: Keep it in sync with vcsserver's
 # HTTPApplication.ALLOWED_EXCEPTIONS
 EXCEPTIONS_MAP = {
     'KeyError': KeyError,
     'URLError': urllib.error.URLError,
 }
 
 
 def _remote_call(url, payload, exceptions_map, session, retries=3):
 
     for attempt in range(retries):
         try:
             response = session.post(url, data=msgpack.packb(payload))
+            break
         except pycurl.error as e:
             error_code, error_message = e.args
             if error_code == pycurl.E_RECV_ERROR:
                 log.warning(f'Received a "Connection reset by peer" error. '
                             f'Retrying... ({attempt + 1}/{retries})')
                 continue  # Retry if connection reset error.
             msg = '{}. \npycurl traceback: {}'.format(e, traceback.format_exc())
             raise exceptions.HttpVCSCommunicationError(msg)
         except Exception as e:
             message = getattr(e, 'message', '')
             if 'Failed to connect' in message:
                 # gevent doesn't return proper pycurl errors
                 raise exceptions.HttpVCSCommunicationError(e)
             else:
                 raise
 
     if response.status_code >= 400:
-        log.error('Call to %s returned non 200 HTTP code: %s',
-                  url, response.status_code)
+        content_type = response.content_type
+        log.error('Call to %s returned non 200 HTTP code: %s [%s]',
+                  url, response.status_code, content_type)
         raise exceptions.HttpVCSCommunicationError(repr(response.content))
 
     try:
-        response = msgpack.unpackb(response.content, raw=False)
+        response = msgpack.unpackb(response.content)
     except Exception:
         log.exception('Failed to decode response from msgpack')
         raise
 
     error = response.get('error')
     if error:
         type_ = error.get('type', 'Exception')
         exc = exceptions_map.get(type_, Exception)
         exc = exc(error.get('message'))
         try:
             exc._vcs_kind = error['_vcs_kind']
         except KeyError:
             pass
 
         try:
             exc._vcs_server_traceback = error['traceback']
             exc._vcs_server_org_exc_name = error['org_exc']
             exc._vcs_server_org_exc_tb = error['org_exc_tb']
         except KeyError:
             pass
 
-        raise exc
+        exc.add_note(attach_exc_details(error))
+        raise exc  # raising the org exception from vcsserver
     return response.get('result')
 
 
+def attach_exc_details(error):
+    note = '-- EXC NOTE -- :\n'
+    note += f'vcs_kind: {error.get("_vcs_kind")}\n'
+    note += f'org_exc: {error.get("_vcs_kind")}\n'
+    note += f'tb: {error.get("traceback")}\n'
+    note += '-- END EXC NOTE --'
+    return note
+
+
 def _streaming_remote_call(url, payload, exceptions_map, session, chunk_size):
     try:
         headers = {
             'X-RC-Method': payload.get('method'),
             'X-RC-Repo-Name': payload.get('_repo_name')
         }
         response = session.post(url, data=msgpack.packb(payload), headers=headers)
     except pycurl.error as e:
         error_code, error_message = e.args
         msg = '{}. \npycurl traceback: {}'.format(e, traceback.format_exc())
         raise exceptions.HttpVCSCommunicationError(msg)
     except Exception as e:
         message = getattr(e, 'message', '')
         if 'Failed to connect' in message:
             # gevent doesn't return proper pycurl errors
             raise exceptions.HttpVCSCommunicationError(e)
         else:
             raise
 
     if response.status_code >= 400:
         log.error('Call to %s returned non 200 HTTP code: %s',
                   url, response.status_code)
         raise exceptions.HttpVCSCommunicationError(repr(response.content))
 
     return response.iter_content(chunk_size=chunk_size)
 
 
 class ServiceConnection(object):
     def __init__(self, server_and_port, backend_endpoint, session_factory):
         self.url = urllib.parse.urljoin('http://%s' % server_and_port, backend_endpoint)
         self._session_factory = session_factory
 
     def __getattr__(self, name):
         def f(*args, **kwargs):
             return self._call(name, *args, **kwargs)
         return f
 
     @exceptions.map_vcs_exceptions
     def _call(self, name, *args, **kwargs):
         payload = {
             'id': str(uuid.uuid4()),
             'method': name,
             'params': {'args': args, 'kwargs': kwargs}
         }
         return _remote_call(
             self.url, payload, EXCEPTIONS_MAP, self._session_factory())
 
 
 class RemoteVCSMaker(object):
 
     def __init__(self, server_and_port, backend_endpoint, backend_type, session_factory):
         self.url = urllib.parse.urljoin('http://%s' % server_and_port, backend_endpoint)
         self.stream_url = urllib.parse.urljoin('http://%s' % server_and_port, backend_endpoint+'/stream')
 
         self._session_factory = session_factory
         self.backend_type = backend_type
 
     @classmethod
     def init_cache_region(cls, repo_id):
-        cache_namespace_uid = 'cache_repo.{}'.format(repo_id)
+        cache_namespace_uid = 'repo.{}'.format(repo_id)
         region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
         return region, cache_namespace_uid
 
     def __call__(self, path, repo_id, config, with_wire=None):
         log.debug('%s RepoMaker call on %s', self.backend_type.upper(), path)
         return RemoteRepo(path, repo_id, config, self, with_wire=with_wire)
 
     def __getattr__(self, name):
         def remote_attr(*args, **kwargs):
             return self._call(name, *args, **kwargs)
         return remote_attr
 
     @exceptions.map_vcs_exceptions
     def _call(self, func_name, *args, **kwargs):
         payload = {
             'id': str(uuid.uuid4()),
             'method': func_name,
             'backend': self.backend_type,
             'params': {'args': args, 'kwargs': kwargs}
         }
         url = self.url
         return _remote_call(url, payload, EXCEPTIONS_MAP, self._session_factory())
 
 
 class RemoteRepo(object):
     CHUNK_SIZE = 16384
 
     def __init__(self, path, repo_id, config, remote_maker, with_wire=None):
         self.url = remote_maker.url
         self.stream_url = remote_maker.stream_url
         self._session = remote_maker._session_factory()
 
         cache_repo_id = self._repo_id_sanitizer(repo_id)
         _repo_name = self._get_repo_name(config, path)
         self._cache_region, self._cache_namespace = \
             remote_maker.init_cache_region(cache_repo_id)
 
         with_wire = with_wire or {}
 
         repo_state_uid = with_wire.get('repo_state_uid') or 'state'
 
         self._wire = {
             "_repo_name": _repo_name,
             "path": path,  # repo path
             "repo_id": repo_id,
             "cache_repo_id": cache_repo_id,
             "config": config,
             "repo_state_uid": repo_state_uid,
             "context": self._create_vcs_cache_context(path, repo_state_uid)
         }
 
         if with_wire:
             self._wire.update(with_wire)
 
         # NOTE(johbo): Trading complexity for performance. Avoiding the call to
         # log.debug brings a few percent gain even if is is not active.
         if log.isEnabledFor(logging.DEBUG):
             self._call_with_logging = True
 
         self.cert_dir = get_cert_path(rhodecode.CONFIG.get('__file__'))
 
     def _get_repo_name(self, config, path):
         repo_store = config.get('paths', '/')
         return path.split(repo_store)[-1].lstrip('/')
 
     def _repo_id_sanitizer(self, repo_id):
         pathless = repo_id.replace('/', '__').replace('-', '_')
         return ''.join(char if ord(char) < 128 else '_{}_'.format(ord(char)) for char in pathless)
 
     def __getattr__(self, name):
 
         if name.startswith('stream:'):
             def repo_remote_attr(*args, **kwargs):
                 return self._call_stream(name, *args, **kwargs)
         else:
             def repo_remote_attr(*args, **kwargs):
                 return self._call(name, *args, **kwargs)
 
         return repo_remote_attr
 
     def _base_call(self, name, *args, **kwargs):
         # TODO: oliver: This is currently necessary pre-call since the
         # config object is being changed for hooking scenarios
         wire = copy.deepcopy(self._wire)
         wire["config"] = wire["config"].serialize()
         wire["config"].append(('vcs', 'ssl_dir', self.cert_dir))
 
         payload = {
             'id': str(uuid.uuid4()),
             'method': name,
             "_repo_name":  wire['_repo_name'],
             'params': {'wire': wire, 'args': args, 'kwargs': kwargs}
         }
 
         context_uid = wire.get('context')
         return context_uid, payload
 
     def get_local_cache(self, name, args):
         cache_on = False
         cache_key = ''
-        local_cache_on = str2bool(rhodecode.CONFIG.get('vcs.methods.cache'))
+        local_cache_on = rhodecode.ConfigGet().get_bool('vcs.methods.cache')
 
         cache_methods = [
             'branches', 'tags', 'bookmarks',
             'is_large_file', 'is_binary',
             'fctx_size', 'stream:fctx_node_data', 'blob_raw_length',
             'node_history',
             'revision', 'tree_items',
             'ctx_list', 'ctx_branch', 'ctx_description',
             'bulk_request',
             'assert_correct_path'
         ]
 
         if local_cache_on and name in cache_methods:
             cache_on = True
             repo_state_uid = self._wire['repo_state_uid']
             call_args = [a for a in args]
             cache_key = compute_key_from_params(repo_state_uid, name, *call_args)
 
         return cache_on, cache_key
 
     @exceptions.map_vcs_exceptions
     def _call(self, name, *args, **kwargs):
         context_uid, payload = self._base_call(name, *args, **kwargs)
         url = self.url
 
         start = time.time()
         cache_on, cache_key = self.get_local_cache(name, args)
 
         @self._cache_region.conditional_cache_on_arguments(
             namespace=self._cache_namespace, condition=cache_on and cache_key)
         def remote_call(_cache_key):
             if self._call_with_logging:
-                args_repr = f'ARG: {str(args):.256}|KW: {str(kwargs):.256}'
+                args_repr = f'ARG: {str(args):.512}|KW: {str(kwargs):.512}'
                 log.debug('Calling %s@%s with args:%r. wire_context: %s cache_on: %s',
                           url, name, args_repr, context_uid, cache_on)
             return _remote_call(url, payload, EXCEPTIONS_MAP, self._session)
 
         result = remote_call(cache_key)
         if self._call_with_logging:
             log.debug('Call %s@%s took: %.4fs. wire_context: %s',
                       url, name, time.time()-start, context_uid)
         return result
 
     @exceptions.map_vcs_exceptions
     def _call_stream(self, name, *args, **kwargs):
         context_uid, payload = self._base_call(name, *args, **kwargs)
         payload['chunk_size'] = self.CHUNK_SIZE
         url = self.stream_url
 
         start = time.time()
         cache_on, cache_key = self.get_local_cache(name, args)
 
         # Cache is a problem because this is a stream
         def streaming_remote_call(_cache_key):
             if self._call_with_logging:
-                args_repr = f'ARG: {str(args):.256}|KW: {str(kwargs):.256}'
+                args_repr = f'ARG: {str(args):.512}|KW: {str(kwargs):.512}'
                 log.debug('Calling %s@%s with args:%r. wire_context: %s cache_on: %s',
                           url, name, args_repr, context_uid, cache_on)
             return _streaming_remote_call(url, payload, EXCEPTIONS_MAP, self._session, self.CHUNK_SIZE)
 
         result = streaming_remote_call(cache_key)
         if self._call_with_logging:
             log.debug('Call %s@%s took: %.4fs. wire_context: %s',
                       url, name, time.time()-start, context_uid)
         return result
 
     def __getitem__(self, key):
         return self.revision(key)
 
     def _create_vcs_cache_context(self, *args):
         """
         Creates a unique string which is passed to the VCSServer on every
         remote call. It is used as cache key in the VCSServer.
         """
         hash_key = '-'.join(map(str, args))
         return str(uuid.uuid5(uuid.NAMESPACE_URL, hash_key))
 
     def invalidate_vcs_cache(self):
         """
         This invalidates the context which is sent to the VCSServer on every
         call to a remote method. It forces the VCSServer to create a fresh
         repository instance on the next call to a remote method.
         """
         self._wire['context'] = str(uuid.uuid4())
 
 
 class VcsHttpProxy(object):
 
     CHUNK_SIZE = 16384
 
     def __init__(self, server_and_port, backend_endpoint):
         retries = Retry(total=5, connect=None, read=None, redirect=None)
 
         adapter = requests.adapters.HTTPAdapter(max_retries=retries)
         self.base_url = urllib.parse.urljoin('http://%s' % server_and_port, backend_endpoint)
         self.session = requests.Session()
         self.session.mount('http://', adapter)
 
     def handle(self, environment, input_data, *args, **kwargs):
         data = {
             'environment': environment,
             'input_data': input_data,
             'args': args,
             'kwargs': kwargs
         }
         result = self.session.post(
             self.base_url, msgpack.packb(data), stream=True)
         return self._get_result(result)
 
     def _deserialize_and_raise(self, error):
         exception = Exception(error['message'])
         try:
             exception._vcs_kind = error['_vcs_kind']
         except KeyError:
             pass
         raise exception
 
     def _iterate(self, result):
         unpacker = msgpack.Unpacker()
         for line in result.iter_content(chunk_size=self.CHUNK_SIZE):
             unpacker.feed(line)
             for chunk in unpacker:
                 yield chunk
 
     def _get_result(self, result):
         iterator = self._iterate(result)
         error = next(iterator)
         if error:
             self._deserialize_and_raise(error)
 
         status = next(iterator)
         headers = next(iterator)
 
         return iterator, status, headers
 
 
 class ThreadlocalSessionFactory(object):
     """
     Creates one CurlSession per thread on demand.
     """
 
     def __init__(self):
         self._thread_local = threading.local()
 
     def __call__(self):
         if not hasattr(self._thread_local, 'curl_session'):
             self._thread_local.curl_session = CurlSession()
         return self._thread_local.curl_session

rhodecode/lib/vcs/conf/settings.py

@@ -1,76 +1,76 b''
 
 
 # Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Internal settings for vcs-lib
 """
 
-# list of default encoding used in safe_unicode/safe_str methods
+# list of default encoding used in safe_str methods
 DEFAULT_ENCODINGS = ['utf8']
 
 
 # Compatibility version when creating SVN repositories. None means newest.
 # Other available options are: pre-1.4-compatible, pre-1.5-compatible,
 # pre-1.6-compatible, pre-1.8-compatible
 SVN_COMPATIBLE_VERSION = None
 
 ALIASES = ['hg', 'git', 'svn']
 
 BACKENDS = {
     'hg': 'rhodecode.lib.vcs.backends.hg.MercurialRepository',
     'git': 'rhodecode.lib.vcs.backends.git.GitRepository',
     'svn': 'rhodecode.lib.vcs.backends.svn.SubversionRepository',
 }
 
 
 ARCHIVE_SPECS = [
     ('tbz2', 'application/x-bzip2', '.tbz2'),
     ('tbz2', 'application/x-bzip2', '.tar.bz2'),
 
     ('tgz', 'application/x-gzip', '.tgz'),
     ('tgz', 'application/x-gzip', '.tar.gz'),
 
     ('zip', 'application/zip', '.zip'),
 ]
 
 HOOKS_PROTOCOL = None
 HOOKS_DIRECT_CALLS = False
 HOOKS_HOST = '127.0.0.1'
 
 
 MERGE_MESSAGE_TMPL = (
     u'Merge pull request !{pr_id} from {source_repo} {source_ref_name}\n\n '
     u'{pr_title}')
 MERGE_DRY_RUN_MESSAGE = 'dry_run_merge_message_from_rhodecode'
 MERGE_DRY_RUN_USER = 'Dry-Run User'
 MERGE_DRY_RUN_EMAIL = '[e-mail]'
 
 
 def available_aliases():
     """
     Mercurial is required for the system to work, so in case vcs.backends does
     not include it, we make sure it will be available internally
     TODO: anderson: refactor vcs.backends so it won't be necessary, VCS server
     should be responsible to dictate available backends.
     """
     aliases = ALIASES[:]
     if 'hg' not in aliases:
         aliases += ['hg']
     return aliases

rhodecode/lib/vcs/exceptions.py

@@ -1,233 +1,234 b''
 
 
 # Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Custom vcs exceptions module.
 """
 import logging
 import functools
-import urllib.request, urllib.error, urllib.parse
+import urllib.error
+import urllib.parse
 import rhodecode
 
 log = logging.getLogger(__name__)
 
 
 class VCSCommunicationError(Exception):
     pass
 
 
 class HttpVCSCommunicationError(VCSCommunicationError):
     pass
 
 
 class VCSError(Exception):
     pass
 
 
 class RepositoryError(VCSError):
     pass
 
 
 class RepositoryRequirementError(RepositoryError):
     pass
 
 
 class UnresolvedFilesInRepo(RepositoryError):
     pass
 
 
 class VCSBackendNotSupportedError(VCSError):
     """
     Exception raised when VCSServer does not support requested backend
     """
 
 
 class EmptyRepositoryError(RepositoryError):
     pass
 
 
 class TagAlreadyExistError(RepositoryError):
     pass
 
 
 class TagDoesNotExistError(RepositoryError):
     pass
 
 
 class BranchAlreadyExistError(RepositoryError):
     pass
 
 
 class BranchDoesNotExistError(RepositoryError):
     pass
 
 
 class CommitError(RepositoryError):
     """
     Exceptions related to an existing commit
     """
 
 
 class CommitDoesNotExistError(CommitError):
     pass
 
 
 class CommittingError(RepositoryError):
     """
     Exceptions happening while creating a new commit
     """
 
 
 class NothingChangedError(CommittingError):
     pass
 
 
 class NodeError(VCSError):
     pass
 
 
 class RemovedFileNodeError(NodeError):
     pass
 
 
 class NodeAlreadyExistsError(CommittingError):
     pass
 
 
 class NodeAlreadyChangedError(CommittingError):
     pass
 
 
 class NodeDoesNotExistError(CommittingError):
     pass
 
 
 class NodeNotChangedError(CommittingError):
     pass
 
 
 class NodeAlreadyAddedError(CommittingError):
     pass
 
 
 class NodeAlreadyRemovedError(CommittingError):
     pass
 
 
 class SubrepoMergeError(RepositoryError):
     """
     This happens if we try to merge a repository which contains subrepos and
     the subrepos cannot be merged. The subrepos are not merged itself but
     their references in the root repo are merged.
     """
 
 
 class ImproperArchiveTypeError(VCSError):
     pass
 
 
 class CommandError(VCSError):
     pass
 
 
 class UnhandledException(VCSError):
     """
     Signals that something unexpected went wrong.
 
     This usually means we have a programming error on the side of the VCSServer
     and should inspect the logfile of the VCSServer to find more details.
     """
 
 
 _EXCEPTION_MAP = {
     'abort': RepositoryError,
     'archive': ImproperArchiveTypeError,
     'error': RepositoryError,
     'lookup': CommitDoesNotExistError,
     'repo_locked': RepositoryError,
     'requirement': RepositoryRequirementError,
     'unhandled': UnhandledException,
     # TODO: johbo: Define our own exception for this and stop abusing
     # urllib's exception class.
     'url_error': urllib.error.URLError,
     'subrepo_merge_error': SubrepoMergeError,
 }
 
 
 def map_vcs_exceptions(func):
     """
     Utility to decorate functions so that plain exceptions are translated.
 
     The translation is based on `exc_map` which maps a `str` indicating
     the error type into an exception class representing this error inside
     of the vcs layer.
     """
 
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
         try:
             return func(*args, **kwargs)
         except Exception as e:
-            from rhodecode.lib.utils2 import str2bool
-            debug = str2bool(rhodecode.CONFIG.get('debug'))
+            debug = rhodecode.ConfigGet().get_bool('debug')
 
             # The error middleware adds information if it finds
             # __traceback_info__ in a frame object. This way the remote
             # traceback information is made available in error reports.
+
             remote_tb = getattr(e, '_vcs_server_traceback', None)
             org_remote_tb = getattr(e, '_vcs_server_org_exc_tb', '')
             __traceback_info__ = None
             if remote_tb:
                 if isinstance(remote_tb, str):
                     remote_tb = [remote_tb]
                 __traceback_info__ = (
                     'Found VCSServer remote traceback information:\n'
                     '{}\n'
                     '+++ BEG SOURCE EXCEPTION +++\n\n'
                     '{}\n'
                     '+++ END SOURCE EXCEPTION +++\n'
                     ''.format('\n'.join(remote_tb), org_remote_tb)
                 )
 
                 # Avoid that remote_tb also appears in the frame
                 del remote_tb
 
             # Special vcs errors had an attribute "_vcs_kind" which is used
             # to translate them to the proper exception class in the vcs
             # client layer.
             kind = getattr(e, '_vcs_kind', None)
             exc_name = getattr(e, '_vcs_server_org_exc_name', None)
 
             if kind:
                 if any(e.args):
                     _args = [a for a in e.args]
                     # replace the first argument with a prefix exc name
                     args = ['{}:{}'.format(exc_name, _args[0] if _args else '?')] + _args[1:]
                 else:
                     args = [__traceback_info__ or '{}: UnhandledException'.format(exc_name)]
                 if debug or __traceback_info__ and kind not in ['unhandled', 'lookup']:
                     # for other than unhandled errors also log the traceback
                     # can be useful for debugging
                     log.error(__traceback_info__)
 
                 raise _EXCEPTION_MAP[kind](*args)
             else:
                 raise
     return wrapper

rhodecode/lib/vcs/nodes.py

@@ -1,876 +1,963 b''
 
 
 # Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Module holding everything related to vcs nodes, with vcs2 architecture.
 """
-
+import functools
 import os
 import stat
 
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
-import rhodecode
 from rhodecode.config.conf import LANGUAGES_EXTENSIONS_MAP
-from rhodecode.lib.utils import safe_unicode, safe_str
-from rhodecode.lib.utils2 import md5
+from rhodecode.lib.str_utils import safe_str, safe_bytes
+from rhodecode.lib.hash_utils import md5
 from rhodecode.lib.vcs import path as vcspath
 from rhodecode.lib.vcs.backends.base import EmptyCommit, FILEMODE_DEFAULT
 from rhodecode.lib.vcs.conf.mtypes import get_mimetypes_db
 from rhodecode.lib.vcs.exceptions import NodeError, RemovedFileNodeError
 
 LARGEFILE_PREFIX = '.hglf'
 
 
 class NodeKind:
     SUBMODULE = -1
     DIR = 1
     FILE = 2
     LARGEFILE = 3
 
 
 class NodeState:
     ADDED = 'added'
     CHANGED = 'changed'
     NOT_CHANGED = 'not changed'
     REMOVED = 'removed'
 
+#TODO: not sure if that should be bytes or str ?
+# most probably bytes because content should be bytes and we check it
+BIN_BYTE_MARKER = b'\0'
+
 
 class NodeGeneratorBase(object):
     """
     Base class for removed added and changed filenodes, it's a lazy generator
     class that will create filenodes only on iteration or call
 
     The len method doesn't need to create filenodes at all
     """
 
     def __init__(self, current_paths, cs):
         self.cs = cs
         self.current_paths = current_paths
 
     def __call__(self):
         return [n for n in self]
 
-    def __getslice__(self, i, j):
-        for p in self.current_paths[i:j]:
+    def __getitem__(self, key):
+        if isinstance(key, slice):
+            for p in self.current_paths[key.start:key.stop]:
             yield self.cs.get_node(p)
 
     def __len__(self):
         return len(self.current_paths)
 
     def __iter__(self):
         for p in self.current_paths:
             yield self.cs.get_node(p)
 
 
 class AddedFileNodesGenerator(NodeGeneratorBase):
     """
     Class holding added files for current commit
     """
 
 
 class ChangedFileNodesGenerator(NodeGeneratorBase):
     """
     Class holding changed files for current commit
     """
 
 
 class RemovedFileNodesGenerator(NodeGeneratorBase):
     """
     Class holding removed files for current commit
     """
     def __iter__(self):
         for p in self.current_paths:
-            yield RemovedFileNode(path=p)
+            yield RemovedFileNode(path=safe_bytes(p))
 
-    def __getslice__(self, i, j):
-        for p in self.current_paths[i:j]:
-            yield RemovedFileNode(path=p)
+    def __getitem__(self, key):
+        if isinstance(key, slice):
+            for p in self.current_paths[key.start:key.stop]:
+                yield RemovedFileNode(path=safe_bytes(p))
 
 
[e-mail]_ordering
 class Node(object):
     """
     Simplest class representing file or directory on repository.  SCM backends
     should use ``FileNode`` and ``DirNode`` subclasses rather than ``Node``
     directly.
 
     Node's ``path`` cannot start with slash as we operate on *relative* paths
     only. Moreover, every single node is identified by the ``path`` attribute,
     so it cannot end with slash, too. Otherwise, path could lead to mistakes.
     """
-    RTLO_MARKER = "\u202E"  # RTLO marker allows swapping text, and certain
+    # RTLO marker allows swapping text, and certain
                              # security attacks could be used with this
+    RTLO_MARKER = "\u202E"
+
     commit = None
 
-    def __init__(self, path, kind):
+    def __init__(self, path: bytes, kind):
         self._validate_path(path)  # can throw exception if path is invalid
-        self.path = safe_str(path.rstrip('/'))  # we store paths as str
-        if path == '' and kind != NodeKind.DIR:
+
+        self.bytes_path = path.rstrip(b'/')  # store for __repr__
+        self.path = safe_str(self.bytes_path)  # we store paths as str
+
+        if self.bytes_path == b'' and kind != NodeKind.DIR:
             raise NodeError("Only DirNode and its subclasses may be "
                             "initialized with empty path")
         self.kind = kind
 
         if self.is_root() and not self.is_dir():
             raise NodeError("Root node cannot be FILE kind")
 
-    def _validate_path(self, path):
-        if path.startswith('/'):
+    def __eq__(self, other):
+        if type(self) is not type(other):
+            return False
+        for attr in ['name', 'path', 'kind']:
+            if getattr(self, attr) != getattr(other, attr):
+                return False
+        if self.is_file():
+            # FileNode compare, we need to fallback to content compare
+            return None
+        else:
+            # For DirNode's check without entering each dir
+            self_nodes_paths = list(sorted(n.path for n in self.nodes))
+            other_nodes_paths = list(sorted(n.path for n in self.nodes))
+            if self_nodes_paths != other_nodes_paths:
+                return False
+        return True
+
+    def __lt__(self, other):
+        if self.kind < other.kind:
+            return True
+        if self.kind > other.kind:
+            return False
+        if self.path < other.path:
+            return True
+        if self.path > other.path:
+            return False
+
+    # def __cmp__(self, other):
+    #     """
+    #     Comparator using name of the node, needed for quick list sorting.
+    #     """
+    #
+    #     kind_cmp = cmp(self.kind, other.kind)
+    #     if kind_cmp:
+    #         if isinstance(self, SubModuleNode):
+    #             # we make submodules equal to dirnode for "sorting" purposes
+    #             return NodeKind.DIR
+    #         return kind_cmp
+    #     return cmp(self.name, other.name)
+
+    def __repr__(self):
+        maybe_path = getattr(self, 'path', 'UNKNOWN_PATH')
+        return f'<{self.__class__.__name__} {maybe_path!r}>'
+
+    def __str__(self):
+        return self.name
+
+    def _validate_path(self, path: bytes):
+        self._assert_bytes(path)
+
+        if path.startswith(b'/'):
             raise NodeError(
-                "Cannot initialize Node objects with slash at "
-                "the beginning as only relative paths are supported. "
-                "Got %s" % (path,))
+                f"Cannot initialize Node objects with slash at "
+                f"the beginning as only relative paths are supported. "
+                f"Got {path}")
+
+    def _assert_bytes(self, value):
+        if not isinstance(value, bytes):
+            raise TypeError(f"Bytes required as input, got {type(value)} of {value}.")
 
     @LazyProperty
     def parent(self):
         parent_path = self.get_parent_path()
         if parent_path:
             if self.commit:
                 return self.commit.get_node(parent_path)
             return DirNode(parent_path)
         return None
 
     @LazyProperty
-    def unicode_path(self):
-        return safe_unicode(self.path)
+    def str_path(self) -> str:
+        return safe_str(self.path)
 
     @LazyProperty
     def has_rtlo(self):
         """Detects if a path has right-to-left-override marker"""
-        return self.RTLO_MARKER in self.unicode_path
-
-    @LazyProperty
-    def unicode_path_safe(self):
-        """
-        Special SAFE representation of path without the right-to-left-override.
-        This should be only used for "showing" the file, cannot be used for any
-        urls etc.
-        """
-        return safe_unicode(self.path).replace(self.RTLO_MARKER, '')
+        return self.RTLO_MARKER in self.str_path
 
     @LazyProperty
     def dir_path(self):
         """
         Returns name of the directory from full path of this vcs node. Empty
         string is returned if there's no directory in the path
         """
         _parts = self.path.rstrip('/').rsplit('/', 1)
         if len(_parts) == 2:
-            return safe_unicode(_parts[0])
+            return _parts[0]
         return ''
 
     @LazyProperty
     def name(self):
         """
         Returns name of the node so if its path
         then only last part is returned.
         """
-        return safe_unicode(self.path.rstrip('/').split('/')[-1])
+        return self.path.rstrip('/').split('/')[-1]
 
     @property
     def kind(self):
         return self._kind
 
     @kind.setter
     def kind(self, kind):
         if hasattr(self, '_kind'):
             raise NodeError("Cannot change node's kind")
         else:
             self._kind = kind
             # Post setter check (path's trailing slash)
             if self.path.endswith('/'):
                 raise NodeError("Node's path cannot end with slash")
 
-    def __cmp__(self, other):
-        """
-        Comparator using name of the node, needed for quick list sorting.
-        """
-
-        kind_cmp = cmp(self.kind, other.kind)
-        if kind_cmp:
-            if isinstance(self, SubModuleNode):
-                # we make submodules equal to dirnode for "sorting" purposes
-                return NodeKind.DIR
-            return kind_cmp
-        return cmp(self.name, other.name)
-
-    def __eq__(self, other):
-        for attr in ['name', 'path', 'kind']:
-            if getattr(self, attr) != getattr(other, attr):
-                return False
-        if self.is_file():
-            if self.content != other.content:
-                return False
-        else:
-            # For DirNode's check without entering each dir
-            self_nodes_paths = list(sorted(n.path for n in self.nodes))
-            other_nodes_paths = list(sorted(n.path for n in self.nodes))
-            if self_nodes_paths != other_nodes_paths:
-                return False
-        return True
-
-    def __ne__(self, other):
-        return not self.__eq__(other)
-
-    def __repr__(self):
-        return '<%s %r>' % (self.__class__.__name__, self.path)
-
-    def __str__(self):
-        return self.__repr__()
-
-    def __unicode__(self):
-        return self.name
-
-    def get_parent_path(self):
+    def get_parent_path(self) -> bytes:
         """
         Returns node's parent path or empty string if node is root.
         """
         if self.is_root():
-            return ''
-        return vcspath.dirname(self.path.rstrip('/')) + '/'
+            return b''
+        str_path = vcspath.dirname(self.path.rstrip('/')) + '/'
+
+        return safe_bytes(str_path)
 
     def is_file(self):
         """
         Returns ``True`` if node's kind is ``NodeKind.FILE``, ``False``
         otherwise.
         """
         return self.kind == NodeKind.FILE
 
     def is_dir(self):
         """
         Returns ``True`` if node's kind is ``NodeKind.DIR``, ``False``
         otherwise.
         """
         return self.kind == NodeKind.DIR
 
     def is_root(self):
         """
         Returns ``True`` if node is a root node and ``False`` otherwise.
         """
         return self.kind == NodeKind.DIR and self.path == ''
 
     def is_submodule(self):
         """
         Returns ``True`` if node's kind is ``NodeKind.SUBMODULE``, ``False``
         otherwise.
         """
         return self.kind == NodeKind.SUBMODULE
 
     def is_largefile(self):
         """
         Returns ``True`` if node's kind is ``NodeKind.LARGEFILE``, ``False``
         otherwise
         """
         return self.kind == NodeKind.LARGEFILE
 
     def is_link(self):
         if self.commit:
             return self.commit.is_link(self.path)
         return False
 
     @LazyProperty
     def added(self):
         return self.state is NodeState.ADDED
 
     @LazyProperty
     def changed(self):
         return self.state is NodeState.CHANGED
 
     @LazyProperty
     def not_changed(self):
         return self.state is NodeState.NOT_CHANGED
 
     @LazyProperty
     def removed(self):
         return self.state is NodeState.REMOVED
 
 
 class FileNode(Node):
     """
     Class representing file nodes.
 
     :attribute: path: path to the node, relative to repository's root
     :attribute: content: if given arbitrary sets content of the file
     :attribute: commit: if given, first time content is accessed, callback
     :attribute: mode: stat mode for a node. Default is `FILEMODE_DEFAULT`.
     """
     _filter_pre_load = []
 
-    def __init__(self, path, content=None, commit=None, mode=None, pre_load=None):
+    def __init__(self, path: bytes, content: bytes | None = None, commit=None, mode=None, pre_load=None):
         """
         Only one of ``content`` and ``commit`` may be given. Passing both
         would raise ``NodeError`` exception.
 
         :param path: relative path to the node
         :param content: content may be passed to constructor
         :param commit: if given, will use it to lazily fetch content
         :param mode: ST_MODE (i.e. 0100644)
         """
         if content and commit:
             raise NodeError("Cannot use both content and commit")
-        super(FileNode, self).__init__(path, kind=NodeKind.FILE)
+
+        super().__init__(path, kind=NodeKind.FILE)
+
         self.commit = commit
+        if content and not isinstance(content, bytes):
+            # File content is one thing that inherently must be bytes
+            # we support passing str too, and convert the content
+            content = safe_bytes(content)
         self._content = content
         self._mode = mode or FILEMODE_DEFAULT
 
         self._set_bulk_properties(pre_load)
 
+    def __eq__(self, other):
+        eq = super(FileNode, self).__eq__(other)
+        if eq is not None:
+            return eq
+        return self.content == other.content
+
+    def __hash__(self):
+        raw_id = getattr(self.commit, 'raw_id', '')
+        return hash((self.path, raw_id))
+
+    def __lt__(self, other):
+        lt = super(FileNode, self).__lt__(other)
+        if lt is not None:
+            return lt
+        return self.content < other.content
+
+    def __repr__(self):
+        short_id = getattr(self.commit, 'short_id', '')
+        return f'<{self.__class__.__name__} path={self.path!r}, short_id={short_id}>'
+
     def _set_bulk_properties(self, pre_load):
         if not pre_load:
             return
         pre_load = [entry for entry in pre_load
                     if entry not in self._filter_pre_load]
         if not pre_load:
             return
 
-        for attr_name in pre_load:
-            result = getattr(self, attr_name)
-            if callable(result):
-                result = result()
-            self.__dict__[attr_name] = result
+        remote = self.commit.get_remote()
+        result = remote.bulk_file_request(self.commit.raw_id, self.path, pre_load)
+
+        for attr, value in result.items():
+            if attr == "flags":
+                self.__dict__['mode'] = safe_str(value)
+            elif attr == "size":
+                self.__dict__['size'] = value
+            elif attr == "data":
+                self.__dict__['_content'] = value
+            elif attr == "is_binary":
+                self.__dict__['is_binary'] = value
+            elif attr == "md5":
+                self.__dict__['md5'] = value
+            else:
+                raise ValueError(f'Unsupported attr in bulk_property: {attr}')
 
     @LazyProperty
     def mode(self):
         """
         Returns lazily mode of the FileNode. If `commit` is not set, would
         use value given at initialization or `FILEMODE_DEFAULT` (default).
         """
         if self.commit:
             mode = self.commit.get_file_mode(self.path)
         else:
             mode = self._mode
         return mode
 
     @LazyProperty
-    def raw_bytes(self):
+    def raw_bytes(self) -> bytes:
         """
         Returns lazily the raw bytes of the FileNode.
         """
         if self.commit:
             if self._content is None:
                 self._content = self.commit.get_file_content(self.path)
             content = self._content
         else:
             content = self._content
         return content
 
+    def content_uncached(self):
+        """
+        Returns lazily content of the FileNode.
+        """
+        if self.commit:
+            content = self.commit.get_file_content(self.path)
+        else:
+            content = self._content
+        return content
+
     def stream_bytes(self):
         """
         Returns an iterator that will stream the content of the file directly from
         vcsserver without loading it to memory.
         """
         if self.commit:
             return self.commit.get_file_content_streamed(self.path)
         raise NodeError("Cannot retrieve stream_bytes without related commit attribute")
 
-    @LazyProperty
-    def md5(self):
-        """
-        Returns md5 of the file node.
-        """
-        return md5(self.raw_bytes)
-
     def metadata_uncached(self):
         """
         Returns md5, binary flag of the file node, without any cache usage.
         """
 
         content = self.content_uncached()
 
-        is_binary = content and '\0' in content
+        is_binary = bool(content and BIN_BYTE_MARKER in content)
         size = 0
         if content:
             size = len(content)
 
         return is_binary, md5(content), size, content
 
-    def content_uncached(self):
-        """
-        Returns lazily content of the FileNode. If possible, would try to
-        decode content from UTF-8.
+    @LazyProperty
+    def content(self) -> bytes:
         """
-        if self.commit:
-            content = self.commit.get_file_content(self.path)
-        else:
-            content = self._content
+        Returns lazily content of the FileNode.
+        """
+        content = self.raw_bytes
+        if content and not isinstance(content, bytes):
+            raise ValueError(f'Content is of type {type(content)} instead of bytes')
         return content
 
     @LazyProperty
-    def content(self):
-        """
-        Returns lazily content of the FileNode. If possible, would try to
-        decode content from UTF-8.
-        """
-        content = self.raw_bytes
-
-        if self.is_binary:
-            return content
-        return safe_unicode(content)
+    def str_content(self) -> str:
+        return safe_str(self.raw_bytes)
 
     @LazyProperty
     def size(self):
         if self.commit:
             return self.commit.get_file_size(self.path)
         raise NodeError(
             "Cannot retrieve size of the file without related "
             "commit attribute")
 
     @LazyProperty
     def message(self):
         if self.commit:
             return self.last_commit.message
         raise NodeError(
             "Cannot retrieve message of the file without related "
             "commit attribute")
 
     @LazyProperty
     def last_commit(self):
         if self.commit:
             pre_load = ["author", "date", "message", "parents"]
             return self.commit.get_path_commit(self.path, pre_load=pre_load)
         raise NodeError(
             "Cannot retrieve last commit of the file without "
             "related commit attribute")
 
     def get_mimetype(self):
         """
         Mimetype is calculated based on the file's content. If ``_mimetype``
         attribute is available, it will be returned (backends which store
         mimetypes or can easily recognize them, should set this private
         attribute to indicate that type should *NOT* be calculated).
         """
 
         if hasattr(self, '_mimetype'):
-            if (isinstance(self._mimetype, (tuple, list,)) and
+            if (isinstance(self._mimetype, (tuple, list)) and
                     len(self._mimetype) == 2):
                 return self._mimetype
             else:
                 raise NodeError('given _mimetype attribute must be an 2 '
                                 'element list or tuple')
 
         db = get_mimetypes_db()
         mtype, encoding = db.guess_type(self.name)
 
         if mtype is None:
             if not self.is_largefile() and self.is_binary:
                 mtype = 'application/octet-stream'
                 encoding = None
             else:
                 mtype = 'text/plain'
                 encoding = None
 
                 # try with pygments
                 try:
                     from pygments.lexers import get_lexer_for_filename
                     mt = get_lexer_for_filename(self.name).mimetypes
                 except Exception:
                     mt = None
 
                 if mt:
                     mtype = mt[0]
 
         return mtype, encoding
 
     @LazyProperty
     def mimetype(self):
         """
         Wrapper around full mimetype info. It returns only type of fetched
         mimetype without the encoding part. use get_mimetype function to fetch
         full set of (type,encoding)
         """
         return self.get_mimetype()[0]
 
     @LazyProperty
     def mimetype_main(self):
         return self.mimetype.split('/')[0]
 
     @classmethod
     def get_lexer(cls, filename, content=None):
         from pygments import lexers
 
         extension = filename.split('.')[-1]
         lexer = None
 
         try:
             lexer = lexers.guess_lexer_for_filename(
                 filename, content, stripnl=False)
         except lexers.ClassNotFound:
-            lexer = None
+            pass
 
         # try our EXTENSION_MAP
         if not lexer:
             try:
                 lexer_class = LANGUAGES_EXTENSIONS_MAP.get(extension)
                 if lexer_class:
                     lexer = lexers.get_lexer_by_name(lexer_class[0])
             except lexers.ClassNotFound:
-                lexer = None
+                pass
 
         if not lexer:
             lexer = lexers.TextLexer(stripnl=False)
 
         return lexer
 
     @LazyProperty
     def lexer(self):
         """
         Returns pygment's lexer class. Would try to guess lexer taking file's
         content, name and mimetype.
         """
-        return self.get_lexer(self.name, self.content)
+        # TODO: this is more proper, but super heavy on investigating the type based on the content
+        #self.get_lexer(self.name, self.content)
+
+        return self.get_lexer(self.name)
 
     @LazyProperty
     def lexer_alias(self):
         """
         Returns first alias of the lexer guessed for this file.
         """
         return self.lexer.aliases[0]
 
     @LazyProperty
     def history(self):
         """
         Returns a list of commit for this file in which the file was changed
         """
         if self.commit is None:
             raise NodeError('Unable to get commit for this FileNode')
         return self.commit.get_path_history(self.path)
 
     @LazyProperty
     def annotate(self):
         """
         Returns a list of three element tuples with lineno, commit and line
         """
         if self.commit is None:
             raise NodeError('Unable to get commit for this FileNode')
         pre_load = ["author", "date", "message", "parents"]
         return self.commit.get_file_annotate(self.path, pre_load=pre_load)
 
     @LazyProperty
     def state(self):
         if not self.commit:
             raise NodeError(
                 "Cannot check state of the node if it's not "
                 "linked with commit")
         elif self.path in (node.path for node in self.commit.added):
             return NodeState.ADDED
         elif self.path in (node.path for node in self.commit.changed):
             return NodeState.CHANGED
         else:
             return NodeState.NOT_CHANGED
 
     @LazyProperty
     def is_binary(self):
         """
         Returns True if file has binary content.
         """
         if self.commit:
             return self.commit.is_node_binary(self.path)
         else:
             raw_bytes = self._content
-            return raw_bytes and '\0' in raw_bytes
+            return bool(raw_bytes and BIN_BYTE_MARKER in raw_bytes)
+
+    @LazyProperty
+    def md5(self):
+        """
+        Returns md5 of the file node.
+        """
+
+        if self.commit:
+            return self.commit.node_md5_hash(self.path)
+        else:
+            raw_bytes = self._content
+            # TODO: this sucks, we're computing md5 on potentially super big stream data...
+            return md5(raw_bytes)
 
     @LazyProperty
     def extension(self):
         """Returns filenode extension"""
         return self.name.split('.')[-1]
 
     @property
     def is_executable(self):
         """
         Returns ``True`` if file has executable flag turned on.
         """
         return bool(self.mode & stat.S_IXUSR)
 
     def get_largefile_node(self):
         """
         Try to return a Mercurial FileNode from this node. It does internal
         checks inside largefile store, if that file exist there it will
         create special instance of LargeFileNode which can get content from
         LF store.
         """
         if self.commit:
             return self.commit.get_largefile_node(self.path)
 
-    def count_lines(self, content, count_empty=False):
+    def count_lines(self, content: str | bytes, count_empty=False):
+        if isinstance(content, str):
+            newline_marker = '\n'
+        elif isinstance(content, bytes):
+            newline_marker = b'\n'
+        else:
+            raise ValueError('content must be bytes or str got {type(content)} instead')
 
         if count_empty:
             all_lines = 0
             empty_lines = 0
             for line in content.splitlines(True):
-                if line == '\n':
+                if line == newline_marker:
                     empty_lines += 1
                 all_lines += 1
 
             return all_lines, all_lines - empty_lines
         else:
             # fast method
-            empty_lines = all_lines = content.count('\n')
+            empty_lines = all_lines = content.count(newline_marker)
             if all_lines == 0 and content:
                 # one-line without a newline
                 empty_lines = all_lines = 1
 
         return all_lines, empty_lines
 
     def lines(self, count_empty=False):
         all_lines, empty_lines = 0, 0
 
         if not self.is_binary:
             content = self.content
             all_lines, empty_lines = self.count_lines(content, count_empty=count_empty)
         return all_lines, empty_lines
 
-    def __repr__(self):
-        return '<%s %r @ %s>' % (self.__class__.__name__, self.path,
-                                 getattr(self.commit, 'short_id', ''))
-
 
 class RemovedFileNode(FileNode):
     """
     Dummy FileNode class - trying to access any public attribute except path,
     name, kind or state (or methods/attributes checking those two) would raise
     RemovedFileNodeError.
     """
     ALLOWED_ATTRIBUTES = [
         'name', 'path', 'state', 'is_root', 'is_file', 'is_dir', 'kind',
-        'added', 'changed', 'not_changed', 'removed'
+        'added', 'changed', 'not_changed', 'removed', 'bytes_path'
     ]
 
     def __init__(self, path):
         """
         :param path: relative path to the node
         """
-        super(RemovedFileNode, self).__init__(path=path)
+        super().__init__(path=path)
 
     def __getattribute__(self, attr):
         if attr.startswith('_') or attr in RemovedFileNode.ALLOWED_ATTRIBUTES:
-            return super(RemovedFileNode, self).__getattribute__(attr)
-        raise RemovedFileNodeError(
-            "Cannot access attribute %s on RemovedFileNode" % attr)
+            return super().__getattribute__(attr)
+        raise RemovedFileNodeError(f"Cannot access attribute {attr} on RemovedFileNode. Not in allowed attributes")
 
     @LazyProperty
     def state(self):
         return NodeState.REMOVED
 
 
 class DirNode(Node):
     """
     DirNode stores list of files and directories within this node.
     Nodes may be used standalone but within repository context they
     lazily fetch data within same repository's commit.
     """
 
-    def __init__(self, path, nodes=(), commit=None):
+    def __init__(self, path, nodes=(), commit=None, default_pre_load=None):
         """
         Only one of ``nodes`` and ``commit`` may be given. Passing both
         would raise ``NodeError`` exception.
 
         :param path: relative path to the node
         :param nodes: content may be passed to constructor
         :param commit: if given, will use it to lazily fetch content
         """
         if nodes and commit:
             raise NodeError("Cannot use both nodes and commit")
         super(DirNode, self).__init__(path, NodeKind.DIR)
         self.commit = commit
         self._nodes = nodes
+        self.default_pre_load = default_pre_load or ['is_binary', 'size']
+
+    def __iter__(self):
+        for node in self.nodes:
+            yield node
+
+    def __eq__(self, other):
+        eq = super(DirNode, self).__eq__(other)
+        if eq is not None:
+            return eq
+        # check without entering each dir
+        self_nodes_paths = list(sorted(n.path for n in self.nodes))
+        other_nodes_paths = list(sorted(n.path for n in self.nodes))
+        return self_nodes_paths == other_nodes_paths
+
+    def __lt__(self, other):
+        lt = super(DirNode, self).__lt__(other)
+        if lt is not None:
+            return lt
+        # check without entering each dir
+        self_nodes_paths = list(sorted(n.path for n in self.nodes))
+        other_nodes_paths = list(sorted(n.path for n in self.nodes))
+        return self_nodes_paths < other_nodes_paths
 
     @LazyProperty
     def content(self):
-        raise NodeError(
-            "%s represents a dir and has no `content` attribute" % self)
+        raise NodeError(f"{self} represents a dir and has no `content` attribute")
 
     @LazyProperty
     def nodes(self):
         if self.commit:
-            nodes = self.commit.get_nodes(self.path)
+            nodes = self.commit.get_nodes(self.path, pre_load=self.default_pre_load)
         else:
             nodes = self._nodes
         self._nodes_dict = dict((node.path, node) for node in nodes)
         return sorted(nodes)
 
     @LazyProperty
     def files(self):
         return sorted((node for node in self.nodes if node.is_file()))
 
     @LazyProperty
     def dirs(self):
         return sorted((node for node in self.nodes if node.is_dir()))
 
-    def __iter__(self):
-        for node in self.nodes:
-            yield node
-
     def get_node(self, path):
         """
         Returns node from within this particular ``DirNode``, so it is now
         allowed to fetch, i.e. node located at 'docs/api/index.rst' from node
         'docs'. In order to access deeper nodes one must fetch nodes between
         them first - this would work::
 
            docs = root.get_node('docs')
            docs.get_node('api').get_node('index.rst')
 
         :param: path - relative to the current node
 
         .. note::
            To access lazily (as in example above) node have to be initialized
            with related commit object - without it node is out of
            context and may know nothing about anything else than nearest
            (located at same level) nodes.
         """
         try:
             path = path.rstrip('/')
             if path == '':
                 raise NodeError("Cannot retrieve node without path")
             self.nodes  # access nodes first in order to set _nodes_dict
             paths = path.split('/')
             if len(paths) == 1:
                 if not self.is_root():
                     path = '/'.join((self.path, paths[0]))
                 else:
                     path = paths[0]
                 return self._nodes_dict[path]
             elif len(paths) > 1:
                 if self.commit is None:
                     raise NodeError("Cannot access deeper nodes without commit")
                 else:
                     path1, path2 = paths[0], '/'.join(paths[1:])
                     return self.get_node(path1).get_node(path2)
             else:
                 raise KeyError
         except KeyError:
-            raise NodeError("Node does not exist at %s" % path)
+            raise NodeError(f"Node does not exist at {path}")
 
     @LazyProperty
     def state(self):
         raise NodeError("Cannot access state of DirNode")
 
     @LazyProperty
     def size(self):
         size = 0
         for root, dirs, files in self.commit.walk(self.path):
             for f in files:
                 size += f.size
 
         return size
 
     @LazyProperty
     def last_commit(self):
         if self.commit:
             pre_load = ["author", "date", "message", "parents"]
             return self.commit.get_path_commit(self.path, pre_load=pre_load)
         raise NodeError(
             "Cannot retrieve last commit of the file without "
             "related commit attribute")
 
     def __repr__(self):
-        return '<%s %r @ %s>' % (self.__class__.__name__, self.path,
-                                 getattr(self.commit, 'short_id', ''))
+        short_id = getattr(self.commit, 'short_id', '')
+        return f'<{self.__class__.__name__} {self.path!r} @ {short_id}>'
 
 
 class RootNode(DirNode):
     """
     DirNode being the root node of the repository.
     """
 
     def __init__(self, nodes=(), commit=None):
-        super(RootNode, self).__init__(path='', nodes=nodes, commit=commit)
+        super(RootNode, self).__init__(path=b'', nodes=nodes, commit=commit)
 
     def __repr__(self):
-        return '<%s>' % self.__class__.__name__
+        return f'<{self.__class__.__name__}>'
 
 
 class SubModuleNode(Node):
     """
     represents a SubModule of Git or SubRepo of Mercurial
     """
     is_binary = False
     size = 0
 
     def __init__(self, name, url=None, commit=None, alias=None):
         self.path = name
         self.kind = NodeKind.SUBMODULE
         self.alias = alias
 
         # we have to use EmptyCommit here since this can point to svn/git/hg
         # submodules we cannot get from repository
         self.commit = EmptyCommit(str(commit), alias=alias)
         self.url = url or self._extract_submodule_url()
 
     def __repr__(self):
-        return '<%s %r @ %s>' % (self.__class__.__name__, self.path,
-                                 getattr(self.commit, 'short_id', ''))
+        short_id = getattr(self.commit, 'short_id', '')
+        return f'<{self.__class__.__name__} {self.path!r} @ {short_id}>'
 
     def _extract_submodule_url(self):
         # TODO: find a way to parse gits submodule file and extract the
         # linking URL
         return self.path
 
     @LazyProperty
     def name(self):
         """
         Returns name of the node so if its path
         then only last part is returned.
         """
-        org = safe_unicode(self.path.rstrip('/').split('/')[-1])
-        return '%s @ %s' % (org, self.commit.short_id)
+        org = safe_str(self.path.rstrip('/').split('/')[-1])
+        return f'{org} @ {self.commit.short_id}'
 
 
 class LargeFileNode(FileNode):
 
     def __init__(self, path, url=None, commit=None, alias=None, org_path=None):
-        self.path = path
-        self.org_path = org_path
+        self._validate_path(path)  # can throw exception if path is invalid
+        self.org_path = org_path  # as stored in VCS as LF pointer
+
+        self.bytes_path = path.rstrip(b'/')  # store for __repr__
+        self.path = safe_str(self.bytes_path)  # we store paths as str
+
         self.kind = NodeKind.LARGEFILE
         self.alias = alias
-        self._content = ''
+        self._content = b''
 
-    def _validate_path(self, path):
+    def _validate_path(self, path: bytes):
         """
-        we override check since the LargeFileNode path is system absolute
+        we override check since the LargeFileNode path is system absolute, but we check for bytes only
         """
-        pass
+        self._assert_bytes(path)
 
     def __repr__(self):
-        return '<%s %r>' % (self.__class__.__name__, self.path)
+        return f'<{self.__class__.__name__} {self.org_path} -> {self.path!r}>'
 
     @LazyProperty
     def size(self):
         return os.stat(self.path).st_size
 
     @LazyProperty
     def raw_bytes(self):
         with open(self.path, 'rb') as f:
             content = f.read()
         return content
 
     @LazyProperty
     def name(self):
         """
         Overwrites name to be the org lf path
         """
         return self.org_path
 
     def stream_bytes(self):
         with open(self.path, 'rb') as stream:
             while True:
                 data = stream.read(16 * 1024)
                 if not data:
                     break
                 yield data

rhodecode/lib/vcs/utils/helpers.py

@@ -1,161 +1,161 b''
 
 
 # Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Utilities aimed to help achieve mostly basic tasks.
 """
 
 
 
 
 import re
 import os
 import time
 import datetime
 import logging
 
 from rhodecode.lib.vcs.conf import settings
 from rhodecode.lib.vcs.exceptions import VCSError, VCSBackendNotSupportedError
 
 
 log = logging.getLogger(__name__)
 
 
 def get_scm(path):
     """
     Returns one of alias from ``ALIASES`` (in order of precedence same as
     shortcuts given in ``ALIASES``) and working dir path for the given
     argument. If no scm-specific directory is found or more than one scm is
     found at that directory, ``VCSError`` is raised.
     """
     if not os.path.isdir(path):
         raise VCSError("Given path %s is not a directory" % path)
 
     found_scms = [(scm, path) for scm in get_scms_for_path(path)]
 
     if len(found_scms) > 1:
         found = ', '.join((x[0] for x in found_scms))
         raise VCSError(
             'More than one [%s] scm found at given path %s' % (found, path))
 
-    if len(found_scms) is 0:
+    if len(found_scms) == 0:
         raise VCSError('No scm found at given path %s' % path)
 
     return found_scms[0]
 
 
 def get_scm_backend(backend_type):
     from rhodecode.lib.vcs.backends import get_backend
     return get_backend(backend_type)
 
 
 def get_scms_for_path(path):
     """
     Returns all scm's found at the given path. If no scm is recognized
     - empty list is returned.
 
     :param path: path to directory which should be checked. May be callable.
 
     :raises VCSError: if given ``path`` is not a directory
     """
     from rhodecode.lib.vcs.backends import get_backend
     if hasattr(path, '__call__'):
         path = path()
     if not os.path.isdir(path):
         raise VCSError("Given path %r is not a directory" % path)
 
     result = []
     for key in settings.available_aliases():
         try:
             backend = get_backend(key)
         except VCSBackendNotSupportedError:
             log.warning('VCSBackendNotSupportedError: %s not supported', key)
             continue
         if backend.is_valid_repository(path):
             result.append(key)
     return result
 
 
 def parse_datetime(text):
     """
     Parses given text and returns ``datetime.datetime`` instance or raises
     ``ValueError``.
 
     :param text: string of desired date/datetime or something more verbose,
       like *yesterday*, *2weeks 3days*, etc.
     """
     if not text:
         raise ValueError('Wrong date: "%s"' % text)
 
     if isinstance(text, datetime.datetime):
         return text
 
     # we limit a format to no include microseconds e.g 2017-10-17t17:48:23.XXXX
     text = text.strip().lower()[:19]
 
     input_formats = (
         '%Y-%m-%d %H:%M:%S',
         '%Y-%m-%dt%H:%M:%S',
         '%Y-%m-%d %H:%M',
         '%Y-%m-%dt%H:%M',
         '%Y-%m-%d',
         '%m/%d/%Y %H:%M:%S',
         '%m/%d/%Yt%H:%M:%S',
         '%m/%d/%Y %H:%M',
         '%m/%d/%Yt%H:%M',
         '%m/%d/%Y',
         '%m/%d/%y %H:%M:%S',
         '%m/%d/%yt%H:%M:%S',
         '%m/%d/%y %H:%M',
         '%m/%d/%yt%H:%M',
         '%m/%d/%y',
     )
     for format_def in input_formats:
         try:
             return datetime.datetime(*time.strptime(text, format_def)[:6])
         except ValueError:
             pass
 
     # Try descriptive texts
     if text == 'tomorrow':
         future = datetime.datetime.now() + datetime.timedelta(days=1)
         args = future.timetuple()[:3] + (23, 59, 59)
         return datetime.datetime(*args)
     elif text == 'today':
         return datetime.datetime(*datetime.datetime.today().timetuple()[:3])
     elif text == 'now':
         return datetime.datetime.now()
     elif text == 'yesterday':
         past = datetime.datetime.now() - datetime.timedelta(days=1)
         return datetime.datetime(*past.timetuple()[:3])
     else:
         days = 0
         matched = re.match(
             r'^((?P<weeks>\d+) ?w(eeks?)?)? ?((?P<days>\d+) ?d(ays?)?)?$', text)
         if matched:
             groupdict = matched.groupdict()
             if groupdict['days']:
                 days += int(matched.groupdict()['days'])
             if groupdict['weeks']:
                 days += int(matched.groupdict()['weeks']) * 7
             past = datetime.datetime.now() - datetime.timedelta(days=days)
             return datetime.datetime(*past.timetuple()[:3])
 
     raise ValueError('Wrong date: "%s"' % text)