models: major update for python3,

- imports fixes
- str vs unicode fixes
- found issues with logic on tests runs

rhodecode/model/changeset_status.py

@@ -178,13 +178,14 b' class ChangesetStatusModel(BaseModel):'
         """
 
         def group_rule(element):
-            review_obj = element[0]
-            rule_data = review_obj.rule_user_group_data()
+            _review_obj = element[0]
+            rule_data = _review_obj.rule_user_group_data()
             if rule_data and rule_data['id']:
                 return rule_data['id']
+            # don't return None, as we cant compare this
+            return 0
 
-        voting_groups = itertools.groupby(
-            sorted(statuses_by_reviewers, key=group_rule), group_rule)
+        voting_groups = itertools.groupby(sorted(statuses_by_reviewers, key=group_rule), group_rule)
 
         voting_by_groups = [(x, list(y)) for x, y in voting_groups]
 

rhodecode/model/comment.py

@@ -296,9 +296,6 b' class CommentsModel(BaseModel):'
         :param extra_recipients: list of extra users to be added to recipients
         """
 
-        if not text:
-            log.warning('Missing text for comment, skipping...')
-            return
         request = get_current_request()
         _ = request.translate
 

rhodecode/model/gist.py

@@ -31,7 +31,7 b' import shutil'
 from pyramid.threadlocal import get_current_request
 
 from rhodecode.lib.utils2 import (
-    safe_unicode, unique_id, safe_int, time_to_datetime, AttributeDict)
+    unique_id, safe_int, safe_str, time_to_datetime, AttributeDict)
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.vcs import VCSError
 from rhodecode.model import BaseModel
@@ -121,7 +121,7 b' class GistModel(BaseModel):'
         :param gist_acl_level: acl level for this gist
         """
         owner = self._get_user(owner)
-        gist_id = safe_unicode(gist_id or unique_id(20))
+        gist_id = safe_str(gist_id or unique_id(20))
         lifetime = safe_int(lifetime, -1)
         gist_expires = time.time() + (lifetime * 60) if lifetime != -1 else -1
         expiration = (time_to_datetime(gist_expires)
@@ -133,13 +133,13 b' class GistModel(BaseModel):'
         gist.gist_access_id = gist_id
         gist.gist_owner = owner.user_id
         gist.gist_expires = gist_expires
-        gist.gist_type = safe_unicode(gist_type)
+        gist.gist_type = safe_str(gist_type)
         gist.acl_level = gist_acl_level
         self.sa.add(gist)
         self.sa.flush()
         if gist_type == Gist.GIST_PUBLIC:
             # use DB ID for easy to use GIST ID
-            gist_id = safe_unicode(gist.gist_id)
+            gist_id = safe_str(gist.gist_id)
             gist.gist_access_id = gist_id
             self.sa.add(gist)
 
@@ -152,7 +152,7 b' class GistModel(BaseModel):'
         # now create single multifile commit
         message = 'added file'
         message += 's: ' if len(gist_mapping) > 1 else ': '
-        message += ', '.join([x for x in gist_mapping])
+        message += ', '.join([safe_str(x) for x in gist_mapping])
 
         # fake RhodeCode Repository object
         fake_repo = AttributeDict({
@@ -218,7 +218,7 b' class GistModel(BaseModel):'
 
         message = 'updated file'
         message += 's: ' if len(gist_mapping) > 1 else ': '
-        message += ', '.join([x for x in gist_mapping])
+        message += ', '.join([safe_str(x) for x in gist_mapping])
 
         # fake RhodeCode Repository object
         fake_repo = AttributeDict({

rhodecode/model/integration.py

@@ -28,12 +28,11 b' import logging'
 
 from sqlalchemy import or_, and_
 
-import rhodecode
 from rhodecode import events
 from rhodecode.integrations.types.base import EEIntegration
 from rhodecode.lib.caching_query import FromCache
 from rhodecode.model import BaseModel
-from rhodecode.model.db import Integration, Repository, RepoGroup, true, false, case
+from rhodecode.model.db import Integration, Repository, RepoGroup, true, false, case, null
 from rhodecode.integrations import integration_type_registry
 
 log = logging.getLogger(__name__)
@@ -53,8 +52,7 b' class IntegrationModel(BaseModel):'
                 raise Exception('integration must be int or Instance'
                                 ' of Integration got %s' % type(integration))
 
-    def create(self, IntegrationType, name, enabled, repo, repo_group,
-        child_repos_only, settings):
+    def create(self, IntegrationType, name, enabled, repo, repo_group, child_repos_only, settings):
         """ Create an IntegrationType integration """
         integration = Integration()
         integration.integration_type = IntegrationType.key
@@ -163,15 +161,15 b' class IntegrationModel(BaseModel):'
         )
 
         global_integrations_filter = and_(
-            Integration.repo_id == None,
-            Integration.repo_group_id == None,
+            Integration.repo_id == null(),
+            Integration.repo_group_id == null(),
             Integration.child_repos_only == false(),
         )
 
         if isinstance(event, events.RepoEvent):
             root_repos_integrations_filter = and_(
-                Integration.repo_id == None,
-                Integration.repo_group_id == None,
+                Integration.repo_id == null(),
+                Integration.repo_group_id == null(),
                 Integration.child_repos_only == true(),
             )
 
@@ -225,7 +223,7 b' class IntegrationModel(BaseModel):'
             query = query.order_by(order_by_criterion)
 
             if cache:
-                cache_key = "get_enabled_repo_integrations_%i" % event.repo.repo_id
+                cache_key = f"get_enabled_repo_integrations_{event.repo.repo_id}"
                 query = query.options(
                     FromCache("sql_cache_short", cache_key))
         else:  # only global integrations

rhodecode/model/notification.py

@@ -332,6 +332,7 b" EMAIL_COMMENT_FILE_SUBJECT_TEMPLATE = ''"
 import cssutils
 # hijack css utils logger and replace with ours
 log = logging.getLogger('rhodecode.cssutils.premailer')
+log.setLevel(logging.INFO)
 cssutils.log.setLog(log)
 
 
@@ -377,7 +378,10 b' class EmailNotificationModel(BaseModel):'
             'rhodecode:templates/email_templates/pull_request_update.mako',
     }
 
-    premailer_instance = premailer.Premailer()
+    premailer_instance = premailer.Premailer(
+        #cssutils_logging_handler=log.handlers[0],
+        #cssutils_logging_level=logging.INFO
+    )
 
     def __init__(self):
         """

rhodecode/model/permission.py

@@ -180,16 +180,24 b' class PermissionModel(BaseModel):'
 
     def _make_new_user_perm(self, user, perm_name):
         log.debug('Creating new user permission:%s', perm_name)
+        new_perm = Permission.get_by_key(perm_name)
+        if not new_perm:
+            raise ValueError(f'permission with name {perm_name} not found')
+
         new = UserToPerm()
         new.user = user
-        new.permission = Permission.get_by_key(perm_name)
+        new.permission = new_perm
         return new
 
     def _make_new_user_group_perm(self, user_group, perm_name):
         log.debug('Creating new user group permission:%s', perm_name)
+        new_perm = Permission.get_by_key(perm_name)
+        if not new_perm:
+            raise ValueError(f'permission with name {perm_name} not found')
+
         new = UserGroupToPerm()
         new.users_group = user_group
-        new.permission = Permission.get_by_key(perm_name)
+        new.permission = new_perm
         return new
 
     def _keep_perm(self, perm_name, keep_fields):
@@ -278,10 +286,10 b' class PermissionModel(BaseModel):'
                 raise ValueError('Missing permission for %s' % (_perm_key,))
 
             if obj_type == 'user':
-                p = self._make_new_user_perm(object, perm_value)
+                p = self._make_new_user_perm(to_object, perm_value)
                 self.sa.add(p)
             if obj_type == 'user_group':
-                p = self._make_new_user_group_perm(object, perm_value)
+                p = self._make_new_user_group_perm(to_object, perm_value)
                 self.sa.add(p)
 
     def _set_new_user_perms(self, user, form_result, preserve=None):
@@ -321,8 +329,8 b' class PermissionModel(BaseModel):'
         def _get_group(perm_name):
             return '.'.join(perm_name.split('.')[:1])
 
-        defined_perms_groups = map(
-            _get_group, (x.permission.permission_name for x in obj_perms))
+        defined_perms_groups = list(map(
+            _get_group, (x.permission.permission_name for x in obj_perms)))
         log.debug('GOT ALREADY DEFINED:%s', obj_perms)
 
         if force:

rhodecode/model/pull_request.py

@@ -23,15 +23,16 b''
 pull request model for RhodeCode
 """
 
-
-import json
 import logging
 import os
 
 import datetime
-import urllib.request, urllib.parse, urllib.error
+import urllib.request
+import urllib.parse
+import urllib.error
 import collections
 
+import dataclasses as dataclasses
 from pyramid.threadlocal import get_current_request
 
 from rhodecode.lib.vcs.nodes import FileNode
@@ -40,11 +41,12 b' from rhodecode.lib import helpers as h, '
 from rhodecode.lib import audit_logger
 from collections import OrderedDict
 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
+from rhodecode.lib.ext_json import sjson as json
 from rhodecode.lib.markup_renderer import (
     DEFAULT_COMMENTS_RENDERER, RstTemplateRenderer)
-from rhodecode.lib.utils2 import (
-    safe_unicode, safe_str, md5_safe, AttributeDict, safe_int,
-    get_current_rhodecode_user)
+from rhodecode.lib.hash_utils import md5_safe
+from rhodecode.lib.str_utils import safe_str
+from rhodecode.lib.utils2 import AttributeDict, get_current_rhodecode_user
 from rhodecode.lib.vcs.backends.base import (
     Reference, MergeResponse, MergeFailureReason, UpdateFailureReason,
     TargetRefMissing, SourceRefMissing)
@@ -55,7 +57,7 b' from rhodecode.model import BaseModel'
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.comment import CommentsModel
 from rhodecode.model.db import (
-    aliased, null, lazyload, and_, or_, func, String, cast, PullRequest, PullRequestReviewers, ChangesetStatus,
+    aliased, null, lazyload, and_, or_, select, func, String, cast, PullRequest, PullRequestReviewers, ChangesetStatus,
     PullRequestVersion, ChangesetComment, Repository, RepoReviewRule, User)
 from rhodecode.model.meta import Session
 from rhodecode.model.notification import NotificationModel, \
@@ -116,9 +118,8 b' def get_diff_info('
         source_scm.get_diff(commit1=source_commit, commit2=target_commit,
                             ignore_whitespace=False, context=3)
 
-    diff_processor = diffs.DiffProcessor(
-        vcs_diff, format='newdiff', diff_limit=None,
-        file_limit=None, show_full_diff=True)
+    diff_processor = diffs.DiffProcessor(vcs_diff, diff_format='newdiff',
+                                         diff_limit=0, file_limit=0, show_full_diff=True)
 
     _parsed = diff_processor.prepare()
 
@@ -317,7 +318,7 b' class PullRequestModel(BaseModel):'
         q = PullRequest.query()
 
         if search_q:
-            like_expression = u'%{}%'.format(safe_unicode(search_q))
+            like_expression = u'%{}%'.format(safe_str(search_q))
             q = q.join(User, User.user_id == PullRequest.user_id)
             q = q.filter(or_(
                 cast(PullRequest.pull_request_id, String).ilike(like_expression),
@@ -489,7 +490,7 b' class PullRequestModel(BaseModel):'
             q = q.filter(pull_request_alias.status.in_(statuses))
 
         if search_q:
-            like_expression = u'%{}%'.format(safe_unicode(search_q))
+            like_expression = u'%{}%'.format(safe_str(search_q))
             q = q.join(User, User.user_id == pull_request_alias.user_id)
             q = q.filter(or_(
                 cast(pull_request_alias.pull_request_id, String).ilike(like_expression),
@@ -562,12 +563,14 b' class PullRequestModel(BaseModel):'
         """
         q = PullRequest.query()
         if user_id:
-            reviewers_subquery = Session().query(
-                PullRequestReviewers.pull_request_id).filter(
-                PullRequestReviewers.user_id == user_id).subquery()
+
+            base_query = select(PullRequestReviewers)\
+                .where(PullRequestReviewers.user_id == user_id)\
+                .with_only_columns(PullRequestReviewers.pull_request_id)
+
             user_filter = or_(
                 PullRequest.user_id == user_id,
-                PullRequest.pull_request_id.in_(reviewers_subquery)
+                PullRequest.pull_request_id.in_(base_query)
             )
             q = PullRequest.query().filter(user_filter)
 
@@ -576,7 +579,7 b' class PullRequestModel(BaseModel):'
             q = q.filter(PullRequest.status.in_(statuses))
 
         if query:
-            like_expression = u'%{}%'.format(safe_unicode(query))
+            like_expression = u'%{}%'.format(safe_str(query))
             q = q.join(User, User.user_id == PullRequest.user_id)
             q = q.filter(or_(
                 cast(PullRequest.pull_request_id, String).ilike(like_expression),
@@ -656,7 +659,7 b' class PullRequestModel(BaseModel):'
             q = q.filter(pull_request_alias.status.in_(statuses))
 
         if query:
-            like_expression = u'%{}%'.format(safe_unicode(query))
+            like_expression = u'%{}%'.format(safe_str(query))
             q = q.join(User, User.user_id == pull_request_alias.user_id)
             q = q.filter(or_(
                 cast(pull_request_alias.pull_request_id, String).ilike(like_expression),
@@ -939,7 +942,8 b' class PullRequestModel(BaseModel):'
 
     def merge_repo(self, pull_request, user, extras):
         repo_type = pull_request.source_repo.repo_type
-        log.debug("Merging pull request %s", pull_request.pull_request_id)
+        log.debug("Merging pull request %s", pull_request)
+
         extras['user_agent'] = '{}/internal-merge'.format(repo_type)
         merge_state = self._merge_pull_request(pull_request, user, extras)
         if merge_state.executed:
@@ -952,14 +956,14 b' class PullRequestModel(BaseModel):'
                 user, pull_request)
 
         else:
-            log.warn("Merge failed, not updating the pull request.")
+            log.warning("Merge failed, not updating the pull request.")
         return merge_state
 
     def _merge_pull_request(self, pull_request, user, extras, merge_msg=None):
         target_vcs = pull_request.target_repo.scm_instance()
         source_vcs = pull_request.source_repo.scm_instance()
 
-        message = safe_unicode(merge_msg or vcs_settings.MERGE_MESSAGE_TMPL).format(
+        message = safe_str(merge_msg or vcs_settings.MERGE_MESSAGE_TMPL).format(
             pr_id=pull_request.pull_request_id,
             pr_title=pull_request.title,
             pr_desc=pull_request.description,
@@ -995,6 +999,7 b' class PullRequestModel(BaseModel):'
                 user_name=user_name, user_email=user.email,
                 message=message, use_rebase=use_rebase,
                 close_branch=close_branch)
+
         return merge_state
 
     def _comment_and_close_pr(self, pull_request, user, merge_state, close_msg=None):
@@ -1003,7 +1008,7 b' class PullRequestModel(BaseModel):'
         close_msg = close_msg or 'Pull request merged and closed'
 
         CommentsModel().create(
-            text=safe_unicode(close_msg),
+            text=safe_str(close_msg),
             repo=pull_request.target_repo.repo_id,
             user=user.user_id,
             pull_request=pull_request.pull_request_id,
@@ -1289,9 +1294,10 b' class PullRequestModel(BaseModel):'
             source_repo, source_ref_id, target_ref_id,
             hide_whitespace_changes=hide_whitespace_changes, diff_context=diff_context)
 
-        old_diff_data = diffs.DiffProcessor(old_diff)
+        # NOTE: this was using diff_format='gitdiff'
+        old_diff_data = diffs.DiffProcessor(old_diff, diff_format='newdiff')
         old_diff_data.prepare()
-        new_diff_data = diffs.DiffProcessor(new_diff)
+        new_diff_data = diffs.DiffProcessor(new_diff, diff_format='newdiff')
         new_diff_data.prepare()
 
         return old_diff_data, new_diff_data
@@ -1598,7 +1604,7 b' class PullRequestModel(BaseModel):'
             return None
         else:
             pr_url = urllib.parse.unquote(self.get_url(pull_request, request=request))
-            return safe_unicode('{pr_url}/repository'.format(pr_url=pr_url))
+            return safe_str('{pr_url}/repository'.format(pr_url=pr_url))
 
     def _notify_reviewers(self, pull_request, user_ids, role, user):
         # notification to reviewers/observers
@@ -2032,8 +2038,8 b' class PullRequestModel(BaseModel):'
         _ = translator or get_current_request().translate
 
         commit_id = safe_str(commit_id) if commit_id else None
-        branch = safe_unicode(branch) if branch else None
-        bookmark = safe_unicode(bookmark) if bookmark else None
+        branch = safe_str(branch) if branch else None
+        bookmark = safe_str(bookmark) if bookmark else None
 
         selected = None
 
@@ -2072,8 +2078,8 b' class PullRequestModel(BaseModel):'
                     u'No commit refs could be found matching: {}'.format(ref))
             elif repo.DEFAULT_BRANCH_NAME in repo.branches:
                 selected = u'branch:{}:{}'.format(
-                    safe_unicode(repo.DEFAULT_BRANCH_NAME),
-                    safe_unicode(repo.branches[repo.DEFAULT_BRANCH_NAME])
+                    safe_str(repo.DEFAULT_BRANCH_NAME),
+                    safe_str(repo.branches[repo.DEFAULT_BRANCH_NAME])
                 )
             elif repo.commit_ids:
                 # make the user select in this case
@@ -2113,7 +2119,7 b' class PullRequestModel(BaseModel):'
         log.debug('calculating diff between '
                   'source_ref:%s and target_ref:%s for repo `%s`',
                   target_ref_id, source_ref_id,
-                  safe_unicode(vcs_repo.path))
+                  safe_str(vcs_repo.path))
 
         vcs_diff = vcs_repo.get_diff(
             commit1=target_commit, commit2=source_commit,
@@ -2373,8 +2379,16 b' class MergeCheck(object):'
         return merge_details
 
 
-ChangeTuple = collections.namedtuple(
-    'ChangeTuple', ['added', 'common', 'removed', 'total'])
[e-mail]
+class ChangeTuple:
+    added: list
+    common: list
+    removed: list
+    total: list
 
-FileChangeTuple = collections.namedtuple(
-    'FileChangeTuple', ['added', 'modified', 'removed'])
+
[e-mail]
+class FileChangeTuple:
+    added: list
+    modified: list
+    removed: list

rhodecode/model/repo.py

@@ -26,6 +26,7 b' import traceback'
 import datetime
 
 from pyramid.threadlocal import get_current_request
+from sqlalchemy.orm import aliased
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 from rhodecode import events
@@ -36,7 +37,7 b' from rhodecode.lib import hooks_base'
 from rhodecode.lib.user_log_filter import user_log_filter
 from rhodecode.lib.utils import make_db_config
 from rhodecode.lib.utils2 import (
-    safe_str, safe_unicode, remove_prefix, obfuscate_url_pw,
+    safe_str, remove_prefix, obfuscate_url_pw,
     get_current_rhodecode_user, safe_int, action_logger_generic)
 from rhodecode.lib.vcs.backends import get_backend
 from rhodecode.model import BaseModel
@@ -78,7 +79,7 b' class RepoModel(BaseModel):'
         repo_to_perm.permission = Permission.get_by_key(default_perm)
 
         repo_to_perm.repository = repository
-        repo_to_perm.user_id = def_user.user_id
+        repo_to_perm.user = def_user
 
         return repo_to_perm
 
@@ -112,7 +113,7 b' class RepoModel(BaseModel):'
     def _extract_id_from_repo_name(self, repo_name):
         if repo_name.startswith('/'):
             repo_name = repo_name.lstrip('/')
-        by_id_match = re.match(r'^_(\d{1,})', repo_name)
+        by_id_match = re.match(r'^_(\d+)', repo_name)
         if by_id_match:
             return by_id_match.groups()[0]
 
@@ -138,7 +139,7 b' class RepoModel(BaseModel):'
 
     def get_repos_for_root(self, root, traverse=False):
         if traverse:
-            like_expression = u'{}%'.format(safe_unicode(root))
+            like_expression = u'{}%'.format(safe_str(root))
             repos = Repository.query().filter(
                 Repository.repo_name.like(like_expression)).all()
         else:
@@ -209,12 +210,12 b' class RepoModel(BaseModel):'
         def quick_menu(repo_name):
             return _render('quick_menu', repo_name)
 
-        def repo_lnk(name, rtype, rstate, private, archived, fork_of):
+        def repo_lnk(name, rtype, rstate, private, archived, fork_repo_name):
             if short_name is not None:
                 short_name_var = short_name
             else:
                 short_name_var = not admin
-            return _render('repo_name', name, rtype, rstate, private, archived, fork_of,
+            return _render('repo_name', name, rtype, rstate, private, archived, fork_repo_name,
                            short_name=short_name_var, admin=False)
 
         def last_change(last_change):
@@ -259,7 +260,7 b' class RepoModel(BaseModel):'
                 "menu": quick_menu(repo.repo_name),
 
                 "name": repo_lnk(repo.repo_name, repo.repo_type, repo.repo_state,
-                                 repo.private, repo.archived, repo.fork),
+                                 repo.private, repo.archived, repo.fork_repo_name),
 
                 "desc": desc(h.escape(repo.description)),
 
@@ -268,7 +269,7 b' class RepoModel(BaseModel):'
                 "last_changeset": last_rev(repo.repo_name, changeset_cache),
                 "last_changeset_raw": changeset_cache.get('revision'),
 
-                "owner": user_profile(repo.User.username),
+                "owner": user_profile(repo.owner_username),
 
                 "state": state(repo.repo_state),
                 "rss": rss_lnk(repo.repo_name),
@@ -309,6 +310,8 b' class RepoModel(BaseModel):'
             ) \
             .count()
 
+        RepoFork = aliased(Repository)
+        OwnerUser = aliased(User)
         base_q = Session.query(
             Repository.repo_id,
             Repository.repo_name,
@@ -317,18 +320,18 b' class RepoModel(BaseModel):'
             Repository.repo_state,
             Repository.private,
             Repository.archived,
-            Repository.fork,
             Repository.updated_on,
             Repository._changeset_cache,
-            User,
+            RepoFork.repo_name.label('fork_repo_name'),
+            OwnerUser.username.label('owner_username'),
             ) \
             .filter(Repository.group_id == repo_group_id) \
             .filter(or_(
                 # generate multiple IN to fix limitation problems
                 *in_filter_generator(Repository.repo_id, allowed_ids))
             ) \
-            .join(User, User.user_id == Repository.user_id) \
-            .group_by(Repository, User)
+            .outerjoin(RepoFork, Repository.fork_id == RepoFork.repo_id) \
+            .join(OwnerUser, Repository.user_id == OwnerUser.user_id)
 
         repos_data_total_filtered_count = base_q.count()
 
@@ -515,8 +518,8 b' class RepoModel(BaseModel):'
         landing_rev = landing_rev or default_landing_ref
 
         try:
-            repo_name = safe_unicode(repo_name)
-            description = safe_unicode(description)
+            repo_name = safe_str(repo_name)
+            description = safe_str(description)
             # repo name is just a name of repository
             # while repo_name_full is a full qualified name that is combined
             # with name and path of group
@@ -979,14 +982,14 b' class RepoModel(BaseModel):'
 
         # check if this path is not a repository
         if is_valid_repo(repo_path, self.repos_path):
-            raise Exception('This path %s is a valid repository' % repo_path)
+            raise Exception(f'This path {repo_path} is a valid repository')
 
         # check if this path is a group
         if is_valid_repo_group(repo_path, self.repos_path):
-            raise Exception('This path %s is a valid group' % repo_path)
+            raise Exception(f'This path {repo_path} is a valid group')
 
         log.info('creating repo %s in %s from url: `%s`',
-                 repo_name, safe_unicode(repo_path),
+                 repo_name, safe_str(repo_path),
                  obfuscate_url_pw(clone_uri))
 
         backend = get_backend(repo_type)
@@ -1016,7 +1019,7 b' class RepoModel(BaseModel):'
             repo.install_hooks()
 
         log.debug('Created repo %s with %s backend',
-                  safe_unicode(repo_name), safe_unicode(repo_type))
+                  safe_str(repo_name), safe_str(repo_type))
         return repo
 
     def _rename_filesystem_repo(self, old, new):
@@ -1038,8 +1041,8 b' class RepoModel(BaseModel):'
 
     def _delete_filesystem_repo(self, repo):
         """
-        removes repo from filesystem, the removal is acctually made by
-        added rm__ prefix into dir, and rename internat .hg/.git dirs so this
+        removes repo from filesystem, the removal is actually made by
+        added rm__ prefix into dir, and rename internal .hg/.git dirs so this
         repository is no longer valid for rhodecode, can be undeleted later on
         by reverting the renames on this repository
 
@@ -1047,7 +1050,7 b' class RepoModel(BaseModel):'
         """
         rm_path = os.path.join(self.repos_path, repo.repo_name)
         repo_group = repo.group
-        log.info("Removing repository %s", rm_path)
+        log.info("delete_filesystem_repo: removing repository %s", rm_path)
         # disable hg/git internal that it doesn't get detected as repo
         alias = repo.repo_type
 
@@ -1094,19 +1097,19 b' class ReadmeFinder:'
     path_re = re.compile(r'^docs?', re.IGNORECASE)
 
     default_priorities = {
-        None: 0,
-        '.text': 2,
-        '.txt': 3,
-        '.rst': 1,
-        '.rest': 2,
-        '.md': 1,
-        '.mkdn': 2,
-        '.mdown': 3,
+        None:        0,
+        '.rst':      1,
+        '.md':       1,
+        '.rest':     2,
+        '.mkdn':     2,
+        '.text':     2,
+        '.txt':      3,
+        '.mdown':    3,
         '.markdown': 4,
     }
 
     path_priority = {
-        'doc': 0,
+        'doc':  0,
         'docs': 1,
     }
 
@@ -1122,7 +1125,7 b' class ReadmeFinder:'
         self._renderer_extensions = self.RENDERER_TO_EXTENSION.get(
             default_renderer, [])
 
-    def search(self, commit, path=u'/'):
+    def search(self, commit, path='/'):
         """
         Find a readme in the given `commit`.
         """

rhodecode/model/repo_group.py

@@ -131,7 +131,7 b' class RepoGroupModel(BaseModel):'
         repo_group_to_perm.permission = Permission.get_by_key(default_perm)
 
         repo_group_to_perm.group = new_group
-        repo_group_to_perm.user_id = def_user.user_id
+        repo_group_to_perm.user = def_user
         return repo_group_to_perm
 
     def _get_group_name_and_parent(self, group_name_full, repo_in_path=False,
@@ -780,10 +780,10 b' class RepoGroupModel(BaseModel):'
             }
             if admin:
                 repo_count = group.repositories.count()
-                children_groups = map(
-                    h.safe_unicode,
+                children_groups = list(map(
+                    h.safe_str,
                     itertools.chain((g.name for g in group.parents),
-                                    (x.name for x in [group])))
+                                    (x.name for x in [group]))))
                 row.update({
                     "action": repo_group_actions(
                         group.group_id, group.group_name, repo_count),

rhodecode/model/scm.py

@@ -30,6 +30,7 b' from sqlalchemy import func'
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 import rhodecode
+from rhodecode.lib.str_utils import safe_bytes
 from rhodecode.lib.vcs import get_backend
 from rhodecode.lib.vcs.exceptions import RepositoryError, NodeNotChangedError
 from rhodecode.lib.vcs.nodes import FileNode
@@ -42,7 +43,7 b' from rhodecode.lib.exceptions import Non'
 from rhodecode.lib import hooks_utils
 from rhodecode.lib.utils import (
     get_filesystem_repos, make_db_config)
-from rhodecode.lib.utils2 import (safe_str, safe_unicode)
+from rhodecode.lib.str_utils import safe_str
 from rhodecode.lib.system_info import get_system_info
 from rhodecode.model import BaseModel
 from rhodecode.model.db import (
@@ -284,8 +285,7 b' class ScmModel(BaseModel):'
             repo.update_commit_cache(config=config, cs_cache=None)
             if delete:
                 cache_namespace_uid = 'cache_repo.{}'.format(repo_id)
-                rc_cache.clear_cache_namespace(
-                    'cache_repo', cache_namespace_uid, invalidate=True)
+                rc_cache.clear_cache_namespace('cache_repo', cache_namespace_uid, method=rc_cache.CLEAR_INVALIDATE)
 
     def toggle_following_repo(self, follow_repo_id, user_id):
 
@@ -443,25 +443,18 b' class ScmModel(BaseModel):'
             raise
 
     def commit_change(self, repo, repo_name, commit, user, author, message,
-                      content, f_path):
+                      content: bytes, f_path: bytes):
         """
         Commits changes
-
-        :param repo: SCM instance
-
         """
         user = self._get_user(user)
 
-        # decoding here will force that we have proper encoded values
-        # in any other case this will throw exceptions and deny commit
-        content = safe_str(content)
-        path = safe_str(f_path)
         # message and author needs to be unicode
         # proper backend should then translate that into required type
-        message = safe_unicode(message)
-        author = safe_unicode(author)
+        message = safe_str(message)
+        author = safe_str(author)
         imc = repo.in_memory_commit
-        imc.change(FileNode(path, content, mode=commit.get_file_mode(f_path)))
+        imc.change(FileNode(f_path, content, mode=commit.get_file_mode(f_path)))
         try:
             # TODO: handle pre-push action !
             tip = imc.commit(
@@ -480,9 +473,9 b' class ScmModel(BaseModel):'
             repo_name=repo_name, repo_type=repo.alias, commit_ids=[tip.raw_id])
         return tip
 
-    def _sanitize_path(self, f_path):
-        if f_path.startswith('/') or f_path.startswith('./') or '../' in f_path:
-            raise NonRelativePathError('%s is not an relative path' % f_path)
+    def _sanitize_path(self, f_path: bytes):
+        if f_path.startswith(b'/') or f_path.startswith(b'./') or b'../' in f_path:
+            raise NonRelativePathError(b'%b is not an relative path' % f_path)
         if f_path:
             f_path = os.path.normpath(f_path)
         return f_path
@@ -531,15 +524,24 b' class ScmModel(BaseModel):'
         """
         _files = list()
         _dirs = list()
+
         try:
             _repo = self._get_repo(repo_name)
             commit = _repo.scm_instance().get_commit(commit_id=commit_id)
             root_path = root_path.lstrip('/')
-            for __, dirs, files in commit.walk(root_path):
+
+            # get RootNode, inject pre-load options before walking
+            top_node = commit.get_node(root_path)
+            extended_info_pre_load = []
+            if extended_info:
+                extended_info_pre_load += ['md5']
+            top_node.default_pre_load = ['is_binary', 'size'] + extended_info_pre_load
+
+            for __, dirs, files in commit.walk(top_node):
 
                 for f in files:
                     _content = None
-                    _data = f_name = f.unicode_path
+                    _data = f_name = f.str_path
 
                     if not flat:
                         _data = {
@@ -561,7 +563,7 b' class ScmModel(BaseModel):'
                                                and f.size > max_file_bytes)
                             full_content = None
                             if not f.is_binary and not over_size_limit:
-                                full_content = safe_str(f.content)
+                                full_content = f.str_content
 
                             _data.update({
                                 "content": full_content,
@@ -569,7 +571,7 b' class ScmModel(BaseModel):'
                     _files.append(_data)
 
                 for d in dirs:
-                    _data = d_name = d.unicode_path
+                    _data = d_name = d.str_path
                     if not flat:
                         _data = {
                             "name": h.escape(d_name),
@@ -577,10 +579,10 b' class ScmModel(BaseModel):'
                             }
                     if extended_info:
                         _data.update({
-                            "md5": None,
-                            "binary": None,
-                            "size": None,
-                            "extension": None,
+                            "md5": "",
+                            "binary": False,
+                            "size": 0,
+                            "extension": "",
                         })
                     if content:
                         _data.update({
@@ -609,7 +611,7 b' class ScmModel(BaseModel):'
                 for f in files:
 
                     _data = {
-                        "name": h.escape(f.unicode_path),
+                        "name": h.escape(f.str_path),
                         "type": "file",
                         }
 
@@ -618,7 +620,7 b' class ScmModel(BaseModel):'
                 for d in dirs:
 
                     _data = {
-                        "name": h.escape(d.unicode_path),
+                        "name": h.escape(d.str_path),
                         "type": "dir",
                         }
 
@@ -634,6 +636,7 b' class ScmModel(BaseModel):'
         """
         retrieve single node from commit
         """
+
         try:
 
             _repo = self._get_repo(repo_name)
@@ -644,7 +647,7 b' class ScmModel(BaseModel):'
                 raise RepositoryError('The given path is a directory')
 
             _content = None
-            f_name = file_node.unicode_path
+            f_name = file_node.str_path
 
             file_data = {
                 "name": h.escape(f_name),
@@ -677,7 +680,7 b' class ScmModel(BaseModel):'
                 full_content = None
                 all_lines = 0
                 if not file_node.is_binary and not over_size_limit:
-                    full_content = safe_unicode(file_node.content)
+                    full_content = safe_str(file_node.content)
                     all_lines, empty_lines = file_node.count_lines(full_content)
 
                 file_data.update({
@@ -693,7 +696,7 b' class ScmModel(BaseModel):'
                 full_content = None
                 all_lines = 0
                 if not is_binary and not over_size_limit:
-                    full_content = safe_unicode(_content)
+                    full_content = safe_str(_content)
                     all_lines, empty_lines = file_node.count_lines(full_content)
 
                 file_data.update({
@@ -718,12 +721,15 b' class ScmModel(BaseModel):'
             _repo = self._get_repo(repo_name)
             commit = _repo.scm_instance().get_commit(commit_id=commit_id)
             root_path = root_path.lstrip('/')
-            for __, dirs, files in commit.walk(root_path):
+            top_node = commit.get_node(root_path)
+            top_node.default_pre_load = []
+
+            for __, dirs, files in commit.walk(top_node):
 
                 for f in files:
                     is_binary, md5, size, _content = f.metadata_uncached()
                     _data = {
-                        "name": f.unicode_path,
+                        "name": f.str_path,
                         "md5": md5,
                         "extension": f.extension,
                         "binary": is_binary,
@@ -759,26 +765,9 b' class ScmModel(BaseModel):'
         user = self._get_user(user)
         scm_instance = repo.scm_instance(cache=False)
 
-        processed_nodes = []
-        for f_path in nodes:
-            f_path = self._sanitize_path(f_path)
-            content = nodes[f_path]['content']
-            f_path = safe_str(f_path)
-            # decoding here will force that we have proper encoded values
-            # in any other case this will throw exceptions and deny commit
-            if isinstance(content, (str,)):
-                content = safe_str(content)
-            elif isinstance(content, (file, cStringIO.OutputType,)):
-                content = content.read()
-            else:
-                raise Exception('Content is of unrecognized type %s' % (
-                    type(content)
-                ))
-            processed_nodes.append((f_path, content))
-
-        message = safe_unicode(message)
+        message = safe_str(message)
         commiter = user.full_contact
-        author = safe_unicode(author) if author else commiter
+        author = safe_str(author) if author else commiter
 
         imc = scm_instance.in_memory_commit
 
@@ -786,13 +775,39 b' class ScmModel(BaseModel):'
             parent_commit = EmptyCommit(alias=scm_instance.alias)
 
         if isinstance(parent_commit, EmptyCommit):
-            # EmptyCommit means we we're editing empty repository
+            # EmptyCommit means we're editing empty repository
             parents = None
         else:
             parents = [parent_commit]
+
+        upload_file_types = (io.BytesIO, io.BufferedRandom)
+        processed_nodes = []
+        for filename, content_dict in nodes.items():
+            if not isinstance(filename, bytes):
+                raise ValueError(f'filename key in nodes needs to be bytes , or {upload_file_types}')
+            content = content_dict['content']
+            if not isinstance(content, upload_file_types + (bytes,)):
+                raise ValueError('content key value in nodes needs to be bytes')
+
+        for f_path in nodes:
+            f_path = self._sanitize_path(f_path)
+            content = nodes[f_path]['content']
+
+            # decoding here will force that we have proper encoded values
+            # in any other case this will throw exceptions and deny commit
+
+            if isinstance(content, bytes):
+                pass
+            elif isinstance(content, upload_file_types):
+                content = content.read()
+            else:
+                raise Exception(f'Content is of unrecognized type {type(content)}, expected {upload_file_types}')
+            processed_nodes.append((f_path, content))
+
         # add multiple nodes
         for path, content in processed_nodes:
             imc.add(FileNode(path, content=content))
+
         # TODO: handle pre push scenario
         tip = imc.commit(message=message,
                          author=author,
@@ -813,9 +828,9 b' class ScmModel(BaseModel):'
         user = self._get_user(user)
         scm_instance = repo.scm_instance(cache=False)
 
-        message = safe_unicode(message)
+        message = safe_str(message)
         commiter = user.full_contact
-        author = safe_unicode(author) if author else commiter
+        author = safe_str(author) if author else commiter
 
         imc = scm_instance.in_memory_commit
 
@@ -897,14 +912,14 b' class ScmModel(BaseModel):'
         processed_nodes = []
         for f_path in nodes:
             f_path = self._sanitize_path(f_path)
-            # content can be empty but for compatabilty it allows same dicts
+            # content can be empty but for compatibility it allows same dicts
             # structure as add_nodes
             content = nodes[f_path].get('content')
-            processed_nodes.append((f_path, content))
+            processed_nodes.append((safe_bytes(f_path), content))
 
-        message = safe_unicode(message)
+        message = safe_str(message)
         commiter = user.full_contact
-        author = safe_unicode(author) if author else commiter
+        author = safe_str(author) if author else commiter
 
         imc = scm_instance.in_memory_commit
 
@@ -994,7 +1009,7 b' class ScmModel(BaseModel):'
         choices = [default_landing_ref]
 
         # branches
-        branch_group = [(u'branch:%s' % safe_unicode(b), safe_unicode(b)) for b in repo.branches]
+        branch_group = [(f'branch:{safe_str(b)}', safe_str(b)) for b in repo.branches]
         if not branch_group:
             # new repo, or without maybe a branch?
             branch_group = default_ref_options
@@ -1006,7 +1021,7 b' class ScmModel(BaseModel):'
         # bookmarks for HG
         if repo.alias == 'hg':
             bookmarks_group = (
-                [(u'book:%s' % safe_unicode(b), safe_unicode(b))
+                [(f'book:{safe_str(b)}', safe_str(b))
                     for b in repo.bookmarks],
                 _("Bookmarks"))
             ref_options.append(bookmarks_group)
@@ -1014,7 +1029,7 b' class ScmModel(BaseModel):'
 
         # tags
         tags_group = (
-            [(u'tag:%s' % safe_unicode(t), safe_unicode(t))
+            [(f'tag:{safe_str(t)}', safe_str(t))
                 for t in repo.tags],
             _("Tags"))
         ref_options.append(tags_group)

rhodecode/model/settings.py

@@ -19,16 +19,16 b''
 
 import os
 import re
-import hashlib
 import logging
 import time
 import functools
 import bleach
 from collections import namedtuple
 
-from pyramid.threadlocal import get_current_request, get_current_registry
+from pyramid.threadlocal import get_current_request
 
 from rhodecode.lib import rc_cache
+from rhodecode.lib.hash_utils import sha1_safe
 from rhodecode.lib.utils2 import (
     Optional, AttributeDict, safe_str, remove_prefix, str2bool)
 from rhodecode.lib.vcs.backends import base
@@ -132,10 +132,9 b' class SettingsModel(BaseModel):'
         if not key:
             # keys are unique so they need appended info
             if self.repo:
-                key = hashlib.sha1(
-                    '{}{}{}'.format(section, val, repository_id)).hexdigest()
+                key = sha1_safe(f'{section}{val}{repository_id}')
             else:
-                key = hashlib.sha1('{}{}'.format(section, val)).hexdigest()
+                key = sha1_safe(f'{section}{val}')
 
         new_ui.ui_key = key
 
@@ -212,30 +211,20 b' class SettingsModel(BaseModel):'
 
     def get_cache_region(self):
         repo = self._get_repo(self.repo) if self.repo else None
-        cache_key = "repo.{}".format(repo.repo_id) if repo else "general_settings"
-        cache_namespace_uid = 'cache_settings.{}'.format(cache_key)
+        cache_key = f"repo.{repo.repo_id}" if repo else "repo.ALL"
+        cache_namespace_uid = f'cache_settings.{cache_key}'
         region = rc_cache.get_or_create_region('cache_general', cache_namespace_uid)
-        return region, cache_key
-
-    def invalidate_settings_cache(self):
-        region, cache_key = self.get_cache_region()
-        log.debug('Invalidation cache region %s for cache_key: %s', region, cache_key)
-        region.invalidate()
+        return region, cache_namespace_uid
 
-    def get_all_settings(self, cache=False, from_request=True):
-        # defines if we use GLOBAL, or PER_REPO
-        repo = self._get_repo(self.repo) if self.repo else None
+    def invalidate_settings_cache(self, hard=False):
+        region, namespace_key = self.get_cache_region()
+        log.debug('Invalidation cache [%s] region %s for cache_key: %s',
+                  'invalidate_settings_cache', region, namespace_key)
 
-        # initially try the requests context, this is the fastest
-        # we only fetch global config
-        if from_request:
-            request = get_current_request()
+        # we use hard cleanup if invalidation is sent
+        rc_cache.clear_cache_namespace(region, namespace_key, method=rc_cache.CLEAR_DELETE)
 
-            if request and not repo and hasattr(request, 'call_context') and hasattr(request.call_context, 'rc_config'):
-                rc_config = request.call_context.rc_config
-                if rc_config:
-                    return rc_config
-
+    def get_cache_call_method(self, cache=True):
         region, cache_key = self.get_cache_region()
 
         @region.conditional_cache_on_arguments(condition=cache)
@@ -245,10 +234,28 b' class SettingsModel(BaseModel):'
                 raise Exception('Could not get application settings !')
 
             settings = {
-                'rhodecode_' + res.app_settings_name: res.app_settings_value
+                f'rhodecode_{res.app_settings_name}': res.app_settings_value
                 for res in q
             }
             return settings
+        return _get_all_settings
+
+    def get_all_settings(self, cache=False, from_request=True):
+        # defines if we use GLOBAL, or PER_REPO
+        repo = self._get_repo(self.repo) if self.repo else None
+
+        # initially try the requests context, this is the fastest
+        # we only fetch global config, NOT for repo-specific
+        if from_request and not repo:
+            request = get_current_request()
+
+            if request and hasattr(request, 'call_context') and hasattr(request.call_context, 'rc_config'):
+                rc_config = request.call_context.rc_config
+                if rc_config:
+                    return rc_config
+
+        _region, cache_key = self.get_cache_region()
+        _get_all_settings = self.get_cache_call_method(cache=cache)
 
         start = time.time()
         result = _get_all_settings('rhodecode_settings', cache_key)
@@ -318,8 +325,7 b' class SettingsModel(BaseModel):'
     def list_enabled_social_plugins(self, settings):
         enabled = []
         for plug in SOCIAL_PLUGINS_LIST:
-            if str2bool(settings.get('rhodecode_auth_{}_enabled'.format(plug)
-                                     )):
+            if str2bool(settings.get(f'rhodecode_auth_{plug}_enabled')):
                 enabled.append(plug)
         return enabled
 

rhodecode/model/ssh_key.py

@@ -28,6 +28,7 b' from cryptography.hazmat.primitives.asym'
 from cryptography.hazmat.primitives import serialization as crypto_serialization
 from cryptography.hazmat.backends import default_backend as crypto_default_backend
 
+from rhodecode.lib.str_utils import safe_bytes, safe_str
 from rhodecode.model import BaseModel
 from rhodecode.model.db import UserSshKeys
 from rhodecode.model.meta import Session
@@ -85,10 +86,13 b' class SshKeyModel(BaseModel):'
             crypto_serialization.Encoding.PEM,
             private_format,
             crypto_serialization.NoEncryption())
+        private_key = safe_str(private_key)
+
         public_key = key.public_key().public_bytes(
             crypto_serialization.Encoding.OpenSSH,
             crypto_serialization.PublicFormat.OpenSSH
         )
+        public_key = safe_str(public_key)
 
         if comment:
             public_key = public_key + " " + comment

rhodecode/model/update.py

@@ -19,7 +19,9 b''
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
-import urllib.request, urllib.error, urllib.parse
+import urllib.request
+import urllib.error
+import urllib.parse
 from packaging.version import Version
 
 import rhodecode

rhodecode/model/user.py

@@ -32,8 +32,9 b' from sqlalchemy.exc import DatabaseError'
 from rhodecode import events
 from rhodecode.lib.user_log_filter import user_log_filter
 from rhodecode.lib.utils2 import (
-    safe_unicode, get_current_rhodecode_user, action_logger_generic,
+    get_current_rhodecode_user, action_logger_generic,
     AttributeDict, str2bool)
+from rhodecode.lib.str_utils import safe_str
 from rhodecode.lib.exceptions import (
     DefaultUserException, UserOwnsReposException, UserOwnsRepoGroupsException,
     UserOwnsUserGroupsException, NotAllowedToCreateUserError,
@@ -87,7 +88,7 b' class UserModel(BaseModel):'
             query = query.filter(User.active == true())
 
         if name_contains:
-            ilike_expression = u'%{}%'.format(safe_unicode(name_contains))
+            ilike_expression = u'%{}%'.format(safe_str(name_contains))
             query = query.filter(
                 or_(
                     User.name.ilike(ilike_expression),
@@ -358,8 +359,8 b' class UserModel(BaseModel):'
             new_user.admin = admin
             new_user.email = email
             new_user.active = active
-            new_user.extern_name = safe_unicode(extern_name)
-            new_user.extern_type = safe_unicode(extern_type)
+            new_user.extern_name = safe_str(extern_name)
+            new_user.extern_type = safe_str(extern_type)
             new_user.name = firstname
             new_user.lastname = lastname
             new_user.description = description
@@ -533,7 +534,7 b' class UserModel(BaseModel):'
 
             left_overs = False
 
-        # if nothing is done we have left overs left
+        # if nothing is done we have leftovers left
         return left_overs
 
     def _handle_user_artifacts(self, username, artifacts, handle_user,
@@ -909,8 +910,8 b' class UserModel(BaseModel):'
             ip_range = ip_range.strip()
             if '-' in ip_range:
                 start_ip, end_ip = ip_range.split('-', 1)
-                start_ip = ipaddress.ip_address(safe_unicode(start_ip.strip()))
-                end_ip = ipaddress.ip_address(safe_unicode(end_ip.strip()))
+                start_ip = ipaddress.ip_address(safe_str(start_ip.strip()))
+                end_ip = ipaddress.ip_address(safe_str(end_ip.strip()))
                 parsed_ip_range = []
 
                 for index in range(int(start_ip), int(end_ip) + 1):

rhodecode/model/user_group.py

@@ -21,7 +21,7 b''
 import logging
 import traceback
 
-from rhodecode.lib.utils2 import safe_str, safe_unicode
+from rhodecode.lib.utils2 import safe_str
 from rhodecode.lib.exceptions import (
     UserGroupAssignedException, RepoGroupAssignmentError)
 from rhodecode.lib.utils2 import (
@@ -58,7 +58,7 b' class UserGroupModel(BaseModel):'
         user_group_to_perm.permission = Permission.get_by_key(default_perm)
 
         user_group_to_perm.user_group = user_group
-        user_group_to_perm.user_id = def_user.user_id
+        user_group_to_perm.user = def_user
         return user_group_to_perm
 
     def update_permissions(
@@ -710,7 +710,7 b' class UserGroupModel(BaseModel):'
             query = query.filter(UserGroup.users_group_active == true())
 
         if name_contains:
-            ilike_expression = u'%{}%'.format(safe_unicode(name_contains))
+            ilike_expression = u'%{}%'.format(safe_str(name_contains))
             query = query.filter(
                 UserGroup.users_group_name.ilike(ilike_expression))\
                 .order_by(func.length(UserGroup.users_group_name))\