# HG changeset patch # User RhodeCode Admin # Date 2023-07-18 09:31:01 # Node ID 175fe6cbec55c1dfe49072670a5645cca74cc705 # Parent aafa637aa364f076ff04e5467ed13edae42c636f models: major update for python3, - imports fixes - str vs unicode fixes - found issues with logic on tests runs diff --git a/rhodecode/model/changeset_status.py b/rhodecode/model/changeset_status.py --- a/rhodecode/model/changeset_status.py +++ b/rhodecode/model/changeset_status.py @@ -178,13 +178,14 @@ class ChangesetStatusModel(BaseModel): """ def group_rule(element): - review_obj = element[0] - rule_data = review_obj.rule_user_group_data() + _review_obj = element[0] + rule_data = _review_obj.rule_user_group_data() if rule_data and rule_data['id']: return rule_data['id'] + # don't return None, as we cant compare this + return 0 - voting_groups = itertools.groupby( - sorted(statuses_by_reviewers, key=group_rule), group_rule) + voting_groups = itertools.groupby(sorted(statuses_by_reviewers, key=group_rule), group_rule) voting_by_groups = [(x, list(y)) for x, y in voting_groups] diff --git a/rhodecode/model/comment.py b/rhodecode/model/comment.py --- a/rhodecode/model/comment.py +++ b/rhodecode/model/comment.py @@ -296,9 +296,6 @@ class CommentsModel(BaseModel): :param extra_recipients: list of extra users to be added to recipients """ - if not text: - log.warning('Missing text for comment, skipping...') - return request = get_current_request() _ = request.translate diff --git a/rhodecode/model/gist.py b/rhodecode/model/gist.py --- a/rhodecode/model/gist.py +++ b/rhodecode/model/gist.py @@ -31,7 +31,7 @@ import shutil from pyramid.threadlocal import get_current_request from rhodecode.lib.utils2 import ( - safe_unicode, unique_id, safe_int, time_to_datetime, AttributeDict) + unique_id, safe_int, safe_str, time_to_datetime, AttributeDict) from rhodecode.lib.ext_json import json from rhodecode.lib.vcs import VCSError from rhodecode.model import BaseModel @@ -121,7 +121,7 @@ class GistModel(BaseModel): :param gist_acl_level: acl level for this gist """ owner = self._get_user(owner) - gist_id = safe_unicode(gist_id or unique_id(20)) + gist_id = safe_str(gist_id or unique_id(20)) lifetime = safe_int(lifetime, -1) gist_expires = time.time() + (lifetime * 60) if lifetime != -1 else -1 expiration = (time_to_datetime(gist_expires) @@ -133,13 +133,13 @@ class GistModel(BaseModel): gist.gist_access_id = gist_id gist.gist_owner = owner.user_id gist.gist_expires = gist_expires - gist.gist_type = safe_unicode(gist_type) + gist.gist_type = safe_str(gist_type) gist.acl_level = gist_acl_level self.sa.add(gist) self.sa.flush() if gist_type == Gist.GIST_PUBLIC: # use DB ID for easy to use GIST ID - gist_id = safe_unicode(gist.gist_id) + gist_id = safe_str(gist.gist_id) gist.gist_access_id = gist_id self.sa.add(gist) @@ -152,7 +152,7 @@ class GistModel(BaseModel): # now create single multifile commit message = 'added file' message += 's: ' if len(gist_mapping) > 1 else ': ' - message += ', '.join([x for x in gist_mapping]) + message += ', '.join([safe_str(x) for x in gist_mapping]) # fake RhodeCode Repository object fake_repo = AttributeDict({ @@ -218,7 +218,7 @@ class GistModel(BaseModel): message = 'updated file' message += 's: ' if len(gist_mapping) > 1 else ': ' - message += ', '.join([x for x in gist_mapping]) + message += ', '.join([safe_str(x) for x in gist_mapping]) # fake RhodeCode Repository object fake_repo = AttributeDict({ diff --git a/rhodecode/model/integration.py b/rhodecode/model/integration.py --- a/rhodecode/model/integration.py +++ b/rhodecode/model/integration.py @@ -28,12 +28,11 @@ import logging from sqlalchemy import or_, and_ -import rhodecode from rhodecode import events from rhodecode.integrations.types.base import EEIntegration from rhodecode.lib.caching_query import FromCache from rhodecode.model import BaseModel -from rhodecode.model.db import Integration, Repository, RepoGroup, true, false, case +from rhodecode.model.db import Integration, Repository, RepoGroup, true, false, case, null from rhodecode.integrations import integration_type_registry log = logging.getLogger(__name__) @@ -53,8 +52,7 @@ class IntegrationModel(BaseModel): raise Exception('integration must be int or Instance' ' of Integration got %s' % type(integration)) - def create(self, IntegrationType, name, enabled, repo, repo_group, - child_repos_only, settings): + def create(self, IntegrationType, name, enabled, repo, repo_group, child_repos_only, settings): """ Create an IntegrationType integration """ integration = Integration() integration.integration_type = IntegrationType.key @@ -163,15 +161,15 @@ class IntegrationModel(BaseModel): ) global_integrations_filter = and_( - Integration.repo_id == None, - Integration.repo_group_id == None, + Integration.repo_id == null(), + Integration.repo_group_id == null(), Integration.child_repos_only == false(), ) if isinstance(event, events.RepoEvent): root_repos_integrations_filter = and_( - Integration.repo_id == None, - Integration.repo_group_id == None, + Integration.repo_id == null(), + Integration.repo_group_id == null(), Integration.child_repos_only == true(), ) @@ -225,7 +223,7 @@ class IntegrationModel(BaseModel): query = query.order_by(order_by_criterion) if cache: - cache_key = "get_enabled_repo_integrations_%i" % event.repo.repo_id + cache_key = f"get_enabled_repo_integrations_{event.repo.repo_id}" query = query.options( FromCache("sql_cache_short", cache_key)) else: # only global integrations diff --git a/rhodecode/model/notification.py b/rhodecode/model/notification.py --- a/rhodecode/model/notification.py +++ b/rhodecode/model/notification.py @@ -332,6 +332,7 @@ EMAIL_COMMENT_FILE_SUBJECT_TEMPLATE = '' import cssutils # hijack css utils logger and replace with ours log = logging.getLogger('rhodecode.cssutils.premailer') +log.setLevel(logging.INFO) cssutils.log.setLog(log) @@ -377,7 +378,10 @@ class EmailNotificationModel(BaseModel): 'rhodecode:templates/email_templates/pull_request_update.mako', } - premailer_instance = premailer.Premailer() + premailer_instance = premailer.Premailer( + #cssutils_logging_handler=log.handlers[0], + #cssutils_logging_level=logging.INFO + ) def __init__(self): """ diff --git a/rhodecode/model/permission.py b/rhodecode/model/permission.py --- a/rhodecode/model/permission.py +++ b/rhodecode/model/permission.py @@ -180,16 +180,24 @@ class PermissionModel(BaseModel): def _make_new_user_perm(self, user, perm_name): log.debug('Creating new user permission:%s', perm_name) + new_perm = Permission.get_by_key(perm_name) + if not new_perm: + raise ValueError(f'permission with name {perm_name} not found') + new = UserToPerm() new.user = user - new.permission = Permission.get_by_key(perm_name) + new.permission = new_perm return new def _make_new_user_group_perm(self, user_group, perm_name): log.debug('Creating new user group permission:%s', perm_name) + new_perm = Permission.get_by_key(perm_name) + if not new_perm: + raise ValueError(f'permission with name {perm_name} not found') + new = UserGroupToPerm() new.users_group = user_group - new.permission = Permission.get_by_key(perm_name) + new.permission = new_perm return new def _keep_perm(self, perm_name, keep_fields): @@ -278,10 +286,10 @@ class PermissionModel(BaseModel): raise ValueError('Missing permission for %s' % (_perm_key,)) if obj_type == 'user': - p = self._make_new_user_perm(object, perm_value) + p = self._make_new_user_perm(to_object, perm_value) self.sa.add(p) if obj_type == 'user_group': - p = self._make_new_user_group_perm(object, perm_value) + p = self._make_new_user_group_perm(to_object, perm_value) self.sa.add(p) def _set_new_user_perms(self, user, form_result, preserve=None): @@ -321,8 +329,8 @@ class PermissionModel(BaseModel): def _get_group(perm_name): return '.'.join(perm_name.split('.')[:1]) - defined_perms_groups = map( - _get_group, (x.permission.permission_name for x in obj_perms)) + defined_perms_groups = list(map( + _get_group, (x.permission.permission_name for x in obj_perms))) log.debug('GOT ALREADY DEFINED:%s', obj_perms) if force: diff --git a/rhodecode/model/pull_request.py b/rhodecode/model/pull_request.py --- a/rhodecode/model/pull_request.py +++ b/rhodecode/model/pull_request.py @@ -23,15 +23,16 @@ pull request model for RhodeCode """ - -import json import logging import os import datetime -import urllib.request, urllib.parse, urllib.error +import urllib.request +import urllib.parse +import urllib.error import collections +import dataclasses as dataclasses from pyramid.threadlocal import get_current_request from rhodecode.lib.vcs.nodes import FileNode @@ -40,11 +41,12 @@ from rhodecode.lib import helpers as h, from rhodecode.lib import audit_logger from collections import OrderedDict from rhodecode.lib.hooks_daemon import prepare_callback_daemon +from rhodecode.lib.ext_json import sjson as json from rhodecode.lib.markup_renderer import ( DEFAULT_COMMENTS_RENDERER, RstTemplateRenderer) -from rhodecode.lib.utils2 import ( - safe_unicode, safe_str, md5_safe, AttributeDict, safe_int, - get_current_rhodecode_user) +from rhodecode.lib.hash_utils import md5_safe +from rhodecode.lib.str_utils import safe_str +from rhodecode.lib.utils2 import AttributeDict, get_current_rhodecode_user from rhodecode.lib.vcs.backends.base import ( Reference, MergeResponse, MergeFailureReason, UpdateFailureReason, TargetRefMissing, SourceRefMissing) @@ -55,7 +57,7 @@ from rhodecode.model import BaseModel from rhodecode.model.changeset_status import ChangesetStatusModel from rhodecode.model.comment import CommentsModel from rhodecode.model.db import ( - aliased, null, lazyload, and_, or_, func, String, cast, PullRequest, PullRequestReviewers, ChangesetStatus, + aliased, null, lazyload, and_, or_, select, func, String, cast, PullRequest, PullRequestReviewers, ChangesetStatus, PullRequestVersion, ChangesetComment, Repository, RepoReviewRule, User) from rhodecode.model.meta import Session from rhodecode.model.notification import NotificationModel, \ @@ -116,9 +118,8 @@ def get_diff_info( source_scm.get_diff(commit1=source_commit, commit2=target_commit, ignore_whitespace=False, context=3) - diff_processor = diffs.DiffProcessor( - vcs_diff, format='newdiff', diff_limit=None, - file_limit=None, show_full_diff=True) + diff_processor = diffs.DiffProcessor(vcs_diff, diff_format='newdiff', + diff_limit=0, file_limit=0, show_full_diff=True) _parsed = diff_processor.prepare() @@ -317,7 +318,7 @@ class PullRequestModel(BaseModel): q = PullRequest.query() if search_q: - like_expression = u'%{}%'.format(safe_unicode(search_q)) + like_expression = u'%{}%'.format(safe_str(search_q)) q = q.join(User, User.user_id == PullRequest.user_id) q = q.filter(or_( cast(PullRequest.pull_request_id, String).ilike(like_expression), @@ -489,7 +490,7 @@ class PullRequestModel(BaseModel): q = q.filter(pull_request_alias.status.in_(statuses)) if search_q: - like_expression = u'%{}%'.format(safe_unicode(search_q)) + like_expression = u'%{}%'.format(safe_str(search_q)) q = q.join(User, User.user_id == pull_request_alias.user_id) q = q.filter(or_( cast(pull_request_alias.pull_request_id, String).ilike(like_expression), @@ -562,12 +563,14 @@ class PullRequestModel(BaseModel): """ q = PullRequest.query() if user_id: - reviewers_subquery = Session().query( - PullRequestReviewers.pull_request_id).filter( - PullRequestReviewers.user_id == user_id).subquery() + + base_query = select(PullRequestReviewers)\ + .where(PullRequestReviewers.user_id == user_id)\ + .with_only_columns(PullRequestReviewers.pull_request_id) + user_filter = or_( PullRequest.user_id == user_id, - PullRequest.pull_request_id.in_(reviewers_subquery) + PullRequest.pull_request_id.in_(base_query) ) q = PullRequest.query().filter(user_filter) @@ -576,7 +579,7 @@ class PullRequestModel(BaseModel): q = q.filter(PullRequest.status.in_(statuses)) if query: - like_expression = u'%{}%'.format(safe_unicode(query)) + like_expression = u'%{}%'.format(safe_str(query)) q = q.join(User, User.user_id == PullRequest.user_id) q = q.filter(or_( cast(PullRequest.pull_request_id, String).ilike(like_expression), @@ -656,7 +659,7 @@ class PullRequestModel(BaseModel): q = q.filter(pull_request_alias.status.in_(statuses)) if query: - like_expression = u'%{}%'.format(safe_unicode(query)) + like_expression = u'%{}%'.format(safe_str(query)) q = q.join(User, User.user_id == pull_request_alias.user_id) q = q.filter(or_( cast(pull_request_alias.pull_request_id, String).ilike(like_expression), @@ -939,7 +942,8 @@ class PullRequestModel(BaseModel): def merge_repo(self, pull_request, user, extras): repo_type = pull_request.source_repo.repo_type - log.debug("Merging pull request %s", pull_request.pull_request_id) + log.debug("Merging pull request %s", pull_request) + extras['user_agent'] = '{}/internal-merge'.format(repo_type) merge_state = self._merge_pull_request(pull_request, user, extras) if merge_state.executed: @@ -952,14 +956,14 @@ class PullRequestModel(BaseModel): user, pull_request) else: - log.warn("Merge failed, not updating the pull request.") + log.warning("Merge failed, not updating the pull request.") return merge_state def _merge_pull_request(self, pull_request, user, extras, merge_msg=None): target_vcs = pull_request.target_repo.scm_instance() source_vcs = pull_request.source_repo.scm_instance() - message = safe_unicode(merge_msg or vcs_settings.MERGE_MESSAGE_TMPL).format( + message = safe_str(merge_msg or vcs_settings.MERGE_MESSAGE_TMPL).format( pr_id=pull_request.pull_request_id, pr_title=pull_request.title, pr_desc=pull_request.description, @@ -995,6 +999,7 @@ class PullRequestModel(BaseModel): user_name=user_name, user_email=user.email, message=message, use_rebase=use_rebase, close_branch=close_branch) + return merge_state def _comment_and_close_pr(self, pull_request, user, merge_state, close_msg=None): @@ -1003,7 +1008,7 @@ class PullRequestModel(BaseModel): close_msg = close_msg or 'Pull request merged and closed' CommentsModel().create( - text=safe_unicode(close_msg), + text=safe_str(close_msg), repo=pull_request.target_repo.repo_id, user=user.user_id, pull_request=pull_request.pull_request_id, @@ -1289,9 +1294,10 @@ class PullRequestModel(BaseModel): source_repo, source_ref_id, target_ref_id, hide_whitespace_changes=hide_whitespace_changes, diff_context=diff_context) - old_diff_data = diffs.DiffProcessor(old_diff) + # NOTE: this was using diff_format='gitdiff' + old_diff_data = diffs.DiffProcessor(old_diff, diff_format='newdiff') old_diff_data.prepare() - new_diff_data = diffs.DiffProcessor(new_diff) + new_diff_data = diffs.DiffProcessor(new_diff, diff_format='newdiff') new_diff_data.prepare() return old_diff_data, new_diff_data @@ -1598,7 +1604,7 @@ class PullRequestModel(BaseModel): return None else: pr_url = urllib.parse.unquote(self.get_url(pull_request, request=request)) - return safe_unicode('{pr_url}/repository'.format(pr_url=pr_url)) + return safe_str('{pr_url}/repository'.format(pr_url=pr_url)) def _notify_reviewers(self, pull_request, user_ids, role, user): # notification to reviewers/observers @@ -2032,8 +2038,8 @@ class PullRequestModel(BaseModel): _ = translator or get_current_request().translate commit_id = safe_str(commit_id) if commit_id else None - branch = safe_unicode(branch) if branch else None - bookmark = safe_unicode(bookmark) if bookmark else None + branch = safe_str(branch) if branch else None + bookmark = safe_str(bookmark) if bookmark else None selected = None @@ -2072,8 +2078,8 @@ class PullRequestModel(BaseModel): u'No commit refs could be found matching: {}'.format(ref)) elif repo.DEFAULT_BRANCH_NAME in repo.branches: selected = u'branch:{}:{}'.format( - safe_unicode(repo.DEFAULT_BRANCH_NAME), - safe_unicode(repo.branches[repo.DEFAULT_BRANCH_NAME]) + safe_str(repo.DEFAULT_BRANCH_NAME), + safe_str(repo.branches[repo.DEFAULT_BRANCH_NAME]) ) elif repo.commit_ids: # make the user select in this case @@ -2113,7 +2119,7 @@ class PullRequestModel(BaseModel): log.debug('calculating diff between ' 'source_ref:%s and target_ref:%s for repo `%s`', target_ref_id, source_ref_id, - safe_unicode(vcs_repo.path)) + safe_str(vcs_repo.path)) vcs_diff = vcs_repo.get_diff( commit1=target_commit, commit2=source_commit, @@ -2373,8 +2379,16 @@ class MergeCheck(object): return merge_details -ChangeTuple = collections.namedtuple( - 'ChangeTuple', ['added', 'common', 'removed', 'total']) +@dataclasses.dataclass +class ChangeTuple: + added: list + common: list + removed: list + total: list -FileChangeTuple = collections.namedtuple( - 'FileChangeTuple', ['added', 'modified', 'removed']) + +@dataclasses.dataclass +class FileChangeTuple: + added: list + modified: list + removed: list diff --git a/rhodecode/model/repo.py b/rhodecode/model/repo.py --- a/rhodecode/model/repo.py +++ b/rhodecode/model/repo.py @@ -26,6 +26,7 @@ import traceback import datetime from pyramid.threadlocal import get_current_request +from sqlalchemy.orm import aliased from zope.cachedescriptors.property import Lazy as LazyProperty from rhodecode import events @@ -36,7 +37,7 @@ from rhodecode.lib import hooks_base from rhodecode.lib.user_log_filter import user_log_filter from rhodecode.lib.utils import make_db_config from rhodecode.lib.utils2 import ( - safe_str, safe_unicode, remove_prefix, obfuscate_url_pw, + safe_str, remove_prefix, obfuscate_url_pw, get_current_rhodecode_user, safe_int, action_logger_generic) from rhodecode.lib.vcs.backends import get_backend from rhodecode.model import BaseModel @@ -78,7 +79,7 @@ class RepoModel(BaseModel): repo_to_perm.permission = Permission.get_by_key(default_perm) repo_to_perm.repository = repository - repo_to_perm.user_id = def_user.user_id + repo_to_perm.user = def_user return repo_to_perm @@ -112,7 +113,7 @@ class RepoModel(BaseModel): def _extract_id_from_repo_name(self, repo_name): if repo_name.startswith('/'): repo_name = repo_name.lstrip('/') - by_id_match = re.match(r'^_(\d{1,})', repo_name) + by_id_match = re.match(r'^_(\d+)', repo_name) if by_id_match: return by_id_match.groups()[0] @@ -138,7 +139,7 @@ class RepoModel(BaseModel): def get_repos_for_root(self, root, traverse=False): if traverse: - like_expression = u'{}%'.format(safe_unicode(root)) + like_expression = u'{}%'.format(safe_str(root)) repos = Repository.query().filter( Repository.repo_name.like(like_expression)).all() else: @@ -209,12 +210,12 @@ class RepoModel(BaseModel): def quick_menu(repo_name): return _render('quick_menu', repo_name) - def repo_lnk(name, rtype, rstate, private, archived, fork_of): + def repo_lnk(name, rtype, rstate, private, archived, fork_repo_name): if short_name is not None: short_name_var = short_name else: short_name_var = not admin - return _render('repo_name', name, rtype, rstate, private, archived, fork_of, + return _render('repo_name', name, rtype, rstate, private, archived, fork_repo_name, short_name=short_name_var, admin=False) def last_change(last_change): @@ -259,7 +260,7 @@ class RepoModel(BaseModel): "menu": quick_menu(repo.repo_name), "name": repo_lnk(repo.repo_name, repo.repo_type, repo.repo_state, - repo.private, repo.archived, repo.fork), + repo.private, repo.archived, repo.fork_repo_name), "desc": desc(h.escape(repo.description)), @@ -268,7 +269,7 @@ class RepoModel(BaseModel): "last_changeset": last_rev(repo.repo_name, changeset_cache), "last_changeset_raw": changeset_cache.get('revision'), - "owner": user_profile(repo.User.username), + "owner": user_profile(repo.owner_username), "state": state(repo.repo_state), "rss": rss_lnk(repo.repo_name), @@ -309,6 +310,8 @@ class RepoModel(BaseModel): ) \ .count() + RepoFork = aliased(Repository) + OwnerUser = aliased(User) base_q = Session.query( Repository.repo_id, Repository.repo_name, @@ -317,18 +320,18 @@ class RepoModel(BaseModel): Repository.repo_state, Repository.private, Repository.archived, - Repository.fork, Repository.updated_on, Repository._changeset_cache, - User, + RepoFork.repo_name.label('fork_repo_name'), + OwnerUser.username.label('owner_username'), ) \ .filter(Repository.group_id == repo_group_id) \ .filter(or_( # generate multiple IN to fix limitation problems *in_filter_generator(Repository.repo_id, allowed_ids)) ) \ - .join(User, User.user_id == Repository.user_id) \ - .group_by(Repository, User) + .outerjoin(RepoFork, Repository.fork_id == RepoFork.repo_id) \ + .join(OwnerUser, Repository.user_id == OwnerUser.user_id) repos_data_total_filtered_count = base_q.count() @@ -515,8 +518,8 @@ class RepoModel(BaseModel): landing_rev = landing_rev or default_landing_ref try: - repo_name = safe_unicode(repo_name) - description = safe_unicode(description) + repo_name = safe_str(repo_name) + description = safe_str(description) # repo name is just a name of repository # while repo_name_full is a full qualified name that is combined # with name and path of group @@ -979,14 +982,14 @@ class RepoModel(BaseModel): # check if this path is not a repository if is_valid_repo(repo_path, self.repos_path): - raise Exception('This path %s is a valid repository' % repo_path) + raise Exception(f'This path {repo_path} is a valid repository') # check if this path is a group if is_valid_repo_group(repo_path, self.repos_path): - raise Exception('This path %s is a valid group' % repo_path) + raise Exception(f'This path {repo_path} is a valid group') log.info('creating repo %s in %s from url: `%s`', - repo_name, safe_unicode(repo_path), + repo_name, safe_str(repo_path), obfuscate_url_pw(clone_uri)) backend = get_backend(repo_type) @@ -1016,7 +1019,7 @@ class RepoModel(BaseModel): repo.install_hooks() log.debug('Created repo %s with %s backend', - safe_unicode(repo_name), safe_unicode(repo_type)) + safe_str(repo_name), safe_str(repo_type)) return repo def _rename_filesystem_repo(self, old, new): @@ -1038,8 +1041,8 @@ class RepoModel(BaseModel): def _delete_filesystem_repo(self, repo): """ - removes repo from filesystem, the removal is acctually made by - added rm__ prefix into dir, and rename internat .hg/.git dirs so this + removes repo from filesystem, the removal is actually made by + added rm__ prefix into dir, and rename internal .hg/.git dirs so this repository is no longer valid for rhodecode, can be undeleted later on by reverting the renames on this repository @@ -1047,7 +1050,7 @@ class RepoModel(BaseModel): """ rm_path = os.path.join(self.repos_path, repo.repo_name) repo_group = repo.group - log.info("Removing repository %s", rm_path) + log.info("delete_filesystem_repo: removing repository %s", rm_path) # disable hg/git internal that it doesn't get detected as repo alias = repo.repo_type @@ -1094,19 +1097,19 @@ class ReadmeFinder: path_re = re.compile(r'^docs?', re.IGNORECASE) default_priorities = { - None: 0, - '.text': 2, - '.txt': 3, - '.rst': 1, - '.rest': 2, - '.md': 1, - '.mkdn': 2, - '.mdown': 3, + None: 0, + '.rst': 1, + '.md': 1, + '.rest': 2, + '.mkdn': 2, + '.text': 2, + '.txt': 3, + '.mdown': 3, '.markdown': 4, } path_priority = { - 'doc': 0, + 'doc': 0, 'docs': 1, } @@ -1122,7 +1125,7 @@ class ReadmeFinder: self._renderer_extensions = self.RENDERER_TO_EXTENSION.get( default_renderer, []) - def search(self, commit, path=u'/'): + def search(self, commit, path='/'): """ Find a readme in the given `commit`. """ diff --git a/rhodecode/model/repo_group.py b/rhodecode/model/repo_group.py --- a/rhodecode/model/repo_group.py +++ b/rhodecode/model/repo_group.py @@ -131,7 +131,7 @@ class RepoGroupModel(BaseModel): repo_group_to_perm.permission = Permission.get_by_key(default_perm) repo_group_to_perm.group = new_group - repo_group_to_perm.user_id = def_user.user_id + repo_group_to_perm.user = def_user return repo_group_to_perm def _get_group_name_and_parent(self, group_name_full, repo_in_path=False, @@ -780,10 +780,10 @@ class RepoGroupModel(BaseModel): } if admin: repo_count = group.repositories.count() - children_groups = map( - h.safe_unicode, + children_groups = list(map( + h.safe_str, itertools.chain((g.name for g in group.parents), - (x.name for x in [group]))) + (x.name for x in [group])))) row.update({ "action": repo_group_actions( group.group_id, group.group_name, repo_count), diff --git a/rhodecode/model/scm.py b/rhodecode/model/scm.py --- a/rhodecode/model/scm.py +++ b/rhodecode/model/scm.py @@ -30,6 +30,7 @@ from sqlalchemy import func from zope.cachedescriptors.property import Lazy as LazyProperty import rhodecode +from rhodecode.lib.str_utils import safe_bytes from rhodecode.lib.vcs import get_backend from rhodecode.lib.vcs.exceptions import RepositoryError, NodeNotChangedError from rhodecode.lib.vcs.nodes import FileNode @@ -42,7 +43,7 @@ from rhodecode.lib.exceptions import Non from rhodecode.lib import hooks_utils from rhodecode.lib.utils import ( get_filesystem_repos, make_db_config) -from rhodecode.lib.utils2 import (safe_str, safe_unicode) +from rhodecode.lib.str_utils import safe_str from rhodecode.lib.system_info import get_system_info from rhodecode.model import BaseModel from rhodecode.model.db import ( @@ -284,8 +285,7 @@ class ScmModel(BaseModel): repo.update_commit_cache(config=config, cs_cache=None) if delete: cache_namespace_uid = 'cache_repo.{}'.format(repo_id) - rc_cache.clear_cache_namespace( - 'cache_repo', cache_namespace_uid, invalidate=True) + rc_cache.clear_cache_namespace('cache_repo', cache_namespace_uid, method=rc_cache.CLEAR_INVALIDATE) def toggle_following_repo(self, follow_repo_id, user_id): @@ -443,25 +443,18 @@ class ScmModel(BaseModel): raise def commit_change(self, repo, repo_name, commit, user, author, message, - content, f_path): + content: bytes, f_path: bytes): """ Commits changes - - :param repo: SCM instance - """ user = self._get_user(user) - # decoding here will force that we have proper encoded values - # in any other case this will throw exceptions and deny commit - content = safe_str(content) - path = safe_str(f_path) # message and author needs to be unicode # proper backend should then translate that into required type - message = safe_unicode(message) - author = safe_unicode(author) + message = safe_str(message) + author = safe_str(author) imc = repo.in_memory_commit - imc.change(FileNode(path, content, mode=commit.get_file_mode(f_path))) + imc.change(FileNode(f_path, content, mode=commit.get_file_mode(f_path))) try: # TODO: handle pre-push action ! tip = imc.commit( @@ -480,9 +473,9 @@ class ScmModel(BaseModel): repo_name=repo_name, repo_type=repo.alias, commit_ids=[tip.raw_id]) return tip - def _sanitize_path(self, f_path): - if f_path.startswith('/') or f_path.startswith('./') or '../' in f_path: - raise NonRelativePathError('%s is not an relative path' % f_path) + def _sanitize_path(self, f_path: bytes): + if f_path.startswith(b'/') or f_path.startswith(b'./') or b'../' in f_path: + raise NonRelativePathError(b'%b is not an relative path' % f_path) if f_path: f_path = os.path.normpath(f_path) return f_path @@ -531,15 +524,24 @@ class ScmModel(BaseModel): """ _files = list() _dirs = list() + try: _repo = self._get_repo(repo_name) commit = _repo.scm_instance().get_commit(commit_id=commit_id) root_path = root_path.lstrip('/') - for __, dirs, files in commit.walk(root_path): + + # get RootNode, inject pre-load options before walking + top_node = commit.get_node(root_path) + extended_info_pre_load = [] + if extended_info: + extended_info_pre_load += ['md5'] + top_node.default_pre_load = ['is_binary', 'size'] + extended_info_pre_load + + for __, dirs, files in commit.walk(top_node): for f in files: _content = None - _data = f_name = f.unicode_path + _data = f_name = f.str_path if not flat: _data = { @@ -561,7 +563,7 @@ class ScmModel(BaseModel): and f.size > max_file_bytes) full_content = None if not f.is_binary and not over_size_limit: - full_content = safe_str(f.content) + full_content = f.str_content _data.update({ "content": full_content, @@ -569,7 +571,7 @@ class ScmModel(BaseModel): _files.append(_data) for d in dirs: - _data = d_name = d.unicode_path + _data = d_name = d.str_path if not flat: _data = { "name": h.escape(d_name), @@ -577,10 +579,10 @@ class ScmModel(BaseModel): } if extended_info: _data.update({ - "md5": None, - "binary": None, - "size": None, - "extension": None, + "md5": "", + "binary": False, + "size": 0, + "extension": "", }) if content: _data.update({ @@ -609,7 +611,7 @@ class ScmModel(BaseModel): for f in files: _data = { - "name": h.escape(f.unicode_path), + "name": h.escape(f.str_path), "type": "file", } @@ -618,7 +620,7 @@ class ScmModel(BaseModel): for d in dirs: _data = { - "name": h.escape(d.unicode_path), + "name": h.escape(d.str_path), "type": "dir", } @@ -634,6 +636,7 @@ class ScmModel(BaseModel): """ retrieve single node from commit """ + try: _repo = self._get_repo(repo_name) @@ -644,7 +647,7 @@ class ScmModel(BaseModel): raise RepositoryError('The given path is a directory') _content = None - f_name = file_node.unicode_path + f_name = file_node.str_path file_data = { "name": h.escape(f_name), @@ -677,7 +680,7 @@ class ScmModel(BaseModel): full_content = None all_lines = 0 if not file_node.is_binary and not over_size_limit: - full_content = safe_unicode(file_node.content) + full_content = safe_str(file_node.content) all_lines, empty_lines = file_node.count_lines(full_content) file_data.update({ @@ -693,7 +696,7 @@ class ScmModel(BaseModel): full_content = None all_lines = 0 if not is_binary and not over_size_limit: - full_content = safe_unicode(_content) + full_content = safe_str(_content) all_lines, empty_lines = file_node.count_lines(full_content) file_data.update({ @@ -718,12 +721,15 @@ class ScmModel(BaseModel): _repo = self._get_repo(repo_name) commit = _repo.scm_instance().get_commit(commit_id=commit_id) root_path = root_path.lstrip('/') - for __, dirs, files in commit.walk(root_path): + top_node = commit.get_node(root_path) + top_node.default_pre_load = [] + + for __, dirs, files in commit.walk(top_node): for f in files: is_binary, md5, size, _content = f.metadata_uncached() _data = { - "name": f.unicode_path, + "name": f.str_path, "md5": md5, "extension": f.extension, "binary": is_binary, @@ -759,26 +765,9 @@ class ScmModel(BaseModel): user = self._get_user(user) scm_instance = repo.scm_instance(cache=False) - processed_nodes = [] - for f_path in nodes: - f_path = self._sanitize_path(f_path) - content = nodes[f_path]['content'] - f_path = safe_str(f_path) - # decoding here will force that we have proper encoded values - # in any other case this will throw exceptions and deny commit - if isinstance(content, (str,)): - content = safe_str(content) - elif isinstance(content, (file, cStringIO.OutputType,)): - content = content.read() - else: - raise Exception('Content is of unrecognized type %s' % ( - type(content) - )) - processed_nodes.append((f_path, content)) - - message = safe_unicode(message) + message = safe_str(message) commiter = user.full_contact - author = safe_unicode(author) if author else commiter + author = safe_str(author) if author else commiter imc = scm_instance.in_memory_commit @@ -786,13 +775,39 @@ class ScmModel(BaseModel): parent_commit = EmptyCommit(alias=scm_instance.alias) if isinstance(parent_commit, EmptyCommit): - # EmptyCommit means we we're editing empty repository + # EmptyCommit means we're editing empty repository parents = None else: parents = [parent_commit] + + upload_file_types = (io.BytesIO, io.BufferedRandom) + processed_nodes = [] + for filename, content_dict in nodes.items(): + if not isinstance(filename, bytes): + raise ValueError(f'filename key in nodes needs to be bytes , or {upload_file_types}') + content = content_dict['content'] + if not isinstance(content, upload_file_types + (bytes,)): + raise ValueError('content key value in nodes needs to be bytes') + + for f_path in nodes: + f_path = self._sanitize_path(f_path) + content = nodes[f_path]['content'] + + # decoding here will force that we have proper encoded values + # in any other case this will throw exceptions and deny commit + + if isinstance(content, bytes): + pass + elif isinstance(content, upload_file_types): + content = content.read() + else: + raise Exception(f'Content is of unrecognized type {type(content)}, expected {upload_file_types}') + processed_nodes.append((f_path, content)) + # add multiple nodes for path, content in processed_nodes: imc.add(FileNode(path, content=content)) + # TODO: handle pre push scenario tip = imc.commit(message=message, author=author, @@ -813,9 +828,9 @@ class ScmModel(BaseModel): user = self._get_user(user) scm_instance = repo.scm_instance(cache=False) - message = safe_unicode(message) + message = safe_str(message) commiter = user.full_contact - author = safe_unicode(author) if author else commiter + author = safe_str(author) if author else commiter imc = scm_instance.in_memory_commit @@ -897,14 +912,14 @@ class ScmModel(BaseModel): processed_nodes = [] for f_path in nodes: f_path = self._sanitize_path(f_path) - # content can be empty but for compatabilty it allows same dicts + # content can be empty but for compatibility it allows same dicts # structure as add_nodes content = nodes[f_path].get('content') - processed_nodes.append((f_path, content)) + processed_nodes.append((safe_bytes(f_path), content)) - message = safe_unicode(message) + message = safe_str(message) commiter = user.full_contact - author = safe_unicode(author) if author else commiter + author = safe_str(author) if author else commiter imc = scm_instance.in_memory_commit @@ -994,7 +1009,7 @@ class ScmModel(BaseModel): choices = [default_landing_ref] # branches - branch_group = [(u'branch:%s' % safe_unicode(b), safe_unicode(b)) for b in repo.branches] + branch_group = [(f'branch:{safe_str(b)}', safe_str(b)) for b in repo.branches] if not branch_group: # new repo, or without maybe a branch? branch_group = default_ref_options @@ -1006,7 +1021,7 @@ class ScmModel(BaseModel): # bookmarks for HG if repo.alias == 'hg': bookmarks_group = ( - [(u'book:%s' % safe_unicode(b), safe_unicode(b)) + [(f'book:{safe_str(b)}', safe_str(b)) for b in repo.bookmarks], _("Bookmarks")) ref_options.append(bookmarks_group) @@ -1014,7 +1029,7 @@ class ScmModel(BaseModel): # tags tags_group = ( - [(u'tag:%s' % safe_unicode(t), safe_unicode(t)) + [(f'tag:{safe_str(t)}', safe_str(t)) for t in repo.tags], _("Tags")) ref_options.append(tags_group) diff --git a/rhodecode/model/settings.py b/rhodecode/model/settings.py --- a/rhodecode/model/settings.py +++ b/rhodecode/model/settings.py @@ -19,16 +19,16 @@ import os import re -import hashlib import logging import time import functools import bleach from collections import namedtuple -from pyramid.threadlocal import get_current_request, get_current_registry +from pyramid.threadlocal import get_current_request from rhodecode.lib import rc_cache +from rhodecode.lib.hash_utils import sha1_safe from rhodecode.lib.utils2 import ( Optional, AttributeDict, safe_str, remove_prefix, str2bool) from rhodecode.lib.vcs.backends import base @@ -132,10 +132,9 @@ class SettingsModel(BaseModel): if not key: # keys are unique so they need appended info if self.repo: - key = hashlib.sha1( - '{}{}{}'.format(section, val, repository_id)).hexdigest() + key = sha1_safe(f'{section}{val}{repository_id}') else: - key = hashlib.sha1('{}{}'.format(section, val)).hexdigest() + key = sha1_safe(f'{section}{val}') new_ui.ui_key = key @@ -212,30 +211,20 @@ class SettingsModel(BaseModel): def get_cache_region(self): repo = self._get_repo(self.repo) if self.repo else None - cache_key = "repo.{}".format(repo.repo_id) if repo else "general_settings" - cache_namespace_uid = 'cache_settings.{}'.format(cache_key) + cache_key = f"repo.{repo.repo_id}" if repo else "repo.ALL" + cache_namespace_uid = f'cache_settings.{cache_key}' region = rc_cache.get_or_create_region('cache_general', cache_namespace_uid) - return region, cache_key - - def invalidate_settings_cache(self): - region, cache_key = self.get_cache_region() - log.debug('Invalidation cache region %s for cache_key: %s', region, cache_key) - region.invalidate() + return region, cache_namespace_uid - def get_all_settings(self, cache=False, from_request=True): - # defines if we use GLOBAL, or PER_REPO - repo = self._get_repo(self.repo) if self.repo else None + def invalidate_settings_cache(self, hard=False): + region, namespace_key = self.get_cache_region() + log.debug('Invalidation cache [%s] region %s for cache_key: %s', + 'invalidate_settings_cache', region, namespace_key) - # initially try the requests context, this is the fastest - # we only fetch global config - if from_request: - request = get_current_request() + # we use hard cleanup if invalidation is sent + rc_cache.clear_cache_namespace(region, namespace_key, method=rc_cache.CLEAR_DELETE) - if request and not repo and hasattr(request, 'call_context') and hasattr(request.call_context, 'rc_config'): - rc_config = request.call_context.rc_config - if rc_config: - return rc_config - + def get_cache_call_method(self, cache=True): region, cache_key = self.get_cache_region() @region.conditional_cache_on_arguments(condition=cache) @@ -245,10 +234,28 @@ class SettingsModel(BaseModel): raise Exception('Could not get application settings !') settings = { - 'rhodecode_' + res.app_settings_name: res.app_settings_value + f'rhodecode_{res.app_settings_name}': res.app_settings_value for res in q } return settings + return _get_all_settings + + def get_all_settings(self, cache=False, from_request=True): + # defines if we use GLOBAL, or PER_REPO + repo = self._get_repo(self.repo) if self.repo else None + + # initially try the requests context, this is the fastest + # we only fetch global config, NOT for repo-specific + if from_request and not repo: + request = get_current_request() + + if request and hasattr(request, 'call_context') and hasattr(request.call_context, 'rc_config'): + rc_config = request.call_context.rc_config + if rc_config: + return rc_config + + _region, cache_key = self.get_cache_region() + _get_all_settings = self.get_cache_call_method(cache=cache) start = time.time() result = _get_all_settings('rhodecode_settings', cache_key) @@ -318,8 +325,7 @@ class SettingsModel(BaseModel): def list_enabled_social_plugins(self, settings): enabled = [] for plug in SOCIAL_PLUGINS_LIST: - if str2bool(settings.get('rhodecode_auth_{}_enabled'.format(plug) - )): + if str2bool(settings.get(f'rhodecode_auth_{plug}_enabled')): enabled.append(plug) return enabled diff --git a/rhodecode/model/ssh_key.py b/rhodecode/model/ssh_key.py --- a/rhodecode/model/ssh_key.py +++ b/rhodecode/model/ssh_key.py @@ -28,6 +28,7 @@ from cryptography.hazmat.primitives.asym from cryptography.hazmat.primitives import serialization as crypto_serialization from cryptography.hazmat.backends import default_backend as crypto_default_backend +from rhodecode.lib.str_utils import safe_bytes, safe_str from rhodecode.model import BaseModel from rhodecode.model.db import UserSshKeys from rhodecode.model.meta import Session @@ -85,10 +86,13 @@ class SshKeyModel(BaseModel): crypto_serialization.Encoding.PEM, private_format, crypto_serialization.NoEncryption()) + private_key = safe_str(private_key) + public_key = key.public_key().public_bytes( crypto_serialization.Encoding.OpenSSH, crypto_serialization.PublicFormat.OpenSSH ) + public_key = safe_str(public_key) if comment: public_key = public_key + " " + comment diff --git a/rhodecode/model/update.py b/rhodecode/model/update.py --- a/rhodecode/model/update.py +++ b/rhodecode/model/update.py @@ -19,7 +19,9 @@ # and proprietary license terms, please see https://rhodecode.com/licenses/ import logging -import urllib.request, urllib.error, urllib.parse +import urllib.request +import urllib.error +import urllib.parse from packaging.version import Version import rhodecode diff --git a/rhodecode/model/user.py b/rhodecode/model/user.py --- a/rhodecode/model/user.py +++ b/rhodecode/model/user.py @@ -32,8 +32,9 @@ from sqlalchemy.exc import DatabaseError from rhodecode import events from rhodecode.lib.user_log_filter import user_log_filter from rhodecode.lib.utils2 import ( - safe_unicode, get_current_rhodecode_user, action_logger_generic, + get_current_rhodecode_user, action_logger_generic, AttributeDict, str2bool) +from rhodecode.lib.str_utils import safe_str from rhodecode.lib.exceptions import ( DefaultUserException, UserOwnsReposException, UserOwnsRepoGroupsException, UserOwnsUserGroupsException, NotAllowedToCreateUserError, @@ -87,7 +88,7 @@ class UserModel(BaseModel): query = query.filter(User.active == true()) if name_contains: - ilike_expression = u'%{}%'.format(safe_unicode(name_contains)) + ilike_expression = u'%{}%'.format(safe_str(name_contains)) query = query.filter( or_( User.name.ilike(ilike_expression), @@ -358,8 +359,8 @@ class UserModel(BaseModel): new_user.admin = admin new_user.email = email new_user.active = active - new_user.extern_name = safe_unicode(extern_name) - new_user.extern_type = safe_unicode(extern_type) + new_user.extern_name = safe_str(extern_name) + new_user.extern_type = safe_str(extern_type) new_user.name = firstname new_user.lastname = lastname new_user.description = description @@ -533,7 +534,7 @@ class UserModel(BaseModel): left_overs = False - # if nothing is done we have left overs left + # if nothing is done we have leftovers left return left_overs def _handle_user_artifacts(self, username, artifacts, handle_user, @@ -909,8 +910,8 @@ class UserModel(BaseModel): ip_range = ip_range.strip() if '-' in ip_range: start_ip, end_ip = ip_range.split('-', 1) - start_ip = ipaddress.ip_address(safe_unicode(start_ip.strip())) - end_ip = ipaddress.ip_address(safe_unicode(end_ip.strip())) + start_ip = ipaddress.ip_address(safe_str(start_ip.strip())) + end_ip = ipaddress.ip_address(safe_str(end_ip.strip())) parsed_ip_range = [] for index in range(int(start_ip), int(end_ip) + 1): diff --git a/rhodecode/model/user_group.py b/rhodecode/model/user_group.py --- a/rhodecode/model/user_group.py +++ b/rhodecode/model/user_group.py @@ -21,7 +21,7 @@ import logging import traceback -from rhodecode.lib.utils2 import safe_str, safe_unicode +from rhodecode.lib.utils2 import safe_str from rhodecode.lib.exceptions import ( UserGroupAssignedException, RepoGroupAssignmentError) from rhodecode.lib.utils2 import ( @@ -58,7 +58,7 @@ class UserGroupModel(BaseModel): user_group_to_perm.permission = Permission.get_by_key(default_perm) user_group_to_perm.user_group = user_group - user_group_to_perm.user_id = def_user.user_id + user_group_to_perm.user = def_user return user_group_to_perm def update_permissions( @@ -710,7 +710,7 @@ class UserGroupModel(BaseModel): query = query.filter(UserGroup.users_group_active == true()) if name_contains: - ilike_expression = u'%{}%'.format(safe_unicode(name_contains)) + ilike_expression = u'%{}%'.format(safe_str(name_contains)) query = query.filter( UserGroup.users_group_name.ilike(ilike_expression))\ .order_by(func.length(UserGroup.users_group_name))\