# HG changeset patch # User RhodeCode Admin # Date 2023-07-18 09:18:37 # Node ID ccd88b7c8d7297cdc8ab61c4c43bf43d49523af1 # Parent bfe9513d4c23ac897ab9009711f7c32dd22337db validators/schemas: python3 fixes str vs unicode and few test breaking fixes diff --git a/rhodecode/model/validation_schema/preparers.py b/rhodecode/model/validation_schema/preparers.py --- a/rhodecode/model/validation_schema/preparers.py +++ b/rhodecode/model/validation_schema/preparers.py @@ -18,7 +18,14 @@ # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ -import unicodedata +from rhodecode.lib.str_utils import convert_special_chars, safe_int, safe_bytes + + +def to_bytes_preparer(value): + + if value: + value = safe_bytes(value) + return value def strip_preparer(value): @@ -49,7 +56,7 @@ def non_ascii_strip_preparer(value): `żołw` converts into `zolw` """ if value: - value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore') + value = convert_special_chars(value) return value @@ -85,4 +92,8 @@ def unique_list_from_str_preparer(value) if isinstance(value, str): value = aslist(value, ',') - return unique_list_preparer(value) \ No newline at end of file + return unique_list_preparer(value) + + +def ensure_value_is_int(value): + return safe_int(value) diff --git a/rhodecode/model/validation_schema/schemas/comment_schema.py b/rhodecode/model/validation_schema/schemas/comment_schema.py --- a/rhodecode/model/validation_schema/schemas/comment_schema.py +++ b/rhodecode/model/validation_schema/schemas/comment_schema.py @@ -60,7 +60,7 @@ class CommentSchema(colander.MappingSche colander.String(), validator=colander.OneOf(ChangesetComment.COMMENT_TYPES), missing=ChangesetComment.COMMENT_TYPE_NOTE) - is_draft = colander.SchemaNode(colander.Boolean(),missing=False) + is_draft = colander.SchemaNode(colander.Boolean(), missing=False) comment_file = colander.SchemaNode(colander.String(), missing=None) comment_line = colander.SchemaNode(colander.String(), missing=None) status_change = colander.SchemaNode( diff --git a/rhodecode/model/validation_schema/schemas/gist_schema.py b/rhodecode/model/validation_schema/schemas/gist_schema.py --- a/rhodecode/model/validation_schema/schemas/gist_schema.py +++ b/rhodecode/model/validation_schema/schemas/gist_schema.py @@ -22,6 +22,7 @@ import os import colander +from rhodecode.lib.str_utils import safe_str from rhodecode.translation import _ from rhodecode.model.validation_schema import preparers @@ -34,7 +35,7 @@ def nodes_to_sequence(nodes, colander_no """ if not isinstance(nodes, dict): - msg = 'Nodes needs to be a dict, got {}'.format(type(nodes)) + msg = f'Nodes needs to be a dict, got {type(nodes)}' raise colander.Invalid(colander_node, msg) out = [] @@ -51,6 +52,7 @@ def nodes_to_sequence(nodes, colander_no def sequence_to_nodes(nodes, colander_node=None): + if not isinstance(nodes, list): msg = 'Nodes needs to be a list, got {}'.format(type(nodes)) raise colander.Invalid(colander_node, msg) @@ -86,33 +88,41 @@ def unique_gist_validator(node, value): from rhodecode.model.db import Gist existing = Gist.get_by_access_id(value) if existing: - msg = _(u'Gist with name {} already exists').format(value) + msg = _('Gist with name {} already exists').format(value) raise colander.Invalid(node, msg) def filename_validator(node, value): if value != os.path.basename(value): - msg = _(u'Filename {} cannot be inside a directory').format(value) + msg = _('Filename {} cannot be inside a directory').format(safe_str(value)) raise colander.Invalid(node, msg) class NodeSchema(colander.MappingSchema): # if we perform rename this will be org filename filename_org = colander.SchemaNode( - colander.String(), - preparer=[preparers.strip_preparer, - preparers.non_ascii_strip_preparer], + colander.String(encoding='utf-8'), + preparer=[ + preparers.strip_preparer, + preparers.non_ascii_strip_preparer, + preparers.to_bytes_preparer, + ], validator=filename_validator, missing=None) filename = colander.SchemaNode( - colander.String(), - preparer=[preparers.strip_preparer, - preparers.non_ascii_strip_preparer], + colander.String(encoding='utf-8'), + preparer=[ + preparers.strip_preparer, + preparers.non_ascii_strip_preparer, + preparers.to_bytes_preparer, + ], validator=filename_validator) content = colander.SchemaNode( - colander.String()) + colander.String(encoding='utf-8'), + preparer=[preparers.to_bytes_preparer]) + mimetype = colander.SchemaNode( colander.String(), missing=None) @@ -164,7 +174,7 @@ class GistSchema(colander.MappingSchema) description = colander.SchemaNode( colander.String(), - missing=u'') + missing='') lifetime = colander.SchemaNode( colander.Integer(), @@ -172,12 +182,13 @@ class GistSchema(colander.MappingSchema) gist_acl_level = colander.SchemaNode( colander.String(), + missing=Gist.ACL_LEVEL_PRIVATE, validator=colander.OneOf([Gist.ACL_LEVEL_PUBLIC, Gist.ACL_LEVEL_PRIVATE])) gist_type = colander.SchemaNode( colander.String(), - missing=Gist.GIST_PUBLIC, + missing=Gist.GIST_PRIVATE, validator=colander.OneOf([Gist.GIST_PRIVATE, Gist.GIST_PUBLIC])) nodes = Nodes() diff --git a/rhodecode/model/validation_schema/schemas/repo_schema.py b/rhodecode/model/validation_schema/schemas/repo_schema.py --- a/rhodecode/model/validation_schema/schemas/repo_schema.py +++ b/rhodecode/model/validation_schema/schemas/repo_schema.py @@ -238,9 +238,18 @@ def deferred_repo_name_validator(node, k @colander.deferred def deferred_repo_group_validator(node, kw): - options = kw.get( - 'repo_repo_group_options') - return colander.OneOf([x for x in options]) + options = kw.get('repo_repo_group_options') + + def repo_group_validator(node, value): + choices = [x for x in options] + err = _('Group ID: `${val}` is not one of allowed ${choices}') + + if preparers.ensure_value_is_int(value) not in choices: + choices = ', '.join(['%s' % x for x in sorted(choices)]) + err = _(err, mapping={'val': value, 'choices': choices}) + raise colander.Invalid(node, err) + + return repo_group_validator @colander.deferred @@ -395,6 +404,7 @@ class RepoSettingsSchema(RepoSchema): colander.Integer(), validator=deferred_repo_group_validator, widget=deferred_repo_group_widget, + preparers=[preparers.ensure_value_is_int], missing='') repo_clone_uri_change = colander.SchemaNode( diff --git a/rhodecode/model/validation_schema/validators.py b/rhodecode/model/validation_schema/validators.py --- a/rhodecode/model/validation_schema/validators.py +++ b/rhodecode/model/validation_schema/validators.py @@ -18,16 +18,15 @@ # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ -import os import re import logging - import ipaddress import colander from rhodecode.translation import _ -from rhodecode.lib.utils2 import glob2re, safe_unicode +from rhodecode.lib.utils2 import glob2re +from rhodecode.lib.str_utils import safe_str from rhodecode.lib.ext_json import json log = logging.getLogger(__name__) @@ -36,9 +35,9 @@ log = logging.getLogger(__name__) def ip_addr_validator(node, value): try: # this raises an ValueError if address is not IpV4 or IpV6 - ipaddress.ip_network(safe_unicode(value), strict=False) + ipaddress.ip_network(safe_str(value), strict=False) except ValueError: - msg = _(u'Please enter a valid IPv4 or IpV6 address') + msg = _('Please enter a valid IPv4 or IpV6 address') raise colander.Invalid(node, msg) @@ -49,9 +48,9 @@ class IpAddrValidator(object): def __call__(self, node, value): try: # this raises an ValueError if address is not IpV4 or IpV6 - ipaddress.ip_network(safe_unicode(value), strict=self.strict) + ipaddress.ip_network(safe_str(value), strict=self.strict) except ValueError: - msg = _(u'Please enter a valid IPv4 or IpV6 address') + msg = _('Please enter a valid IPv4 or IpV6 address') raise colander.Invalid(node, msg) @@ -59,7 +58,7 @@ def glob_validator(node, value): try: re.compile('^' + glob2re(value) + '$') except Exception: - msg = _(u'Invalid glob pattern') + msg = _('Invalid glob pattern') raise colander.Invalid(node, msg) @@ -136,19 +135,21 @@ class CloneUriValidator(object): url_validator(value, self.repo_type, config) except InvalidCloneUrl as e: log.warning(e) - raise colander.Invalid(node, e.message) - except Exception: + raise colander.Invalid(node, str(e)) + except Exception as e: log.exception('Url validation failed') - msg = _(u'invalid clone url or credentials for {repo_type} repository').format( - repo_type=self.repo_type) + reason = repr(e) + reason = reason.replace('<', '<').replace('>', '>') + msg = _('invalid clone url or credentials for {repo_type} repository. Reason: {reason}')\ + .format(reason=reason, repo_type=self.repo_type) raise colander.Invalid(node, msg) def json_validator(node, value): try: json.loads(value) - except (Exception,) as e: - msg = _(u'Please enter a valid json object') + except (Exception,): + msg = _('Please enter a valid json object') raise colander.Invalid(node, msg) @@ -156,5 +157,5 @@ def json_validator_with_exc(node, value) try: json.loads(value) except (Exception,) as e: - msg = _(u'Please enter a valid json object: `{}`'.format(e)) + msg = _('Please enter a valid json object: `{}`'.format(e)) raise colander.Invalid(node, msg)