validators/schemas: python3 fixes str vs unicode and few test breaking fixes

rhodecode/model/validation_schema/preparers.py

@@ -18,7 +18,14 b''
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
-import unicodedata
+from rhodecode.lib.str_utils import convert_special_chars, safe_int, safe_bytes
+
+
+def to_bytes_preparer(value):
+
+    if value:
+        value = safe_bytes(value)
+    return value
 
 
 def strip_preparer(value):
@@ -49,7 +56,7 b' def non_ascii_strip_preparer(value):'
         `żołw` converts into `zolw`
     """
     if value:
-        value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore')
+        value = convert_special_chars(value)
     return value
 
 
@@ -85,4 +92,8 b' def unique_list_from_str_preparer(value)'
 
     if isinstance(value, str):
         value = aslist(value, ',')
-    return unique_list_preparer(value)
  No newline at end of file
+    return unique_list_preparer(value)
+
+
+def ensure_value_is_int(value):
+    return safe_int(value)

rhodecode/model/validation_schema/schemas/comment_schema.py

@@ -60,7 +60,7 b' class CommentSchema(colander.MappingSche'
         colander.String(),
         validator=colander.OneOf(ChangesetComment.COMMENT_TYPES),
         missing=ChangesetComment.COMMENT_TYPE_NOTE)
-    is_draft = colander.SchemaNode(colander.Boolean(),missing=False)
+    is_draft = colander.SchemaNode(colander.Boolean(), missing=False)
     comment_file = colander.SchemaNode(colander.String(), missing=None)
     comment_line = colander.SchemaNode(colander.String(), missing=None)
     status_change = colander.SchemaNode(

rhodecode/model/validation_schema/schemas/gist_schema.py

@@ -22,6 +22,7 b' import os'
 
 import colander
 
+from rhodecode.lib.str_utils import safe_str
 from rhodecode.translation import _
 from rhodecode.model.validation_schema import preparers
 
@@ -34,7 +35,7 b' def nodes_to_sequence(nodes, colander_no'
 
     """
     if not isinstance(nodes, dict):
-        msg = 'Nodes needs to be a dict, got {}'.format(type(nodes))
+        msg = f'Nodes needs to be a dict, got {type(nodes)}'
         raise colander.Invalid(colander_node, msg)
     out = []
 
@@ -51,6 +52,7 b' def nodes_to_sequence(nodes, colander_no'
 
 
 def sequence_to_nodes(nodes, colander_node=None):
+
     if not isinstance(nodes, list):
         msg = 'Nodes needs to be a list, got {}'.format(type(nodes))
         raise colander.Invalid(colander_node, msg)
@@ -86,33 +88,41 b' def unique_gist_validator(node, value):'
     from rhodecode.model.db import Gist
     existing = Gist.get_by_access_id(value)
     if existing:
-        msg = _(u'Gist with name {} already exists').format(value)
+        msg = _('Gist with name {} already exists').format(value)
         raise colander.Invalid(node, msg)
 
 
 def filename_validator(node, value):
     if value != os.path.basename(value):
-        msg = _(u'Filename {} cannot be inside a directory').format(value)
+        msg = _('Filename {} cannot be inside a directory').format(safe_str(value))
         raise colander.Invalid(node, msg)
 
 
 class NodeSchema(colander.MappingSchema):
     # if we perform rename this will be org filename
     filename_org = colander.SchemaNode(
-        colander.String(),
-        preparer=[preparers.strip_preparer,
-                  preparers.non_ascii_strip_preparer],
+        colander.String(encoding='utf-8'),
+        preparer=[
+            preparers.strip_preparer,
+            preparers.non_ascii_strip_preparer,
+            preparers.to_bytes_preparer,
+        ],
         validator=filename_validator,
         missing=None)
 
     filename = colander.SchemaNode(
-        colander.String(),
-        preparer=[preparers.strip_preparer,
-                  preparers.non_ascii_strip_preparer],
+        colander.String(encoding='utf-8'),
+        preparer=[
+            preparers.strip_preparer,
+            preparers.non_ascii_strip_preparer,
+            preparers.to_bytes_preparer,
+        ],
         validator=filename_validator)
 
     content = colander.SchemaNode(
-        colander.String())
+        colander.String(encoding='utf-8'),
+        preparer=[preparers.to_bytes_preparer])
+
     mimetype = colander.SchemaNode(
         colander.String(),
         missing=None)
@@ -164,7 +174,7 b' class GistSchema(colander.MappingSchema)'
 
     description = colander.SchemaNode(
         colander.String(),
-        missing=u'')
+        missing='')
 
     lifetime = colander.SchemaNode(
         colander.Integer(),
@@ -172,12 +182,13 b' class GistSchema(colander.MappingSchema)'
 
     gist_acl_level = colander.SchemaNode(
         colander.String(),
+        missing=Gist.ACL_LEVEL_PRIVATE,
         validator=colander.OneOf([Gist.ACL_LEVEL_PUBLIC,
                                   Gist.ACL_LEVEL_PRIVATE]))
 
     gist_type = colander.SchemaNode(
         colander.String(),
-        missing=Gist.GIST_PUBLIC,
+        missing=Gist.GIST_PRIVATE,
         validator=colander.OneOf([Gist.GIST_PRIVATE, Gist.GIST_PUBLIC]))
 
     nodes = Nodes()

rhodecode/model/validation_schema/schemas/repo_schema.py

@@ -238,9 +238,18 b' def deferred_repo_name_validator(node, k'
 
 @colander.deferred
 def deferred_repo_group_validator(node, kw):
-    options = kw.get(
-        'repo_repo_group_options')
-    return colander.OneOf([x for x in options])
+    options = kw.get('repo_repo_group_options')
+
+    def repo_group_validator(node, value):
+        choices = [x for x in options]
+        err = _('Group ID: `${val}` is not one of allowed ${choices}')
+
+        if preparers.ensure_value_is_int(value) not in choices:
+            choices = ', '.join(['%s' % x for x in sorted(choices)])
+            err = _(err, mapping={'val': value, 'choices': choices})
+            raise colander.Invalid(node, err)
+
+    return repo_group_validator
 
 
 @colander.deferred
@@ -395,6 +404,7 b' class RepoSettingsSchema(RepoSchema):'
         colander.Integer(),
         validator=deferred_repo_group_validator,
         widget=deferred_repo_group_widget,
+        preparers=[preparers.ensure_value_is_int],
         missing='')
 
     repo_clone_uri_change = colander.SchemaNode(

rhodecode/model/validation_schema/validators.py

@@ -18,16 +18,15 b''
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
-import os
 import re
 import logging
 
-
 import ipaddress
 import colander
 
 from rhodecode.translation import _
-from rhodecode.lib.utils2 import glob2re, safe_unicode
+from rhodecode.lib.utils2 import glob2re
+from rhodecode.lib.str_utils import safe_str
 from rhodecode.lib.ext_json import json
 
 log = logging.getLogger(__name__)
@@ -36,9 +35,9 b' log = logging.getLogger(__name__)'
 def ip_addr_validator(node, value):
     try:
         # this raises an ValueError if address is not IpV4 or IpV6
-        ipaddress.ip_network(safe_unicode(value), strict=False)
+        ipaddress.ip_network(safe_str(value), strict=False)
     except ValueError:
-        msg = _(u'Please enter a valid IPv4 or IpV6 address')
+        msg = _('Please enter a valid IPv4 or IpV6 address')
         raise colander.Invalid(node, msg)
 
 
@@ -49,9 +48,9 b' class IpAddrValidator(object):'
     def __call__(self, node, value):
         try:
             # this raises an ValueError if address is not IpV4 or IpV6
-            ipaddress.ip_network(safe_unicode(value), strict=self.strict)
+            ipaddress.ip_network(safe_str(value), strict=self.strict)
         except ValueError:
-            msg = _(u'Please enter a valid IPv4 or IpV6 address')
+            msg = _('Please enter a valid IPv4 or IpV6 address')
             raise colander.Invalid(node, msg)
 
 
@@ -59,7 +58,7 b' def glob_validator(node, value):'
     try:
         re.compile('^' + glob2re(value) + '$')
     except Exception:
-        msg = _(u'Invalid glob pattern')
+        msg = _('Invalid glob pattern')
         raise colander.Invalid(node, msg)
 
 
@@ -136,19 +135,21 b' class CloneUriValidator(object):'
             url_validator(value, self.repo_type, config)
         except InvalidCloneUrl as e:
             log.warning(e)
-            raise colander.Invalid(node, e.message)
-        except Exception:
+            raise colander.Invalid(node, str(e))
+        except Exception as e:
             log.exception('Url validation failed')
-            msg = _(u'invalid clone url or credentials for {repo_type} repository').format(
-                repo_type=self.repo_type)
+            reason = repr(e)
+            reason = reason.replace('<', '&lt;').replace('>', '&gt;')
+            msg = _('invalid clone url or credentials for {repo_type} repository. Reason: {reason}')\
+                .format(reason=reason, repo_type=self.repo_type)
             raise colander.Invalid(node, msg)
 
 
 def json_validator(node, value):
     try:
         json.loads(value)
-    except (Exception,) as e:
-        msg = _(u'Please enter a valid json object')
+    except (Exception,):
+        msg = _('Please enter a valid json object')
         raise colander.Invalid(node, msg)
 
 
@@ -156,5 +157,5 b' def json_validator_with_exc(node, value)'
     try:
         json.loads(value)
     except (Exception,) as e:
-        msg = _(u'Please enter a valid json object: `{}`'.format(e))
+        msg = _('Please enter a valid json object: `{}`'.format(e))
         raise colander.Invalid(node, msg)