helpers: python3 port

rhodecode/lib/helpers.py

@@ -29,15 +29,16 b' import collections'
 import os
 import random
 import hashlib
-from io import StringIO
+import io
 import textwrap
-import urllib.request, urllib.parse, urllib.error
+import urllib.request
+import urllib.parse
+import urllib.error
 import math
 import logging
 import re
 import time
 import string
-import hashlib
 import regex
 from collections import OrderedDict
 
@@ -70,19 +71,22 b' from webhelpers2.html.tags import ('
     form as insecure_form,
     auto_discovery_link, checkbox, end_form, file,
     hidden, image, javascript_link, link_to, link_to_if, link_to_unless, ol,
-    select as raw_select, stylesheet_link, submit, text, password, textarea,
+    stylesheet_link, submit, text, password, textarea,
     ul, radio, Options)
 
 from webhelpers2.number import format_byte_size
+# python3.11 backport fixes for webhelpers2
+from rhodecode.lib._vendor.webhelpers_backports import raw_select
 
 from rhodecode.lib.action_parser import action_parser
 from rhodecode.lib.pagination import Page, RepoPage, SqlPage
 from rhodecode.lib import ext_json
 from rhodecode.lib.ext_json import json
-from rhodecode.lib.str_utils import safe_bytes
+from rhodecode.lib.str_utils import safe_bytes, convert_special_chars
 from rhodecode.lib.utils import repo_name_slug, get_custom_lexer
+from rhodecode.lib.str_utils import safe_str
 from rhodecode.lib.utils2 import (
-    str2bool, safe_unicode, safe_str,
+    str2bool,
     get_commit_safe, datetime_to_time, time_to_datetime, time_to_utcdatetime,
     AttributeDict, safe_int, md5, md5_safe, get_host_info)
 from rhodecode.lib.markup_renderer import MarkupRenderer, relative_links
@@ -123,11 +127,11 b' def asset(path, ver=None, **kwargs):'
 
 
 default_html_escape_table = {
-    ord('&'): u'&',
-    ord('<'): u'&lt;',
-    ord('>'): u'&gt;',
-    ord('"'): u'&quot;',
-    ord("'"): u'&#39;',
+    ord('&'): '&amp;',
+    ord('<'): '&lt;',
+    ord('>'): '&gt;',
+    ord('"'): '&quot;',
+    ord("'"): '&#39;',
 }
 
 
@@ -265,14 +269,12 b' class _ToolTip(object):'
 
 tooltip = _ToolTip()
 
-files_icon = u'<i class="file-breadcrumb-copy tooltip icon-clipboard clipboard-action" data-clipboard-text="{}" title="Copy file path"></i>'
+files_icon = '<i class="file-breadcrumb-copy tooltip icon-clipboard clipboard-action" data-clipboard-text="{}" title="Copy file path"></i>'
 
 
 def files_breadcrumbs(repo_name, repo_type, commit_id, file_path, landing_ref_name=None, at_ref=None,
                       limit_items=False, linkify_last_item=False, hide_last_item=False,
                       copy_path_icon=True):
-    if isinstance(file_path, str):
-        file_path = safe_unicode(file_path)
 
     if at_ref:
         route_qry = {'at': at_ref}
@@ -282,7 +284,7 b' def files_breadcrumbs(repo_name, repo_ty'
         default_landing_ref = commit_id
 
     # first segment is a `HOME` link to repo files root location
-    root_name = literal(u'<i class="icon-home"></i>')
+    root_name = literal('<i class="icon-home"></i>')
 
     url_segments = [
         link_to(
@@ -345,7 +347,6 b' def files_breadcrumbs(repo_name, repo_ty'
 
 
 def files_url_data(request):
-    import urllib.request, urllib.parse, urllib.error
     matchdict = request.matchdict
 
     if 'f_path' not in matchdict:
@@ -426,17 +427,17 b' class CodeHtmlFormatter(HtmlFormatter):'
     My code Html Formatter for source codes
     """
 
-    def wrap(self, source, outfile):
+    def wrap(self, source):
         return self._wrap_div(self._wrap_pre(self._wrap_code(source)))
 
     def _wrap_code(self, source):
         for cnt, it in enumerate(source):
             i, t = it
-            t = '<div id="L%s">%s</div>' % (cnt + 1, t)
+            t = f'<div id="L{cnt+1}">{t}</div>'
             yield i, t
 
     def _wrap_tablelinenos(self, inner):
-        dummyoutfile = StringIO.StringIO()
+        dummyoutfile = io.StringIO()
         lncount = 0
         for t, line in inner:
             if t:
@@ -594,7 +595,7 b' def unique_color_generator(n=10000, satu'
         h %= 1
         HSV_tuple = [h, saturation, lightness]
         RGB_tuple = hsv_to_rgb(*HSV_tuple)
-        yield map(lambda x: str(int(x * 256)), RGB_tuple)
+        yield [str(int(x * 256)) for x in RGB_tuple]
 
 
 def color_hasher(n=10000, saturation=0.10, lightness=0.95):
@@ -692,7 +693,7 b' class _Message(object):'
     __unicode__ = __str__
 
     def __html__(self):
-        return escape(safe_unicode(self.message))
+        return escape(safe_str(self.message))
 
 
 class Flash(object):
@@ -771,7 +772,7 b' class Flash(object):'
         for message in messages:
             payloads.append({
                 'message': {
-                    'message': u'{}'.format(message.message),
+                    'message': '{}'.format(message.message),
                     'level': message.category,
                     'force': True,
                     'subdata': message.sub_data
@@ -816,8 +817,7 b' def hide_credentials(url):'
     from rhodecode.lib.utils2 import credentials_filter
     return credentials_filter(url)
 
-
-import pytz
+import zoneinfo
 import tzlocal
 local_timezone = tzlocal.get_localzone()
 
@@ -829,9 +829,10 b' def get_timezone(datetime_iso, time_is_l'
     if time_is_local and isinstance(datetime_iso, datetime) and not datetime_iso.tzinfo:
         force_timezone = os.environ.get('RC_TIMEZONE', '')
         if force_timezone:
-            force_timezone = pytz.timezone(force_timezone)
+            force_timezone = zoneinfo.ZoneInfo(force_timezone)
         timezone = force_timezone or local_timezone
-        offset = timezone.localize(datetime_iso).strftime('%z')
+
+        offset = datetime_iso.replace(tzinfo=timezone).strftime('%z')
         tzinfo = '{}:{}'.format(offset[:-2], offset[-2:])
     return tzinfo
 
@@ -883,9 +884,9 b' def format_date(date):'
 
     if date:
         _fmt = "%a, %d %b %Y %H:%M:%S"
-        return safe_unicode(date.strftime(_fmt))
-
-    return u""
+        return safe_str(date.strftime(_fmt))
+
+    return ""
 
 
 class _RepoChecker(object):
@@ -1021,7 +1022,8 b' def author_string(email):'
 
 def person_by_id(id_, show_attr="username_and_name"):
     # attr to return from fetched user
-    person_getter = lambda usr: getattr(usr, show_attr)
+    def person_getter(usr):
+        return getattr(usr, show_attr)
 
     #maybe it's an ID ?
     if str(id_).isdigit() or isinstance(id_, int):
@@ -1074,7 +1076,7 b' def extract_metatags(value):'
     if not value:
         return tags, ''
 
-    for key, val in tags_paterns.items():
+    for key, val in list(tags_paterns.items()):
         pat, replace_html = val
         tags.extend([(key, x.group()) for x in pat.finditer(value)])
         value = pat.sub('', value)
@@ -1093,8 +1095,8 b' def style_metatag(tag_type, value):'
     tag_data = tags_paterns.get(tag_type)
     if tag_data:
         pat, replace_html = tag_data
-        # convert to plain `unicode` instead of a markup tag to be used in
-        # regex expressions. safe_unicode doesn't work here
+        # convert to plain `str` instead of a markup tag to be used in
+        # regex expressions. safe_str doesn't work here
         html_value = pat.sub(replace_html, value)
 
     return html_value
@@ -1117,7 +1119,7 b' def bool2icon(value, show_at_false=True)'
 
 
 def b64(inp):
-    return base64.b64encode(inp)
+    return base64.b64encode(safe_bytes(inp))
 
 #==============================================================================
 # PERMS
@@ -1230,23 +1232,22 b' class InitialsGravatar(object):'
         return color_bank[pos]
 
     def normalize_email(self, email_address):
-        import unicodedata
         # default host used to fill in the fake/missing email
         default_host = 'localhost'
 
         if not email_address:
-            email_address = '%s@%s' % (User.DEFAULT_USER, default_host)
-
-        email_address = safe_unicode(email_address)
-
-        if u'@' not in email_address:
-            email_address = u'%s@%s' % (email_address, default_host)
-
-        if email_address.endswith(u'@'):
-            email_address = u'%s%s' % (email_address, default_host)
-
-        email_address = unicodedata.normalize('NFKD', email_address)\
-            .encode('ascii', 'ignore')
+            email_address = f'{User.DEFAULT_USER}@{default_host}'
+
+        email_address = safe_str(email_address)
+
+        if '@' not in email_address:
+            email_address = f'{email_address}@{default_host}'
+
+        if email_address.endswith('@'):
+            email_address = f'{email_address}{default_host}'
+
+        email_address = convert_special_chars(email_address)
+
         return email_address
 
     def get_initials(self):
@@ -1266,12 +1267,11 b' class InitialsGravatar(object):'
         Function also normalizes the non-ascii characters to they ascii
         representation, eg Ą => A
         """
-        import unicodedata
         # replace non-ascii to ascii
-        first_name = unicodedata.normalize(
-            'NFKD', safe_unicode(self.first_name)).encode('ascii', 'ignore')
-        last_name = unicodedata.normalize(
-            'NFKD', safe_unicode(self.last_name)).encode('ascii', 'ignore')
+        first_name = convert_special_chars(self.first_name)
+        last_name = convert_special_chars(self.last_name)
+        # multi word last names, Guido Von Rossum, we take the last part only
+        last_name = last_name.split(' ', 1)[-1]
 
         # do NFKD encoding, and also make sure email has proper format
         email_address = self.normalize_email(self.email_address)
@@ -1286,9 +1286,9 b' class InitialsGravatar(object):'
         else:
             initials = [prefix[0], server[0]]
 
-        # then try to replace either first_name or last_name
+        # get first letter of first and last names to create initials
         fn_letter = (first_name or " ")[0].strip()
-        ln_letter = (last_name.split(' ', 1)[-1] or " ")[0].strip()
+        ln_letter = (last_name or " ")[0].strip()
 
         if fn_letter:
             initials[0] = fn_letter
@@ -1305,6 +1305,7 b' class InitialsGravatar(object):'
         viewBox="-15 -10 439.165 429.164"
 
         xml:space="preserve"
+        font-family="{font_family}
         style="background:{background};" >
 
         <path d="M204.583,216.671c50.664,0,91.74-48.075,
@@ -1374,8 +1375,8 b' class InitialsGravatar(object):'
         return img_data
 
     def generate_svg(self, svg_type=None):
-        img_data = self.get_img_data(svg_type)
-        return "data:image/svg+xml;base64,%s" % base64.b64encode(img_data)
+        img_data = safe_bytes(self.get_img_data(svg_type))
+        return "data:image/svg+xml;base64,%s" % safe_str(base64.b64encode(img_data))
 
 
 def initials_gravatar(request, email_address, first_name, last_name, size=30, store_on_disk=False):
@@ -1421,7 +1422,7 b' def initials_gravatar(request, email_add'
                 file_uid=store_uid, filename=metadata["filename"],
                 file_hash=metadata["sha256"], file_size=metadata["size"],
                 file_display_name=filename,
-                file_description=u'user gravatar `{}`'.format(safe_unicode(filename)),
+                file_description=f'user gravatar `{safe_str(filename)}`',
                 hidden=True, check_acl=False, user_id=1
             )
             Session().add(entry)
@@ -1694,7 +1695,7 b' def process_patterns(text_string, repo_n'
 
     log.debug('Got %s pattern entries to process', len(active_entries))
 
-    for uid, entry in active_entries.items():
+    for uid, entry in list(active_entries.items()):
 
         if not (entry['pat'] and entry['url']):
             log.debug('skipping due to missing data')
@@ -1841,10 +1842,10 b" def render(source, renderer='rst', menti"
                 for issue in issues:
                     issues_container_callback(issue)
 
-        return literal(
-            '<div class="rst-block">%s</div>' %
-            maybe_convert_relative_links(
-                MarkupRenderer.rst(source, mentions=mentions)))
+        rendered_block = maybe_convert_relative_links(
+                MarkupRenderer.rst(source, mentions=mentions))
+
+        return literal(f'<div class="rst-block">{rendered_block}</div>')
 
     elif renderer == 'markdown':
         if repo_name:
@@ -1856,18 +1857,14 b" def render(source, renderer='rst', menti"
                 for issue in issues:
                     issues_container_callback(issue)
 
-
-        return literal(
-            '<div class="markdown-block">%s</div>' %
-            maybe_convert_relative_links(
-                MarkupRenderer.markdown(source, flavored=True,
-                                        mentions=mentions)))
+        rendered_block = maybe_convert_relative_links(
+            MarkupRenderer.markdown(source, flavored=True, mentions=mentions))
+        return literal(f'<div class="markdown-block">{rendered_block}</div>')
 
     elif renderer == 'jupyter':
-        return literal(
-            '<div class="ipynb">%s</div>' %
-            maybe_convert_relative_links(
-                MarkupRenderer.jupyter(source)))
+        rendered_block = maybe_convert_relative_links(
+            MarkupRenderer.jupyter(source))
+        return literal(f'<div class="ipynb">{rendered_block}</div>')
 
     # None means just show the file-source
     return None
@@ -2020,10 +2017,10 b' def get_visual_attr(tmpl_context_var, at'
 
 def get_last_path_part(file_node):
     if not file_node.path:
-        return u'/'
-
-    path = safe_unicode(file_node.path.split('/')[-1])
-    return u'../' + path
+        return '/'
+
+    path = safe_str(file_node.path.split('/')[-1])
+    return '../' + path
 
 
 def route_url(*args, **kwargs):