html_sanitizer: abstracted bleach into own function/code for later replacement

- fixed bleach related test failures

rhodecode/tests/lib/test_html_sanitizer.py

@@ -0,0 +1,38 b''
+
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import pytest
+
+from rhodecode.lib.html_filters import sanitize_html
+
+
[e-mail](
+    "src_html, expected_html",
+    [
+        ('<div>ITEM</div>', '<div>ITEM</div>'),
+        ('<div>ITEM</div> <!-- comment here -->', '<div>ITEM</div> <!-- comment here -->'),
+        ('<div style="not-allowed:true">ITEM</div>', '<div style="">ITEM</div>'),
+        ('<div onload="ACTION">ITEM</div>', '<div>ITEM</div>'),
+        ('<a onload="ACTION" style="color:red">ITEM</a>', '<a style="color:red;">ITEM</a>'),
+        ('<img src="/file.png"></img>', '<img src="/file.png">'),
+        ('<img src="/file.png"></img>', '<img src="/file.png">'),
+    ])
+def test_html_sanitizer_options(src_html, expected_html):
+    parsed_html = sanitize_html(src_html)
+    assert parsed_html == expected_html

rhodecode/lib/helpers.py

@@ -1,2160 +1,2160 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Helper functions
 
 Consists of functions to typically be used within templates, but also
 available to Controllers. This module is available to both as 'h'.
 """
 import base64
 import collections
 
 import os
 import random
 import hashlib
 import io
 import textwrap
 import urllib.request
 import urllib.parse
 import urllib.error
 import math
 import logging
 import re
 import time
 import string
 import regex
 from collections import OrderedDict
 
 import pygments
 import itertools
 import fnmatch
-import bleach
 
 from datetime import datetime
 from functools import partial
 from pygments.formatters.html import HtmlFormatter
 from pygments.lexers import (
     get_lexer_by_name, get_lexer_for_filename, get_lexer_for_mimetype)
 
 from pyramid.threadlocal import get_current_request
 from tempita import looper
 from webhelpers2.html import literal, HTML, escape
 from webhelpers2.html._autolink import _auto_link_urls
 from webhelpers2.html.tools import (
     button_to, highlight, js_obfuscate, strip_links, strip_tags)
 
 from webhelpers2.text import (
     chop_at, collapse, convert_accented_entities,
     convert_misc_entities, lchop, plural, rchop, remove_formatting,
     replace_whitespace, urlify, truncate, wrap_paragraphs)
 from webhelpers2.date import time_ago_in_words
 
 from webhelpers2.html.tags import (
     _input, NotGiven, _make_safe_id_component as safeid,
     form as insecure_form,
     auto_discovery_link, checkbox, end_form, file,
     hidden, image, javascript_link, link_to, link_to_if, link_to_unless, ol,
     stylesheet_link, submit, text, password, textarea,
     ul, radio, Options)
 
 from webhelpers2.number import format_byte_size
 # python3.11 backport fixes for webhelpers2
 from rhodecode.lib._vendor.webhelpers_backports import raw_select
 
 from rhodecode.lib.action_parser import action_parser
+from rhodecode.lib.html_filters import sanitize_html
 from rhodecode.lib.pagination import Page, RepoPage, SqlPage
 from rhodecode.lib import ext_json
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.str_utils import safe_bytes, convert_special_chars
 from rhodecode.lib.utils import repo_name_slug, get_custom_lexer
 from rhodecode.lib.str_utils import safe_str
 from rhodecode.lib.utils2 import (
     str2bool,
     get_commit_safe, datetime_to_time, time_to_datetime, time_to_utcdatetime,
     AttributeDict, safe_int, md5, md5_safe, get_host_info)
 from rhodecode.lib.markup_renderer import MarkupRenderer, relative_links
 from rhodecode.lib.vcs.exceptions import CommitDoesNotExistError
 from rhodecode.lib.vcs.backends.base import BaseChangeset, EmptyCommit
 from rhodecode.lib.vcs.conf.settings import ARCHIVE_SPECS
 from rhodecode.lib.index.search_utils import get_matching_line_offsets
 from rhodecode.config.conf import DATE_FORMAT, DATETIME_FORMAT
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.db import Permission, User, Repository, UserApiKeys, FileStore
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.settings import IssueTrackerSettingsModel
 
 
 log = logging.getLogger(__name__)
 
 
 DEFAULT_USER = User.DEFAULT_USER
 DEFAULT_USER_EMAIL = User.DEFAULT_USER_EMAIL
 
 
 def asset(path, ver=None, **kwargs):
     """
     Helper to generate a static asset file path for rhodecode assets
 
     eg. h.asset('images/image.png', ver='3923')
 
     :param path: path of asset
     :param ver: optional version query param to append as ?ver=
     """
     request = get_current_request()
     query = {}
     query.update(kwargs)
     if ver:
         query = {'ver': ver}
     return request.static_path(
         f'rhodecode:public/{path}', _query=query)
 
 
 default_html_escape_table = {
     ord('&'): '&amp;',
     ord('<'): '&lt;',
     ord('>'): '&gt;',
     ord('"'): '&quot;',
     ord("'"): '&#39;',
 }
 
 
 def html_escape(text, html_escape_table=default_html_escape_table):
     """Produce entities within text."""
     return text.translate(html_escape_table)
 
 
 def str_json(*args, **kwargs):
     return ext_json.str_json(*args, **kwargs)
 
 
 def formatted_str_json(*args, **kwargs):
     return ext_json.formatted_str_json(*args, **kwargs)
 
 
 def chop_at_smart(s, sub, inclusive=False, suffix_if_chopped=None):
     """
     Truncate string ``s`` at the first occurrence of ``sub``.
 
     If ``inclusive`` is true, truncate just after ``sub`` rather than at it.
     """
     suffix_if_chopped = suffix_if_chopped or ''
     pos = s.find(sub)
     if pos == -1:
         return s
 
     if inclusive:
         pos += len(sub)
 
     chopped = s[:pos]
     left = s[pos:].strip()
 
     if left and suffix_if_chopped:
         chopped += suffix_if_chopped
 
     return chopped
 
 
 def shorter(text, size=20, prefix=False):
     postfix = '...'
     if len(text) > size:
         if prefix:
             # shorten in front
             return postfix + text[-(size - len(postfix)):]
         else:
             return text[:size - len(postfix)] + postfix
     return text
 
 
 def reset(name, value=None, id=NotGiven, type="reset", **attrs):
     """
     Reset button
     """
     return _input(type, name, value, id, attrs)
 
 
 def select(name, selected_values, options, id=NotGiven, **attrs):
 
     if isinstance(options, (list, tuple)):
         options_iter = options
         # Handle old value,label lists ... where value also can be value,label lists
         options = Options()
         for opt in options_iter:
             if isinstance(opt, tuple) and len(opt) == 2:
                 value, label = opt
             elif isinstance(opt, str):
                 value = label = opt
             else:
                 raise ValueError('invalid select option type %r' % type(opt))
 
             if isinstance(value, (list, tuple)):
                 option_group = options.add_optgroup(label)
                 for opt2 in value:
                     if isinstance(opt2, tuple) and len(opt2) == 2:
                         group_value, group_label = opt2
                     elif isinstance(opt2, str):
                         group_value = group_label = opt2
                     else:
                         raise ValueError('invalid select option type %r' % type(opt2))
 
                     option_group.add_option(group_label, group_value)
             else:
                 options.add_option(label, value)
 
     return raw_select(name, selected_values, options, id=id, **attrs)
 
 
 def branding(name, length=40):
     return truncate(name, length, indicator="")
 
 
 def FID(raw_id, path):
     """
     Creates a unique ID for filenode based on it's hash of path and commit
     it's safe to use in urls
 
     :param raw_id:
     :param path:
     """
 
     return 'c-%s-%s' % (short_id(raw_id), md5_safe(path)[:12])
 
 
 class _GetError(object):
     """Get error from form_errors, and represent it as span wrapped error
     message
 
     :param field_name: field to fetch errors for
     :param form_errors: form errors dict
     """
 
     def __call__(self, field_name, form_errors):
         tmpl = """<span class="error_msg">%s</span>"""
         if form_errors and field_name in form_errors:
             return literal(tmpl % form_errors.get(field_name))
 
 
 get_error = _GetError()
 
 
 class _ToolTip(object):
 
     def __call__(self, tooltip_title, trim_at=50):
         """
         Special function just to wrap our text into nice formatted
         autowrapped text
 
         :param tooltip_title:
         """
         tooltip_title = escape(tooltip_title)
         tooltip_title = tooltip_title.replace('<', '&lt;').replace('>', '&gt;')
         return tooltip_title
 
 
 tooltip = _ToolTip()
 
 files_icon = '<i class="file-breadcrumb-copy tooltip icon-clipboard clipboard-action" data-clipboard-text="{}" title="Copy file path"></i>'
 
 
 def files_breadcrumbs(repo_name, repo_type, commit_id, file_path, landing_ref_name=None, at_ref=None,
                       limit_items=False, linkify_last_item=False, hide_last_item=False,
                       copy_path_icon=True):
 
     if at_ref:
         route_qry = {'at': at_ref}
         default_landing_ref = at_ref or landing_ref_name or commit_id
     else:
         route_qry = None
         default_landing_ref = commit_id
 
     # first segment is a `HOME` link to repo files root location
     root_name = literal('<i class="icon-home"></i>')
 
     url_segments = [
         link_to(
             root_name,
             repo_files_by_ref_url(
                 repo_name,
                 repo_type,
                 f_path=None,  # None here is a special case for SVN repos,
                               # that won't prefix with a ref
                 ref_name=default_landing_ref,
                 commit_id=commit_id,
                 query=route_qry
             )
         )]
 
     path_segments = file_path.split('/')
     last_cnt = len(path_segments) - 1
     for cnt, segment in enumerate(path_segments):
         if not segment:
             continue
         segment_html = escape(segment)
 
         last_item = cnt == last_cnt
 
         if last_item and hide_last_item:
             # iterate over and hide last element
             continue
 
         if last_item and linkify_last_item is False:
             # plain version
             url_segments.append(segment_html)
         else:
             url_segments.append(
                 link_to(
                     segment_html,
                     repo_files_by_ref_url(
                         repo_name,
                         repo_type,
                         f_path='/'.join(path_segments[:cnt + 1]),
                         ref_name=default_landing_ref,
                         commit_id=commit_id,
                         query=route_qry
                     ),
                 ))
 
     limited_url_segments = url_segments[:1] + ['...'] + url_segments[-5:]
     if limit_items and len(limited_url_segments) < len(url_segments):
         url_segments = limited_url_segments
 
     full_path = file_path
     if copy_path_icon:
         icon = files_icon.format(escape(full_path))
     else:
         icon = ''
 
     if file_path == '':
         return root_name
     else:
         return literal(' / '.join(url_segments) + icon)
 
 
 def files_url_data(request):
     matchdict = request.matchdict
 
     if 'f_path' not in matchdict:
         matchdict['f_path'] = ''
     else:
         matchdict['f_path'] = urllib.parse.quote(safe_str(matchdict['f_path']))
     if 'commit_id' not in matchdict:
         matchdict['commit_id'] = 'tip'
 
     return ext_json.str_json(matchdict)
 
 
 def repo_files_by_ref_url(db_repo_name, db_repo_type, f_path, ref_name, commit_id, query=None, ):
     _is_svn = is_svn(db_repo_type)
     final_f_path = f_path
 
     if _is_svn:
         """
         For SVN the ref_name cannot be used as a commit_id, it needs to be prefixed with
         actually commit_id followed by the ref_name. This should be done only in case
         This is a initial landing url, without additional paths.
 
         like: /1000/tags/1.0.0/?at=tags/1.0.0
         """
 
         if ref_name and ref_name != 'tip':
             # NOTE(marcink): for svn the ref_name is actually the stored path, so we prefix it
             # for SVN we only do this magic prefix if it's root, .eg landing revision
             # of files link. If we are in the tree we don't need this since we traverse the url
             # that has everything stored
             if f_path in ['', '/']:
                 final_f_path = '/'.join([ref_name, f_path])
 
         # SVN always needs a commit_id explicitly, without a named REF
         default_commit_id = commit_id
     else:
         """
         For git and mercurial we construct a new URL using the names instead of commit_id
         like: /master/some_path?at=master
         """
         # We currently do not support branches with slashes
         if '/' in ref_name:
             default_commit_id = commit_id
         else:
             default_commit_id = ref_name
 
     # sometimes we pass f_path as None, to indicate explicit no prefix,
     # we translate it to string to not have None
     final_f_path = final_f_path or ''
 
     files_url = route_path(
         'repo_files',
         repo_name=db_repo_name,
         commit_id=default_commit_id,
         f_path=final_f_path,
         _query=query
     )
     return files_url
 
 
 def code_highlight(code, lexer, formatter, use_hl_filter=False):
     """
     Lex ``code`` with ``lexer`` and format it with the formatter ``formatter``.
 
     If ``outfile`` is given and a valid file object (an object
     with a ``write`` method), the result will be written to it, otherwise
     it is returned as a string.
     """
     if use_hl_filter:
         # add HL filter
         from rhodecode.lib.index import search_utils
         lexer.add_filter(search_utils.ElasticSearchHLFilter())
     return pygments.format(pygments.lex(code, lexer), formatter)
 
 
 class CodeHtmlFormatter(HtmlFormatter):
     """
     My code Html Formatter for source codes
     """
 
     def wrap(self, source):
         return self._wrap_div(self._wrap_pre(self._wrap_code(source)))
 
     def _wrap_code(self, source):
         for cnt, it in enumerate(source):
             i, t = it
             t = f'<div id="L{cnt+1}">{t}</div>'
             yield i, t
 
     def _wrap_tablelinenos(self, inner):
         dummyoutfile = io.StringIO()
         lncount = 0
         for t, line in inner:
             if t:
                 lncount += 1
             dummyoutfile.write(line)
 
         fl = self.linenostart
         mw = len(str(lncount + fl - 1))
         sp = self.linenospecial
         st = self.linenostep
         la = self.lineanchors
         aln = self.anchorlinenos
         nocls = self.noclasses
         if sp:
             lines = []
 
             for i in range(fl, fl + lncount):
                 if i % st == 0:
                     if i % sp == 0:
                         if aln:
                             lines.append('<a href="#%s%d" class="special">%*d</a>' %
                                          (la, i, mw, i))
                         else:
                             lines.append('<span class="special">%*d</span>' % (mw, i))
                     else:
                         if aln:
                             lines.append('<a href="#%s%d">%*d</a>' % (la, i, mw, i))
                         else:
                             lines.append('%*d' % (mw, i))
                 else:
                     lines.append('')
             ls = '\n'.join(lines)
         else:
             lines = []
             for i in range(fl, fl + lncount):
                 if i % st == 0:
                     if aln:
                         lines.append('<a href="#%s%d">%*d</a>' % (la, i, mw, i))
                     else:
                         lines.append('%*d' % (mw, i))
                 else:
                     lines.append('')
             ls = '\n'.join(lines)
 
         # in case you wonder about the seemingly redundant <div> here: since the
         # content in the other cell also is wrapped in a div, some browsers in
         # some configurations seem to mess up the formatting...
         if nocls:
             yield 0, ('<table class="%stable">' % self.cssclass +
                       '<tr><td><div class="linenodiv" '
                       'style="background-color: #f0f0f0; padding-right: 10px">'
                       '<pre style="line-height: 125%">' +
                       ls + '</pre></div></td><td id="hlcode" class="code">')
         else:
             yield 0, ('<table class="%stable">' % self.cssclass +
                       '<tr><td class="linenos"><div class="linenodiv"><pre>' +
                       ls + '</pre></div></td><td id="hlcode" class="code">')
         yield 0, dummyoutfile.getvalue()
         yield 0, '</td></tr></table>'
 
 
 class SearchContentCodeHtmlFormatter(CodeHtmlFormatter):
     def __init__(self, **kw):
         # only show these line numbers if set
         self.only_lines = kw.pop('only_line_numbers', [])
         self.query_terms = kw.pop('query_terms', [])
         self.max_lines = kw.pop('max_lines', 5)
         self.line_context = kw.pop('line_context', 3)
         self.url = kw.pop('url', None)
 
         super(CodeHtmlFormatter, self).__init__(**kw)
 
     def _wrap_code(self, source):
         for cnt, it in enumerate(source):
             i, t = it
             t = '<pre>%s</pre>' % t
             yield i, t
 
     def _wrap_tablelinenos(self, inner):
         yield 0, '<table class="code-highlight %stable">' % self.cssclass
 
         last_shown_line_number = 0
         current_line_number = 1
 
         for t, line in inner:
             if not t:
                 yield t, line
                 continue
 
             if current_line_number in self.only_lines:
                 if last_shown_line_number + 1 != current_line_number:
                     yield 0, '<tr>'
                     yield 0, '<td class="line">...</td>'
                     yield 0, '<td id="hlcode" class="code"></td>'
                     yield 0, '</tr>'
 
                 yield 0, '<tr>'
                 if self.url:
                     yield 0, '<td class="line"><a href="%s#L%i">%i</a></td>' % (
                         self.url, current_line_number, current_line_number)
                 else:
                     yield 0, '<td class="line"><a href="">%i</a></td>' % (
                         current_line_number)
                 yield 0, '<td id="hlcode" class="code">' + line + '</td>'
                 yield 0, '</tr>'
 
                 last_shown_line_number = current_line_number
 
             current_line_number += 1
 
         yield 0, '</table>'
 
 
 def hsv_to_rgb(h, s, v):
     """ Convert hsv color values to rgb """
 
     if s == 0.0:
         return v, v, v
     i = int(h * 6.0)  # XXX assume int() truncates!
     f = (h * 6.0) - i
     p = v * (1.0 - s)
     q = v * (1.0 - s * f)
     t = v * (1.0 - s * (1.0 - f))
     i = i % 6
     if i == 0:
         return v, t, p
     if i == 1:
         return q, v, p
     if i == 2:
         return p, v, t
     if i == 3:
         return p, q, v
     if i == 4:
         return t, p, v
     if i == 5:
         return v, p, q
 
 
 def unique_color_generator(n=10000, saturation=0.10, lightness=0.95):
     """
     Generator for getting n of evenly distributed colors using
     hsv color and golden ratio. It always return same order of colors
 
     :param n: number of colors to generate
     :param saturation: saturation of returned colors
     :param lightness: lightness of returned colors
     :returns: RGB tuple
     """
 
     golden_ratio = 0.618033988749895
     h = 0.22717784590367374
 
     for _ in range(n):
         h += golden_ratio
         h %= 1
         HSV_tuple = [h, saturation, lightness]
         RGB_tuple = hsv_to_rgb(*HSV_tuple)
         yield [str(int(x * 256)) for x in RGB_tuple]
 
 
 def color_hasher(n=10000, saturation=0.10, lightness=0.95):
     """
     Returns a function which when called with an argument returns a unique
     color for that argument, eg.
 
     :param n: number of colors to generate
     :param saturation: saturation of returned colors
     :param lightness: lightness of returned colors
     :returns: css RGB string
 
     >>> color_hash = color_hasher()
     >>> color_hash('hello')
     'rgb(34, 12, 59)'
     >>> color_hash('hello')
     'rgb(34, 12, 59)'
     >>> color_hash('other')
     'rgb(90, 224, 159)'
     """
 
     color_dict = {}
     cgenerator = unique_color_generator(
         saturation=saturation, lightness=lightness)
 
     def get_color_string(thing):
         if thing in color_dict:
             col = color_dict[thing]
         else:
             col = color_dict[thing] = next(cgenerator)
         return "rgb(%s)" % (', '.join(col))
 
     return get_color_string
 
 
 def get_lexer_safe(mimetype=None, filepath=None):
     """
     Tries to return a relevant pygments lexer using mimetype/filepath name,
     defaulting to plain text if none could be found
     """
     lexer = None
     try:
         if mimetype:
             lexer = get_lexer_for_mimetype(mimetype)
         if not lexer:
             lexer = get_lexer_for_filename(filepath)
     except pygments.util.ClassNotFound:
         pass
 
     if not lexer:
         lexer = get_lexer_by_name('text')
 
     return lexer
 
 
 def get_lexer_for_filenode(filenode):
     lexer = get_custom_lexer(filenode.extension) or filenode.lexer
     return lexer
 
 
 def pygmentize(filenode, **kwargs):
     """
     pygmentize function using pygments
 
     :param filenode:
     """
     lexer = get_lexer_for_filenode(filenode)
     return literal(code_highlight(filenode.content, lexer,
                                   CodeHtmlFormatter(**kwargs)))
 
 
 def is_following_repo(repo_name, user_id):
     from rhodecode.model.scm import ScmModel
     return ScmModel().is_following_repo(repo_name, user_id)
 
 
 class _Message(object):
     """A message returned by ``Flash.pop_messages()``.
 
     Converting the message to a string returns the message text. Instances
     also have the following attributes:
 
     * ``message``: the message text.
     * ``category``: the category specified when the message was created.
     """
 
     def __init__(self, category, message, sub_data=None):
         self.category = category
         self.message = message
         self.sub_data = sub_data or {}
 
     def __str__(self):
         return self.message
 
     __unicode__ = __str__
 
     def __html__(self):
         return escape(safe_str(self.message))
 
 
 class Flash(object):
     # List of allowed categories.  If None, allow any category.
     categories = ["warning", "notice", "error", "success"]
 
     # Default category if none is specified.
     default_category = "notice"
 
     def __init__(self, session_key="flash", categories=None,
                  default_category=None):
         """
         Instantiate a ``Flash`` object.
 
         ``session_key`` is the key to save the messages under in the user's
         session.
 
         ``categories`` is an optional list which overrides the default list
         of categories.
 
         ``default_category`` overrides the default category used for messages
         when none is specified.
         """
         self.session_key = session_key
         if categories is not None:
             self.categories = categories
         if default_category is not None:
             self.default_category = default_category
         if self.categories and self.default_category not in self.categories:
             raise ValueError(
                 "unrecognized default category %r" % (self.default_category,))
 
     def pop_messages(self, session=None, request=None):
         """
         Return all accumulated messages and delete them from the session.
 
         The return value is a list of ``Message`` objects.
         """
         messages = []
 
         if not session:
             if not request:
                 request = get_current_request()
             session = request.session
 
         # Pop the 'old' pylons flash messages. They are tuples of the form
         # (category, message)
         for cat, msg in session.pop(self.session_key, []):
             messages.append(_Message(cat, msg))
 
         # Pop the 'new' pyramid flash messages for each category as list
         # of strings.
         for cat in self.categories:
             for msg in session.pop_flash(queue=cat):
                 sub_data = {}
                 if hasattr(msg, 'rsplit'):
                     flash_data = msg.rsplit('|DELIM|', 1)
                     org_message = flash_data[0]
                     if len(flash_data) > 1:
                         sub_data = json.loads(flash_data[1])
                 else:
                     org_message = msg
 
                 messages.append(_Message(cat, org_message, sub_data=sub_data))
 
         # Map messages from the default queue to the 'notice' category.
         for msg in session.pop_flash():
             messages.append(_Message('notice', msg))
 
         session.save()
         return messages
 
     def json_alerts(self, session=None, request=None):
         payloads = []
         messages = flash.pop_messages(session=session, request=request) or []
         for message in messages:
             payloads.append({
                 'message': {
                     'message': '{}'.format(message.message),
                     'level': message.category,
                     'force': True,
                     'subdata': message.sub_data
                 }
             })
         return safe_str(json.dumps(payloads))
 
     def __call__(self, message, category=None, ignore_duplicate=True,
                  session=None, request=None):
 
         if not session:
             if not request:
                 request = get_current_request()
             session = request.session
 
         session.flash(
             message, queue=category, allow_duplicate=not ignore_duplicate)
 
 
 flash = Flash()
 
 #==============================================================================
 # SCM FILTERS available via h.
 #==============================================================================
 from rhodecode.lib.vcs.utils import author_name, author_email
 from rhodecode.lib.utils2 import age, age_from_seconds
 from rhodecode.model.db import User, ChangesetStatus
 
 
 email = author_email
 
 
 def capitalize(raw_text):
     return raw_text.capitalize()
 
 
 def short_id(long_id):
     return long_id[:12]
 
 
 def hide_credentials(url):
     from rhodecode.lib.utils2 import credentials_filter
     return credentials_filter(url)
 
 import zoneinfo
 import tzlocal
 local_timezone = tzlocal.get_localzone()
 
 
 def get_timezone(datetime_iso, time_is_local=False):
     tzinfo = '+00:00'
 
     # detect if we have a timezone info, otherwise, add it
     if time_is_local and isinstance(datetime_iso, datetime) and not datetime_iso.tzinfo:
         force_timezone = os.environ.get('RC_TIMEZONE', '')
         if force_timezone:
             force_timezone = zoneinfo.ZoneInfo(force_timezone)
         timezone = force_timezone or local_timezone
 
         offset = datetime_iso.replace(tzinfo=timezone).strftime('%z')
         tzinfo = '{}:{}'.format(offset[:-2], offset[-2:])
     return tzinfo
 
 
 def age_component(datetime_iso, value=None, time_is_local=False, tooltip=True):
     title = value or format_date(datetime_iso)
     tzinfo = get_timezone(datetime_iso, time_is_local=time_is_local)
 
     return literal(
         '<time class="timeago {cls}" title="{tt_title}" datetime="{dt}{tzinfo}">{title}</time>'.format(
             cls='tooltip' if tooltip else '',
             tt_title=('{title}{tzinfo}'.format(title=title, tzinfo=tzinfo)) if tooltip else '',
             title=title, dt=datetime_iso, tzinfo=tzinfo
         ))
 
 
 def _shorten_commit_id(commit_id, commit_len=None):
     if commit_len is None:
         request = get_current_request()
         commit_len = request.call_context.visual.show_sha_length
     return commit_id[:commit_len]
 
 
 def show_id(commit, show_idx=None, commit_len=None):
     """
     Configurable function that shows ID
     by default it's r123:fffeeefffeee
 
     :param commit: commit instance
     """
     if show_idx is None:
         request = get_current_request()
         show_idx = request.call_context.visual.show_revision_number
 
     raw_id = _shorten_commit_id(commit.raw_id, commit_len=commit_len)
     if show_idx:
         return 'r%s:%s' % (commit.idx, raw_id)
     else:
         return '%s' % (raw_id, )
 
 
 def format_date(date):
     """
     use a standardized formatting for dates used in RhodeCode
 
     :param date: date/datetime object
     :return: formatted date
     """
 
     if date:
         _fmt = "%a, %d %b %Y %H:%M:%S"
         return safe_str(date.strftime(_fmt))
 
     return ""
 
 
 class _RepoChecker(object):
 
     def __init__(self, backend_alias):
         self._backend_alias = backend_alias
 
     def __call__(self, repository):
         if hasattr(repository, 'alias'):
             _type = repository.alias
         elif hasattr(repository, 'repo_type'):
             _type = repository.repo_type
         else:
             _type = repository
         return _type == self._backend_alias
 
 
 is_git = _RepoChecker('git')
 is_hg = _RepoChecker('hg')
 is_svn = _RepoChecker('svn')
 
 
 def get_repo_type_by_name(repo_name):
     repo = Repository.get_by_repo_name(repo_name)
     if repo:
         return repo.repo_type
 
 
 def is_svn_without_proxy(repository):
     if is_svn(repository):
         from rhodecode.model.settings import VcsSettingsModel
         conf = VcsSettingsModel().get_ui_settings_as_config_obj()
         return not str2bool(conf.get('vcs_svn_proxy', 'http_requests_enabled'))
     return False
 
 
 def discover_user(author):
     """
     Tries to discover RhodeCode User based on the author string. Author string
     is typically `FirstName LastName <[e-mail]>`
     """
 
     # if author is already an instance use it for extraction
     if isinstance(author, User):
         return author
 
     # Valid email in the attribute passed, see if they're in the system
     _email = author_email(author)
     if _email != '':
         user = User.get_by_email(_email, case_insensitive=True, cache=True)
         if user is not None:
             return user
 
     # Maybe it's a username, we try to extract it and fetch by username ?
     _author = author_name(author)
     user = User.get_by_username(_author, case_insensitive=True, cache=True)
     if user is not None:
         return user
 
     return None
 
 
 def email_or_none(author):
     # extract email from the commit string
     _email = author_email(author)
 
     # If we have an email, use it, otherwise
     # see if it contains a username we can get an email from
     if _email != '':
         return _email
     else:
         user = User.get_by_username(
             author_name(author), case_insensitive=True, cache=True)
 
     if user is not None:
             return user.email
 
     # No valid email, not a valid user in the system, none!
     return None
 
 
 def link_to_user(author, length=0, **kwargs):
     user = discover_user(author)
     # user can be None, but if we have it already it means we can re-use it
     # in the person() function, so we save 1 intensive-query
     if user:
         author = user
 
     display_person = person(author, 'username_or_name_or_email')
     if length:
         display_person = shorter(display_person, length)
 
     if user and user.username != user.DEFAULT_USER:
         return link_to(
             escape(display_person),
             route_path('user_profile', username=user.username),
             **kwargs)
     else:
         return escape(display_person)
 
 
 def link_to_group(users_group_name, **kwargs):
     return link_to(
         escape(users_group_name),
         route_path('user_group_profile', user_group_name=users_group_name),
         **kwargs)
 
 
 def person(author, show_attr="username_and_name"):
     user = discover_user(author)
     if user:
         return getattr(user, show_attr)
     else:
         _author = author_name(author)
         _email = email(author)
         return _author or _email
 
 
 def author_string(email):
     if email:
         user = User.get_by_email(email, case_insensitive=True, cache=True)
         if user:
             if user.first_name or user.last_name:
                 return '%s %s &lt;%s&gt;' % (
                     user.first_name, user.last_name, email)
             else:
                 return email
         else:
             return email
     else:
         return None
 
 
 def person_by_id(id_, show_attr="username_and_name"):
     # attr to return from fetched user
     def person_getter(usr):
         return getattr(usr, show_attr)
 
     #maybe it's an ID ?
     if str(id_).isdigit() or isinstance(id_, int):
         id_ = int(id_)
         user = User.get(id_)
         if user is not None:
             return person_getter(user)
     return id_
 
 
 def gravatar_with_user(request, author, show_disabled=False, tooltip=False):
     _render = request.get_partial_renderer('rhodecode:templates/base/base.mako')
     return _render('gravatar_with_user', author, show_disabled=show_disabled, tooltip=tooltip)
 
 
 tags_paterns = OrderedDict((
     ('lang', (re.compile(r'\[(lang|language)\ \=\&gt;\ *([a-zA-Z\-\/\#\+\.]*)\]'),
              '<div class="metatag" tag="lang">\\2</div>')),
 
     ('see', (re.compile(r'\[see\ \=\&gt;\ *([a-zA-Z0-9\/\=\?\&amp;\ \:\/\.\-]*)\]'),
             '<div class="metatag" tag="see">see: \\1 </div>')),
 
     ('url', (re.compile(r'\[url\ \=\&gt;\ \[([a-zA-Z0-9\ \.\-\_]+)\]\((http://|https://|/)(.*?)\)\]'),
             '<div class="metatag" tag="url"> <a href="\\2\\3">\\1</a> </div>')),
 
     ('license', (re.compile(r'\[license\ \=\&gt;\ *([a-zA-Z0-9\/\=\?\&amp;\ \:\/\.\-]*)\]'),
                 '<div class="metatag" tag="license"><a href="http:\/\/www.opensource.org/licenses/\\1">\\1</a></div>')),
 
     ('ref', (re.compile(r'\[(requires|recommends|conflicts|base)\ \=\&gt;\ *([a-zA-Z0-9\-\/]*)\]'),
             '<div class="metatag" tag="ref \\1">\\1: <a href="/\\2">\\2</a></div>')),
 
     ('state', (re.compile(r'\[(stable|featured|stale|dead|dev|deprecated)\]'),
               '<div class="metatag" tag="state \\1">\\1</div>')),
 
     # label in grey
     ('label', (re.compile(r'\[([a-z]+)\]'),
               '<div class="metatag" tag="label">\\1</div>')),
 
     # generic catch all in grey
     ('generic', (re.compile(r'\[([a-zA-Z0-9\.\-\_]+)\]'),
                 '<div class="metatag" tag="generic">\\1</div>')),
 ))
 
 
 def extract_metatags(value):
     """
     Extract supported meta-tags from given text value
     """
     tags = []
     if not value:
         return tags, ''
 
     for key, val in list(tags_paterns.items()):
         pat, replace_html = val
         tags.extend([(key, x.group()) for x in pat.finditer(value)])
         value = pat.sub('', value)
 
     return tags, value
 
 
 def style_metatag(tag_type, value):
     """
     converts tags from value into html equivalent
     """
     if not value:
         return ''
 
     html_value = value
     tag_data = tags_paterns.get(tag_type)
     if tag_data:
         pat, replace_html = tag_data
         # convert to plain `str` instead of a markup tag to be used in
         # regex expressions. safe_str doesn't work here
         html_value = pat.sub(replace_html, value)
 
     return html_value
 
 
 def bool2icon(value, show_at_false=True):
     """
     Returns boolean value of a given value, represented as html element with
     classes that will represent icons
 
     :param value: given value to convert to html node
     """
 
     if value:  # does bool conversion
         return HTML.tag('i', class_="icon-true", title='True')
     else:  # not true as bool
         if show_at_false:
             return HTML.tag('i', class_="icon-false", title='False')
         return HTML.tag('i')
 
 
 def b64(inp):
     return base64.b64encode(safe_bytes(inp))
 
 #==============================================================================
 # PERMS
 #==============================================================================
 from rhodecode.lib.auth import (
     HasPermissionAny, HasPermissionAll,
     HasRepoPermissionAny, HasRepoPermissionAll, HasRepoGroupPermissionAll,
     HasRepoGroupPermissionAny, HasRepoPermissionAnyApi, get_csrf_token,
     csrf_token_key, AuthUser)
 
 
 #==============================================================================
 # GRAVATAR URL
 #==============================================================================
 class InitialsGravatar(object):
     def __init__(self, email_address, first_name, last_name, size=30,
                  background=None, text_color='#fff'):
         self.size = size
         self.first_name = first_name
         self.last_name = last_name
         self.email_address = email_address
         self.background = background or self.str2color(email_address)
         self.text_color = text_color
 
     def get_color_bank(self):
         """
         returns a predefined list of colors that gravatars can use.
         Those are randomized distinct colors that guarantee readability and
         uniqueness.
 
         generated with: http://phrogz.net/css/distinct-colors.html
         """
         return [
             '#bf3030', '#a67f53', '#00ff00', '#5989b3', '#392040', '#d90000',
             '#402910', '#204020', '#79baf2', '#a700b3', '#bf6060', '#7f5320',
             '#008000', '#003059', '#ee00ff', '#ff0000', '#8c4b00', '#007300',
             '#005fb3', '#de73e6', '#ff4040', '#ffaa00', '#3df255', '#203140',
             '#47004d', '#591616', '#664400', '#59b365', '#0d2133', '#83008c',
             '#592d2d', '#bf9f60', '#73e682', '#1d3f73', '#73006b', '#402020',
             '#b2862d', '#397341', '#597db3', '#e600d6', '#a60000', '#736039',
             '#00b318', '#79aaf2', '#330d30', '#ff8080', '#403010', '#16591f',
             '#002459', '#8c4688', '#e50000', '#ffbf40', '#00732e', '#102340',
             '#bf60ac', '#8c4646', '#cc8800', '#00a642', '#1d3473', '#b32d98',
             '#660e00', '#ffd580', '#80ffb2', '#7391e6', '#733967', '#d97b6c',
             '#8c5e00', '#59b389', '#3967e6', '#590047', '#73281d', '#665200',
             '#00e67a', '#2d50b3', '#8c2377', '#734139', '#b2982d', '#16593a',
             '#001859', '#ff00aa', '#a65e53', '#ffcc00', '#0d3321', '#2d3959',
             '#731d56', '#401610', '#4c3d00', '#468c6c', '#002ca6', '#d936a3',
             '#d94c36', '#403920', '#36d9a3', '#0d1733', '#592d4a', '#993626',
             '#cca300', '#00734d', '#46598c', '#8c005e', '#7f1100', '#8c7000',
             '#00a66f', '#7382e6', '#b32d74', '#d9896c', '#ffe680', '#1d7362',
             '#364cd9', '#73003d', '#d93a00', '#998a4d', '#59b3a1', '#5965b3',
             '#e5007a', '#73341d', '#665f00', '#00b38f', '#0018b3', '#59163a',
             '#b2502d', '#bfb960', '#00ffcc', '#23318c', '#a6537f', '#734939',
             '#b2a700', '#104036', '#3d3df2', '#402031', '#e56739', '#736f39',
             '#79f2ea', '#000059', '#401029', '#4c1400', '#ffee00', '#005953',
             '#101040', '#990052', '#402820', '#403d10', '#00ffee', '#0000d9',
             '#ff80c4', '#a66953', '#eeff00', '#00ccbe', '#8080ff', '#e673a1',
             '#a62c00', '#474d00', '#1a3331', '#46468c', '#733950', '#662900',
             '#858c23', '#238c85', '#0f0073', '#b20047', '#d9986c', '#becc00',
             '#396f73', '#281d73', '#ff0066', '#ff6600', '#dee673', '#59adb3',
             '#6559b3', '#590024', '#b2622d', '#98b32d', '#36ced9', '#332d59',
             '#40001a', '#733f1d', '#526600', '#005359', '#242040', '#bf6079',
             '#735039', '#cef23d', '#007780', '#5630bf', '#66001b', '#b24700',
             '#acbf60', '#1d6273', '#25008c', '#731d34', '#a67453', '#50592d',
             '#00ccff', '#6600ff', '#ff0044', '#4c1f00', '#8a994d', '#79daf2',
             '#a173e6', '#d93662', '#402310', '#aaff00', '#2d98b3', '#8c40ff',
             '#592d39', '#ff8c40', '#354020', '#103640', '#1a0040', '#331a20',
             '#331400', '#334d00', '#1d5673', '#583973', '#7f0022', '#4c3626',
             '#88cc00', '#36a3d9', '#3d0073', '#d9364c', '#33241a', '#698c23',
             '#5995b3', '#300059', '#e57382', '#7f3300', '#366600', '#00aaff',
             '#3a1659', '#733941', '#663600', '#74b32d', '#003c59', '#7f53a6',
             '#73000f', '#ff8800', '#baf279', '#79caf2', '#291040', '#a6293a',
             '#b2742d', '#587339', '#0077b3', '#632699', '#400009', '#d9a66c',
             '#294010', '#2d4a59', '#aa00ff', '#4c131b', '#b25f00', '#5ce600',
             '#267399', '#a336d9', '#990014', '#664e33', '#86bf60', '#0088ff',
             '#7700b3', '#593a16', '#073300', '#1d4b73', '#ac60bf', '#e59539',
             '#4f8c46', '#368dd9', '#5c0073'
         ]
 
     def rgb_to_hex_color(self, rgb_tuple):
         """
         Converts an rgb_tuple passed to an hex color.
 
         :param rgb_tuple: tuple with 3 ints represents rgb color space
         """
         return '#' + ("".join(map(chr, rgb_tuple)).encode('hex'))
 
     def email_to_int_list(self, email_str):
         """
         Get every byte of the hex digest value of email and turn it to integer.
         It's going to be always between 0-255
         """
         digest = md5_safe(email_str.lower())
         return [int(digest[i * 2:i * 2 + 2], 16) for i in range(16)]
 
     def pick_color_bank_index(self, email_str, color_bank):
         return self.email_to_int_list(email_str)[0] % len(color_bank)
 
     def str2color(self, email_str):
         """
         Tries to map in a stable algorithm an email to color
 
         :param email_str:
         """
         color_bank = self.get_color_bank()
         # pick position (module it's length so we always find it in the
         # bank even if it's smaller than 256 values
         pos = self.pick_color_bank_index(email_str, color_bank)
         return color_bank[pos]
 
     def normalize_email(self, email_address):
         # default host used to fill in the fake/missing email
         default_host = 'localhost'
 
         if not email_address:
             email_address = f'{User.DEFAULT_USER}@{default_host}'
 
         email_address = safe_str(email_address)
 
         if '@' not in email_address:
             email_address = f'{email_address}@{default_host}'
 
         if email_address.endswith('@'):
             email_address = f'{email_address}{default_host}'
 
         email_address = convert_special_chars(email_address)
 
         return email_address
 
     def get_initials(self):
         """
         Returns 2 letter initials calculated based on the input.
         The algorithm picks first given email address, and takes first letter
         of part before @, and then the first letter of server name. In case
         the part before @ is in a format of `somestring.somestring2` it replaces
         the server letter with first letter of somestring2
 
         In case function was initialized with both first and lastname, this
         overrides the extraction from email by first letter of the first and
         last name. We add special logic to that functionality, In case Full name
         is compound, like Guido Von Rossum, we use last part of the last name
         (Von Rossum) picking `R`.
 
         Function also normalizes the non-ascii characters to they ascii
         representation, eg Ą => A
         """
         # replace non-ascii to ascii
         first_name = convert_special_chars(self.first_name)
         last_name = convert_special_chars(self.last_name)
         # multi word last names, Guido Von Rossum, we take the last part only
         last_name = last_name.split(' ', 1)[-1]
 
         # do NFKD encoding, and also make sure email has proper format
         email_address = self.normalize_email(self.email_address)
 
         # first push the email initials
         prefix, server = email_address.split('@', 1)
 
         # check if prefix is maybe a 'first_name.last_name' syntax
         _dot_split = prefix.rsplit('.', 1)
         if len(_dot_split) == 2 and _dot_split[1]:
             initials = [_dot_split[0][0], _dot_split[1][0]]
         else:
             initials = [prefix[0], server[0]]
 
         # get first letter of first and last names to create initials
         fn_letter = (first_name or " ")[0].strip()
         ln_letter = (last_name or " ")[0].strip()
 
         if fn_letter:
             initials[0] = fn_letter
 
         if ln_letter:
             initials[1] = ln_letter
 
         return ''.join(initials).upper()
 
     def get_img_data_by_type(self, font_family, img_type):
         default_user = """
         <svg xmlns="http://www.w3.org/2000/svg"
         version="1.1" x="0px" y="0px" width="{size}" height="{size}"
         viewBox="-15 -10 439.165 429.164"
 
         xml:space="preserve"
         font-family="{font_family}
         style="background:{background};" >
 
         <path d="M204.583,216.671c50.664,0,91.74-48.075,
                  91.74-107.378c0-82.237-41.074-107.377-91.74-107.377
                  c-50.668,0-91.74,25.14-91.74,107.377C112.844,
                  168.596,153.916,216.671,
                  204.583,216.671z" fill="{text_color}"/>
         <path d="M407.164,374.717L360.88,
                  270.454c-2.117-4.771-5.836-8.728-10.465-11.138l-71.83-37.392
                  c-1.584-0.823-3.502-0.663-4.926,0.415c-20.316,
                  15.366-44.203,23.488-69.076,23.488c-24.877,
                  0-48.762-8.122-69.078-23.488
                  c-1.428-1.078-3.346-1.238-4.93-0.415L58.75,
                  259.316c-4.631,2.41-8.346,6.365-10.465,11.138L2.001,374.717
                  c-3.191,7.188-2.537,15.412,1.75,22.005c4.285,
                  6.592,11.537,10.526,19.4,10.526h362.861c7.863,0,15.117-3.936,
                  19.402-10.527 C409.699,390.129,
                  410.355,381.902,407.164,374.717z" fill="{text_color}"/>
         </svg>""".format(
             size=self.size,
             background='#979797',  # @grey4
             text_color=self.text_color,
             font_family=font_family)
 
         return {
             "default_user": default_user
         }[img_type]
 
     def get_img_data(self, svg_type=None):
         """
         generates the svg metadata for image
         """
         fonts = [
             '-apple-system',
              'BlinkMacSystemFont',
              'Segoe UI',
              'Roboto',
              'Oxygen-Sans',
              'Ubuntu',
              'Cantarell',
              'Helvetica Neue',
              'sans-serif'
         ]
         font_family = ','.join(fonts)
         if svg_type:
             return self.get_img_data_by_type(font_family, svg_type)
 
         initials = self.get_initials()
         img_data = """
         <svg xmlns="http://www.w3.org/2000/svg" pointer-events="none"
              width="{size}" height="{size}"
              style="width: 100%; height: 100%; background-color: {background}"
              viewBox="0 0 {size} {size}">
             <text text-anchor="middle" y="50%" x="50%" dy="0.35em"
                   pointer-events="auto" fill="{text_color}"
                   font-family="{font_family}"
                   style="font-weight: 400; font-size: {f_size}px;">{text}
             </text>
         </svg>""".format(
             size=self.size,
             f_size=self.size/2.05,  # scale the text inside the box nicely
             background=self.background,
             text_color=self.text_color,
             text=initials.upper(),
             font_family=font_family)
 
         return img_data
 
     def generate_svg(self, svg_type=None):
         img_data = safe_bytes(self.get_img_data(svg_type))
         return "data:image/svg+xml;base64,%s" % safe_str(base64.b64encode(img_data))
 
 
 def initials_gravatar(request, email_address, first_name, last_name, size=30, store_on_disk=False):
 
     svg_type = None
     if email_address == User.DEFAULT_USER_EMAIL:
         svg_type = 'default_user'
 
     klass = InitialsGravatar(email_address, first_name, last_name, size)
 
     if store_on_disk:
         from rhodecode.apps.file_store import utils as store_utils
         from rhodecode.apps.file_store.exceptions import FileNotAllowedException, \
             FileOverSizeException
         from rhodecode.model.db import Session
 
         image_key = md5_safe(email_address.lower()
                              + first_name.lower() + last_name.lower())
 
         storage = store_utils.get_file_storage(request.registry.settings)
         filename = '{}.svg'.format(image_key)
         subdir = 'gravatars'
         # since final name has a counter, we apply the 0
         uid = storage.apply_counter(0, store_utils.uid_filename(filename, randomized=False))
         store_uid = os.path.join(subdir, uid)
 
         db_entry = FileStore.get_by_store_uid(store_uid)
         if db_entry:
             return request.route_path('download_file', fid=store_uid)
 
         img_data = klass.get_img_data(svg_type=svg_type)
         img_file = store_utils.bytes_to_file_obj(img_data)
 
         try:
             store_uid, metadata = storage.save_file(
                 img_file, filename, directory=subdir,
                 extensions=['.svg'], randomized_name=False)
         except (FileNotAllowedException, FileOverSizeException):
             raise
 
         try:
             entry = FileStore.create(
                 file_uid=store_uid, filename=metadata["filename"],
                 file_hash=metadata["sha256"], file_size=metadata["size"],
                 file_display_name=filename,
                 file_description=f'user gravatar `{safe_str(filename)}`',
                 hidden=True, check_acl=False, user_id=1
             )
             Session().add(entry)
             Session().commit()
             log.debug('Stored upload in DB as %s', entry)
         except Exception:
             raise
 
         return request.route_path('download_file', fid=store_uid)
 
     else:
         return klass.generate_svg(svg_type=svg_type)
 
 
 def gravatar_external(request, gravatar_url_tmpl, email_address, size=30):
     return safe_str(gravatar_url_tmpl)\
         .replace('{email}', email_address) \
         .replace('{md5email}', md5_safe(email_address.lower())) \
         .replace('{netloc}', request.host) \
         .replace('{scheme}', request.scheme) \
         .replace('{size}', safe_str(size))
 
 
 def gravatar_url(email_address, size=30, request=None):
     request = request or get_current_request()
     _use_gravatar = request.call_context.visual.use_gravatar
 
     email_address = email_address or User.DEFAULT_USER_EMAIL
     if isinstance(email_address, str):
         # hashlib crashes on unicode items
         email_address = safe_str(email_address)
 
     # empty email or default user
     if not email_address or email_address == User.DEFAULT_USER_EMAIL:
         return initials_gravatar(request, User.DEFAULT_USER_EMAIL, '', '', size=size)
 
     if _use_gravatar:
         gravatar_url_tmpl = request.call_context.visual.gravatar_url \
                             or User.DEFAULT_GRAVATAR_URL
         return gravatar_external(request, gravatar_url_tmpl, email_address, size=size)
 
     else:
         return initials_gravatar(request, email_address, '', '', size=size)
 
 
 def breadcrumb_repo_link(repo):
     """
     Makes a breadcrumbs path link to repo
 
     ex::
         group >> subgroup >> repo
 
     :param repo: a Repository instance
     """
 
     path = [
         link_to(group.name, route_path('repo_group_home', repo_group_name=group.group_name),
                 title='last change:{}'.format(format_date(group.last_commit_change)))
         for group in repo.groups_with_parents
     ] + [
         link_to(repo.just_name, route_path('repo_summary', repo_name=repo.repo_name),
                 title='last change:{}'.format(format_date(repo.last_commit_change)))
     ]
 
     return literal(' &raquo; '.join(path))
 
 
 def breadcrumb_repo_group_link(repo_group):
     """
     Makes a breadcrumbs path link to repo
 
     ex::
         group >> subgroup
 
     :param repo_group: a Repository Group instance
     """
 
     path = [
         link_to(group.name,
                 route_path('repo_group_home', repo_group_name=group.group_name),
                 title='last change:{}'.format(format_date(group.last_commit_change)))
         for group in repo_group.parents
     ] + [
         link_to(repo_group.name,
                 route_path('repo_group_home', repo_group_name=repo_group.group_name),
                 title='last change:{}'.format(format_date(repo_group.last_commit_change)))
     ]
 
     return literal(' &raquo; '.join(path))
 
 
 def format_byte_size_binary(file_size):
     """
     Formats file/folder sizes to standard.
     """
     if file_size is None:
         file_size = 0
 
     formatted_size = format_byte_size(file_size, binary=True)
     return formatted_size
 
 
 def urlify_text(text_, safe=True, **href_attrs):
     """
     Extract urls from text and make html links out of them
     """
 
     url_pat = re.compile(r'''(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@#.&+]'''
                          '''|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)''')
 
     def url_func(match_obj):
         url_full = match_obj.groups()[0]
         a_options = dict(href_attrs)
         a_options['href'] = url_full
         a_text = url_full
         return HTML.tag("a", a_text, **a_options)
 
     _new_text = url_pat.sub(url_func, text_)
 
     if safe:
         return literal(_new_text)
     return _new_text
 
 
 def urlify_commits(text_, repo_name):
     """
     Extract commit ids from text and make link from them
 
     :param text_:
     :param repo_name: repo name to build the URL with
     """
 
     url_pat = re.compile(r'(^|\s)([0-9a-fA-F]{12,40})($|\s)')
 
     def url_func(match_obj):
         commit_id = match_obj.groups()[1]
         pref = match_obj.groups()[0]
         suf = match_obj.groups()[2]
 
         tmpl = (
             '%(pref)s<a class="tooltip-hovercard %(cls)s" href="%(url)s" data-hovercard-alt="%(hovercard_alt)s" data-hovercard-url="%(hovercard_url)s">'
             '%(commit_id)s</a>%(suf)s'
         )
         return tmpl % {
             'pref': pref,
             'cls': 'revision-link',
             'url': route_url(
                 'repo_commit', repo_name=repo_name, commit_id=commit_id),
             'commit_id': commit_id,
             'suf': suf,
             'hovercard_alt': 'Commit: {}'.format(commit_id),
             'hovercard_url': route_url(
                 'hovercard_repo_commit', repo_name=repo_name, commit_id=commit_id)
         }
 
     new_text = url_pat.sub(url_func, text_)
 
     return new_text
 
 
 def _process_url_func(match_obj, repo_name, uid, entry,
                       return_raw_data=False, link_format='html'):
     pref = ''
     if match_obj.group().startswith(' '):
         pref = ' '
 
     issue_id = ''.join(match_obj.groups())
 
     if link_format == 'html':
         tmpl = (
             '%(pref)s<a class="tooltip %(cls)s" href="%(url)s" title="%(title)s">'
             '%(issue-prefix)s%(id-repr)s'
             '</a>')
     elif link_format == 'html+hovercard':
         tmpl = (
             '%(pref)s<a class="tooltip-hovercard %(cls)s" href="%(url)s" data-hovercard-url="%(hovercard_url)s">'
             '%(issue-prefix)s%(id-repr)s'
             '</a>')
     elif link_format in ['rst', 'rst+hovercard']:
         tmpl = '`%(issue-prefix)s%(id-repr)s <%(url)s>`_'
     elif link_format in ['markdown', 'markdown+hovercard']:
         tmpl = '[%(pref)s%(issue-prefix)s%(id-repr)s](%(url)s)'
     else:
         raise ValueError('Bad link_format:{}'.format(link_format))
 
     (repo_name_cleaned,
      parent_group_name) = RepoGroupModel()._get_group_name_and_parent(repo_name)
 
     # variables replacement
     named_vars = {
         'id': issue_id,
         'repo': repo_name,
         'repo_name': repo_name_cleaned,
         'group_name': parent_group_name,
         # set dummy keys so we always have them
         'hostname': '',
         'netloc': '',
         'scheme': ''
     }
 
     request = get_current_request()
     if request:
         # exposes, hostname, netloc, scheme
         host_data = get_host_info(request)
         named_vars.update(host_data)
 
     # named regex variables
     named_vars.update(match_obj.groupdict())
     _url = string.Template(entry['url']).safe_substitute(**named_vars)
     desc = string.Template(escape(entry['desc'])).safe_substitute(**named_vars)
     hovercard_url = string.Template(entry.get('hovercard_url', '')).safe_substitute(**named_vars)
 
     def quote_cleaner(input_str):
         """Remove quotes as it's HTML"""
         return input_str.replace('"', '')
 
     data = {
         'pref': pref,
         'cls': quote_cleaner('issue-tracker-link'),
         'url': quote_cleaner(_url),
         'id-repr': issue_id,
         'issue-prefix': entry['pref'],
         'serv': entry['url'],
-        'title': bleach.clean(desc, strip=True),
+        'title': sanitize_html(desc, strip=True),
         'hovercard_url': hovercard_url
     }
 
     if return_raw_data:
         return {
             'id': issue_id,
             'url': _url
         }
     return tmpl % data
 
 
 def get_active_pattern_entries(repo_name):
     repo = None
     if repo_name:
         # Retrieving repo_name to avoid invalid repo_name to explode on
         # IssueTrackerSettingsModel but still passing invalid name further down
         repo = Repository.get_by_repo_name(repo_name, cache=True)
 
     settings_model = IssueTrackerSettingsModel(repo=repo)
     active_entries = settings_model.get_settings(cache=True)
     return active_entries
 
 
 pr_pattern_re = regex.compile(r'(?:(?:^!)|(?: !))(\d+)')
 
 allowed_link_formats = [
     'html', 'rst', 'markdown', 'html+hovercard', 'rst+hovercard', 'markdown+hovercard']
 
 compile_cache = {
 
 }
 
 
 def process_patterns(text_string, repo_name, link_format='html', active_entries=None):
 
     if link_format not in allowed_link_formats:
         raise ValueError('Link format can be only one of:{} got {}'.format(
                          allowed_link_formats, link_format))
     issues_data = []
     errors = []
     new_text = text_string
 
     if active_entries is None:
         log.debug('Fetch active issue tracker patterns for repo: %s', repo_name)
         active_entries = get_active_pattern_entries(repo_name)
 
     log.debug('Got %s pattern entries to process', len(active_entries))
 
     for uid, entry in list(active_entries.items()):
 
         if not (entry['pat'] and entry['url']):
             log.debug('skipping due to missing data')
             continue
 
         log.debug('issue tracker entry: uid: `%s` PAT:%s URL:%s PREFIX:%s',
                   uid, entry['pat'], entry['url'], entry['pref'])
 
         if entry.get('pat_compiled'):
             pattern = entry['pat_compiled']
         elif entry['pat'] in compile_cache:
             pattern = compile_cache[entry['pat']]
         else:
             try:
                 pattern = regex.compile(r'%s' % entry['pat'])
             except regex.error as e:
                 regex_err = ValueError('{}:{}'.format(entry['pat'], e))
                 log.exception('issue tracker pattern: `%s` failed to compile', regex_err)
                 errors.append(regex_err)
                 continue
             compile_cache[entry['pat']] = pattern
 
         data_func = partial(
             _process_url_func, repo_name=repo_name, entry=entry, uid=uid,
             return_raw_data=True)
 
         for match_obj in pattern.finditer(text_string):
             issues_data.append(data_func(match_obj))
 
         url_func = partial(
             _process_url_func, repo_name=repo_name, entry=entry, uid=uid,
             link_format=link_format)
 
         new_text = pattern.sub(url_func, new_text)
         log.debug('processed prefix:uid `%s`', uid)
 
     # finally use global replace, eg !123 -> pr-link, those will not catch
     # if already similar pattern exists
     server_url = '${scheme}://${netloc}'
     pr_entry = {
         'pref': '!',
         'url': server_url + '/_admin/pull-requests/${id}',
         'desc': 'Pull Request !${id}',
         'hovercard_url': server_url + '/_hovercard/pull_request/${id}'
     }
     pr_url_func = partial(
         _process_url_func, repo_name=repo_name, entry=pr_entry, uid=None,
         link_format=link_format+'+hovercard')
     new_text = pr_pattern_re.sub(pr_url_func, new_text)
     log.debug('processed !pr pattern')
 
     return new_text, issues_data, errors
 
 
 def urlify_commit_message(commit_text, repository=None, active_pattern_entries=None,
                           issues_container_callback=None, error_container=None):
     """
     Parses given text message and makes proper links.
     issues are linked to given issue-server, and rest is a commit link
     """
 
     def escaper(_text):
         return _text.replace('<', '&lt;').replace('>', '&gt;')
 
     new_text = escaper(commit_text)
 
     # extract http/https links and make them real urls
     new_text = urlify_text(new_text, safe=False)
 
     # urlify commits - extract commit ids and make link out of them, if we have
     # the scope of repository present.
     if repository:
         new_text = urlify_commits(new_text, repository)
 
     # process issue tracker patterns
     new_text, issues, errors = process_patterns(
         new_text, repository or '', active_entries=active_pattern_entries)
 
     if issues_container_callback is not None:
         for issue in issues:
             issues_container_callback(issue)
 
     if error_container is not None:
         error_container.extend(errors)
 
     return literal(new_text)
 
 
 def render_binary(repo_name, file_obj):
     """
     Choose how to render a binary file
     """
 
     # unicode
     filename = file_obj.name
 
     # images
     for ext in ['*.png', '*.jpeg', '*.jpg', '*.ico', '*.gif']:
         if fnmatch.fnmatch(filename, pat=ext):
             src = route_path(
                 'repo_file_raw', repo_name=repo_name,
                 commit_id=file_obj.commit.raw_id,
                 f_path=file_obj.path)
 
             return literal(
                 '<img class="rendered-binary" alt="rendered-image" src="{}">'.format(src))
 
 
 def renderer_from_filename(filename, exclude=None):
     """
     choose a renderer based on filename, this works only for text based files
     """
 
     # ipython
     for ext in ['*.ipynb']:
         if fnmatch.fnmatch(filename, pat=ext):
             return 'jupyter'
 
     is_markup = MarkupRenderer.renderer_from_filename(filename, exclude=exclude)
     if is_markup:
         return is_markup
     return None
 
 
 def render(source, renderer='rst', mentions=False, relative_urls=None,
            repo_name=None, active_pattern_entries=None, issues_container_callback=None):
 
     def maybe_convert_relative_links(html_source):
         if relative_urls:
             return relative_links(html_source, relative_urls)
         return html_source
 
     if renderer == 'plain':
         return literal(
             MarkupRenderer.plain(source, leading_newline=False))
 
     elif renderer == 'rst':
         if repo_name:
             # process patterns on comments if we pass in repo name
             source, issues, errors = process_patterns(
                 source, repo_name, link_format='rst',
                 active_entries=active_pattern_entries)
             if issues_container_callback is not None:
                 for issue in issues:
                     issues_container_callback(issue)
 
         rendered_block = maybe_convert_relative_links(
                 MarkupRenderer.rst(source, mentions=mentions))
 
         return literal(f'<div class="rst-block">{rendered_block}</div>')
 
     elif renderer == 'markdown':
         if repo_name:
             # process patterns on comments if we pass in repo name
             source, issues, errors = process_patterns(
                 source, repo_name, link_format='markdown',
                 active_entries=active_pattern_entries)
             if issues_container_callback is not None:
                 for issue in issues:
                     issues_container_callback(issue)
 
         rendered_block = maybe_convert_relative_links(
             MarkupRenderer.markdown(source, flavored=True, mentions=mentions))
         return literal(f'<div class="markdown-block">{rendered_block}</div>')
 
     elif renderer == 'jupyter':
         rendered_block = maybe_convert_relative_links(
             MarkupRenderer.jupyter(source))
         return literal(f'<div class="ipynb">{rendered_block}</div>')
 
     # None means just show the file-source
     return None
 
 
 def commit_status(repo, commit_id):
     return ChangesetStatusModel().get_status(repo, commit_id)
 
 
 def commit_status_lbl(commit_status):
     return dict(ChangesetStatus.STATUSES).get(commit_status)
 
 
 def commit_time(repo_name, commit_id):
     repo = Repository.get_by_repo_name(repo_name)
     commit = repo.get_commit(commit_id=commit_id)
     return commit.date
 
 
 def get_permission_name(key):
     return dict(Permission.PERMS).get(key)
 
 
 def journal_filter_help(request):
     _ = request.translate
     from rhodecode.lib.audit_logger import ACTIONS
     actions = '\n'.join(textwrap.wrap(', '.join(sorted(ACTIONS.keys())), 80))
 
     return _(
         'Example filter terms:\n' +
         '     repository:vcs\n' +
         '     username:marcin\n' +
         '     username:(NOT marcin)\n' +
         '     action:*push*\n' +
         '     ip:127.0.0.1\n' +
         '     date:20120101\n' +
         '     date:[20120101100000 TO 20120102]\n' +
         '\n' +
         'Actions: {actions}\n' +
         '\n' +
         'Generate wildcards using \'*\' character:\n' +
         '     "repository:vcs*" - search everything starting with \'vcs\'\n' +
         '     "repository:*vcs*" - search for repository containing \'vcs\'\n' +
         '\n' +
         'Optional AND / OR operators in queries\n' +
         '     "repository:vcs OR repository:test"\n' +
         '     "username:test AND repository:test*"\n'
     ).format(actions=actions)
 
 
 def not_mapped_error(repo_name):
     from rhodecode.translation import _
     flash(_('%s repository is not mapped to db perhaps'
             ' it was created or renamed from the filesystem'
             ' please run the application again'
             ' in order to rescan repositories') % repo_name, category='error')
 
 
 def ip_range(ip_addr):
     from rhodecode.model.db import UserIpMap
     s, e = UserIpMap._get_ip_range(ip_addr)
     return '%s - %s' % (s, e)
 
 
 def form(url, method='post', needs_csrf_token=True, **attrs):
     """Wrapper around webhelpers.tags.form to prevent CSRF attacks."""
     if method.lower() != 'get' and needs_csrf_token:
         raise Exception(
             'Forms to POST/PUT/DELETE endpoints should have (in general) a ' +
             'CSRF token. If the endpoint does not require such token you can ' +
             'explicitly set the parameter needs_csrf_token to false.')
 
     return insecure_form(url, method=method, **attrs)
 
 
 def secure_form(form_url, method="POST", multipart=False, **attrs):
     """Start a form tag that points the action to an url. This
     form tag will also include the hidden field containing
     the auth token.
 
     The url options should be given either as a string, or as a
     ``url()`` function. The method for the form defaults to POST.
 
     Options:
 
     ``multipart``
         If set to True, the enctype is set to "multipart/form-data".
     ``method``
         The method to use when submitting the form, usually either
         "GET" or "POST". If "PUT", "DELETE", or another verb is used, a
         hidden input with name _method is added to simulate the verb
         over POST.
 
     """
 
     if 'request' in attrs:
         session = attrs['request'].session
         del attrs['request']
     else:
         raise ValueError(
             'Calling this form requires request= to be passed as argument')
 
     _form = insecure_form(form_url, method, multipart, **attrs)
     token = literal(
         '<input type="hidden" name="{}" value="{}">'.format(
             csrf_token_key, get_csrf_token(session)))
 
     return literal("%s\n%s" % (_form, token))
 
 
 def dropdownmenu(name, selected, options, enable_filter=False, **attrs):
     select_html = select(name, selected, options, **attrs)
 
     select2 = """
         <script>
             $(document).ready(function() {
                   $('#%s').select2({
                       containerCssClass: 'drop-menu %s',
                       dropdownCssClass: 'drop-menu-dropdown',
                       dropdownAutoWidth: true%s
                   });
             });
         </script>
     """
 
     filter_option = """,
                         minimumResultsForSearch: -1
     """
     input_id = attrs.get('id') or name
     extra_classes = ' '.join(attrs.pop('extra_classes', []))
     filter_enabled = "" if enable_filter else filter_option
     select_script = literal(select2 % (input_id, extra_classes, filter_enabled))
 
     return literal(select_html+select_script)
 
 
 def get_visual_attr(tmpl_context_var, attr_name):
     """
     A safe way to get a variable from visual variable of template context
 
     :param tmpl_context_var: instance of tmpl_context, usually present as `c`
     :param attr_name: name of the attribute we fetch from the c.visual
     """
     visual = getattr(tmpl_context_var, 'visual', None)
     if not visual:
         return
     else:
         return getattr(visual, attr_name, None)
 
 
 def get_last_path_part(file_node):
     if not file_node.path:
         return '/'
 
     path = safe_str(file_node.path.split('/')[-1])
     return '../' + path
 
 
 def route_url(*args, **kwargs):
     """
     Wrapper around pyramids `route_url` (fully qualified url) function.
     """
     req = get_current_request()
     return req.route_url(*args, **kwargs)
 
 
 def route_path(*args, **kwargs):
     """
     Wrapper around pyramids `route_path` function.
     """
     req = get_current_request()
     return req.route_path(*args, **kwargs)
 
 
 def route_path_or_none(*args, **kwargs):
     try:
         return route_path(*args, **kwargs)
     except KeyError:
         return None
 
 
 def current_route_path(request, **kw):
     new_args = request.GET.mixed()
     new_args.update(kw)
     return request.current_route_path(_query=new_args)
 
 
 def curl_api_example(method, args):
     args_json = json.dumps(OrderedDict([
         ('id', 1),
         ('auth_token', 'SECRET'),
         ('method', method),
         ('args', args)
     ]))
 
     return "curl {api_url} -X POST -H 'content-type:text/plain' --data-binary '{args_json}'".format(
         api_url=route_url('apiv2'),
         args_json=args_json
     )
 
 
 def api_call_example(method, args):
     """
     Generates an API call example via CURL
     """
     curl_call = curl_api_example(method, args)
 
     return literal(
         curl_call +
         "<br/><br/>SECRET can be found in <a href=\"{token_url}\">auth-tokens</a> page, "
         "and needs to be of `api calls` role."
         .format(token_url=route_url('my_account_auth_tokens')))
 
 
 def notification_description(notification, request):
     """
     Generate notification human readable description based on notification type
     """
     from rhodecode.model.notification import NotificationModel
     return NotificationModel().make_description(
         notification, translate=request.translate)
 
 
 def go_import_header(request, db_repo=None):
     """
     Creates a header for go-import functionality in Go Lang
     """
 
     if not db_repo:
         return
     if 'go-get' not in request.GET:
         return
 
     clone_url = db_repo.clone_url()
     prefix = re.split(r'^https?:\/\/', clone_url)[-1]
     # we have a repo and go-get flag,
     return literal('<meta name="go-import" content="{} {} {}">'.format(
         prefix, db_repo.repo_type, clone_url))
 
 
 def reviewer_as_json(*args, **kwargs):
     from rhodecode.apps.repository.utils import reviewer_as_json as _reviewer_as_json
     return _reviewer_as_json(*args, **kwargs)
 
 
 def get_repo_view_type(request):
     route_name = request.matched_route.name
     route_to_view_type = {
         'repo_changelog': 'commits',
         'repo_commits': 'commits',
         'repo_files': 'files',
         'repo_summary': 'summary',
         'repo_commit': 'commit'
     }
 
     return route_to_view_type.get(route_name)
 
 
 def is_active(menu_entry, selected):
     """
     Returns active class for selecting menus in templates
     <li class=${h.is_active('settings', current_active)}></li>
     """
     if not isinstance(menu_entry, list):
         menu_entry = [menu_entry]
 
     if selected in menu_entry:
         return "active"
 
 
 class IssuesRegistry(object):
     """
     issue_registry = IssuesRegistry()
     some_func(issues_callback=issues_registry(...))
     """
 
     def __init__(self):
         self.issues = []
         self.unique_issues = collections.defaultdict(lambda: [])
 
     def __call__(self, commit_dict=None):
         def callback(issue):
             if commit_dict and issue:
                 issue['commit'] = commit_dict
             self.issues.append(issue)
             self.unique_issues[issue['id']].append(issue)
         return callback
 
     def get_issues(self):
         return self.issues
 
     @property
     def issues_unique_count(self):
         return len(set(i['id'] for i in self.issues))

rhodecode/lib/html_filters.py

@@ -1,23 +1,62 b''
 # Copyright (C) 2020-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
+import functools
+import logging
+from .html_sanitizer_defs import markdown_attrs, markdown_tags, all_tags, all_styles
+
+
+log = logging.getLogger(__name__)
+
+
 # base64 filter e.g ${ example | base64,n }
 def base64(text):
     from rhodecode.lib.str_utils import base64_to_str
     return base64_to_str(text)
 
+
+def sanitize_html(text, **kwargs):
+    # TODO: replace this with https://nh3.readthedocs.io/en/latest
+    # bleach is abandoned and deprecated :/
+    import bleach
+    from bleach.css_sanitizer import CSSSanitizer
+
+    css_sanitizer = CSSSanitizer(allowed_css_properties=all_styles)
+
+    markdown = kwargs.pop('markdown', False)
+
+    allowed_attrs = markdown_attrs
+
+    cleaner = functools.partial(bleach.clean,
+                                tags=all_tags,
+                                attributes=allowed_attrs,
+                                css_sanitizer=css_sanitizer,
+                                strip_comments=False, **kwargs)
+
+    if markdown:
+        cleaner = functools.partial(bleach.clean,
+                                    tags=markdown_tags,
+                                    attributes=markdown_attrs,
+                                    css_sanitizer=css_sanitizer,
+                                    strip_comments=False, **kwargs)
+
+    try:
+        return cleaner(text)
+    except Exception:
+        log.exception('Failed to sanitize html')
+        return 'TEXT CANNOT BE PARSED USING HTML SANITIZE'

rhodecode/lib/html_sanitizer_defs.py

@@ -1,408 +1,409 b''
 all_tags = [
     "a", "abbr", "acronym", "address", "applet", "area", "article", "aside", "audio",
     "b", "base", "basefont", "bdi", "bdo", "bgsound", "big", "blink", "blockquote", "body", "br", "button",
     "canvas", "caption", "center", "cite", "code", "col", "colgroup", "command", "content",
     "data", "datalist", "dd", "del", "detals", "dfn", "dialog", "dir", "div", "dl", "dt",
     "element", "em", "embed",
     "fieldset", "figcaption", "figure", "font", "footer", "form", "frame", "frameset",
     "h1", "h2", "h3", "h4", "h5", "h6", "head", "header", "hgroup", "hr", "html",
     "i", "iframe", "image", "img", "input", "ins", "isindex",
     "kbd", "keygen",
     "label", "legend", "li", "link", "listing",
     "main", "map", "mark", "marquee", "menu", "menuitem", "meta", "meter", "multicol",
     "nav", "nobr", "noembed", "noframes", "noscript",
     "object", "ol", "optgroup", "option", "output",
     "p", "param", "picture", "plaintext", "pre", "progress",
     "q",
     "rp", "rt", "ruby",
     "s", "samp", "script", "section", "select", "shadow", "small", "source", "spacer", "span", "strike", "strong", "style", "sub", "summary", "sup",
     "table", "tbody", "td", "template", "textarea", "tfoot", "th", "thead", "time", "title", "tr", "track", "tt",
     "u", "ul",
     "var", "video",
     "wbr",
     "xmp",
 ]
 
 # List tags that, if included in a page, could break markup or open XSS.
 generally_xss_unsafe = [
     "applet", "audio",
     "bgsound", "body",
     "canvas",
     "embed",
     "frame", "frameset",
     "head", "html",
     "iframe",
     "link",
     "meta",
     "object",
     "param",
     "source", "script",
     "ruby", "rt",
     "title", "track",
     "video",
     "xmp"
 ]
 
 # Tags that, if included on the page, will probably not break markup or open
 # XSS.  Note that these must be combined with attribute whitelisting, or things
 # like <img> and <style> could still be unsafe.
 generally_xss_safe = list(set(all_tags) - set(generally_xss_unsafe))
 generally_xss_safe.sort()
 
 # Tags suitable for rendering markdown
 markdown_tags = [
     "h1", "h2", "h3", "h4", "h5", "h6",
     "b", "i", "strong", "em", "tt",
     "p", "br",
     "span", "div", "blockquote", "code", "hr", "pre", "del",
     "ul", "ol", "li",
     "dl", "dd", "dt",
     "table", "thead", "tbody", "tfoot", "tr", "th", "td",
     "img",
     "a",
     "input",
     "details",
-    "summary"
+    "summary",
+    "div"
 ]
 
 markdown_attrs = {
     "*": ["class", "style", "align"],
     "img": ["src", "alt", "title", "width", "height", "hspace", "align"],
     "a": ["href", "alt", "title", "name", "data-hovercard-alt", "data-hovercard-url"],
     "abbr": ["title"],
     "acronym": ["title"],
     "pre": ["lang"],
     "input": ["type", "disabled", "checked"],
     "strong": ["title", "data-hovercard-alt", "data-hovercard-url"],
 }
 
 standard_styles = [
     # Taken from https://developer.mozilla.org/en-US/docs/Web/CSS/Reference
     # This includes pseudo-classes, pseudo-elements, @-rules, units, and
     # selectors in addition to properties, but it doesn't matter for our
     # purposes -- we don't need to filter styles..
     ":active", "::after (:after)", "align-content", "align-items", "align-self",
     "all", "<angle>", "animation", "animation-delay", "animation-direction",
     "animation-duration", "animation-fill-mode", "animation-iteration-count",
     "animation-name", "animation-play-state", "animation-timing-function",
     "@annotation", "annotation()", "attr()", "::backdrop", "backface-visibility",
     "background", "background-attachment", "background-blend-mode",
     "background-clip", "background-color", "background-image", "background-origin",
     "background-position", "background-repeat", "background-size", "<basic-shape>",
     "::before (:before)", "<blend-mode>", "blur()", "border", "border-bottom",
     "border-bottom-color", "border-bottom-left-radius",
     "border-bottom-right-radius", "border-bottom-style", "border-bottom-width",
     "border-collapse", "border-color", "border-image", "border-image-outset",
     "border-image-repeat", "border-image-slice", "border-image-source",
     "border-image-width", "border-left", "border-left-color", "border-left-style",
     "border-left-width", "border-radius", "border-right", "border-right-color",
     "border-right-style", "border-right-width", "border-spacing", "border-style",
     "border-top", "border-top-color", "border-top-left-radius",
     "border-top-right-radius", "border-top-style", "border-top-width",
     "border-width", "bottom", "box-decoration-break", "box-shadow", "box-sizing",
     "break-after", "break-before", "break-inside", "brightness()", "calc()",
     "caption-side", "ch", "@character-variant", "character-variant()", "@charset",
     ":checked", "circle()", "clear", "clip", "clip-path", "cm", "color", "<color>",
     "columns", "column-count", "column-fill", "column-gap", "column-rule",
     "column-rule-color", "column-rule-style", "column-rule-width", "column-span",
     "column-width", "content", "contrast()", "<counter>", "counter-increment",
     "counter-reset", "@counter-style", "cubic-bezier()", "cursor",
     "<custom-ident>", ":default", "deg", ":dir()", "direction", ":disabled",
     "display", "@document", "dpcm", "dpi", "dppx", "drop-shadow()", "element()",
     "ellipse()", "em", ":empty", "empty-cells", ":enabled", "ex", "filter",
     ":first", ":first-child", "::first-letter", "::first-line",
     ":first-of-type", "flex", "flex-basis", "flex-direction",
     "flex-flow", "flex-grow", "flex-shrink", "flex-wrap", "float", ":focus",
     "font", "@font-face", "font-family", "font-feature-settings",
     "@font-feature-values", "font-kerning", "font-language-override", "font-size",
     "font-size-adjust", "font-stretch", "font-style", "font-synthesis",
     "font-variant", "font-variant-alternates", "font-variant-caps",
     "font-variant-east-asian", "font-variant-ligatures", "font-variant-numeric",
     "font-variant-position", "font-weight", "<frequency>", ":fullscreen", "grad",
     "<gradient>", "grayscale()", "grid", "grid-area", "grid-auto-columns",
     "grid-auto-flow", "grid-auto-position", "grid-auto-rows", "grid-column",
     "grid-column-start", "grid-column-end", "grid-row", "grid-row-start",
     "grid-row-end", "grid-template", "grid-template-areas", "grid-template-rows",
     "grid-template-columns", "height", ":hover", "hsl()", "hsla()", "hue-rotate()",
     "hyphens", "hz", "<image>", "image()", "image-rendering", "image-resolution",
     "image-orientation", "ime-mode", "@import", "in", ":indeterminate", "inherit",
     "initial", ":in-range", "inset()", "<integer>", ":invalid", "invert()",
     "isolation", "justify-content", "@keyframes", "khz", ":lang()", ":last-child",
     ":last-of-type", "left", ":left", "<length>", "letter-spacing",
     "linear-gradient()", "line-break", "line-height", ":link", "list-style",
     "list-style-image", "list-style-position", "list-style-type", "margin",
     "margin-bottom", "margin-left", "margin-right", "margin-top", "marks", "mask",
     "mask-type", "matrix()", "matrix3d()", "max-height", "max-width", "@media",
     "min-height", "minmax()", "min-width", "mix-blend-mode", "mm", "ms",
     "@namespace", ":not()", ":nth-child()", ":nth-last-child()",
     ":nth-last-of-type()", ":nth-of-type()", "<number>", "object-fit",
     "object-position", ":only-child", ":only-of-type", "opacity", "opacity()",
     ":optional", "order", "@ornaments", "ornaments()", "orphans", "outline",
     "outline-color", "outline-offset", "outline-style", "outline-width",
     ":out-of-range", "overflow", "overflow-wrap", "overflow-x", "overflow-y",
     "padding", "padding-bottom", "padding-left", "padding-right", "padding-top",
     "@page", "page-break-after", "page-break-before", "page-break-inside", "pc",
     "<percentage>", "perspective", "perspective()", "perspective-origin",
     "pointer-events", "polygon()", "position", "<position>", "pt", "px", "quotes",
     "rad", "radial-gradient()", "<ratio>", ":read-only", ":read-write", "rect()",
     "rem", "repeat()", "::repeat-index", "::repeat-item",
     "repeating-linear-gradient()", "repeating-radial-gradient()", ":required",
     "resize", "<resolution>", "rgb()", "rgba()", "right", ":right", ":root",
     "rotate()", "rotatex()", "rotatey()", "rotatez()", "rotate3d()", "ruby-align",
     "ruby-merge", "ruby-position", "s", "saturate()", "scale()", "scalex()",
     "scaley()", "scalez()", "scale3d()", ":scope", "scroll-behavior",
     "::selection", "sepia()", "<shape>", "shape-image-threshold", "shape-margin",
     "shape-outside", "skew()", "skewx()", "skewy()", "steps()", "<string>",
     "@styleset", "styleset()", "@stylistic", "stylistic()", "@supports", "@swash",
     "swash()", "symbol()", "table-layout", "tab-size", ":target", "text-align",
     "text-align-last", "text-combine-upright", "text-decoration",
     "text-decoration-color", "text-decoration-line", "text-decoration-style",
     "text-indent", "text-orientation", "text-overflow", "text-rendering",
     "text-shadow", "text-transform", "text-underline-position", "<time>",
     "<timing-function>", "top", "touch-action", "transform", "transform-origin",
     "transform-style", "transition", "transition-delay", "transition-duration",
     "transition-property", "transition-timing-function", "translate()",
     "translatex()", "translatey()", "translatez()", "translate3d()", "turn",
     "unicode-bidi", "unicode-range", "unset", "<uri>", "url()", "<user-ident>",
     ":valid", "::value", "var()", "vertical-align", "vh", "@viewport",
     "visibility", ":visited", "vmax", "vmin", "vw", "white-space", "widows",
     "width", "will-change", "word-break", "word-spacing", "word-wrap",
     "writing-mode", "z-index",
 
 ]
 
 webkit_prefixed_styles = [
     # Webkit-prefixed styles
     # https://developer.mozilla.org/en-US/docs/Web/CSS/Reference/Webkit_Extensions
     "-webkit-animation", "-webkit-animation-delay", "-webkit-animation-direction",
     "-webkit-animation-duration", "-webkit-animation-fill-mode",
     "-webkit-animation-iteration-count", "-webkit-animation-name",
     "-webkit-animation-play-state", "-webkit-animation-timing-function",
     "-webkit-backface-visibility", "-webkit-border-image", "-webkit-column-count",
     "-webkit-column-gap", "-webkit-column-width", "-webkit-column-rule",
     "-webkit-column-rule-width", "-webkit-column-rule-style",
     "-webkit-column-rule-color", "-webkit-columns", "-webkit-column-span",
     "-webkit-font-feature-settings", "-webkit-font-kerning",
     "-webkit-font-size-delta", "-webkit-font-variant-ligatures",
     "-webkit-grid-column", "-webkit-grid-row", "-webkit-hyphens", "-webkit-mask",
     "-webkit-mask-clip", "-webkit-mask-composite", "-webkit-mask-image",
     "-webkit-mask-origin", "-webkit-mask-position", "-webkit-mask-repeat",
     "-webkit-mask-size", "-webkit-perspective", "-webkit-perspective-origin",
     "-webkit-region-fragment", "-webkit-shape-outside", "-webkit-text-emphasis",
     "-webkit-text-emphasis-color", "-webkit-text-emphasis-position",
     "-webkit-text-emphasis-style", "-webkit-transform", "-webkit-transform-origin",
     "-webkit-transform-style", "-webkit-transition", "-webkit-transition-delay",
     "-webkit-transition-duration", "-webkit-transition-property",
     "-webkit-transition-timing-function", "-epub-word-break", "-epub-writing-mode",
     # WebKit-prefixed properties with an unprefixed counterpart
     "-webkit-background-clip", "-webkit-background-origin",
     "-webkit-background-size", "-webkit-border-bottom-left-radius",
     "-webkit-border-bottom-right-radius", "-webkit-border-radius",
     "-webkit-border-top-left-radius", "-webkit-border-top-right-radius",
     "-webkit-box-sizing", "-epub-caption-side", "-webkit-opacity",
     "-epub-text-transform",
 ]
 
 mozilla_prefixed_styles = [
     "-moz-column-count", "-moz-column-fill", "-moz-column-gap",
     "-moz-column-width", "-moz-column-rule", "-moz-column-rule-width",
     "-moz-column-rule-style", "-moz-column-rule-color",
     "-moz-font-feature-settings", "-moz-font-language-override", "-moz-hyphens",
     "-moz-text-align-last", "-moz-text-decoration-color",
     "-moz-text-decoration-line", "-moz-text-decoration-style",
 ]
 
 all_prefixed_styles = [
     # From http://peter.sh/experiments/vendor-prefixed-css-property-overview/
     "-ms-accelerator", "-webkit-app-region", "-webkit-appearance",
     "-webkit-appearance", "-moz-appearance", "-webkit-aspect-ratio",
     "-webkit-backdrop-filter", "backface-visibility",
     "-webkit-backface-visibility", "backface-visibility", "backface-visibility",
     "-webkit-background-composite", "-webkit-background-composite", "-moz-binding",
     "-ms-block-progression", "-webkit-border-after", "-webkit-border-after",
     "-webkit-border-after-color", "-webkit-border-after-color",
     "-webkit-border-after-style", "-webkit-border-after-style",
     "-webkit-border-after-width", "-webkit-border-after-width",
     "-webkit-border-before", "-webkit-border-before",
     "-webkit-border-before-color", "-webkit-border-before-color",
     "-webkit-border-before-style", "-webkit-border-before-style",
     "-webkit-border-before-width", "-webkit-border-before-width",
     "-moz-border-bottom-colors", "-webkit-border-end", "-webkit-border-end",
     "-moz-border-end", "-webkit-border-end-color", "-webkit-border-end-color",
     "-moz-border-end-color", "-webkit-border-end-style",
     "-webkit-border-end-style", "-moz-border-end-style",
     "-webkit-border-end-width", "-webkit-border-end-width",
     "-moz-border-end-width", "-webkit-border-fit",
     "-webkit-border-horizontal-spacing", "-webkit-border-horizontal-spacing",
     "-moz-border-left-colors", "-moz-border-right-colors", "-webkit-border-start",
     "-webkit-border-start", "-moz-border-start", "-webkit-border-start-color",
     "-webkit-border-start-color", "-moz-border-start-color",
     "-webkit-border-start-style", "-webkit-border-start-style",
     "-moz-border-start-style", "-webkit-border-start-width",
     "-webkit-border-start-width", "-moz-border-start-width",
     "-moz-border-top-colors", "-webkit-border-vertical-spacing",
     "-webkit-border-vertical-spacing", "-webkit-box-align", "-webkit-box-align",
     "-moz-box-align", "-webkit-box-decoration-break",
     "-webkit-box-decoration-break", "box-decoration-break",
     "-webkit-box-direction", "-webkit-box-direction", "-moz-box-direction",
     "-webkit-box-flex", "-webkit-box-flex", "-moz-box-flex",
     "-webkit-box-flex-group", "-webkit-box-flex-group", "-webkit-box-lines",
     "-webkit-box-lines", "-webkit-box-ordinal-group", "-webkit-box-ordinal-group",
     "-moz-box-ordinal-group", "-webkit-box-orient", "-webkit-box-orient",
     "-moz-box-orient", "-webkit-box-pack", "-webkit-box-pack", "-moz-box-pack",
     "-webkit-box-reflect", "-webkit-box-reflect", "clip-path", "-webkit-clip-path",
     "clip-path", "clip-path", "-webkit-color-correction", "-webkit-column-axis",
     "-webkit-column-break-after", "-webkit-column-break-after",
     "-webkit-column-break-before", "-webkit-column-break-before",
     "-webkit-column-break-inside", "-webkit-column-break-inside",
     "-webkit-column-count", "column-count", "-moz-column-count", "column-count",
     "column-fill", "column-fill", "-moz-column-fill", "column-fill",
     "-webkit-column-gap", "column-gap", "-moz-column-gap", "column-gap",
     "-webkit-column-rule", "column-rule", "-moz-column-rule", "column-rule",
     "-webkit-column-rule-color", "column-rule-color", "-moz-column-rule-color",
     "column-rule-color", "-webkit-column-rule-style", "column-rule-style",
     "-moz-column-rule-style", "column-rule-style", "-webkit-column-rule-width",
     "column-rule-width", "-moz-column-rule-width", "column-rule-width",
     "-webkit-column-span", "column-span", "column-span", "-webkit-column-width",
     "column-width", "-moz-column-width", "column-width", "-webkit-columns",
     "columns", "-moz-columns", "columns", "-ms-content-zoom-chaining",
     "-ms-content-zoom-limit", "-ms-content-zoom-limit-max",
     "-ms-content-zoom-limit-min", "-ms-content-zoom-snap",
     "-ms-content-zoom-snap-points", "-ms-content-zoom-snap-type",
     "-ms-content-zooming", "-moz-control-character-visibility",
     "-webkit-cursor-visibility", "-webkit-dashboard-region", "filter",
     "-webkit-filter", "filter", "filter", "-ms-flex-align", "-ms-flex-item-align",
     "-ms-flex-line-pack", "-ms-flex-negative", "-ms-flex-order", "-ms-flex-pack",
     "-ms-flex-positive", "-ms-flex-preferred-size", "-moz-float-edge",
     "-webkit-flow-from", "-ms-flow-from", "-webkit-flow-into", "-ms-flow-into",
     "-webkit-font-feature-settings", "-webkit-font-feature-settings",
     "font-feature-settings", "font-feature-settings", "font-kerning",
     "-webkit-font-kerning", "font-kerning", "-webkit-font-size-delta",
     "-webkit-font-size-delta", "-webkit-font-smoothing", "-webkit-font-smoothing",
     "font-variant-ligatures", "-webkit-font-variant-ligatures",
     "font-variant-ligatures", "-moz-force-broken-image-icon", "grid",
     "-webkit-grid", "grid", "grid-area", "-webkit-grid-area", "grid-area",
     "grid-auto-columns", "-webkit-grid-auto-columns", "grid-auto-columns",
     "grid-auto-flow", "-webkit-grid-auto-flow", "grid-auto-flow", "grid-auto-rows",
     "-webkit-grid-auto-rows", "grid-auto-rows", "grid-column",
     "-webkit-grid-column", "grid-column", "-ms-grid-column",
     "-ms-grid-column-align", "grid-column-end", "-webkit-grid-column-end",
     "grid-column-end", "-ms-grid-column-span", "grid-column-start",
     "-webkit-grid-column-start", "grid-column-start", "-ms-grid-columns",
     "grid-row", "-webkit-grid-row", "grid-row", "-ms-grid-row",
     "-ms-grid-row-align", "grid-row-end", "-webkit-grid-row-end", "grid-row-end",
     "-ms-grid-row-span", "grid-row-start", "-webkit-grid-row-start",
     "grid-row-start", "-ms-grid-rows", "grid-template", "-webkit-grid-template",
     "grid-template", "grid-template-areas", "-webkit-grid-template-areas",
     "grid-template-areas", "grid-template-columns",
     "-webkit-grid-template-columns", "grid-template-columns", "grid-template-rows",
     "-webkit-grid-template-rows", "grid-template-rows", "-ms-high-contrast-adjust",
     "-webkit-highlight", "-webkit-hyphenate-character",
     "-webkit-hyphenate-character", "-webkit-hyphenate-limit-after",
     "-webkit-hyphenate-limit-before", "-ms-hyphenate-limit-chars",
     "-webkit-hyphenate-limit-lines", "-ms-hyphenate-limit-lines",
     "-ms-hyphenate-limit-zone", "-webkit-hyphens", "-moz-hyphens", "-ms-hyphens",
     "-moz-image-region", "-ms-ime-align", "-webkit-initial-letter",
     "-ms-interpolation-mode", "justify-self", "-webkit-justify-self",
     "-webkit-line-align", "-webkit-line-box-contain", "-webkit-line-box-contain",
     "-webkit-line-break", "-webkit-line-break", "line-break", "-webkit-line-clamp",
     "-webkit-line-clamp", "-webkit-line-grid", "-webkit-line-snap",
     "-webkit-locale", "-webkit-locale", "-webkit-logical-height",
     "-webkit-logical-height", "-webkit-logical-width", "-webkit-logical-width",
     "-webkit-margin-after", "-webkit-margin-after",
     "-webkit-margin-after-collapse", "-webkit-margin-after-collapse",
     "-webkit-margin-before", "-webkit-margin-before",
     "-webkit-margin-before-collapse", "-webkit-margin-before-collapse",
     "-webkit-margin-bottom-collapse", "-webkit-margin-bottom-collapse",
     "-webkit-margin-collapse", "-webkit-margin-collapse", "-webkit-margin-end",
     "-webkit-margin-end", "-moz-margin-end", "-webkit-margin-start",
     "-webkit-margin-start", "-moz-margin-start", "-webkit-margin-top-collapse",
     "-webkit-margin-top-collapse", "-webkit-marquee", "-webkit-marquee-direction",
     "-webkit-marquee-increment", "-webkit-marquee-repetition",
     "-webkit-marquee-speed", "-webkit-marquee-style", "mask", "-webkit-mask",
     "mask", "-webkit-mask-box-image", "-webkit-mask-box-image",
     "-webkit-mask-box-image-outset", "-webkit-mask-box-image-outset",
     "-webkit-mask-box-image-repeat", "-webkit-mask-box-image-repeat",
     "-webkit-mask-box-image-slice", "-webkit-mask-box-image-slice",
     "-webkit-mask-box-image-source", "-webkit-mask-box-image-source",
     "-webkit-mask-box-image-width", "-webkit-mask-box-image-width",
     "-webkit-mask-clip", "-webkit-mask-clip", "-webkit-mask-composite",
     "-webkit-mask-composite", "-webkit-mask-image", "-webkit-mask-image",
     "-webkit-mask-origin", "-webkit-mask-origin", "-webkit-mask-position",
     "-webkit-mask-position", "-webkit-mask-position-x", "-webkit-mask-position-x",
     "-webkit-mask-position-y", "-webkit-mask-position-y", "-webkit-mask-repeat",
     "-webkit-mask-repeat", "-webkit-mask-repeat-x", "-webkit-mask-repeat-x",
     "-webkit-mask-repeat-y", "-webkit-mask-repeat-y", "-webkit-mask-size",
     "-webkit-mask-size", "mask-source-type", "-webkit-mask-source-type",
     "-moz-math-display", "-moz-math-variant", "-webkit-max-logical-height",
     "-webkit-max-logical-height", "-webkit-max-logical-width",
     "-webkit-max-logical-width", "-webkit-min-logical-height",
     "-webkit-min-logical-height", "-webkit-min-logical-width",
     "-webkit-min-logical-width", "-webkit-nbsp-mode", "-moz-orient",
     "-moz-osx-font-smoothing", "-moz-outline-radius",
     "-moz-outline-radius-bottomleft", "-moz-outline-radius-bottomright",
     "-moz-outline-radius-topleft", "-moz-outline-radius-topright",
     "-webkit-overflow-scrolling", "-ms-overflow-style", "-webkit-padding-after",
     "-webkit-padding-after", "-webkit-padding-before", "-webkit-padding-before",
     "-webkit-padding-end", "-webkit-padding-end", "-moz-padding-end",
     "-webkit-padding-start", "-webkit-padding-start", "-moz-padding-start",
     "perspective", "-webkit-perspective", "perspective", "perspective",
     "perspective-origin", "-webkit-perspective-origin", "perspective-origin",
     "perspective-origin", "-webkit-perspective-origin-x",
     "-webkit-perspective-origin-x", "perspective-origin-x",
     "-webkit-perspective-origin-y", "-webkit-perspective-origin-y",
     "perspective-origin-y", "-webkit-print-color-adjust",
     "-webkit-print-color-adjust", "-webkit-region-break-after",
     "-webkit-region-break-before", "-webkit-region-break-inside",
     "-webkit-region-fragment", "-webkit-rtl-ordering", "-webkit-rtl-ordering",
     "-webkit-ruby-position", "-webkit-ruby-position", "ruby-position",
     "-moz-script-level", "-moz-script-min-size", "-moz-script-size-multiplier",
     "-ms-scroll-chaining", "-ms-scroll-limit", "-ms-scroll-limit-x-max",
     "-ms-scroll-limit-x-min", "-ms-scroll-limit-y-max", "-ms-scroll-limit-y-min",
     "-ms-scroll-rails", "-webkit-scroll-snap-coordinate",
     "-webkit-scroll-snap-destination", "-webkit-scroll-snap-points-x",
     "-ms-scroll-snap-points-x", "-webkit-scroll-snap-points-y",
     "-ms-scroll-snap-points-y", "-webkit-scroll-snap-type", "-ms-scroll-snap-type",
     "-ms-scroll-snap-x", "-ms-scroll-snap-y", "-ms-scroll-translation",
     "-ms-scrollbar-3dlight-color", "shape-image-threshold",
     "-webkit-shape-image-threshold", "shape-margin", "-webkit-shape-margin",
     "shape-outside", "-webkit-shape-outside", "-moz-stack-sizing", "tab-size",
     "tab-size", "-moz-tab-size", "-webkit-tap-highlight-color",
     "-webkit-tap-highlight-color", "text-align-last", "-webkit-text-align-last",
     "-moz-text-align-last", "text-align-last", "-webkit-text-combine",
     "-webkit-text-combine", "-ms-text-combine-horizontal", "text-decoration-color",
     "-webkit-text-decoration-color", "text-decoration-color",
     "text-decoration-color", "text-decoration-line",
     "-webkit-text-decoration-line", "text-decoration-line",
     "-webkit-text-decoration-skip", "text-decoration-style",
     "-webkit-text-decoration-style", "text-decoration-style",
     "-webkit-text-decorations-in-effect", "-webkit-text-decorations-in-effect",
     "-webkit-text-emphasis", "text-emphasis", "-webkit-text-emphasis-color",
     "text-emphasis-color", "-webkit-text-emphasis-position",
     "text-emphasis-position", "-webkit-text-emphasis-style", "text-emphasis-style",
     "-webkit-text-fill-color", "-webkit-text-fill-color", "text-justify",
     "-webkit-text-justify", "text-justify", "-webkit-text-orientation",
     "-webkit-text-orientation", "text-orientation", "-webkit-text-security",
     "-webkit-text-security", "-webkit-text-size-adjust", "-moz-text-size-adjust",
     "-ms-text-size-adjust", "-webkit-text-stroke", "-webkit-text-stroke",
     "-webkit-text-stroke-color", "-webkit-text-stroke-color",
     "-webkit-text-stroke-width", "-webkit-text-stroke-width",
     "text-underline-position", "-webkit-text-underline-position",
     "text-underline-position", "-webkit-touch-callout", "-ms-touch-select",
     "transform", "-webkit-transform", "transform", "transform", "transform-origin",
     "-webkit-transform-origin", "transform-origin", "transform-origin",
     "-webkit-transform-origin-x", "-webkit-transform-origin-x",
     "transform-origin-x", "-webkit-transform-origin-y",
     "-webkit-transform-origin-y", "transform-origin-y",
     "-webkit-transform-origin-z", "-webkit-transform-origin-z",
     "transform-origin-z", "transform-style", "-webkit-transform-style",
     "transform-style", "transform-style", "-webkit-user-drag", "-webkit-user-drag",
     "-moz-user-focus", "-moz-user-input", "-webkit-user-modify",
     "-webkit-user-modify", "-moz-user-modify", "-webkit-user-select",
     "-webkit-user-select", "-moz-user-select", "-ms-user-select",
     "-moz-window-dragging", "-moz-window-shadow", "-ms-wrap-flow",
     "-ms-wrap-margin", "-ms-wrap-through", "writing-mode", "-webkit-writing-mode",
     "writing-mode", "writing-mode",
 ]
 
 all_styles = standard_styles + all_prefixed_styles
  No newline at end of file

rhodecode/lib/markup_renderer.py

@@ -1,547 +1,537 b''
 
 
 # Copyright (C) 2011-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 Renderer for markup languages with ability to parse using rst or markdown
 """
 
 import re
 import os
 import lxml
 import logging
 import urllib.parse
-import bleach
 import pycmarkgfm
 
 from mako.lookup import TemplateLookup
 from mako.template import Template as MakoTemplate
 
 from docutils.core import publish_parts
 from docutils.parsers.rst import directives
 from docutils import writers
 from docutils.writers import html4css1
 import markdown
 
-from rhodecode.lib.utils2 import safe_str, md5_safe, MENTIONS_REGEX
+from rhodecode.lib.utils2 import safe_str, MENTIONS_REGEX
 
 log = logging.getLogger(__name__)
 
 # default renderer used to generate automated comments
 DEFAULT_COMMENTS_RENDERER = 'rst'
 
 try:
     from lxml.html import fromstring
     from lxml.html import tostring
 except ImportError:
     log.exception('Failed to import lxml')
     fromstring = None
     tostring = None
 
 
 class CustomHTMLTranslator(writers.html4css1.HTMLTranslator):
     """
     Custom HTML Translator used for sandboxing potential
     JS injections in ref links
     """
     def visit_literal_block(self, node):
         self.body.append(self.starttag(node, 'pre', CLASS='codehilite literal-block'))
 
     def visit_reference(self, node):
         if 'refuri' in node.attributes:
             refuri = node['refuri']
             if ':' in refuri:
                 prefix, link = refuri.lstrip().split(':', 1)
                 prefix = prefix or ''
 
                 if prefix.lower() == 'javascript':
                     # we don't allow javascript type of refs...
                     node['refuri'] = 'javascript:alert("SandBoxedJavascript")'
 
         # old style class requires this...
         return html4css1.HTMLTranslator.visit_reference(self, node)
 
 
 class RhodeCodeWriter(writers.html4css1.Writer):
     def __init__(self):
         super(RhodeCodeWriter, self).__init__()
         self.translator_class = CustomHTMLTranslator
 
 
 def relative_links(html_source, server_paths):
     if not html_source:
         return html_source
 
     if not fromstring and tostring:
         return html_source
 
     try:
         doc = lxml.html.fromstring(html_source)
     except Exception:
         return html_source
 
     for el in doc.cssselect('img, video'):
         src = el.attrib.get('src')
         if src:
             el.attrib['src'] = relative_path(src, server_paths['raw'])
 
     for el in doc.cssselect('a:not(.gfm)'):
         src = el.attrib.get('href')
         if src:
             raw_mode = el.attrib['href'].endswith('?raw=1')
             if raw_mode:
                 el.attrib['href'] = relative_path(src, server_paths['raw'])
             else:
                 el.attrib['href'] = relative_path(src, server_paths['standard'])
 
     return lxml.html.tostring(doc, encoding='unicode')
 
 
 def relative_path(path, request_path, is_repo_file=None):
     """
     relative link support, path is a rel path, and request_path is current
     server path (not absolute)
 
     e.g.
 
     path = '../logo.png'
     request_path= '/repo/files/path/file.md'
     produces: '/repo/files/logo.png'
     """
     # TODO(marcink): unicode/str support ?
     # maybe=> safe_str(urllib.quote(safe_str(final_path), '/:'))
 
     def dummy_check(p):
         return True  # assume default is a valid file path
 
     is_repo_file = is_repo_file or dummy_check
     if not path:
         return request_path
 
     path = safe_str(path)
     request_path = safe_str(request_path)
 
     if path.startswith(('data:', 'javascript:', '#', ':')):
         # skip data, anchor, invalid links
         return path
 
     is_absolute = bool(urllib.parse.urlparse(path).netloc)
     if is_absolute:
         return path
 
     if not request_path:
         return path
 
     if path.startswith('/'):
         path = path[1:]
 
     if path.startswith('./'):
         path = path[2:]
 
     parts = request_path.split('/')
     # compute how deep we need to traverse the request_path
     depth = 0
 
     if is_repo_file(request_path):
         # if request path is a VALID file, we use a relative path with
         # one level up
         depth += 1
 
     while path.startswith('../'):
         depth += 1
         path = path[3:]
 
     if depth > 0:
         parts = parts[:-depth]
 
     parts.append(path)
     final_path = '/'.join(parts).lstrip('/')
 
     return '/' + final_path
 
 
 _cached_markdown_renderer = None
 
 
 def get_markdown_renderer(extensions, output_format):
     global _cached_markdown_renderer
 
     if _cached_markdown_renderer is None:
         _cached_markdown_renderer = markdown.Markdown(
             extensions=extensions + ['legacy_attrs'],
             output_format=output_format)
     return _cached_markdown_renderer
 
 
 def get_markdown_renderer_flavored(extensions, output_format):
     """
     Dummy wrapper to mimic markdown API and render github HTML rendered
 
     """
     md = get_markdown_renderer(extensions, output_format)
 
     class GFM(object):
         def convert(self, source):
             with pycmarkgfm.parse_gfm(source) as document:
                 parsed_md = document.to_commonmark()
                 return md.convert(parsed_md)
 
     return GFM()
 
 
 class MarkupRenderer(object):
     RESTRUCTUREDTEXT_DISALLOWED_DIRECTIVES = ['include', 'meta', 'raw']
 
     MARKDOWN_PAT = re.compile(r'\.(md|mkdn?|mdown|markdown)$', re.IGNORECASE)
     RST_PAT = re.compile(r'\.re?st$', re.IGNORECASE)
     JUPYTER_PAT = re.compile(r'\.(ipynb)$', re.IGNORECASE)
     PLAIN_PAT = re.compile(r'^readme$', re.IGNORECASE)
 
     URL_PAT = re.compile(r'(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]'
                          r'|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)')
 
     MENTION_PAT = re.compile(MENTIONS_REGEX)
 
     extensions = ['markdown.extensions.codehilite', 'markdown.extensions.extra',
                   'markdown.extensions.def_list', 'markdown.extensions.sane_lists']
 
     output_format = 'html4'
 
     # extension together with weights. Lower is first means we control how
     # extensions are attached to readme names with those.
     PLAIN_EXTS = [
         # prefer no extension
         ('', 0),  # special case that renders READMES names without extension
         ('.text', 2), ('.TEXT', 2),
         ('.txt', 3), ('.TXT', 3)
     ]
 
     RST_EXTS = [
         ('.rst', 1), ('.rest', 1),
         ('.RST', 2), ('.REST', 2)
     ]
 
     MARKDOWN_EXTS = [
         ('.md', 1), ('.MD', 1),
         ('.mkdn', 2), ('.MKDN', 2),
         ('.mdown', 3), ('.MDOWN', 3),
         ('.markdown', 4), ('.MARKDOWN', 4)
     ]
 
     def _detect_renderer(self, source, filename=None):
         """
         runs detection of what renderer should be used for generating html
         from a markup language
 
         filename can be also explicitly a renderer name
 
         :param source:
         :param filename:
         """
 
         if MarkupRenderer.MARKDOWN_PAT.findall(filename):
             detected_renderer = 'markdown'
         elif MarkupRenderer.RST_PAT.findall(filename):
             detected_renderer = 'rst'
         elif MarkupRenderer.JUPYTER_PAT.findall(filename):
             detected_renderer = 'jupyter'
         elif MarkupRenderer.PLAIN_PAT.findall(filename):
             detected_renderer = 'plain'
         else:
             detected_renderer = 'plain'
 
         return getattr(MarkupRenderer, detected_renderer)
 
     @classmethod
     def sanitize_html(cls, text):
-        # TODO: replace this with https://nh3.readthedocs.io/en/latest
-        # bleach is abandoned and deprecated :/
-
-        from .bleach_whitelist import markdown_attrs, markdown_tags
-        allowed_tags = markdown_tags
-        allowed_attrs = markdown_attrs
-
-        try:
-            return bleach.clean(text, tags=allowed_tags, attributes=allowed_attrs)
-        except Exception:
-            return 'TEXT CANNOT BE PARSED USING SANITIZE'
+        from .html_filters import sanitize_html
+        return sanitize_html(text, markdown=True)
 
     @classmethod
     def renderer_from_filename(cls, filename, exclude):
         """
         Detect renderer markdown/rst from filename and optionally use exclude
         list to remove some options. This is mostly used in helpers.
         Returns None when no renderer can be detected.
         """
         def _filter(elements):
             if isinstance(exclude, (list, tuple)):
                 return [x for x in elements if x not in exclude]
             return elements
 
         if filename.endswith(
                 tuple(_filter([x[0] for x in cls.MARKDOWN_EXTS if x[0]]))):
             return 'markdown'
         if filename.endswith(tuple(_filter([x[0] for x in cls.RST_EXTS if x[0]]))):
             return 'rst'
 
         return None
 
     def render(self, source, filename=None):
         """
         Renders a given filename using detected renderer
         it detects renderers based on file extension or mimetype.
         At last it will just do a simple html replacing new lines with <br/>
         """
 
         renderer = self._detect_renderer(source, filename)
         readme_data = renderer(source)
         return readme_data
 
     @classmethod
     def urlify_text(cls, text):
         def url_func(match_obj):
             url_full = match_obj.groups()[0]
             return f'<a href="{url_full}">{url_full}</a>'
 
         return cls.URL_PAT.sub(url_func, text)
 
     @classmethod
     def convert_mentions(cls, text, mode):
         mention_pat = cls.MENTION_PAT
 
         def wrapp(match_obj):
             uname = match_obj.groups()[0]
             hovercard_url = "pyroutes.url('hovercard_username', {'username': '%s'});" % uname
 
             if mode == 'markdown':
                 tmpl = '<strong class="tooltip-hovercard" data-hovercard-alt="{uname}" data-hovercard-url="{hovercard_url}">@{uname}</strong>'
             elif mode == 'rst':
                 tmpl = ' **@{uname}** '
             else:
                 raise ValueError('mode must be rst or markdown')
 
             return tmpl.format(**{'uname': uname,
                                   'hovercard_url': hovercard_url})
 
         return mention_pat.sub(wrapp, text).strip()
 
     @classmethod
     def plain(cls, source, universal_newline=True, leading_newline=True):
         source = safe_str(source)
         if universal_newline:
             newline = '\n'
             source = newline.join(source.splitlines())
 
         rendered_source = cls.urlify_text(source)
         source = ''
         if leading_newline:
             source += '<br />'
         source += rendered_source.replace("\n", '<br />')
 
         rendered = cls.sanitize_html(source)
         return rendered
 
     @classmethod
     def markdown(cls, source, safe=True, flavored=True, mentions=False,
                  clean_html=True):
         """
         returns markdown rendered code cleaned by the bleach library
         """
 
         if flavored:
             markdown_renderer = get_markdown_renderer_flavored(
                 cls.extensions, cls.output_format)
         else:
             markdown_renderer = get_markdown_renderer(
                 cls.extensions, cls.output_format)
 
         if mentions:
             mention_hl = cls.convert_mentions(source, mode='markdown')
             # we extracted mentions render with this using Mentions false
             return cls.markdown(mention_hl, safe=safe, flavored=flavored,
                                 mentions=False)
 
         try:
             rendered = markdown_renderer.convert(source)
 
         except Exception:
             log.exception('Error when rendering Markdown')
             if safe:
                 log.debug('Fallback to render in plain mode')
                 rendered = cls.plain(source)
             else:
                 raise
 
         if clean_html:
             rendered = cls.sanitize_html(rendered)
         return rendered
 
     @classmethod
     def rst(cls, source, safe=True, mentions=False, clean_html=False):
 
         if mentions:
             mention_hl = cls.convert_mentions(source, mode='rst')
             # we extracted mentions render with this using Mentions false
             return cls.rst(mention_hl, safe=safe, mentions=False)
 
         source = safe_str(source)
         try:
             docutils_settings = dict(
                 [(alias, None) for alias in
                  cls.RESTRUCTUREDTEXT_DISALLOWED_DIRECTIVES])
 
             docutils_settings.update({
                 'input_encoding': 'unicode',
                 'report_level': 4,
                 'syntax_highlight': 'short',
             })
 
             for k, v in list(docutils_settings.items()):
                 directives.register_directive(k, v)
 
             parts = publish_parts(source=source,
                                   writer=RhodeCodeWriter(),
                                   settings_overrides=docutils_settings)
             rendered = parts["fragment"]
             if clean_html:
                 rendered = cls.sanitize_html(rendered)
             return parts['html_title'] + rendered
         except Exception:
             log.exception('Error when rendering RST')
             if safe:
                 log.debug('Fallback to render in plain mode')
                 return cls.plain(source)
             else:
                 raise
 
     @classmethod
     def jupyter(cls, source, safe=True):
         from rhodecode.lib import helpers
 
         from traitlets.config import Config
         import nbformat
         from nbconvert import HTMLExporter
         from nbconvert.preprocessors import Preprocessor
 
         class CustomHTMLExporter(HTMLExporter):
             def _template_file_default(self):
                 return 'basic'
 
         class Sandbox(Preprocessor):
 
             def preprocess(self, nb, resources):
                 sandbox_text = 'SandBoxed(IPython.core.display.Javascript object)'
                 for cell in nb['cells']:
                     if not safe:
                         continue
 
                     if 'outputs' in cell:
                         for cell_output in cell['outputs']:
                             if 'data' in cell_output:
                                 if 'application/javascript' in cell_output['data']:
                                     cell_output['data']['text/plain'] = sandbox_text
                                     cell_output['data'].pop('application/javascript', None)
 
                     if 'source' in cell and cell['cell_type'] == 'markdown':
                         # sanitize similar like in markdown
                         cell['source'] = cls.sanitize_html(cell['source'])
 
                 return nb, resources
 
         def _sanitize_resources(input_resources):
             """
             Skip/sanitize some of the CSS generated and included in jupyter
             so it doesn't messes up UI so much
             """
 
             # TODO(marcink): probably we should replace this with whole custom
             # CSS set that doesn't screw up, but jupyter generated html has some
             # special markers, so it requires Custom HTML exporter template with
             # _default_template_path_default, to achieve that
 
             # strip the reset CSS
             input_resources[0] = input_resources[0][input_resources[0].find('/*! Source'):]
             return input_resources
 
         def as_html(notebook):
             conf = Config()
             conf.CustomHTMLExporter.preprocessors = [Sandbox]
             html_exporter = CustomHTMLExporter(config=conf)
 
             (body, resources) = html_exporter.from_notebook_node(notebook)
             header = '<!-- ## IPYTHON NOTEBOOK RENDERING ## -->'
             js = MakoTemplate(r'''
             <!-- MathJax configuration -->
             <script type="text/x-mathjax-config">
             MathJax.Hub.Config({
                 jax: ["input/TeX","output/HTML-CSS", "output/PreviewHTML"],
                 extensions: ["tex2jax.js","MathMenu.js","MathZoom.js", "fast-preview.js", "AssistiveMML.js", "[Contrib]/a11y/accessibility-menu.js"],
                 TeX: {
                     extensions: ["AMSmath.js","AMSsymbols.js","noErrors.js","noUndefined.js"]
                 },
                 tex2jax: {
                     inlineMath: [ ['$','$'], ["\\(","\\)"] ],
                     displayMath: [ ['$$','$$'], ["\\[","\\]"] ],
                     processEscapes: true,
                     processEnvironments: true
                 },
                 // Center justify equations in code and markdown cells. Elsewhere
                 // we use CSS to left justify single line equations in code cells.
                 displayAlign: 'center',
                 "HTML-CSS": {
                     styles: {'.MathJax_Display': {"margin": 0}},
                     linebreaks: { automatic: true },
                     availableFonts: ["STIX", "TeX"]
                 },
                 showMathMenu: false
             });
             </script>
             <!-- End of MathJax configuration -->
             <script src="${h.asset('js/src/math_jax/MathJax.js')}"></script>
             ''').render(h=helpers)
 
             css = MakoTemplate(r'''
             <link rel="stylesheet" type="text/css" href="${h.asset('css/style-ipython.css', ver=ver)}" media="screen"/>
             ''').render(h=helpers, ver='ver1')
 
             body = '\n'.join([header, css, js, body])
             return body, resources
 
         notebook = nbformat.reads(source, as_version=4)
         (body, resources) = as_html(notebook)
         return body
 
 
 class RstTemplateRenderer(object):
 
     def __init__(self):
         base = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
         rst_template_dirs = [os.path.join(base, 'templates', 'rst_templates')]
         self.template_store = TemplateLookup(
             directories=rst_template_dirs,
             input_encoding='utf-8',
             imports=['from rhodecode.lib import helpers as h'])
 
     def _get_template(self, templatename):
         return self.template_store.get_template(templatename)
 
     def render(self, template_name, **kwargs):
         template = self._get_template(template_name)
         return template.render(**kwargs)

rhodecode/model/settings.py

@@ -1,923 +1,923 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import re
 import logging
 import time
 import functools
-import bleach
 from collections import namedtuple
 
 from pyramid.threadlocal import get_current_request
 
 from rhodecode.lib import rc_cache
 from rhodecode.lib.hash_utils import sha1_safe
+from rhodecode.lib.html_filters import sanitize_html
 from rhodecode.lib.utils2 import (
     Optional, AttributeDict, safe_str, remove_prefix, str2bool)
 from rhodecode.lib.vcs.backends import base
 from rhodecode.lib.statsd_client import StatsdClient
 from rhodecode.model import BaseModel
 from rhodecode.model.db import (
     RepoRhodeCodeUi, RepoRhodeCodeSetting, RhodeCodeUi, RhodeCodeSetting)
 from rhodecode.model.meta import Session
 
 
 log = logging.getLogger(__name__)
 
 
 UiSetting = namedtuple(
     'UiSetting', ['section', 'key', 'value', 'active'])
 
 SOCIAL_PLUGINS_LIST = ['github', 'bitbucket', 'twitter', 'google']
 
 
 class SettingNotFound(Exception):
     def __init__(self, setting_id):
         msg = f'Setting `{setting_id}` is not found'
         super().__init__(msg)
 
 
 class SettingsModel(BaseModel):
     BUILTIN_HOOKS = (
         RhodeCodeUi.HOOK_REPO_SIZE, RhodeCodeUi.HOOK_PUSH,
         RhodeCodeUi.HOOK_PRE_PUSH, RhodeCodeUi.HOOK_PRETX_PUSH,
         RhodeCodeUi.HOOK_PULL, RhodeCodeUi.HOOK_PRE_PULL,
         RhodeCodeUi.HOOK_PUSH_KEY,)
     HOOKS_SECTION = 'hooks'
 
     def __init__(self, sa=None, repo=None):
         self.repo = repo
         self.UiDbModel = RepoRhodeCodeUi if repo else RhodeCodeUi
         self.SettingsDbModel = (
             RepoRhodeCodeSetting if repo else RhodeCodeSetting)
         super().__init__(sa)
 
     def get_ui_by_key(self, key):
         q = self.UiDbModel.query()
         q = q.filter(self.UiDbModel.ui_key == key)
         q = self._filter_by_repo(RepoRhodeCodeUi, q)
         return q.scalar()
 
     def get_ui_by_section(self, section):
         q = self.UiDbModel.query()
         q = q.filter(self.UiDbModel.ui_section == section)
         q = self._filter_by_repo(RepoRhodeCodeUi, q)
         return q.all()
 
     def get_ui_by_section_and_key(self, section, key):
         q = self.UiDbModel.query()
         q = q.filter(self.UiDbModel.ui_section == section)
         q = q.filter(self.UiDbModel.ui_key == key)
         q = self._filter_by_repo(RepoRhodeCodeUi, q)
         return q.scalar()
 
     def get_ui(self, section=None, key=None):
         q = self.UiDbModel.query()
         q = self._filter_by_repo(RepoRhodeCodeUi, q)
 
         if section:
             q = q.filter(self.UiDbModel.ui_section == section)
         if key:
             q = q.filter(self.UiDbModel.ui_key == key)
 
         # TODO: mikhail: add caching
         result = [
             UiSetting(
                 section=safe_str(r.ui_section), key=safe_str(r.ui_key),
                 value=safe_str(r.ui_value), active=r.ui_active
             )
             for r in q.all()
         ]
         return result
 
     def get_builtin_hooks(self):
         q = self.UiDbModel.query()
         q = q.filter(self.UiDbModel.ui_key.in_(self.BUILTIN_HOOKS))
         return self._get_hooks(q)
 
     def get_custom_hooks(self):
         q = self.UiDbModel.query()
         q = q.filter(~self.UiDbModel.ui_key.in_(self.BUILTIN_HOOKS))
         return self._get_hooks(q)
 
     def create_ui_section_value(self, section, val, key=None, active=True):
         new_ui = self.UiDbModel()
         new_ui.ui_section = section
         new_ui.ui_value = val
         new_ui.ui_active = active
 
         repository_id = ''
         if self.repo:
             repo = self._get_repo(self.repo)
             repository_id = repo.repo_id
             new_ui.repository_id = repository_id
 
         if not key:
             # keys are unique so they need appended info
             if self.repo:
                 key = sha1_safe(f'{section}{val}{repository_id}')
             else:
                 key = sha1_safe(f'{section}{val}')
 
         new_ui.ui_key = key
 
         Session().add(new_ui)
         return new_ui
 
     def create_or_update_hook(self, key, value):
         ui = (
             self.get_ui_by_section_and_key(self.HOOKS_SECTION, key) or
             self.UiDbModel())
         ui.ui_section = self.HOOKS_SECTION
         ui.ui_active = True
         ui.ui_key = key
         ui.ui_value = value
 
         if self.repo:
             repo = self._get_repo(self.repo)
             repository_id = repo.repo_id
             ui.repository_id = repository_id
 
         Session().add(ui)
         return ui
 
     def delete_ui(self, id_):
         ui = self.UiDbModel.get(id_)
         if not ui:
             raise SettingNotFound(id_)
         Session().delete(ui)
 
     def get_setting_by_name(self, name):
         q = self._get_settings_query()
         q = q.filter(self.SettingsDbModel.app_settings_name == name)
         return q.scalar()
 
     def create_or_update_setting(
             self, name, val=Optional(''), type_=Optional('unicode')):
         """
         Creates or updates RhodeCode setting. If updates is triggered it will
         only update parameters that are explicitly set Optional instance will
         be skipped
 
         :param name:
         :param val:
         :param type_:
         :return:
         """
 
         res = self.get_setting_by_name(name)
         repo = self._get_repo(self.repo) if self.repo else None
 
         if not res:
             val = Optional.extract(val)
             type_ = Optional.extract(type_)
 
             args = (
                 (repo.repo_id, name, val, type_)
                 if repo else (name, val, type_))
             res = self.SettingsDbModel(*args)
 
         else:
             if self.repo:
                 res.repository_id = repo.repo_id
 
             res.app_settings_name = name
             if not isinstance(type_, Optional):
                 # update if set
                 res.app_settings_type = type_
             if not isinstance(val, Optional):
                 # update if set
                 res.app_settings_value = val
 
         Session().add(res)
         return res
 
     def get_cache_region(self):
         repo = self._get_repo(self.repo) if self.repo else None
         cache_key = f"repo.{repo.repo_id}" if repo else "repo.ALL"
         cache_namespace_uid = f'cache_settings.{cache_key}'
         region = rc_cache.get_or_create_region('cache_general', cache_namespace_uid)
         return region, cache_namespace_uid
 
     def invalidate_settings_cache(self, hard=False):
         region, namespace_key = self.get_cache_region()
         log.debug('Invalidation cache [%s] region %s for cache_key: %s',
                   'invalidate_settings_cache', region, namespace_key)
 
         # we use hard cleanup if invalidation is sent
         rc_cache.clear_cache_namespace(region, namespace_key, method=rc_cache.CLEAR_DELETE)
 
     def get_cache_call_method(self, cache=True):
         region, cache_key = self.get_cache_region()
 
         @region.conditional_cache_on_arguments(condition=cache)
         def _get_all_settings(name, key):
             q = self._get_settings_query()
             if not q:
                 raise Exception('Could not get application settings !')
 
             settings = {
                 f'rhodecode_{res.app_settings_name}': res.app_settings_value
                 for res in q
             }
             return settings
         return _get_all_settings
 
     def get_all_settings(self, cache=False, from_request=True):
         # defines if we use GLOBAL, or PER_REPO
         repo = self._get_repo(self.repo) if self.repo else None
 
         # initially try the requests context, this is the fastest
         # we only fetch global config, NOT for repo-specific
         if from_request and not repo:
             request = get_current_request()
 
             if request and hasattr(request, 'call_context') and hasattr(request.call_context, 'rc_config'):
                 rc_config = request.call_context.rc_config
                 if rc_config:
                     return rc_config
 
         _region, cache_key = self.get_cache_region()
         _get_all_settings = self.get_cache_call_method(cache=cache)
 
         start = time.time()
         result = _get_all_settings('rhodecode_settings', cache_key)
         compute_time = time.time() - start
         log.debug('cached method:%s took %.4fs', _get_all_settings.__name__, compute_time)
 
         statsd = StatsdClient.statsd
         if statsd:
             elapsed_time_ms = round(1000.0 * compute_time)  # use ms only
             statsd.timing("rhodecode_settings_timing.histogram", elapsed_time_ms,
                           use_decimals=False)
 
         log.debug('Fetching app settings for key: %s took: %.4fs: cache: %s', cache_key, compute_time, cache)
 
         return result
 
     def get_auth_settings(self):
         q = self._get_settings_query()
         q = q.filter(
             self.SettingsDbModel.app_settings_name.startswith('auth_'))
         rows = q.all()
         auth_settings = {
             row.app_settings_name: row.app_settings_value for row in rows}
         return auth_settings
 
     def get_auth_plugins(self):
         auth_plugins = self.get_setting_by_name("auth_plugins")
         return auth_plugins.app_settings_value
 
     def get_default_repo_settings(self, strip_prefix=False):
         q = self._get_settings_query()
         q = q.filter(
             self.SettingsDbModel.app_settings_name.startswith('default_'))
         rows = q.all()
 
         result = {}
         for row in rows:
             key = row.app_settings_name
             if strip_prefix:
                 key = remove_prefix(key, prefix='default_')
             result.update({key: row.app_settings_value})
         return result
 
     def get_repo(self):
         repo = self._get_repo(self.repo)
         if not repo:
             raise Exception(
                 'Repository `{}` cannot be found inside the database'.format(
                     self.repo))
         return repo
 
     def _filter_by_repo(self, model, query):
         if self.repo:
             repo = self.get_repo()
             query = query.filter(model.repository_id == repo.repo_id)
         return query
 
     def _get_hooks(self, query):
         query = query.filter(self.UiDbModel.ui_section == self.HOOKS_SECTION)
         query = self._filter_by_repo(RepoRhodeCodeUi, query)
         return query.all()
 
     def _get_settings_query(self):
         q = self.SettingsDbModel.query()
         return self._filter_by_repo(RepoRhodeCodeSetting, q)
 
     def list_enabled_social_plugins(self, settings):
         enabled = []
         for plug in SOCIAL_PLUGINS_LIST:
             if str2bool(settings.get(f'rhodecode_auth_{plug}_enabled')):
                 enabled.append(plug)
         return enabled
 
 
 def assert_repo_settings(func):
     @functools.wraps(func)
     def _wrapper(self, *args, **kwargs):
         if not self.repo_settings:
             raise Exception('Repository is not specified')
         return func(self, *args, **kwargs)
     return _wrapper
 
 
 class IssueTrackerSettingsModel(object):
     INHERIT_SETTINGS = 'inherit_issue_tracker_settings'
     SETTINGS_PREFIX = 'issuetracker_'
 
     def __init__(self, sa=None, repo=None):
         self.global_settings = SettingsModel(sa=sa)
         self.repo_settings = SettingsModel(sa=sa, repo=repo) if repo else None
 
     @property
     def inherit_global_settings(self):
         if not self.repo_settings:
             return True
         setting = self.repo_settings.get_setting_by_name(self.INHERIT_SETTINGS)
         return setting.app_settings_value if setting else True
 
     @inherit_global_settings.setter
     def inherit_global_settings(self, value):
         if self.repo_settings:
             settings = self.repo_settings.create_or_update_setting(
                 self.INHERIT_SETTINGS, value, type_='bool')
             Session().add(settings)
 
     def _get_keyname(self, key, uid, prefix=''):
         return '{}{}{}_{}'.format(
             prefix, self.SETTINGS_PREFIX, key, uid)
 
     def _make_dict_for_settings(self, qs):
         prefix_match = self._get_keyname('pat', '', 'rhodecode_')
 
         issuetracker_entries = {}
         # create keys
         for k, v in qs.items():
             if k.startswith(prefix_match):
                 uid = k[len(prefix_match):]
                 issuetracker_entries[uid] = None
 
         def url_cleaner(input_str):
             input_str = input_str.replace('"', '').replace("'", '')
-            input_str = bleach.clean(input_str, strip=True)
+            input_str = sanitize_html(input_str, strip=True)
             return input_str
 
         # populate
         for uid in issuetracker_entries:
             url_data = qs.get(self._get_keyname('url', uid, 'rhodecode_'))
 
             pat = qs.get(self._get_keyname('pat', uid, 'rhodecode_'))
             try:
                 pat_compiled = re.compile(r'%s' % pat)
             except re.error:
                 pat_compiled = None
 
             issuetracker_entries[uid] = AttributeDict({
                 'pat': pat,
                 'pat_compiled': pat_compiled,
                 'url': url_cleaner(
                     qs.get(self._get_keyname('url', uid, 'rhodecode_')) or ''),
-                'pref': bleach.clean(
+                'pref': sanitize_html(
                     qs.get(self._get_keyname('pref', uid, 'rhodecode_')) or ''),
                 'desc': qs.get(
                     self._get_keyname('desc', uid, 'rhodecode_')),
             })
 
         return issuetracker_entries
 
     def get_global_settings(self, cache=False):
         """
         Returns list of global issue tracker settings
         """
         defaults = self.global_settings.get_all_settings(cache=cache)
         settings = self._make_dict_for_settings(defaults)
         return settings
 
     def get_repo_settings(self, cache=False):
         """
         Returns list of issue tracker settings per repository
         """
         if not self.repo_settings:
             raise Exception('Repository is not specified')
         all_settings = self.repo_settings.get_all_settings(cache=cache)
         settings = self._make_dict_for_settings(all_settings)
         return settings
 
     def get_settings(self, cache=False):
         if self.inherit_global_settings:
             return self.get_global_settings(cache=cache)
         else:
             return self.get_repo_settings(cache=cache)
 
     def delete_entries(self, uid):
         if self.repo_settings:
             all_patterns = self.get_repo_settings()
             settings_model = self.repo_settings
         else:
             all_patterns = self.get_global_settings()
             settings_model = self.global_settings
         entries = all_patterns.get(uid, [])
 
         for del_key in entries:
             setting_name = self._get_keyname(del_key, uid)
             entry = settings_model.get_setting_by_name(setting_name)
             if entry:
                 Session().delete(entry)
 
         Session().commit()
 
     def create_or_update_setting(
             self, name, val=Optional(''), type_=Optional('unicode')):
         if self.repo_settings:
             setting = self.repo_settings.create_or_update_setting(
                 name, val, type_)
         else:
             setting = self.global_settings.create_or_update_setting(
                 name, val, type_)
         return setting
 
 
 class VcsSettingsModel(object):
 
     INHERIT_SETTINGS = 'inherit_vcs_settings'
     GENERAL_SETTINGS = (
         'use_outdated_comments',
         'pr_merge_enabled',
         'hg_use_rebase_for_merging',
         'hg_close_branch_before_merging',
         'git_use_rebase_for_merging',
         'git_close_branch_before_merging',
         'diff_cache',
     )
 
     HOOKS_SETTINGS = (
         ('hooks', 'changegroup.repo_size'),
         ('hooks', 'changegroup.push_logger'),
         ('hooks', 'outgoing.pull_logger'),
     )
     HG_SETTINGS = (
         ('extensions', 'largefiles'),
         ('phases', 'publish'),
         ('extensions', 'evolve'),
         ('extensions', 'topic'),
         ('experimental', 'evolution'),
         ('experimental', 'evolution.exchange'),
     )
     GIT_SETTINGS = (
         ('vcs_git_lfs', 'enabled'),
     )
     GLOBAL_HG_SETTINGS = (
         ('extensions', 'largefiles'),
         ('largefiles', 'usercache'),
         ('phases', 'publish'),
         ('extensions', 'hgsubversion'),
         ('extensions', 'evolve'),
         ('extensions', 'topic'),
         ('experimental', 'evolution'),
         ('experimental', 'evolution.exchange'),
     )
 
     GLOBAL_GIT_SETTINGS = (
         ('vcs_git_lfs', 'enabled'),
         ('vcs_git_lfs', 'store_location')
     )
 
     GLOBAL_SVN_SETTINGS = (
         ('vcs_svn_proxy', 'http_requests_enabled'),
         ('vcs_svn_proxy', 'http_server_url')
     )
 
     SVN_BRANCH_SECTION = 'vcs_svn_branch'
     SVN_TAG_SECTION = 'vcs_svn_tag'
     SSL_SETTING = ('web', 'push_ssl')
     PATH_SETTING = ('paths', '/')
 
     def __init__(self, sa=None, repo=None):
         self.global_settings = SettingsModel(sa=sa)
         self.repo_settings = SettingsModel(sa=sa, repo=repo) if repo else None
         self._ui_settings = (
             self.HG_SETTINGS + self.GIT_SETTINGS + self.HOOKS_SETTINGS)
         self._svn_sections = (self.SVN_BRANCH_SECTION, self.SVN_TAG_SECTION)
 
     @property
     @assert_repo_settings
     def inherit_global_settings(self):
         setting = self.repo_settings.get_setting_by_name(self.INHERIT_SETTINGS)
         return setting.app_settings_value if setting else True
 
     @inherit_global_settings.setter
     @assert_repo_settings
     def inherit_global_settings(self, value):
         self.repo_settings.create_or_update_setting(
             self.INHERIT_SETTINGS, value, type_='bool')
 
     def get_global_svn_branch_patterns(self):
         return self.global_settings.get_ui_by_section(self.SVN_BRANCH_SECTION)
 
     @assert_repo_settings
     def get_repo_svn_branch_patterns(self):
         return self.repo_settings.get_ui_by_section(self.SVN_BRANCH_SECTION)
 
     def get_global_svn_tag_patterns(self):
         return self.global_settings.get_ui_by_section(self.SVN_TAG_SECTION)
 
     @assert_repo_settings
     def get_repo_svn_tag_patterns(self):
         return self.repo_settings.get_ui_by_section(self.SVN_TAG_SECTION)
 
     def get_global_settings(self):
         return self._collect_all_settings(global_=True)
 
     @assert_repo_settings
     def get_repo_settings(self):
         return self._collect_all_settings(global_=False)
 
     @assert_repo_settings
     def get_repo_settings_inherited(self):
         global_settings = self.get_global_settings()
         global_settings.update(self.get_repo_settings())
         return global_settings
 
     @assert_repo_settings
     def create_or_update_repo_settings(
             self, data, inherit_global_settings=False):
         from rhodecode.model.scm import ScmModel
 
         self.inherit_global_settings = inherit_global_settings
 
         repo = self.repo_settings.get_repo()
         if not inherit_global_settings:
             if repo.repo_type == 'svn':
                 self.create_repo_svn_settings(data)
             else:
                 self.create_or_update_repo_hook_settings(data)
                 self.create_or_update_repo_pr_settings(data)
 
             if repo.repo_type == 'hg':
                 self.create_or_update_repo_hg_settings(data)
 
             if repo.repo_type == 'git':
                 self.create_or_update_repo_git_settings(data)
 
         ScmModel().mark_for_invalidation(repo.repo_name, delete=True)
 
     @assert_repo_settings
     def create_or_update_repo_hook_settings(self, data):
         for section, key in self.HOOKS_SETTINGS:
             data_key = self._get_form_ui_key(section, key)
             if data_key not in data:
                 raise ValueError(
                     f'The given data does not contain {data_key} key')
 
             active = data.get(data_key)
             repo_setting = self.repo_settings.get_ui_by_section_and_key(
                 section, key)
             if not repo_setting:
                 global_setting = self.global_settings.\
                     get_ui_by_section_and_key(section, key)
                 self.repo_settings.create_ui_section_value(
                     section, global_setting.ui_value, key=key, active=active)
             else:
                 repo_setting.ui_active = active
                 Session().add(repo_setting)
 
     def update_global_hook_settings(self, data):
         for section, key in self.HOOKS_SETTINGS:
             data_key = self._get_form_ui_key(section, key)
             if data_key not in data:
                 raise ValueError(
                     f'The given data does not contain {data_key} key')
             active = data.get(data_key)
             repo_setting = self.global_settings.get_ui_by_section_and_key(
                 section, key)
             repo_setting.ui_active = active
             Session().add(repo_setting)
 
     @assert_repo_settings
     def create_or_update_repo_pr_settings(self, data):
         return self._create_or_update_general_settings(
             self.repo_settings, data)
 
     def create_or_update_global_pr_settings(self, data):
         return self._create_or_update_general_settings(
             self.global_settings, data)
 
     @assert_repo_settings
     def create_repo_svn_settings(self, data):
         return self._create_svn_settings(self.repo_settings, data)
 
     def _set_evolution(self, settings, is_enabled):
         if is_enabled:
             # if evolve is active set evolution=all
 
             self._create_or_update_ui(
                 settings, *('experimental', 'evolution'), value='all',
                 active=True)
             self._create_or_update_ui(
                 settings, *('experimental', 'evolution.exchange'), value='yes',
                 active=True)
             # if evolve is active set topics server support
             self._create_or_update_ui(
                 settings, *('extensions', 'topic'), value='',
                 active=True)
 
         else:
             self._create_or_update_ui(
                 settings, *('experimental', 'evolution'), value='',
                 active=False)
             self._create_or_update_ui(
                 settings, *('experimental', 'evolution.exchange'), value='no',
                 active=False)
             self._create_or_update_ui(
                 settings, *('extensions', 'topic'), value='',
                 active=False)
 
     @assert_repo_settings
     def create_or_update_repo_hg_settings(self, data):
         largefiles, phases, evolve = \
             self.HG_SETTINGS[:3]
         largefiles_key, phases_key, evolve_key = \
             self._get_settings_keys(self.HG_SETTINGS[:3], data)
 
         self._create_or_update_ui(
             self.repo_settings, *largefiles, value='',
             active=data[largefiles_key])
         self._create_or_update_ui(
             self.repo_settings, *evolve, value='',
             active=data[evolve_key])
         self._set_evolution(self.repo_settings, is_enabled=data[evolve_key])
 
         self._create_or_update_ui(
             self.repo_settings, *phases, value=safe_str(data[phases_key]))
 
     def create_or_update_global_hg_settings(self, data):
         largefiles, largefiles_store, phases, hgsubversion, evolve \
             = self.GLOBAL_HG_SETTINGS[:5]
         largefiles_key, largefiles_store_key, phases_key, subversion_key, evolve_key \
             = self._get_settings_keys(self.GLOBAL_HG_SETTINGS[:5], data)
 
         self._create_or_update_ui(
             self.global_settings, *largefiles, value='',
             active=data[largefiles_key])
         self._create_or_update_ui(
             self.global_settings, *largefiles_store, value=data[largefiles_store_key])
         self._create_or_update_ui(
             self.global_settings, *phases, value=safe_str(data[phases_key]))
         self._create_or_update_ui(
             self.global_settings, *hgsubversion, active=data[subversion_key])
         self._create_or_update_ui(
             self.global_settings, *evolve, value='',
             active=data[evolve_key])
         self._set_evolution(self.global_settings, is_enabled=data[evolve_key])
 
     def create_or_update_repo_git_settings(self, data):
         # NOTE(marcink): # comma makes unpack work properly
         lfs_enabled, \
             = self.GIT_SETTINGS
 
         lfs_enabled_key, \
             = self._get_settings_keys(self.GIT_SETTINGS, data)
 
         self._create_or_update_ui(
             self.repo_settings, *lfs_enabled, value=data[lfs_enabled_key],
             active=data[lfs_enabled_key])
 
     def create_or_update_global_git_settings(self, data):
         lfs_enabled, lfs_store_location \
             = self.GLOBAL_GIT_SETTINGS
         lfs_enabled_key, lfs_store_location_key \
             = self._get_settings_keys(self.GLOBAL_GIT_SETTINGS, data)
 
         self._create_or_update_ui(
             self.global_settings, *lfs_enabled, value=data[lfs_enabled_key],
             active=data[lfs_enabled_key])
         self._create_or_update_ui(
             self.global_settings, *lfs_store_location,
             value=data[lfs_store_location_key])
 
     def create_or_update_global_svn_settings(self, data):
         # branch/tags patterns
         self._create_svn_settings(self.global_settings, data)
 
         http_requests_enabled, http_server_url = self.GLOBAL_SVN_SETTINGS
         http_requests_enabled_key, http_server_url_key = self._get_settings_keys(
             self.GLOBAL_SVN_SETTINGS, data)
 
         self._create_or_update_ui(
             self.global_settings, *http_requests_enabled,
             value=safe_str(data[http_requests_enabled_key]))
         self._create_or_update_ui(
             self.global_settings, *http_server_url,
             value=data[http_server_url_key])
 
     def update_global_ssl_setting(self, value):
         self._create_or_update_ui(
             self.global_settings, *self.SSL_SETTING, value=value)
 
     def update_global_path_setting(self, value):
         self._create_or_update_ui(
             self.global_settings, *self.PATH_SETTING, value=value)
 
     @assert_repo_settings
     def delete_repo_svn_pattern(self, id_):
         ui = self.repo_settings.UiDbModel.get(id_)
         if ui and ui.repository.repo_name == self.repo_settings.repo:
             # only delete if it's the same repo as initialized settings
             self.repo_settings.delete_ui(id_)
         else:
             # raise error as if we wouldn't find this option
             self.repo_settings.delete_ui(-1)
 
     def delete_global_svn_pattern(self, id_):
         self.global_settings.delete_ui(id_)
 
     @assert_repo_settings
     def get_repo_ui_settings(self, section=None, key=None):
         global_uis = self.global_settings.get_ui(section, key)
         repo_uis = self.repo_settings.get_ui(section, key)
 
         filtered_repo_uis = self._filter_ui_settings(repo_uis)
         filtered_repo_uis_keys = [
             (s.section, s.key) for s in filtered_repo_uis]
 
         def _is_global_ui_filtered(ui):
             return (
                 (ui.section, ui.key) in filtered_repo_uis_keys
                 or ui.section in self._svn_sections)
 
         filtered_global_uis = [
             ui for ui in global_uis if not _is_global_ui_filtered(ui)]
 
         return filtered_global_uis + filtered_repo_uis
 
     def get_global_ui_settings(self, section=None, key=None):
         return self.global_settings.get_ui(section, key)
 
     def get_ui_settings_as_config_obj(self, section=None, key=None):
         config = base.Config()
 
         ui_settings = self.get_ui_settings(section=section, key=key)
 
         for entry in ui_settings:
             config.set(entry.section, entry.key, entry.value)
 
         return config
 
     def get_ui_settings(self, section=None, key=None):
         if not self.repo_settings or self.inherit_global_settings:
             return self.get_global_ui_settings(section, key)
         else:
             return self.get_repo_ui_settings(section, key)
 
     def get_svn_patterns(self, section=None):
         if not self.repo_settings:
             return self.get_global_ui_settings(section)
         else:
             return self.get_repo_ui_settings(section)
 
     @assert_repo_settings
     def get_repo_general_settings(self):
         global_settings = self.global_settings.get_all_settings()
         repo_settings = self.repo_settings.get_all_settings()
         filtered_repo_settings = self._filter_general_settings(repo_settings)
         global_settings.update(filtered_repo_settings)
         return global_settings
 
     def get_global_general_settings(self):
         return self.global_settings.get_all_settings()
 
     def get_general_settings(self):
         if not self.repo_settings or self.inherit_global_settings:
             return self.get_global_general_settings()
         else:
             return self.get_repo_general_settings()
 
     def get_repos_location(self):
         return self.global_settings.get_ui_by_key('/').ui_value
 
     def _filter_ui_settings(self, settings):
         filtered_settings = [
             s for s in settings if self._should_keep_setting(s)]
         return filtered_settings
 
     def _should_keep_setting(self, setting):
         keep = (
             (setting.section, setting.key) in self._ui_settings or
             setting.section in self._svn_sections)
         return keep
 
     def _filter_general_settings(self, settings):
         keys = [f'rhodecode_{key}' for key in self.GENERAL_SETTINGS]
         return {
             k: settings[k]
             for k in settings if k in keys}
 
     def _collect_all_settings(self, global_=False):
         settings = self.global_settings if global_ else self.repo_settings
         result = {}
 
         for section, key in self._ui_settings:
             ui = settings.get_ui_by_section_and_key(section, key)
             result_key = self._get_form_ui_key(section, key)
 
             if ui:
                 if section in ('hooks', 'extensions'):
                     result[result_key] = ui.ui_active
                 elif result_key in ['vcs_git_lfs_enabled']:
                     result[result_key] = ui.ui_active
                 else:
                     result[result_key] = ui.ui_value
 
         for name in self.GENERAL_SETTINGS:
             setting = settings.get_setting_by_name(name)
             if setting:
                 result_key = f'rhodecode_{name}'
                 result[result_key] = setting.app_settings_value
 
         return result
 
     def _get_form_ui_key(self, section, key):
         return '{section}_{key}'.format(
             section=section, key=key.replace('.', '_'))
 
     def _create_or_update_ui(
             self, settings, section, key, value=None, active=None):
         ui = settings.get_ui_by_section_and_key(section, key)
         if not ui:
             active = True if active is None else active
             settings.create_ui_section_value(
                 section, value, key=key, active=active)
         else:
             if active is not None:
                 ui.ui_active = active
             if value is not None:
                 ui.ui_value = value
             Session().add(ui)
 
     def _create_svn_settings(self, settings, data):
         svn_settings = {
             'new_svn_branch': self.SVN_BRANCH_SECTION,
             'new_svn_tag': self.SVN_TAG_SECTION
         }
         for key in svn_settings:
             if data.get(key):
                 settings.create_ui_section_value(svn_settings[key], data[key])
 
     def _create_or_update_general_settings(self, settings, data):
         for name in self.GENERAL_SETTINGS:
             data_key = f'rhodecode_{name}'
             if data_key not in data:
                 raise ValueError(
                     f'The given data does not contain {data_key} key')
             setting = settings.create_or_update_setting(
                 name, data[data_key], 'bool')
             Session().add(setting)
 
     def _get_settings_keys(self, settings, data):
         data_keys = [self._get_form_ui_key(*s) for s in settings]
         for data_key in data_keys:
             if data_key not in data:
                 raise ValueError(
                     f'The given data does not contain {data_key} key')
         return data_keys
 
     def create_largeobjects_dirs_if_needed(self, repo_store_path):
         """
         This is subscribed to the `pyramid.events.ApplicationCreated` event. It
         does a repository scan if enabled in the settings.
         """
 
         from rhodecode.lib.vcs.backends.hg import largefiles_store
         from rhodecode.lib.vcs.backends.git import lfs_store
 
         paths = [
             largefiles_store(repo_store_path),
             lfs_store(repo_store_path)]
 
         for path in paths:
             if os.path.isdir(path):
                 continue
             if os.path.isfile(path):
                 continue
             # not a file nor dir, we try to create it
             try:
                 os.makedirs(path)
             except Exception:
                 log.warning('Failed to create largefiles dir:%s', path)