code: fixes to escape characters improperly used

rhodecode/lib/diffs.py

@@ -816,7 +816,7 b' class DiffProcessor(object):'
         return b''.join(raw_diff), chunks, stats
 
     def _safe_id(self, idstring):
-        """Make a string safe for including in an id attribute.
+        r"""Make a string safe for including in an id attribute.
 
         The HTML spec says that id attributes 'must begin with
         a letter ([A-Za-z]) and may be followed by any number
@@ -828,8 +828,8 b' class DiffProcessor(object):'
         Whitespace is transformed into underscores, and then
         anything which is not a hyphen or a character that
         matches \w (alphanumerics and underscore) is removed.
+        """
 
-        """
         # Transform all whitespace to underscore
         idstring = re.sub(r'\s', "_", f'{idstring}')
         # Remove everything that is not a hyphen or a member of \w

rhodecode/lib/helpers.py

@@ -1038,33 +1038,71 b' def gravatar_with_user(request, author, '
     return _render('gravatar_with_user', author, show_disabled=show_disabled, tooltip=tooltip)
 
 
-tags_paterns = OrderedDict((
-    ('lang', (re.compile(r'\[(lang|language)\ \=\>\ *([a-zA-Z\-\/\#\+\.]*)\]'),
-             '<div class="metatag" tag="lang">\\2</div>')),
-
-    ('see', (re.compile(r'\[see\ \=\&gt;\ *([a-zA-Z0-9\/\=\?\&amp;\ \:\/\.\-]*)\]'),
-            '<div class="metatag" tag="see">see: \\1 </div>')),
-
-    ('url', (re.compile(r'\[url\ \=\&gt;\ \[([a-zA-Z0-9\ \.\-\_]+)\]\((http://|https://|/)(.*?)\)\]'),
-            '<div class="metatag" tag="url"> <a href="\\2\\3">\\1</a> </div>')),
-
-    ('license', (re.compile(r'\[license\ \=\&gt;\ *([a-zA-Z0-9\/\=\?\&amp;\ \:\/\.\-]*)\]'),
-                '<div class="metatag" tag="license"><a href="http:\/\/www.opensource.org/licenses/\\1">\\1</a></div>')),
-
-    ('ref', (re.compile(r'\[(requires|recommends|conflicts|base)\ \=\&gt;\ *([a-zA-Z0-9\-\/]*)\]'),
-            '<div class="metatag" tag="ref \\1">\\1: <a href="/\\2">\\2</a></div>')),
-
-    ('state', (re.compile(r'\[(stable|featured|stale|dead|dev|deprecated)\]'),
-              '<div class="metatag" tag="state \\1">\\1</div>')),
-
+tags_patterns = OrderedDict(
+    (
+        (
+            "lang",
+            (
+                re.compile(r"\[(lang|language)\ \=\&gt;\ *([a-zA-Z\-\/\#\+\.]*)\]"),
+                '<div class="metatag" tag="lang">\\2</div>',
+            ),
+        ),
+        (
+            "see",
+            (
+                re.compile(r"\[see\ \=\&gt;\ *([a-zA-Z0-9\/\=\?\&amp;\ \:\/\.\-]*)\]"),
+                '<div class="metatag" tag="see">see: \\1 </div>',
+            ),
+        ),
+        (
+            "url",
+            (
+                re.compile(
+                    r"\[url\ \=\&gt;\ \[([a-zA-Z0-9\ \.\-\_]+)\]\((http://|https://|/)(.*?)\)\]"
+                ),
+                '<div class="metatag" tag="url"> <a href="\\2\\3">\\1</a> </div>',
+            ),
+        ),
+        (
+            "license",
+            (
+                re.compile(
+                    r"\[license\ \=\&gt;\ *([a-zA-Z0-9\/\=\?\&amp;\ \:\/\.\-]*)\]"
+                ),
+                r'<div class="metatag" tag="license"><a href="http:\/\/www.opensource.org/licenses/\\1">\\1</a></div>',
+            ),
+        ),
+        (
+            "ref",
+            (
+                re.compile(
+                    r"\[(requires|recommends|conflicts|base)\ \=\&gt;\ *([a-zA-Z0-9\-\/]*)\]"
+                ),
+                '<div class="metatag" tag="ref \\1">\\1: <a href="/\\2">\\2</a></div>',
+            ),
+        ),
+        (
+            "state",
+            (
+                re.compile(r"\[(stable|featured|stale|dead|dev|deprecated)\]"),
+                '<div class="metatag" tag="state \\1">\\1</div>',
+            ),
+        ),
     # label in grey
-    ('label', (re.compile(r'\[([a-z]+)\]'),
-              '<div class="metatag" tag="label">\\1</div>')),
-
+        (
+            "label",
+            (re.compile(r"\[([a-z]+)\]"), '<div class="metatag" tag="label">\\1</div>'),
+        ),
     # generic catch all in grey
-    ('generic', (re.compile(r'\[([a-zA-Z0-9\.\-\_]+)\]'),
-                '<div class="metatag" tag="generic">\\1</div>')),
-))
+        (
+            "generic",
+            (
+                re.compile(r"\[([a-zA-Z0-9\.\-\_]+)\]"),
+                '<div class="metatag" tag="generic">\\1</div>',
+            ),
+        ),
+    )
+)
 
 
 def extract_metatags(value):
@@ -1075,7 +1113,7 b' def extract_metatags(value):'
     if not value:
         return tags, ''
 
-    for key, val in list(tags_paterns.items()):
+    for key, val in list(tags_patterns.items()):
         pat, replace_html = val
         tags.extend([(key, x.group()) for x in pat.finditer(value)])
         value = pat.sub('', value)
@@ -1091,7 +1129,7 b' def style_metatag(tag_type, value):'
         return ''
 
     html_value = value
-    tag_data = tags_paterns.get(tag_type)
+    tag_data = tags_patterns.get(tag_type)
     if tag_data:
         pat, replace_html = tag_data
         # convert to plain `str` instead of a markup tag to be used in
@@ -1530,7 +1568,7 b' def urlify_text(text_, safe=True, **href'
     """
 
     url_pat = re.compile(r'''(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@#.&+]'''
-                         '''|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)''')
+                         r'''|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)''')
 
     def url_func(match_obj):
         url_full = match_obj.groups()[0]

rhodecode/lib/index/search_utils.py

@@ -148,7 +148,7 b' def normalize_text_for_matching(x):'
     Replaces all non alfanum characters to spaces and lower cases the string,
     useful for comparing two text strings without punctuation
     """
-    return re.sub(r'[^\w]', ' ', x.lower())
+    return re.sub(r'\W', ' ', x.lower())
 
 
 def get_matching_line_offsets(lines, terms=None, markers=None):

rhodecode/tests/lib/test_helpers.py

@@ -28,14 +28,14 b' from rhodecode.tests import no_newline_i'
 
 
 @pytest.mark.parametrize('url, expected_url', [
-    ('http://rc.com', '<a href="http://rc.com">http://rc.com</a>'),
-    ('http://rc.com/test', '<a href="http://rc.com/test">http://rc.com/test</a>'),
-    ('http://rc.com/!foo', '<a href="http://rc.com/!foo">http://rc.com/!foo</a>'),
-    ('http://rc.com/&foo', '<a href="http://rc.com/&amp;foo">http://rc.com/&amp;foo</a>'),
-    ('http://rc.com/?foo-1&bar=1', '<a href="http://rc.com/?foo-1&amp;bar=1">http://rc.com/?foo-1&amp;bar=1</a>'),
-    ('http://rc.com?foo-1&bar=1', '<a href="http://rc.com?foo-1&amp;bar=1">http://rc.com?foo-1&amp;bar=1</a>'),
-    ('http://rc.com/#foo', '<a href="http://rc.com/#foo">http://rc.com/#foo</a>'),
-    ('http://rc.com/@foo', '<a href="http://rc.com/@foo">http://rc.com/@foo</a>'),
+    (r'https://rc.com', '<a href="https://rc.com">http://rc.com</a>'),
+    (r'https://rc.com/test', '<a href="https://rc.com/test">https://rc.com/test</a>'),
+    (r'https://rc.com/!foo', '<a href="https://rc.com/!foo">https://rc.com/!foo</a>'),
+    (r'https://rc.com/&foo', '<a href="https://rc.com/&amp;foo">https://rc.com/&amp;foo</a>'),
+    (r'https://rc.com/?foo-1&bar=1', '<a href="https://rc.com/?foo-1&amp;bar=1">https://rc.com/?foo-1&amp;bar=1</a>'),
+    (r'https://rc.com?foo-1&bar=1', '<a href="https://rc.com?foo-1&amp;bar=1">https://rc.com?foo-1&amp;bar=1</a>'),
+    (r'https://rc.com/#foo', '<a href="https://rc.com/#foo">https://rc.com/#foo</a>'),
+    (r'https://rc.com/@foo', '<a href="https://rc.com/@foo">https://rc.com/@foo</a>'),
 ])
 def test_urlify_text(url, expected_url):
     assert helpers.urlify_text(url) == expected_url
@@ -95,12 +95,12 b' def test_format_binary():'
 
 
 @pytest.mark.parametrize('text_string, pattern, expected', [
-    ('No issue here', '(?:#)(?P<issue_id>\d+)', []),
+    ('No issue here', r'(?:#)(?P<issue_id>\d+)', []),
     ('Fix #42', '(?:#)(?P<issue_id>\d+)',
-     [{'url': 'http://r.io/{repo}/i/42', 'id': '42'}]),
+     [{'url': 'https://r.io/{repo}/i/42', 'id': '42'}]),
     ('Fix #42, #53', '(?:#)(?P<issue_id>\d+)', [
-     {'url': 'http://r.io/{repo}/i/42', 'id': '42'},
-     {'url': 'http://r.io/{repo}/i/53', 'id': '53'}]),
+     {'url': 'https://r.io/{repo}/i/42', 'id': '42'},
+     {'url': 'https://r.io/{repo}/i/53', 'id': '53'}]),
     ('Fix #42', '(?:#)?<issue_id>\d+)', []),  # Broken regex
 ])
 def test_extract_issues(backend, text_string, pattern, expected):
@@ -109,7 +109,7 b' def test_extract_issues(backend, text_st'
         '123': {
             'uid': '123',
             'pat': pattern,
-            'url': 'http://r.io/${repo}/i/${issue_id}',
+            'url': r'https://r.io/${repo}/i/${issue_id}',
             'pref': '#',
             'desc': 'Test Pattern'
         }

rhodecode/tests/models/schemas/test_schema_types.py

@@ -36,29 +36,29 b' class TestGroupNameType(object):'
         assert result == expected
 
     @pytest.mark.parametrize('given, expected', [
-        ('//group1/group2//', 'group1/group2'),
-        ('//group1///group2//', 'group1/group2'),
-        ('group1/group2///group3', 'group1/group2/group3'),
-        ('v1.2', 'v1.2'),
-        ('/v1.2', 'v1.2'),
-        ('.dirs', '.dirs'),
-        ('..dirs', '.dirs'),
-        ('./..dirs', '.dirs'),
-        ('dir/;name;/;[];/sub', 'dir/name/sub'),
-        (',/,/,d,,,', 'd'),
-        ('/;/#/,d,,,', 'd'),
-        ('long../../..name', 'long./.name'),
-        ('long../..name', 'long./.name'),
-        ('../', ''),
-        ('\'../"../', ''),
-        ('c,/,/..//./,c,,,/.d/../.........c', 'c/c/.d/.c'),
-        ('c,/,/..//./,c,,,', 'c/c'),
-        ('d../..d', 'd./.d'),
-        ('d../../d', 'd./d'),
+        (r'//group1/group2//', 'group1/group2'),
+        (r'//group1///group2//', 'group1/group2'),
+        (r'group1/group2///group3', 'group1/group2/group3'),
+        (r'v1.2', 'v1.2'),
+        (r'/v1.2', 'v1.2'),
+        (r'.dirs', '.dirs'),
+        (r'..dirs', '.dirs'),
+        (r'./..dirs', '.dirs'),
+        (r'dir/;name;/;[];/sub', 'dir/name/sub'),
+        (r',/,/,d,,,', 'd'),
+        (r'/;/#/,d,,,', 'd'),
+        (r'long../../..name', 'long./.name'),
+        (r'long../..name', 'long./.name'),
+        (r'../', ''),
+        (r'\'../"../', ''),
+        (r'c,/,/..//./,c,,,/.d/../.........c', 'c/c/.d/.c'),
+        (r'c,/,/..//./,c,,,', 'c/c'),
+        (r'd../..d', 'd./.d'),
+        (r'd../../d', 'd./d'),
 
-        ('d\;\./\,\./d', 'd./d'),
-        ('d\.\./\.\./d', 'd./d'),
-        ('d\.\./\..\../d', 'd./d'),
+        (r'd\;\./\,\./d', 'd./d'),
+        (r'd\.\./\.\./d', 'd./d'),
+        (r'd\.\./\..\../d', 'd./d'),
     ])
     def test_deserialize_clean_up_name(self, given, expected):
         class TestSchema(colander.Schema):