issue-trackers: implemented more sophisticated ticket data extraction based on advanced regex module, and special

look-behind regex. This is a recommended ticket extraction from Atlassian itself (for Jira) and we also
implemented it for Redmine, or tickets

- bonus: report errors on patterns in preview window

pkgs/python-packages.nix

@@ -1816,6 +1816,17 b' self: super: {'
       license = [ pkgs.lib.licenses.mit ];
     };
   };
+  "regex" = super.buildPythonPackage {
+    name = "regex-2020.9.27";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/93/8c/17f45cdfb39b13d4b5f909e4b4c2917abcbdef9c0036919a0399769148cf/regex-2020.9.27.tar.gz";
+      sha256 = "179ngfzwbsjvn5vhyzdahvmg0f7acahkwwy9bpjy1pv08bm2mwx6";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.psfl ];
+    };
+  };
   "redis" = super.buildPythonPackage {
     name = "redis-3.4.1";
     doCheck = false;
@@ -1946,6 +1957,7 b' self: super: {'
       self."tzlocal"
       self."pyzmq"
       self."py-gfm"
+      self."regex"
       self."redis"
       self."repoze.lru"
       self."requests"

requirements.txt

@@ -56,6 +56,7 b' pytz==2019.3'
 tzlocal==1.5.1
 pyzmq==14.6.0
 py-gfm==0.1.4
+regex==2020.9.27
 redis==3.4.1
 repoze.lru==0.7
 requests==2.22.0

rhodecode/apps/admin/views/settings.py

@@ -474,9 +474,18 b' class AdminSettingsView(BaseAppView):'
         route_name='admin_settings_issuetracker_test', request_method='POST',
         renderer='string', xhr=True)
     def settings_issuetracker_test(self):
-        return h.urlify_commit_message(
+        error_container = []
+
+        urlified_commit = h.urlify_commit_message(
             self.request.POST.get('test_text', ''),
-            'repo_group/test_repo1')
+            'repo_group/test_repo1', error_container=error_container)
+        if error_container:
+            def converter(inp):
+                return h.html_escape(unicode(inp))
+
+            return 'ERRORS: ' + '\n'.join(map(converter, error_container))
+
+        return urlified_commit
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')

rhodecode/events/repo.py

@@ -113,7 +113,7 b' def _commits_as_dict(event, commit_ids, '
                 cs_data['permalink_url'] = RepoModel().get_commit_url(
                     repo, cs_data['raw_id'], request=event.request,
                     permalink=True)
-                urlified_message, issues_data = process_patterns(
+                urlified_message, issues_data, errors = process_patterns(
                     cs_data['message'], repo.repo_name)
                 cs_data['issues'] = issues_data
                 cs_data['message_html'] = urlify_commit_message(

rhodecode/lib/helpers.py

@@ -38,6 +38,7 b' import re'
 import time
 import string
 import hashlib
+import regex
 from collections import OrderedDict
 
 import pygments
@@ -1653,7 +1654,7 b' def get_active_pattern_entries(repo_name'
     return active_entries
 
 
-pr_pattern_re = re.compile(r'(?:(?:^!)|(?: !))(\d+)')
+pr_pattern_re = regex.compile(r'(?:(?:^!)|(?: !))(\d+)')
 
 allowed_link_formats = [
     'html', 'rst', 'markdown', 'html+hovercard', 'rst+hovercard', 'markdown+hovercard']
@@ -1670,6 +1671,7 b' def process_patterns(text_string, repo_n'
         active_entries = get_active_pattern_entries(repo_name)
 
     issues_data = []
+    errors = []
     new_text = text_string
 
     log.debug('Got %s entries to process', len(active_entries))
@@ -1687,9 +1689,11 b' def process_patterns(text_string, repo_n'
             pattern = entry['pat_compiled']
         else:
             try:
-                pattern = re.compile(r'%s' % entry['pat'])
-            except re.error:
-                log.exception('issue tracker pattern: `%s` failed to compile', entry['pat'])
+                pattern = regex.compile(r'%s' % entry['pat'])
+            except regex.error as e:
+                regex_err = ValueError('{}:{}'.format(entry['pat'], e))
+                log.exception('issue tracker pattern: `%s` failed to compile', regex_err)
+                errors.append(regex_err)
                 continue
 
         data_func = partial(
@@ -1721,11 +1725,11 b' def process_patterns(text_string, repo_n'
     new_text = pr_pattern_re.sub(pr_url_func, new_text)
     log.debug('processed !pr pattern')
 
-    return new_text, issues_data
+    return new_text, issues_data, errors
 
 
 def urlify_commit_message(commit_text, repository=None, active_pattern_entries=None,
-                          issues_container=None):
+                          issues_container=None, error_container=None):
     """
     Parses given text message and makes proper links.
     issues are linked to given issue-server, and rest is a commit link
@@ -1745,12 +1749,15 b' def urlify_commit_message(commit_text, r'
         new_text = urlify_commits(new_text, repository)
 
     # process issue tracker patterns
-    new_text, issues = process_patterns(new_text, repository or '',
-                                        active_entries=active_pattern_entries)
+    new_text, issues, errors = process_patterns(
+        new_text, repository or '', active_entries=active_pattern_entries)
 
     if issues_container is not None:
         issues_container.extend(issues)
 
+    if error_container is not None:
+        error_container.extend(errors)
+
     return literal(new_text)
 
 
@@ -1805,7 +1812,7 b" def render(source, renderer='rst', menti"
     elif renderer == 'rst':
         if repo_name:
             # process patterns on comments if we pass in repo name
-            source, issues = process_patterns(
+            source, issues, errors = process_patterns(
                 source, repo_name, link_format='rst',
                 active_entries=active_pattern_entries)
             if issues_container is not None:
@@ -1819,7 +1826,7 b" def render(source, renderer='rst', menti"
     elif renderer == 'markdown':
         if repo_name:
             # process patterns on comments if we pass in repo name
-            source, issues = process_patterns(
+            source, issues, errors = process_patterns(
                 source, repo_name, link_format='markdown',
                 active_entries=active_pattern_entries)
             if issues_container is not None:

rhodecode/templates/base/issue_tracker_settings.mako

@@ -16,8 +16,8 b' examples = ['
     ),
 
     (
-    'Redmine',
-    '(^#|\s#)(?P<issue_id>\d+)',
+    'Tickets with #123 (Redmine etc)',
+    '(?<![a-zA-Z0-9_/]{1,10}-?)(#)(?P<issue_id>\d+)',
     'https://myissueserver.com/${repo}/issue/${issue_id}',
     ''
     ),
@@ -38,14 +38,15 b' examples = ['
 
     (
     'JIRA - All tickets',
-    '(^|\s\w+-\d+)',
-    'https://myjira.com/browse/${id}',
+    # official JIRA ticket pattern
+    '(?<![a-zA-Z0-9_/#]-?)(?P<issue_id>[A-Z]{1,6}-(?:[1-9][0-9]{0,7}))',
+    'https://myjira.com/browse/${issue_id}',
     ''
     ),
 
     (
-    'JIRA - Project (JRA)',
-    '(?:(^|\s)(?P<issue_id>(?:JRA-|JRA-)(?:\d+)))',
+    'JIRA - Single project (JRA-XXXXXXXX)',
+    '(?<![a-zA-Z0-9_/#]-?)(?P<issue_id>JRA-(?:[1-9][0-9]{0,7}))',
     'https://myjira.com/${issue_id}',
     ''
     ),
@@ -275,12 +276,18 b' examples = ['
                 <div class='textarea-full'>
                     <textarea id="test_pattern_data" rows="12">
 This is an example text for testing issue tracker patterns.
-This commit fixes ticket #451 and ticket #910.
-Following tickets will get mentioned:
+This commit fixes ticket #451 and ticket #910, reference for JRA-401.
+The following tickets will get mentioned:
 #123
-#456
-JRA-123
-JRA-456
+#456 and PROJ-101
+JRA-123 and #123
+PROJ-456
+
+[my artifact](http://something.com/JRA-1234-build.zip)
+
+- #1001
+- JRA-998
+
 Open a pull request !101 to contribute !
 Added tag v1.3.0 for commit 0f3b629be725
 

rhodecode/tests/lib/test_helpers.py

@@ -121,7 +121,7 b' def test_extract_issues(backend, text_st'
 
     with mock.patch.object(IssueTrackerSettingsModel,
                            'get_settings', get_settings_mock):
-        text, issues = helpers.process_patterns(text_string, repo.repo_name)
+        text, issues, errors = helpers.process_patterns(text_string, repo.repo_name)
 
     expected = copy.deepcopy(expected)
     for item in expected:
@@ -159,7 +159,7 b' def test_process_patterns_repo(backend, '
 
     with mock.patch.object(IssueTrackerSettingsModel,
                            'get_settings', get_settings_mock):
-        processed_text, issues = helpers.process_patterns(
+        processed_text, issues, error = helpers.process_patterns(
             text_string, repo.repo_name, link_format)
 
     assert processed_text == expected_text.format(repo=repo.repo_name)
@@ -186,7 +186,7 b' def test_process_patterns_no_repo(text_s'
 
     with mock.patch.object(IssueTrackerSettingsModel,
                            'get_global_settings', get_settings_mock):
-        processed_text, issues = helpers.process_patterns(
+        processed_text, issues, errors = helpers.process_patterns(
             text_string, '')
 
     assert processed_text == expected_text
@@ -211,7 +211,7 b' def test_process_patterns_non_existent_r'
 
     with mock.patch.object(IssueTrackerSettingsModel,
                            'get_global_settings', get_settings_mock):
-        processed_text, issues = helpers.process_patterns(
+        processed_text, issues, errors = helpers.process_patterns(
             text_string, 'do-not-exist')
 
     assert processed_text == expected_text