webhook: abstract webhook handler to base for easier re-usage.

rhodecode/integrations/types/base.py

@@ -19,8 +19,13 b''
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import colander
+import string
+import collections
+import logging
 from rhodecode.translation import _
 
+log = logging.getLogger(__name__)
+
 
 class IntegrationTypeBase(object):
     """ Base class for IntegrationType plugins """
@@ -142,6 +147,122 b' WEBHOOK_URL_VARS = ['
 CI_URL_VARS = WEBHOOK_URL_VARS
 
 
+class WebhookDataHandler(object):
+    name = 'webhook'
+
+    def __init__(self, template_url, headers):
+        self.template_url = template_url
+        self.headers = headers
+
+    def get_base_parsed_template(self, data):
+        """
+        initially parses the passed in template with some common variables
+        available on ALL calls
+        """
+        # note: make sure to update the `WEBHOOK_URL_VARS` if this changes
+        common_vars = {
+            'repo_name': data['repo']['repo_name'],
+            'repo_type': data['repo']['repo_type'],
+            'repo_id': data['repo']['repo_id'],
+            'repo_url': data['repo']['url'],
+            'username': data['actor']['username'],
+            'user_id': data['actor']['user_id'],
+            'event_name': data['name']
+        }
+
+        extra_vars = {}
+        for extra_key, extra_val in data['repo']['extra_fields'].items():
+            extra_vars['extra__{}'.format(extra_key)] = extra_val
+        common_vars.update(extra_vars)
+
+        template_url = self.template_url.replace('${extra:', '${extra__')
+        return string.Template(template_url).safe_substitute(**common_vars)
+
+    def repo_push_event_handler(self, event, data):
+        url = self.get_base_parsed_template(data)
+        url_cals = []
+        branch_data = collections.OrderedDict()
+        for obj in data['push']['branches']:
+            branch_data[obj['name']] = obj
+
+        branches_commits = collections.OrderedDict()
+        for commit in data['push']['commits']:
+            if commit.get('git_ref_change'):
+                # special case for GIT that allows creating tags,
+                # deleting branches without associated commit
+                continue
+
+            if commit['branch'] not in branches_commits:
+                branch_commits = {'branch': branch_data[commit['branch']],
+                                  'commits': []}
+                branches_commits[commit['branch']] = branch_commits
+
+            branch_commits = branches_commits[commit['branch']]
+            branch_commits['commits'].append(commit)
+
+        if '${branch}' in url:
+            # call it multiple times, for each branch if used in variables
+            for branch, commit_ids in branches_commits.items():
+                branch_url = string.Template(url).safe_substitute(branch=branch)
+                # call further down for each commit if used
+                if '${commit_id}' in branch_url:
+                    for commit_data in commit_ids['commits']:
+                        commit_id = commit_data['raw_id']
+                        commit_url = string.Template(branch_url).safe_substitute(
+                            commit_id=commit_id)
+                        # register per-commit call
+                        log.debug(
+                            'register %s call(%s) to url %s',
+                            self.name, event, commit_url)
+                        url_cals.append(
+                            (commit_url, self.headers, data))
+
+                else:
+                    # register per-branch call
+                    log.debug(
+                        'register %s call(%s) to url %s',
+                        self.name, event, branch_url)
+                    url_cals.append(
+                        (branch_url, self.headers, data))
+
+        else:
+            log.debug(
+                'register %s call(%s) to url %s', self.name, event, url)
+            url_cals.append((url, self.headers, data))
+
+        return url_cals
+
+    def repo_create_event_handler(self, event, data):
+        url = self.get_base_parsed_template(data)
+        log.debug(
+            'register %s call(%s) to url %s', self.name, event, url)
+        return [(url, self.headers, data)]
+
+    def pull_request_event_handler(self, event, data):
+        url = self.get_base_parsed_template(data)
+        log.debug(
+            'register %s call(%s) to url %s', self.name, event, url)
+        url = string.Template(url).safe_substitute(
+            pull_request_id=data['pullrequest']['pull_request_id'],
+            pull_request_url=data['pullrequest']['url'],
+            pull_request_shadow_url=data['pullrequest']['shadow_url'],)
+        return [(url, self.headers, data)]
+
+    def __call__(self, event, data):
+        from rhodecode import events
+
+        if isinstance(event, events.RepoPushEvent):
+            return self.repo_push_event_handler(event, data)
+        elif isinstance(event, events.RepoCreateEvent):
+            return self.repo_create_event_handler(event, data)
+        elif isinstance(event, events.PullRequestEvent):
+            return self.pull_request_event_handler(event, data)
+        else:
+            raise ValueError(
+                'event type `%s` not in supported list: %s' % (
+                    event.__class__, events))
+
+
 def get_auth(settings):
     from requests.auth import HTTPBasicAuth
     username = settings.get('username')
@@ -151,6 +272,10 b' def get_auth(settings):'
     return None
 
 
+def get_web_token(settings):
+    return settings['secret_token']
+
+
 def get_url_vars(url_vars):
     return '\n'.join(
         '{} - {}'.format('${' + key + '}', explanation)

rhodecode/integrations/types/webhook.py

@@ -19,8 +19,6 b''
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 from __future__ import unicode_literals
-import string
-import collections
 
 import deform
 import deform.widget
@@ -34,7 +32,8 b' import rhodecode'
 from rhodecode import events
 from rhodecode.translation import _
 from rhodecode.integrations.types.base import (
-    IntegrationTypeBase, get_auth, get_url_vars, WEBHOOK_URL_VARS)
+    IntegrationTypeBase, get_auth, get_web_token, get_url_vars,
+    WebhookDataHandler, WEBHOOK_URL_VARS)
 from rhodecode.lib.celerylib import run_task, async_task, RequestContextTask
 from rhodecode.model.validation_schema import widgets
 
@@ -46,113 +45,6 b' log = logging.getLogger(__name__)'
 URL_VARS = get_url_vars(WEBHOOK_URL_VARS)
 
 
-class WebhookHandler(object):
-    def __init__(self, template_url, secret_token, headers):
-        self.template_url = template_url
-        self.secret_token = secret_token
-        self.headers = headers
-
-    def get_base_parsed_template(self, data):
-        """
-        initially parses the passed in template with some common variables
-        available on ALL calls
-        """
-        # note: make sure to update the `WEBHOOK_URL_VARS` if this changes
-        common_vars = {
-            'repo_name': data['repo']['repo_name'],
-            'repo_type': data['repo']['repo_type'],
-            'repo_id': data['repo']['repo_id'],
-            'repo_url': data['repo']['url'],
-            'username': data['actor']['username'],
-            'user_id': data['actor']['user_id'],
-            'event_name': data['name']
-        }
-
-        extra_vars = {}
-        for extra_key, extra_val in data['repo']['extra_fields'].items():
-            extra_vars['extra__{}'.format(extra_key)] = extra_val
-        common_vars.update(extra_vars)
-
-        template_url = self.template_url.replace('${extra:', '${extra__')
-        return string.Template(template_url).safe_substitute(**common_vars)
-
-    def repo_push_event_handler(self, event, data):
-        url = self.get_base_parsed_template(data)
-        url_cals = []
-        branch_data = collections.OrderedDict()
-        for obj in data['push']['branches']:
-            branch_data[obj['name']] = obj
-
-        branches_commits = collections.OrderedDict()
-        for commit in data['push']['commits']:
-            if commit.get('git_ref_change'):
-                # special case for GIT that allows creating tags,
-                # deleting branches without associated commit
-                continue
-
-            if commit['branch'] not in branches_commits:
-                branch_commits = {'branch': branch_data[commit['branch']],
-                                  'commits': []}
-                branches_commits[commit['branch']] = branch_commits
-
-            branch_commits = branches_commits[commit['branch']]
-            branch_commits['commits'].append(commit)
-
-        if '${branch}' in url:
-            # call it multiple times, for each branch if used in variables
-            for branch, commit_ids in branches_commits.items():
-                branch_url = string.Template(url).safe_substitute(branch=branch)
-                # call further down for each commit if used
-                if '${commit_id}' in branch_url:
-                    for commit_data in commit_ids['commits']:
-                        commit_id = commit_data['raw_id']
-                        commit_url = string.Template(branch_url).safe_substitute(
-                            commit_id=commit_id)
-                        # register per-commit call
-                        log.debug(
-                            'register webhook call(%s) to url %s', event, commit_url)
-                        url_cals.append((commit_url, self.secret_token, self.headers, data))
-
-                else:
-                    # register per-branch call
-                    log.debug(
-                        'register webhook call(%s) to url %s', event, branch_url)
-                    url_cals.append((branch_url, self.secret_token, self.headers, data))
-
-        else:
-            log.debug(
-                'register webhook call(%s) to url %s', event, url)
-            url_cals.append((url, self.secret_token, self.headers, data))
-
-        return url_cals
-
-    def repo_create_event_handler(self, event, data):
-        url = self.get_base_parsed_template(data)
-        log.debug(
-            'register webhook call(%s) to url %s', event, url)
-        return [(url, self.secret_token, self.headers, data)]
-
-    def pull_request_event_handler(self, event, data):
-        url = self.get_base_parsed_template(data)
-        log.debug(
-            'register webhook call(%s) to url %s', event, url)
-        url = string.Template(url).safe_substitute(
-            pull_request_id=data['pullrequest']['pull_request_id'],
-            pull_request_url=data['pullrequest']['url'],
-            pull_request_shadow_url=data['pullrequest']['shadow_url'],)
-        return [(url, self.secret_token, self.headers, data)]
-
-    def __call__(self, event, data):
-        if isinstance(event, events.RepoPushEvent):
-            return self.repo_push_event_handler(event, data)
-        elif isinstance(event, events.RepoCreateEvent):
-            return self.repo_create_event_handler(event, data)
-        elif isinstance(event, events.PullRequestEvent):
-            return self.pull_request_event_handler(event, data)
-        else:
-            raise ValueError('event type not supported: %s' % events)
-
-
 class WebhookSettingsSchema(colander.Schema):
     url = colander.SchemaNode(
         colander.String(),
@@ -243,7 +135,7 b' class WebhookSettingsSchema(colander.Sch'
 class WebhookIntegrationType(IntegrationTypeBase):
     key = 'webhook'
     display_name = _('Webhook')
-    description = _('Post json events to a Webhook endpoint')
+    description = _('send JSON data to a url endpoint')
 
     @classmethod
     def icon(cls):
@@ -275,8 +167,8 b' class WebhookIntegrationType(Integration'
         return schema
 
     def send_event(self, event):
-        log.debug('handling event %s with Webhook integration %s',
-            event.name, self)
+        log.debug(
+            'handling event %s with Webhook integration %s', event.name, self)
 
         if event.__class__ not in self.valid_events:
             log.debug('event not valid: %r' % event)
@@ -295,8 +187,7 b' class WebhookIntegrationType(Integration'
         if head_key and head_val:
             headers = {head_key: head_val}
 
-        handler = WebhookHandler(
-            template_url, self.settings['secret_token'], headers)
+        handler = WebhookDataHandler(template_url, headers)
 
         url_calls = handler(event, data)
         log.debug('webhook: calling following urls: %s',
@@ -353,7 +244,9 b' def post_to_webhook(url_calls, settings)'
     }  # updated below with custom ones, allows override
 
     auth = get_auth(settings)
-    for url, token, headers, data in url_calls:
+    token = get_web_token(settings)
+
+    for url, headers, data in url_calls:
         req_session = requests.Session()
         req_session.mount(  # retry max N times
             'http://', requests.adapters.HTTPAdapter(max_retries=retries))

rhodecode/tests/integrations/test_webhook.py

@@ -22,12 +22,13 b' import pytest'
 
 from rhodecode import events
 from rhodecode.lib.utils2 import AttributeDict
-from rhodecode.integrations.types.webhook import WebhookHandler
+from rhodecode.integrations.types.webhook import WebhookDataHandler
 
 
 @pytest.fixture
 def base_data():
     return {
+        'name': 'event',
         'repo': {
             'repo_name': 'foo',
             'repo_type': 'hg',
@@ -44,31 +45,40 b' def base_data():'
 
 def test_webhook_parse_url_invalid_event():
     template_url = 'http://server.com/${repo_name}/build'
-    handler = WebhookHandler(
-        template_url, 'secret_token', {'exmaple-header':'header-values'})
+    handler = WebhookDataHandler(
+        template_url, {'exmaple-header': 'header-values'})
+    event = events.RepoDeleteEvent('')
     with pytest.raises(ValueError) as err:
-        handler(events.RepoDeleteEvent(''), {})
-    assert str(err.value).startswith('event type not supported')
+        handler(event, {})
+
+    err = str(err.value)
+    assert err.startswith(
+        'event type `%s` not in supported list' % event.__class__)
 
 
 @pytest.mark.parametrize('template,expected_urls', [
-    ('http://server.com/${repo_name}/build', ['http://server.com/foo/build']),
-    ('http://server.com/${repo_name}/${repo_type}', ['http://server.com/foo/hg']),
-    ('http://${server}.com/${repo_name}/${repo_id}', ['http://${server}.com/foo/12']),
-    ('http://server.com/${branch}/build', ['http://server.com/${branch}/build']),
+    ('http://server.com/${repo_name}/build',
+     ['http://server.com/foo/build']),
+    ('http://server.com/${repo_name}/${repo_type}',
+     ['http://server.com/foo/hg']),
+    ('http://${server}.com/${repo_name}/${repo_id}',
+     ['http://${server}.com/foo/12']),
+    ('http://server.com/${branch}/build',
+     ['http://server.com/${branch}/build']),
 ])
 def test_webook_parse_url_for_create_event(base_data, template, expected_urls):
     headers = {'exmaple-header': 'header-values'}
-    handler = WebhookHandler(
-        template, 'secret_token', headers)
+    handler = WebhookDataHandler(template, headers)
     urls = handler(events.RepoCreateEvent(''), base_data)
     assert urls == [
-        (url, 'secret_token', headers, base_data) for url in expected_urls]
+        (url, headers, base_data) for url in expected_urls]
 
 
 @pytest.mark.parametrize('template,expected_urls', [
-    ('http://server.com/${repo_name}/${pull_request_id}', ['http://server.com/foo/999']),
-    ('http://server.com/${repo_name}/${pull_request_url}', ['http://server.com/foo/http://pr-url.com']),
+    ('http://server.com/${repo_name}/${pull_request_id}',
+     ['http://server.com/foo/999']),
+    ('http://server.com/${repo_name}/${pull_request_url}',
+     ['http://server.com/foo/http://pr-url.com']),
 ])
 def test_webook_parse_url_for_pull_request_event(
         base_data, template, expected_urls):
@@ -76,23 +86,25 b' def test_webook_parse_url_for_pull_reque'
     base_data['pullrequest'] = {
         'pull_request_id': 999,
         'url': 'http://pr-url.com',
+        'shadow_url': 'http://pr-url.com/repository'
     }
     headers = {'exmaple-header': 'header-values'}
-    handler = WebhookHandler(
-        template, 'secret_token', headers)
+    handler = WebhookDataHandler(template, headers)
     urls = handler(events.PullRequestCreateEvent(
         AttributeDict({'target_repo': 'foo'})), base_data)
     assert urls == [
-        (url, 'secret_token', headers, base_data) for url in expected_urls]
+        (url, headers, base_data) for url in expected_urls]
 
 
 @pytest.mark.parametrize('template,expected_urls', [
-    ('http://server.com/${branch}/build', ['http://server.com/stable/build',
-                                           'http://server.com/dev/build']),
-    ('http://server.com/${branch}/${commit_id}', ['http://server.com/stable/stable-xxx',
-                                                  'http://server.com/stable/stable-yyy',
-                                                  'http://server.com/dev/dev-xxx',
-                                                  'http://server.com/dev/dev-yyy']),
+    ('http://server.com/${branch}/build',
+        ['http://server.com/stable/build',
+         'http://server.com/dev/build']),
+    ('http://server.com/${branch}/${commit_id}',
+        ['http://server.com/stable/stable-xxx',
+         'http://server.com/stable/stable-yyy',
+         'http://server.com/dev/dev-xxx',
+         'http://server.com/dev/dev-yyy']),
 ])
 def test_webook_parse_url_for_push_event(
         baseapp, repo_push_event, base_data, template, expected_urls):
@@ -104,8 +116,7 b' def test_webook_parse_url_for_push_event'
                     {'branch': 'dev', 'raw_id': 'dev-yyy'}]
     }
     headers = {'exmaple-header': 'header-values'}
-    handler = WebhookHandler(
-        template, 'secret_token', headers)
+    handler = WebhookDataHandler(template, headers)
     urls = handler(repo_push_event, base_data)
     assert urls == [
-        (url, 'secret_token', headers, base_data) for url in expected_urls]
+        (url, headers, base_data) for url in expected_urls]