webhooks: added variables into the call URL. Fixes #4211

rhodecode/tests/integrations/conftest.py

@@ -0,0 +1,52 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+
+import pytest
+from rhodecode import events
+
+
[e-mail]
+def repo_push_event(backend, user_regular):
+    commits = [
+        {'message': 'ancestor commit fixes #15'},
+        {'message': 'quick fixes'},
+        {'message': 'change that fixes #41, #2'},
+        {'message': 'this is because 5b23c3532 broke stuff'},
+        {'message': 'last commit'},
+    ]
+    commit_ids = backend.create_master_repo(commits).values()
+    repo = backend.create_repo()
+    scm_extras = {
+        'ip': '127.0.0.1',
+        'username': user_regular.username,
+        'action': '',
+        'repository': repo.repo_name,
+        'scm': repo.scm_instance().alias,
+        'config': '',
+        'server_url': 'http://example.com',
+        'make_lock': None,
+        'locked_by': [None],
+        'commit_ids': commit_ids,
+    }
+
+    return events.RepoPushEvent(repo_name=repo.repo_name,
+                                pushed_commit_ids=commit_ids,
+                                extras=scm_extras)

rhodecode/tests/integrations/test_webhook.py

@@ -0,0 +1,93 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import pytest
+
+from rhodecode import events
+from rhodecode.lib.utils2 import AttributeDict
+from rhodecode.integrations.types.webhook import WebhookHandler
+
+
[e-mail]
+def base_data():
+    return {
+        'repo': {
+            'repo_name': 'foo',
+            'repo_type': 'hg',
+            'repo_id': '12',
+            'url': 'http://repo.url/foo'
+        }
+    }
+
+
+def test_webhook_parse_url_invalid_event():
+    template_url = 'http://server.com/${repo_name}/build'
+    handler = WebhookHandler(template_url, 'secret_token')
+    with pytest.raises(ValueError) as err:
+        handler(events.RepoDeleteEvent(''), {})
+    assert str(err.value).startswith('event type not supported')
+
+
[e-mail]('template,expected_urls', [
+    ('http://server.com/${repo_name}/build', ['http://server.com/foo/build']),
+    ('http://server.com/${repo_name}/${repo_type}', ['http://server.com/foo/hg']),
+    ('http://${server}.com/${repo_name}/${repo_id}', ['http://${server}.com/foo/12']),
+    ('http://server.com/${branch}/build', ['http://server.com/${branch}/build']),
+])
+def test_webook_parse_url_for_create_event(base_data, template, expected_urls):
+    handler = WebhookHandler(template, 'secret_token')
+    urls = handler(events.RepoCreateEvent(''), base_data)
+    assert urls == [(url, 'secret_token', base_data) for url in expected_urls]
+
+
[e-mail]('template,expected_urls', [
+    ('http://server.com/${repo_name}/${pull_request_id}', ['http://server.com/foo/999']),
+    ('http://server.com/${repo_name}/${pull_request_url}', ['http://server.com/foo/http://pr-url.com']),
+])
+def test_webook_parse_url_for_pull_request_event(base_data, template, expected_urls):
+    base_data['pullrequest'] = {
+        'pull_request_id': 999,
+        'url': 'http://pr-url.com',
+    }
+    handler = WebhookHandler(template, 'secret_token')
+    urls = handler(events.PullRequestCreateEvent(
+        AttributeDict({'target_repo': 'foo'})), base_data)
+    assert urls == [(url, 'secret_token', base_data) for url in expected_urls]
+
+
[e-mail]('template,expected_urls', [
+    ('http://server.com/${branch}/build', ['http://server.com/stable/build',
+                                           'http://server.com/dev/build']),
+    ('http://server.com/${branch}/${commit_id}', ['http://server.com/stable/stable-xxx',
+                                                  'http://server.com/stable/stable-yyy',
+                                                  'http://server.com/dev/dev-xxx',
+                                                  'http://server.com/dev/dev-yyy']),
+])
+def test_webook_parse_url_for_push_event(pylonsapp, repo_push_event, base_data, template, expected_urls):
+    base_data['push'] = {
+        'branches': [{'name': 'stable'}, {'name': 'dev'}],
+        'commits': [{'branch': 'stable', 'raw_id': 'stable-xxx'},
+                    {'branch': 'stable', 'raw_id': 'stable-yyy'},
+                    {'branch': 'dev', 'raw_id': 'dev-xxx'},
+                    {'branch': 'dev', 'raw_id': 'dev-yyy'}]
+    }
+    handler = WebhookHandler(template, 'secret_token')
+    urls = handler(repo_push_event, base_data)
+    assert urls == [(url, 'secret_token', base_data) for url in expected_urls]

rhodecode/integrations/types/webhook.py

@@ -19,13 +19,15 b''
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 from __future__ import unicode_literals
+import string
+from collections import OrderedDict
 
 import deform
 import logging
 import requests
 import colander
 from celery.task import task
-from mako.template import Template
+from requests.packages.urllib3.util.retry import Retry
 
 from rhodecode import events
 from rhodecode.translation import _
@@ -33,12 +35,127 b' from rhodecode.integrations.types.base i'
 
 log = logging.getLogger(__name__)
 
+# updating this required to update the `base_vars` passed in url calling func
+WEBHOOK_URL_VARS = [
+    'repo_name',
+    'repo_type',
+    'repo_id',
+    'repo_url',
+
+    # special attrs below that we handle, using multi-call
+    'branch',
+    'commit_id',
+
+    # pr events vars
+    'pull_request_id',
+    'pull_request_url',
+
+]
+URL_VARS = ', '.join('${' + x + '}' for x in WEBHOOK_URL_VARS)
+
+
+class WebhookHandler(object):
+    def __init__(self, template_url, secret_token):
+        self.template_url = template_url
+        self.secret_token = secret_token
+
+    def get_base_parsed_template(self, data):
+        """
+        initially parses the passed in template with some common variables
+        available on ALL calls
+        """
+        # note: make sure to update the `WEBHOOK_URL_VARS` if this changes
+        common_vars = {
+            'repo_name': data['repo']['repo_name'],
+            'repo_type': data['repo']['repo_type'],
+            'repo_id': data['repo']['repo_id'],
+            'repo_url': data['repo']['url'],
+        }
+
+        return string.Template(
+            self.template_url).safe_substitute(**common_vars)
+
+    def repo_push_event_handler(self, event, data):
+        url = self.get_base_parsed_template(data)
+        url_cals = []
+        branch_data = OrderedDict()
+        for obj in data['push']['branches']:
+            branch_data[obj['name']] = obj
+
+        branches_commits = OrderedDict()
+        for commit in data['push']['commits']:
+            if commit['branch'] not in branches_commits:
+                branch_commits = {'branch': branch_data[commit['branch']],
+                                  'commits': []}
+                branches_commits[commit['branch']] = branch_commits
+
+            branch_commits = branches_commits[commit['branch']]
+            branch_commits['commits'].append(commit)
+
+        if '${branch}' in url:
+            # call it multiple times, for each branch if used in variables
+            for branch, commit_ids in branches_commits.items():
+                branch_url = string.Template(url).safe_substitute(branch=branch)
+                # call further down for each commit if used
+                if '${commit_id}' in branch_url:
+                    for commit_data in commit_ids['commits']:
+                        commit_id = commit_data['raw_id']
+                        commit_url = string.Template(branch_url).safe_substitute(
+                            commit_id=commit_id)
+                        # register per-commit call
+                        log.debug(
+                            'register webhook call(%s) to url %s', event, commit_url)
+                        url_cals.append((commit_url, self.secret_token, data))
+
+                else:
+                    # register per-branch call
+                    log.debug(
+                        'register webhook call(%s) to url %s', event, branch_url)
+                    url_cals.append((branch_url, self.secret_token, data))
+
+        else:
+            log.debug(
+                'register webhook call(%s) to url %s', event, url)
+            url_cals.append((url, self.secret_token, data))
+
+        return url_cals
+
+    def repo_create_event_handler(self, event, data):
+        url = self.get_base_parsed_template(data)
+        log.debug(
+            'register webhook call(%s) to url %s', event, url)
+        return [(url, self.secret_token, data)]
+
+    def pull_request_event_handler(self, event, data):
+        url = self.get_base_parsed_template(data)
+        log.debug(
+            'register webhook call(%s) to url %s', event, url)
+        url = string.Template(url).safe_substitute(
+            pull_request_id=data['pullrequest']['pull_request_id'],
+            pull_request_url=data['pullrequest']['url'])
+        return [(url, self.secret_token, data)]
+
+    def __call__(self, event, data):
+        if isinstance(event, events.RepoPushEvent):
+            return self.repo_push_event_handler(event, data)
+        elif isinstance(event, events.RepoCreateEvent):
+            return self.repo_create_event_handler(event, data)
+        elif isinstance(event, events.PullRequestEvent):
+            return self.pull_request_event_handler(event, data)
+        else:
+            raise ValueError('event type not supported: %s' % events)
+
 
 class WebhookSettingsSchema(colander.Schema):
     url = colander.SchemaNode(
         colander.String(),
         title=_('Webhook URL'),
-        description=_('URL of the webhook to receive POST event.'),
+        description=
+            _('URL of the webhook to receive POST event. Following variables '
+              'are allowed to be used: {vars}. Some of the variables would '
+              'trigger multiple calls, like ${{branch}} or ${{commit_id}}. '
+              'Webhook will be called as many times as unique objects in '
+              'data in such cases.').format(vars=URL_VARS),
         missing=colander.required,
         required=True,
         validator=colander.url,
@@ -58,8 +175,6 b' class WebhookSettingsSchema(colander.Sch'
     )
 
 
-
-
 class WebhookIntegrationType(IntegrationTypeBase):
     key = 'webhook'
     display_name = _('Webhook')
@@ -104,14 +219,30 b' class WebhookIntegrationType(Integration'
             return
 
         data = event.as_dict()
-        post_to_webhook(data, self.settings)
+        template_url = self.settings['url']
+
+        handler = WebhookHandler(template_url, self.settings['secret_token'])
+        url_calls = handler(event, data)
+        log.debug('webhook: calling following urls: %s',
+                  [x[0] for x in url_calls])
+        post_to_webhook(url_calls)
 
 
 @task(ignore_result=True)
-def post_to_webhook(data, settings):
-    log.debug('sending event:%s to webhook %s', data['name'], settings['url'])
-    resp = requests.post(settings['url'], json={
-        'token': settings['secret_token'],
-        'event': data
-    })
-    resp.raise_for_status()  # raise exception on a failed request
+def post_to_webhook(url_calls):
+    max_retries = 3
+    for url, token, data in url_calls:
+        # retry max N times
+        retries = Retry(
+            total=max_retries,
+            backoff_factor=0.15,
+            status_forcelist=[500, 502, 503, 504])
+        req_session = requests.Session()
+        req_session.mount(
+            'http://', requests.adapters.HTTPAdapter(max_retries=retries))
+
+        resp = req_session.post(url, json={
+            'token': token,
+            'event': data
+        })
+        resp.raise_for_status()  # raise exception on a failed request

rhodecode/tests/integrations/test_slack.py

@@ -19,41 +19,12 b''
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import pytest
-import requests
-from mock import Mock, patch
+from mock import patch
 
 from rhodecode import events
 from rhodecode.model.db import Session, Integration
 from rhodecode.integrations.types.slack import SlackIntegrationType
 
[e-mail]
-def repo_push_event(backend, user_regular):
-    commits = [
-        {'message': 'ancestor commit fixes #15'},
-        {'message': 'quick fixes'},
-        {'message': 'change that fixes #41, #2'},
-        {'message': 'this is because 5b23c3532 broke stuff'},
-        {'message': 'last commit'},
-    ]
-    commit_ids = backend.create_master_repo(commits).values()
-    repo = backend.create_repo()
-    scm_extras = {
-        'ip': '127.0.0.1',
-        'username': user_regular.username,
-        'action': '',
-        'repository': repo.repo_name,
-        'scm': repo.scm_instance().alias,
-        'config': '',
-        'server_url': 'http://example.com',
-        'make_lock': None,
-        'locked_by': [None],
-        'commit_ids': commit_ids,
-    }
-
-    return events.RepoPushEvent(repo_name=repo.repo_name,
-                                pushed_commit_ids=commit_ids,
-                                extras=scm_extras)
-
 
 @pytest.fixture
 def slack_settings():

rhodecode/translation.py

@@ -21,6 +21,7 b' from pyramid.i18n import TranslationStri'
 # Create a translation string factory for the 'rhodecode' domain.
 _ = TranslationStringFactory('rhodecode')
 
+
 class LazyString(object):
     def __init__(self, *args, **kw):
         self.args = args