diff --git a/rhodecode/integrations/types/webhook.py b/rhodecode/integrations/types/webhook.py
--- a/rhodecode/integrations/types/webhook.py
+++ b/rhodecode/integrations/types/webhook.py
@@ -19,13 +19,15 @@
# and proprietary license terms, please see https://rhodecode.com/licenses/
from __future__ import unicode_literals
+import string
+from collections import OrderedDict
import deform
import logging
import requests
import colander
from celery.task import task
-from mako.template import Template
+from requests.packages.urllib3.util.retry import Retry
from rhodecode import events
from rhodecode.translation import _
@@ -33,12 +35,127 @@ from rhodecode.integrations.types.base i
log = logging.getLogger(__name__)
+# updating this required to update the `base_vars` passed in url calling func
+WEBHOOK_URL_VARS = [
+ 'repo_name',
+ 'repo_type',
+ 'repo_id',
+ 'repo_url',
+
+ # special attrs below that we handle, using multi-call
+ 'branch',
+ 'commit_id',
+
+ # pr events vars
+ 'pull_request_id',
+ 'pull_request_url',
+
+]
+URL_VARS = ', '.join('${' + x + '}' for x in WEBHOOK_URL_VARS)
+
+
+class WebhookHandler(object):
+ def __init__(self, template_url, secret_token):
+ self.template_url = template_url
+ self.secret_token = secret_token
+
+ def get_base_parsed_template(self, data):
+ """
+ initially parses the passed in template with some common variables
+ available on ALL calls
+ """
+ # note: make sure to update the `WEBHOOK_URL_VARS` if this changes
+ common_vars = {
+ 'repo_name': data['repo']['repo_name'],
+ 'repo_type': data['repo']['repo_type'],
+ 'repo_id': data['repo']['repo_id'],
+ 'repo_url': data['repo']['url'],
+ }
+
+ return string.Template(
+ self.template_url).safe_substitute(**common_vars)
+
+ def repo_push_event_handler(self, event, data):
+ url = self.get_base_parsed_template(data)
+ url_cals = []
+ branch_data = OrderedDict()
+ for obj in data['push']['branches']:
+ branch_data[obj['name']] = obj
+
+ branches_commits = OrderedDict()
+ for commit in data['push']['commits']:
+ if commit['branch'] not in branches_commits:
+ branch_commits = {'branch': branch_data[commit['branch']],
+ 'commits': []}
+ branches_commits[commit['branch']] = branch_commits
+
+ branch_commits = branches_commits[commit['branch']]
+ branch_commits['commits'].append(commit)
+
+ if '${branch}' in url:
+ # call it multiple times, for each branch if used in variables
+ for branch, commit_ids in branches_commits.items():
+ branch_url = string.Template(url).safe_substitute(branch=branch)
+ # call further down for each commit if used
+ if '${commit_id}' in branch_url:
+ for commit_data in commit_ids['commits']:
+ commit_id = commit_data['raw_id']
+ commit_url = string.Template(branch_url).safe_substitute(
+ commit_id=commit_id)
+ # register per-commit call
+ log.debug(
+ 'register webhook call(%s) to url %s', event, commit_url)
+ url_cals.append((commit_url, self.secret_token, data))
+
+ else:
+ # register per-branch call
+ log.debug(
+ 'register webhook call(%s) to url %s', event, branch_url)
+ url_cals.append((branch_url, self.secret_token, data))
+
+ else:
+ log.debug(
+ 'register webhook call(%s) to url %s', event, url)
+ url_cals.append((url, self.secret_token, data))
+
+ return url_cals
+
+ def repo_create_event_handler(self, event, data):
+ url = self.get_base_parsed_template(data)
+ log.debug(
+ 'register webhook call(%s) to url %s', event, url)
+ return [(url, self.secret_token, data)]
+
+ def pull_request_event_handler(self, event, data):
+ url = self.get_base_parsed_template(data)
+ log.debug(
+ 'register webhook call(%s) to url %s', event, url)
+ url = string.Template(url).safe_substitute(
+ pull_request_id=data['pullrequest']['pull_request_id'],
+ pull_request_url=data['pullrequest']['url'])
+ return [(url, self.secret_token, data)]
+
+ def __call__(self, event, data):
+ if isinstance(event, events.RepoPushEvent):
+ return self.repo_push_event_handler(event, data)
+ elif isinstance(event, events.RepoCreateEvent):
+ return self.repo_create_event_handler(event, data)
+ elif isinstance(event, events.PullRequestEvent):
+ return self.pull_request_event_handler(event, data)
+ else:
+ raise ValueError('event type not supported: %s' % events)
+
class WebhookSettingsSchema(colander.Schema):
url = colander.SchemaNode(
colander.String(),
title=_('Webhook URL'),
- description=_('URL of the webhook to receive POST event.'),
+ description=
+ _('URL of the webhook to receive POST event. Following variables '
+ 'are allowed to be used: {vars}. Some of the variables would '
+ 'trigger multiple calls, like ${{branch}} or ${{commit_id}}. '
+ 'Webhook will be called as many times as unique objects in '
+ 'data in such cases.').format(vars=URL_VARS),
missing=colander.required,
required=True,
validator=colander.url,
@@ -58,8 +175,6 @@ class WebhookSettingsSchema(colander.Sch
)
-
-
class WebhookIntegrationType(IntegrationTypeBase):
key = 'webhook'
display_name = _('Webhook')
@@ -104,14 +219,30 @@ class WebhookIntegrationType(Integration
return
data = event.as_dict()
- post_to_webhook(data, self.settings)
+ template_url = self.settings['url']
+
+ handler = WebhookHandler(template_url, self.settings['secret_token'])
+ url_calls = handler(event, data)
+ log.debug('webhook: calling following urls: %s',
+ [x[0] for x in url_calls])
+ post_to_webhook(url_calls)
@task(ignore_result=True)
-def post_to_webhook(data, settings):
- log.debug('sending event:%s to webhook %s', data['name'], settings['url'])
- resp = requests.post(settings['url'], json={
- 'token': settings['secret_token'],
- 'event': data
- })
- resp.raise_for_status() # raise exception on a failed request
+def post_to_webhook(url_calls):
+ max_retries = 3
+ for url, token, data in url_calls:
+ # retry max N times
+ retries = Retry(
+ total=max_retries,
+ backoff_factor=0.15,
+ status_forcelist=[500, 502, 503, 504])
+ req_session = requests.Session()
+ req_session.mount(
+ 'http://', requests.adapters.HTTPAdapter(max_retries=retries))
+
+ resp = req_session.post(url, json={
+ 'token': token,
+ 'event': data
+ })
+ resp.raise_for_status() # raise exception on a failed request
diff --git a/rhodecode/tests/integrations/conftest.py b/rhodecode/tests/integrations/conftest.py
new file mode 100644
--- /dev/null
+++ b/rhodecode/tests/integrations/conftest.py
@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+
+import pytest
+from rhodecode import events
+
+
+@pytest.fixture
+def repo_push_event(backend, user_regular):
+ commits = [
+ {'message': 'ancestor commit fixes #15'},
+ {'message': 'quick fixes'},
+ {'message': 'change that fixes #41, #2'},
+ {'message': 'this is because 5b23c3532 broke stuff'},
+ {'message': 'last commit'},
+ ]
+ commit_ids = backend.create_master_repo(commits).values()
+ repo = backend.create_repo()
+ scm_extras = {
+ 'ip': '127.0.0.1',
+ 'username': user_regular.username,
+ 'action': '',
+ 'repository': repo.repo_name,
+ 'scm': repo.scm_instance().alias,
+ 'config': '',
+ 'server_url': 'http://example.com',
+ 'make_lock': None,
+ 'locked_by': [None],
+ 'commit_ids': commit_ids,
+ }
+
+ return events.RepoPushEvent(repo_name=repo.repo_name,
+ pushed_commit_ids=commit_ids,
+ extras=scm_extras)
diff --git a/rhodecode/tests/integrations/test_slack.py b/rhodecode/tests/integrations/test_slack.py
--- a/rhodecode/tests/integrations/test_slack.py
+++ b/rhodecode/tests/integrations/test_slack.py
@@ -19,41 +19,12 @@
# and proprietary license terms, please see https://rhodecode.com/licenses/
import pytest
-import requests
-from mock import Mock, patch
+from mock import patch
from rhodecode import events
from rhodecode.model.db import Session, Integration
from rhodecode.integrations.types.slack import SlackIntegrationType
-@pytest.fixture
-def repo_push_event(backend, user_regular):
- commits = [
- {'message': 'ancestor commit fixes #15'},
- {'message': 'quick fixes'},
- {'message': 'change that fixes #41, #2'},
- {'message': 'this is because 5b23c3532 broke stuff'},
- {'message': 'last commit'},
- ]
- commit_ids = backend.create_master_repo(commits).values()
- repo = backend.create_repo()
- scm_extras = {
- 'ip': '127.0.0.1',
- 'username': user_regular.username,
- 'action': '',
- 'repository': repo.repo_name,
- 'scm': repo.scm_instance().alias,
- 'config': '',
- 'server_url': 'http://example.com',
- 'make_lock': None,
- 'locked_by': [None],
- 'commit_ids': commit_ids,
- }
-
- return events.RepoPushEvent(repo_name=repo.repo_name,
- pushed_commit_ids=commit_ids,
- extras=scm_extras)
-
@pytest.fixture
def slack_settings():
diff --git a/rhodecode/tests/integrations/test_webhook.py b/rhodecode/tests/integrations/test_webhook.py
new file mode 100644
--- /dev/null
+++ b/rhodecode/tests/integrations/test_webhook.py
@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import pytest
+
+from rhodecode import events
+from rhodecode.lib.utils2 import AttributeDict
+from rhodecode.integrations.types.webhook import WebhookHandler
+
+
+@pytest.fixture
+def base_data():
+ return {
+ 'repo': {
+ 'repo_name': 'foo',
+ 'repo_type': 'hg',
+ 'repo_id': '12',
+ 'url': 'http://repo.url/foo'
+ }
+ }
+
+
+def test_webhook_parse_url_invalid_event():
+ template_url = 'http://server.com/${repo_name}/build'
+ handler = WebhookHandler(template_url, 'secret_token')
+ with pytest.raises(ValueError) as err:
+ handler(events.RepoDeleteEvent(''), {})
+ assert str(err.value).startswith('event type not supported')
+
+
+@pytest.mark.parametrize('template,expected_urls', [
+ ('http://server.com/${repo_name}/build', ['http://server.com/foo/build']),
+ ('http://server.com/${repo_name}/${repo_type}', ['http://server.com/foo/hg']),
+ ('http://${server}.com/${repo_name}/${repo_id}', ['http://${server}.com/foo/12']),
+ ('http://server.com/${branch}/build', ['http://server.com/${branch}/build']),
+])
+def test_webook_parse_url_for_create_event(base_data, template, expected_urls):
+ handler = WebhookHandler(template, 'secret_token')
+ urls = handler(events.RepoCreateEvent(''), base_data)
+ assert urls == [(url, 'secret_token', base_data) for url in expected_urls]
+
+
+@pytest.mark.parametrize('template,expected_urls', [
+ ('http://server.com/${repo_name}/${pull_request_id}', ['http://server.com/foo/999']),
+ ('http://server.com/${repo_name}/${pull_request_url}', ['http://server.com/foo/http://pr-url.com']),
+])
+def test_webook_parse_url_for_pull_request_event(base_data, template, expected_urls):
+ base_data['pullrequest'] = {
+ 'pull_request_id': 999,
+ 'url': 'http://pr-url.com',
+ }
+ handler = WebhookHandler(template, 'secret_token')
+ urls = handler(events.PullRequestCreateEvent(
+ AttributeDict({'target_repo': 'foo'})), base_data)
+ assert urls == [(url, 'secret_token', base_data) for url in expected_urls]
+
+
+@pytest.mark.parametrize('template,expected_urls', [
+ ('http://server.com/${branch}/build', ['http://server.com/stable/build',
+ 'http://server.com/dev/build']),
+ ('http://server.com/${branch}/${commit_id}', ['http://server.com/stable/stable-xxx',
+ 'http://server.com/stable/stable-yyy',
+ 'http://server.com/dev/dev-xxx',
+ 'http://server.com/dev/dev-yyy']),
+])
+def test_webook_parse_url_for_push_event(pylonsapp, repo_push_event, base_data, template, expected_urls):
+ base_data['push'] = {
+ 'branches': [{'name': 'stable'}, {'name': 'dev'}],
+ 'commits': [{'branch': 'stable', 'raw_id': 'stable-xxx'},
+ {'branch': 'stable', 'raw_id': 'stable-yyy'},
+ {'branch': 'dev', 'raw_id': 'dev-xxx'},
+ {'branch': 'dev', 'raw_id': 'dev-yyy'}]
+ }
+ handler = WebhookHandler(template, 'secret_token')
+ urls = handler(repo_push_event, base_data)
+ assert urls == [(url, 'secret_token', base_data) for url in expected_urls]
diff --git a/rhodecode/translation.py b/rhodecode/translation.py
--- a/rhodecode/translation.py
+++ b/rhodecode/translation.py
@@ -21,6 +21,7 @@ from pyramid.i18n import TranslationStri
# Create a translation string factory for the 'rhodecode' domain.
_ = TranslationStringFactory('rhodecode')
+
class LazyString(object):
def __init__(self, *args, **kw):
self.args = args