diff --git a/rhodecode/integrations/types/webhook.py b/rhodecode/integrations/types/webhook.py --- a/rhodecode/integrations/types/webhook.py +++ b/rhodecode/integrations/types/webhook.py @@ -19,13 +19,15 @@ # and proprietary license terms, please see https://rhodecode.com/licenses/ from __future__ import unicode_literals +import string +from collections import OrderedDict import deform import logging import requests import colander from celery.task import task -from mako.template import Template +from requests.packages.urllib3.util.retry import Retry from rhodecode import events from rhodecode.translation import _ @@ -33,12 +35,127 @@ from rhodecode.integrations.types.base i log = logging.getLogger(__name__) +# updating this required to update the `base_vars` passed in url calling func +WEBHOOK_URL_VARS = [ + 'repo_name', + 'repo_type', + 'repo_id', + 'repo_url', + + # special attrs below that we handle, using multi-call + 'branch', + 'commit_id', + + # pr events vars + 'pull_request_id', + 'pull_request_url', + +] +URL_VARS = ', '.join('${' + x + '}' for x in WEBHOOK_URL_VARS) + + +class WebhookHandler(object): + def __init__(self, template_url, secret_token): + self.template_url = template_url + self.secret_token = secret_token + + def get_base_parsed_template(self, data): + """ + initially parses the passed in template with some common variables + available on ALL calls + """ + # note: make sure to update the `WEBHOOK_URL_VARS` if this changes + common_vars = { + 'repo_name': data['repo']['repo_name'], + 'repo_type': data['repo']['repo_type'], + 'repo_id': data['repo']['repo_id'], + 'repo_url': data['repo']['url'], + } + + return string.Template( + self.template_url).safe_substitute(**common_vars) + + def repo_push_event_handler(self, event, data): + url = self.get_base_parsed_template(data) + url_cals = [] + branch_data = OrderedDict() + for obj in data['push']['branches']: + branch_data[obj['name']] = obj + + branches_commits = OrderedDict() + for commit in data['push']['commits']: + if commit['branch'] not in branches_commits: + branch_commits = {'branch': branch_data[commit['branch']], + 'commits': []} + branches_commits[commit['branch']] = branch_commits + + branch_commits = branches_commits[commit['branch']] + branch_commits['commits'].append(commit) + + if '${branch}' in url: + # call it multiple times, for each branch if used in variables + for branch, commit_ids in branches_commits.items(): + branch_url = string.Template(url).safe_substitute(branch=branch) + # call further down for each commit if used + if '${commit_id}' in branch_url: + for commit_data in commit_ids['commits']: + commit_id = commit_data['raw_id'] + commit_url = string.Template(branch_url).safe_substitute( + commit_id=commit_id) + # register per-commit call + log.debug( + 'register webhook call(%s) to url %s', event, commit_url) + url_cals.append((commit_url, self.secret_token, data)) + + else: + # register per-branch call + log.debug( + 'register webhook call(%s) to url %s', event, branch_url) + url_cals.append((branch_url, self.secret_token, data)) + + else: + log.debug( + 'register webhook call(%s) to url %s', event, url) + url_cals.append((url, self.secret_token, data)) + + return url_cals + + def repo_create_event_handler(self, event, data): + url = self.get_base_parsed_template(data) + log.debug( + 'register webhook call(%s) to url %s', event, url) + return [(url, self.secret_token, data)] + + def pull_request_event_handler(self, event, data): + url = self.get_base_parsed_template(data) + log.debug( + 'register webhook call(%s) to url %s', event, url) + url = string.Template(url).safe_substitute( + pull_request_id=data['pullrequest']['pull_request_id'], + pull_request_url=data['pullrequest']['url']) + return [(url, self.secret_token, data)] + + def __call__(self, event, data): + if isinstance(event, events.RepoPushEvent): + return self.repo_push_event_handler(event, data) + elif isinstance(event, events.RepoCreateEvent): + return self.repo_create_event_handler(event, data) + elif isinstance(event, events.PullRequestEvent): + return self.pull_request_event_handler(event, data) + else: + raise ValueError('event type not supported: %s' % events) + class WebhookSettingsSchema(colander.Schema): url = colander.SchemaNode( colander.String(), title=_('Webhook URL'), - description=_('URL of the webhook to receive POST event.'), + description= + _('URL of the webhook to receive POST event. Following variables ' + 'are allowed to be used: {vars}. Some of the variables would ' + 'trigger multiple calls, like ${{branch}} or ${{commit_id}}. ' + 'Webhook will be called as many times as unique objects in ' + 'data in such cases.').format(vars=URL_VARS), missing=colander.required, required=True, validator=colander.url, @@ -58,8 +175,6 @@ class WebhookSettingsSchema(colander.Sch ) - - class WebhookIntegrationType(IntegrationTypeBase): key = 'webhook' display_name = _('Webhook') @@ -104,14 +219,30 @@ class WebhookIntegrationType(Integration return data = event.as_dict() - post_to_webhook(data, self.settings) + template_url = self.settings['url'] + + handler = WebhookHandler(template_url, self.settings['secret_token']) + url_calls = handler(event, data) + log.debug('webhook: calling following urls: %s', + [x[0] for x in url_calls]) + post_to_webhook(url_calls) @task(ignore_result=True) -def post_to_webhook(data, settings): - log.debug('sending event:%s to webhook %s', data['name'], settings['url']) - resp = requests.post(settings['url'], json={ - 'token': settings['secret_token'], - 'event': data - }) - resp.raise_for_status() # raise exception on a failed request +def post_to_webhook(url_calls): + max_retries = 3 + for url, token, data in url_calls: + # retry max N times + retries = Retry( + total=max_retries, + backoff_factor=0.15, + status_forcelist=[500, 502, 503, 504]) + req_session = requests.Session() + req_session.mount( + 'http://', requests.adapters.HTTPAdapter(max_retries=retries)) + + resp = req_session.post(url, json={ + 'token': token, + 'event': data + }) + resp.raise_for_status() # raise exception on a failed request diff --git a/rhodecode/tests/integrations/conftest.py b/rhodecode/tests/integrations/conftest.py new file mode 100644 --- /dev/null +++ b/rhodecode/tests/integrations/conftest.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2016 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + + +import pytest +from rhodecode import events + + +@pytest.fixture +def repo_push_event(backend, user_regular): + commits = [ + {'message': 'ancestor commit fixes #15'}, + {'message': 'quick fixes'}, + {'message': 'change that fixes #41, #2'}, + {'message': 'this is because 5b23c3532 broke stuff'}, + {'message': 'last commit'}, + ] + commit_ids = backend.create_master_repo(commits).values() + repo = backend.create_repo() + scm_extras = { + 'ip': '127.0.0.1', + 'username': user_regular.username, + 'action': '', + 'repository': repo.repo_name, + 'scm': repo.scm_instance().alias, + 'config': '', + 'server_url': 'http://example.com', + 'make_lock': None, + 'locked_by': [None], + 'commit_ids': commit_ids, + } + + return events.RepoPushEvent(repo_name=repo.repo_name, + pushed_commit_ids=commit_ids, + extras=scm_extras) diff --git a/rhodecode/tests/integrations/test_slack.py b/rhodecode/tests/integrations/test_slack.py --- a/rhodecode/tests/integrations/test_slack.py +++ b/rhodecode/tests/integrations/test_slack.py @@ -19,41 +19,12 @@ # and proprietary license terms, please see https://rhodecode.com/licenses/ import pytest -import requests -from mock import Mock, patch +from mock import patch from rhodecode import events from rhodecode.model.db import Session, Integration from rhodecode.integrations.types.slack import SlackIntegrationType -@pytest.fixture -def repo_push_event(backend, user_regular): - commits = [ - {'message': 'ancestor commit fixes #15'}, - {'message': 'quick fixes'}, - {'message': 'change that fixes #41, #2'}, - {'message': 'this is because 5b23c3532 broke stuff'}, - {'message': 'last commit'}, - ] - commit_ids = backend.create_master_repo(commits).values() - repo = backend.create_repo() - scm_extras = { - 'ip': '127.0.0.1', - 'username': user_regular.username, - 'action': '', - 'repository': repo.repo_name, - 'scm': repo.scm_instance().alias, - 'config': '', - 'server_url': 'http://example.com', - 'make_lock': None, - 'locked_by': [None], - 'commit_ids': commit_ids, - } - - return events.RepoPushEvent(repo_name=repo.repo_name, - pushed_commit_ids=commit_ids, - extras=scm_extras) - @pytest.fixture def slack_settings(): diff --git a/rhodecode/tests/integrations/test_webhook.py b/rhodecode/tests/integrations/test_webhook.py new file mode 100644 --- /dev/null +++ b/rhodecode/tests/integrations/test_webhook.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2016 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +import pytest + +from rhodecode import events +from rhodecode.lib.utils2 import AttributeDict +from rhodecode.integrations.types.webhook import WebhookHandler + + +@pytest.fixture +def base_data(): + return { + 'repo': { + 'repo_name': 'foo', + 'repo_type': 'hg', + 'repo_id': '12', + 'url': 'http://repo.url/foo' + } + } + + +def test_webhook_parse_url_invalid_event(): + template_url = 'http://server.com/${repo_name}/build' + handler = WebhookHandler(template_url, 'secret_token') + with pytest.raises(ValueError) as err: + handler(events.RepoDeleteEvent(''), {}) + assert str(err.value).startswith('event type not supported') + + +@pytest.mark.parametrize('template,expected_urls', [ + ('http://server.com/${repo_name}/build', ['http://server.com/foo/build']), + ('http://server.com/${repo_name}/${repo_type}', ['http://server.com/foo/hg']), + ('http://${server}.com/${repo_name}/${repo_id}', ['http://${server}.com/foo/12']), + ('http://server.com/${branch}/build', ['http://server.com/${branch}/build']), +]) +def test_webook_parse_url_for_create_event(base_data, template, expected_urls): + handler = WebhookHandler(template, 'secret_token') + urls = handler(events.RepoCreateEvent(''), base_data) + assert urls == [(url, 'secret_token', base_data) for url in expected_urls] + + +@pytest.mark.parametrize('template,expected_urls', [ + ('http://server.com/${repo_name}/${pull_request_id}', ['http://server.com/foo/999']), + ('http://server.com/${repo_name}/${pull_request_url}', ['http://server.com/foo/http://pr-url.com']), +]) +def test_webook_parse_url_for_pull_request_event(base_data, template, expected_urls): + base_data['pullrequest'] = { + 'pull_request_id': 999, + 'url': 'http://pr-url.com', + } + handler = WebhookHandler(template, 'secret_token') + urls = handler(events.PullRequestCreateEvent( + AttributeDict({'target_repo': 'foo'})), base_data) + assert urls == [(url, 'secret_token', base_data) for url in expected_urls] + + +@pytest.mark.parametrize('template,expected_urls', [ + ('http://server.com/${branch}/build', ['http://server.com/stable/build', + 'http://server.com/dev/build']), + ('http://server.com/${branch}/${commit_id}', ['http://server.com/stable/stable-xxx', + 'http://server.com/stable/stable-yyy', + 'http://server.com/dev/dev-xxx', + 'http://server.com/dev/dev-yyy']), +]) +def test_webook_parse_url_for_push_event(pylonsapp, repo_push_event, base_data, template, expected_urls): + base_data['push'] = { + 'branches': [{'name': 'stable'}, {'name': 'dev'}], + 'commits': [{'branch': 'stable', 'raw_id': 'stable-xxx'}, + {'branch': 'stable', 'raw_id': 'stable-yyy'}, + {'branch': 'dev', 'raw_id': 'dev-xxx'}, + {'branch': 'dev', 'raw_id': 'dev-yyy'}] + } + handler = WebhookHandler(template, 'secret_token') + urls = handler(repo_push_event, base_data) + assert urls == [(url, 'secret_token', base_data) for url in expected_urls] diff --git a/rhodecode/translation.py b/rhodecode/translation.py --- a/rhodecode/translation.py +++ b/rhodecode/translation.py @@ -21,6 +21,7 @@ from pyramid.i18n import TranslationStri # Create a translation string factory for the 'rhodecode' domain. _ = TranslationStringFactory('rhodecode') + class LazyString(object): def __init__(self, *args, **kw): self.args = args