diff --git a/rhodecode/integrations/types/base.py b/rhodecode/integrations/types/base.py --- a/rhodecode/integrations/types/base.py +++ b/rhodecode/integrations/types/base.py @@ -19,8 +19,13 @@ # and proprietary license terms, please see https://rhodecode.com/licenses/ import colander +import string +import collections +import logging from rhodecode.translation import _ +log = logging.getLogger(__name__) + class IntegrationTypeBase(object): """ Base class for IntegrationType plugins """ @@ -142,6 +147,122 @@ WEBHOOK_URL_VARS = [ CI_URL_VARS = WEBHOOK_URL_VARS +class WebhookDataHandler(object): + name = 'webhook' + + def __init__(self, template_url, headers): + self.template_url = template_url + self.headers = headers + + def get_base_parsed_template(self, data): + """ + initially parses the passed in template with some common variables + available on ALL calls + """ + # note: make sure to update the `WEBHOOK_URL_VARS` if this changes + common_vars = { + 'repo_name': data['repo']['repo_name'], + 'repo_type': data['repo']['repo_type'], + 'repo_id': data['repo']['repo_id'], + 'repo_url': data['repo']['url'], + 'username': data['actor']['username'], + 'user_id': data['actor']['user_id'], + 'event_name': data['name'] + } + + extra_vars = {} + for extra_key, extra_val in data['repo']['extra_fields'].items(): + extra_vars['extra__{}'.format(extra_key)] = extra_val + common_vars.update(extra_vars) + + template_url = self.template_url.replace('${extra:', '${extra__') + return string.Template(template_url).safe_substitute(**common_vars) + + def repo_push_event_handler(self, event, data): + url = self.get_base_parsed_template(data) + url_cals = [] + branch_data = collections.OrderedDict() + for obj in data['push']['branches']: + branch_data[obj['name']] = obj + + branches_commits = collections.OrderedDict() + for commit in data['push']['commits']: + if commit.get('git_ref_change'): + # special case for GIT that allows creating tags, + # deleting branches without associated commit + continue + + if commit['branch'] not in branches_commits: + branch_commits = {'branch': branch_data[commit['branch']], + 'commits': []} + branches_commits[commit['branch']] = branch_commits + + branch_commits = branches_commits[commit['branch']] + branch_commits['commits'].append(commit) + + if '${branch}' in url: + # call it multiple times, for each branch if used in variables + for branch, commit_ids in branches_commits.items(): + branch_url = string.Template(url).safe_substitute(branch=branch) + # call further down for each commit if used + if '${commit_id}' in branch_url: + for commit_data in commit_ids['commits']: + commit_id = commit_data['raw_id'] + commit_url = string.Template(branch_url).safe_substitute( + commit_id=commit_id) + # register per-commit call + log.debug( + 'register %s call(%s) to url %s', + self.name, event, commit_url) + url_cals.append( + (commit_url, self.headers, data)) + + else: + # register per-branch call + log.debug( + 'register %s call(%s) to url %s', + self.name, event, branch_url) + url_cals.append( + (branch_url, self.headers, data)) + + else: + log.debug( + 'register %s call(%s) to url %s', self.name, event, url) + url_cals.append((url, self.headers, data)) + + return url_cals + + def repo_create_event_handler(self, event, data): + url = self.get_base_parsed_template(data) + log.debug( + 'register %s call(%s) to url %s', self.name, event, url) + return [(url, self.headers, data)] + + def pull_request_event_handler(self, event, data): + url = self.get_base_parsed_template(data) + log.debug( + 'register %s call(%s) to url %s', self.name, event, url) + url = string.Template(url).safe_substitute( + pull_request_id=data['pullrequest']['pull_request_id'], + pull_request_url=data['pullrequest']['url'], + pull_request_shadow_url=data['pullrequest']['shadow_url'],) + return [(url, self.headers, data)] + + def __call__(self, event, data): + from rhodecode import events + + if isinstance(event, events.RepoPushEvent): + return self.repo_push_event_handler(event, data) + elif isinstance(event, events.RepoCreateEvent): + return self.repo_create_event_handler(event, data) + elif isinstance(event, events.PullRequestEvent): + return self.pull_request_event_handler(event, data) + else: + raise ValueError( + 'event type `%s` not in supported list: %s' % ( + event.__class__, events)) + + def get_auth(settings): from requests.auth import HTTPBasicAuth username = settings.get('username') @@ -151,6 +272,10 @@ def get_auth(settings): return None +def get_web_token(settings): + return settings['secret_token'] + + def get_url_vars(url_vars): return '\n'.join( '{} - {}'.format('${' + key + '}', explanation) diff --git a/rhodecode/integrations/types/webhook.py b/rhodecode/integrations/types/webhook.py --- a/rhodecode/integrations/types/webhook.py +++ b/rhodecode/integrations/types/webhook.py @@ -19,8 +19,6 @@ # and proprietary license terms, please see https://rhodecode.com/licenses/ from __future__ import unicode_literals -import string -import collections import deform import deform.widget @@ -34,7 +32,8 @@ import rhodecode from rhodecode import events from rhodecode.translation import _ from rhodecode.integrations.types.base import ( - IntegrationTypeBase, get_auth, get_url_vars, WEBHOOK_URL_VARS) + IntegrationTypeBase, get_auth, get_web_token, get_url_vars, + WebhookDataHandler, WEBHOOK_URL_VARS) from rhodecode.lib.celerylib import run_task, async_task, RequestContextTask from rhodecode.model.validation_schema import widgets @@ -46,113 +45,6 @@ log = logging.getLogger(__name__) URL_VARS = get_url_vars(WEBHOOK_URL_VARS) -class WebhookHandler(object): - def __init__(self, template_url, secret_token, headers): - self.template_url = template_url - self.secret_token = secret_token - self.headers = headers - - def get_base_parsed_template(self, data): - """ - initially parses the passed in template with some common variables - available on ALL calls - """ - # note: make sure to update the `WEBHOOK_URL_VARS` if this changes - common_vars = { - 'repo_name': data['repo']['repo_name'], - 'repo_type': data['repo']['repo_type'], - 'repo_id': data['repo']['repo_id'], - 'repo_url': data['repo']['url'], - 'username': data['actor']['username'], - 'user_id': data['actor']['user_id'], - 'event_name': data['name'] - } - - extra_vars = {} - for extra_key, extra_val in data['repo']['extra_fields'].items(): - extra_vars['extra__{}'.format(extra_key)] = extra_val - common_vars.update(extra_vars) - - template_url = self.template_url.replace('${extra:', '${extra__') - return string.Template(template_url).safe_substitute(**common_vars) - - def repo_push_event_handler(self, event, data): - url = self.get_base_parsed_template(data) - url_cals = [] - branch_data = collections.OrderedDict() - for obj in data['push']['branches']: - branch_data[obj['name']] = obj - - branches_commits = collections.OrderedDict() - for commit in data['push']['commits']: - if commit.get('git_ref_change'): - # special case for GIT that allows creating tags, - # deleting branches without associated commit - continue - - if commit['branch'] not in branches_commits: - branch_commits = {'branch': branch_data[commit['branch']], - 'commits': []} - branches_commits[commit['branch']] = branch_commits - - branch_commits = branches_commits[commit['branch']] - branch_commits['commits'].append(commit) - - if '${branch}' in url: - # call it multiple times, for each branch if used in variables - for branch, commit_ids in branches_commits.items(): - branch_url = string.Template(url).safe_substitute(branch=branch) - # call further down for each commit if used - if '${commit_id}' in branch_url: - for commit_data in commit_ids['commits']: - commit_id = commit_data['raw_id'] - commit_url = string.Template(branch_url).safe_substitute( - commit_id=commit_id) - # register per-commit call - log.debug( - 'register webhook call(%s) to url %s', event, commit_url) - url_cals.append((commit_url, self.secret_token, self.headers, data)) - - else: - # register per-branch call - log.debug( - 'register webhook call(%s) to url %s', event, branch_url) - url_cals.append((branch_url, self.secret_token, self.headers, data)) - - else: - log.debug( - 'register webhook call(%s) to url %s', event, url) - url_cals.append((url, self.secret_token, self.headers, data)) - - return url_cals - - def repo_create_event_handler(self, event, data): - url = self.get_base_parsed_template(data) - log.debug( - 'register webhook call(%s) to url %s', event, url) - return [(url, self.secret_token, self.headers, data)] - - def pull_request_event_handler(self, event, data): - url = self.get_base_parsed_template(data) - log.debug( - 'register webhook call(%s) to url %s', event, url) - url = string.Template(url).safe_substitute( - pull_request_id=data['pullrequest']['pull_request_id'], - pull_request_url=data['pullrequest']['url'], - pull_request_shadow_url=data['pullrequest']['shadow_url'],) - return [(url, self.secret_token, self.headers, data)] - - def __call__(self, event, data): - if isinstance(event, events.RepoPushEvent): - return self.repo_push_event_handler(event, data) - elif isinstance(event, events.RepoCreateEvent): - return self.repo_create_event_handler(event, data) - elif isinstance(event, events.PullRequestEvent): - return self.pull_request_event_handler(event, data) - else: - raise ValueError('event type not supported: %s' % events) - - class WebhookSettingsSchema(colander.Schema): url = colander.SchemaNode( colander.String(), @@ -243,7 +135,7 @@ class WebhookSettingsSchema(colander.Sch class WebhookIntegrationType(IntegrationTypeBase): key = 'webhook' display_name = _('Webhook') - description = _('Post json events to a Webhook endpoint') + description = _('send JSON data to a url endpoint') @classmethod def icon(cls): @@ -275,8 +167,8 @@ class WebhookIntegrationType(Integration return schema def send_event(self, event): - log.debug('handling event %s with Webhook integration %s', - event.name, self) + log.debug( + 'handling event %s with Webhook integration %s', event.name, self) if event.__class__ not in self.valid_events: log.debug('event not valid: %r' % event) @@ -295,8 +187,7 @@ class WebhookIntegrationType(Integration if head_key and head_val: headers = {head_key: head_val} - handler = WebhookHandler( - template_url, self.settings['secret_token'], headers) + handler = WebhookDataHandler(template_url, headers) url_calls = handler(event, data) log.debug('webhook: calling following urls: %s', @@ -353,7 +244,9 @@ def post_to_webhook(url_calls, settings) } # updated below with custom ones, allows override auth = get_auth(settings) - for url, token, headers, data in url_calls: + token = get_web_token(settings) + + for url, headers, data in url_calls: req_session = requests.Session() req_session.mount( # retry max N times 'http://', requests.adapters.HTTPAdapter(max_retries=retries)) diff --git a/rhodecode/tests/integrations/test_webhook.py b/rhodecode/tests/integrations/test_webhook.py --- a/rhodecode/tests/integrations/test_webhook.py +++ b/rhodecode/tests/integrations/test_webhook.py @@ -22,12 +22,13 @@ import pytest from rhodecode import events from rhodecode.lib.utils2 import AttributeDict -from rhodecode.integrations.types.webhook import WebhookHandler +from rhodecode.integrations.types.webhook import WebhookDataHandler @pytest.fixture def base_data(): return { + 'name': 'event', 'repo': { 'repo_name': 'foo', 'repo_type': 'hg', @@ -44,31 +45,40 @@ def base_data(): def test_webhook_parse_url_invalid_event(): template_url = 'http://server.com/${repo_name}/build' - handler = WebhookHandler( - template_url, 'secret_token', {'exmaple-header':'header-values'}) + handler = WebhookDataHandler( + template_url, {'exmaple-header': 'header-values'}) + event = events.RepoDeleteEvent('') with pytest.raises(ValueError) as err: - handler(events.RepoDeleteEvent(''), {}) - assert str(err.value).startswith('event type not supported') + handler(event, {}) + + err = str(err.value) + assert err.startswith( + 'event type `%s` not in supported list' % event.__class__) @pytest.mark.parametrize('template,expected_urls', [ - ('http://server.com/${repo_name}/build', ['http://server.com/foo/build']), - ('http://server.com/${repo_name}/${repo_type}', ['http://server.com/foo/hg']), - ('http://${server}.com/${repo_name}/${repo_id}', ['http://${server}.com/foo/12']), - ('http://server.com/${branch}/build', ['http://server.com/${branch}/build']), + ('http://server.com/${repo_name}/build', + ['http://server.com/foo/build']), + ('http://server.com/${repo_name}/${repo_type}', + ['http://server.com/foo/hg']), + ('http://${server}.com/${repo_name}/${repo_id}', + ['http://${server}.com/foo/12']), + ('http://server.com/${branch}/build', + ['http://server.com/${branch}/build']), ]) def test_webook_parse_url_for_create_event(base_data, template, expected_urls): headers = {'exmaple-header': 'header-values'} - handler = WebhookHandler( - template, 'secret_token', headers) + handler = WebhookDataHandler(template, headers) urls = handler(events.RepoCreateEvent(''), base_data) assert urls == [ - (url, 'secret_token', headers, base_data) for url in expected_urls] + (url, headers, base_data) for url in expected_urls] @pytest.mark.parametrize('template,expected_urls', [ - ('http://server.com/${repo_name}/${pull_request_id}', ['http://server.com/foo/999']), - ('http://server.com/${repo_name}/${pull_request_url}', ['http://server.com/foo/http://pr-url.com']), + ('http://server.com/${repo_name}/${pull_request_id}', + ['http://server.com/foo/999']), + ('http://server.com/${repo_name}/${pull_request_url}', + ['http://server.com/foo/http://pr-url.com']), ]) def test_webook_parse_url_for_pull_request_event( base_data, template, expected_urls): @@ -76,23 +86,25 @@ def test_webook_parse_url_for_pull_reque base_data['pullrequest'] = { 'pull_request_id': 999, 'url': 'http://pr-url.com', + 'shadow_url': 'http://pr-url.com/repository' } headers = {'exmaple-header': 'header-values'} - handler = WebhookHandler( - template, 'secret_token', headers) + handler = WebhookDataHandler(template, headers) urls = handler(events.PullRequestCreateEvent( AttributeDict({'target_repo': 'foo'})), base_data) assert urls == [ - (url, 'secret_token', headers, base_data) for url in expected_urls] + (url, headers, base_data) for url in expected_urls] @pytest.mark.parametrize('template,expected_urls', [ - ('http://server.com/${branch}/build', ['http://server.com/stable/build', - 'http://server.com/dev/build']), - ('http://server.com/${branch}/${commit_id}', ['http://server.com/stable/stable-xxx', - 'http://server.com/stable/stable-yyy', - 'http://server.com/dev/dev-xxx', - 'http://server.com/dev/dev-yyy']), + ('http://server.com/${branch}/build', + ['http://server.com/stable/build', + 'http://server.com/dev/build']), + ('http://server.com/${branch}/${commit_id}', + ['http://server.com/stable/stable-xxx', + 'http://server.com/stable/stable-yyy', + 'http://server.com/dev/dev-xxx', + 'http://server.com/dev/dev-yyy']), ]) def test_webook_parse_url_for_push_event( baseapp, repo_push_event, base_data, template, expected_urls): @@ -104,8 +116,7 @@ def test_webook_parse_url_for_push_event {'branch': 'dev', 'raw_id': 'dev-yyy'}] } headers = {'exmaple-header': 'header-values'} - handler = WebhookHandler( - template, 'secret_token', headers) + handler = WebhookDataHandler(template, headers) urls = handler(repo_push_event, base_data) assert urls == [ - (url, 'secret_token', headers, base_data) for url in expected_urls] + (url, headers, base_data) for url in expected_urls]