##// END OF EJS Templates
webhooks: added variables into the call URL. Fixes #4211
marcink -
r938:87d3b112 default
parent child Browse files
Show More
@@ -0,0 +1,52 b''
1 # -*- coding: utf-8 -*-
2
3 # Copyright (C) 2010-2016 RhodeCode GmbH
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
21
22 import pytest
23 from rhodecode import events
24
25
26 @pytest.fixture
27 def repo_push_event(backend, user_regular):
28 commits = [
29 {'message': 'ancestor commit fixes #15'},
30 {'message': 'quick fixes'},
31 {'message': 'change that fixes #41, #2'},
32 {'message': 'this is because 5b23c3532 broke stuff'},
33 {'message': 'last commit'},
34 ]
35 commit_ids = backend.create_master_repo(commits).values()
36 repo = backend.create_repo()
37 scm_extras = {
38 'ip': '127.0.0.1',
39 'username': user_regular.username,
40 'action': '',
41 'repository': repo.repo_name,
42 'scm': repo.scm_instance().alias,
43 'config': '',
44 'server_url': 'http://example.com',
45 'make_lock': None,
46 'locked_by': [None],
47 'commit_ids': commit_ids,
48 }
49
50 return events.RepoPushEvent(repo_name=repo.repo_name,
51 pushed_commit_ids=commit_ids,
52 extras=scm_extras)
@@ -0,0 +1,93 b''
1 # -*- coding: utf-8 -*-
2
3 # Copyright (C) 2010-2016 RhodeCode GmbH
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
21 import pytest
22
23 from rhodecode import events
24 from rhodecode.lib.utils2 import AttributeDict
25 from rhodecode.integrations.types.webhook import WebhookHandler
26
27
28 @pytest.fixture
29 def base_data():
30 return {
31 'repo': {
32 'repo_name': 'foo',
33 'repo_type': 'hg',
34 'repo_id': '12',
35 'url': 'http://repo.url/foo'
36 }
37 }
38
39
40 def test_webhook_parse_url_invalid_event():
41 template_url = 'http://server.com/${repo_name}/build'
42 handler = WebhookHandler(template_url, 'secret_token')
43 with pytest.raises(ValueError) as err:
44 handler(events.RepoDeleteEvent(''), {})
45 assert str(err.value).startswith('event type not supported')
46
47
48 @pytest.mark.parametrize('template,expected_urls', [
49 ('http://server.com/${repo_name}/build', ['http://server.com/foo/build']),
50 ('http://server.com/${repo_name}/${repo_type}', ['http://server.com/foo/hg']),
51 ('http://${server}.com/${repo_name}/${repo_id}', ['http://${server}.com/foo/12']),
52 ('http://server.com/${branch}/build', ['http://server.com/${branch}/build']),
53 ])
54 def test_webook_parse_url_for_create_event(base_data, template, expected_urls):
55 handler = WebhookHandler(template, 'secret_token')
56 urls = handler(events.RepoCreateEvent(''), base_data)
57 assert urls == [(url, 'secret_token', base_data) for url in expected_urls]
58
59
60 @pytest.mark.parametrize('template,expected_urls', [
61 ('http://server.com/${repo_name}/${pull_request_id}', ['http://server.com/foo/999']),
62 ('http://server.com/${repo_name}/${pull_request_url}', ['http://server.com/foo/http://pr-url.com']),
63 ])
64 def test_webook_parse_url_for_pull_request_event(base_data, template, expected_urls):
65 base_data['pullrequest'] = {
66 'pull_request_id': 999,
67 'url': 'http://pr-url.com',
68 }
69 handler = WebhookHandler(template, 'secret_token')
70 urls = handler(events.PullRequestCreateEvent(
71 AttributeDict({'target_repo': 'foo'})), base_data)
72 assert urls == [(url, 'secret_token', base_data) for url in expected_urls]
73
74
75 @pytest.mark.parametrize('template,expected_urls', [
76 ('http://server.com/${branch}/build', ['http://server.com/stable/build',
77 'http://server.com/dev/build']),
78 ('http://server.com/${branch}/${commit_id}', ['http://server.com/stable/stable-xxx',
79 'http://server.com/stable/stable-yyy',
80 'http://server.com/dev/dev-xxx',
81 'http://server.com/dev/dev-yyy']),
82 ])
83 def test_webook_parse_url_for_push_event(pylonsapp, repo_push_event, base_data, template, expected_urls):
84 base_data['push'] = {
85 'branches': [{'name': 'stable'}, {'name': 'dev'}],
86 'commits': [{'branch': 'stable', 'raw_id': 'stable-xxx'},
87 {'branch': 'stable', 'raw_id': 'stable-yyy'},
88 {'branch': 'dev', 'raw_id': 'dev-xxx'},
89 {'branch': 'dev', 'raw_id': 'dev-yyy'}]
90 }
91 handler = WebhookHandler(template, 'secret_token')
92 urls = handler(repo_push_event, base_data)
93 assert urls == [(url, 'secret_token', base_data) for url in expected_urls]
@@ -19,13 +19,15 b''
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 from __future__ import unicode_literals
21 from __future__ import unicode_literals
22 import string
23 from collections import OrderedDict
22
24
23 import deform
25 import deform
24 import logging
26 import logging
25 import requests
27 import requests
26 import colander
28 import colander
27 from celery.task import task
29 from celery.task import task
28 from mako.template import Template
30 from requests.packages.urllib3.util.retry import Retry
29
31
30 from rhodecode import events
32 from rhodecode import events
31 from rhodecode.translation import _
33 from rhodecode.translation import _
@@ -33,12 +35,127 b' from rhodecode.integrations.types.base i'
33
35
34 log = logging.getLogger(__name__)
36 log = logging.getLogger(__name__)
35
37
38 # updating this required to update the `base_vars` passed in url calling func
39 WEBHOOK_URL_VARS = [
40 'repo_name',
41 'repo_type',
42 'repo_id',
43 'repo_url',
44
45 # special attrs below that we handle, using multi-call
46 'branch',
47 'commit_id',
48
49 # pr events vars
50 'pull_request_id',
51 'pull_request_url',
52
53 ]
54 URL_VARS = ', '.join('${' + x + '}' for x in WEBHOOK_URL_VARS)
55
56
57 class WebhookHandler(object):
58 def __init__(self, template_url, secret_token):
59 self.template_url = template_url
60 self.secret_token = secret_token
61
62 def get_base_parsed_template(self, data):
63 """
64 initially parses the passed in template with some common variables
65 available on ALL calls
66 """
67 # note: make sure to update the `WEBHOOK_URL_VARS` if this changes
68 common_vars = {
69 'repo_name': data['repo']['repo_name'],
70 'repo_type': data['repo']['repo_type'],
71 'repo_id': data['repo']['repo_id'],
72 'repo_url': data['repo']['url'],
73 }
74
75 return string.Template(
76 self.template_url).safe_substitute(**common_vars)
77
78 def repo_push_event_handler(self, event, data):
79 url = self.get_base_parsed_template(data)
80 url_cals = []
81 branch_data = OrderedDict()
82 for obj in data['push']['branches']:
83 branch_data[obj['name']] = obj
84
85 branches_commits = OrderedDict()
86 for commit in data['push']['commits']:
87 if commit['branch'] not in branches_commits:
88 branch_commits = {'branch': branch_data[commit['branch']],
89 'commits': []}
90 branches_commits[commit['branch']] = branch_commits
91
92 branch_commits = branches_commits[commit['branch']]
93 branch_commits['commits'].append(commit)
94
95 if '${branch}' in url:
96 # call it multiple times, for each branch if used in variables
97 for branch, commit_ids in branches_commits.items():
98 branch_url = string.Template(url).safe_substitute(branch=branch)
99 # call further down for each commit if used
100 if '${commit_id}' in branch_url:
101 for commit_data in commit_ids['commits']:
102 commit_id = commit_data['raw_id']
103 commit_url = string.Template(branch_url).safe_substitute(
104 commit_id=commit_id)
105 # register per-commit call
106 log.debug(
107 'register webhook call(%s) to url %s', event, commit_url)
108 url_cals.append((commit_url, self.secret_token, data))
109
110 else:
111 # register per-branch call
112 log.debug(
113 'register webhook call(%s) to url %s', event, branch_url)
114 url_cals.append((branch_url, self.secret_token, data))
115
116 else:
117 log.debug(
118 'register webhook call(%s) to url %s', event, url)
119 url_cals.append((url, self.secret_token, data))
120
121 return url_cals
122
123 def repo_create_event_handler(self, event, data):
124 url = self.get_base_parsed_template(data)
125 log.debug(
126 'register webhook call(%s) to url %s', event, url)
127 return [(url, self.secret_token, data)]
128
129 def pull_request_event_handler(self, event, data):
130 url = self.get_base_parsed_template(data)
131 log.debug(
132 'register webhook call(%s) to url %s', event, url)
133 url = string.Template(url).safe_substitute(
134 pull_request_id=data['pullrequest']['pull_request_id'],
135 pull_request_url=data['pullrequest']['url'])
136 return [(url, self.secret_token, data)]
137
138 def __call__(self, event, data):
139 if isinstance(event, events.RepoPushEvent):
140 return self.repo_push_event_handler(event, data)
141 elif isinstance(event, events.RepoCreateEvent):
142 return self.repo_create_event_handler(event, data)
143 elif isinstance(event, events.PullRequestEvent):
144 return self.pull_request_event_handler(event, data)
145 else:
146 raise ValueError('event type not supported: %s' % events)
147
36
148
37 class WebhookSettingsSchema(colander.Schema):
149 class WebhookSettingsSchema(colander.Schema):
38 url = colander.SchemaNode(
150 url = colander.SchemaNode(
39 colander.String(),
151 colander.String(),
40 title=_('Webhook URL'),
152 title=_('Webhook URL'),
41 description=_('URL of the webhook to receive POST event.'),
153 description=
154 _('URL of the webhook to receive POST event. Following variables '
155 'are allowed to be used: {vars}. Some of the variables would '
156 'trigger multiple calls, like ${{branch}} or ${{commit_id}}. '
157 'Webhook will be called as many times as unique objects in '
158 'data in such cases.').format(vars=URL_VARS),
42 missing=colander.required,
159 missing=colander.required,
43 required=True,
160 required=True,
44 validator=colander.url,
161 validator=colander.url,
@@ -58,8 +175,6 b' class WebhookSettingsSchema(colander.Sch'
58 )
175 )
59
176
60
177
61
62
63 class WebhookIntegrationType(IntegrationTypeBase):
178 class WebhookIntegrationType(IntegrationTypeBase):
64 key = 'webhook'
179 key = 'webhook'
65 display_name = _('Webhook')
180 display_name = _('Webhook')
@@ -104,14 +219,30 b' class WebhookIntegrationType(Integration'
104 return
219 return
105
220
106 data = event.as_dict()
221 data = event.as_dict()
107 post_to_webhook(data, self.settings)
222 template_url = self.settings['url']
223
224 handler = WebhookHandler(template_url, self.settings['secret_token'])
225 url_calls = handler(event, data)
226 log.debug('webhook: calling following urls: %s',
227 [x[0] for x in url_calls])
228 post_to_webhook(url_calls)
108
229
109
230
110 @task(ignore_result=True)
231 @task(ignore_result=True)
111 def post_to_webhook(data, settings):
232 def post_to_webhook(url_calls):
112 log.debug('sending event:%s to webhook %s', data['name'], settings['url'])
233 max_retries = 3
113 resp = requests.post(settings['url'], json={
234 for url, token, data in url_calls:
114 'token': settings['secret_token'],
235 # retry max N times
236 retries = Retry(
237 total=max_retries,
238 backoff_factor=0.15,
239 status_forcelist=[500, 502, 503, 504])
240 req_session = requests.Session()
241 req_session.mount(
242 'http://', requests.adapters.HTTPAdapter(max_retries=retries))
243
244 resp = req_session.post(url, json={
245 'token': token,
115 'event': data
246 'event': data
116 })
247 })
117 resp.raise_for_status() # raise exception on a failed request
248 resp.raise_for_status() # raise exception on a failed request
@@ -19,41 +19,12 b''
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 import pytest
21 import pytest
22 import requests
22 from mock import patch
23 from mock import Mock, patch
24
23
25 from rhodecode import events
24 from rhodecode import events
26 from rhodecode.model.db import Session, Integration
25 from rhodecode.model.db import Session, Integration
27 from rhodecode.integrations.types.slack import SlackIntegrationType
26 from rhodecode.integrations.types.slack import SlackIntegrationType
28
27
29 @pytest.fixture
30 def repo_push_event(backend, user_regular):
31 commits = [
32 {'message': 'ancestor commit fixes #15'},
33 {'message': 'quick fixes'},
34 {'message': 'change that fixes #41, #2'},
35 {'message': 'this is because 5b23c3532 broke stuff'},
36 {'message': 'last commit'},
37 ]
38 commit_ids = backend.create_master_repo(commits).values()
39 repo = backend.create_repo()
40 scm_extras = {
41 'ip': '127.0.0.1',
42 'username': user_regular.username,
43 'action': '',
44 'repository': repo.repo_name,
45 'scm': repo.scm_instance().alias,
46 'config': '',
47 'server_url': 'http://example.com',
48 'make_lock': None,
49 'locked_by': [None],
50 'commit_ids': commit_ids,
51 }
52
53 return events.RepoPushEvent(repo_name=repo.repo_name,
54 pushed_commit_ids=commit_ids,
55 extras=scm_extras)
56
57
28
58 @pytest.fixture
29 @pytest.fixture
59 def slack_settings():
30 def slack_settings():
@@ -21,6 +21,7 b' from pyramid.i18n import TranslationStri'
21 # Create a translation string factory for the 'rhodecode' domain.
21 # Create a translation string factory for the 'rhodecode' domain.
22 _ = TranslationStringFactory('rhodecode')
22 _ = TranslationStringFactory('rhodecode')
23
23
24
24 class LazyString(object):
25 class LazyString(object):
25 def __init__(self, *args, **kw):
26 def __init__(self, *args, **kw):
26 self.args = args
27 self.args = args
General Comments 0
You need to be logged in to leave comments. Login now