##// END OF EJS Templates
webhook: abstract webhook handler to base for easier re-usage.
marcink -
r2585:70fa3b0c default
parent child
Show More
@@ -19,8 +19,13
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 import colander
21 import colander
22 import string
23 import collections
24 import logging
22 from rhodecode.translation import _
25 from rhodecode.translation import _
23
26
27 log = logging.getLogger(__name__)
28
24
29
25 class IntegrationTypeBase(object):
30 class IntegrationTypeBase(object):
26 """ Base class for IntegrationType plugins """
31 """ Base class for IntegrationType plugins """
@@ -142,6 +147,122 WEBHOOK_URL_VARS = [
142 CI_URL_VARS = WEBHOOK_URL_VARS
147 CI_URL_VARS = WEBHOOK_URL_VARS
143
148
144
149
150 class WebhookDataHandler(object):
151 name = 'webhook'
152
153 def __init__(self, template_url, headers):
154 self.template_url = template_url
155 self.headers = headers
156
157 def get_base_parsed_template(self, data):
158 """
159 initially parses the passed in template with some common variables
160 available on ALL calls
161 """
162 # note: make sure to update the `WEBHOOK_URL_VARS` if this changes
163 common_vars = {
164 'repo_name': data['repo']['repo_name'],
165 'repo_type': data['repo']['repo_type'],
166 'repo_id': data['repo']['repo_id'],
167 'repo_url': data['repo']['url'],
168 'username': data['actor']['username'],
169 'user_id': data['actor']['user_id'],
170 'event_name': data['name']
171 }
172
173 extra_vars = {}
174 for extra_key, extra_val in data['repo']['extra_fields'].items():
175 extra_vars['extra__{}'.format(extra_key)] = extra_val
176 common_vars.update(extra_vars)
177
178 template_url = self.template_url.replace('${extra:', '${extra__')
179 return string.Template(template_url).safe_substitute(**common_vars)
180
181 def repo_push_event_handler(self, event, data):
182 url = self.get_base_parsed_template(data)
183 url_cals = []
184 branch_data = collections.OrderedDict()
185 for obj in data['push']['branches']:
186 branch_data[obj['name']] = obj
187
188 branches_commits = collections.OrderedDict()
189 for commit in data['push']['commits']:
190 if commit.get('git_ref_change'):
191 # special case for GIT that allows creating tags,
192 # deleting branches without associated commit
193 continue
194
195 if commit['branch'] not in branches_commits:
196 branch_commits = {'branch': branch_data[commit['branch']],
197 'commits': []}
198 branches_commits[commit['branch']] = branch_commits
199
200 branch_commits = branches_commits[commit['branch']]
201 branch_commits['commits'].append(commit)
202
203 if '${branch}' in url:
204 # call it multiple times, for each branch if used in variables
205 for branch, commit_ids in branches_commits.items():
206 branch_url = string.Template(url).safe_substitute(branch=branch)
207 # call further down for each commit if used
208 if '${commit_id}' in branch_url:
209 for commit_data in commit_ids['commits']:
210 commit_id = commit_data['raw_id']
211 commit_url = string.Template(branch_url).safe_substitute(
212 commit_id=commit_id)
213 # register per-commit call
214 log.debug(
215 'register %s call(%s) to url %s',
216 self.name, event, commit_url)
217 url_cals.append(
218 (commit_url, self.headers, data))
219
220 else:
221 # register per-branch call
222 log.debug(
223 'register %s call(%s) to url %s',
224 self.name, event, branch_url)
225 url_cals.append(
226 (branch_url, self.headers, data))
227
228 else:
229 log.debug(
230 'register %s call(%s) to url %s', self.name, event, url)
231 url_cals.append((url, self.headers, data))
232
233 return url_cals
234
235 def repo_create_event_handler(self, event, data):
236 url = self.get_base_parsed_template(data)
237 log.debug(
238 'register %s call(%s) to url %s', self.name, event, url)
239 return [(url, self.headers, data)]
240
241 def pull_request_event_handler(self, event, data):
242 url = self.get_base_parsed_template(data)
243 log.debug(
244 'register %s call(%s) to url %s', self.name, event, url)
245 url = string.Template(url).safe_substitute(
246 pull_request_id=data['pullrequest']['pull_request_id'],
247 pull_request_url=data['pullrequest']['url'],
248 pull_request_shadow_url=data['pullrequest']['shadow_url'],)
249 return [(url, self.headers, data)]
250
251 def __call__(self, event, data):
252 from rhodecode import events
253
254 if isinstance(event, events.RepoPushEvent):
255 return self.repo_push_event_handler(event, data)
256 elif isinstance(event, events.RepoCreateEvent):
257 return self.repo_create_event_handler(event, data)
258 elif isinstance(event, events.PullRequestEvent):
259 return self.pull_request_event_handler(event, data)
260 else:
261 raise ValueError(
262 'event type `%s` not in supported list: %s' % (
263 event.__class__, events))
264
265
145 def get_auth(settings):
266 def get_auth(settings):
146 from requests.auth import HTTPBasicAuth
267 from requests.auth import HTTPBasicAuth
147 username = settings.get('username')
268 username = settings.get('username')
@@ -151,6 +272,10 def get_auth(settings):
151 return None
272 return None
152
273
153
274
275 def get_web_token(settings):
276 return settings['secret_token']
277
278
154 def get_url_vars(url_vars):
279 def get_url_vars(url_vars):
155 return '\n'.join(
280 return '\n'.join(
156 '{} - {}'.format('${' + key + '}', explanation)
281 '{} - {}'.format('${' + key + '}', explanation)
@@ -19,8 +19,6
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 from __future__ import unicode_literals
21 from __future__ import unicode_literals
22 import string
23 import collections
24
22
25 import deform
23 import deform
26 import deform.widget
24 import deform.widget
@@ -34,7 +32,8 import rhodecode
34 from rhodecode import events
32 from rhodecode import events
35 from rhodecode.translation import _
33 from rhodecode.translation import _
36 from rhodecode.integrations.types.base import (
34 from rhodecode.integrations.types.base import (
37 IntegrationTypeBase, get_auth, get_url_vars, WEBHOOK_URL_VARS)
35 IntegrationTypeBase, get_auth, get_web_token, get_url_vars,
36 WebhookDataHandler, WEBHOOK_URL_VARS)
38 from rhodecode.lib.celerylib import run_task, async_task, RequestContextTask
37 from rhodecode.lib.celerylib import run_task, async_task, RequestContextTask
39 from rhodecode.model.validation_schema import widgets
38 from rhodecode.model.validation_schema import widgets
40
39
@@ -46,113 +45,6 log = logging.getLogger(__name__)
46 URL_VARS = get_url_vars(WEBHOOK_URL_VARS)
45 URL_VARS = get_url_vars(WEBHOOK_URL_VARS)
47
46
48
47
49 class WebhookHandler(object):
50 def __init__(self, template_url, secret_token, headers):
51 self.template_url = template_url
52 self.secret_token = secret_token
53 self.headers = headers
54
55 def get_base_parsed_template(self, data):
56 """
57 initially parses the passed in template with some common variables
58 available on ALL calls
59 """
60 # note: make sure to update the `WEBHOOK_URL_VARS` if this changes
61 common_vars = {
62 'repo_name': data['repo']['repo_name'],
63 'repo_type': data['repo']['repo_type'],
64 'repo_id': data['repo']['repo_id'],
65 'repo_url': data['repo']['url'],
66 'username': data['actor']['username'],
67 'user_id': data['actor']['user_id'],
68 'event_name': data['name']
69 }
70
71 extra_vars = {}
72 for extra_key, extra_val in data['repo']['extra_fields'].items():
73 extra_vars['extra__{}'.format(extra_key)] = extra_val
74 common_vars.update(extra_vars)
75
76 template_url = self.template_url.replace('${extra:', '${extra__')
77 return string.Template(template_url).safe_substitute(**common_vars)
78
79 def repo_push_event_handler(self, event, data):
80 url = self.get_base_parsed_template(data)
81 url_cals = []
82 branch_data = collections.OrderedDict()
83 for obj in data['push']['branches']:
84 branch_data[obj['name']] = obj
85
86 branches_commits = collections.OrderedDict()
87 for commit in data['push']['commits']:
88 if commit.get('git_ref_change'):
89 # special case for GIT that allows creating tags,
90 # deleting branches without associated commit
91 continue
92
93 if commit['branch'] not in branches_commits:
94 branch_commits = {'branch': branch_data[commit['branch']],
95 'commits': []}
96 branches_commits[commit['branch']] = branch_commits
97
98 branch_commits = branches_commits[commit['branch']]
99 branch_commits['commits'].append(commit)
100
101 if '${branch}' in url:
102 # call it multiple times, for each branch if used in variables
103 for branch, commit_ids in branches_commits.items():
104 branch_url = string.Template(url).safe_substitute(branch=branch)
105 # call further down for each commit if used
106 if '${commit_id}' in branch_url:
107 for commit_data in commit_ids['commits']:
108 commit_id = commit_data['raw_id']
109 commit_url = string.Template(branch_url).safe_substitute(
110 commit_id=commit_id)
111 # register per-commit call
112 log.debug(
113 'register webhook call(%s) to url %s', event, commit_url)
114 url_cals.append((commit_url, self.secret_token, self.headers, data))
115
116 else:
117 # register per-branch call
118 log.debug(
119 'register webhook call(%s) to url %s', event, branch_url)
120 url_cals.append((branch_url, self.secret_token, self.headers, data))
121
122 else:
123 log.debug(
124 'register webhook call(%s) to url %s', event, url)
125 url_cals.append((url, self.secret_token, self.headers, data))
126
127 return url_cals
128
129 def repo_create_event_handler(self, event, data):
130 url = self.get_base_parsed_template(data)
131 log.debug(
132 'register webhook call(%s) to url %s', event, url)
133 return [(url, self.secret_token, self.headers, data)]
134
135 def pull_request_event_handler(self, event, data):
136 url = self.get_base_parsed_template(data)
137 log.debug(
138 'register webhook call(%s) to url %s', event, url)
139 url = string.Template(url).safe_substitute(
140 pull_request_id=data['pullrequest']['pull_request_id'],
141 pull_request_url=data['pullrequest']['url'],
142 pull_request_shadow_url=data['pullrequest']['shadow_url'],)
143 return [(url, self.secret_token, self.headers, data)]
144
145 def __call__(self, event, data):
146 if isinstance(event, events.RepoPushEvent):
147 return self.repo_push_event_handler(event, data)
148 elif isinstance(event, events.RepoCreateEvent):
149 return self.repo_create_event_handler(event, data)
150 elif isinstance(event, events.PullRequestEvent):
151 return self.pull_request_event_handler(event, data)
152 else:
153 raise ValueError('event type not supported: %s' % events)
154
155
156 class WebhookSettingsSchema(colander.Schema):
48 class WebhookSettingsSchema(colander.Schema):
157 url = colander.SchemaNode(
49 url = colander.SchemaNode(
158 colander.String(),
50 colander.String(),
@@ -243,7 +135,7 class WebhookSettingsSchema(colander.Sch
243 class WebhookIntegrationType(IntegrationTypeBase):
135 class WebhookIntegrationType(IntegrationTypeBase):
244 key = 'webhook'
136 key = 'webhook'
245 display_name = _('Webhook')
137 display_name = _('Webhook')
246 description = _('Post json events to a Webhook endpoint')
138 description = _('send JSON data to a url endpoint')
247
139
248 @classmethod
140 @classmethod
249 def icon(cls):
141 def icon(cls):
@@ -275,8 +167,8 class WebhookIntegrationType(Integration
275 return schema
167 return schema
276
168
277 def send_event(self, event):
169 def send_event(self, event):
278 log.debug('handling event %s with Webhook integration %s',
170 log.debug(
279 event.name, self)
171 'handling event %s with Webhook integration %s', event.name, self)
280
172
281 if event.__class__ not in self.valid_events:
173 if event.__class__ not in self.valid_events:
282 log.debug('event not valid: %r' % event)
174 log.debug('event not valid: %r' % event)
@@ -295,8 +187,7 class WebhookIntegrationType(Integration
295 if head_key and head_val:
187 if head_key and head_val:
296 headers = {head_key: head_val}
188 headers = {head_key: head_val}
297
189
298 handler = WebhookHandler(
190 handler = WebhookDataHandler(template_url, headers)
299 template_url, self.settings['secret_token'], headers)
300
191
301 url_calls = handler(event, data)
192 url_calls = handler(event, data)
302 log.debug('webhook: calling following urls: %s',
193 log.debug('webhook: calling following urls: %s',
@@ -353,7 +244,9 def post_to_webhook(url_calls, settings)
353 } # updated below with custom ones, allows override
244 } # updated below with custom ones, allows override
354
245
355 auth = get_auth(settings)
246 auth = get_auth(settings)
356 for url, token, headers, data in url_calls:
247 token = get_web_token(settings)
248
249 for url, headers, data in url_calls:
357 req_session = requests.Session()
250 req_session = requests.Session()
358 req_session.mount( # retry max N times
251 req_session.mount( # retry max N times
359 'http://', requests.adapters.HTTPAdapter(max_retries=retries))
252 'http://', requests.adapters.HTTPAdapter(max_retries=retries))
@@ -22,12 +22,13 import pytest
22
22
23 from rhodecode import events
23 from rhodecode import events
24 from rhodecode.lib.utils2 import AttributeDict
24 from rhodecode.lib.utils2 import AttributeDict
25 from rhodecode.integrations.types.webhook import WebhookHandler
25 from rhodecode.integrations.types.webhook import WebhookDataHandler
26
26
27
27
28 @pytest.fixture
28 @pytest.fixture
29 def base_data():
29 def base_data():
30 return {
30 return {
31 'name': 'event',
31 'repo': {
32 'repo': {
32 'repo_name': 'foo',
33 'repo_name': 'foo',
33 'repo_type': 'hg',
34 'repo_type': 'hg',
@@ -44,31 +45,40 def base_data():
44
45
45 def test_webhook_parse_url_invalid_event():
46 def test_webhook_parse_url_invalid_event():
46 template_url = 'http://server.com/${repo_name}/build'
47 template_url = 'http://server.com/${repo_name}/build'
47 handler = WebhookHandler(
48 handler = WebhookDataHandler(
48 template_url, 'secret_token', {'exmaple-header':'header-values'})
49 template_url, {'exmaple-header': 'header-values'})
50 event = events.RepoDeleteEvent('')
49 with pytest.raises(ValueError) as err:
51 with pytest.raises(ValueError) as err:
50 handler(events.RepoDeleteEvent(''), {})
52 handler(event, {})
51 assert str(err.value).startswith('event type not supported')
53
54 err = str(err.value)
55 assert err.startswith(
56 'event type `%s` not in supported list' % event.__class__)
52
57
53
58
54 @pytest.mark.parametrize('template,expected_urls', [
59 @pytest.mark.parametrize('template,expected_urls', [
55 ('http://server.com/${repo_name}/build', ['http://server.com/foo/build']),
60 ('http://server.com/${repo_name}/build',
56 ('http://server.com/${repo_name}/${repo_type}', ['http://server.com/foo/hg']),
61 ['http://server.com/foo/build']),
57 ('http://${server}.com/${repo_name}/${repo_id}', ['http://${server}.com/foo/12']),
62 ('http://server.com/${repo_name}/${repo_type}',
58 ('http://server.com/${branch}/build', ['http://server.com/${branch}/build']),
63 ['http://server.com/foo/hg']),
64 ('http://${server}.com/${repo_name}/${repo_id}',
65 ['http://${server}.com/foo/12']),
66 ('http://server.com/${branch}/build',
67 ['http://server.com/${branch}/build']),
59 ])
68 ])
60 def test_webook_parse_url_for_create_event(base_data, template, expected_urls):
69 def test_webook_parse_url_for_create_event(base_data, template, expected_urls):
61 headers = {'exmaple-header': 'header-values'}
70 headers = {'exmaple-header': 'header-values'}
62 handler = WebhookHandler(
71 handler = WebhookDataHandler(template, headers)
63 template, 'secret_token', headers)
64 urls = handler(events.RepoCreateEvent(''), base_data)
72 urls = handler(events.RepoCreateEvent(''), base_data)
65 assert urls == [
73 assert urls == [
66 (url, 'secret_token', headers, base_data) for url in expected_urls]
74 (url, headers, base_data) for url in expected_urls]
67
75
68
76
69 @pytest.mark.parametrize('template,expected_urls', [
77 @pytest.mark.parametrize('template,expected_urls', [
70 ('http://server.com/${repo_name}/${pull_request_id}', ['http://server.com/foo/999']),
78 ('http://server.com/${repo_name}/${pull_request_id}',
71 ('http://server.com/${repo_name}/${pull_request_url}', ['http://server.com/foo/http://pr-url.com']),
79 ['http://server.com/foo/999']),
80 ('http://server.com/${repo_name}/${pull_request_url}',
81 ['http://server.com/foo/http://pr-url.com']),
72 ])
82 ])
73 def test_webook_parse_url_for_pull_request_event(
83 def test_webook_parse_url_for_pull_request_event(
74 base_data, template, expected_urls):
84 base_data, template, expected_urls):
@@ -76,23 +86,25 def test_webook_parse_url_for_pull_reque
76 base_data['pullrequest'] = {
86 base_data['pullrequest'] = {
77 'pull_request_id': 999,
87 'pull_request_id': 999,
78 'url': 'http://pr-url.com',
88 'url': 'http://pr-url.com',
89 'shadow_url': 'http://pr-url.com/repository'
79 }
90 }
80 headers = {'exmaple-header': 'header-values'}
91 headers = {'exmaple-header': 'header-values'}
81 handler = WebhookHandler(
92 handler = WebhookDataHandler(template, headers)
82 template, 'secret_token', headers)
83 urls = handler(events.PullRequestCreateEvent(
93 urls = handler(events.PullRequestCreateEvent(
84 AttributeDict({'target_repo': 'foo'})), base_data)
94 AttributeDict({'target_repo': 'foo'})), base_data)
85 assert urls == [
95 assert urls == [
86 (url, 'secret_token', headers, base_data) for url in expected_urls]
96 (url, headers, base_data) for url in expected_urls]
87
97
88
98
89 @pytest.mark.parametrize('template,expected_urls', [
99 @pytest.mark.parametrize('template,expected_urls', [
90 ('http://server.com/${branch}/build', ['http://server.com/stable/build',
100 ('http://server.com/${branch}/build',
91 'http://server.com/dev/build']),
101 ['http://server.com/stable/build',
92 ('http://server.com/${branch}/${commit_id}', ['http://server.com/stable/stable-xxx',
102 'http://server.com/dev/build']),
93 'http://server.com/stable/stable-yyy',
103 ('http://server.com/${branch}/${commit_id}',
94 'http://server.com/dev/dev-xxx',
104 ['http://server.com/stable/stable-xxx',
95 'http://server.com/dev/dev-yyy']),
105 'http://server.com/stable/stable-yyy',
106 'http://server.com/dev/dev-xxx',
107 'http://server.com/dev/dev-yyy']),
96 ])
108 ])
97 def test_webook_parse_url_for_push_event(
109 def test_webook_parse_url_for_push_event(
98 baseapp, repo_push_event, base_data, template, expected_urls):
110 baseapp, repo_push_event, base_data, template, expected_urls):
@@ -104,8 +116,7 def test_webook_parse_url_for_push_event
104 {'branch': 'dev', 'raw_id': 'dev-yyy'}]
116 {'branch': 'dev', 'raw_id': 'dev-yyy'}]
105 }
117 }
106 headers = {'exmaple-header': 'header-values'}
118 headers = {'exmaple-header': 'header-values'}
107 handler = WebhookHandler(
119 handler = WebhookDataHandler(template, headers)
108 template, 'secret_token', headers)
109 urls = handler(repo_push_event, base_data)
120 urls = handler(repo_push_event, base_data)
110 assert urls == [
121 assert urls == [
111 (url, 'secret_token', headers, base_data) for url in expected_urls]
122 (url, headers, base_data) for url in expected_urls]
General Comments 0
You need to be logged in to leave comments. Login now