##// END OF EJS Templates
webhook: abstract webhook handler to base for easier re-usage.
marcink -
r2585:70fa3b0c default
parent child Browse files
Show More
@@ -19,8 +19,13 b''
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20
21 21 import colander
22 import string
23 import collections
24 import logging
22 25 from rhodecode.translation import _
23 26
27 log = logging.getLogger(__name__)
28
24 29
25 30 class IntegrationTypeBase(object):
26 31 """ Base class for IntegrationType plugins """
@@ -142,6 +147,122 b' WEBHOOK_URL_VARS = ['
142 147 CI_URL_VARS = WEBHOOK_URL_VARS
143 148
144 149
150 class WebhookDataHandler(object):
151 name = 'webhook'
152
153 def __init__(self, template_url, headers):
154 self.template_url = template_url
155 self.headers = headers
156
157 def get_base_parsed_template(self, data):
158 """
159 initially parses the passed in template with some common variables
160 available on ALL calls
161 """
162 # note: make sure to update the `WEBHOOK_URL_VARS` if this changes
163 common_vars = {
164 'repo_name': data['repo']['repo_name'],
165 'repo_type': data['repo']['repo_type'],
166 'repo_id': data['repo']['repo_id'],
167 'repo_url': data['repo']['url'],
168 'username': data['actor']['username'],
169 'user_id': data['actor']['user_id'],
170 'event_name': data['name']
171 }
172
173 extra_vars = {}
174 for extra_key, extra_val in data['repo']['extra_fields'].items():
175 extra_vars['extra__{}'.format(extra_key)] = extra_val
176 common_vars.update(extra_vars)
177
178 template_url = self.template_url.replace('${extra:', '${extra__')
179 return string.Template(template_url).safe_substitute(**common_vars)
180
181 def repo_push_event_handler(self, event, data):
182 url = self.get_base_parsed_template(data)
183 url_cals = []
184 branch_data = collections.OrderedDict()
185 for obj in data['push']['branches']:
186 branch_data[obj['name']] = obj
187
188 branches_commits = collections.OrderedDict()
189 for commit in data['push']['commits']:
190 if commit.get('git_ref_change'):
191 # special case for GIT that allows creating tags,
192 # deleting branches without associated commit
193 continue
194
195 if commit['branch'] not in branches_commits:
196 branch_commits = {'branch': branch_data[commit['branch']],
197 'commits': []}
198 branches_commits[commit['branch']] = branch_commits
199
200 branch_commits = branches_commits[commit['branch']]
201 branch_commits['commits'].append(commit)
202
203 if '${branch}' in url:
204 # call it multiple times, for each branch if used in variables
205 for branch, commit_ids in branches_commits.items():
206 branch_url = string.Template(url).safe_substitute(branch=branch)
207 # call further down for each commit if used
208 if '${commit_id}' in branch_url:
209 for commit_data in commit_ids['commits']:
210 commit_id = commit_data['raw_id']
211 commit_url = string.Template(branch_url).safe_substitute(
212 commit_id=commit_id)
213 # register per-commit call
214 log.debug(
215 'register %s call(%s) to url %s',
216 self.name, event, commit_url)
217 url_cals.append(
218 (commit_url, self.headers, data))
219
220 else:
221 # register per-branch call
222 log.debug(
223 'register %s call(%s) to url %s',
224 self.name, event, branch_url)
225 url_cals.append(
226 (branch_url, self.headers, data))
227
228 else:
229 log.debug(
230 'register %s call(%s) to url %s', self.name, event, url)
231 url_cals.append((url, self.headers, data))
232
233 return url_cals
234
235 def repo_create_event_handler(self, event, data):
236 url = self.get_base_parsed_template(data)
237 log.debug(
238 'register %s call(%s) to url %s', self.name, event, url)
239 return [(url, self.headers, data)]
240
241 def pull_request_event_handler(self, event, data):
242 url = self.get_base_parsed_template(data)
243 log.debug(
244 'register %s call(%s) to url %s', self.name, event, url)
245 url = string.Template(url).safe_substitute(
246 pull_request_id=data['pullrequest']['pull_request_id'],
247 pull_request_url=data['pullrequest']['url'],
248 pull_request_shadow_url=data['pullrequest']['shadow_url'],)
249 return [(url, self.headers, data)]
250
251 def __call__(self, event, data):
252 from rhodecode import events
253
254 if isinstance(event, events.RepoPushEvent):
255 return self.repo_push_event_handler(event, data)
256 elif isinstance(event, events.RepoCreateEvent):
257 return self.repo_create_event_handler(event, data)
258 elif isinstance(event, events.PullRequestEvent):
259 return self.pull_request_event_handler(event, data)
260 else:
261 raise ValueError(
262 'event type `%s` not in supported list: %s' % (
263 event.__class__, events))
264
265
145 266 def get_auth(settings):
146 267 from requests.auth import HTTPBasicAuth
147 268 username = settings.get('username')
@@ -151,6 +272,10 b' def get_auth(settings):'
151 272 return None
152 273
153 274
275 def get_web_token(settings):
276 return settings['secret_token']
277
278
154 279 def get_url_vars(url_vars):
155 280 return '\n'.join(
156 281 '{} - {}'.format('${' + key + '}', explanation)
@@ -19,8 +19,6 b''
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20
21 21 from __future__ import unicode_literals
22 import string
23 import collections
24 22
25 23 import deform
26 24 import deform.widget
@@ -34,7 +32,8 b' import rhodecode'
34 32 from rhodecode import events
35 33 from rhodecode.translation import _
36 34 from rhodecode.integrations.types.base import (
37 IntegrationTypeBase, get_auth, get_url_vars, WEBHOOK_URL_VARS)
35 IntegrationTypeBase, get_auth, get_web_token, get_url_vars,
36 WebhookDataHandler, WEBHOOK_URL_VARS)
38 37 from rhodecode.lib.celerylib import run_task, async_task, RequestContextTask
39 38 from rhodecode.model.validation_schema import widgets
40 39
@@ -46,113 +45,6 b' log = logging.getLogger(__name__)'
46 45 URL_VARS = get_url_vars(WEBHOOK_URL_VARS)
47 46
48 47
49 class WebhookHandler(object):
50 def __init__(self, template_url, secret_token, headers):
51 self.template_url = template_url
52 self.secret_token = secret_token
53 self.headers = headers
54
55 def get_base_parsed_template(self, data):
56 """
57 initially parses the passed in template with some common variables
58 available on ALL calls
59 """
60 # note: make sure to update the `WEBHOOK_URL_VARS` if this changes
61 common_vars = {
62 'repo_name': data['repo']['repo_name'],
63 'repo_type': data['repo']['repo_type'],
64 'repo_id': data['repo']['repo_id'],
65 'repo_url': data['repo']['url'],
66 'username': data['actor']['username'],
67 'user_id': data['actor']['user_id'],
68 'event_name': data['name']
69 }
70
71 extra_vars = {}
72 for extra_key, extra_val in data['repo']['extra_fields'].items():
73 extra_vars['extra__{}'.format(extra_key)] = extra_val
74 common_vars.update(extra_vars)
75
76 template_url = self.template_url.replace('${extra:', '${extra__')
77 return string.Template(template_url).safe_substitute(**common_vars)
78
79 def repo_push_event_handler(self, event, data):
80 url = self.get_base_parsed_template(data)
81 url_cals = []
82 branch_data = collections.OrderedDict()
83 for obj in data['push']['branches']:
84 branch_data[obj['name']] = obj
85
86 branches_commits = collections.OrderedDict()
87 for commit in data['push']['commits']:
88 if commit.get('git_ref_change'):
89 # special case for GIT that allows creating tags,
90 # deleting branches without associated commit
91 continue
92
93 if commit['branch'] not in branches_commits:
94 branch_commits = {'branch': branch_data[commit['branch']],
95 'commits': []}
96 branches_commits[commit['branch']] = branch_commits
97
98 branch_commits = branches_commits[commit['branch']]
99 branch_commits['commits'].append(commit)
100
101 if '${branch}' in url:
102 # call it multiple times, for each branch if used in variables
103 for branch, commit_ids in branches_commits.items():
104 branch_url = string.Template(url).safe_substitute(branch=branch)
105 # call further down for each commit if used
106 if '${commit_id}' in branch_url:
107 for commit_data in commit_ids['commits']:
108 commit_id = commit_data['raw_id']
109 commit_url = string.Template(branch_url).safe_substitute(
110 commit_id=commit_id)
111 # register per-commit call
112 log.debug(
113 'register webhook call(%s) to url %s', event, commit_url)
114 url_cals.append((commit_url, self.secret_token, self.headers, data))
115
116 else:
117 # register per-branch call
118 log.debug(
119 'register webhook call(%s) to url %s', event, branch_url)
120 url_cals.append((branch_url, self.secret_token, self.headers, data))
121
122 else:
123 log.debug(
124 'register webhook call(%s) to url %s', event, url)
125 url_cals.append((url, self.secret_token, self.headers, data))
126
127 return url_cals
128
129 def repo_create_event_handler(self, event, data):
130 url = self.get_base_parsed_template(data)
131 log.debug(
132 'register webhook call(%s) to url %s', event, url)
133 return [(url, self.secret_token, self.headers, data)]
134
135 def pull_request_event_handler(self, event, data):
136 url = self.get_base_parsed_template(data)
137 log.debug(
138 'register webhook call(%s) to url %s', event, url)
139 url = string.Template(url).safe_substitute(
140 pull_request_id=data['pullrequest']['pull_request_id'],
141 pull_request_url=data['pullrequest']['url'],
142 pull_request_shadow_url=data['pullrequest']['shadow_url'],)
143 return [(url, self.secret_token, self.headers, data)]
144
145 def __call__(self, event, data):
146 if isinstance(event, events.RepoPushEvent):
147 return self.repo_push_event_handler(event, data)
148 elif isinstance(event, events.RepoCreateEvent):
149 return self.repo_create_event_handler(event, data)
150 elif isinstance(event, events.PullRequestEvent):
151 return self.pull_request_event_handler(event, data)
152 else:
153 raise ValueError('event type not supported: %s' % events)
154
155
156 48 class WebhookSettingsSchema(colander.Schema):
157 49 url = colander.SchemaNode(
158 50 colander.String(),
@@ -243,7 +135,7 b' class WebhookSettingsSchema(colander.Sch'
243 135 class WebhookIntegrationType(IntegrationTypeBase):
244 136 key = 'webhook'
245 137 display_name = _('Webhook')
246 description = _('Post json events to a Webhook endpoint')
138 description = _('send JSON data to a url endpoint')
247 139
248 140 @classmethod
249 141 def icon(cls):
@@ -275,8 +167,8 b' class WebhookIntegrationType(Integration'
275 167 return schema
276 168
277 169 def send_event(self, event):
278 log.debug('handling event %s with Webhook integration %s',
279 event.name, self)
170 log.debug(
171 'handling event %s with Webhook integration %s', event.name, self)
280 172
281 173 if event.__class__ not in self.valid_events:
282 174 log.debug('event not valid: %r' % event)
@@ -295,8 +187,7 b' class WebhookIntegrationType(Integration'
295 187 if head_key and head_val:
296 188 headers = {head_key: head_val}
297 189
298 handler = WebhookHandler(
299 template_url, self.settings['secret_token'], headers)
190 handler = WebhookDataHandler(template_url, headers)
300 191
301 192 url_calls = handler(event, data)
302 193 log.debug('webhook: calling following urls: %s',
@@ -353,7 +244,9 b' def post_to_webhook(url_calls, settings)'
353 244 } # updated below with custom ones, allows override
354 245
355 246 auth = get_auth(settings)
356 for url, token, headers, data in url_calls:
247 token = get_web_token(settings)
248
249 for url, headers, data in url_calls:
357 250 req_session = requests.Session()
358 251 req_session.mount( # retry max N times
359 252 'http://', requests.adapters.HTTPAdapter(max_retries=retries))
@@ -22,12 +22,13 b' import pytest'
22 22
23 23 from rhodecode import events
24 24 from rhodecode.lib.utils2 import AttributeDict
25 from rhodecode.integrations.types.webhook import WebhookHandler
25 from rhodecode.integrations.types.webhook import WebhookDataHandler
26 26
27 27
28 28 @pytest.fixture
29 29 def base_data():
30 30 return {
31 'name': 'event',
31 32 'repo': {
32 33 'repo_name': 'foo',
33 34 'repo_type': 'hg',
@@ -44,31 +45,40 b' def base_data():'
44 45
45 46 def test_webhook_parse_url_invalid_event():
46 47 template_url = 'http://server.com/${repo_name}/build'
47 handler = WebhookHandler(
48 template_url, 'secret_token', {'exmaple-header':'header-values'})
48 handler = WebhookDataHandler(
49 template_url, {'exmaple-header': 'header-values'})
50 event = events.RepoDeleteEvent('')
49 51 with pytest.raises(ValueError) as err:
50 handler(events.RepoDeleteEvent(''), {})
51 assert str(err.value).startswith('event type not supported')
52 handler(event, {})
53
54 err = str(err.value)
55 assert err.startswith(
56 'event type `%s` not in supported list' % event.__class__)
52 57
53 58
54 59 @pytest.mark.parametrize('template,expected_urls', [
55 ('http://server.com/${repo_name}/build', ['http://server.com/foo/build']),
56 ('http://server.com/${repo_name}/${repo_type}', ['http://server.com/foo/hg']),
57 ('http://${server}.com/${repo_name}/${repo_id}', ['http://${server}.com/foo/12']),
58 ('http://server.com/${branch}/build', ['http://server.com/${branch}/build']),
60 ('http://server.com/${repo_name}/build',
61 ['http://server.com/foo/build']),
62 ('http://server.com/${repo_name}/${repo_type}',
63 ['http://server.com/foo/hg']),
64 ('http://${server}.com/${repo_name}/${repo_id}',
65 ['http://${server}.com/foo/12']),
66 ('http://server.com/${branch}/build',
67 ['http://server.com/${branch}/build']),
59 68 ])
60 69 def test_webook_parse_url_for_create_event(base_data, template, expected_urls):
61 70 headers = {'exmaple-header': 'header-values'}
62 handler = WebhookHandler(
63 template, 'secret_token', headers)
71 handler = WebhookDataHandler(template, headers)
64 72 urls = handler(events.RepoCreateEvent(''), base_data)
65 73 assert urls == [
66 (url, 'secret_token', headers, base_data) for url in expected_urls]
74 (url, headers, base_data) for url in expected_urls]
67 75
68 76
69 77 @pytest.mark.parametrize('template,expected_urls', [
70 ('http://server.com/${repo_name}/${pull_request_id}', ['http://server.com/foo/999']),
71 ('http://server.com/${repo_name}/${pull_request_url}', ['http://server.com/foo/http://pr-url.com']),
78 ('http://server.com/${repo_name}/${pull_request_id}',
79 ['http://server.com/foo/999']),
80 ('http://server.com/${repo_name}/${pull_request_url}',
81 ['http://server.com/foo/http://pr-url.com']),
72 82 ])
73 83 def test_webook_parse_url_for_pull_request_event(
74 84 base_data, template, expected_urls):
@@ -76,20 +86,22 b' def test_webook_parse_url_for_pull_reque'
76 86 base_data['pullrequest'] = {
77 87 'pull_request_id': 999,
78 88 'url': 'http://pr-url.com',
89 'shadow_url': 'http://pr-url.com/repository'
79 90 }
80 91 headers = {'exmaple-header': 'header-values'}
81 handler = WebhookHandler(
82 template, 'secret_token', headers)
92 handler = WebhookDataHandler(template, headers)
83 93 urls = handler(events.PullRequestCreateEvent(
84 94 AttributeDict({'target_repo': 'foo'})), base_data)
85 95 assert urls == [
86 (url, 'secret_token', headers, base_data) for url in expected_urls]
96 (url, headers, base_data) for url in expected_urls]
87 97
88 98
89 99 @pytest.mark.parametrize('template,expected_urls', [
90 ('http://server.com/${branch}/build', ['http://server.com/stable/build',
100 ('http://server.com/${branch}/build',
101 ['http://server.com/stable/build',
91 102 'http://server.com/dev/build']),
92 ('http://server.com/${branch}/${commit_id}', ['http://server.com/stable/stable-xxx',
103 ('http://server.com/${branch}/${commit_id}',
104 ['http://server.com/stable/stable-xxx',
93 105 'http://server.com/stable/stable-yyy',
94 106 'http://server.com/dev/dev-xxx',
95 107 'http://server.com/dev/dev-yyy']),
@@ -104,8 +116,7 b' def test_webook_parse_url_for_push_event'
104 116 {'branch': 'dev', 'raw_id': 'dev-yyy'}]
105 117 }
106 118 headers = {'exmaple-header': 'header-values'}
107 handler = WebhookHandler(
108 template, 'secret_token', headers)
119 handler = WebhookDataHandler(template, headers)
109 120 urls = handler(repo_push_event, base_data)
110 121 assert urls == [
111 (url, 'secret_token', headers, base_data) for url in expected_urls]
122 (url, headers, base_data) for url in expected_urls]
General Comments 0
You need to be logged in to leave comments. Login now