##// END OF EJS Templates
integrations: parse pushed tags, and lightweight tags for git....
marcink -
r2422:ac36fdfd default
parent child
Show More
@@ -0,0 +1,141
1 # -*- coding: utf-8 -*-
2
3 # Copyright (C) 2010-2017 RhodeCode GmbH
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
21 """
22 Test suite for making push/pull operations, on specially modified INI files
23
24 .. important::
25
26 You must have git >= 1.8.5 for tests to work fine. With 68b939b git started
27 to redirect things to stderr instead of stdout.
28 """
29
30 import pytest
31 import requests
32
33 from rhodecode import events
34 from rhodecode.model.db import Integration
35 from rhodecode.model.integration import IntegrationModel
36 from rhodecode.model.meta import Session
37
38 from rhodecode.tests import GIT_REPO, HG_REPO
39 from rhodecode.tests.other.vcs_operations import Command, _add_files_and_push
40 from rhodecode.integrations.types.webhook import WebhookIntegrationType
41
42
43 def check_connection():
44 try:
45 response = requests.get('http://httpbin.org')
46 return response.status_code == 200
47 except Exception as e:
48 print(e)
49
50 return False
51
52
53 connection_available = pytest.mark.skipif(
54 not check_connection(), reason="No outside internet connection available")
55
56
57 @pytest.fixture
58 def enable_webhook_push_integration(request):
59 integration = Integration()
60 integration.integration_type = WebhookIntegrationType.key
61 Session().add(integration)
62
63 settings = dict(
64 url='http://httpbin.org',
65 secret_token='secret',
66 username=None,
67 password=None,
68 custom_header_key=None,
69 custom_header_val=None,
70 method_type='get',
71 events=[events.RepoPushEvent.name],
72 log_data=True
73 )
74
75 IntegrationModel().update_integration(
76 integration,
77 name='IntegrationWebhookTest',
78 enabled=True,
79 settings=settings,
80 repo=None,
81 repo_group=None,
82 child_repos_only=False,
83 )
84 Session().commit()
85 integration_id = integration.integration_id
86
87 @request.addfinalizer
88 def cleanup():
89 integration = Integration.get(integration_id)
90 Session().delete(integration)
91 Session().commit()
92
93
94 @pytest.mark.usefixtures(
95 "disable_locking", "disable_anonymous_user",
96 "enable_webhook_push_integration")
97 class TestVCSOperationsOnCustomIniConfig(object):
98
99 def test_push_tag_with_commit_hg(self, rc_web_server, tmpdir):
100 clone_url = rc_web_server.repo_clone_url(HG_REPO)
101 stdout, stderr = Command('/tmp').execute(
102 'hg clone', clone_url, tmpdir.strpath)
103
104 push_url = rc_web_server.repo_clone_url(HG_REPO)
105 _add_files_and_push(
106 'hg', tmpdir.strpath, clone_url=push_url,
107 tags=[{'name': 'v1.0.0', 'commit': 'added tag v1.0.0'}])
108
109 rc_log = rc_web_server.get_rc_log()
110 assert 'ERROR' not in rc_log
111 assert "'name': u'v1.0.0'" in rc_log
112
113 def test_push_tag_with_commit_git(
114 self, rc_web_server, tmpdir):
115 clone_url = rc_web_server.repo_clone_url(GIT_REPO)
116 stdout, stderr = Command('/tmp').execute(
117 'git clone', clone_url, tmpdir.strpath)
118
119 push_url = rc_web_server.repo_clone_url(GIT_REPO)
120 _add_files_and_push(
121 'git', tmpdir.strpath, clone_url=push_url,
122 tags=[{'name': 'v1.0.0', 'commit': 'added tag v1.0.0'}])
123
124 rc_log = rc_web_server.get_rc_log()
125 assert 'ERROR' not in rc_log
126 assert "'name': u'v1.0.0'" in rc_log
127
128 def test_push_tag_with_no_commit_git(
129 self, rc_web_server, tmpdir):
130 clone_url = rc_web_server.repo_clone_url(GIT_REPO)
131 stdout, stderr = Command('/tmp').execute(
132 'git clone', clone_url, tmpdir.strpath)
133
134 push_url = rc_web_server.repo_clone_url(GIT_REPO)
135 _add_files_and_push(
136 'git', tmpdir.strpath, clone_url=push_url,
137 tags=[{'name': 'v1.0.0', 'commit': 'added tag v1.0.0'}])
138
139 rc_log = rc_web_server.get_rc_log()
140 assert 'ERROR' not in rc_log
141 assert "'name': u'v1.0.0'" in rc_log
@@ -18,6 +18,7
18
18
19 import collections
19 import collections
20 import logging
20 import logging
21 import datetime
21
22
22 from rhodecode.translation import lazy_ugettext
23 from rhodecode.translation import lazy_ugettext
23 from rhodecode.model.db import User, Repository, Session
24 from rhodecode.model.db import User, Repository, Session
@@ -58,22 +59,60 def _commits_as_dict(event, commit_ids,
58 return commits # return early if we have the commits we need
59 return commits # return early if we have the commits we need
59
60
60 vcs_repo = repo.scm_instance(cache=False)
61 vcs_repo = repo.scm_instance(cache=False)
62
61 try:
63 try:
62 # use copy of needed_commits since we modify it while iterating
64 # use copy of needed_commits since we modify it while iterating
63 for commit_id in list(needed_commits):
65 for commit_id in list(needed_commits):
64 try:
66 if commit_id.startswith('tag=>'):
65 cs = vcs_repo.get_changeset(commit_id)
67 raw_id = commit_id[5:]
66 except CommitDoesNotExistError:
68 cs_data = {
67 continue # maybe its in next repo
69 'raw_id': commit_id, 'short_id': commit_id,
70 'branch': None,
71 'git_ref_change': 'tag_add',
72 'message': 'Added new tag {}'.format(raw_id),
73 'author': event.actor.full_contact,
74 'date': datetime.datetime.now(),
75 'refs': {
76 'branches': [],
77 'bookmarks': [],
78 'tags': []
79 }
80 }
81 commits.append(cs_data)
68
82
69 cs_data = cs.__json__()
83 elif commit_id.startswith('delete_branch=>'):
70 cs_data['refs'] = cs._get_refs()
84 raw_id = commit_id[15:]
85 cs_data = {
86 'raw_id': commit_id, 'short_id': commit_id,
87 'branch': None,
88 'git_ref_change': 'branch_delete',
89 'message': 'Deleted branch {}'.format(raw_id),
90 'author': event.actor.full_contact,
91 'date': datetime.datetime.now(),
92 'refs': {
93 'branches': [],
94 'bookmarks': [],
95 'tags': []
96 }
97 }
98 commits.append(cs_data)
99
100 else:
101 try:
102 cs = vcs_repo.get_changeset(commit_id)
103 except CommitDoesNotExistError:
104 continue # maybe its in next repo
105
106 cs_data = cs.__json__()
107 cs_data['refs'] = cs._get_refs()
108
71 cs_data['mentions'] = extract_mentioned_users(cs_data['message'])
109 cs_data['mentions'] = extract_mentioned_users(cs_data['message'])
72 cs_data['reviewers'] = reviewers
110 cs_data['reviewers'] = reviewers
73 cs_data['url'] = RepoModel().get_commit_url(
111 cs_data['url'] = RepoModel().get_commit_url(
74 repo, cs_data['raw_id'], request=event.request)
112 repo, cs_data['raw_id'], request=event.request)
75 cs_data['permalink_url'] = RepoModel().get_commit_url(
113 cs_data['permalink_url'] = RepoModel().get_commit_url(
76 repo, cs_data['raw_id'], request=event.request, permalink=True)
114 repo, cs_data['raw_id'], request=event.request,
115 permalink=True)
77 urlified_message, issues_data = process_patterns(
116 urlified_message, issues_data = process_patterns(
78 cs_data['message'], repo.repo_name)
117 cs_data['message'], repo.repo_name)
79 cs_data['issues'] = issues_data
118 cs_data['issues'] = issues_data
@@ -85,8 +124,8 def _commits_as_dict(event, commit_ids,
85
124
86 needed_commits.remove(commit_id)
125 needed_commits.remove(commit_id)
87
126
88 except Exception as e:
127 except Exception:
89 log.exception(e)
128 log.exception('Failed to extract commits data')
90 # we don't send any commits when crash happens, only full list
129 # we don't send any commits when crash happens, only full list
91 # matters we short circuit then.
130 # matters we short circuit then.
92 return []
131 return []
@@ -248,6 +287,7 class RepoPushEvent(RepoVCSEvent):
248 def __init__(self, repo_name, pushed_commit_ids, extras):
287 def __init__(self, repo_name, pushed_commit_ids, extras):
249 super(RepoPushEvent, self).__init__(repo_name, extras)
288 super(RepoPushEvent, self).__init__(repo_name, extras)
250 self.pushed_commit_ids = pushed_commit_ids
289 self.pushed_commit_ids = pushed_commit_ids
290 self.new_refs = extras.new_refs
251
291
252 def as_dict(self):
292 def as_dict(self):
253 data = super(RepoPushEvent, self).as_dict()
293 data = super(RepoPushEvent, self).as_dict()
@@ -256,6 +296,10 class RepoPushEvent(RepoVCSEvent):
256 return '{}/changelog?branch={}'.format(
296 return '{}/changelog?branch={}'.format(
257 data['repo']['url'], branch_name)
297 data['repo']['url'], branch_name)
258
298
299 def tag_url(tag_name):
300 return '{}/files/{}/'.format(
301 data['repo']['url'], tag_name)
302
259 commits = _commits_as_dict(
303 commits = _commits_as_dict(
260 self, commit_ids=self.pushed_commit_ids, repos=[self.repo])
304 self, commit_ids=self.pushed_commit_ids, repos=[self.repo])
261
305
@@ -265,8 +309,21 class RepoPushEvent(RepoVCSEvent):
265 last_branch = commit['branch']
309 last_branch = commit['branch']
266 issues = _issues_as_dict(commits)
310 issues = _issues_as_dict(commits)
267
311
268 branches = set(
312 branches = set()
269 commit['branch'] for commit in commits if commit['branch'])
313 tags = set()
314 for commit in commits:
315 if commit['refs']['tags']:
316 for tag in commit['refs']['tags']:
317 tags.add(tag)
318 if commit['branch']:
319 branches.add(commit['branch'])
320
321 # maybe we have branches in new_refs ?
322 try:
323 branches = branches.union(set(self.new_refs['branches']))
324 except Exception:
325 pass
326
270 branches = [
327 branches = [
271 {
328 {
272 'name': branch,
329 'name': branch,
@@ -275,9 +332,24 class RepoPushEvent(RepoVCSEvent):
275 for branch in branches
332 for branch in branches
276 ]
333 ]
277
334
335 # maybe we have branches in new_refs ?
336 try:
337 tags = tags.union(set(self.new_refs['tags']))
338 except Exception:
339 pass
340
341 tags = [
342 {
343 'name': tag,
344 'url': tag_url(tag)
345 }
346 for tag in tags
347 ]
348
278 data['push'] = {
349 data['push'] = {
279 'commits': commits,
350 'commits': commits,
280 'issues': issues,
351 'issues': issues,
281 'branches': branches,
352 'branches': branches,
353 'tags': tags,
282 }
354 }
283 return data
355 return data
@@ -110,6 +110,11 class WebhookHandler(object):
110
110
111 branches_commits = OrderedDict()
111 branches_commits = OrderedDict()
112 for commit in data['push']['commits']:
112 for commit in data['push']['commits']:
113 if commit.get('git_ref_change'):
114 # special case for GIT that allows creating tags,
115 # deleting branches without associated commit
116 continue
117
113 if commit['branch'] not in branches_commits:
118 if commit['branch'] not in branches_commits:
114 branch_commits = {'branch': branch_data[commit['branch']],
119 branch_commits = {'branch': branch_data[commit['branch']],
115 'commits': []}
120 'commits': []}
@@ -378,7 +383,8 def post_to_webhook(url_calls, settings)
378
383
379 log.debug('calling Webhook with method: %s, and auth:%s',
384 log.debug('calling Webhook with method: %s, and auth:%s',
380 call_method, auth)
385 call_method, auth)
381
386 if settings.get('log_data'):
387 log.debug('calling webhook with data: %s', data)
382 resp = call_method(url, json={
388 resp = call_method(url, json={
383 'token': token,
389 'token': token,
384 'event': data
390 'event': data
@@ -404,7 +404,6 class RepoIntegrationsView(IntegrationSe
404 c.repo_name = self.db_repo.repo_name
404 c.repo_name = self.db_repo.repo_name
405 c.repository_pull_requests = ScmModel().get_pull_requests(self.repo)
405 c.repository_pull_requests = ScmModel().get_pull_requests(self.repo)
406
406
407
408 return c
407 return c
409
408
410 @LoginRequired()
409 @LoginRequired()
@@ -198,7 +198,6 class IntegrationOptionsSchemaBase(colan
198 )
198 )
199
199
200
200
201
202 def make_integration_schema(IntegrationType, settings=None):
201 def make_integration_schema(IntegrationType, settings=None):
203 """
202 """
204 Return a colander schema for an integration type
203 Return a colander schema for an integration type
@@ -21,6 +21,7
21
21
22 import pytest
22 import pytest
23 from rhodecode import events
23 from rhodecode import events
24 from rhodecode.lib.utils2 import AttributeDict
24
25
25
26
26 @pytest.fixture
27 @pytest.fixture
@@ -34,7 +35,7 def repo_push_event(backend, user_regula
34 ]
35 ]
35 commit_ids = backend.create_master_repo(commits).values()
36 commit_ids = backend.create_master_repo(commits).values()
36 repo = backend.create_repo()
37 repo = backend.create_repo()
37 scm_extras = {
38 scm_extras = AttributeDict({
38 'ip': '127.0.0.1',
39 'ip': '127.0.0.1',
39 'username': user_regular.username,
40 'username': user_regular.username,
40 'user_id': user_regular.user_id,
41 'user_id': user_regular.user_id,
@@ -46,7 +47,7 def repo_push_event(backend, user_regula
46 'make_lock': None,
47 'make_lock': None,
47 'locked_by': [None],
48 'locked_by': [None],
48 'commit_ids': commit_ids,
49 'commit_ids': commit_ids,
49 }
50 })
50
51
51 return events.RepoPushEvent(repo_name=repo.repo_name,
52 return events.RepoPushEvent(repo_name=repo.repo_name,
52 pushed_commit_ids=commit_ids,
53 pushed_commit_ids=commit_ids,
@@ -77,12 +77,13 class Command(object):
77 assert self.process.returncode == 0
77 assert self.process.returncode == 0
78
78
79
79
80 def _add_files_and_push(vcs, dest, clone_url=None, **kwargs):
80 def _add_files_and_push(vcs, dest, clone_url=None, tags=None, **kwargs):
81 """
81 """
82 Generate some files, add it to DEST repo and push back
82 Generate some files, add it to DEST repo and push back
83 vcs is git or hg and defines what VCS we want to make those files for
83 vcs is git or hg and defines what VCS we want to make those files for
84 """
84 """
85 # commit some stuff into this repo
85 # commit some stuff into this repo
86 tags = tags or []
86 cwd = path = jn(dest)
87 cwd = path = jn(dest)
87 added_file = jn(path, '%ssetup.py' % tempfile._RandomNameSequence().next())
88 added_file = jn(path, '%ssetup.py' % tempfile._RandomNameSequence().next())
88 Command(cwd).execute('touch %s' % added_file)
89 Command(cwd).execute('touch %s' % added_file)
@@ -92,7 +93,7 def _add_files_and_push(vcs, dest, clone
92 git_ident = "git config user.name {} && git config user.email {}".format(
93 git_ident = "git config user.name {} && git config user.email {}".format(
93 'Marcin Kuźminski', 'me@email.com')
94 'Marcin Kuźminski', 'me@email.com')
94
95
95 for i in xrange(kwargs.get('files_no', 3)):
96 for i in range(kwargs.get('files_no', 3)):
96 cmd = """echo 'added_line%s' >> %s""" % (i, added_file)
97 cmd = """echo 'added_line%s' >> %s""" % (i, added_file)
97 Command(cwd).execute(cmd)
98 Command(cwd).execute(cmd)
98 if vcs == 'hg':
99 if vcs == 'hg':
@@ -104,6 +105,22 def _add_files_and_push(vcs, dest, clone
104 git_ident, i, added_file)
105 git_ident, i, added_file)
105 Command(cwd).execute(cmd)
106 Command(cwd).execute(cmd)
106
107
108 for tag in tags:
109 if vcs == 'hg':
110 stdout, stderr = Command(cwd).execute(
111 'hg tag', tag['name'])
112 elif vcs == 'git':
113 if tag['commit']:
114 # annotated tag
115 stdout, stderr = Command(cwd).execute(
116 """%s && git tag -a %s -m "%s" """ % (
117 git_ident, tag['name'], tag['commit']))
118 else:
119 # lightweight tag
120 stdout, stderr = Command(cwd).execute(
121 """%s && git tag %s""" % (
122 git_ident, tag['name']))
123
107 # PUSH it back
124 # PUSH it back
108 stdout = stderr = None
125 stdout = stderr = None
109 if vcs == 'hg':
126 if vcs == 'hg':
@@ -111,7 +128,8 def _add_files_and_push(vcs, dest, clone
111 'hg push --verbose', clone_url)
128 'hg push --verbose', clone_url)
112 elif vcs == 'git':
129 elif vcs == 'git':
113 stdout, stderr = Command(cwd).execute(
130 stdout, stderr = Command(cwd).execute(
114 """%s && git push --verbose %s master""" % (
131 """%s &&
132 git push --verbose --tags %s master""" % (
115 git_ident, clone_url))
133 git_ident, clone_url))
116
134
117 return stdout, stderr
135 return stdout, stderr
@@ -57,20 +57,24 def assert_no_running_instance(url):
57 "Port is not free at %s, cannot start web interface" % url)
57 "Port is not free at %s, cannot start web interface" % url)
58
58
59
59
60 def get_port(pyramid_config):
61 config = ConfigParser.ConfigParser()
62 config.read(pyramid_config)
63 return config.get('server:main', 'port')
64
65
60 def get_host_url(pyramid_config):
66 def get_host_url(pyramid_config):
61 """Construct the host url using the port in the test configuration."""
67 """Construct the host url using the port in the test configuration."""
62 config = ConfigParser.ConfigParser()
68 return '127.0.0.1:%s' % get_port(pyramid_config)
63 config.read(pyramid_config)
64
65 return '127.0.0.1:%s' % config.get('server:main', 'port')
66
69
67
70
68 class RcWebServer(object):
71 class RcWebServer(object):
69 """
72 """
70 Represents a running RCE web server used as a test fixture.
73 Represents a running RCE web server used as a test fixture.
71 """
74 """
72 def __init__(self, pyramid_config):
75 def __init__(self, pyramid_config, log_file):
73 self.pyramid_config = pyramid_config
76 self.pyramid_config = pyramid_config
77 self.log_file = log_file
74
78
75 def repo_clone_url(self, repo_name, **kwargs):
79 def repo_clone_url(self, repo_name, **kwargs):
76 params = {
80 params = {
@@ -86,6 +90,10 class RcWebServer(object):
86 def host_url(self):
90 def host_url(self):
87 return 'http://' + get_host_url(self.pyramid_config)
91 return 'http://' + get_host_url(self.pyramid_config)
88
92
93 def get_rc_log(self):
94 with open(self.log_file) as f:
95 return f.read()
96
89
97
90 @pytest.fixture(scope="module")
98 @pytest.fixture(scope="module")
91 def rcextensions(request, baseapp, tmpdir_factory):
99 def rcextensions(request, baseapp, tmpdir_factory):
@@ -155,12 +163,11 def rc_web_server(
155 env = os.environ.copy()
163 env = os.environ.copy()
156 env['RC_NO_TMP_PATH'] = '1'
164 env['RC_NO_TMP_PATH'] = '1'
157
165
158 rc_log = RC_LOG
166 rc_log = list(RC_LOG.partition('.log'))
159 server_out = open(rc_log, 'w')
167 rc_log.insert(1, get_port(rc_web_server_config))
168 rc_log = ''.join(rc_log)
160
169
161 # TODO: Would be great to capture the output and err of the subprocess
170 server_out = open(rc_log, 'w')
162 # and make it available in a section of the py.test report in case of an
163 # error.
164
171
165 host_url = 'http://' + get_host_url(rc_web_server_config)
172 host_url = 'http://' + get_host_url(rc_web_server_config)
166 assert_no_running_instance(host_url)
173 assert_no_running_instance(host_url)
@@ -184,7 +191,7 def rc_web_server(
184 server_out.flush()
191 server_out.flush()
185 server_out.close()
192 server_out.close()
186
193
187 return RcWebServer(rc_web_server_config)
194 return RcWebServer(rc_web_server_config, log_file=rc_log)
188
195
189
196
190 @pytest.fixture
197 @pytest.fixture
General Comments 0
You need to be logged in to leave comments. Login now