# HG changeset patch # User Marcin Kuzminski # Date 2017-12-03 21:37:36 # Node ID ac36fdfd7ee0cc98ec9e00985ba0c9223c60e539 # Parent 3ca1f08e0ea3a9f348e91e11a7a0fe8d2079e4a8 integrations: parse pushed tags, and lightweight tags for git. - now aggregated as 'tags' key - handles the case for email/webhook integrations diff --git a/rhodecode/events/repo.py b/rhodecode/events/repo.py --- a/rhodecode/events/repo.py +++ b/rhodecode/events/repo.py @@ -18,6 +18,7 @@ import collections import logging +import datetime from rhodecode.translation import lazy_ugettext from rhodecode.model.db import User, Repository, Session @@ -58,22 +59,60 @@ def _commits_as_dict(event, commit_ids, return commits # return early if we have the commits we need vcs_repo = repo.scm_instance(cache=False) + try: # use copy of needed_commits since we modify it while iterating for commit_id in list(needed_commits): - try: - cs = vcs_repo.get_changeset(commit_id) - except CommitDoesNotExistError: - continue # maybe its in next repo + if commit_id.startswith('tag=>'): + raw_id = commit_id[5:] + cs_data = { + 'raw_id': commit_id, 'short_id': commit_id, + 'branch': None, + 'git_ref_change': 'tag_add', + 'message': 'Added new tag {}'.format(raw_id), + 'author': event.actor.full_contact, + 'date': datetime.datetime.now(), + 'refs': { + 'branches': [], + 'bookmarks': [], + 'tags': [] + } + } + commits.append(cs_data) - cs_data = cs.__json__() - cs_data['refs'] = cs._get_refs() + elif commit_id.startswith('delete_branch=>'): + raw_id = commit_id[15:] + cs_data = { + 'raw_id': commit_id, 'short_id': commit_id, + 'branch': None, + 'git_ref_change': 'branch_delete', + 'message': 'Deleted branch {}'.format(raw_id), + 'author': event.actor.full_contact, + 'date': datetime.datetime.now(), + 'refs': { + 'branches': [], + 'bookmarks': [], + 'tags': [] + } + } + commits.append(cs_data) + + else: + try: + cs = vcs_repo.get_changeset(commit_id) + except CommitDoesNotExistError: + continue # maybe its in next repo + + cs_data = cs.__json__() + cs_data['refs'] = cs._get_refs() + cs_data['mentions'] = extract_mentioned_users(cs_data['message']) cs_data['reviewers'] = reviewers cs_data['url'] = RepoModel().get_commit_url( repo, cs_data['raw_id'], request=event.request) cs_data['permalink_url'] = RepoModel().get_commit_url( - repo, cs_data['raw_id'], request=event.request, permalink=True) + repo, cs_data['raw_id'], request=event.request, + permalink=True) urlified_message, issues_data = process_patterns( cs_data['message'], repo.repo_name) cs_data['issues'] = issues_data @@ -85,8 +124,8 @@ def _commits_as_dict(event, commit_ids, needed_commits.remove(commit_id) - except Exception as e: - log.exception(e) + except Exception: + log.exception('Failed to extract commits data') # we don't send any commits when crash happens, only full list # matters we short circuit then. return [] @@ -248,6 +287,7 @@ class RepoPushEvent(RepoVCSEvent): def __init__(self, repo_name, pushed_commit_ids, extras): super(RepoPushEvent, self).__init__(repo_name, extras) self.pushed_commit_ids = pushed_commit_ids + self.new_refs = extras.new_refs def as_dict(self): data = super(RepoPushEvent, self).as_dict() @@ -256,6 +296,10 @@ class RepoPushEvent(RepoVCSEvent): return '{}/changelog?branch={}'.format( data['repo']['url'], branch_name) + def tag_url(tag_name): + return '{}/files/{}/'.format( + data['repo']['url'], tag_name) + commits = _commits_as_dict( self, commit_ids=self.pushed_commit_ids, repos=[self.repo]) @@ -265,8 +309,21 @@ class RepoPushEvent(RepoVCSEvent): last_branch = commit['branch'] issues = _issues_as_dict(commits) - branches = set( - commit['branch'] for commit in commits if commit['branch']) + branches = set() + tags = set() + for commit in commits: + if commit['refs']['tags']: + for tag in commit['refs']['tags']: + tags.add(tag) + if commit['branch']: + branches.add(commit['branch']) + + # maybe we have branches in new_refs ? + try: + branches = branches.union(set(self.new_refs['branches'])) + except Exception: + pass + branches = [ { 'name': branch, @@ -275,9 +332,24 @@ class RepoPushEvent(RepoVCSEvent): for branch in branches ] + # maybe we have branches in new_refs ? + try: + tags = tags.union(set(self.new_refs['tags'])) + except Exception: + pass + + tags = [ + { + 'name': tag, + 'url': tag_url(tag) + } + for tag in tags + ] + data['push'] = { 'commits': commits, 'issues': issues, 'branches': branches, + 'tags': tags, } return data diff --git a/rhodecode/integrations/types/webhook.py b/rhodecode/integrations/types/webhook.py --- a/rhodecode/integrations/types/webhook.py +++ b/rhodecode/integrations/types/webhook.py @@ -110,6 +110,11 @@ class WebhookHandler(object): branches_commits = OrderedDict() for commit in data['push']['commits']: + if commit.get('git_ref_change'): + # special case for GIT that allows creating tags, + # deleting branches without associated commit + continue + if commit['branch'] not in branches_commits: branch_commits = {'branch': branch_data[commit['branch']], 'commits': []} @@ -378,7 +383,8 @@ def post_to_webhook(url_calls, settings) log.debug('calling Webhook with method: %s, and auth:%s', call_method, auth) - + if settings.get('log_data'): + log.debug('calling webhook with data: %s', data) resp = call_method(url, json={ 'token': token, 'event': data diff --git a/rhodecode/integrations/views.py b/rhodecode/integrations/views.py --- a/rhodecode/integrations/views.py +++ b/rhodecode/integrations/views.py @@ -404,7 +404,6 @@ class RepoIntegrationsView(IntegrationSe c.repo_name = self.db_repo.repo_name c.repository_pull_requests = ScmModel().get_pull_requests(self.repo) - return c @LoginRequired() diff --git a/rhodecode/model/validation_schema/schemas/integration_schema.py b/rhodecode/model/validation_schema/schemas/integration_schema.py --- a/rhodecode/model/validation_schema/schemas/integration_schema.py +++ b/rhodecode/model/validation_schema/schemas/integration_schema.py @@ -198,7 +198,6 @@ class IntegrationOptionsSchemaBase(colan ) - def make_integration_schema(IntegrationType, settings=None): """ Return a colander schema for an integration type diff --git a/rhodecode/tests/integrations/conftest.py b/rhodecode/tests/integrations/conftest.py --- a/rhodecode/tests/integrations/conftest.py +++ b/rhodecode/tests/integrations/conftest.py @@ -21,6 +21,7 @@ import pytest from rhodecode import events +from rhodecode.lib.utils2 import AttributeDict @pytest.fixture @@ -34,7 +35,7 @@ def repo_push_event(backend, user_regula ] commit_ids = backend.create_master_repo(commits).values() repo = backend.create_repo() - scm_extras = { + scm_extras = AttributeDict({ 'ip': '127.0.0.1', 'username': user_regular.username, 'user_id': user_regular.user_id, @@ -46,7 +47,7 @@ def repo_push_event(backend, user_regula 'make_lock': None, 'locked_by': [None], 'commit_ids': commit_ids, - } + }) return events.RepoPushEvent(repo_name=repo.repo_name, pushed_commit_ids=commit_ids, diff --git a/rhodecode/tests/other/vcs_operations/__init__.py b/rhodecode/tests/other/vcs_operations/__init__.py --- a/rhodecode/tests/other/vcs_operations/__init__.py +++ b/rhodecode/tests/other/vcs_operations/__init__.py @@ -77,12 +77,13 @@ class Command(object): assert self.process.returncode == 0 -def _add_files_and_push(vcs, dest, clone_url=None, **kwargs): +def _add_files_and_push(vcs, dest, clone_url=None, tags=None, **kwargs): """ Generate some files, add it to DEST repo and push back vcs is git or hg and defines what VCS we want to make those files for """ # commit some stuff into this repo + tags = tags or [] cwd = path = jn(dest) added_file = jn(path, '%ssetup.py' % tempfile._RandomNameSequence().next()) Command(cwd).execute('touch %s' % added_file) @@ -92,7 +93,7 @@ def _add_files_and_push(vcs, dest, clone git_ident = "git config user.name {} && git config user.email {}".format( 'Marcin Kuźminski', 'me@email.com') - for i in xrange(kwargs.get('files_no', 3)): + for i in range(kwargs.get('files_no', 3)): cmd = """echo 'added_line%s' >> %s""" % (i, added_file) Command(cwd).execute(cmd) if vcs == 'hg': @@ -104,6 +105,22 @@ def _add_files_and_push(vcs, dest, clone git_ident, i, added_file) Command(cwd).execute(cmd) + for tag in tags: + if vcs == 'hg': + stdout, stderr = Command(cwd).execute( + 'hg tag', tag['name']) + elif vcs == 'git': + if tag['commit']: + # annotated tag + stdout, stderr = Command(cwd).execute( + """%s && git tag -a %s -m "%s" """ % ( + git_ident, tag['name'], tag['commit'])) + else: + # lightweight tag + stdout, stderr = Command(cwd).execute( + """%s && git tag %s""" % ( + git_ident, tag['name'])) + # PUSH it back stdout = stderr = None if vcs == 'hg': @@ -111,7 +128,8 @@ def _add_files_and_push(vcs, dest, clone 'hg push --verbose', clone_url) elif vcs == 'git': stdout, stderr = Command(cwd).execute( - """%s && git push --verbose %s master""" % ( + """%s && + git push --verbose --tags %s master""" % ( git_ident, clone_url)) return stdout, stderr diff --git a/rhodecode/tests/other/vcs_operations/conftest.py b/rhodecode/tests/other/vcs_operations/conftest.py --- a/rhodecode/tests/other/vcs_operations/conftest.py +++ b/rhodecode/tests/other/vcs_operations/conftest.py @@ -57,20 +57,24 @@ def assert_no_running_instance(url): "Port is not free at %s, cannot start web interface" % url) +def get_port(pyramid_config): + config = ConfigParser.ConfigParser() + config.read(pyramid_config) + return config.get('server:main', 'port') + + def get_host_url(pyramid_config): """Construct the host url using the port in the test configuration.""" - config = ConfigParser.ConfigParser() - config.read(pyramid_config) - - return '127.0.0.1:%s' % config.get('server:main', 'port') + return '127.0.0.1:%s' % get_port(pyramid_config) class RcWebServer(object): """ Represents a running RCE web server used as a test fixture. """ - def __init__(self, pyramid_config): + def __init__(self, pyramid_config, log_file): self.pyramid_config = pyramid_config + self.log_file = log_file def repo_clone_url(self, repo_name, **kwargs): params = { @@ -86,6 +90,10 @@ class RcWebServer(object): def host_url(self): return 'http://' + get_host_url(self.pyramid_config) + def get_rc_log(self): + with open(self.log_file) as f: + return f.read() + @pytest.fixture(scope="module") def rcextensions(request, baseapp, tmpdir_factory): @@ -155,12 +163,11 @@ def rc_web_server( env = os.environ.copy() env['RC_NO_TMP_PATH'] = '1' - rc_log = RC_LOG - server_out = open(rc_log, 'w') + rc_log = list(RC_LOG.partition('.log')) + rc_log.insert(1, get_port(rc_web_server_config)) + rc_log = ''.join(rc_log) - # TODO: Would be great to capture the output and err of the subprocess - # and make it available in a section of the py.test report in case of an - # error. + server_out = open(rc_log, 'w') host_url = 'http://' + get_host_url(rc_web_server_config) assert_no_running_instance(host_url) @@ -184,7 +191,7 @@ def rc_web_server( server_out.flush() server_out.close() - return RcWebServer(rc_web_server_config) + return RcWebServer(rc_web_server_config, log_file=rc_log) @pytest.fixture diff --git a/rhodecode/tests/other/vcs_operations/test_vcs_operations_tag_push.py b/rhodecode/tests/other/vcs_operations/test_vcs_operations_tag_push.py new file mode 100644 --- /dev/null +++ b/rhodecode/tests/other/vcs_operations/test_vcs_operations_tag_push.py @@ -0,0 +1,141 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2017 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +""" +Test suite for making push/pull operations, on specially modified INI files + +.. important:: + + You must have git >= 1.8.5 for tests to work fine. With 68b939b git started + to redirect things to stderr instead of stdout. +""" + +import pytest +import requests + +from rhodecode import events +from rhodecode.model.db import Integration +from rhodecode.model.integration import IntegrationModel +from rhodecode.model.meta import Session + +from rhodecode.tests import GIT_REPO, HG_REPO +from rhodecode.tests.other.vcs_operations import Command, _add_files_and_push +from rhodecode.integrations.types.webhook import WebhookIntegrationType + + +def check_connection(): + try: + response = requests.get('http://httpbin.org') + return response.status_code == 200 + except Exception as e: + print(e) + + return False + + +connection_available = pytest.mark.skipif( + not check_connection(), reason="No outside internet connection available") + + +@pytest.fixture +def enable_webhook_push_integration(request): + integration = Integration() + integration.integration_type = WebhookIntegrationType.key + Session().add(integration) + + settings = dict( + url='http://httpbin.org', + secret_token='secret', + username=None, + password=None, + custom_header_key=None, + custom_header_val=None, + method_type='get', + events=[events.RepoPushEvent.name], + log_data=True + ) + + IntegrationModel().update_integration( + integration, + name='IntegrationWebhookTest', + enabled=True, + settings=settings, + repo=None, + repo_group=None, + child_repos_only=False, + ) + Session().commit() + integration_id = integration.integration_id + + @request.addfinalizer + def cleanup(): + integration = Integration.get(integration_id) + Session().delete(integration) + Session().commit() + + +@pytest.mark.usefixtures( + "disable_locking", "disable_anonymous_user", + "enable_webhook_push_integration") +class TestVCSOperationsOnCustomIniConfig(object): + + def test_push_tag_with_commit_hg(self, rc_web_server, tmpdir): + clone_url = rc_web_server.repo_clone_url(HG_REPO) + stdout, stderr = Command('/tmp').execute( + 'hg clone', clone_url, tmpdir.strpath) + + push_url = rc_web_server.repo_clone_url(HG_REPO) + _add_files_and_push( + 'hg', tmpdir.strpath, clone_url=push_url, + tags=[{'name': 'v1.0.0', 'commit': 'added tag v1.0.0'}]) + + rc_log = rc_web_server.get_rc_log() + assert 'ERROR' not in rc_log + assert "'name': u'v1.0.0'" in rc_log + + def test_push_tag_with_commit_git( + self, rc_web_server, tmpdir): + clone_url = rc_web_server.repo_clone_url(GIT_REPO) + stdout, stderr = Command('/tmp').execute( + 'git clone', clone_url, tmpdir.strpath) + + push_url = rc_web_server.repo_clone_url(GIT_REPO) + _add_files_and_push( + 'git', tmpdir.strpath, clone_url=push_url, + tags=[{'name': 'v1.0.0', 'commit': 'added tag v1.0.0'}]) + + rc_log = rc_web_server.get_rc_log() + assert 'ERROR' not in rc_log + assert "'name': u'v1.0.0'" in rc_log + + def test_push_tag_with_no_commit_git( + self, rc_web_server, tmpdir): + clone_url = rc_web_server.repo_clone_url(GIT_REPO) + stdout, stderr = Command('/tmp').execute( + 'git clone', clone_url, tmpdir.strpath) + + push_url = rc_web_server.repo_clone_url(GIT_REPO) + _add_files_and_push( + 'git', tmpdir.strpath, clone_url=push_url, + tags=[{'name': 'v1.0.0', 'commit': 'added tag v1.0.0'}]) + + rc_log = rc_web_server.get_rc_log() + assert 'ERROR' not in rc_log + assert "'name': u'v1.0.0'" in rc_log