integrations: parse pushed tags, and lightweight tags for git.

- now aggregated as 'tags' key
- handles the case for email/webhook integrations

rhodecode/tests/other/vcs_operations/test_vcs_operations_tag_push.py

@@ -0,0 +1,141 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2017 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+"""
+Test suite for making push/pull operations, on specially modified INI files
+
+.. important::
+
+   You must have git >= 1.8.5 for tests to work fine. With 68b939b git started
+   to redirect things to stderr instead of stdout.
+"""
+
+import pytest
+import requests
+
+from rhodecode import events
+from rhodecode.model.db import Integration
+from rhodecode.model.integration import IntegrationModel
+from rhodecode.model.meta import Session
+
+from rhodecode.tests import GIT_REPO, HG_REPO
+from rhodecode.tests.other.vcs_operations import Command, _add_files_and_push
+from rhodecode.integrations.types.webhook import WebhookIntegrationType
+
+
+def check_connection():
+    try:
+        response = requests.get('http://httpbin.org')
+        return response.status_code == 200
+    except Exception as e:
+        print(e)
+
+    return False
+
+
+connection_available = pytest.mark.skipif(
+    not check_connection(), reason="No outside internet connection available")
+
+
[e-mail]
+def enable_webhook_push_integration(request):
+    integration = Integration()
+    integration.integration_type = WebhookIntegrationType.key
+    Session().add(integration)
+
+    settings = dict(
+        url='http://httpbin.org',
+        secret_token='secret',
+        username=None,
+        password=None,
+        custom_header_key=None,
+        custom_header_val=None,
+        method_type='get',
+        events=[events.RepoPushEvent.name],
+        log_data=True
+    )
+
+    IntegrationModel().update_integration(
+        integration,
+        name='IntegrationWebhookTest',
+        enabled=True,
+        settings=settings,
+        repo=None,
+        repo_group=None,
+        child_repos_only=False,
+    )
+    Session().commit()
+    integration_id = integration.integration_id
+
+    @request.addfinalizer
+    def cleanup():
+        integration = Integration.get(integration_id)
+        Session().delete(integration)
+        Session().commit()
+
+
[e-mail](
+    "disable_locking", "disable_anonymous_user",
+    "enable_webhook_push_integration")
+class TestVCSOperationsOnCustomIniConfig(object):
+
+    def test_push_tag_with_commit_hg(self, rc_web_server, tmpdir):
+        clone_url = rc_web_server.repo_clone_url(HG_REPO)
+        stdout, stderr = Command('/tmp').execute(
+            'hg clone', clone_url, tmpdir.strpath)
+
+        push_url = rc_web_server.repo_clone_url(HG_REPO)
+        _add_files_and_push(
+            'hg', tmpdir.strpath, clone_url=push_url,
+            tags=[{'name': 'v1.0.0', 'commit': 'added tag v1.0.0'}])
+
+        rc_log = rc_web_server.get_rc_log()
+        assert 'ERROR' not in rc_log
+        assert "'name': u'v1.0.0'" in rc_log
+
+    def test_push_tag_with_commit_git(
+            self, rc_web_server, tmpdir):
+        clone_url = rc_web_server.repo_clone_url(GIT_REPO)
+        stdout, stderr = Command('/tmp').execute(
+            'git clone', clone_url, tmpdir.strpath)
+
+        push_url = rc_web_server.repo_clone_url(GIT_REPO)
+        _add_files_and_push(
+            'git', tmpdir.strpath, clone_url=push_url,
+            tags=[{'name': 'v1.0.0', 'commit': 'added tag v1.0.0'}])
+
+        rc_log = rc_web_server.get_rc_log()
+        assert 'ERROR' not in rc_log
+        assert "'name': u'v1.0.0'" in rc_log
+
+    def test_push_tag_with_no_commit_git(
+            self, rc_web_server, tmpdir):
+        clone_url = rc_web_server.repo_clone_url(GIT_REPO)
+        stdout, stderr = Command('/tmp').execute(
+            'git clone', clone_url, tmpdir.strpath)
+
+        push_url = rc_web_server.repo_clone_url(GIT_REPO)
+        _add_files_and_push(
+            'git', tmpdir.strpath, clone_url=push_url,
+            tags=[{'name': 'v1.0.0', 'commit': 'added tag v1.0.0'}])
+
+        rc_log = rc_web_server.get_rc_log()
+        assert 'ERROR' not in rc_log
+        assert "'name': u'v1.0.0'" in rc_log

rhodecode/events/repo.py

@@ -18,6 +18,7 b''
 
 import collections
 import logging
+import datetime
 
 from rhodecode.translation import lazy_ugettext
 from rhodecode.model.db import User, Repository, Session
@@ -58,22 +59,60 b' def _commits_as_dict(event, commit_ids, '
             return commits  # return early if we have the commits we need
 
         vcs_repo = repo.scm_instance(cache=False)
+
         try:
             # use copy of needed_commits since we modify it while iterating
             for commit_id in list(needed_commits):
-                try:
-                    cs = vcs_repo.get_changeset(commit_id)
-                except CommitDoesNotExistError:
-                    continue  # maybe its in next repo
+                if commit_id.startswith('tag=>'):
+                    raw_id = commit_id[5:]
+                    cs_data = {
+                        'raw_id': commit_id, 'short_id': commit_id,
+                        'branch': None,
+                        'git_ref_change': 'tag_add',
+                        'message': 'Added new tag {}'.format(raw_id),
+                        'author': event.actor.full_contact,
+                        'date': datetime.datetime.now(),
+                        'refs': {
+                            'branches': [],
+                            'bookmarks': [],
+                            'tags': []
+                        }
+                    }
+                    commits.append(cs_data)
 
-                cs_data = cs.__json__()
-                cs_data['refs'] = cs._get_refs()
+                elif commit_id.startswith('delete_branch=>'):
+                    raw_id = commit_id[15:]
+                    cs_data = {
+                        'raw_id': commit_id, 'short_id': commit_id,
+                        'branch': None,
+                        'git_ref_change': 'branch_delete',
+                        'message': 'Deleted branch {}'.format(raw_id),
+                        'author': event.actor.full_contact,
+                        'date': datetime.datetime.now(),
+                        'refs': {
+                            'branches': [],
+                            'bookmarks': [],
+                            'tags': []
+                        }
+                    }
+                    commits.append(cs_data)
+
+                else:
+                    try:
+                        cs = vcs_repo.get_changeset(commit_id)
+                    except CommitDoesNotExistError:
+                        continue  # maybe its in next repo
+
+                    cs_data = cs.__json__()
+                    cs_data['refs'] = cs._get_refs()
+
                 cs_data['mentions'] = extract_mentioned_users(cs_data['message'])
                 cs_data['reviewers'] = reviewers
                 cs_data['url'] = RepoModel().get_commit_url(
                     repo, cs_data['raw_id'], request=event.request)
                 cs_data['permalink_url'] = RepoModel().get_commit_url(
-                    repo, cs_data['raw_id'], request=event.request, permalink=True)
+                    repo, cs_data['raw_id'], request=event.request,
+                    permalink=True)
                 urlified_message, issues_data = process_patterns(
                     cs_data['message'], repo.repo_name)
                 cs_data['issues'] = issues_data
@@ -85,8 +124,8 b' def _commits_as_dict(event, commit_ids, '
 
                 needed_commits.remove(commit_id)
 
-        except Exception as e:
-            log.exception(e)
+        except Exception:
+            log.exception('Failed to extract commits data')
             # we don't send any commits when crash happens, only full list
             # matters we short circuit then.
             return []
@@ -248,6 +287,7 b' class RepoPushEvent(RepoVCSEvent):'
     def __init__(self, repo_name, pushed_commit_ids, extras):
         super(RepoPushEvent, self).__init__(repo_name, extras)
         self.pushed_commit_ids = pushed_commit_ids
+        self.new_refs = extras.new_refs
 
     def as_dict(self):
         data = super(RepoPushEvent, self).as_dict()
@@ -256,6 +296,10 b' class RepoPushEvent(RepoVCSEvent):'
             return '{}/changelog?branch={}'.format(
                 data['repo']['url'], branch_name)
 
+        def tag_url(tag_name):
+            return '{}/files/{}/'.format(
+                data['repo']['url'], tag_name)
+
         commits = _commits_as_dict(
             self, commit_ids=self.pushed_commit_ids, repos=[self.repo])
 
@@ -265,8 +309,21 b' class RepoPushEvent(RepoVCSEvent):'
             last_branch = commit['branch']
         issues = _issues_as_dict(commits)
 
-        branches = set(
-            commit['branch'] for commit in commits if commit['branch'])
+        branches = set()
+        tags = set()
+        for commit in commits:
+            if commit['refs']['tags']:
+                for tag in commit['refs']['tags']:
+                    tags.add(tag)
+            if commit['branch']:
+                branches.add(commit['branch'])
+
+        # maybe we have branches in new_refs ?
+        try:
+            branches = branches.union(set(self.new_refs['branches']))
+        except Exception:
+            pass
+
         branches = [
             {
                 'name': branch,
@@ -275,9 +332,24 b' class RepoPushEvent(RepoVCSEvent):'
             for branch in branches
         ]
 
+        # maybe we have branches in new_refs ?
+        try:
+            tags = tags.union(set(self.new_refs['tags']))
+        except Exception:
+            pass
+
+        tags = [
+            {
+                'name': tag,
+                'url': tag_url(tag)
+            }
+            for tag in tags
+        ]
+
         data['push'] = {
             'commits': commits,
             'issues': issues,
             'branches': branches,
+            'tags': tags,
         }
         return data

rhodecode/integrations/types/webhook.py

@@ -110,6 +110,11 b' class WebhookHandler(object):'
 
         branches_commits = OrderedDict()
         for commit in data['push']['commits']:
+            if commit.get('git_ref_change'):
+                # special case for GIT that allows creating tags,
+                # deleting branches without associated commit
+                continue
+
             if commit['branch'] not in branches_commits:
                 branch_commits = {'branch': branch_data[commit['branch']],
                                   'commits': []}
@@ -378,7 +383,8 b' def post_to_webhook(url_calls, settings)'
 
         log.debug('calling Webhook with method: %s, and auth:%s',
                   call_method, auth)
-
+        if settings.get('log_data'):
+            log.debug('calling webhook with data: %s', data)
         resp = call_method(url, json={
             'token': token,
             'event': data

rhodecode/integrations/views.py

@@ -404,7 +404,6 b' class RepoIntegrationsView(IntegrationSe'
         c.repo_name = self.db_repo.repo_name
         c.repository_pull_requests = ScmModel().get_pull_requests(self.repo)
 
-
         return c
 
     @LoginRequired()

rhodecode/model/validation_schema/schemas/integration_schema.py

@@ -198,7 +198,6 b' class IntegrationOptionsSchemaBase(colan'
     )
 
 
-
 def make_integration_schema(IntegrationType, settings=None):
     """
     Return a colander schema for an integration type

rhodecode/tests/integrations/conftest.py

@@ -21,6 +21,7 b''
 
 import pytest
 from rhodecode import events
+from rhodecode.lib.utils2 import AttributeDict
 
 
 @pytest.fixture
@@ -34,7 +35,7 b' def repo_push_event(backend, user_regula'
     ]
     commit_ids = backend.create_master_repo(commits).values()
     repo = backend.create_repo()
-    scm_extras = {
+    scm_extras = AttributeDict({
         'ip': '127.0.0.1',
         'username': user_regular.username,
         'user_id': user_regular.user_id,
@@ -46,7 +47,7 b' def repo_push_event(backend, user_regula'
         'make_lock': None,
         'locked_by': [None],
         'commit_ids': commit_ids,
-    }
+    })
 
     return events.RepoPushEvent(repo_name=repo.repo_name,
                                 pushed_commit_ids=commit_ids,

rhodecode/tests/other/vcs_operations/__init__.py

@@ -77,12 +77,13 b' class Command(object):'
         assert self.process.returncode == 0
 
 
-def _add_files_and_push(vcs, dest, clone_url=None, **kwargs):
+def _add_files_and_push(vcs, dest, clone_url=None, tags=None, **kwargs):
     """
     Generate some files, add it to DEST repo and push back
     vcs is git or hg and defines what VCS we want to make those files for
     """
     # commit some stuff into this repo
+    tags = tags or []
     cwd = path = jn(dest)
     added_file = jn(path, '%ssetup.py' % tempfile._RandomNameSequence().next())
     Command(cwd).execute('touch %s' % added_file)
@@ -92,7 +93,7 b' def _add_files_and_push(vcs, dest, clone'
     git_ident = "git config user.name {} && git config user.email {}".format(
             'Marcin Kuźminski', '[e-mail]')
 
-    for i in xrange(kwargs.get('files_no', 3)):
+    for i in range(kwargs.get('files_no', 3)):
         cmd = """echo 'added_line%s' >> %s""" % (i, added_file)
         Command(cwd).execute(cmd)
         if vcs == 'hg':
@@ -104,6 +105,22 b' def _add_files_and_push(vcs, dest, clone'
                 git_ident, i, added_file)
         Command(cwd).execute(cmd)
 
+    for tag in tags:
+        if vcs == 'hg':
+            stdout, stderr = Command(cwd).execute(
+                'hg tag', tag['name'])
+        elif vcs == 'git':
+            if tag['commit']:
+                # annotated tag
+                stdout, stderr = Command(cwd).execute(
+                    """%s && git tag -a %s -m "%s" """ % (
+                        git_ident, tag['name'], tag['commit']))
+            else:
+                # lightweight tag
+                stdout, stderr = Command(cwd).execute(
+                    """%s && git tag %s""" % (
+                        git_ident, tag['name']))
+
     # PUSH it back
     stdout = stderr = None
     if vcs == 'hg':
@@ -111,7 +128,8 b' def _add_files_and_push(vcs, dest, clone'
             'hg push --verbose', clone_url)
     elif vcs == 'git':
         stdout, stderr = Command(cwd).execute(
-            """%s && git push --verbose %s master""" % (
+            """%s && 
+            git push --verbose --tags %s master""" % (
                 git_ident, clone_url))
 
     return stdout, stderr

rhodecode/tests/other/vcs_operations/conftest.py

@@ -57,20 +57,24 b' def assert_no_running_instance(url):'
             "Port is not free at %s, cannot start web interface" % url)
 
 
+def get_port(pyramid_config):
+    config = ConfigParser.ConfigParser()
+    config.read(pyramid_config)
+    return config.get('server:main', 'port')
+
+
 def get_host_url(pyramid_config):
     """Construct the host url using the port in the test configuration."""
-    config = ConfigParser.ConfigParser()
-    config.read(pyramid_config)
-
-    return '127.0.0.1:%s' % config.get('server:main', 'port')
+    return '127.0.0.1:%s' % get_port(pyramid_config)
 
 
 class RcWebServer(object):
     """
     Represents a running RCE web server used as a test fixture.
     """
-    def __init__(self, pyramid_config):
+    def __init__(self, pyramid_config, log_file):
         self.pyramid_config = pyramid_config
+        self.log_file = log_file
 
     def repo_clone_url(self, repo_name, **kwargs):
         params = {
@@ -86,6 +90,10 b' class RcWebServer(object):'
     def host_url(self):
         return 'http://' + get_host_url(self.pyramid_config)
 
+    def get_rc_log(self):
+        with open(self.log_file) as f:
+            return f.read()
+
 
 @pytest.fixture(scope="module")
 def rcextensions(request, baseapp, tmpdir_factory):
@@ -155,12 +163,11 b' def rc_web_server('
     env = os.environ.copy()
     env['RC_NO_TMP_PATH'] = '1'
 
-    rc_log = RC_LOG
-    server_out = open(rc_log, 'w')
+    rc_log = list(RC_LOG.partition('.log'))
+    rc_log.insert(1, get_port(rc_web_server_config))
+    rc_log = ''.join(rc_log)
 
-    # TODO: Would be great to capture the output and err of the subprocess
-    # and make it available in a section of the py.test report in case of an
-    # error.
+    server_out = open(rc_log, 'w')
 
     host_url = 'http://' + get_host_url(rc_web_server_config)
     assert_no_running_instance(host_url)
@@ -184,7 +191,7 b' def rc_web_server('
         server_out.flush()
         server_out.close()
 
-    return RcWebServer(rc_web_server_config)
+    return RcWebServer(rc_web_server_config, log_file=rc_log)
 
 
 @pytest.fixture