integrations: use a common logic for parsing the commits branches inside the integrations that require it.

There was 3 duplicated parts on each integration which now uses a small class with helper to unify the code.

rhodecode/integrations/types/base.py

@@ -1,288 +1,299 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import colander
 import string
 import collections
 import logging
 from rhodecode.translation import _
 
 log = logging.getLogger(__name__)
 
 
 class IntegrationTypeBase(object):
     """ Base class for IntegrationType plugins """
     is_dummy = False
     description = ''
 
     @classmethod
     def icon(cls):
         return '''
         <?xml version="1.0" encoding="UTF-8" standalone="no"?>
         <svg
            xmlns:dc="http://purl.org/dc/elements/1.1/"
            xmlns:cc="http://creativecommons.org/ns#"
            xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
            xmlns:svg="http://www.w3.org/2000/svg"
            xmlns="http://www.w3.org/2000/svg"
            xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
            xmlns:inkscape="http://setwww.inkscape.org/namespaces/inkscape"
            viewBox="0 -256 1792 1792"
            id="svg3025"
            version="1.1"
            inkscape:version="0.48.3.1 r9886"
            width="100%"
            height="100%"
            sodipodi:docname="cog_font_awesome.svg">
           <metadata
              id="metadata3035">
             <rdf:RDF>
               <cc:Work
                  rdf:about="">
                 <dc:format>image/svg+xml</dc:format>
                 <dc:type
                    rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
               </cc:Work>
             </rdf:RDF>
           </metadata>
           <defs
              id="defs3033" />
           <sodipodi:namedview
              pagecolor="#ffffff"
              bordercolor="#666666"
              borderopacity="1"
              objecttolerance="10"
              gridtolerance="10"
              guidetolerance="10"
              inkscape:pageopacity="0"
              inkscape:pageshadow="2"
              inkscape:window-width="640"
              inkscape:window-height="480"
              id="namedview3031"
              showgrid="false"
              inkscape:zoom="0.13169643"
              inkscape:cx="896"
              inkscape:cy="896"
              inkscape:window-x="0"
              inkscape:window-y="25"
              inkscape:window-maximized="0"
              inkscape:current-layer="svg3025" />
           <g
              transform="matrix(1,0,0,-1,121.49153,1285.4237)"
              id="g3027">
             <path
                d="m 1024,640 q 0,106 -75,181 -75,75 -181,75 -106,0 -181,-75 -75,-75 -75,-181 0,-106 75,-181 75,-75 181,-75 106,0 181,75 75,75 75,181 z m 512,109 V 527 q 0,-12 -8,-23 -8,-11 -20,-13 l -185,-28 q -19,-54 -39,-91 35,-50 107,-138 10,-12 10,-25 0,-13 -9,-23 -27,-37 -99,-108 -72,-71 -94,-71 -12,0 -26,9 l -138,108 q -44,-23 -91,-38 -16,-136 -29,-186 -7,-28 -36,-28 H 657 q -14,0 -24.5,8.5 Q 622,-111 621,-98 L 593,86 q -49,16 -90,37 L 362,16 Q 352,7 337,7 323,7 312,18 186,132 147,186 q -7,10 -7,23 0,12 8,23 15,21 51,66.5 36,45.5 54,70.5 -27,50 -41,99 L 29,495 Q 16,497 8,507.5 0,518 0,531 v 222 q 0,12 8,23 8,11 19,13 l 186,28 q 14,46 39,92 -40,57 -107,138 -10,12 -10,24 0,10 9,23 26,36 98.5,107.5 72.5,71.5 94.5,71.5 13,0 26,-10 l 138,-107 q 44,23 91,38 16,136 29,186 7,28 36,28 h 222 q 14,0 24.5,-8.5 Q 914,1391 915,1378 l 28,-184 q 49,-16 90,-37 l 142,107 q 9,9 24,9 13,0 25,-10 129,-119 165,-170 7,-8 7,-22 0,-12 -8,-23 -15,-21 -51,-66.5 -36,-45.5 -54,-70.5 26,-50 41,-98 l 183,-28 q 13,-2 21,-12.5 8,-10.5 8,-23.5 z"
                id="path3029"
                inkscape:connector-curvature="0"
                style="fill:currentColor" />
           </g>
         </svg>
         '''
 
     def __init__(self, settings):
         """
         :param settings: dict of settings to be used for the integration
         """
         self.settings = settings
 
     def settings_schema(self):
         """
         A colander schema of settings for the integration type
         """
         return colander.Schema()
 
 
 class EEIntegration(IntegrationTypeBase):
     description = 'Integration available in RhodeCode EE edition.'
     is_dummy = True
 
     def __init__(self, name, key, settings=None):
         self.display_name = name
         self.key = key
         super(EEIntegration, self).__init__(settings)
 
 
 # Helpers #
 WEBHOOK_URL_VARS = [
     ('event_name', 'Unique name of the event type, e.g pullrequest-update'),
     ('repo_name', 'Full name of the repository'),
     ('repo_type', 'VCS type of repository'),
     ('repo_id', 'Unique id of repository'),
     ('repo_url', 'Repository url'),
     # extra repo fields
     ('extra:<extra_key_name>', 'Extra repo variables, read from its settings.'),
 
     # special attrs below that we handle, using multi-call
     ('branch', 'Name of each brach submitted, if any.'),
     ('commit_id', 'Id of each commit submitted, if any.'),
 
     # pr events vars
     ('pull_request_id', 'Unique ID of the pull request.'),
     ('pull_request_title', 'Title of the pull request.'),
     ('pull_request_url', 'Pull request url.'),
     ('pull_request_shadow_url', 'Pull request shadow repo clone url.'),
     ('pull_request_commits_uid', 'Calculated UID of all commits inside the PR. '
                                  'Changes after PR update'),
 
     # user who triggers the call
     ('username', 'User who triggered the call.'),
     ('user_id', 'User id who triggered the call.'),
 ]
 
 # common vars for url template used for CI plugins. Shared with webhook
 CI_URL_VARS = WEBHOOK_URL_VARS
 
 
-class WebhookDataHandler(object):
+class CommitParsingDataHandler(object):
+
+    def aggregate_branch_data(self, branches, commits):
+        branch_data = collections.OrderedDict()
+        for obj in branches:
+            branch_data[obj['name']] = obj
+
+        branches_commits = collections.OrderedDict()
+        for commit in commits:
+            if commit.get('git_ref_change'):
+                # special case for GIT that allows creating tags,
+                # deleting branches without associated commit
+                continue
+            commit_branch = commit['branch']
+
+            if commit_branch not in branches_commits:
+                _branch = branch_data[commit_branch] \
+                    if commit_branch else commit_branch
+                branch_commits = {'branch': _branch,
+                                  'commits': []}
+                branches_commits[commit_branch] = branch_commits
+
+            branch_commits = branches_commits[commit_branch]
+            branch_commits['commits'].append(commit)
+        return branches_commits
+
+
+class WebhookDataHandler(CommitParsingDataHandler):
     name = 'webhook'
 
     def __init__(self, template_url, headers):
         self.template_url = template_url
         self.headers = headers
 
     def get_base_parsed_template(self, data):
         """
         initially parses the passed in template with some common variables
         available on ALL calls
         """
         # note: make sure to update the `WEBHOOK_URL_VARS` if this changes
         common_vars = {
             'repo_name': data['repo']['repo_name'],
             'repo_type': data['repo']['repo_type'],
             'repo_id': data['repo']['repo_id'],
             'repo_url': data['repo']['url'],
             'username': data['actor']['username'],
             'user_id': data['actor']['user_id'],
             'event_name': data['name']
         }
 
         extra_vars = {}
         for extra_key, extra_val in data['repo']['extra_fields'].items():
             extra_vars['extra__{}'.format(extra_key)] = extra_val
         common_vars.update(extra_vars)
 
         template_url = self.template_url.replace('${extra:', '${extra__')
         return string.Template(template_url).safe_substitute(**common_vars)
 
     def repo_push_event_handler(self, event, data):
         url = self.get_base_parsed_template(data)
         url_cals = []
-        branch_data = collections.OrderedDict()
-        for obj in data['push']['branches']:
-            branch_data[obj['name']] = obj
 
-        branches_commits = collections.OrderedDict()
-        for commit in data['push']['commits']:
-            if commit.get('git_ref_change'):
-                # special case for GIT that allows creating tags,
-                # deleting branches without associated commit
-                continue
-
-            if commit['branch'] not in branches_commits:
-                branch_commits = {'branch': branch_data[commit['branch']],
-                                  'commits': []}
-                branches_commits[commit['branch']] = branch_commits
-
-            branch_commits = branches_commits[commit['branch']]
-            branch_commits['commits'].append(commit)
-
+        branches_commits = self.aggregate_branch_data(
+            data['push']['branches'], data['push']['commits'])
         if '${branch}' in url:
             # call it multiple times, for each branch if used in variables
             for branch, commit_ids in branches_commits.items():
                 branch_url = string.Template(url).safe_substitute(branch=branch)
                 # call further down for each commit if used
                 if '${commit_id}' in branch_url:
                     for commit_data in commit_ids['commits']:
                         commit_id = commit_data['raw_id']
                         commit_url = string.Template(branch_url).safe_substitute(
                             commit_id=commit_id)
                         # register per-commit call
                         log.debug(
                             'register %s call(%s) to url %s',
                             self.name, event, commit_url)
                         url_cals.append(
                             (commit_url, self.headers, data))
 
                 else:
                     # register per-branch call
                     log.debug(
                         'register %s call(%s) to url %s',
                         self.name, event, branch_url)
                     url_cals.append(
                         (branch_url, self.headers, data))
 
         else:
             log.debug(
                 'register %s call(%s) to url %s', self.name, event, url)
             url_cals.append((url, self.headers, data))
 
         return url_cals
 
     def repo_create_event_handler(self, event, data):
         url = self.get_base_parsed_template(data)
         log.debug(
             'register %s call(%s) to url %s', self.name, event, url)
         return [(url, self.headers, data)]
 
     def pull_request_event_handler(self, event, data):
         url = self.get_base_parsed_template(data)
         log.debug(
             'register %s call(%s) to url %s', self.name, event, url)
         url = string.Template(url).safe_substitute(
             pull_request_id=data['pullrequest']['pull_request_id'],
             pull_request_title=data['pullrequest']['title'],
             pull_request_url=data['pullrequest']['url'],
             pull_request_shadow_url=data['pullrequest']['shadow_url'],
             pull_request_commits_uid=data['pullrequest']['commits_uid'],
         )
         return [(url, self.headers, data)]
 
     def __call__(self, event, data):
         from rhodecode import events
 
         if isinstance(event, events.RepoPushEvent):
             return self.repo_push_event_handler(event, data)
         elif isinstance(event, events.RepoCreateEvent):
             return self.repo_create_event_handler(event, data)
         elif isinstance(event, events.PullRequestEvent):
             return self.pull_request_event_handler(event, data)
         else:
             raise ValueError(
                 'event type `%s` not in supported list: %s' % (
                     event.__class__, events))
 
 
 def get_auth(settings):
     from requests.auth import HTTPBasicAuth
     username = settings.get('username')
     password = settings.get('password')
     if username and password:
         return HTTPBasicAuth(username, password)
     return None
 
 
 def get_web_token(settings):
     return settings['secret_token']
 
 
 def get_url_vars(url_vars):
     return '\n'.join(
         '{} - {}'.format('${' + key + '}', explanation)
         for key, explanation in url_vars)

rhodecode/integrations/types/hipchat.py

@@ -1,256 +1,246 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 from __future__ import unicode_literals
 import deform
 import logging
 import requests
 import colander
 import textwrap
-from collections import OrderedDict
 from mako.template import Template
 from rhodecode import events
 from rhodecode.translation import _
 from rhodecode.lib import helpers as h
 from rhodecode.lib.celerylib import run_task, async_task, RequestContextTask
 from rhodecode.lib.colander_utils import strip_whitespace
-from rhodecode.integrations.types.base import IntegrationTypeBase
+from rhodecode.integrations.types.base import (
+    IntegrationTypeBase, CommitParsingDataHandler)
 
 log = logging.getLogger(__name__)
 
 
 class HipchatSettingsSchema(colander.Schema):
     color_choices = [
         ('yellow', _('Yellow')),
         ('red', _('Red')),
         ('green', _('Green')),
         ('purple', _('Purple')),
         ('gray', _('Gray')),
     ]
 
     server_url = colander.SchemaNode(
         colander.String(),
         title=_('Hipchat server URL'),
         description=_('Hipchat integration url.'),
         default='',
         preparer=strip_whitespace,
         validator=colander.url,
         widget=deform.widget.TextInputWidget(
             placeholder='https://?.hipchat.com/v2/room/?/notification?auth_token=?',
         ),
     )
     notify = colander.SchemaNode(
         colander.Bool(),
         title=_('Notify'),
         description=_('Make a notification to the users in room.'),
         missing=False,
         default=False,
     )
     color = colander.SchemaNode(
         colander.String(),
         title=_('Color'),
         description=_('Background color of message.'),
         missing='',
         validator=colander.OneOf([x[0] for x in color_choices]),
         widget=deform.widget.Select2Widget(
             values=color_choices,
         ),
     )
 
 
 repo_push_template = Template('''
 <b>${data['actor']['username']}</b> pushed to repo <a href="${data['repo']['url']}">${data['repo']['repo_name']}</a>:
 <br>
 <ul>
 %for branch, branch_commits in branches_commits.items():
     <li>
         <a href="${branch_commits['branch']['url']}">branch: ${branch_commits['branch']['name']}</a>
         <ul>
     %for commit in branch_commits['commits']:
             <li><a href="${commit['url']}">${commit['short_id']}</a> - ${commit['message_html']}</li>
     %endfor
         </ul>
     </li>
 %endfor
 ''')
 
 
-class HipchatIntegrationType(IntegrationTypeBase):
+class HipchatIntegrationType(IntegrationTypeBase, CommitParsingDataHandler):
     key = 'hipchat'
     display_name = _('Hipchat')
     description = _('Send events such as repo pushes and pull requests to '
                     'your hipchat channel.')
 
     @classmethod
     def icon(cls):
         return '''<?xml version="1.0" encoding="utf-8"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" viewBox="0 0 1000 1000" enable-background="new 0 0 1000 1000" xml:space="preserve"><g><g transform="translate(0.000000,511.000000) scale(0.100000,-0.100000)"><path fill="#205281" d="M4197.1,4662.4c-1661.5-260.4-3018-1171.6-3682.6-2473.3C219.9,1613.6,100,1120.3,100,462.6c0-1014,376.8-1918.4,1127-2699.4C2326.7-3377.6,3878.5-3898.3,5701-3730.5l486.5,44.5l208.9-123.3c637.2-373.4,1551.8-640.6,2240.4-650.9c304.9-6.9,335.7,0,417.9,75.4c185,174.7,147.3,411.1-89.1,548.1c-315.2,181.6-620,544.7-733.1,870.1l-51.4,157.6l472.7,472.7c349.4,349.4,520.7,551.5,657.7,774.2c784.5,1281.2,784.5,2788.5,0,4052.6c-236.4,376.8-794.8,966-1178.4,1236.7c-572.1,407.7-1264.1,709.1-1993.7,870.1c-267.2,58.2-479.6,75.4-1038,82.2C4714.4,4686.4,4310.2,4679.6,4197.1,4662.4z M5947.6,3740.9c1856.7-380.3,3127.6-1709.4,3127.6-3275c0-1000.3-534.4-1949.2-1466.2-2600.1c-188.4-133.6-287.8-226.1-301.5-284.4c-41.1-157.6,263.8-938.6,397.4-1020.8c20.5-10.3,34.3-44.5,34.3-75.4c0-167.8-811.9,195.3-1363.4,609.8l-181.6,137l-332.3-58.2c-445.3-78.8-1281.2-78.8-1702.6,0C2796-2569.2,1734.1-1832.6,1220.2-801.5C983.8-318.5,905,51.5,929,613.3c27.4,640.6,243.2,1192.1,685.1,1740.3c620,770.8,1661.5,1305.2,2822.8,1452.5C4806.9,3854,5553.7,3819.7,5947.6,3740.9z"/><path fill="#205281" d="M2381.5-345.9c-75.4-106.2-68.5-167.8,34.3-322c332.3-500.2,1010.6-928.4,1760.8-1120.2c417.9-106.2,1226.4-106.2,1644.3,0c712.5,181.6,1270.9,517.3,1685.4,1014C7681-561.7,7715.3-424.7,7616-325.4c-89.1,89.1-167.9,65.1-431.7-133.6c-835.8-630.3-2028-856.4-3086.5-585.8C3683.3-938.6,3142-685,2830.3-448.7C2576.8-253.4,2463.7-229.4,2381.5-345.9z"/></g></g><!-- Svg Vector Icons : http://www.onlinewebfonts.com/icon --></svg>'''
 
     valid_events = [
         events.PullRequestCloseEvent,
         events.PullRequestMergeEvent,
         events.PullRequestUpdateEvent,
         events.PullRequestCommentEvent,
         events.PullRequestReviewEvent,
         events.PullRequestCreateEvent,
         events.RepoPushEvent,
         events.RepoCreateEvent,
     ]
 
     def send_event(self, event):
         if event.__class__ not in self.valid_events:
             log.debug('event not valid: %r' % event)
             return
 
         if event.name not in self.settings['events']:
             log.debug('event ignored: %r' % event)
             return
 
         data = event.as_dict()
 
         text = '<b>%s<b> caused a <b>%s</b> event' % (
             data['actor']['username'], event.name)
 
         log.debug('handling hipchat event for %s' % event.name)
 
         if isinstance(event, events.PullRequestCommentEvent):
             text = self.format_pull_request_comment_event(event, data)
         elif isinstance(event, events.PullRequestReviewEvent):
             text = self.format_pull_request_review_event(event, data)
         elif isinstance(event, events.PullRequestEvent):
             text = self.format_pull_request_event(event, data)
         elif isinstance(event, events.RepoPushEvent):
             text = self.format_repo_push_event(data)
         elif isinstance(event, events.RepoCreateEvent):
             text = self.format_repo_create_event(data)
         else:
             log.error('unhandled event type: %r' % event)
 
         run_task(post_text_to_hipchat, self.settings, text)
 
     def settings_schema(self):
         schema = HipchatSettingsSchema()
         schema.add(colander.SchemaNode(
             colander.Set(),
             widget=deform.widget.CheckboxChoiceWidget(
                 values=sorted(
                     [(e.name, e.display_name) for e in self.valid_events]
                 )
             ),
             description="Events activated for this integration",
             name='events'
         ))
 
         return schema
 
     def format_pull_request_comment_event(self, event, data):
         comment_text = data['comment']['text']
         if len(comment_text) > 200:
             comment_text = '{comment_text}<a href="{comment_url}">...<a/>'.format(
                 comment_text=h.html_escape(comment_text[:200]),
                 comment_url=data['comment']['url'],
             )
 
         comment_status = ''
         if data['comment']['status']:
             comment_status = '[{}]: '.format(data['comment']['status'])
 
         return (textwrap.dedent(
             '''
             {user} commented on pull request <a href="{pr_url}">{number}</a> - {pr_title}:
             >>> {comment_status}{comment_text}
             ''').format(
                 comment_status=comment_status,
                 user=data['actor']['username'],
                 number=data['pullrequest']['pull_request_id'],
                 pr_url=data['pullrequest']['url'],
                 pr_status=data['pullrequest']['status'],
                 pr_title=h.html_escape(data['pullrequest']['title']),
                 comment_text=h.html_escape(comment_text)
             )
         )
 
     def format_pull_request_review_event(self, event, data):
         return (textwrap.dedent(
             '''
             Status changed to {pr_status} for pull request <a href="{pr_url}">#{number}</a> - {pr_title}
             ''').format(
                 user=data['actor']['username'],
                 number=data['pullrequest']['pull_request_id'],
                 pr_url=data['pullrequest']['url'],
                 pr_status=data['pullrequest']['status'],
                 pr_title=h.html_escape(data['pullrequest']['title']),
             )
         )
 
     def format_pull_request_event(self, event, data):
         action = {
             events.PullRequestCloseEvent: 'closed',
             events.PullRequestMergeEvent: 'merged',
             events.PullRequestUpdateEvent: 'updated',
             events.PullRequestCreateEvent: 'created',
         }.get(event.__class__, str(event.__class__))
 
         return ('Pull request <a href="{url}">#{number}</a> - {title} '
                 '{action} by <b>{user}</b>').format(
             user=data['actor']['username'],
             number=data['pullrequest']['pull_request_id'],
             url=data['pullrequest']['url'],
             title=h.html_escape(data['pullrequest']['title']),
             action=action
         )
 
     def format_repo_push_event(self, data):
-        branch_data = {branch['name']: branch
-                       for branch in data['push']['branches']}
-
-        branches_commits = OrderedDict()
-        for commit in data['push']['commits']:
-            if commit['branch'] not in branches_commits:
-                branch_commits = {'branch': branch_data[commit['branch']],
-                                  'commits': []}
-                branches_commits[commit['branch']] = branch_commits
-
-            branch_commits = branches_commits[commit['branch']]
-            branch_commits['commits'].append(commit)
+        branches_commits = self.aggregate_branch_data(
+            data['push']['branches'], data['push']['commits'])
 
         result = repo_push_template.render(
             data=data,
             branches_commits=branches_commits,
         )
         return result
 
     def format_repo_create_event(self, data):
         return '<a href="{}">{}</a> ({}) repository created by <b>{}</b>'.format(
             data['repo']['url'],
             h.html_escape(data['repo']['repo_name']),
             data['repo']['repo_type'],
             data['actor']['username'],
         )
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def post_text_to_hipchat(settings, text):
     log.debug('sending %s to hipchat %s' % (text, settings['server_url']))
     resp = requests.post(settings['server_url'], json={
         "message": text,
         "color": settings.get('color', 'yellow'),
         "notify": settings.get('notify', False),
     })
     resp.raise_for_status()  # raise exception on a failed request

rhodecode/integrations/types/slack.py

@@ -1,338 +1,329 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 from __future__ import unicode_literals
 import re
 import time
 import textwrap
 import logging
 
 import deform
 import requests
 import colander
 from mako.template import Template
-from collections import OrderedDict
 
 from rhodecode import events
 from rhodecode.translation import _
 from rhodecode.lib import helpers as h
 from rhodecode.lib.celerylib import run_task, async_task, RequestContextTask
 from rhodecode.lib.colander_utils import strip_whitespace
-from rhodecode.integrations.types.base import IntegrationTypeBase
+from rhodecode.integrations.types.base import (
+    IntegrationTypeBase, CommitParsingDataHandler)
 
 log = logging.getLogger(__name__)
 
 
 class SlackSettingsSchema(colander.Schema):
     service = colander.SchemaNode(
         colander.String(),
         title=_('Slack service URL'),
         description=h.literal(_(
             'This can be setup at the '
             '<a href="https://my.slack.com/services/new/incoming-webhook/">'
             'slack app manager</a>')),
         default='',
         preparer=strip_whitespace,
         validator=colander.url,
         widget=deform.widget.TextInputWidget(
             placeholder='https://hooks.slack.com/services/...',
         ),
     )
     username = colander.SchemaNode(
         colander.String(),
         title=_('Username'),
         description=_('Username to show notifications coming from.'),
         missing='Rhodecode',
         preparer=strip_whitespace,
         widget=deform.widget.TextInputWidget(
             placeholder='Rhodecode'
         ),
     )
     channel = colander.SchemaNode(
         colander.String(),
         title=_('Channel'),
         description=_('Channel to send notifications to.'),
         missing='',
         preparer=strip_whitespace,
         widget=deform.widget.TextInputWidget(
             placeholder='#general'
         ),
     )
     icon_emoji = colander.SchemaNode(
         colander.String(),
         title=_('Emoji'),
         description=_('Emoji to use eg. :studio_microphone:'),
         missing='',
         preparer=strip_whitespace,
         widget=deform.widget.TextInputWidget(
             placeholder=':studio_microphone:'
         ),
     )
 
 
-class SlackIntegrationType(IntegrationTypeBase):
+class SlackIntegrationType(IntegrationTypeBase, CommitParsingDataHandler):
     key = 'slack'
     display_name = _('Slack')
     description = _('Send events such as repo pushes and pull requests to '
                     'your slack channel.')
 
     @classmethod
     def icon(cls):
         return '''<?xml version="1.0" encoding="UTF-8" standalone="no"?><svg viewBox="0 0 256 256" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" preserveAspectRatio="xMidYMid"><g><path d="M165.963541,15.8384262 C162.07318,3.86308197 149.212328,-2.69009836 137.239082,1.20236066 C125.263738,5.09272131 118.710557,17.9535738 122.603016,29.9268197 L181.550164,211.292328 C185.597902,222.478689 197.682361,228.765377 209.282098,225.426885 C221.381246,221.943607 228.756984,209.093246 224.896,197.21023 C224.749115,196.756984 165.963541,15.8384262 165.963541,15.8384262" fill="#DFA22F"></path><path d="M74.6260984,45.515541 C70.7336393,33.5422951 57.8727869,26.9891148 45.899541,30.8794754 C33.9241967,34.7698361 27.3710164,47.6306885 31.2634754,59.6060328 L90.210623,240.971541 C94.2583607,252.157902 106.34282,258.44459 117.942557,255.104 C130.041705,251.62282 137.417443,238.772459 133.556459,226.887344 C133.409574,226.436197 74.6260984,45.515541 74.6260984,45.515541" fill="#3CB187"></path><path d="M240.161574,166.045377 C252.136918,162.155016 258.688,149.294164 254.797639,137.31882 C250.907279,125.345574 238.046426,118.792393 226.07318,122.682754 L44.7076721,181.632 C33.5213115,185.677639 27.234623,197.762098 30.5731148,209.361836 C34.0563934,221.460984 46.9067541,228.836721 58.7897705,224.975738 C59.2430164,224.828852 240.161574,166.045377 240.161574,166.045377" fill="#CE1E5B"></path><path d="M82.507541,217.270557 C94.312918,213.434754 109.528131,208.491016 125.855475,203.186361 C122.019672,191.380984 117.075934,176.163672 111.76918,159.83423 L68.4191475,173.924721 L82.507541,217.270557" fill="#392538"></path><path d="M173.847082,187.591344 C190.235279,182.267803 205.467279,177.31777 217.195016,173.507148 C213.359213,161.70177 208.413377,146.480262 203.106623,130.146623 L159.75659,144.237115 L173.847082,187.591344" fill="#BB242A"></path><path d="M210.484459,74.7058361 C222.457705,70.8154754 229.010885,57.954623 225.120525,45.9792787 C221.230164,34.0060328 208.369311,27.4528525 196.393967,31.3432131 L15.028459,90.292459 C3.84209836,94.3380984 -2.44459016,106.422557 0.896,118.022295 C4.37718033,130.121443 17.227541,137.49718 29.1126557,133.636197 C29.5638033,133.489311 210.484459,74.7058361 210.484459,74.7058361" fill="#72C5CD"></path><path d="M52.8220328,125.933115 C64.6274098,122.097311 79.8468197,117.151475 96.1762623,111.84682 C90.8527213,95.4565246 85.9026885,80.2245246 82.0920656,68.4946885 L38.731541,82.5872787 L52.8220328,125.933115" fill="#248C73"></path><path d="M144.159475,96.256 C160.551869,90.9303607 175.785967,85.9803279 187.515803,82.1676066 C182.190164,65.7752131 177.240131,50.5390164 173.42741,38.807082 L130.068984,52.8996721 L144.159475,96.256" fill="#62803A"></path></g></svg>'''
 
     valid_events = [
         events.PullRequestCloseEvent,
         events.PullRequestMergeEvent,
         events.PullRequestUpdateEvent,
         events.PullRequestCommentEvent,
         events.PullRequestReviewEvent,
         events.PullRequestCreateEvent,
         events.RepoPushEvent,
         events.RepoCreateEvent,
     ]
 
     def send_event(self, event):
         if event.__class__ not in self.valid_events:
             log.debug('event not valid: %r' % event)
             return
 
         if event.name not in self.settings['events']:
             log.debug('event ignored: %r' % event)
             return
 
         data = event.as_dict()
 
         # defaults
         title = '*%s* caused a *%s* event' % (
             data['actor']['username'], event.name)
         text = '*%s* caused a *%s* event' % (
             data['actor']['username'], event.name)
         fields = None
         overrides = None
 
         log.debug('handling slack event for %s' % event.name)
 
         if isinstance(event, events.PullRequestCommentEvent):
             (title, text, fields, overrides) \
                 = self.format_pull_request_comment_event(event, data)
         elif isinstance(event, events.PullRequestReviewEvent):
             title, text = self.format_pull_request_review_event(event, data)
         elif isinstance(event, events.PullRequestEvent):
             title, text = self.format_pull_request_event(event, data)
         elif isinstance(event, events.RepoPushEvent):
             title, text = self.format_repo_push_event(data)
         elif isinstance(event, events.RepoCreateEvent):
             title, text = self.format_repo_create_event(data)
         else:
             log.error('unhandled event type: %r' % event)
 
         run_task(post_text_to_slack, self.settings, title, text, fields, overrides)
 
     def settings_schema(self):
         schema = SlackSettingsSchema()
         schema.add(colander.SchemaNode(
             colander.Set(),
             widget=deform.widget.CheckboxChoiceWidget(
                 values=sorted(
                     [(e.name, e.display_name) for e in self.valid_events]
                 )
             ),
             description="Events activated for this integration",
             name='events'
         ))
 
         return schema
 
     def format_pull_request_comment_event(self, event, data):
         comment_text = data['comment']['text']
         if len(comment_text) > 200:
             comment_text = '<{comment_url}|{comment_text}...>'.format(
                 comment_text=comment_text[:200],
                 comment_url=data['comment']['url'],
             )
 
         fields = None
         overrides = None
         status_text = None
 
         if data['comment']['status']:
             status_color = {
                 'approved': '#0ac878',
                 'rejected': '#e85e4d'}.get(data['comment']['status'])
 
             if status_color:
                 overrides = {"color": status_color}
 
             status_text = data['comment']['status']
 
         if data['comment']['file']:
             fields = [
                 {
                     "title": "file",
                     "value": data['comment']['file']
                 },
                 {
                     "title": "line",
                     "value": data['comment']['line']
                 }
             ]
 
         title = Template(textwrap.dedent(r'''
         *${data['actor']['username']}* left ${data['comment']['type']} on pull request <${data['pullrequest']['url']}|#${data['pullrequest']['pull_request_id']}>:
         ''')).render(data=data, comment=event.comment)
 
         text = Template(textwrap.dedent(r'''
         *pull request title*: ${pr_title}
         % if status_text:
         *submitted status*: `${status_text}`
         % endif
         >>> ${comment_text}
         ''')).render(comment_text=comment_text,
                      pr_title=data['pullrequest']['title'],
                      status_text=status_text)
 
         return title, text, fields, overrides
 
     def format_pull_request_review_event(self, event, data):
         title = Template(textwrap.dedent(r'''
         *${data['actor']['username']}* changed status of pull request <${data['pullrequest']['url']}|#${data['pullrequest']['pull_request_id']} to `${data['pullrequest']['status']}`>:
         ''')).render(data=data)
 
         text = Template(textwrap.dedent(r'''
         *pull request title*: ${pr_title}
         ''')).render(
             pr_title=data['pullrequest']['title'],
         )
 
         return title, text
 
     def format_pull_request_event(self, event, data):
         action = {
             events.PullRequestCloseEvent: 'closed',
             events.PullRequestMergeEvent: 'merged',
             events.PullRequestUpdateEvent: 'updated',
             events.PullRequestCreateEvent: 'created',
         }.get(event.__class__, str(event.__class__))
 
         title = Template(textwrap.dedent(r'''
         *${data['actor']['username']}* `${action}` pull request <${data['pullrequest']['url']}|#${data['pullrequest']['pull_request_id']}>:
         ''')).render(data=data, action=action)
 
         text = Template(textwrap.dedent(r'''
         *pull request title*: ${pr_title}
         %if data['pullrequest']['commits']:
         *commits*: ${len(data['pullrequest']['commits'])}
         %endif
         ''')).render(
             pr_title=data['pullrequest']['title'],
             data=data
         )
 
         return title, text
 
     def format_repo_push_event(self, data):
-        branch_data = {branch['name']: branch
-                       for branch in data['push']['branches']}
 
-        branches_commits = OrderedDict()
-        for commit in data['push']['commits']:
-            if commit['branch'] not in branches_commits:
-                branch_commits = {'branch': branch_data[commit['branch']],
-                                  'commits': []}
-                branches_commits[commit['branch']] = branch_commits
-
-            branch_commits = branches_commits[commit['branch']]
-            branch_commits['commits'].append(commit)
+        branches_commits = self.aggregate_branch_data(
+            data['push']['branches'], data['push']['commits'])
 
         title = Template(r'''
         *${data['actor']['username']}* pushed to repo <${data['repo']['url']}|${data['repo']['repo_name']}>:
         ''').render(data=data)
 
         repo_push_template = Template(textwrap.dedent(r'''
         %for branch, branch_commits in branches_commits.items():
         ${len(branch_commits['commits'])} ${'commit' if len(branch_commits['commits']) == 1 else 'commits'} on branch: <${branch_commits['branch']['url']}|${branch_commits['branch']['name']}>
         %for commit in branch_commits['commits']:
         `<${commit['url']}|${commit['short_id']}>` - ${commit['message_html']|html_to_slack_links}
         %endfor
         %endfor
         '''))
 
         text = repo_push_template.render(
             data=data,
             branches_commits=branches_commits,
             html_to_slack_links=html_to_slack_links,
         )
 
         return title, text
 
     def format_repo_create_event(self, data):
         title = Template(r'''
         *${data['actor']['username']}* created new repository ${data['repo']['repo_name']}:
         ''').render(data=data)
 
         text = Template(textwrap.dedent(r'''
         repo_url: ${data['repo']['url']}
         repo_type: ${data['repo']['repo_type']}
         ''')).render(data=data)
 
         return title, text
 
 
 def html_to_slack_links(message):
     return re.compile(r'<a .*?href=["\'](.+?)".*?>(.+?)</a>').sub(
         r'<\1|\2>', message)
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def post_text_to_slack(settings, title, text, fields=None, overrides=None):
     log.debug('sending %s (%s) to slack %s' % (
         title, text, settings['service']))
 
     fields = fields or []
     overrides = overrides or {}
 
     message_data = {
                 "fallback": text,
                 "color": "#427cc9",
                 "pretext": title,
                 #"author_name": "Bobby Tables",
                 #"author_link": "http://flickr.com/bobby/",
                 #"author_icon": "http://flickr.com/icons/bobby.jpg",
                 #"title": "Slack API Documentation",
                 #"title_link": "https://api.slack.com/",
                 "text": text,
                 "fields": fields,
                 #"image_url": "http://my-website.com/path/to/image.jpg",
                 #"thumb_url": "http://example.com/path/to/thumb.png",
                 "footer": "RhodeCode",
                 #"footer_icon": "",
                 "ts": time.time(),
                 "mrkdwn_in": ["pretext", "text"]
     }
     message_data.update(overrides)
     json_message = {
         "icon_emoji": settings.get('icon_emoji', ':studio_microphone:'),
         "channel": settings.get('channel', ''),
         "username": settings.get('username', 'Rhodecode'),
         "attachments": [message_data]
     }
 
     resp = requests.post(settings['service'], json=json_message)
     resp.raise_for_status()  # raise exception on a failed request