rhodecode/integrations/types/base.py

# Copyright (C) 2012-2020 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/

import colander
import string
import collections
import logging
import requests
import urllib.request
import urllib.parse
import urllib.error
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry

from mako import exceptions

from rhodecode.lib.utils2 import safe_str
from rhodecode.translation import _


log = logging.getLogger(__name__)


class UrlTmpl(string.Template):

    def safe_substitute(self, **kws):
        # url encode the kw for usage in url
        kws = {k: urllib.parse.quote(safe_str(v)) for k, v in kws.items()}
        return super(UrlTmpl, self).safe_substitute(**kws)


class IntegrationTypeBase(object):
    """ Base class for IntegrationType plugins """
    is_dummy = False
    description = ''

    @classmethod
    def icon(cls):
        return '''
        <?xml version="1.0" encoding="UTF-8" standalone="no"?>
        <svg
           xmlns:dc="http://purl.org/dc/elements/1.1/"
           xmlns:cc="http://creativecommons.org/ns#"
           xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
           xmlns:svg="http://www.w3.org/2000/svg"
           xmlns="http://www.w3.org/2000/svg"
           xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
           xmlns:inkscape="http://setwww.inkscape.org/namespaces/inkscape"
           viewBox="0 -256 1792 1792"
           id="svg3025"
           version="1.1"
           inkscape:version="0.48.3.1 r9886"
           width="100%"
           height="100%"
           sodipodi:docname="cog_font_awesome.svg">
          <metadata
             id="metadata3035">
            <rdf:RDF>
              <cc:Work
                 rdf:about="">
                <dc:format>image/svg+xml</dc:format>
                <dc:type
                   rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
              </cc:Work>
            </rdf:RDF>
          </metadata>
          <defs
             id="defs3033" />
          <sodipodi:namedview
             pagecolor="#ffffff"
             bordercolor="#666666"
             borderopacity="1"
             objecttolerance="10"
             gridtolerance="10"
             guidetolerance="10"
             inkscape:pageopacity="0"
             inkscape:pageshadow="2"
             inkscape:window-width="640"
             inkscape:window-height="480"
             id="namedview3031"
             showgrid="false"
             inkscape:zoom="0.13169643"
             inkscape:cx="896"
             inkscape:cy="896"
             inkscape:window-x="0"
             inkscape:window-y="25"
             inkscape:window-maximized="0"
             inkscape:current-layer="svg3025" />
          <g
             transform="matrix(1,0,0,-1,121.49153,1285.4237)"
             id="g3027">
            <path
               d="m 1024,640 q 0,106 -75,181 -75,75 -181,75 -106,0 -181,-75 -75,-75 -75,-181 0,-106 75,-181 75,-75 181,-75 106,0 181,75 75,75 75,181 z m 512,109 V 527 q 0,-12 -8,-23 -8,-11 -20,-13 l -185,-28 q -19,-54 -39,-91 35,-50 107,-138 10,-12 10,-25 0,-13 -9,-23 -27,-37 -99,-108 -72,-71 -94,-71 -12,0 -26,9 l -138,108 q -44,-23 -91,-38 -16,-136 -29,-186 -7,-28 -36,-28 H 657 q -14,0 -24.5,8.5 Q 622,-111 621,-98 L 593,86 q -49,16 -90,37 L 362,16 Q 352,7 337,7 323,7 312,18 186,132 147,186 q -7,10 -7,23 0,12 8,23 15,21 51,66.5 36,45.5 54,70.5 -27,50 -41,99 L 29,495 Q 16,497 8,507.5 0,518 0,531 v 222 q 0,12 8,23 8,11 19,13 l 186,28 q 14,46 39,92 -40,57 -107,138 -10,12 -10,24 0,10 9,23 26,36 98.5,107.5 72.5,71.5 94.5,71.5 13,0 26,-10 l 138,-107 q 44,23 91,38 16,136 29,186 7,28 36,28 h 222 q 14,0 24.5,-8.5 Q 914,1391 915,1378 l 28,-184 q 49,-16 90,-37 l 142,107 q 9,9 24,9 13,0 25,-10 129,-119 165,-170 7,-8 7,-22 0,-12 -8,-23 -15,-21 -51,-66.5 -36,-45.5 -54,-70.5 26,-50 41,-98 l 183,-28 q 13,-2 21,-12.5 8,-10.5 8,-23.5 z"
               id="path3029"
               inkscape:connector-curvature="0"
               style="fill:currentColor" />
          </g>
        </svg>
        '''

    def __init__(self, settings):
        """
        :param settings: dict of settings to be used for the integration
        """
        self.settings = settings

    def settings_schema(self):
        """
        A colander schema of settings for the integration type
        """
        return colander.Schema()

    def event_enabled(self, event):
        """
        Checks if submitted event is enabled based on the plugin settings
        :param event:
        :return: bool
        """
        allowed_events = self.settings.get('events') or []
        if event.name not in allowed_events:
            log.debug('event ignored: %r event %s not in allowed set of events %s',
                      event, event.name, allowed_events)
            return False
        return True


class EEIntegration(IntegrationTypeBase):
    description = 'Integration available in RhodeCode EE edition.'
    is_dummy = True

    def __init__(self, name, key, settings=None):
        self.display_name = name
        self.key = key
        super(EEIntegration, self).__init__(settings)


# Helpers #
# updating this required to update the `common_vars` as well.
WEBHOOK_URL_VARS = [
    # GENERAL
    ('General', [
        ('event_name', 'Unique name of the event type, e.g pullrequest-update'),
        ('repo_name', 'Full name of the repository'),
        ('repo_type', 'VCS type of repository'),
        ('repo_id', 'Unique id of repository'),
        ('repo_url', 'Repository url'),
        ]
    ),
    # extra repo fields
    ('Repository', [
        ('extra:<extra_key_name>', 'Extra repo variables, read from its settings.'),
        ]
    ),
    # special attrs below that we handle, using multi-call
    ('Commit push - Multicalls', [
        ('branch', 'Name of each branch submitted, if any.'),
        ('branch_head', 'Head ID of pushed branch (full sha of last commit), if any.'),
        ('commit_id', 'ID (full sha) of each commit submitted, if any.'),
        ]
    ),
    # pr events vars
    ('Pull request', [
        ('pull_request_id', 'Unique ID of the pull request.'),
        ('pull_request_title', 'Title of the pull request.'),
        ('pull_request_url', 'Pull request url.'),
        ('pull_request_shadow_url', 'Pull request shadow repo clone url.'),
        ('pull_request_commits_uid', 'Calculated UID of all commits inside the PR. '
                                     'Changes after PR update'),
        ]
    ),
    # commit comment event vars
    ('Commit comment', [
        ('commit_comment_id', 'Unique ID of the comment made on a commit.'),
        ('commit_comment_text', 'Text of commit comment.'),
        ('commit_comment_type', 'Type of comment, e.g note/todo.'),

        ('commit_comment_f_path', 'Optionally path of file for inline comments.'),
        ('commit_comment_line_no', 'Line number of the file: eg o10, or n200'),

        ('commit_comment_commit_id', 'Commit id that comment was left at.'),
        ('commit_comment_commit_branch', 'Commit branch that comment was left at'),
        ('commit_comment_commit_message', 'Commit message that comment was left at'),
        ]
    ),
    # user who triggers the call
    ('Caller', [
        ('username', 'User who triggered the call.'),
        ('user_id', 'User id who triggered the call.'),
        ]
    ),
]

# common vars for url template used for CI plugins. Shared with webhook
CI_URL_VARS = WEBHOOK_URL_VARS


class CommitParsingDataHandler(object):

    def aggregate_branch_data(self, branches, commits):
        branch_data = collections.OrderedDict()
        for obj in branches:
            branch_data[obj['name']] = obj

        branches_commits = collections.OrderedDict()
        for commit in commits:
            if commit.get('git_ref_change'):
                # special case for GIT that allows creating tags,
                # deleting branches without associated commit
                continue
            commit_branch = commit['branch']

            if commit_branch not in branches_commits:
                _branch = branch_data[commit_branch] \
                    if commit_branch else commit_branch
                branch_commits = {'branch': _branch,
                                  'branch_head': '',
                                  'commits': []}
                branches_commits[commit_branch] = branch_commits

            branch_commits = branches_commits[commit_branch]
            branch_commits['commits'].append(commit)
            branch_commits['branch_head'] = commit['raw_id']
        return branches_commits


class WebhookDataHandler(CommitParsingDataHandler):
    name = 'webhook'

    def __init__(self, template_url, headers):
        self.template_url = template_url
        self.headers = headers

    def get_base_parsed_template(self, data):
        """
        initially parses the passed in template with some common variables
        available on ALL calls
        """
        # note: make sure to update the `WEBHOOK_URL_VARS` if this changes
        common_vars = {
            'repo_name': data['repo']['repo_name'],
            'repo_type': data['repo']['repo_type'],
            'repo_id': data['repo']['repo_id'],
            'repo_url': data['repo']['url'],
            'username': data['actor']['username'],
            'user_id': data['actor']['user_id'],
            'event_name': data['name']
        }

        extra_vars = {}
        for extra_key, extra_val in data['repo']['extra_fields'].items():
            extra_vars['extra__{}'.format(extra_key)] = extra_val
        common_vars.update(extra_vars)

        template_url = self.template_url.replace('${extra:', '${extra__')
        for k, v in common_vars.items():
            template_url = UrlTmpl(template_url).safe_substitute(**{k: v})
        return template_url

    def repo_push_event_handler(self, event, data):
        url = self.get_base_parsed_template(data)
        url_calls = []

        branches_commits = self.aggregate_branch_data(
            data['push']['branches'], data['push']['commits'])
        if '${branch}' in url or '${branch_head}' in url or '${commit_id}' in url:
            # call it multiple times, for each branch if used in variables
            for branch, commit_ids in branches_commits.items():
                branch_url = UrlTmpl(url).safe_substitute(branch=branch)

                if '${branch_head}' in branch_url:
                    # last commit in the aggregate is the head of the branch
                    branch_head = commit_ids['branch_head']
                    branch_url = UrlTmpl(branch_url).safe_substitute(branch_head=branch_head)

                # call further down for each commit if used
                if '${commit_id}' in branch_url:
                    for commit_data in commit_ids['commits']:
                        commit_id = commit_data['raw_id']
                        commit_url = UrlTmpl(branch_url).safe_substitute(commit_id=commit_id)
                        # register per-commit call
                        log.debug(
                            'register %s call(%s) to url %s',
                            self.name, event, commit_url)
                        url_calls.append(
                            (commit_url, self.headers, data))

                else:
                    # register per-branch call
                    log.debug('register %s call(%s) to url %s',
                              self.name, event, branch_url)
                    url_calls.append((branch_url, self.headers, data))

        else:
            log.debug('register %s call(%s) to url %s', self.name, event, url)
            url_calls.append((url, self.headers, data))

        return url_calls

    def repo_commit_comment_handler(self, event, data):
        url = self.get_base_parsed_template(data)
        log.debug('register %s call(%s) to url %s', self.name, event, url)
        comment_vars = [
            ('commit_comment_id', data['comment']['comment_id']),
            ('commit_comment_text', data['comment']['comment_text']),
            ('commit_comment_type', data['comment']['comment_type']),

            ('commit_comment_f_path', data['comment']['comment_f_path']),
            ('commit_comment_line_no', data['comment']['comment_line_no']),

            ('commit_comment_commit_id', data['commit']['commit_id']),
            ('commit_comment_commit_branch', data['commit']['commit_branch']),
            ('commit_comment_commit_message', data['commit']['commit_message']),
        ]
        for k, v in comment_vars:
            url = UrlTmpl(url).safe_substitute(**{k: v})

        return [(url, self.headers, data)]

    def repo_commit_comment_edit_handler(self, event, data):
        url = self.get_base_parsed_template(data)
        log.debug('register %s call(%s) to url %s', self.name, event, url)
        comment_vars = [
            ('commit_comment_id', data['comment']['comment_id']),
            ('commit_comment_text', data['comment']['comment_text']),
            ('commit_comment_type', data['comment']['comment_type']),

            ('commit_comment_f_path', data['comment']['comment_f_path']),
            ('commit_comment_line_no', data['comment']['comment_line_no']),

            ('commit_comment_commit_id', data['commit']['commit_id']),
            ('commit_comment_commit_branch', data['commit']['commit_branch']),
            ('commit_comment_commit_message', data['commit']['commit_message']),
        ]
        for k, v in comment_vars:
            url = UrlTmpl(url).safe_substitute(**{k: v})

        return [(url, self.headers, data)]

    def repo_create_event_handler(self, event, data):
        url = self.get_base_parsed_template(data)
        log.debug('register %s call(%s) to url %s', self.name, event, url)
        return [(url, self.headers, data)]

    def pull_request_event_handler(self, event, data):
        url = self.get_base_parsed_template(data)
        log.debug('register %s call(%s) to url %s', self.name, event, url)
        pr_vars = [
            ('pull_request_id', data['pullrequest']['pull_request_id']),
            ('pull_request_title', data['pullrequest']['title']),
            ('pull_request_url', data['pullrequest']['url']),
            ('pull_request_shadow_url', data['pullrequest']['shadow_url']),
            ('pull_request_commits_uid', data['pullrequest']['commits_uid']),
        ]
        for k, v in pr_vars:
            url = UrlTmpl(url).safe_substitute(**{k: v})

        return [(url, self.headers, data)]

    def __call__(self, event, data):
        from rhodecode import events

        if isinstance(event, events.RepoPushEvent):
            return self.repo_push_event_handler(event, data)
        elif isinstance(event, events.RepoCreateEvent):
            return self.repo_create_event_handler(event, data)
        elif isinstance(event, events.RepoCommitCommentEvent):
            return self.repo_commit_comment_handler(event, data)
        elif isinstance(event, events.RepoCommitCommentEditEvent):
            return self.repo_commit_comment_edit_handler(event, data)
        elif isinstance(event, events.PullRequestEvent):
            return self.pull_request_event_handler(event, data)
        else:
            raise ValueError(
                'event type `{}` has no handler defined'.format(event.__class__))


def get_auth(settings):
    from requests.auth import HTTPBasicAuth
    username = settings.get('username')
    password = settings.get('password')
    if username and password:
        return HTTPBasicAuth(username, password)
    return None


def get_web_token(settings):
    return settings['secret_token']


def get_url_vars(url_vars):
    items = []

    for section, section_items in url_vars:
        items.append('\n*{}*'.format(section))
        for key, explanation in section_items:
            items.append('  {} - {}'.format('${' + key + '}', explanation))
    return '\n'.join(items)


def render_with_traceback(template, *args, **kwargs):
    try:
        return template.render(*args, **kwargs)
    except Exception:
        log.error(exceptions.text_error_template().render())
        raise


STATUS_400 = (400, 401, 403)
STATUS_500 = (500, 502, 504)


def requests_retry_call(
        retries=3, backoff_factor=0.3, status_forcelist=STATUS_400+STATUS_500,
        session=None):
    """
    session = requests_retry_session()
    response = session.get('http://example.com')

    :param retries:
    :param backoff_factor:
    :param status_forcelist:
    :param session:
    """
    session = session or requests.Session()
    retry = Retry(
        total=retries,
        read=retries,
        connect=retries,
        backoff_factor=backoff_factor,
        status_forcelist=status_forcelist,
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session