# HG changeset patch # User Marcin Kuzminski # Date 2018-10-23 09:45:45 # Node ID 6afdd8e7b848153a2163dd60be89deca12aa36de # Parent fb303e8b6b46160be0acf26fe1d77f907ddeb4ec rcextensions: new builtin rcextensions. - ability to expose output to users from rcextensions - new docs - linking to the code makes the tests always check the integrity of rcextensions - added new hook variables and described them to end users diff --git a/default.nix b/default.nix --- a/default.nix +++ b/default.nix @@ -201,6 +201,9 @@ let cp configs/production.ini $out/etc echo "[DONE]: saved enterprise-ce production.ini into $out/etc" + cp -r rhodecode/config/rcextensions $out/etc/rcextensions.tmpl + echo "[DONE]: saved enterprise-ce rcextensions into $out/etc/rcextensions.tmpl" + # python based programs need to be wrapped mkdir -p $out/bin diff --git a/docs/extensions/config-ext.rst b/docs/extensions/config-ext.rst deleted file mode 100644 --- a/docs/extensions/config-ext.rst +++ /dev/null @@ -1,150 +0,0 @@ -.. _config-ext: - -Configure |RCX| ---------------- - -To get the the built in plugins and extensions working the way you want them -to, you have to configure them to work with your services. An overview of -what needs to be done is: - -* :ref:`config-rcx-plugin` to carry out your desired actions once its hook is - triggered. There are default actions built in, but you may wish to alter - those. -* :ref:`config-rcx-hook` to execute actions for the plugin, when certain - actions are carried out with |RCE|. - -.. _config-rcx-plugin: - -Tweak a Default Plugin -^^^^^^^^^^^^^^^^^^^^^^ - -Each of the default plugins comes with a standard configuration, but you may -wish to change those settings. In this example, the Redmine plugin watches -for the words defined in the ``HASH_REGEX`` variable and takes actions if one -of those words is used in conjunction with a ``#{number}``, which matches a -ticket number in Redmine. You can configure this to work differently based on -the `Redmine documentation`_. - -.. code-block:: python - :emphasize-lines: 3-5, 37 - - import re - - HASH_REGEX = re.compile( - r"(?:fix|fixes|fixing|close|closes|closing)\s*#([0-9]+)\b", - re.IGNORECASE) - - - def link_to_commit(repo_url, commit_id): - rev_url = '%s/changeset/%s' % (repo_url, commit_id) - return '"%s":%s' % (commit_id[:6], rev_url) - - - def run(*args, **kwargs): - issues = kwargs['RM_ISSUES'] - if not issues: - return 0 - - # repo extra fields can control this, they should be propagated with - # extract repo fields - tracker_url = kwargs.get('redmine_tracker_url') or kwargs['RM_URL'] - project_id = kwargs.get('redmine_project_id') or kwargs['RM_PROJECT'] - api_key = kwargs.get('redmine_api_key') or kwargs['RM_APIKEY'] - - if project_id: - from redmine import Redmine - remote_redmine = Redmine(tracker_url, key=api_key) - project = remote_redmine.project.get(project_id) - repo_url = '%(server_url)s/%(repository)s' % kwargs - # for each fetched issue id make a redmine api call - for _id, details in issues.items(): - commits = ', '.join([link_to_commit(repo_url, - x['raw_id'],) - for x in details]) - issue = project.issues.get(int(_id)) - if issue: - issue.notes = 'Issue resolved by %s' % (commits,) - issue.status_id = 3 # Resolved - issue.save() - - -.. _config-rcx-hook: - -Configure a Hook -^^^^^^^^^^^^^^^^ - -To configure the default hooks in the -:file:`/home/{user}/.rccontrol/{instance-id}/rcextensions/__init.py__` file, -use the following steps. - -1. Configure the connection details, either in the file or import from a - dictionary. For these connection scenarios the following details need to - be configured. - -* **REDMINE_URL** = '' -* **REDMINE_API_KEY** = '' -* **SLACK_API_URL** = '?token=' -* **SLACK_API_KEY** = '' - -2. You will also need to configure other variables, such as the - **SLACK_ROOM** or **RM_PROJECT** (Redmine Project). These set where the - commit message is posted. Various hooks can take different variables and - they are documented in the file. - -3. Inside each hook you can then configure it to carry out actions - per service. In this example, the push hook is pushing to the Redmine and - Slack plugins on each push if the hook criteria are matched. - -.. code-block:: python - :emphasize-lines: 21-29, 37-44 - - def _push_hook(*args, **kwargs): - kwargs['commit_ids'] = kwargs['pushed_revs'] - - call = load_extension('extra_fields.py') - if call: - repo_extra_fields = call(**kwargs) - # now update if we have extra fields, they have precedence - # this way users can store any configuration inside - # the database per repo - for key, data in repo_extra_fields.items(): - kwargs[key] = data['field_value'] - - # fetch pushed commits - call = load_extension('extract_commits.py') - extracted_commits = {} - if call: - extracted_commits = call(**kwargs) - # store the commits for the next call chain - kwargs['COMMITS'] = extracted_commits - - # slack ! - call = load_extension('slack.py') - if call: - kwargs['INCOMING_WEBHOOK_URL'] = SLACK_API_URL - kwargs['SLACK_TOKEN'] = SLACK_API_KEY - kwargs['SLACK_ROOM'] = '#slack-channel' - kwargs['SLACK_FROM'] = 'Slack-Message-Poster' - kwargs['SLACK_FROM_ICON_EMOJI'] = ':slack-emoji:' - call(**kwargs) - - # fetch issues from given commits - call = load_extension('extract_issues.py') - issues = {} - if call: - issues = call(**kwargs) - - # redmine smart commits - call = load_extension('redmine_smart_commits.py') - if call: - kwargs['RM_URL'] = REDMINE_URL - kwargs['RM_APIKEY'] = REDMINE_API_KEY - kwargs['RM_PROJECT'] = None # uses extra_fields from repo - kwargs['RM_ISSUES'] = issues - call(**kwargs) - - return 0 - PUSH_HOOK = _push_hook - - -.. _Redmine documentation: http://www.redmine.org/projects/redmine/wiki/Rest_api diff --git a/docs/extensions/example-ext.py b/docs/extensions/example-ext.py deleted file mode 100644 --- a/docs/extensions/example-ext.py +++ /dev/null @@ -1,823 +0,0 @@ -""" -rcextensions module. - -""" - - -import os -import imp -import string -import functools - -here = os.path.dirname(os.path.abspath(__file__)) -registered_extensions = dict() - -class DotDict(dict): - - def __contains__(self, k): - try: - return dict.__contains__(self, k) or hasattr(self, k) - except: - return False - - # only called if k not found in normal places - def __getattr__(self, k): - try: - return object.__getattribute__(self, k) - except AttributeError: - try: - return self[k] - except KeyError: - raise AttributeError(k) - - def __setattr__(self, k, v): - try: - object.__getattribute__(self, k) - except AttributeError: - try: - self[k] = v - except: - raise AttributeError(k) - else: - object.__setattr__(self, k, v) - - def __delattr__(self, k): - try: - object.__getattribute__(self, k) - except AttributeError: - try: - del self[k] - except KeyError: - raise AttributeError(k) - else: - object.__delattr__(self, k) - - def toDict(self): - return unserialize(self) - - def __repr__(self): - keys = list(self.iterkeys()) - keys.sort() - args = ', '.join(['%s=%r' % (key, self[key]) for key in keys]) - return '%s(%s)' % (self.__class__.__name__, args) - - @staticmethod - def fromDict(d): - return serialize(d) - - -def serialize(x): - if isinstance(x, dict): - return DotDict((k, serialize(v)) for k, v in x.iteritems()) - elif isinstance(x, (list, tuple)): - return type(x)(serialize(v) for v in x) - else: - return x - - -def unserialize(x): - if isinstance(x, dict): - return dict((k, unserialize(v)) for k, v in x.iteritems()) - elif isinstance(x, (list, tuple)): - return type(x)(unserialize(v) for v in x) - else: - return x - - -def load_extension(filename, async=False): - """ - use to load extensions inside rcextension folder. - for example:: - - callback = load_extension('email.py', async=False) - if callback: - callback('foobar') - - put file named email.py inside rcextensions folder to load it. Changing - async=True will make the call of the plugin async, it's useful for - blocking calls like sending an email or notification with APIs. - """ - mod = ''.join(filename.split('.')[:-1]) - loaded = imp.load_source(mod, os.path.join(here, filename)) - - callback = getattr(loaded, 'run', None) - if not callback: - raise Exception('Plugin missing `run` method') - if async: - # modify callback so it's actually an async call - def _async_callback(*args, **kwargs): - import threading - thr = threading.Thread(target=callback, args=args, kwargs=kwargs) - thr.start() - if kwargs.get('_async_block'): - del kwargs['_async_block'] - thr.join() - - return _async_callback - return callback - - -def _verify_kwargs(expected_parameters, kwargs): - """ - Verify that exactly `expected_parameters` are passed in as `kwargs`. - """ - expected_parameters = set(expected_parameters) - kwargs_keys = set(kwargs.keys()) - if kwargs_keys != expected_parameters: - missing_kwargs = expected_parameters - kwargs_keys - unexpected_kwargs = kwargs_keys - expected_parameters - raise AssertionError( - "Missing parameters: %r, unexpected parameters: %s" % - (missing_kwargs, unexpected_kwargs)) - - -def verify_kwargs(required_args): - """ - decorator to verify extension calls arguments. - - :param required_args: - """ - def wrap(func): - def wrapper(*args, **kwargs): - _verify_kwargs(required_args, kwargs) - return func(*args, **kwargs) - return wrapper - return wrap - - -def register(name=None): - def wrap(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - # register load_extensions in kwargs, so we can chain plugins - kwargs['_load_extension'] = load_extension - # append this path for us to use added plugins or modules - import sys - _cur_path = os.path.dirname(os.path.abspath(__file__)) - if _cur_path not in sys.path: - sys.path.append(_cur_path) - - registered_extensions[func.__name__] = func - return func(*args, **kwargs) - return wrapper - return wrap - -# ============================================================================= -# END OF UTILITY FUNCTIONS HERE -# ============================================================================= - -# Additional mappings that are not present in the pygments lexers -# used for building stats -# format is {'ext':['Names']} eg. {'py':['Python']} note: there can be -# more than one name for extension -# NOTE: that this will override any mappings in LANGUAGES_EXTENSIONS_MAP -# build by pygments -EXTRA_MAPPINGS = {} - -# additional lexer definitions for custom files it's overrides pygments lexers, -# and uses defined name of lexer to colorize the files. Format is {'ext': -# 'lexer_name'} List of lexers can be printed running: -# >> python -c "import pprint;from pygments import lexers; -# pprint.pprint([(x[0], x[1]) for x in lexers.get_all_lexers()]);" - -EXTRA_LEXERS = {} - - -CONFIG = DotDict( - slack=DotDict( - api_key='api-key', - api_url='slack-incoming-hook-url', - default_room='#slack-channel', - default_plugin_config={}, - ), - redmine=DotDict( - api_key='api-key', - default_tracker_url='https://redmine.tracker.url', - default_project_id=None, - default_status_resolved_id=3 - ), -) - -# slack conf -CONFIG.slack.default_plugin_config = { - 'INCOMING_WEBHOOK_URL': CONFIG.slack.api_url, - 'SLACK_TOKEN': CONFIG.slack.api_key, - 'SLACK_ROOM': CONFIG.slack.default_room, - 'SLACK_FROM': 'RhodeCode', - 'SLACK_FROM_ICON_EMOJI': ':rhodecode:', -} - -# redmine smart_pr configuration -def configure_redmine_smart_pr(issues, kwargs): - kwargs['REDMINE_ISSUES'] = issues - kwargs['redmine_tracker_url'] = kwargs.pop( - 'redmine_tracker_url', '') or CONFIG.redmine.default_tracker_url - kwargs['redmine_api_key'] = kwargs.pop( - 'redmine_api_key', '') or CONFIG.redmine.api_key - kwargs['redmine_project_id'] = kwargs.pop( - 'redmine_project_id', '') or CONFIG.redmine.default_project_id - - -@register('CREATE_REPO_HOOK') -@verify_kwargs( - ['_load_extension', 'repo_name', 'repo_type', 'description', 'private', - 'created_on', 'enable_downloads', 'repo_id', 'user_id', 'enable_statistics', - 'clone_uri', 'fork_id', 'group_id', 'created_by']) -def _create_repo_hook(*args, **kwargs): - """ - POST CREATE REPOSITORY HOOK. This function will be executed after - each repository is created. kwargs available: - - :param repo_name: - :param repo_type: - :param description: - :param private: - :param created_on: - :param enable_downloads: - :param repo_id: - :param user_id: - :param enable_statistics: - :param clone_uri: - :param fork_id: - :param group_id: - :param created_by: - """ - return 0 -CREATE_REPO_HOOK = _create_repo_hook - - -@register('CREATE_REPO_GROUP_HOOK') -@verify_kwargs( - ['_load_extension', 'group_name', 'group_parent_id', 'group_description', - 'group_id', 'user_id', 'created_by', 'created_on', 'enable_locking']) -def _create_repo_group_hook(*args, **kwargs): - """ - POST CREATE REPOSITORY GROUP HOOK, this function will be - executed after each repository group is created. kwargs available: - - :param group_name: - :param group_parent_id: - :param group_description: - :param group_id: - :param user_id: - :param created_by: - :param created_on: - :param enable_locking: - """ - return 0 -CREATE_REPO_GROUP_HOOK = _create_repo_group_hook - - -@register('PRE_CREATE_USER_HOOK') -@verify_kwargs( - ['_load_extension', 'username', 'password', 'email', 'firstname', - 'lastname', 'active', 'admin', 'created_by']) -def _pre_create_user_hook(*args, **kwargs): - """ - PRE CREATE USER HOOK, this function will be executed before each - user is created, it returns a tuple of bool, reason. - If bool is False the user creation will be stopped and reason - will be displayed to the user. kwargs available: - - :param username: - :param password: - :param email: - :param firstname: - :param lastname: - :param active: - :param admin: - :param created_by: - """ - - reason = 'allowed' - return True, reason -PRE_CREATE_USER_HOOK = _pre_create_user_hook - - -@register('CREATE_USER_HOOK') -@verify_kwargs( - ['_load_extension', 'username', 'full_name_or_username', 'full_contact', - 'user_id', 'name', 'firstname', 'short_contact', 'admin', 'lastname', - 'ip_addresses', 'extern_type', 'extern_name', 'email', 'api_key', - 'api_keys', 'last_login', 'full_name', 'active', 'password', 'emails', - 'inherit_default_permissions', 'created_by', 'created_on']) -def _create_user_hook(*args, **kwargs): - """ - POST CREATE USER HOOK, this function will be executed after each user is created - kwargs available: - - :param username: - :param full_name_or_username: - :param full_contact: - :param user_id: - :param name: - :param firstname: - :param short_contact: - :param admin: - :param lastname: - :param ip_addresses: - :param extern_type: - :param extern_name: - :param email: - :param api_key: - :param api_keys: - :param last_login: - :param full_name: - :param active: - :param password: - :param emails: - :param inherit_default_permissions: - :param created_by: - :param created_on: - """ - return 0 -CREATE_USER_HOOK = _create_user_hook - - -@register('DELETE_REPO_HOOK') -@verify_kwargs( - ['_load_extension', 'repo_name', 'repo_type', 'description', 'private', - 'created_on', 'enable_downloads', 'repo_id', 'user_id', 'enable_statistics', - 'clone_uri', 'fork_id', 'group_id', 'deleted_by', 'deleted_on']) -def _delete_repo_hook(*args, **kwargs): - """ - POST DELETE REPOSITORY HOOK, this function will be executed after - each repository deletion kwargs available: - - :param repo_name: - :param repo_type: - :param description: - :param private: - :param created_on: - :param enable_downloads: - :param repo_id: - :param user_id: - :param enable_statistics: - :param clone_uri: - :param fork_id: - :param group_id: - :param deleted_by: - :param deleted_on: - """ - return 0 -DELETE_REPO_HOOK = _delete_repo_hook - - -@register('DELETE_USER_HOOK') -@verify_kwargs( - ['_load_extension', 'username', 'full_name_or_username', 'full_contact', - 'user_id', 'name', 'firstname', 'short_contact', 'admin', 'lastname', - 'ip_addresses', 'email', 'api_key', 'last_login', 'full_name', 'active', - 'password', 'emails', 'inherit_default_permissions', 'deleted_by' - ]) -def _delete_user_hook(*args, **kwargs): - """ - POST DELETE USER HOOK, this function will be executed after each - user is deleted kwargs available: - - :param username: - :param full_name_or_username: - :param full_contact: - :param user_id: - :param name: - :param firstname: - :param short_contact: - :param admin: - :param lastname: - :param ip_addresses: - :param ldap_dn: - :param email: - :param api_key: - :param last_login: - :param full_name: - :param active: - :param password: - :param emails: - :param inherit_default_permissions: - :param deleted_by: - """ - return 0 -DELETE_USER_HOOK = _delete_user_hook - - -@register('PRE_PUSH_HOOK') -@verify_kwargs( - ['_load_extension', 'server_url', 'config', 'scm', 'username', - 'ip', 'action', 'repository', 'repo_store_path']) -def _pre_push_hook(*args, **kwargs): - """ - Post push hook - kwargs available: - - :param server_url: url of instance that triggered this hook - :param config: path to .ini config used - :param scm: type of VS 'git' or 'hg' - :param username: name of user who pushed - :param ip: ip of who pushed - :param action: push - :param repository: repository name - :param repo_store_path: full path to where repositories are stored - """ - return 0 -PRE_PUSH_HOOK = _pre_push_hook - - -@register('PUSH_HOOK') -@verify_kwargs( - ['_load_extension', 'server_url', 'config', 'scm', 'username', - 'ip', 'action', 'repository', 'repo_store_path', 'pushed_revs']) -def _push_hook(*args, **kwargs): - """ - POST PUSH HOOK, this function will be executed after each push it's - executed after the build-in hook that RhodeCode uses for logging pushes - kwargs available: - - :param server_url: url of instance that triggered this hook - :param config: path to .ini config used - :param scm: type of VS 'git' or 'hg' - :param username: name of user who pushed - :param ip: ip of who pushed - :param action: push - :param repository: repository name - :param repo_store_path: full path to where repositories are stored - :param pushed_revs: list of pushed commit ids - """ - # backward compat - kwargs['commit_ids'] = kwargs['pushed_revs'] - - # fetch extra fields from repository - call = load_extension('extra_fields.py') - _extra_fields = {} - if call: - repo_extra_fields = call(**kwargs) - # now update if we have extra fields, they have precedence - # this way users can store any configuration inside the database per - # repo - for key, data in repo_extra_fields.items(): - kwargs[key] = data['field_value'] - _extra_fields[key] = data['field_value'] - - # fetch pushed commits, from commit_ids list - call = load_extension('extract_commits.py') - extracted_commits = {} - if call: - extracted_commits = call(**kwargs) - # store the commits for the next call chain - kwargs['COMMITS'] = extracted_commits - - # slack ! - call = load_extension('slack_push_notify.py') - if call: - kwargs.update(CONFIG.slack.default_plugin_config) - call(**kwargs) - - # fetch redmine issues from given commits - call = load_extension('extract_redmine_issues.py') - issues = {} - if call: - issues = call(**kwargs) - - # redmine smart commits - call = load_extension('redmine_smart_commits.py') - if call: - kwargs['REDMINE_ISSUES'] = issues - - kwargs['redmine_tracker_url'] = kwargs.pop( - 'redmine_tracker_url', '') or CONFIG.redmine.default_tracker_url - kwargs['redmine_api_key'] = kwargs.pop( - 'redmine_api_key', '') or CONFIG.redmine.api_key - kwargs['redmine_status_resolved_id'] = kwargs.pop( - 'redmine_status_resolved_id', '') or CONFIG.redmine.default_status_resolved_id - kwargs['redmine_project_id'] = kwargs.pop( - 'redmine_project_id', '') or CONFIG.redmine.default_project_id - call(**kwargs) - - return 0 -PUSH_HOOK = _push_hook - - -@register('PRE_PULL_HOOK') -@verify_kwargs( - ['_load_extension', 'server_url', 'config', 'scm', 'username', 'ip', - 'action', 'repository']) -def _pre_pull_hook(*args, **kwargs): - """ - Post pull hook - kwargs available:: - - :param server_url: url of instance that triggered this hook - :param config: path to .ini config used - :param scm: type of VS 'git' or 'hg' - :param username: name of user who pulled - :param ip: ip of who pulled - :param action: pull - :param repository: repository name - """ - return 0 -PRE_PULL_HOOK = _pre_pull_hook - - -@register('PULL_HOOK') -@verify_kwargs( - ['_load_extension', 'server_url', 'config', 'scm', 'username', 'ip', - 'action', 'repository']) -def _pull_hook(*args, **kwargs): - """ - POST PULL HOOK, this function will be executed after each push it's - executed after the build-in hook that RhodeCode uses for logging pulls - - kwargs available: - - :param server_url: url of instance that triggered this hook - :param config: path to .ini config used - :param scm: type of VS 'git' or 'hg' - :param username: name of user who pulled - :param ip: ip of who pulled - :param action: pull - :param repository: repository name - """ - return 0 -PULL_HOOK = _pull_hook - - -# ============================================================================= -# PULL REQUEST RELATED HOOKS -# ============================================================================= -@register('CREATE_PULL_REQUEST') -@verify_kwargs( - ['_load_extension', 'server_url', 'config', 'scm', 'username', 'ip', - 'action', 'repository', 'pull_request_id', 'url', 'title', 'description', - 'status', 'created_on', 'updated_on', 'commit_ids', 'review_status', - 'mergeable', 'source', 'target', 'author', 'reviewers']) -def _create_pull_request_hook(*args, **kwargs): - """ - - """ - # extract extra fields and default reviewers from target - kwargs['REPOSITORY'] = kwargs['target']['repository'] - - call = load_extension('extra_fields.py') - if call: - repo_extra_fields = call(**kwargs) - # now update if we have extra fields, they have precedence - # this way users can store any configuration inside the database per - # repo - for key, data in repo_extra_fields.items(): - kwargs[key] = data['field_value'] - - call = load_extension('default_reviewers.py') - if call: - # read default_reviewers key propagated from extra fields - kwargs['default_reviewers'] = map(string.strip, kwargs.pop( - 'default_reviewers', '').split(',')) - call(**kwargs) - - # extract below from source repo as commits are there - kwargs['REPOSITORY'] = kwargs['source']['repository'] - - # # fetch pushed commits, from commit_ids list - # call = load_extension('extract_commits.py') - # extracted_commits = {} - # if call: - # extracted_commits = call(**kwargs) - # # store the commits for the next call chain - # kwargs['COMMITS'] = extracted_commits - # - # # fetch issues from given commits - # call = load_extension('extract_redmine_issues.py') - # issues = {} - # if call: - # issues = call(**kwargs) - # - # # redmine smart pr update - # call = load_extension('redmine_pr_flow.py') - # if call: - # # updates kwargs on the fly - # configure_redmine_smart_pr(issues=issues, kwargs=kwargs) - # call(**kwargs) - # - # # slack notification on merging PR - # call = load_extension('slack_message.py') - # if call: - # kwargs.update(CONFIG.slack.default_plugin_config) - # kwargs['SLACK_ROOM'] = '#develop' - # kwargs['SLACK_MESSAGE'] = 'Pull request <%s|#%s> (%s) was created.' % ( - # kwargs.get('url'), kwargs.get('pull_request_id'), kwargs.get('title')) - # - # call(**kwargs) - - return 0 -CREATE_PULL_REQUEST = _create_pull_request_hook - - -@register('REVIEW_PULL_REQUEST') -@verify_kwargs( - ['_load_extension', 'server_url', 'config', 'scm', 'username', 'ip', - 'action', 'repository', 'pull_request_id', 'url', 'title', 'description', - 'status', 'created_on', 'updated_on', 'commit_ids', 'review_status', - 'mergeable', 'source', 'target', 'author', 'reviewers']) -def _review_pull_request_hook(*args, **kwargs): - """ - - """ - # extract extra fields and default reviewers from target - kwargs['REPOSITORY'] = kwargs['target']['repository'] - - # fetch extra fields - call = load_extension('extra_fields.py') - if call: - repo_extra_fields = call(**kwargs) - # now update if we have extra fields, they have precedence - # this way users can store any configuration inside the database per - # repo - for key, data in repo_extra_fields.items(): - kwargs[key] = data['field_value'] - - # extract below from source repo as commits are there - kwargs['REPOSITORY'] = kwargs['source']['repository'] - - # fetch pushed commits, from commit_ids list - call = load_extension('extract_commits.py') - extracted_commits = {} - if call: - extracted_commits = call(**kwargs) - # store the commits for the next call chain - kwargs['COMMITS'] = extracted_commits - - # fetch issues from given commits - call = load_extension('extract_redmine_issues.py') - issues = {} - if call: - issues = call(**kwargs) - - # redmine smart pr update - call = load_extension('redmine_pr_flow.py') - if call: - # updates kwargs on the fly - configure_redmine_smart_pr(issues=issues, kwargs=kwargs) - call(**kwargs) - - return 0 -REVIEW_PULL_REQUEST = _review_pull_request_hook - - -@register('UPDATE_PULL_REQUEST') -@verify_kwargs( - ['_load_extension', 'server_url', 'config', 'scm', 'username', 'ip', - 'action', 'repository', 'pull_request_id', 'url', 'title', 'description', - 'status', 'created_on', 'updated_on', 'commit_ids', 'review_status', - 'mergeable', 'source', 'target', 'author', 'reviewers']) -def _update_pull_request_hook(*args, **kwargs): - """ - - """ - # extract extra fields and default reviewers from target - kwargs['REPOSITORY'] = kwargs['target']['repository'] - - # fetch extra fields - call = load_extension('extra_fields.py') - if call: - repo_extra_fields = call(**kwargs) - # now update if we have extra fields, they have precedence - # this way users can store any configuration inside the database per - # repo - for key, data in repo_extra_fields.items(): - kwargs[key] = data['field_value'] - - # extract below from source repo as commits are there - kwargs['REPOSITORY'] = kwargs['source']['repository'] - - # fetch pushed commits, from commit_ids list - call = load_extension('extract_commits.py') - extracted_commits = {} - if call: - extracted_commits = call(**kwargs) - # store the commits for the next call chain - kwargs['COMMITS'] = extracted_commits - - # fetch issues from given commits - call = load_extension('extract_redmine_issues.py') - issues = {} - if call: - issues = call(**kwargs) - - # redmine smart pr updated - call = load_extension('redmine_pr_flow.py') - if call: - # updates kwargs on the fly - configure_redmine_smart_pr(issues=issues, kwargs=kwargs) - call(**kwargs) - - return 0 -UPDATE_PULL_REQUEST = _update_pull_request_hook - - -@register('MERGE_PULL_REQUEST') -@verify_kwargs( - ['_load_extension', 'server_url', 'config', 'scm', 'username', 'ip', - 'action', 'repository', 'pull_request_id', 'url', 'title', 'description', - 'status', 'created_on', 'updated_on', 'commit_ids', 'review_status', - 'mergeable', 'source', 'target', 'author', 'reviewers']) -def _merge_pull_request_hook(*args, **kwargs): - """ - - """ - # extract extra fields and default reviewers from target - kwargs['REPOSITORY'] = kwargs['target']['repository'] - - # fetch extra fields - call = load_extension('extra_fields.py') - if call: - repo_extra_fields = call(**kwargs) - # now update if we have extra fields, they have precedence - # this way users can store any configuration inside the database per - # repo - for key, data in repo_extra_fields.items(): - kwargs[key] = data['field_value'] - - # extract below from source repo as commits are there - kwargs['REPOSITORY'] = kwargs['source']['repository'] - - # fetch pushed commits, from commit_ids list - call = load_extension('extract_commits.py') - extracted_commits = {} - if call: - extracted_commits = call(**kwargs) - # store the commits for the next call chain - kwargs['COMMITS'] = extracted_commits - - # fetch issues from given commits - call = load_extension('extract_redmine_issues.py') - issues = {} - if call: - issues = call(**kwargs) - - # redmine smart pr update - call = load_extension('redmine_pr_flow.py') - if call: - # updates kwargs on the fly - configure_redmine_smart_pr(issues=issues, kwargs=kwargs) - call(**kwargs) - - # slack notification on merging PR - call = load_extension('slack_message.py') - if call: - kwargs.update(CONFIG.slack.default_plugin_config) - kwargs['SLACK_ROOM'] = '#develop' - kwargs['SLACK_MESSAGE'] = 'Pull request <%s|#%s> (%s) was merged.' % ( - kwargs.get('url'), kwargs.get('pull_request_id'), kwargs.get('title')) - call(**kwargs) - - return 0 -MERGE_PULL_REQUEST = _merge_pull_request_hook - - -@register('CLOSE_PULL_REQUEST') -@verify_kwargs( - ['_load_extension', 'server_url', 'config', 'scm', 'username', 'ip', - 'action', 'repository', 'pull_request_id', 'url', 'title', 'description', - 'status', 'created_on', 'updated_on', 'commit_ids', 'review_status', - 'mergeable', 'source', 'target', 'author', 'reviewers']) -def _close_pull_request_hook(*args, **kwargs): - """ - - """ - # extract extra fields and default reviewers from target - kwargs['REPOSITORY'] = kwargs['target']['repository'] - - # fetch extra fields - call = load_extension('extra_fields.py') - if call: - repo_extra_fields = call(**kwargs) - # now update if we have extra fields, they have precedence - # this way users can store any configuration inside the database per - # repo - for key, data in repo_extra_fields.items(): - kwargs[key] = data['field_value'] - - # extract below from source repo as commits are there - kwargs['REPOSITORY'] = kwargs['source']['repository'] - - # fetch pushed commits, from commit_ids list - call = load_extension('extract_commits.py') - extracted_commits = {} - if call: - extracted_commits = call(**kwargs) - # store the commits for the next call chain - kwargs['COMMITS'] = extracted_commits - - # fetch issues from given commits - call = load_extension('extract_redmine_issues.py') - issues = {} - if call: - issues = call(**kwargs) - - # redmine smart pr update - call = load_extension('redmine_pr_flow.py') - if call: - # updates kwargs on the fly - configure_redmine_smart_pr(issues=issues, kwargs=kwargs) - call(**kwargs) - - return 0 -CLOSE_PULL_REQUEST = _close_pull_request_hook diff --git a/docs/extensions/extensions-hooks.rst b/docs/extensions/extensions-hooks.rst --- a/docs/extensions/extensions-hooks.rst +++ b/docs/extensions/extensions-hooks.rst @@ -14,12 +14,23 @@ so to clarify what is meant each time, r between software components and can be used to trigger plugins, or their extensions. -.. toctree:: + +Hooks +----- + +Within |RCM| there are two types of supported hooks. - rcx - install-ext - config-ext - extensions - hooks - full-blown-example - int-slack +* **Internal built-in hooks**: The internal |hg|, |git| or |svn| hooks are + triggered by different VCS operations, like push, pull, + or clone and are non-configurable, but you can add your own VCS hooks, + see :ref:`custom-hooks`. +* **Custom rcextensions hooks**: User defined hooks centre around the lifecycle of + certain actions such are |repo| creation, user creation etc. The actions + these hooks trigger can be rejected based on the API permissions of the + user calling them. + +On instructions how to use the custom `rcextensions` +see :ref:`integrations-rcextensions` section. + + + diff --git a/docs/extensions/extensions.rst b/docs/extensions/extensions.rst deleted file mode 100644 --- a/docs/extensions/extensions.rst +++ /dev/null @@ -1,59 +0,0 @@ -.. _dev-plug: - -Developing Plugins/Extensions ------------------------------ - -An Extension or a Plugin is simply a |PY| module with a ``run`` method that -expects a number of parameters, depending on which event it is listening -for. To get an extension working, use the following steps: - -1. Create an extension or plugin using the below example. -2. Save the plugin inside the - :file:`/home/{user}/.rccontrol/{instance-id}/rcextensions` folder. -3. Add a hook to the - :file:`/home/{user}/.rccontrol/{instance-id}/rcextensions/__init__.py` file. - For more information, see :ref:`event-listener`. -4. Restart your |RCM| instance. - -Extension example -^^^^^^^^^^^^^^^^^ - -In the following example, the ``run`` method listens for a push to a |repo| -and parses the commit. - -.. code-block:: python - - def run(*args, **kwargs): - - revs = kwargs.get('pushed_revs') - if not revs: - return 0 - - from rhodecode.lib.utils2 import extract_mentioned_users - from rhodecode.model.db import Repository - - repo = Repository.get_by_repo_name(kwargs['repository']) - changesets = [] - reviewers = [] - - # reviewer fields from extra_fields, users can store their custom - # reviewers inside the extra fields to pre-define a set of people who - # will get notifications about changesets - field_key = kwargs.get('reviewers_extra_field') - if field_key: - for xfield in repo.extra_fields: - if xfield.field_key == field_key: - reviewers.extend(xfield.field_value.split()) - - vcs_repo = repo.scm_instance_no_cache() - for rev in kwargs['pushed_revs']: - cs = vcs_repo.get_changeset(rev) # or get_commit. See API doc - cs_data = cs.__json__() - cs_data['mentions'] = extract_mentioned_users(cs_data['message']) - cs_data['reviewers'] = reviewers - # optionally add more logic to parse the commits, like reading extra - # fields of repository to read managers of reviewers - changesets.append(cs_data) - - return changesets - \ No newline at end of file diff --git a/docs/extensions/full-blown-example.rst b/docs/extensions/full-blown-example.rst deleted file mode 100644 --- a/docs/extensions/full-blown-example.rst +++ /dev/null @@ -1,19 +0,0 @@ -.. _int-full-blown: - -Extensions Extended Example ---------------------------- - -This example -:file:`/home/{user}/.rccontrol/{instance-id}/rcextensions/__init.py__` file -has been highlighted to show a Redmine integration in full. To extend your -|RCE| instances, use the below example to integrate with other -applications. - -This example file also contains a Slack integration, but it is not -highlighted. - - -.. literalinclude:: example-ext.py - :language: python - :emphasize-lines: 186,193-198,210-218,474-496,648-660,749-760,810-822 - :linenos: diff --git a/docs/extensions/hooks.rst b/docs/extensions/hooks.rst deleted file mode 100644 --- a/docs/extensions/hooks.rst +++ /dev/null @@ -1,86 +0,0 @@ -.. _hooks: - -Hooks ------ - -Within |RCM| there are two types of supported hooks. - -* **Internal built-in hooks**: The internal |hg| or |git| hooks are - triggered by different VCS operations, like push, pull, - or clone and are non-configurable, but you can add your own VCS hooks, - see :ref:`custom-hooks`. -* **User defined hooks**: User defined hooks centre around the lifecycle of - certain actions such are |repo| creation, user creation etc. The actions - these hooks trigger can be rejected based on the API permissions of the - user calling them. - -Those custom hooks can be called using |RCT|, see :ref:`rc-tools`. To create -a custom hook, see the :ref:`event-listener` section. - -.. _event-listener: - -Making your Extension listen for Events -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -To create a hook to work with a plugin or extension, -you need configure a listener in the -:file:`/home/{user}/{instance-id}/rcextensions/__init__.py` file, -and use the ``load_extension`` method. - -Use the following example to create your extensions. -In this example: - -* The hook is calling the ``('my_post_push_extension.py')`` extension. -* The hook is listening to |RCM| for pushes to |repos|. -* This highlighted code is the hook, and configured in the ``__init__.py`` file. -* It is inserted into the ``def _pushhook(*args, **kwargs)`` section, - if it is not in the default ``__ini__.py`` file, use the below - non-highlighted section to create it. - -.. code-block:: python - :emphasize-lines: 23-38 - - # ========================================================================== - # POST PUSH HOOK - # ========================================================================== - - # this function will be executed after each push is executed after the - # build-in hook that RhodeCode uses for logging pushes - def _pushhook(*args, **kwargs): - """ - Post push hook - kwargs available: - - :param server_url: url of instance that triggered this hook - :param config: path to .ini config used - :param scm: type of VS 'git' or 'hg' - :param username: name of user who pushed - :param ip: ip of who pushed - :param action: push - :param repository: repository name - :param repo_store_path: full path to where repositories are stored - :param pushed_revs: list of pushed revisions - """ - - # Your hook code goes in here - call = load_extension('my_post_push_extension.py') - if call: - # extra arguments in kwargs - call_kwargs = dict() - call_kwargs.update(kwargs) - my_kw = { - 'reviewers_extra_field': 'reviewers', - # defines if we have a comma - # separated list of reviewers - # in this repo stored in extra_fields - } - call_kwargs.update(my_kw) # pass in hook args - parsed_revs = call(**call_kwargs) - # returns list of dicts with changesets data - - # Default code - return 0 - PUSH_HOOK = _pushhook - -Once your plugin and hook are configured, restart your instance of |RCM| and -your event listener will triggered as soon as a user pushes to a |repo|. diff --git a/docs/extensions/install-ext.rst b/docs/extensions/install-ext.rst deleted file mode 100644 --- a/docs/extensions/install-ext.rst +++ /dev/null @@ -1,55 +0,0 @@ -.. _install-rcx: - -Install |RCX| -------------- - -To install |RCX|, you need to have |RCT| installed. See the :ref:`install-tools` -and :ref:`tools-cli` sections. Use the ``--plugins`` option with -the ``rhodecode-extensions`` argument. - -Upgrading |RCX| -^^^^^^^^^^^^^^^ - -.. important:: - - You should back up any plugins or extensions that you have created before - continuing. - -To upgrade your |RCX|, use the following example: - -1. From inside the |RCT| virtualenv, upgrade to the latest version: - -.. code-block:: bash - - (venv)$ pip install -U https://my.rhodecode.com/dl/rhodecode-tools/latest - Downloading/unpacking https://my.rhodecode.com/dl/rhodecode-tools/latest - Downloading latest (143kB): 143kB downloaded - Running setup.py (path:/tmp/pip-9qYsxf-build/setup.py) egg_info - for package from https://my.rhodecode.com/dl/rhodecode-tools/latest - -2. Once |RCT| are upgraded to the latest version, you can install the latest - extensions using the following example: - -.. code-block:: bash - - (venv)$ rhodecode-extensions --instance-name=enterprise-1 \ - --ini-file=rhodecode.ini --plugins - - Extension file already exists, do you want to overwrite it? [y/N]: y - Writen new extensions file to rcextensions - Copied hipchat_push_notify.py plugin to rcextensions - Copied jira_pr_flow.py plugin to rcextensions - Copied default_reviewers.py plugin to rcextensions - Copied extract_commits.py plugin to rcextensions - Copied extract_issues.py plugin to rcextensions - Copied redmine_pr_flow.py plugin to rcextensions - Copied extra_fields.py plugin to rcextensions - Copied jira_smart_commits.py plugin to rcextensions - Copied http_notify.py plugin to rcextensions - Copied slack_push_notify.py plugin to rcextensions - Copied slack_message.py plugin to rcextensions - Copied extract_jira_issues.py plugin to rcextensions - Copied extract_redmine_issues.py plugin to rcextensions - Copied redmine_smart_commits.py plugin to rcextensions - Copied send_mail.py plugin to rcextensions - diff --git a/docs/extensions/int-slack.rst b/docs/extensions/int-slack.rst deleted file mode 100644 --- a/docs/extensions/int-slack.rst +++ /dev/null @@ -1,151 +0,0 @@ -.. _slack-int: - -Integrate Slack -=============== - -To integrate |RCE| and Slack, you need to configure some things on the Slack -side of the integration, and some things on the |RCE| side. - -On the Slack side you need to allow incoming webhooks, see their -documentation on this, `Slack Webhooks`_. You will also need to get an -Authorization Token from Slack that will allow |RCE| to post to your account. - -On the |RCE| side, this is an overview of what you need to do: - -1. Configure the built-in Slack extensions to post to the correct Slack URL. -2. Set your Slack authentication details in the |RCX| :file:`__init.py__` file. -3. Configure the different hooks in the :file:`__init.py__` file to extract - whatever information you want from |RCE|, and then using the Slack extensions - post that information to your Slack channel. - -.. hint:: - - The below examples should help you to get started. Once you have your - integration up and running, there is a more detailed Slack integration in - the :ref:`int-full-blown` section. - -Configure Built-in Extensions ------------------------------ - -|RCE| comes with 3 Slack extensions: ``slack_message.py``, -``slack_push_notify.py``, and ``slack.py``. The default -location is :file:`/home/{user}/.rccontrol/{instance-id}/rcextensions`. - -To enable these to post to your Slack account, configure each of -these files with the following Slack details. - -.. code-block:: python - - BASE_URL = 'https://your.slack.com/api/link' - INCOMING_WEBHOOK_URL = 'https://hooks.slack.com/services/your/hook/link' - API_VERSION = 1 - -Configure |RCE| to Post to Slack --------------------------------- - -In the |RCX| :file:`__init.py__` file, configure your Slack authentication -details. The default location is -:file:`/home/{user}/.rccontrol/{instance-id}/rcextensions` - -.. code-block:: python - - CONFIG = DotDict( - slack=DotDict( - api_key='api-key', - api_url='slack-incoming-hook-url', - default_room='#slack-channel', - default_plugin_config={}, - ), - ) - - # slack conf - CONFIG.slack.default_plugin_config = { - 'INCOMING_WEBHOOK_URL': CONFIG.slack.api_url, - 'SLACK_TOKEN': CONFIG.slack.api_key, - 'SLACK_ROOM': CONFIG.slack.default_room, - 'SLACK_FROM': 'RhodeCode', - 'SLACK_FROM_ICON_EMOJI': ':rhodecode:', - } - -Add Push Notifications to Slack -------------------------------- - -To add notification to Slack when someone pushes to |RCE|, configure the push -hook to extract the commits pushed, and then call the built-in -``slack_push_notify.py`` extension to post them into your chosen Slack -channel. To do this, add the following code to the push hook section of the -:file:`__init.py__` file - -.. code-block:: python - :emphasize-lines: 10-16,18-22 - - def _push_hook(*args, **kwargs): - """ - POST PUSH HOOK, this function will be executed after each push, it's - executed after the build-in hook that RhodeCode uses for logging pushes - kwargs available: - """ - # backward compat - kwargs['commit_ids'] = kwargs['pushed_revs'] - - # fetch pushed commits, from commit_ids list - call = load_extension('extract_commits.py') - extracted_commits = {} - if call: - extracted_commits = call(**kwargs) - # store the commits for the next call chain - kwargs['COMMITS'] = extracted_commits - - # slack ! - call = load_extension('slack_push_notify.py') - if call: - kwargs.update(CONFIG.slack.default_plugin_config) - call(**kwargs) - return 0 - PUSH_HOOK = _push_hook - - -Add Pull Request Notifications to Slack ---------------------------------------- - -To add |pr| notifications to Slack, use the following example. This example -shows a merged |pr| notification. You can add similar notifications to the -following hooks in the :file:`__init.py__` file, and for those examples see -the :ref:`int-full-blown` section: - -* ``_create_pull_request_hook`` -* ``_review_pull_request_hook`` -* ``_update_pull_request_hook`` -* ``_close_pull_request_hook`` - -.. code-block:: python - :emphasize-lines: 5-23 - - def _merge_pull_request_hook(*args, **kwargs): - """ - - """ - # extract below from source repo as commits are there - kwargs['REPOSITORY'] = kwargs['source']['repository'] - - # fetch pushed commits, from commit_ids list - call = load_extension('extract_commits.py') - extracted_commits = {} - if call: - extracted_commits = call(**kwargs) - # store the commits for the next call chain - kwargs['COMMITS'] = extracted_commits - - # slack notification on merging PR - call = load_extension('slack_message.py') - if call: - kwargs.update(CONFIG.slack.default_plugin_config) - kwargs['SLACK_ROOM'] = '#develop' - kwargs['SLACK_MESSAGE'] = 'Pull request <%s|#%s> (%s) was merged.' % ( - kwargs.get('url'), kwargs.get('pull_request_id'), kwargs.get('title')) - call(**kwargs) - - return 0 - MERGE_PULL_REQUEST = _merge_pull_request_hook - -.. _Slack Webhooks: https://api.slack.com/incoming-webhooks diff --git a/docs/extensions/rcx.rst b/docs/extensions/rcx.rst deleted file mode 100644 --- a/docs/extensions/rcx.rst +++ /dev/null @@ -1,53 +0,0 @@ -.. _rc-ext: - -|RCX| ------ - -|RCX| add additional functionality for push/pull/create/delete |repo| hooks. -These hooks can be used to send signals to build-bots such as `Jenkins`_. It -also adds built in plugin and extension support. Once installed, you will see -a :file:`rcextensions` folder in the instance directory, for example: -:file:`home/{user}/.rccontrol/{instance-id}/rcextensions` - -Built-in Plugins -^^^^^^^^^^^^^^^^ - -* A number of `Jira`_ plugins, enabling you to integrate with that issue - tracker: ``extract_jira_issues.py``, ``jira_pr_flow.py``, - ``jira_smart_commits.py`` -* A number of `Redmine`_ plugins, enabling you to integrate with that issue - tracker: ``extract_redmine_issues.py``, ``redmine_pr_flow.py``, - ``redmine_smart_commits.py``. -* ``hipchat.py`` and ``hipchat_push.py`` enable you to integrate with - `HipChat`_ and set channel or user notifications. -* ``slack.py``, ``slack_message.py``, and ``slack_push_notify.py`` enable - you to integrate with `Slack`_ and set channel or user notifications. - -Built-in Extensions -^^^^^^^^^^^^^^^^^^^ - -* ``commit_parser.py``: Enables you to parse commit messages, - and set a list of users to get notifications about change sets. -* ``default_reviewers.py``: Enables you to add default reviewers to a |pr|. -* ``extra_fields.py``: Returns a list of extra fields added to a |repo|. -* ``http_notify``: Enables you to send data over a web hook. -* ``mail.py`` : This extension uses the |RCE| mail configuration from the - instance :file:`rhodecode.ini` file to send email. -* ``push_post.py``: Enables you to set up push based actions such as - automated Jenkins builds. - -Event Listeners -^^^^^^^^^^^^^^^ - -To enable the extensions to listen to the different events that they are -configured for, you need to also set up an event listener (hook). Event -listeners are configured in the -:file:`/home/{user}/.rccontrol/{instance-id}/rcextensions/__init.__.py` file. - -For more details, see the example hook in :ref:`event-listener`. - -.. _Jenkins: http://jenkins-ci.org/ -.. _HipChat: https://www.hipchat.com/ -.. _Slack: https://slack.com/ -.. _Redmine: http://www.redmine.org/ -.. _Jira: https://www.atlassian.com/software/jira diff --git a/docs/integrations/integrations-rcextensions.rst b/docs/integrations/integrations-rcextensions.rst new file mode 100644 --- /dev/null +++ b/docs/integrations/integrations-rcextensions.rst @@ -0,0 +1,44 @@ +.. _integrations-rcextensions: + + +rcextensions integrations +========================= + + +Since RhodeCode 4.14 release rcextensions aren't part of rhodecode-tools, and instead +they are shipped with the new or upgraded installations. + +The rcextensions template `rcextensions.tmpl` is created in the `etc/` directory +of enterprise or community installation. It's always re-created and updated on upgrades. + + +Activating rcextensions ++++++++++++++++++++++++ + +To activate rcextensions simply copy or rename the created template rcextensions +into the path where the rhodecode.ini file is located:: + + pushd ~/rccontrol/enterprise-1/ + or + pushd ~/rccontrol/community-1/ + + mv etc/rcextensions.tmpl rcextensions + + +rcextensions are loaded when |RCE| starts. So a restart is required after activation or +change of code in rcextensions. + +Simply restart only the enterprise/community instance:: + + rccontrol restart enterprise-1 + or + rccontrol restart community-1 + + +Example usage ++++++++++++++ + + +To see examples of usage please check the examples directory under: + +https://code.rhodecode.com/rhodecode-enterprise-ce/files/stable/rhodecode/config/rcextensions/examples diff --git a/docs/integrations/integrations.rst b/docs/integrations/integrations.rst --- a/docs/integrations/integrations.rst +++ b/docs/integrations/integrations.rst @@ -3,7 +3,7 @@ Integrations ------------ -Rhodecode supports integrations with external services for various events, +|RCE| supports integrations with external services for various events, such as commit pushes and pull requests. Multiple integrations of the same type can be added at the same time; this is useful for posting different events to different Slack channels, for example. @@ -11,18 +11,20 @@ different Slack channels, for example. Supported integrations ^^^^^^^^^^^^^^^^^^^^^^ -============================ ============ ===================================== -Type/Name |RC| Edition Description -============================ ============ ===================================== -:ref:`integrations-slack` |RCCEshort| https://slack.com/ -:ref:`integrations-hipchat` |RCCEshort| https://www.hipchat.com/ -:ref:`integrations-webhook` |RCCEshort| POST events as `json` to a custom url -:ref:`integrations-ci` |RCCEshort| Trigger Builds for Common CI Systems -:ref:`integrations-email` |RCCEshort| Send repo push commits by email -:ref:`integrations-jenkins` |RCEEshort| Trigger Builds for Jenkins CI System -:ref:`integrations-redmine` |RCEEshort| Close/Resolve/Reference Redmine issues -:ref:`integrations-jira` |RCEEshort| Close/Resolve/Reference JIRA issues -============================ ============ ===================================== +================================ ============ ======================================== +Type/Name |RC| Edition Description +================================ ============ ======================================== +:ref:`integrations-webhook` |RCCEshort| Trigger events as `json` to a custom url +:ref:`integrations-slack` |RCCEshort| Integrate with https://slack.com/ +:ref:`integrations-hipchat` |RCCEshort| Integrate with https://www.hipchat.com/ +:ref:`integrations-email` |RCCEshort| Send repo push commits by email +:ref:`integrations-ci` |RCCEshort| Trigger Builds for Common CI Systems +:ref:`integrations-rcextensions` |RCCEshort| Advanced low-level integration framework + +:ref:`integrations-jenkins` |RCEEshort| Trigger Builds for Jenkins CI System +:ref:`integrations-redmine` |RCEEshort| Close/Resolve/Reference Redmine issues +:ref:`integrations-jira` |RCEEshort| Close/Resolve/Reference JIRA issues +================================ ============ ======================================== .. _creating-integrations: @@ -55,3 +57,4 @@ See pages specific to each type of integ email ci jenkins + integrations-rcextensions diff --git a/docs/tools/tools-cli.rst b/docs/tools/tools-cli.rst --- a/docs/tools/tools-cli.rst +++ b/docs/tools/tools-cli.rst @@ -273,107 +273,8 @@ Use this to create or update a |RCE| con rhodecode-extensions -------------------- -|RCT| adds additional mapping for :ref:`indexing-ref`, statistics, and adds -additional code for push/pull/create/delete |repo| hooks. These hooks can be -used to send signals to build-bots such as jenkins. Options: - -.. rst-class:: dl-horizontal - - \-c, - -config - Create a configuration file. The default file is created - in ``~/.rhoderc`` - - \-h, - -help - Show help messages. - - \-F, - -format {json,pretty} - Set the formatted representation. - - \-I, - -install-dir - Set the location of the |RCE| installation. The default location is - :file:`/home/{user}/.rccontrol/`. - - \ - -ini-file - Path to the :file:`rhodecode.ini` file for that instance. - - \ - -instance-name - Set the instance name. - - \ - -plugins - Add plugins to your |RCE| installation. See the - :ref:`extensions-hooks-ref` section for more details. - - \ - -version - Display your |RCT| version. - - -Once installed, you will see a :file:`rcextensions` folder in the instance -directory, for example :file:`home/{user}/.rccontrol/{instance-id}/rcextensions` - -To install ``rcextensions``, use the following example: - -.. code-block:: bash - - # install extensions on the given instance - # If using virtualenv prior to RCE 350 - (venv)$ rhodecode-extensions --instance-name=enterprise-1 \ - --ini-file=rhodecode.ini - Writen new extensions file to rcextensions - - # install extensions with additional plugins on the given instance - (venv)$ rhodecode-extensions --instance-name=enterprise-1 \ - --ini-file=rhodecode.ini --plugins - Writen new extensions file to rcextensions - - # installing extensions from 350 onwards - # as they are packaged with RCE - $ .rccontrol/enterprise-4/profile/bin/rhodecode-extensions --plugins \ - --instance-name=enterprise-4 --ini-file=rhodecode.ini - - Writen new extensions file to rcextensions - -See the new extensions inside this directory for more details about the -additional hooks available, for example see the ``push_post.py`` file. - -.. code-block:: python - - import urllib - import urllib2 - - def run(*args, **kwargs): - """ - Extra params - - :param URL: url to send the data to - """ - - url = kwargs.pop('URL', None) - if url: - from rhodecode.lib.compat import json - from rhodecode.model.db import Repository - - repo = Repository.get_by_repo_name(kwargs['repository']) - changesets = [] - vcs_repo = repo.scm_instance_no_cache() - for r in kwargs['pushed_revs']: - cs = vcs_repo.get_changeset(r) - changesets.append(json.dumps(cs)) - - kwargs['pushed_revs'] = changesets - headers = { - 'User-Agent': 'RhodeCode-SCM web hook', - 'Content-type': 'application/x-www-form-urlencoded; charset=UTF-8', - 'Accept': 'text/javascript, text/html, application/xml, ' - 'text/xml, */*', - 'Accept-Encoding': 'gzip,deflate,sdch', - } - - data = kwargs - data = urllib.urlencode(data) - req = urllib2.Request(url, data, headers) - response = urllib2.urlopen(req) - response.read() - return 0 +The `rcextensions` since version 4.14 are now shipped together with |RCE| please check +the using :ref:`integrations-rcextensions` section. rhodecode-gist diff --git a/docs/tools/tools-overview.rst b/docs/tools/tools-overview.rst --- a/docs/tools/tools-overview.rst +++ b/docs/tools/tools-overview.rst @@ -54,23 +54,13 @@ packaged with |RCE| by default. .. code-block:: bash - $ .rccontrol/enterprise-4/profile/bin/rhodecode-extensions --plugins \ - --instance-name=enterprise-4 --ini-file=rhodecode.ini + $ .rccontrol/enterprise-4/profile/bin/rhodecode-api --instance-name=enterprise-4 get_ip [11:56:57 on 05/10/2018] - Writen new extensions file to rcextensions - Copied hipchat_push_notify.py plugin to rcextensions - Copied jira_pr_flow.py plugin to rcextensions - Copied default_reviewers.py plugin to rcextensions - Copied extract_commits.py plugin to rcextensions - Copied extract_issues.py plugin to rcextensions - Copied redmine_pr_flow.py plugin to rcextensions - Copied extra_fields.py plugin to rcextensions - Copied jira_smart_commits.py plugin to rcextensions - Copied http_notify.py plugin to rcextensions - Copied slack_push_notify.py plugin to rcextensions - Copied slack_message.py plugin to rcextensions - Copied extract_jira_issues.py plugin to rcextensions - Copied extract_redmine_issues.py plugin to rcextensions - Copied redmine_smart_commits.py plugin to rcextensions - Copied send_mail.py plugin to rcextensions - + { + "error": null, + "id": 1000, + "result": { + "server_ip_addr": "1.2.3.4", + "user_ips": [] + } + } diff --git a/rhodecode/apps/repository/tests/test_repo_pullrequests.py b/rhodecode/apps/repository/tests/test_repo_pullrequests.py --- a/rhodecode/apps/repository/tests/test_repo_pullrequests.py +++ b/rhodecode/apps/repository/tests/test_repo_pullrequests.py @@ -578,11 +578,11 @@ class TestPullrequestsView(object): assert actions[-1].action_data['commit_ids'] == pr_commit_ids # Check post_push rcextension was really executed - push_calls = rhodecode.EXTENSIONS.calls['post_push'] + push_calls = rhodecode.EXTENSIONS.calls['_push_hook'] assert len(push_calls) == 1 unused_last_call_args, last_call_kwargs = push_calls[0] assert last_call_kwargs['action'] == 'push' - assert last_call_kwargs['pushed_revs'] == pr_commit_ids + assert last_call_kwargs['commit_ids'] == pr_commit_ids def test_merge_pull_request_disabled(self, pr_util, csrf_token): pull_request = pr_util.create_pull_request(mergeable=False) diff --git a/rhodecode/config/rcextensions/__init__.py b/rhodecode/config/rcextensions/__init__.py new file mode 100644 --- /dev/null +++ b/rhodecode/config/rcextensions/__init__.py @@ -0,0 +1,56 @@ +# Copyright (C) 2016-2018 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +""" +rcextensions module, please edit `hooks.py` to over write hooks logic +""" + +from .hooks import ( + _create_repo_hook, + _create_repo_group_hook, + _pre_create_user_hook, + _create_user_hook, + _delete_repo_hook, + _delete_user_hook, + _pre_push_hook, + _push_hook, + _pre_pull_hook, + _pull_hook, + _create_pull_request_hook, + _review_pull_request_hook, + _update_pull_request_hook, + _merge_pull_request_hook, + _close_pull_request_hook, +) + +# set as module attributes, we use those to call hooks. *do not change this* +CREATE_REPO_HOOK = _create_repo_hook +CREATE_REPO_GROUP_HOOK = _create_repo_group_hook +PRE_CREATE_USER_HOOK = _pre_create_user_hook +CREATE_USER_HOOK = _create_user_hook +DELETE_REPO_HOOK = _delete_repo_hook +DELETE_USER_HOOK = _delete_user_hook +PRE_PUSH_HOOK = _pre_push_hook +PUSH_HOOK = _push_hook +PRE_PULL_HOOK = _pre_pull_hook +PULL_HOOK = _pull_hook +CREATE_PULL_REQUEST = _create_pull_request_hook +REVIEW_PULL_REQUEST = _review_pull_request_hook +UPDATE_PULL_REQUEST = _update_pull_request_hook +MERGE_PULL_REQUEST = _merge_pull_request_hook +CLOSE_PULL_REQUEST = _close_pull_request_hook diff --git a/rhodecode/config/rcextensions/examples/http_call_on_push.py b/rhodecode/config/rcextensions/examples/http_call_on_push.py new file mode 100644 --- /dev/null +++ b/rhodecode/config/rcextensions/examples/http_call_on_push.py @@ -0,0 +1,36 @@ +# Example to trigger a HTTP call via an HTTP helper via post_push hook + + +@has_kwargs({ + 'server_url': 'url of instance that triggered this hook', + 'config': 'path to .ini config used', + 'scm': 'type of version control "git", "hg", "svn"', + 'username': 'username of actor who triggered this event', + 'ip': 'ip address of actor who triggered this hook', + 'action': '', + 'repository': 'repository name', + 'repo_store_path': 'full path to where repositories are stored', + 'commit_ids': '', + 'hook_type': '', + 'user_agent': '', +}) +def _push_hook(*args, **kwargs): + """ + POST PUSH HOOK, this function will be executed after each push it's + executed after the build-in hook that RhodeCode uses for logging pushes + """ + + from .helpers import http_call, extra_fields + # returns list of dicts with key-val fetched from extra fields + repo_extra_fields = extra_fields.run(**kwargs) + + if repo_extra_fields.get('endpoint_url'): + endpoint = repo_extra_fields['endpoint_url'] + if endpoint: + data = { + 'some_key': 'val' + } + response = http_call.run(url=endpoint, json_data=data) + return HookResponse(0, 'Called endpoint {}, with response {}'.format(endpoint, response)) + + return HookResponse(0, '') diff --git a/rhodecode/config/rcextensions/examples/trigger_ci_call.py b/rhodecode/config/rcextensions/examples/trigger_ci_call.py new file mode 100644 --- /dev/null +++ b/rhodecode/config/rcextensions/examples/trigger_ci_call.py @@ -0,0 +1,36 @@ +# Example to trigger a CI call via an HTTP helper via post_push hook + + +@has_kwargs({ + 'server_url': 'url of instance that triggered this hook', + 'config': 'path to .ini config used', + 'scm': 'type of version control "git", "hg", "svn"', + 'username': 'username of actor who triggered this event', + 'ip': 'ip address of actor who triggered this hook', + 'action': '', + 'repository': 'repository name', + 'repo_store_path': 'full path to where repositories are stored', + 'commit_ids': '', + 'hook_type': '', + 'user_agent': '', +}) +def _push_hook(*args, **kwargs): + """ + POST PUSH HOOK, this function will be executed after each push it's + executed after the build-in hook that RhodeCode uses for logging pushes + """ + + from .helpers import http_call, extra_fields + # returns list of dicts with key-val fetched from extra fields + repo_extra_fields = extra_fields.run(**kwargs) + + if repo_extra_fields.get('endpoint_url'): + endpoint = repo_extra_fields['endpoint_url'] + if endpoint: + data = { + 'some_key': 'val' + } + response = http_call.run(url=endpoint, json_data=data) + return HookResponse(0, 'Called endpoint {}, with response {}'.format(endpoint, response)) + + return HookResponse(0, '') diff --git a/rhodecode/config/rcextensions/examples/validate_author.py b/rhodecode/config/rcextensions/examples/validate_author.py new file mode 100644 diff --git a/rhodecode/config/rcextensions/examples/validate_commit_message.py b/rhodecode/config/rcextensions/examples/validate_commit_message.py new file mode 100644 diff --git a/rhodecode/config/rcextensions/helpers/__init__.py b/rhodecode/config/rcextensions/helpers/__init__.py new file mode 100644 --- /dev/null +++ b/rhodecode/config/rcextensions/helpers/__init__.py @@ -0,0 +1,17 @@ +# Copyright (C) 2016-2018 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ diff --git a/rhodecode/config/rcextensions/helpers/extra_fields.py b/rhodecode/config/rcextensions/helpers/extra_fields.py new file mode 100644 --- /dev/null +++ b/rhodecode/config/rcextensions/helpers/extra_fields.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2016-2018 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +""" +us in hooks:: + + from .helpers import extra_fields + # returns list of dicts with key-val fetched from extra fields + repo_extra_fields = extra_fields.run(**kwargs) + +""" + + +def run(*args, **kwargs): + from rhodecode.model.db import Repository + # use temp name then the main one propagated + repo_name = kwargs.pop('REPOSITORY', None) or kwargs['repository'] + repo = Repository.get_by_repo_name(repo_name) + + fields = {} + for field in repo.extra_fields: + fields[field.field_key] = field.get_dict() + + return fields diff --git a/rhodecode/config/rcextensions/helpers/extract_post_commits.py b/rhodecode/config/rcextensions/helpers/extract_post_commits.py new file mode 100644 --- /dev/null +++ b/rhodecode/config/rcextensions/helpers/extract_post_commits.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2016-2018 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +""" +Extract and serialize commits taken from a list of commit_ids. This should +be used in post_push hook + +us in hooks:: + + from .helpers import extract_post_commits + # returns list of dicts with key-val fetched from extra fields + commit_list = extract_post_commits.run(**kwargs) +""" +import traceback + + +def run(*args, **kwargs): + from rhodecode.lib.utils2 import extract_mentioned_users + from rhodecode.model.db import Repository + + commit_ids = kwargs.get('commit_ids') + if not commit_ids: + return 0 + + # use temp name then the main one propagated + repo_name = kwargs.pop('REPOSITORY', None) or kwargs['repository'] + + repo = Repository.get_by_repo_name(repo_name) + commits = [] + + vcs_repo = repo.scm_instance(cache=False) + try: + for commit_id in commit_ids: + cs = vcs_repo.get_changeset(commit_id) + cs_data = cs.__json__() + cs_data['mentions'] = extract_mentioned_users(cs_data['message']) + # optionally add more logic to parse the commits, like reading extra + # fields of repository to read managers of reviewers ? + commits.append(cs_data) + except Exception: + print(traceback.format_exc()) + # we don't send any commits when crash happens, only full list matters + # we short circuit then. + return [] + return commits diff --git a/rhodecode/config/rcextensions/helpers/extract_pre_commits.py b/rhodecode/config/rcextensions/helpers/extract_pre_commits.py new file mode 100644 --- /dev/null +++ b/rhodecode/config/rcextensions/helpers/extract_pre_commits.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2016-2018 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +""" +us in hooks:: + + from .helpers import extract_pre_commits + # returns list of dicts with key-val fetched from extra fields + commit_list = extract_pre_commits.run(**kwargs) + +""" +import re +import collections + + +def get_hg_commits(repo, refs): + commits = [] + return commits + + +def get_git_commits(repo, refs): + commits = [] + return commits + + +def run(*args, **kwargs): + from rhodecode.model.db import Repository + + vcs_type = kwargs['scm'] + # use temp name then the main one propagated + repo_name = kwargs.pop('REPOSITORY', None) or kwargs['repository'] + + repo = Repository.get_by_repo_name(repo_name) + vcs_repo = repo.scm_instance(cache=False) + + commits = [] + + for rev_data in kwargs['commit_ids']: + new_environ = dict((k, v) for k, v in rev_data['hg_env']) + + if vcs_type == 'git': + commits = get_git_commits(vcs_repo, kwargs['commit_ids']) + + if vcs_type == 'hg': + commits = get_hg_commits(vcs_repo, kwargs['commit_ids']) + + return commits diff --git a/rhodecode/config/rcextensions/helpers/http_call.py b/rhodecode/config/rcextensions/helpers/http_call.py new file mode 100644 --- /dev/null +++ b/rhodecode/config/rcextensions/helpers/http_call.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2016-2018 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +""" +us in hooks:: + + from .helpers import http_call + # returns response after making a POST call + response = http_call.run(url=url, json_data=data) + +""" + +from rhodecode.integrations.types.base import requests_retry_call + + +def run(url, json_data, method='post'): + requests_session = requests_retry_call() + requests_session.verify = True # Verify SSL + resp = requests_session.post(url, json=json_data, timeout=60) + return resp.raise_for_status() # raise exception on a failed request diff --git a/rhodecode/config/rcextensions/hooks.py b/rhodecode/config/rcextensions/hooks.py new file mode 100644 --- /dev/null +++ b/rhodecode/config/rcextensions/hooks.py @@ -0,0 +1,431 @@ +# Copyright (C) 2016-2018 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +from .utils import DotDict, HookResponse, has_kwargs + + +# Config shortcut to keep, all configuration in one place +# Example: api_key = CONFIG.my_config.api_key +CONFIG = DotDict( + my_config=DotDict( + api_key='', + ), + +) + + +@has_kwargs({ + 'repo_name': '', + 'repo_type': '', + 'description': '', + 'private': '', + 'created_on': '', + 'enable_downloads': '', + 'repo_id': '', + 'user_id': '', + 'enable_statistics': '', + 'clone_uri': '', + 'fork_id': '', + 'group_id': '', + 'created_by': '' +}) +def _create_repo_hook(*args, **kwargs): + """ + POST CREATE REPOSITORY HOOK. This function will be executed after + each repository is created. kwargs available: + + """ + return HookResponse(0, '') + + +@has_kwargs({ + 'group_name': '', + 'group_parent_id': '', + 'group_description': '', + 'group_id': '', + 'user_id': '', + 'created_by': '', + 'created_on': '', + 'enable_locking': '' +}) +def _create_repo_group_hook(*args, **kwargs): + """ + POST CREATE REPOSITORY GROUP HOOK, this function will be + executed after each repository group is created. kwargs available: + """ + return HookResponse(0, '') + + +@has_kwargs({ + 'username': '', + 'password': '', + 'email': '', + 'firstname': '', + 'lastname': '', + 'active': '', + 'admin': '', + 'created_by': '', +}) +def _pre_create_user_hook(*args, **kwargs): + """ + PRE CREATE USER HOOK, this function will be executed before each + user is created, it returns a tuple of bool, reason. + If bool is False the user creation will be stopped and reason + will be displayed to the user. + + Return HookResponse(1, reason) to block user creation + + """ + + reason = 'allowed' + return HookResponse(0, reason) + + +@has_kwargs({ + 'username': '', + 'full_name_or_username': '', + 'full_contact': '', + 'user_id': '', + 'name': '', + 'firstname': '', + 'short_contact': '', + 'admin': '', + 'lastname': '', + 'ip_addresses': '', + 'extern_type': '', + 'extern_name': '', + 'email': '', + 'api_key': '', + 'api_keys': '', + 'last_login': '', + 'full_name': '', + 'active': '', + 'password': '', + 'emails': '', + 'inherit_default_permissions': '', + 'created_by': '', + 'created_on': '', +}) +def _create_user_hook(*args, **kwargs): + """ + POST CREATE USER HOOK, this function will be executed after each user is created + """ + return HookResponse(0, '') + + +@has_kwargs({ + 'repo_name': '', + 'repo_type': '', + 'description': '', + 'private': '', + 'created_on': '', + 'enable_downloads': '', + 'repo_id': '', + 'user_id': '', + 'enable_statistics': '', + 'clone_uri': '', + 'fork_id': '', + 'group_id': '', + 'deleted_by': '', + 'deleted_on': '', +}) +def _delete_repo_hook(*args, **kwargs): + """ + POST DELETE REPOSITORY HOOK, this function will be executed after + each repository deletion + """ + return HookResponse(0, '') + + +@has_kwargs({ + 'username': '', + 'full_name_or_username': '', + 'full_contact': '', + 'user_id': '', + 'name': '', + 'short_contact': '', + 'admin': '', + 'firstname': '', + 'lastname': '', + 'ip_addresses': '', + 'email': '', + 'api_key': '', + 'last_login': '', + 'full_name': '', + 'active': '', + 'password': '', + 'emails': '', + 'inherit_default_permissions': '', + 'deleted_by': '', + }) +def _delete_user_hook(*args, **kwargs): + """ + POST DELETE USER HOOK, this function will be executed after each + user is deleted kwargs available: + """ + return HookResponse(0, '') + + +# ============================================================================= +# PUSH/PULL RELATED HOOKS +# ============================================================================= +@has_kwargs({ + 'server_url': 'url of instance that triggered this hook', + 'config': 'path to .ini config used', + 'scm': 'type of version control "git", "hg", "svn"', + 'username': 'username of actor who triggered this event', + 'ip': 'ip address of actor who triggered this hook', + 'action': '', + 'repository': 'repository name', + 'repo_store_path': 'full path to where repositories are stored', + 'commit_ids': 'pre transaction metadata for commit ids', + 'hook_type': '', + 'user_agent': 'Client user agent, e.g git or mercurial CLI version', +}) +def _pre_push_hook(*args, **kwargs): + """ + Post push hook + To stop version control from storing the transaction and send a message to user + use non-zero HookResponse with a message, e.g return HookResponse(1, 'Not allowed') + + This message will be shown back to client during PUSH operation + + Commit ids might look like that:: + + [{u'hg_env|git_env': ..., + u'multiple_heads': [], + u'name': u'default', + u'new_rev': u'd0befe0692e722e01d5677f27a104631cf798b69', + u'old_rev': u'd0befe0692e722e01d5677f27a104631cf798b69', + u'ref': u'', + u'total_commits': 2, + u'type': u'branch'}] + """ + return HookResponse(0, '') + + +@has_kwargs({ + 'server_url': 'url of instance that triggered this hook', + 'config': 'path to .ini config used', + 'scm': 'type of version control "git", "hg", "svn"', + 'username': 'username of actor who triggered this event', + 'ip': 'ip address of actor who triggered this hook', + 'action': '', + 'repository': 'repository name', + 'repo_store_path': 'full path to where repositories are stored', + 'commit_ids': 'list of pushed commit_ids (sha1)', + 'hook_type': '', + 'user_agent': 'Client user agent, e.g git or mercurial CLI version', +}) +def _push_hook(*args, **kwargs): + """ + POST PUSH HOOK, this function will be executed after each push it's + executed after the build-in hook that RhodeCode uses for logging pushes + """ + return HookResponse(0, '') + + +@has_kwargs({ + 'server_url': 'url of instance that triggered this hook', + 'repo_store_path': 'full path to where repositories are stored', + 'config': 'path to .ini config used', + 'scm': 'type of version control "git", "hg", "svn"', + 'username': 'username of actor who triggered this event', + 'ip': 'ip address of actor who triggered this hook', + 'action': '', + 'repository': 'repository name', + 'hook_type': '', + 'user_agent': 'Client user agent, e.g git or mercurial CLI version', +}) +def _pre_pull_hook(*args, **kwargs): + """ + Post pull hook + """ + return HookResponse(0, '') + + +@has_kwargs({ + 'server_url': 'url of instance that triggered this hook', + 'repo_store_path': 'full path to where repositories are stored', + 'config': 'path to .ini config used', + 'scm': 'type of version control "git", "hg", "svn"', + 'username': 'username of actor who triggered this event', + 'ip': 'ip address of actor who triggered this hook', + 'action': '', + 'repository': 'repository name', + 'hook_type': '', + 'user_agent': 'Client user agent, e.g git or mercurial CLI version', +}) +def _pull_hook(*args, **kwargs): + """ + This hook will be executed after each code pull. + """ + return HookResponse(0, '') + + +# ============================================================================= +# PULL REQUEST RELATED HOOKS +# ============================================================================= +@has_kwargs({ + 'server_url': 'url of instance that triggered this hook', + 'config': 'path to .ini config used', + 'scm': 'type of version control "git", "hg", "svn"', + 'username': 'username of actor who triggered this event', + 'ip': 'ip address of actor who triggered this hook', + 'action': '', + 'repository': 'repository name', + 'pull_request_id': '', + 'url': '', + 'title': '', + 'description': '', + 'status': '', + 'created_on': '', + 'updated_on': '', + 'commit_ids': '', + 'review_status': '', + 'mergeable': '', + 'source': '', + 'target': '', + 'author': '', + 'reviewers': '', +}) +def _create_pull_request_hook(*args, **kwargs): + """ + This hook will be executed after creation of a pull request. + """ + return HookResponse(0, '') + + +@has_kwargs({ + 'server_url': 'url of instance that triggered this hook', + 'config': 'path to .ini config used', + 'scm': 'type of version control "git", "hg", "svn"', + 'username': 'username of actor who triggered this event', + 'ip': 'ip address of actor who triggered this hook', + 'action': '', + 'repository': 'repository name', + 'pull_request_id': '', + 'url': '', + 'title': '', + 'description': '', + 'status': '', + 'created_on': '', + 'updated_on': '', + 'commit_ids': '', + 'review_status': '', + 'mergeable': '', + 'source': '', + 'target': '', + 'author': '', + 'reviewers': '', +}) +def _review_pull_request_hook(*args, **kwargs): + """ + This hook will be executed after review action was made on a pull request. + """ + return HookResponse(0, '') + + +@has_kwargs({ + 'server_url': 'url of instance that triggered this hook', + 'config': 'path to .ini config used', + 'scm': 'type of version control "git", "hg", "svn"', + 'username': 'username of actor who triggered this event', + 'ip': 'ip address of actor who triggered this hook', + 'action': '', + 'repository': 'repository name', + 'pull_request_id': '', + 'url': '', + 'title': '', + 'description': '', + 'status': '', + 'created_on': '', + 'updated_on': '', + 'commit_ids': '', + 'review_status': '', + 'mergeable': '', + 'source': '', + 'target': '', + 'author': '', + 'reviewers': '', +}) +def _update_pull_request_hook(*args, **kwargs): + """ + This hook will be executed after pull requests has been updated with new commits. + """ + return HookResponse(0, '') + + +@has_kwargs({ + 'server_url': 'url of instance that triggered this hook', + 'config': 'path to .ini config used', + 'scm': 'type of version control "git", "hg", "svn"', + 'username': 'username of actor who triggered this event', + 'ip': 'ip address of actor who triggered this hook', + 'action': '', + 'repository': 'repository name', + 'pull_request_id': '', + 'url': '', + 'title': '', + 'description': '', + 'status': '', + 'created_on': '', + 'updated_on': '', + 'commit_ids': '', + 'review_status': '', + 'mergeable': '', + 'source': '', + 'target': '', + 'author': '', + 'reviewers': '', +}) +def _merge_pull_request_hook(*args, **kwargs): + """ + This hook will be executed after merge of a pull request. + """ + return HookResponse(0, '') + + +@has_kwargs({ + 'server_url': 'url of instance that triggered this hook', + 'config': 'path to .ini config used', + 'scm': 'type of version control "git", "hg", "svn"', + 'username': 'username of actor who triggered this event', + 'ip': 'ip address of actor who triggered this hook', + 'action': '', + 'repository': 'repository name', + 'pull_request_id': '', + 'url': '', + 'title': '', + 'description': '', + 'status': '', + 'created_on': '', + 'updated_on': '', + 'commit_ids': '', + 'review_status': '', + 'mergeable': '', + 'source': '', + 'target': '', + 'author': '', + 'reviewers': '', +}) +def _close_pull_request_hook(*args, **kwargs): + """ + This hook will be executed after close of a pull request. + """ + return HookResponse(0, '') diff --git a/rhodecode/config/rcextensions/mapping.py b/rhodecode/config/rcextensions/mapping.py new file mode 100644 --- /dev/null +++ b/rhodecode/config/rcextensions/mapping.py @@ -0,0 +1,21 @@ +# ============================================================================= +# END OF UTILITY FUNCTIONS HERE +# ============================================================================= + +# Additional mappings that are not present in the pygments lexers +# used for building stats +# format is {'ext':['Names']} eg. {'py':['Python']} note: there can be +# more than one name for extension +# NOTE: that this will override any mappings in LANGUAGES_EXTENSIONS_MAP +# build by pygments +EXTRA_MAPPINGS = {'html': ['Text']} + +# additional lexer definitions for custom files it's overrides pygments lexers, +# and uses defined name of lexer to colorize the files. Format is {'ext': +# 'lexer_name'} List of lexers can be printed running: +# >> python -c "import pprint;from pygments import lexers; +# pprint.pprint([(x[0], x[1]) for x in lexers.get_all_lexers()]);" + +EXTRA_LEXERS = { + 'tt': 'vbnet' +} diff --git a/rhodecode/config/rcextensions/utils.py b/rhodecode/config/rcextensions/utils.py new file mode 100644 --- /dev/null +++ b/rhodecode/config/rcextensions/utils.py @@ -0,0 +1,147 @@ +# Copyright (C) 2016-2018 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +import os +import functools +import collections + + +class HookResponse(object): + def __init__(self, status, output): + self.status = status + self.output = output + + def __add__(self, other): + other_status = getattr(other, 'status', 0) + new_status = max(self.status, other_status) + other_output = getattr(other, 'output', '') + new_output = self.output + other_output + + return HookResponse(new_status, new_output) + + def __bool__(self): + return self.status == 0 + + +class DotDict(dict): + + def __contains__(self, k): + try: + return dict.__contains__(self, k) or hasattr(self, k) + except: + return False + + # only called if k not found in normal places + def __getattr__(self, k): + try: + return object.__getattribute__(self, k) + except AttributeError: + try: + return self[k] + except KeyError: + raise AttributeError(k) + + def __setattr__(self, k, v): + try: + object.__getattribute__(self, k) + except AttributeError: + try: + self[k] = v + except: + raise AttributeError(k) + else: + object.__setattr__(self, k, v) + + def __delattr__(self, k): + try: + object.__getattribute__(self, k) + except AttributeError: + try: + del self[k] + except KeyError: + raise AttributeError(k) + else: + object.__delattr__(self, k) + + def toDict(self): + return unserialize(self) + + def __repr__(self): + keys = list(self.keys()) + keys.sort() + args = ', '.join(['%s=%r' % (key, self[key]) for key in keys]) + return '%s(%s)' % (self.__class__.__name__, args) + + @staticmethod + def fromDict(d): + return serialize(d) + + +def serialize(x): + if isinstance(x, dict): + return DotDict((k, serialize(v)) for k, v in x.items()) + elif isinstance(x, (list, tuple)): + return type(x)(serialize(v) for v in x) + else: + return x + + +def unserialize(x): + if isinstance(x, dict): + return dict((k, unserialize(v)) for k, v in x.items()) + elif isinstance(x, (list, tuple)): + return type(x)(unserialize(v) for v in x) + else: + return x + + +def _verify_kwargs(func_name, expected_parameters, kwargs): + """ + Verify that exactly `expected_parameters` are passed in as `kwargs`. + """ + expected_parameters = set(expected_parameters) + kwargs_keys = set(kwargs.keys()) + if kwargs_keys != expected_parameters: + missing_kwargs = expected_parameters - kwargs_keys + unexpected_kwargs = kwargs_keys - expected_parameters + raise AssertionError( + "func:%s: missing parameters: %r, unexpected parameters: %s" % + (func_name, missing_kwargs, unexpected_kwargs)) + + +def has_kwargs(required_args): + """ + decorator to verify extension calls arguments. + + :param required_args: + """ + def wrap(func): + def wrapper(*args, **kwargs): + _verify_kwargs(func.func_name, required_args.keys(), kwargs) + # in case there's `calls` defined on module we store the data + maybe_log_call(func.func_name, args, kwargs) + return func(*args, **kwargs) + return wrapper + return wrap + + +def maybe_log_call(name, args, kwargs): + from rhodecode.config import rcextensions + if hasattr(rcextensions, 'calls'): + calls = rcextensions.calls + calls[name].append((args, kwargs)) diff --git a/rhodecode/lib/fakemod.py b/rhodecode/lib/fakemod.py deleted file mode 100644 --- a/rhodecode/lib/fakemod.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright (C) 2010-2018 RhodeCode GmbH -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License, version 3 -# (only), as published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -# This program is dual-licensed. If you wish to learn more about the -# RhodeCode Enterprise Edition, including its added features, Support services, -# and proprietary license terms, please see https://rhodecode.com/licenses/ - -import imp - - -def create_module(name, path): - """ - Returns module created *on the fly*. Returned module would have name same - as given ``name`` and would contain code read from file at the given - ``path`` (it may also be a zip or package containing *__main__* module). - """ - module = imp.new_module(name) - module.__file__ = path - execfile(path, module.__dict__) - return module diff --git a/rhodecode/lib/hooks_base.py b/rhodecode/lib/hooks_base.py --- a/rhodecode/lib/hooks_base.py +++ b/rhodecode/lib/hooks_base.py @@ -39,7 +39,21 @@ from rhodecode.model.db import Repositor log = logging.getLogger(__name__) -HookResponse = collections.namedtuple('HookResponse', ('status', 'output')) +class HookResponse(object): + def __init__(self, status, output): + self.status = status + self.output = output + + def __add__(self, other): + other_status = getattr(other, 'status', 0) + new_status = max(self.status, other_status) + other_output = getattr(other, 'output', '') + new_output = self.output + other_output + + return HookResponse(new_status, new_output) + + def __bool__(self): + return self.status == 0 def is_shadow_repo(extras): @@ -110,6 +124,7 @@ def pre_push(extras): else: raise _http_ret + hook_response = '' if not is_shadow_repo(extras): if extras.commit_ids and extras.check_branch_perms: @@ -152,11 +167,12 @@ def pre_push(extras): # Propagate to external components. This is done after checking the # lock, for consistent behavior. - pre_push_extension(repo_store_path=Repository.base_path(), **extras) + hook_response = pre_push_extension( + repo_store_path=Repository.base_path(), **extras) events.trigger(events.RepoPrePushEvent( repo_name=extras.repository, extras=extras)) - return HookResponse(0, output) + return HookResponse(0, output) + hook_response def pre_pull(extras): @@ -182,12 +198,15 @@ def pre_pull(extras): # Propagate to external components. This is done after checking the # lock, for consistent behavior. + hook_response = '' if not is_shadow_repo(extras): - pre_pull_extension(**extras) + extras.hook_type = extras.hook_type or 'pre_pull' + hook_response = pre_pull_extension( + repo_store_path=Repository.base_path(), **extras) events.trigger(events.RepoPrePullEvent( repo_name=extras.repository, extras=extras)) - return HookResponse(0, output) + return HookResponse(0, output) + hook_response def post_pull(extras): @@ -198,16 +217,9 @@ def post_pull(extras): ip_addr=extras.ip) repo = audit_logger.RepoWrap(repo_name=extras.repository) audit_logger.store( - 'user.pull', action_data={ - 'user_agent': extras.user_agent}, + 'user.pull', action_data={'user_agent': extras.user_agent}, user=audit_user, repo=repo, commit=True) - # Propagate to external components. - if not is_shadow_repo(extras): - post_pull_extension(**extras) - events.trigger(events.RepoPullEvent( - repo_name=extras.repository, extras=extras)) - output = '' # make lock is a tri state False, True, None. We only make lock on True if extras.make_lock is True and not is_shadow_repo(extras): @@ -227,7 +239,16 @@ def post_pull(extras): # 2xx Codes don't raise exceptions output += _http_ret.title - return HookResponse(0, output) + # Propagate to external components. + hook_response = '' + if not is_shadow_repo(extras): + extras.hook_type = extras.hook_type or 'post_pull' + hook_response = post_pull_extension( + repo_store_path=Repository.base_path(), **extras) + events.trigger(events.RepoPullEvent( + repo_name=extras.repository, extras=extras)) + + return HookResponse(0, output) + hook_response def post_push(extras): @@ -245,16 +266,6 @@ def post_push(extras): user=audit_user, repo=repo, commit=True) # Propagate to external components. - if not is_shadow_repo(extras): - post_push_extension( - repo_store_path=Repository.base_path(), - pushed_revs=commit_ids, - **extras) - events.trigger(events.RepoPushEvent( - repo_name=extras.repository, - pushed_commit_ids=commit_ids, - extras=extras)) - output = '' # make lock is a tri state False, True, None. We only release lock on False if extras.make_lock is False and not is_shadow_repo(extras): @@ -285,8 +296,16 @@ def post_push(extras): output += 'RhodeCode: open pull request link: {}\n'.format( tmpl.format(ref_type='bookmark', ref_name=book_name)) + hook_response = '' + if not is_shadow_repo(extras): + hook_response = post_push_extension( + repo_store_path=Repository.base_path(), + **extras) + events.trigger(events.RepoPushEvent( + repo_name=extras.repository, pushed_commit_ids=commit_ids, extras=extras)) + output += 'RhodeCode: push completed\n' - return HookResponse(0, output) + return HookResponse(0, output) + hook_response def _locked_by_explanation(repo_name, user_name, reason): @@ -299,8 +318,10 @@ def _locked_by_explanation(repo_name, us def check_allowed_create_user(user_dict, created_by, **kwargs): # pre create hooks if pre_create_user.is_active(): - allowed, reason = pre_create_user(created_by=created_by, **user_dict) + hook_result = pre_create_user(created_by=created_by, **user_dict) + allowed = hook_result.status == 0 if not allowed: + reason = hook_result.output raise UserCreationError(reason) @@ -319,8 +340,15 @@ class ExtensionCallback(object): def __call__(self, *args, **kwargs): log.debug('Calling extension callback for %s', self._hook_name) + kwargs_to_pass = {} + for key in self._kwargs_keys: + try: + kwargs_to_pass[key] = kwargs[key] + except KeyError: + log.error('Failed to fetch %s key. Expected keys: %s', + key, self._kwargs_keys) + raise - kwargs_to_pass = dict((key, kwargs[key]) for key in self._kwargs_keys) # backward compat for removed api_key for old hooks. THis was it works # with older rcextensions that require api_key present if self._hook_name in ['CREATE_USER_HOOK', 'DELETE_USER_HOOK']: @@ -343,28 +371,28 @@ pre_pull_extension = ExtensionCallback( hook_name='PRE_PULL_HOOK', kwargs_keys=( 'server_url', 'config', 'scm', 'username', 'ip', 'action', - 'repository')) + 'repository', 'hook_type', 'user_agent', 'repo_store_path',)) post_pull_extension = ExtensionCallback( hook_name='PULL_HOOK', kwargs_keys=( 'server_url', 'config', 'scm', 'username', 'ip', 'action', - 'repository')) + 'repository', 'hook_type', 'user_agent', 'repo_store_path',)) pre_push_extension = ExtensionCallback( hook_name='PRE_PUSH_HOOK', kwargs_keys=( 'server_url', 'config', 'scm', 'username', 'ip', 'action', - 'repository', 'repo_store_path', 'commit_ids')) + 'repository', 'repo_store_path', 'commit_ids', 'hook_type', 'user_agent',)) post_push_extension = ExtensionCallback( hook_name='PUSH_HOOK', kwargs_keys=( 'server_url', 'config', 'scm', 'username', 'ip', 'action', - 'repository', 'repo_store_path', 'pushed_revs')) + 'repository', 'repo_store_path', 'commit_ids', 'hook_type', 'user_agent',)) pre_create_user = ExtensionCallback( diff --git a/rhodecode/lib/hooks_utils.py b/rhodecode/lib/hooks_utils.py --- a/rhodecode/lib/hooks_utils.py +++ b/rhodecode/lib/hooks_utils.py @@ -47,7 +47,7 @@ def _get_rc_scm_extras(username, repo_na def trigger_post_push_hook( - username, action, repo_name, repo_alias, commit_ids): + username, action, hook_type, repo_name, repo_alias, commit_ids): """ Triggers push action hooks @@ -59,6 +59,7 @@ def trigger_post_push_hook( """ extras = _get_rc_scm_extras(username, repo_name, repo_alias, action) extras.commit_ids = commit_ids + extras.hook_type = hook_type hooks_base.post_push(extras) diff --git a/rhodecode/lib/utils.py b/rhodecode/lib/utils.py --- a/rhodecode/lib/utils.py +++ b/rhodecode/lib/utils.py @@ -28,6 +28,7 @@ import json import logging import os import re +import sys import shutil import tempfile import traceback @@ -43,7 +44,6 @@ from mako import exceptions from pyramid.threadlocal import get_current_registry from rhodecode.lib.request import Request -from rhodecode.lib.fakemod import create_module from rhodecode.lib.vcs.backends.base import Config from rhodecode.lib.vcs.exceptions import VCSError from rhodecode.lib.vcs.utils.helpers import get_scm, get_scm_backend @@ -631,21 +631,21 @@ def load_rcextensions(root_path): import rhodecode from rhodecode.config import conf - path = os.path.join(root_path, 'rcextensions', '__init__.py') - if os.path.isfile(path): - rcext = create_module('rc', path) - EXT = rhodecode.EXTENSIONS = rcext - log.debug('Found rcextensions now loading %s...', rcext) + path = os.path.join(root_path) + sys.path.append(path) + try: + rcextensions = __import__('rcextensions') + except ImportError: + log.warn('Unable to load rcextensions from %s', path) + rcextensions = None + + if rcextensions: + log.debug('Found rcextensions module loaded %s...', rcextensions) + rhodecode.EXTENSIONS = rcextensions # Additional mappings that are not present in the pygments lexers - conf.LANGUAGES_EXTENSIONS_MAP.update(getattr(EXT, 'EXTRA_MAPPINGS', {})) - - # auto check if the module is not missing any data, set to default if is - # this will help autoupdate new feature of rcext module - #from rhodecode.config import rcextensions - #for k in dir(rcextensions): - # if not k.startswith('_') and not hasattr(EXT, k): - # setattr(EXT, k, getattr(rcextensions, k)) + conf.LANGUAGES_EXTENSIONS_MAP.update( + getattr(rhodecode.EXTENSIONS, 'EXTRA_MAPPINGS', {})) def get_custom_lexer(extension): diff --git a/rhodecode/model/pull_request.py b/rhodecode/model/pull_request.py --- a/rhodecode/model/pull_request.py +++ b/rhodecode/model/pull_request.py @@ -590,6 +590,7 @@ class PullRequestModel(BaseModel): def merge_repo(self, pull_request, user, extras): log.debug("Merging pull request %s", pull_request.pull_request_id) + extras['user_agent'] = 'internal-merge' merge_state = self._merge_pull_request(pull_request, user, extras) if merge_state.executed: log.debug( diff --git a/rhodecode/model/scm.py b/rhodecode/model/scm.py --- a/rhodecode/model/scm.py +++ b/rhodecode/model/scm.py @@ -465,8 +465,8 @@ class ScmModel(BaseModel): # We trigger the post-push action hooks_utils.trigger_post_push_hook( - username=user.username, action='push_local', repo_name=repo_name, - repo_alias=repo.alias, commit_ids=[tip.raw_id]) + username=user.username, action='push_local', hook_type='post_push', + repo_name=repo_name, repo_alias=repo.alias, commit_ids=[tip.raw_id]) return tip def _sanitize_path(self, f_path): @@ -644,6 +644,7 @@ class ScmModel(BaseModel): hooks_utils.trigger_post_push_hook( username=user.username, action='push_local', repo_name=repo.repo_name, repo_alias=scm_instance.alias, + hook_type='post_push', commit_ids=[tip.raw_id]) return tip @@ -708,7 +709,7 @@ class ScmModel(BaseModel): if trigger_push_hook: hooks_utils.trigger_post_push_hook( - username=user.username, action='push_local', + username=user.username, action='push_local', hook_type='post_push', repo_name=repo.repo_name, repo_alias=scm_instance.alias, commit_ids=[tip.raw_id]) @@ -768,7 +769,7 @@ class ScmModel(BaseModel): self.mark_for_invalidation(repo.repo_name) if trigger_push_hook: hooks_utils.trigger_post_push_hook( - username=user.username, action='push_local', + username=user.username, action='push_local', hook_type='post_push', repo_name=repo.repo_name, repo_alias=scm_instance.alias, commit_ids=[tip.raw_id]) return tip diff --git a/rhodecode/tests/events/test_repo.py b/rhodecode/tests/events/test_repo.py --- a/rhodecode/tests/events/test_repo.py +++ b/rhodecode/tests/events/test_repo.py @@ -45,9 +45,10 @@ def scm_extras(user_regular, repo_stub): 'repo_store': '', 'server_url': 'http://example.com', 'make_lock': None, - 'user-agent': 'some-client', + 'user_agent': 'some-client', 'locked_by': [None], 'commit_ids': ['a' * 40] * 3, + 'hook_type': 'scm_extras_test', 'is_shadow_repo': False, }) return extras diff --git a/rhodecode/tests/lib/test_hooks_base.py b/rhodecode/tests/lib/test_hooks_base.py --- a/rhodecode/tests/lib/test_hooks_base.py +++ b/rhodecode/tests/lib/test_hooks_base.py @@ -38,6 +38,7 @@ def test_post_push_truncates_commits(use 'user_agent': 'some-client', 'locked_by': [None], 'commit_ids': ['abcde12345' * 4] * 30000, + 'hook_type': 'large_push_test_type', 'is_shadow_repo': False, } extras = utils2.AttributeDict(extras) @@ -76,6 +77,7 @@ def hook_extras(user_regular, repo_stub) 'user_agent': 'some-client', 'locked_by': [None], 'commit_ids': [], + 'hook_type': 'test_type', 'is_shadow_repo': False, }) return extras @@ -92,7 +94,12 @@ def test_hooks_propagate(func, extension Tests that our hook code propagates to rhodecode extensions and triggers the appropriate event. """ - extension_mock = mock.Mock() + class ExtensionMock(mock.Mock): + @property + def output(self): + return 'MOCK' + + extension_mock = ExtensionMock() events_mock = mock.Mock() patches = { 'Repository': mock.Mock(), diff --git a/rhodecode/tests/models/test_pullrequest.py b/rhodecode/tests/models/test_pullrequest.py --- a/rhodecode/tests/models/test_pullrequest.py +++ b/rhodecode/tests/models/test_pullrequest.py @@ -393,6 +393,7 @@ class TestIntegrationMerge(object): def test_merge_triggers_push_hooks( self, pr_util, user_admin, capture_rcextensions, merge_extras, extra_config): + pull_request = pr_util.create_pull_request( approved=True, mergeable=True) # TODO: johbo: Needed for sqlite, try to find an automatic way for it @@ -404,8 +405,8 @@ class TestIntegrationMerge(object): pull_request, user_admin, extras=merge_extras) assert merge_state.executed - assert 'pre_push' in capture_rcextensions - assert 'post_push' in capture_rcextensions + assert '_pre_push_hook' in capture_rcextensions + assert '_push_hook' in capture_rcextensions def test_merge_can_be_rejected_by_pre_push_hook( self, pr_util, user_admin, capture_rcextensions, merge_extras): diff --git a/rhodecode/tests/other/example_rcextensions.py b/rhodecode/tests/other/example_rcextensions.py deleted file mode 100644 --- a/rhodecode/tests/other/example_rcextensions.py +++ /dev/null @@ -1,521 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright (C) 2010-2018 RhodeCode GmbH -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License, version 3 -# (only), as published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -# This program is dual-licensed. If you wish to learn more about the -# RhodeCode Enterprise Edition, including its added features, Support services, -# and proprietary license terms, please see https://rhodecode.com/licenses/ - -""" -Reference example of the rcextensions module. - -This module is also used during integration tests to verify that all showed -examples are valid. -""" - -import collections -import os -import imp - -here = os.path.dirname(os.path.abspath(__file__)) - - -def load_extension(filename, async=False): - """ - use to load extensions inside rcextension folder. - for example:: - - callback = load_extension('email.py', async=False) - if callback: - callback('foobar') - - put file named email.py inside rcextensions folder to load it. Changing - async=True will make the call of the plugin async, it's useful for - blocking calls like sending an email or notification with APIs. - """ - mod = ''.join(filename.split('.')[:-1]) - loaded = imp.load_source(mod, os.path.join(here, filename)) - - callback = getattr(loaded, 'run', None) - if not callback: - raise Exception('Plugin missing `run` method') - if async: - # modify callback so it's actually an async call - def _async_callback(*args, **kwargs): - import threading - thr = threading.Thread(target=callback, args=args, kwargs=kwargs) - thr.start() - if kwargs.get('_async_block'): - del kwargs['_async_block'] - thr.join() - - return _async_callback - return callback - - -# Additional mappings that are not present in the pygments lexers -# used for building stats -# format is {'ext':['Names']} eg. {'py':['Python']} note: there can be -# more than one name for extension -# NOTE: that this will override any mappings in LANGUAGES_EXTENSIONS_MAP -# build by pygments -EXTRA_MAPPINGS = {} - -# additional lexer definitions for custom files it's overrides pygments lexers, -# and uses defined name of lexer to colorize the files. Format is {'ext': -# 'lexer_name'} List of lexers can be printed running: -# >> python -c "import pprint;from pygments import lexers; -# pprint.pprint([(x[0], x[1]) for x in lexers.get_all_lexers()]);" - -EXTRA_LEXERS = {} - - -calls = collections.defaultdict(list) - - -def log_call(name): - def wrap(f): - def wrapper(*args, **kwargs): - calls[name].append((args, kwargs)) - return f(*args, **kwargs) - return wrapper - return wrap - - -# ============================================================================= -# POST CREATE REPOSITORY HOOK -# ============================================================================= -# this function will be executed after each repository is created - - -def _crrepohook(*args, **kwargs): - """ - Post create repository HOOK - kwargs available: - :param repo_name: - :param repo_type: - :param description: - :param private: - :param created_on: - :param enable_downloads: - :param repo_id: - :param user_id: - :param enable_statistics: - :param clone_uri: - :param fork_id: - :param group_id: - :param created_by: - """ - - expected_parameters = ( - 'repo_name', 'repo_type', 'description', 'private', 'created_on', - 'enable_downloads', 'repo_id', 'user_id', 'enable_statistics', - 'clone_uri', 'fork_id', 'group_id', 'created_by') - _verify_kwargs(expected_parameters, kwargs) - - return 0 - -CREATE_REPO_HOOK = _crrepohook - - -# ============================================================================= -# POST CREATE REPOSITORY GROUP HOOK -# ============================================================================= -# this function will be executed after each repository group is created - - -def _crrepogrouphook(*args, **kwargs): - """ - Post create repository group HOOK - kwargs available: - - :param group_name: - :param group_parent_id: - :param group_description: - :param group_id: - :param user_id: - :param created_by: - :param created_on: - :param enable_locking: - """ - - expected_parameters = ( - 'group_name', 'group_parent_id', 'group_description', - 'group_id', 'user_id', 'created_by', 'created_on', - 'enable_locking') - _verify_kwargs(expected_parameters, kwargs) - - return 0 - -CREATE_REPO_GROUP_HOOK = _crrepogrouphook - -# ============================================================================= -# PRE CREATE USER HOOK -# ============================================================================= -# this function will be executed before each user is created - - -def _pre_cruserhook(*args, **kwargs): - """ - Pre create user HOOK, it returns a tuple of bool, reason. - If bool is False the user creation will be stopped and reason - will be displayed to the user. - kwargs available: - :param username: - :param password: - :param email: - :param firstname: - :param lastname: - :param active: - :param admin: - :param created_by: - """ - - expected_parameters = ( - 'username', 'password', 'email', 'firstname', 'lastname', 'active', - 'admin', 'created_by') - _verify_kwargs(expected_parameters, kwargs) - - reason = 'allowed' - return True, reason -PRE_CREATE_USER_HOOK = _pre_cruserhook - -# ============================================================================= -# POST CREATE USER HOOK -# ============================================================================= -# this function will be executed after each user is created - - -def _cruserhook(*args, **kwargs): - """ - Post create user HOOK - kwargs available: - :param username: - :param full_name_or_username: - :param full_contact: - :param user_id: - :param name: - :param firstname: - :param short_contact: - :param admin: - :param lastname: - :param ip_addresses: - :param extern_type: - :param extern_name: - :param email: - :param api_key: - :parma api_keys: - :param last_login: - :param full_name: - :param active: - :param password: - :param emails: - :param inherit_default_permissions: - :param created_by: - :param created_on: - """ - - expected_parameters = ( - 'username', 'full_name_or_username', 'full_contact', 'user_id', - 'name', 'firstname', 'short_contact', 'admin', 'lastname', - 'ip_addresses', 'extern_type', 'extern_name', - 'email', 'api_key', 'api_keys', 'last_login', - 'full_name', 'active', 'password', 'emails', - 'inherit_default_permissions', 'created_by', 'created_on') - _verify_kwargs(expected_parameters, kwargs) - - return 0 -CREATE_USER_HOOK = _cruserhook - - -# ============================================================================= -# POST DELETE REPOSITORY HOOK -# ============================================================================= -# this function will be executed after each repository deletion - - -def _dlrepohook(*args, **kwargs): - """ - Post delete repository HOOK - kwargs available: - :param repo_name: - :param repo_type: - :param description: - :param private: - :param created_on: - :param enable_downloads: - :param repo_id: - :param user_id: - :param enable_statistics: - :param clone_uri: - :param fork_id: - :param group_id: - :param deleted_by: - :param deleted_on: - """ - - expected_parameters = ( - 'repo_name', 'repo_type', 'description', 'private', 'created_on', - 'enable_downloads', 'repo_id', 'user_id', 'enable_statistics', - 'clone_uri', 'fork_id', 'group_id', 'deleted_by', 'deleted_on') - _verify_kwargs(expected_parameters, kwargs) - - return 0 -DELETE_REPO_HOOK = _dlrepohook - - -# ============================================================================= -# POST DELETE USER HOOK -# ============================================================================= -# this function will be executed after each user is deleted - - -def _dluserhook(*args, **kwargs): - """ - Post delete user HOOK - kwargs available: - :param username: - :param full_name_or_username: - :param full_contact: - :param user_id: - :param name: - :param firstname: - :param short_contact: - :param admin: - :param lastname: - :param ip_addresses: - :param ldap_dn: - :param email: - :param api_key: - :param last_login: - :param full_name: - :param active: - :param password: - :param emails: - :param inherit_default_permissions: - :param deleted_by: - """ - - expected_parameters = ( - 'username', 'full_name_or_username', 'full_contact', 'user_id', - 'name', 'firstname', 'short_contact', 'admin', 'lastname', - 'ip_addresses', - # TODO: johbo: Check what's the status with the ldap_dn parameter - # 'ldap_dn', - 'email', 'api_key', 'last_login', - 'full_name', 'active', 'password', 'emails', - 'inherit_default_permissions', 'deleted_by') - _verify_kwargs(expected_parameters, kwargs) - - return 0 -DELETE_USER_HOOK = _dluserhook - - -# ============================================================================= -# PRE PUSH HOOK -# ============================================================================= - - -# this function will be executed after each push it's executed after the -# build-in hook that RhodeCode uses for logging pushes -@log_call('pre_push') -def _prepushhook(*args, **kwargs): - """ - Pre push hook - kwargs available: - - :param server_url: url of instance that triggered this hook - :param config: path to .ini config used - :param scm: type of VS 'git' or 'hg' - :param username: name of user who pushed - :param ip: ip of who pushed - :param action: push - :param repository: repository name - :param repo_store_path: full path to where repositories are stored - :param commit_ids: list of pushed commit ids - """ - - expected_parameters = ( - 'server_url', 'config', 'scm', 'username', 'ip', 'action', - 'repository', 'repo_store_path', 'commit_ids') - _verify_kwargs(expected_parameters, kwargs) - - return 0 -PRE_PUSH_HOOK = _prepushhook - - -# ============================================================================= -# POST PUSH HOOK -# ============================================================================= - - -# this function will be executed after each push it's executed after the -# build-in hook that RhodeCode uses for logging pushes -@log_call('post_push') -def _pushhook(*args, **kwargs): - """ - Post push hook - kwargs available: - - :param server_url: url of instance that triggered this hook - :param config: path to .ini config used - :param scm: type of VS 'git' or 'hg' - :param username: name of user who pushed - :param ip: ip of who pushed - :param action: push - :param repository: repository name - :param repo_store_path: full path to where repositories are stored - :param pushed_revs: list of pushed revisions - """ - - expected_parameters = ( - 'server_url', 'config', 'scm', 'username', 'ip', 'action', - 'repository', 'repo_store_path', 'pushed_revs') - _verify_kwargs(expected_parameters, kwargs) - - return 0 -PUSH_HOOK = _pushhook - - -# ============================================================================= -# PRE PULL HOOK -# ============================================================================= - -# this function will be executed after each push it's executed after the -# build-in hook that RhodeCode uses for logging pulls -def _prepullhook(*args, **kwargs): - """ - Post pull hook - kwargs available:: - - :param server_url: url of instance that triggered this hook - :param config: path to .ini config used - :param scm: type of VS 'git' or 'hg' - :param username: name of user who pulled - :param ip: ip of who pulled - :param action: pull - :param repository: repository name - """ - - expected_parameters = ( - 'server_url', 'config', 'scm', 'username', 'ip', 'action', - 'repository') - _verify_kwargs(expected_parameters, kwargs) - - return 0 -PRE_PULL_HOOK = _prepullhook - - - -# ============================================================================= -# POST PULL HOOK -# ============================================================================= - -# this function will be executed after each push it's executed after the -# build-in hook that RhodeCode uses for logging pulls -def _pullhook(*args, **kwargs): - """ - Post pull hook - kwargs available:: - - :param server_url: url of instance that triggered this hook - :param config: path to .ini config used - :param scm: type of VS 'git' or 'hg' - :param username: name of user who pulled - :param ip: ip of who pulled - :param action: pull - :param repository: repository name - """ - - expected_parameters = ( - 'server_url', 'config', 'scm', 'username', 'ip', 'action', - 'repository') - _verify_kwargs(expected_parameters, kwargs) - - return 0 -PULL_HOOK = _pullhook - - -# ============================================================================= -# PULL REQUEST RELATED HOOKS -# ============================================================================= - -def _create_pull_request_hook(*args, **kwargs): - expected_parameters = ( - 'server_url', 'config', 'scm', 'username', 'ip', 'action', - 'repository', 'pull_request_id', 'url', 'title', 'description', - 'status', 'created_on', 'updated_on', 'commit_ids', 'review_status', - 'mergeable', 'source', 'target', 'author', 'reviewers') - _verify_kwargs(expected_parameters, kwargs) - return 0 -CREATE_PULL_REQUEST = _create_pull_request_hook - - -def _merge_pull_request_hook(*args, **kwargs): - expected_parameters = ( - 'server_url', 'config', 'scm', 'username', 'ip', 'action', - 'repository', 'pull_request_id', 'url', 'title', 'description', - 'status', 'created_on', 'updated_on', 'commit_ids', 'review_status', - 'mergeable', 'source', 'target', 'author', 'reviewers') - _verify_kwargs(expected_parameters, kwargs) - return 0 -MERGE_PULL_REQUEST = _merge_pull_request_hook - - -def _close_pull_request_hook(*args, **kwargs): - expected_parameters = ( - 'server_url', 'config', 'scm', 'username', 'ip', 'action', - 'repository', 'pull_request_id', 'url', 'title', 'description', - 'status', 'created_on', 'updated_on', 'commit_ids', 'review_status', - 'mergeable', 'source', 'target', 'author', 'reviewers') - _verify_kwargs(expected_parameters, kwargs) - return 0 -CLOSE_PULL_REQUEST = _close_pull_request_hook - - -def _review_pull_request_hook(*args, **kwargs): - expected_parameters = ( - 'server_url', 'config', 'scm', 'username', 'ip', 'action', - 'repository', 'pull_request_id', 'url', 'title', 'description', - 'status', 'created_on', 'updated_on', 'commit_ids', 'review_status', - 'mergeable', 'source', 'target', 'author', 'reviewers') - _verify_kwargs(expected_parameters, kwargs) - return 0 -REVIEW_PULL_REQUEST = _review_pull_request_hook - - -def _update_pull_request_hook(*args, **kwargs): - expected_parameters = ( - 'server_url', 'config', 'scm', 'username', 'ip', 'action', - 'repository', 'pull_request_id', 'url', 'title', 'description', - 'status', 'created_on', 'updated_on', 'commit_ids', 'review_status', - 'mergeable', 'source', 'target', 'author', 'reviewers') - _verify_kwargs(expected_parameters, kwargs) - return 0 -UPDATE_PULL_REQUEST = _update_pull_request_hook - - -def _verify_kwargs(expected_parameters, kwargs): - """ - Verify that exactly `expected_parameters` are passed in as `kwargs`. - """ - expected_parameters = set(expected_parameters) - kwargs_keys = set(kwargs.keys()) - if kwargs_keys != expected_parameters: - missing_kwargs = expected_parameters - kwargs_keys - unexpected_kwargs = kwargs_keys - expected_parameters - raise AssertionError( - "rcextensions: Missing parameters: %r, unexpected parameters: %s" % - (missing_kwargs, unexpected_kwargs)) diff --git a/rhodecode/tests/plugin.py b/rhodecode/tests/plugin.py --- a/rhodecode/tests/plugin.py +++ b/rhodecode/tests/plugin.py @@ -154,10 +154,11 @@ def activate_example_rcextensions(reques """ Patch in an example rcextensions module which verifies passed in kwargs. """ - from rhodecode.tests.other import example_rcextensions + from rhodecode.config import rcextensions old_extensions = rhodecode.EXTENSIONS - rhodecode.EXTENSIONS = example_rcextensions + rhodecode.EXTENSIONS = rcextensions + rhodecode.EXTENSIONS.calls = collections.defaultdict(list) @request.addfinalizer def cleanup():