events: ensure stable execution of integrations

rhodecode/model/integration.py

@@ -1,221 +1,241 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 Model for integrations
 """
 
 
 import logging
 
 from sqlalchemy import or_, and_
 
 import rhodecode
 from rhodecode import events
 from rhodecode.integrations.types.base import EEIntegration
 from rhodecode.lib.caching_query import FromCache
 from rhodecode.model import BaseModel
-from rhodecode.model.db import Integration, Repository, RepoGroup, true, false
+from rhodecode.model.db import Integration, Repository, RepoGroup, true, false, case
 from rhodecode.integrations import integration_type_registry
 
 log = logging.getLogger(__name__)
 
 
 class IntegrationModel(BaseModel):
 
     cls = Integration
 
     def __get_integration(self, integration):
         if isinstance(integration, Integration):
             return integration
         elif isinstance(integration, (int, long)):
             return self.sa.query(Integration).get(integration)
         else:
             if integration:
                 raise Exception('integration must be int, long or Instance'
                                 ' of Integration got %s' % type(integration))
 
     def create(self, IntegrationType, name, enabled, repo, repo_group,
         child_repos_only, settings):
         """ Create an IntegrationType integration """
         integration = Integration()
         integration.integration_type = IntegrationType.key
         self.sa.add(integration)
         self.update_integration(integration, name, enabled, repo, repo_group,
                                 child_repos_only, settings)
         self.sa.commit()
         return integration
 
     def update_integration(self, integration, name, enabled, repo, repo_group,
         child_repos_only, settings):
         integration = self.__get_integration(integration)
 
         integration.repo = repo
         integration.repo_group = repo_group
         integration.child_repos_only = child_repos_only
         integration.name = name
         integration.enabled = enabled
         integration.settings = settings
 
         return integration
 
     def delete(self, integration):
         integration = self.__get_integration(integration)
         if integration:
             self.sa.delete(integration)
             return True
         return False
 
     def get_integration_handler(self, integration):
         TypeClass = integration_type_registry.get(integration.integration_type)
         if not TypeClass:
             log.error('No class could be found for integration type: {}'.format(
                 integration.integration_type))
             return None
         elif isinstance(TypeClass, EEIntegration) or issubclass(TypeClass, EEIntegration):
             log.error('EE integration cannot be '
                       'executed for integration type: {}'.format(
                 integration.integration_type))
             return None
 
         return TypeClass(integration.settings)
 
     def send_event(self, integration, event):
         """ Send an event to an integration """
         handler = self.get_integration_handler(integration)
         if handler:
             log.debug(
                 'events: sending event %s on integration %s using handler %s',
                 event, integration, handler)
             handler.send_event(event)
 
     def get_integrations(self, scope, IntegrationType=None):
         """
         Return integrations for a scope, which must be one of:
 
         'all' - every integration, global/repogroup/repo
         'global' - global integrations only
         <Repository> instance - integrations for this repo only
         <RepoGroup> instance - integrations for this repogroup only
         """
 
         if isinstance(scope, Repository):
             query = self.sa.query(Integration).filter(
                 Integration.repo == scope)
         elif isinstance(scope, RepoGroup):
             query = self.sa.query(Integration).filter(
                 Integration.repo_group == scope)
         elif scope == 'global':
             # global integrations
             query = self.sa.query(Integration).filter(
                 and_(Integration.repo_id == None, Integration.repo_group_id == None)
             )
         elif scope == 'root-repos':
             query = self.sa.query(Integration).filter(
                 and_(Integration.repo_id == None,
                      Integration.repo_group_id == None,
                      Integration.child_repos_only == true())
             )
         elif scope == 'all':
             query = self.sa.query(Integration)
         else:
             raise Exception(
                 "invalid `scope`, must be one of: "
                 "['global', 'all', <Repository>, <RepoGroup>]")
 
         if IntegrationType is not None:
             query = query.filter(
                 Integration.integration_type==IntegrationType.key)
 
         result = []
         for integration in query.all():
             IntType = integration_type_registry.get(integration.integration_type)
             result.append((IntType, integration))
         return result
 
     def get_for_event(self, event, cache=False):
         """
         Get integrations that match an event
         """
+        # base query
         query = self.sa.query(
             Integration
         ).filter(
             Integration.enabled == true()
         )
 
         global_integrations_filter = and_(
             Integration.repo_id == None,
             Integration.repo_group_id == None,
-            Integration.child_repos_only == False,
+            Integration.child_repos_only == false(),
         )
 
         if isinstance(event, events.RepoEvent):
             root_repos_integrations_filter = and_(
                 Integration.repo_id == None,
                 Integration.repo_group_id == None,
                 Integration.child_repos_only == true(),
             )
 
             clauses = [
                 global_integrations_filter,
             ]
+            cases = [
+                (global_integrations_filter, 1),
+                (root_repos_integrations_filter, 2),
+            ]
 
-            # repo integrations
-            if event.repo.repo_id:  # pre create events dont have a repo_id yet
-                clauses.append(
-                    Integration.repo_id == event.repo.repo_id
+            # repo group integrations
+            if event.repo.group:
+                # repo group with only root level repos
+                group_child_repos_filter = and_(
+                    Integration.repo_group_id == event.repo.group.group_id,
+                    Integration.child_repos_only == true()
                 )
 
-            if event.repo.group:
-                clauses.append(
-                    and_(
-                        Integration.repo_group_id == event.repo.group.group_id,
-                        Integration.child_repos_only == true()
-                    )
+                clauses.append(group_child_repos_filter)
+                cases.append(
+                    (group_child_repos_filter, 3),
                 )
+
                 # repo group cascade to kids
-                clauses.append(
-                    and_(
-                        Integration.repo_group_id.in_(
-                            [group.group_id for group in
-                             event.repo.groups_with_parents]
-                        ),
-                        Integration.child_repos_only == false()
-                    )
+                group_recursive_repos_filter = and_(
+                    Integration.repo_group_id.in_(
+                        [group.group_id for group in event.repo.groups_with_parents]
+                    ),
+                    Integration.child_repos_only == false()
+                )
+                clauses.append(group_recursive_repos_filter)
+                cases.append(
+                    (group_recursive_repos_filter, 4),
                 )
 
             if not event.repo.group:  # root repo
                 clauses.append(root_repos_integrations_filter)
 
+            # repo integrations
+            if event.repo.repo_id:  # pre create events dont have a repo_id yet
+                specific_repo_filter = Integration.repo_id == event.repo.repo_id
+                clauses.append(specific_repo_filter)
+                cases.append(
+                    (specific_repo_filter, 5),
+                )
+
+            order_by_criterion = case(cases)
+
             query = query.filter(or_(*clauses))
+            query = query.order_by(order_by_criterion)
 
             if cache:
                 cache_key = "get_enabled_repo_integrations_%i" % event.repo.repo_id
                 query = query.options(
                     FromCache("sql_cache_short", cache_key))
         else:  # only global integrations
+            order_by_criterion = Integration.integration_id
+
             query = query.filter(global_integrations_filter)
+            query = query.order_by(order_by_criterion)
             if cache:
                 query = query.options(
                     FromCache("sql_cache_short", "get_enabled_global_integrations"))
 
         result = query.all()
         return result

rhodecode/tests/integrations/test_integration.py

@@ -1,216 +1,216 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import time
 import pytest
 
 from rhodecode import events
 from rhodecode.tests.fixture import Fixture
 from rhodecode.model.db import Session, Integration
 from rhodecode.model.integration import IntegrationModel
 
 
 class TestDeleteScopesDeletesIntegrations(object):
     def test_delete_repo_with_integration_deletes_integration(
             self, repo_integration_stub):
 
         Session().delete(repo_integration_stub.repo)
         Session().commit()
         Session().expire_all()
         integration = Integration.get(repo_integration_stub.integration_id)
         assert integration is None
 
     def test_delete_repo_group_with_integration_deletes_integration(
             self, repogroup_integration_stub):
 
         Session().delete(repogroup_integration_stub.repo_group)
         Session().commit()
         Session().expire_all()
         integration = Integration.get(repogroup_integration_stub.integration_id)
         assert integration is None
 
 
 @pytest.fixture
 def integration_repos(request, StubIntegrationType, stub_integration_settings):
     """
     Create repositories and integrations for testing, and destroy them after
 
     Structure:
         root_repo
         parent_group/
             parent_repo
             child_group/
                 child_repo
         other_group/
             other_repo
     """
     fixture = Fixture()
 
 
     parent_group_id = 'int_test_parent_group_%s' % time.time()
     parent_group = fixture.create_repo_group(parent_group_id)
 
     other_group_id = 'int_test_other_group_%s' % time.time()
     other_group = fixture.create_repo_group(other_group_id)
 
     child_group_id = (
         parent_group_id + '/' + 'int_test_child_group_%s' % time.time())
     child_group = fixture.create_repo_group(child_group_id)
 
     parent_repo_id = 'int_test_parent_repo_%s' % time.time()
     parent_repo = fixture.create_repo(parent_repo_id, repo_group=parent_group)
 
     child_repo_id = 'int_test_child_repo_%s' % time.time()
     child_repo = fixture.create_repo(child_repo_id, repo_group=child_group)
 
     other_repo_id = 'int_test_other_repo_%s' % time.time()
     other_repo = fixture.create_repo(other_repo_id, repo_group=other_group)
 
     root_repo_id = 'int_test_repo_root_%s' % time.time()
     root_repo = fixture.create_repo(root_repo_id)
 
     integrations = {}
     for name, repo, repo_group, child_repos_only in [
             ('global', None, None, None),
             ('root_repos', None, None, True),
             ('parent_repo', parent_repo, None, None),
             ('child_repo', child_repo, None, None),
             ('other_repo', other_repo, None, None),
             ('root_repo', root_repo, None, None),
             ('parent_group', None, parent_group, True),
             ('parent_group_recursive', None, parent_group, False),
             ('child_group', None, child_group, True),
             ('child_group_recursive', None, child_group, False),
             ('other_group', None, other_group, True),
             ('other_group_recursive', None, other_group, False),
         ]:
         integrations[name] = IntegrationModel().create(
             StubIntegrationType, settings=stub_integration_settings,
             enabled=True, name='test %s integration' % name,
             repo=repo, repo_group=repo_group, child_repos_only=child_repos_only)
 
     Session().commit()
 
     def _cleanup():
         for integration in integrations.values():
             Session.delete(integration)
 
         fixture.destroy_repo(root_repo)
         fixture.destroy_repo(child_repo)
         fixture.destroy_repo(parent_repo)
         fixture.destroy_repo(other_repo)
         fixture.destroy_repo_group(child_group)
         fixture.destroy_repo_group(parent_group)
         fixture.destroy_repo_group(other_group)
 
     request.addfinalizer(_cleanup)
 
     return {
         'integrations': integrations,
         'repos': {
             'root_repo': root_repo,
             'other_repo': other_repo,
             'parent_repo': parent_repo,
-                'child_repo': child_repo,
+            'child_repo': child_repo,
         }
     }
 
 
 def test_enabled_integration_repo_scopes(integration_repos):
     integrations = integration_repos['integrations']
     repos = integration_repos['repos']
 
     triggered_integrations = IntegrationModel().get_for_event(
         events.RepoEvent(repos['root_repo']))
 
     assert triggered_integrations == [
         integrations['global'],
         integrations['root_repos'],
         integrations['root_repo'],
     ]
 
     triggered_integrations = IntegrationModel().get_for_event(
         events.RepoEvent(repos['other_repo']))
 
     assert triggered_integrations == [
         integrations['global'],
-        integrations['other_repo'],
         integrations['other_group'],
         integrations['other_group_recursive'],
+        integrations['other_repo'],
     ]
 
     triggered_integrations = IntegrationModel().get_for_event(
         events.RepoEvent(repos['parent_repo']))
 
     assert triggered_integrations == [
         integrations['global'],
-        integrations['parent_repo'],
         integrations['parent_group'],
         integrations['parent_group_recursive'],
+        integrations['parent_repo'],
     ]
 
     triggered_integrations = IntegrationModel().get_for_event(
         events.RepoEvent(repos['child_repo']))
 
     assert triggered_integrations == [
         integrations['global'],
-        integrations['child_repo'],
+        integrations['child_group'],
         integrations['parent_group_recursive'],
-        integrations['child_group'],
         integrations['child_group_recursive'],
+        integrations['child_repo'],
     ]
 
 
 def test_disabled_integration_repo_scopes(integration_repos):
     integrations = integration_repos['integrations']
     repos = integration_repos['repos']
 
     for integration in integrations.values():
         integration.enabled = False
     Session().commit()
 
     triggered_integrations = IntegrationModel().get_for_event(
         events.RepoEvent(repos['root_repo']))
 
     assert triggered_integrations == []
 
     triggered_integrations = IntegrationModel().get_for_event(
         events.RepoEvent(repos['parent_repo']))
 
     assert triggered_integrations == []
 
     triggered_integrations = IntegrationModel().get_for_event(
         events.RepoEvent(repos['child_repo']))
 
     assert triggered_integrations == []
 
     triggered_integrations = IntegrationModel().get_for_event(
         events.RepoEvent(repos['other_repo']))
 
     assert triggered_integrations == []
 
 
 def test_enabled_non_repo_integrations(integration_repos):
     integrations = integration_repos['integrations']
 
     triggered_integrations = IntegrationModel().get_for_event(
         events.UserPreCreate({}))
 
     assert triggered_integrations == [integrations['global']]