black: reformat source

backend/setup.py

@@ -1,81 +1,89 b''
 import os
 import re
 
 from setuptools import setup, find_packages
 
 here = os.path.abspath(os.path.dirname(__file__))
-README = open(os.path.join(here, 'README.rst')).read()
-CHANGES = open(os.path.join(here, 'CHANGELOG.rst')).read()
+README = open(os.path.join(here, "README.rst")).read()
+CHANGES = open(os.path.join(here, "CHANGELOG.rst")).read()
 
-REQUIREMENTS = open(os.path.join(here, 'requirements.txt')).readlines()
+REQUIREMENTS = open(os.path.join(here, "requirements.txt")).readlines()
 
-compiled = re.compile('([^=><]*).*')
+compiled = re.compile("([^=><]*).*")
 
 
 def parse_req(req):
     return compiled.search(req).group(1).strip()
 
 
 requires = [_f for _f in map(parse_req, REQUIREMENTS) if _f]
 
 
 def _get_meta_var(name, data, callback_handler=None):
     import re
-    matches = re.compile(r'(?:%s)\s*=\s*(.*)' % name).search(data)
+
+    matches = re.compile(r"(?:%s)\s*=\s*(.*)" % name).search(data)
     if matches:
         if not callable(callback_handler):
             callback_handler = lambda v: v
 
         return callback_handler(eval(matches.groups()[0]))
 
 
-with open(os.path.join(here, 'src', 'appenlight', '__init__.py'), 'r') as _meta:
+with open(os.path.join(here, "src", "appenlight", "__init__.py"), "r") as _meta:
     _metadata = _meta.read()
 
-with open(os.path.join(here, 'VERSION'), 'r') as _meta_version:
+with open(os.path.join(here, "VERSION"), "r") as _meta_version:
     __version__ = _meta_version.read().strip()
 
-__license__ = _get_meta_var('__license__', _metadata)
-__author__ = _get_meta_var('__author__', _metadata)
-__url__ = _get_meta_var('__url__', _metadata)
-
-found_packages = find_packages('src')
-found_packages.append('appenlight.migrations.versions')
-setup(name='appenlight',
-      description='appenlight',
-      long_description=README + '\n\n' + CHANGES,
-      classifiers=[
-          "Programming Language :: Python",
-          "Framework :: Pylons",
-          "Topic :: Internet :: WWW/HTTP",
-          "Topic :: Internet :: WWW/HTTP :: WSGI :: Application",
-      ],
-      version=__version__,
-      license=__license__,
-      author=__author__,
-      url=__url__,
-      keywords='web wsgi bfg pylons pyramid',
-      package_dir={'': 'src'},
-      packages=found_packages,
-      include_package_data=True,
-      zip_safe=False,
-      test_suite='appenlight',
-      install_requires=requires,
-      extras_require={
-          "dev": ["coverage", "pytest", "pyramid", "tox", "mock", "pytest-mock", "webtest"],
-          "lint": ["black"],
-      },
-      entry_points={
-          'paste.app_factory': [
-              'main = appenlight:main'
-          ],
-          'console_scripts': [
-              'appenlight-cleanup = appenlight.scripts.cleanup:main',
-              'appenlight-initializedb = appenlight.scripts.initialize_db:main',
-              'appenlight-migratedb = appenlight.scripts.migratedb:main',
-              'appenlight-reindex-elasticsearch = appenlight.scripts.reindex_elasticsearch:main',
-              'appenlight-static = appenlight.scripts.static:main',
-              'appenlight-make-config = appenlight.scripts.make_config:main',
-          ]
-      }
-      )
+__license__ = _get_meta_var("__license__", _metadata)
+__author__ = _get_meta_var("__author__", _metadata)
+__url__ = _get_meta_var("__url__", _metadata)
+
+found_packages = find_packages("src")
+found_packages.append("appenlight.migrations.versions")
+setup(
+    name="appenlight",
+    description="appenlight",
+    long_description=README + "\n\n" + CHANGES,
+    classifiers=[
+        "Programming Language :: Python",
+        "Framework :: Pylons",
+        "Topic :: Internet :: WWW/HTTP",
+        "Topic :: Internet :: WWW/HTTP :: WSGI :: Application",
+    ],
+    version=__version__,
+    license=__license__,
+    author=__author__,
+    url=__url__,
+    keywords="web wsgi bfg pylons pyramid",
+    package_dir={"": "src"},
+    packages=found_packages,
+    include_package_data=True,
+    zip_safe=False,
+    test_suite="appenlight",
+    install_requires=requires,
+    extras_require={
+        "dev": [
+            "coverage",
+            "pytest",
+            "pyramid",
+            "tox",
+            "mock",
+            "pytest-mock",
+            "webtest",
+        ],
+        "lint": ["black"],
+    },
+    entry_points={
+        "paste.app_factory": ["main = appenlight:main"],
+        "console_scripts": [
+            "appenlight-cleanup = appenlight.scripts.cleanup:main",
+            "appenlight-initializedb = appenlight.scripts.initialize_db:main",
+            "appenlight-migratedb = appenlight.scripts.migratedb:main",
+            "appenlight-reindex-elasticsearch = appenlight.scripts.reindex_elasticsearch:main",
+            "appenlight-static = appenlight.scripts.static:main",
+            "appenlight-make-config = appenlight.scripts.make_config:main",
+        ],
+    },
+)

backend/src/appenlight/__init__.py

@@ -1,225 +1,254 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import datetime
 import logging
 from elasticsearch import Elasticsearch
 import redis
 import os
 import pkg_resources
 from pkg_resources import iter_entry_points
 
 import appenlight.lib.jinja2_filters as jinja2_filters
 import appenlight.lib.encryption as encryption
 
 from pyramid.config import PHASE3_CONFIG
 from pyramid.authentication import AuthTktAuthenticationPolicy
 from pyramid.authorization import ACLAuthorizationPolicy
 from pyramid_mailer.interfaces import IMailer
 from pyramid.renderers import JSON
 from pyramid_redis_sessions import session_factory_from_settings
 from pyramid.settings import asbool, aslist
 from pyramid.security import AllPermissionsList
 from pyramid_authstack import AuthenticationStackPolicy
 from redlock import Redlock
 from sqlalchemy import engine_from_config
 
 from appenlight.celery import configure_celery
-from appenlight.lib.configurator import (CythonCompatConfigurator,
-                                         register_appenlight_plugin)
+from appenlight.lib.configurator import (
+    CythonCompatConfigurator,
+    register_appenlight_plugin,
+)
 from appenlight.lib import cache_regions
 from appenlight.lib.ext_json import json
 from appenlight.security import groupfinder, AuthTokenAuthenticationPolicy
 
-__license__ = 'Apache 2.0'
-__author__ = 'RhodeCode GmbH'
-__url__ = 'http://rhodecode.com'
+__license__ = "Apache 2.0"
+__author__ = "RhodeCode GmbH"
+__url__ = "http://rhodecode.com"
 __version__ = pkg_resources.get_distribution("appenlight").parsed_version
 
 json_renderer = JSON(serializer=json.dumps, indent=4)
 
 log = logging.getLogger(__name__)
 
 
 def datetime_adapter(obj, request):
     return obj.isoformat()
 
 
 def all_permissions_adapter(obj, request):
-    return '__all_permissions__'
+    return "__all_permissions__"
 
 
 json_renderer.add_adapter(datetime.datetime, datetime_adapter)
 json_renderer.add_adapter(AllPermissionsList, all_permissions_adapter)
 
 
 def main(global_config, **settings):
     """ This function returns a Pyramid WSGI application.
     """
     auth_tkt_policy = AuthTktAuthenticationPolicy(
-        settings['authtkt.secret'],
-        hashalg='sha512',
+        settings["authtkt.secret"],
+        hashalg="sha512",
         callback=groupfinder,
         max_age=2592000,
-        secure=asbool(settings.get('authtkt.secure', 'false')))
-    auth_token_policy = AuthTokenAuthenticationPolicy(
-        callback=groupfinder
+        secure=asbool(settings.get("authtkt.secure", "false")),
     )
+    auth_token_policy = AuthTokenAuthenticationPolicy(callback=groupfinder)
     authorization_policy = ACLAuthorizationPolicy()
     authentication_policy = AuthenticationStackPolicy()
-    authentication_policy.add_policy('auth_tkt', auth_tkt_policy)
-    authentication_policy.add_policy('auth_token', auth_token_policy)
+    authentication_policy.add_policy("auth_tkt", auth_tkt_policy)
+    authentication_policy.add_policy("auth_token", auth_token_policy)
     # set crypto key
-    encryption.ENCRYPTION_SECRET = settings.get('encryption_secret')
+    encryption.ENCRYPTION_SECRET = settings.get("encryption_secret")
     # import this later so encyption key can be monkeypatched
     from appenlight.models import DBSession, register_datastores
 
     # registration
-    settings['appenlight.disable_registration'] = asbool(
-        settings.get('appenlight.disable_registration'))
+    settings["appenlight.disable_registration"] = asbool(
+        settings.get("appenlight.disable_registration")
+    )
 
     # update config with cometd info
-    settings['cometd_servers'] = {'server': settings['cometd.server'],
-                                  'secret': settings['cometd.secret']}
+    settings["cometd_servers"] = {
+        "server": settings["cometd.server"],
+        "secret": settings["cometd.secret"],
+    }
 
     # Create the Pyramid Configurator.
-    settings['_mail_url'] = settings['mailing.app_url']
+    settings["_mail_url"] = settings["mailing.app_url"]
     config = CythonCompatConfigurator(
         settings=settings,
         authentication_policy=authentication_policy,
         authorization_policy=authorization_policy,
-        root_factory='appenlight.security.RootFactory',
-        default_permission='view')
+        root_factory="appenlight.security.RootFactory",
+        default_permission="view",
+    )
     # custom registry variables
 
     # resource type information
-    config.registry.resource_types = ['resource', 'application']
+    config.registry.resource_types = ["resource", "application"]
     # plugin information
     config.registry.appenlight_plugins = {}
 
-    config.set_default_csrf_options(require_csrf=True, header='X-XSRF-TOKEN')
-    config.add_view_deriver('appenlight.predicates.csrf_view',
-                            name='csrf_view')
+    config.set_default_csrf_options(require_csrf=True, header="X-XSRF-TOKEN")
+    config.add_view_deriver("appenlight.predicates.csrf_view", name="csrf_view")
 
     # later, when config is available
-    dogpile_config = {'url': settings['redis.url'],
-                      "redis_expiration_time": 86400,
-                      "redis_distributed_lock": True}
+    dogpile_config = {
+        "url": settings["redis.url"],
+        "redis_expiration_time": 86400,
+        "redis_distributed_lock": True,
+    }
     cache_regions.regions = cache_regions.CacheRegions(dogpile_config)
     config.registry.cache_regions = cache_regions.regions
-    engine = engine_from_config(settings, 'sqlalchemy.',
-                                json_serializer=json.dumps)
+    engine = engine_from_config(settings, "sqlalchemy.", json_serializer=json.dumps)
     DBSession.configure(bind=engine)
 
     # json rederer that serializes datetime
-    config.add_renderer('json', json_renderer)
-    config.add_request_method('appenlight.lib.request.es_conn', 'es_conn', property=True)
-    config.add_request_method('appenlight.lib.request.get_user', 'user',
-                                reify=True, property=True)
-    config.add_request_method('appenlight.lib.request.get_csrf_token',
-                                'csrf_token', reify=True, property=True)
-    config.add_request_method('appenlight.lib.request.safe_json_body',
-                                'safe_json_body', reify=True, property=True)
-    config.add_request_method('appenlight.lib.request.unsafe_json_body',
-                                'unsafe_json_body', reify=True, property=True)
-    config.add_request_method('appenlight.lib.request.add_flash_to_headers',
-                              'add_flash_to_headers')
-    config.add_request_method('appenlight.lib.request.get_authomatic',
-                              'authomatic', reify=True)
-
-    config.include('pyramid_redis_sessions')
-    config.include('pyramid_tm')
-    config.include('pyramid_jinja2')
-    config.include('pyramid_mailer')
-    config.include('appenlight_client.ext.pyramid_tween')
-    config.include('ziggurat_foundations.ext.pyramid.sign_in')
-    es_server_list = aslist(settings['elasticsearch.nodes'])
-    redis_url = settings['redis.url']
-    log.warning('Elasticsearch server list: {}'.format(es_server_list))
-    log.warning('Redis server: {}'.format(redis_url))
+    config.add_renderer("json", json_renderer)
+    config.add_request_method(
+        "appenlight.lib.request.es_conn", "es_conn", property=True
+    )
+    config.add_request_method(
+        "appenlight.lib.request.get_user", "user", reify=True, property=True
+    )
+    config.add_request_method(
+        "appenlight.lib.request.get_csrf_token", "csrf_token", reify=True, property=True
+    )
+    config.add_request_method(
+        "appenlight.lib.request.safe_json_body",
+        "safe_json_body",
+        reify=True,
+        property=True,
+    )
+    config.add_request_method(
+        "appenlight.lib.request.unsafe_json_body",
+        "unsafe_json_body",
+        reify=True,
+        property=True,
+    )
+    config.add_request_method(
+        "appenlight.lib.request.add_flash_to_headers", "add_flash_to_headers"
+    )
+    config.add_request_method(
+        "appenlight.lib.request.get_authomatic", "authomatic", reify=True
+    )
+
+    config.include("pyramid_redis_sessions")
+    config.include("pyramid_tm")
+    config.include("pyramid_jinja2")
+    config.include("pyramid_mailer")
+    config.include("appenlight_client.ext.pyramid_tween")
+    config.include("ziggurat_foundations.ext.pyramid.sign_in")
+    es_server_list = aslist(settings["elasticsearch.nodes"])
+    redis_url = settings["redis.url"]
+    log.warning("Elasticsearch server list: {}".format(es_server_list))
+    log.warning("Redis server: {}".format(redis_url))
     config.registry.es_conn = Elasticsearch(es_server_list)
     config.registry.redis_conn = redis.StrictRedis.from_url(redis_url)
 
-    config.registry.redis_lockmgr = Redlock([settings['redis.redlock.url'], ],
-                                            retry_count=0, retry_delay=0)
+    config.registry.redis_lockmgr = Redlock(
+        [settings["redis.redlock.url"]], retry_count=0, retry_delay=0
+    )
     # mailer bw compat
     config.registry.mailer = config.registry.getUtility(IMailer)
 
     # Configure sessions
     session_factory = session_factory_from_settings(settings)
     config.set_session_factory(session_factory)
 
     # Configure renderers and event subscribers
-    config.add_jinja2_extension('jinja2.ext.loopcontrols')
-    config.add_jinja2_search_path('appenlight:templates')
+    config.add_jinja2_extension("jinja2.ext.loopcontrols")
+    config.add_jinja2_search_path("appenlight:templates")
     # event subscribers
-    config.add_subscriber("appenlight.subscribers.application_created",
-                          "pyramid.events.ApplicationCreated")
-    config.add_subscriber("appenlight.subscribers.add_renderer_globals",
-                          "pyramid.events.BeforeRender")
-    config.add_subscriber('appenlight.subscribers.new_request',
-                          'pyramid.events.NewRequest')
-    config.add_view_predicate('context_type_class',
-                              'appenlight.predicates.contextTypeClass')
-
-    register_datastores(es_conn=config.registry.es_conn,
-                        redis_conn=config.registry.redis_conn,
-                        redis_lockmgr=config.registry.redis_lockmgr)
+    config.add_subscriber(
+        "appenlight.subscribers.application_created",
+        "pyramid.events.ApplicationCreated",
+    )
+    config.add_subscriber(
+        "appenlight.subscribers.add_renderer_globals", "pyramid.events.BeforeRender"
+    )
+    config.add_subscriber(
+        "appenlight.subscribers.new_request", "pyramid.events.NewRequest"
+    )
+    config.add_view_predicate(
+        "context_type_class", "appenlight.predicates.contextTypeClass"
+    )
+
+    register_datastores(
+        es_conn=config.registry.es_conn,
+        redis_conn=config.registry.redis_conn,
+        redis_lockmgr=config.registry.redis_lockmgr,
+    )
 
     # base stuff and scan
 
     # need to ensure webassets exists otherwise config.override_asset()
     # throws exception
-    if not os.path.exists(settings['webassets.dir']):
-        os.mkdir(settings['webassets.dir'])
-    config.add_static_view(path='appenlight:webassets',
-                           name='static', cache_max_age=3600)
-    config.override_asset(to_override='appenlight:webassets/',
-                          override_with=settings['webassets.dir'])
-
-    config.include('appenlight.views')
-    config.include('appenlight.views.admin')
-    config.scan(ignore=['appenlight.migrations', 'appenlight.scripts',
-                        'appenlight.tests'])
-
-    config.add_directive('register_appenlight_plugin',
-                         register_appenlight_plugin)
-
-    for entry_point in iter_entry_points(group='appenlight.plugins'):
+    if not os.path.exists(settings["webassets.dir"]):
+        os.mkdir(settings["webassets.dir"])
+    config.add_static_view(
+        path="appenlight:webassets", name="static", cache_max_age=3600
+    )
+    config.override_asset(
+        to_override="appenlight:webassets/", override_with=settings["webassets.dir"]
+    )
+
+    config.include("appenlight.views")
+    config.include("appenlight.views.admin")
+    config.scan(
+        ignore=["appenlight.migrations", "appenlight.scripts", "appenlight.tests"]
+    )
+
+    config.add_directive("register_appenlight_plugin", register_appenlight_plugin)
+
+    for entry_point in iter_entry_points(group="appenlight.plugins"):
         plugin = entry_point.load()
         plugin.includeme(config)
 
     # include other appenlight plugins explictly if needed
-    includes = aslist(settings.get('appenlight.includes', []))
+    includes = aslist(settings.get("appenlight.includes", []))
     for inc in includes:
         config.include(inc)
 
     # run this after everything registers in configurator
 
     def pre_commit():
         jinja_env = config.get_jinja2_environment()
-        jinja_env.filters['tojson'] = json.dumps
-        jinja_env.filters['toJSONUnsafe'] = jinja2_filters.toJSONUnsafe
+        jinja_env.filters["tojson"] = json.dumps
+        jinja_env.filters["toJSONUnsafe"] = jinja2_filters.toJSONUnsafe
 
     config.action(None, pre_commit, order=PHASE3_CONFIG + 999)
 
     def wrap_config_celery():
         configure_celery(config.registry)
 
     config.action(None, wrap_config_celery, order=PHASE3_CONFIG + 999)
 
     app = config.make_wsgi_app()
     return app

backend/src/appenlight/celery/__init__.py

@@ -1,171 +1,181 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 
 from datetime import timedelta
 from celery import Celery
 from celery.bin import Option
 from celery.schedules import crontab
 from celery.signals import worker_init, task_revoked, user_preload_options
 from celery.signals import task_prerun, task_retry, task_failure, task_success
 from kombu.serialization import register
 from pyramid.paster import bootstrap
 from pyramid.request import Request
 from pyramid.scripting import prepare
 from pyramid.settings import asbool
 from pyramid.threadlocal import get_current_request
 
 from appenlight.celery.encoders import json_dumps, json_loads
 from appenlight_client.ext.celery import register_signals
 
 log = logging.getLogger(__name__)
 
-register('date_json', json_dumps, json_loads,
-         content_type='application/x-date_json',
-         content_encoding='utf-8')
+register(
+    "date_json",
+    json_dumps,
+    json_loads,
+    content_type="application/x-date_json",
+    content_encoding="utf-8",
+)
 
 celery = Celery()
 
-celery.user_options['preload'].add(
-    Option('--ini', dest='ini', default=None,
-           help='Specifies pyramid configuration file location.')
+celery.user_options["preload"].add(
+    Option(
+        "--ini",
+        dest="ini",
+        default=None,
+        help="Specifies pyramid configuration file location.",
+    )
 )
 
 
 @user_preload_options.connect
 def on_preload_parsed(options, **kwargs):
     """
     This actually configures celery from pyramid config file
     """
-    celery.conf['INI_PYRAMID'] = options['ini']
+    celery.conf["INI_PYRAMID"] = options["ini"]
     import appenlight_client.client as e_client
-    ini_location = options['ini']
+
+    ini_location = options["ini"]
     if not ini_location:
-        raise Exception('You need to pass pyramid ini location using '
-                        '--ini=filename.ini argument to the worker')
+        raise Exception(
+            "You need to pass pyramid ini location using "
+            "--ini=filename.ini argument to the worker"
+        )
     env = bootstrap(ini_location[0])
-    api_key = env['request'].registry.settings['appenlight.api_key']
-    tr_config = env['request'].registry.settings.get(
-        'appenlight.transport_config')
-    CONFIG = e_client.get_config({'appenlight.api_key': api_key})
+    api_key = env["request"].registry.settings["appenlight.api_key"]
+    tr_config = env["request"].registry.settings.get("appenlight.transport_config")
+    CONFIG = e_client.get_config({"appenlight.api_key": api_key})
     if tr_config:
-        CONFIG['appenlight.transport_config'] = tr_config
+        CONFIG["appenlight.transport_config"] = tr_config
     APPENLIGHT_CLIENT = e_client.Client(CONFIG)
     # log.addHandler(APPENLIGHT_CLIENT.log_handler)
     register_signals(APPENLIGHT_CLIENT)
     celery.pyramid = env
 
 
 celery_config = {
-    'CELERY_IMPORTS': ["appenlight.celery.tasks", ],
-    'CELERYD_TASK_TIME_LIMIT': 60,
-    'CELERYD_MAX_TASKS_PER_CHILD': 1000,
-    'CELERY_IGNORE_RESULT': True,
-    'CELERY_ACCEPT_CONTENT': ['date_json'],
-    'CELERY_TASK_SERIALIZER': 'date_json',
-    'CELERY_RESULT_SERIALIZER': 'date_json',
-    'BROKER_URL': None,
-    'CELERYD_CONCURRENCY': None,
-    'CELERY_TIMEZONE': None,
-    'CELERYBEAT_SCHEDULE': {
-        'alerting_reports': {
-            'task': 'appenlight.celery.tasks.alerting_reports',
-            'schedule': timedelta(seconds=60)
+    "CELERY_IMPORTS": ["appenlight.celery.tasks"],
+    "CELERYD_TASK_TIME_LIMIT": 60,
+    "CELERYD_MAX_TASKS_PER_CHILD": 1000,
+    "CELERY_IGNORE_RESULT": True,
+    "CELERY_ACCEPT_CONTENT": ["date_json"],
+    "CELERY_TASK_SERIALIZER": "date_json",
+    "CELERY_RESULT_SERIALIZER": "date_json",
+    "BROKER_URL": None,
+    "CELERYD_CONCURRENCY": None,
+    "CELERY_TIMEZONE": None,
+    "CELERYBEAT_SCHEDULE": {
+        "alerting_reports": {
+            "task": "appenlight.celery.tasks.alerting_reports",
+            "schedule": timedelta(seconds=60),
         },
-        'close_alerts': {
-            'task': 'appenlight.celery.tasks.close_alerts',
-            'schedule': timedelta(seconds=60)
-        }
-    }
+        "close_alerts": {
+            "task": "appenlight.celery.tasks.close_alerts",
+            "schedule": timedelta(seconds=60),
+        },
+    },
 }
 celery.config_from_object(celery_config)
 
 
 def configure_celery(pyramid_registry):
     settings = pyramid_registry.settings
-    celery_config['BROKER_URL'] = settings['celery.broker_url']
-    celery_config['CELERYD_CONCURRENCY'] = settings['celery.concurrency']
-    celery_config['CELERY_TIMEZONE'] = settings['celery.timezone']
+    celery_config["BROKER_URL"] = settings["celery.broker_url"]
+    celery_config["CELERYD_CONCURRENCY"] = settings["celery.concurrency"]
+    celery_config["CELERY_TIMEZONE"] = settings["celery.timezone"]
 
-    notifications_seconds = int(settings.get('tasks.notifications_reports.interval', 60))
+    notifications_seconds = int(
+        settings.get("tasks.notifications_reports.interval", 60)
+    )
 
-    celery_config['CELERYBEAT_SCHEDULE']['notifications'] = {
-        'task': 'appenlight.celery.tasks.notifications_reports',
-        'schedule': timedelta(seconds=notifications_seconds)
+    celery_config["CELERYBEAT_SCHEDULE"]["notifications"] = {
+        "task": "appenlight.celery.tasks.notifications_reports",
+        "schedule": timedelta(seconds=notifications_seconds),
     }
 
-    celery_config['CELERYBEAT_SCHEDULE']['daily_digest'] = {
-        'task': 'appenlight.celery.tasks.daily_digest',
-        'schedule': crontab(minute=1, hour='4,12,20')
+    celery_config["CELERYBEAT_SCHEDULE"]["daily_digest"] = {
+        "task": "appenlight.celery.tasks.daily_digest",
+        "schedule": crontab(minute=1, hour="4,12,20"),
     }
 
-    if asbool(settings.get('celery.always_eager')):
-        celery_config['CELERY_ALWAYS_EAGER'] = True
-        celery_config['CELERY_EAGER_PROPAGATES_EXCEPTIONS'] = True
+    if asbool(settings.get("celery.always_eager")):
+        celery_config["CELERY_ALWAYS_EAGER"] = True
+        celery_config["CELERY_EAGER_PROPAGATES_EXCEPTIONS"] = True
 
     for plugin in pyramid_registry.appenlight_plugins.values():
-        if plugin.get('celery_tasks'):
-            celery_config['CELERY_IMPORTS'].extend(plugin['celery_tasks'])
-        if plugin.get('celery_beats'):
-            for name, config in plugin['celery_beats']:
-                celery_config['CELERYBEAT_SCHEDULE'][name] = config
+        if plugin.get("celery_tasks"):
+            celery_config["CELERY_IMPORTS"].extend(plugin["celery_tasks"])
+        if plugin.get("celery_beats"):
+            for name, config in plugin["celery_beats"]:
+                celery_config["CELERYBEAT_SCHEDULE"][name] = config
     celery.config_from_object(celery_config)
 
 
 @task_prerun.connect
 def task_prerun_signal(task_id, task, args, kwargs, **kwaargs):
-    if hasattr(celery, 'pyramid'):
+    if hasattr(celery, "pyramid"):
         env = celery.pyramid
-        env = prepare(registry=env['request'].registry)
-        proper_base_url = env['request'].registry.settings['mailing.app_url']
-        tmp_req = Request.blank('/', base_url=proper_base_url)
+        env = prepare(registry=env["request"].registry)
+        proper_base_url = env["request"].registry.settings["mailing.app_url"]
+        tmp_req = Request.blank("/", base_url=proper_base_url)
         # ensure tasks generate url for right domain from config
-        env['request'].environ['HTTP_HOST'] = tmp_req.environ['HTTP_HOST']
-        env['request'].environ['SERVER_PORT'] = tmp_req.environ['SERVER_PORT']
-        env['request'].environ['SERVER_NAME'] = tmp_req.environ['SERVER_NAME']
-        env['request'].environ['wsgi.url_scheme'] = \
-            tmp_req.environ['wsgi.url_scheme']
+        env["request"].environ["HTTP_HOST"] = tmp_req.environ["HTTP_HOST"]
+        env["request"].environ["SERVER_PORT"] = tmp_req.environ["SERVER_PORT"]
+        env["request"].environ["SERVER_NAME"] = tmp_req.environ["SERVER_NAME"]
+        env["request"].environ["wsgi.url_scheme"] = tmp_req.environ["wsgi.url_scheme"]
     get_current_request().tm.begin()
 
 
 @task_success.connect
 def task_success_signal(result, **kwargs):
     get_current_request().tm.commit()
-    if hasattr(celery, 'pyramid'):
+    if hasattr(celery, "pyramid"):
         celery.pyramid["closer"]()
 
 
 @task_retry.connect
 def task_retry_signal(request, reason, einfo, **kwargs):
     get_current_request().tm.abort()
-    if hasattr(celery, 'pyramid'):
+    if hasattr(celery, "pyramid"):
         celery.pyramid["closer"]()
 
 
 @task_failure.connect
-def task_failure_signal(task_id, exception, args, kwargs, traceback, einfo,
-                        **kwaargs):
+def task_failure_signal(task_id, exception, args, kwargs, traceback, einfo, **kwaargs):
     get_current_request().tm.abort()
-    if hasattr(celery, 'pyramid'):
+    if hasattr(celery, "pyramid"):
         celery.pyramid["closer"]()
 
 
 @task_revoked.connect
 def task_revoked_signal(request, terminated, signum, expired, **kwaargs):
     get_current_request().tm.abort()
-    if hasattr(celery, 'pyramid'):
+    if hasattr(celery, "pyramid"):
         celery.pyramid["closer"]()

backend/src/appenlight/celery/encoders.py

@@ -1,60 +1,51 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import json
 from datetime import datetime, date, timedelta
 
-DATE_FORMAT = '%Y-%m-%dT%H:%M:%S.%f'
+DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%f"
 
 
 class DateEncoder(json.JSONEncoder):
     def default(self, obj):
         if isinstance(obj, datetime):
-            return {
-                '__type__': '__datetime__',
-                'iso': obj.strftime(DATE_FORMAT)
-            }
+            return {"__type__": "__datetime__", "iso": obj.strftime(DATE_FORMAT)}
         elif isinstance(obj, date):
-            return {
-                '__type__': '__date__',
-                'iso': obj.strftime(DATE_FORMAT)
-            }
+            return {"__type__": "__date__", "iso": obj.strftime(DATE_FORMAT)}
         elif isinstance(obj, timedelta):
-            return {
-                '__type__': '__timedelta__',
-                'seconds': obj.total_seconds()
-            }
+            return {"__type__": "__timedelta__", "seconds": obj.total_seconds()}
         else:
             return json.JSONEncoder.default(self, obj)
 
 
 def date_decoder(dct):
-    if '__type__' in dct:
-        if dct['__type__'] == '__datetime__':
-            return datetime.strptime(dct['iso'], DATE_FORMAT)
-        elif dct['__type__'] == '__date__':
-            return datetime.strptime(dct['iso'], DATE_FORMAT).date()
-        elif dct['__type__'] == '__timedelta__':
-            return timedelta(seconds=dct['seconds'])
+    if "__type__" in dct:
+        if dct["__type__"] == "__datetime__":
+            return datetime.strptime(dct["iso"], DATE_FORMAT)
+        elif dct["__type__"] == "__date__":
+            return datetime.strptime(dct["iso"], DATE_FORMAT).date()
+        elif dct["__type__"] == "__timedelta__":
+            return timedelta(seconds=dct["seconds"])
     return dct
 
 
 def json_dumps(obj):
     return json.dumps(obj, cls=DateEncoder)
 
 
 def json_loads(obj):
-    return json.loads(obj.decode('utf8'), object_hook=date_decoder)
+    return json.loads(obj.decode("utf8"), object_hook=date_decoder)

backend/src/appenlight/celery/tasks.py

@@ -1,657 +1,708 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import bisect
 import collections
 import math
 from datetime import datetime, timedelta
 
 import sqlalchemy as sa
 import elasticsearch.exceptions
 import elasticsearch.helpers
 
 from celery.utils.log import get_task_logger
 from zope.sqlalchemy import mark_changed
 from pyramid.threadlocal import get_current_request, get_current_registry
 from ziggurat_foundations.models.services.resource import ResourceService
 
 from appenlight.celery import celery
 from appenlight.models.report_group import ReportGroup
 from appenlight.models import DBSession, Datastores
 from appenlight.models.report import Report
 from appenlight.models.log import Log
 from appenlight.models.metric import Metric
 from appenlight.models.event import Event
 
 from appenlight.models.services.application import ApplicationService
 from appenlight.models.services.event import EventService
 from appenlight.models.services.log import LogService
 from appenlight.models.services.report import ReportService
 from appenlight.models.services.report_group import ReportGroupService
 from appenlight.models.services.user import UserService
 from appenlight.models.tag import Tag
 from appenlight.lib import print_traceback
 from appenlight.lib.utils import parse_proto, in_batches
 from appenlight.lib.ext_json import json
 from appenlight.lib.redis_keys import REDIS_KEYS
 from appenlight.lib.enums import ReportType
 
 log = get_task_logger(__name__)
 
-sample_boundries = list(range(100, 1000, 100)) + \
-                   list(range(1000, 10000, 1000)) + \
-                   list(range(10000, 100000, 5000))
+sample_boundries = (
+    list(range(100, 1000, 100))
+    + list(range(1000, 10000, 1000))
+    + list(range(10000, 100000, 5000))
+)
 
 
 def pick_sample(total_occurences, report_type=None):
     every = 1.0
     position = bisect.bisect_left(sample_boundries, total_occurences)
     if position > 0:
         if report_type == ReportType.not_found:
             divide = 10.0
         else:
             divide = 100.0
         every = sample_boundries[position - 1] / divide
     return total_occurences % every == 0
 
 
 @celery.task(queue="default", default_retry_delay=1, max_retries=2)
 def test_exception_task():
-    log.error('test celery log', extra={'location': 'celery'})
-    log.warning('test celery log', extra={'location': 'celery'})
-    raise Exception('Celery exception test')
+    log.error("test celery log", extra={"location": "celery"})
+    log.warning("test celery log", extra={"location": "celery"})
+    raise Exception("Celery exception test")
 
 
 @celery.task(queue="default", default_retry_delay=1, max_retries=2)
 def test_retry_exception_task():
     try:
         import time
 
         time.sleep(1.3)
-        log.error('test retry celery log', extra={'location': 'celery'})
-        log.warning('test retry celery log', extra={'location': 'celery'})
-        raise Exception('Celery exception test')
+        log.error("test retry celery log", extra={"location": "celery"})
+        log.warning("test retry celery log", extra={"location": "celery"})
+        raise Exception("Celery exception test")
     except Exception as exc:
         if celery.conf["CELERY_EAGER_PROPAGATES_EXCEPTIONS"]:
             raise
         test_retry_exception_task.retry(exc=exc)
 
 
 @celery.task(queue="reports", default_retry_delay=600, max_retries=144)
 def add_reports(resource_id, request_params, dataset, **kwargs):
-    proto_version = parse_proto(request_params.get('protocol_version', ''))
+    proto_version = parse_proto(request_params.get("protocol_version", ""))
     current_time = datetime.utcnow().replace(second=0, microsecond=0)
     try:
         # we will store solr docs here for single insert
         es_report_docs = {}
         es_report_group_docs = {}
         resource = ApplicationService.by_id(resource_id)
 
         tags = []
         es_slow_calls_docs = {}
         es_reports_stats_rows = {}
         for report_data in dataset:
             # build report details for later
             added_details = 0
             report = Report()
             report.set_data(report_data, resource, proto_version)
             report._skip_ft_index = True
 
             # find latest group in this months partition
             report_group = ReportGroupService.by_hash_and_resource(
                 report.resource_id,
                 report.grouping_hash,
-                since_when=datetime.utcnow().date().replace(day=1)
+                since_when=datetime.utcnow().date().replace(day=1),
             )
-            occurences = report_data.get('occurences', 1)
+            occurences = report_data.get("occurences", 1)
             if not report_group:
                 # total reports will be +1 moment later
-                report_group = ReportGroup(grouping_hash=report.grouping_hash,
-                                           occurences=0, total_reports=0,
-                                           last_report=0,
-                                           priority=report.priority,
-                                           error=report.error,
-                                           first_timestamp=report.start_time)
+                report_group = ReportGroup(
+                    grouping_hash=report.grouping_hash,
+                    occurences=0,
+                    total_reports=0,
+                    last_report=0,
+                    priority=report.priority,
+                    error=report.error,
+                    first_timestamp=report.start_time,
+                )
                 report_group._skip_ft_index = True
                 report_group.report_type = report.report_type
             report.report_group_time = report_group.first_timestamp
-            add_sample = pick_sample(report_group.occurences,
-                                     report_type=report_group.report_type)
+            add_sample = pick_sample(
+                report_group.occurences, report_type=report_group.report_type
+            )
             if add_sample:
                 resource.report_groups.append(report_group)
                 report_group.reports.append(report)
                 added_details += 1
                 DBSession.flush()
                 if report.partition_id not in es_report_docs:
                     es_report_docs[report.partition_id] = []
                 es_report_docs[report.partition_id].append(report.es_doc())
                 tags.extend(list(report.tags.items()))
                 slow_calls = report.add_slow_calls(report_data, report_group)
                 DBSession.flush()
                 for s_call in slow_calls:
                     if s_call.partition_id not in es_slow_calls_docs:
                         es_slow_calls_docs[s_call.partition_id] = []
-                    es_slow_calls_docs[s_call.partition_id].append(
-                        s_call.es_doc())
+                    es_slow_calls_docs[s_call.partition_id].append(s_call.es_doc())
                     # try generating new stat rows if needed
             else:
                 # required for postprocessing to not fail later
                 report.report_group = report_group
 
-            stat_row = ReportService.generate_stat_rows(
-                report, resource, report_group)
+            stat_row = ReportService.generate_stat_rows(report, resource, report_group)
             if stat_row.partition_id not in es_reports_stats_rows:
                 es_reports_stats_rows[stat_row.partition_id] = []
-            es_reports_stats_rows[stat_row.partition_id].append(
-                stat_row.es_doc())
+            es_reports_stats_rows[stat_row.partition_id].append(stat_row.es_doc())
 
             # see if we should mark 10th occurence of report
             last_occurences_10 = int(math.floor(report_group.occurences / 10))
-            curr_occurences_10 = int(math.floor(
-                (report_group.occurences + report.occurences) / 10))
-            last_occurences_100 = int(
-                math.floor(report_group.occurences / 100))
-            curr_occurences_100 = int(math.floor(
-                (report_group.occurences + report.occurences) / 100))
+            curr_occurences_10 = int(
+                math.floor((report_group.occurences + report.occurences) / 10)
+            )
+            last_occurences_100 = int(math.floor(report_group.occurences / 100))
+            curr_occurences_100 = int(
+                math.floor((report_group.occurences + report.occurences) / 100)
+            )
             notify_occurences_10 = last_occurences_10 != curr_occurences_10
             notify_occurences_100 = last_occurences_100 != curr_occurences_100
             report_group.occurences = ReportGroup.occurences + occurences
             report_group.last_timestamp = report.start_time
             report_group.summed_duration = ReportGroup.summed_duration + report.duration
             summed_duration = ReportGroup.summed_duration + report.duration
             summed_occurences = ReportGroup.occurences + occurences
             report_group.average_duration = summed_duration / summed_occurences
             report_group.run_postprocessing(report)
             if added_details:
                 report_group.total_reports = ReportGroup.total_reports + 1
                 report_group.last_report = report.id
-            report_group.set_notification_info(notify_10=notify_occurences_10,
-                                               notify_100=notify_occurences_100)
+            report_group.set_notification_info(
+                notify_10=notify_occurences_10, notify_100=notify_occurences_100
+            )
             DBSession.flush()
             report_group.get_report().notify_channel(report_group)
             if report_group.partition_id not in es_report_group_docs:
                 es_report_group_docs[report_group.partition_id] = []
             es_report_group_docs[report_group.partition_id].append(
-                report_group.es_doc())
+                report_group.es_doc()
+            )
 
-            action = 'REPORT'
-            log_msg = '%s: %s %s, client: %s, proto: %s' % (
+            action = "REPORT"
+            log_msg = "%s: %s %s, client: %s, proto: %s" % (
                 action,
-                report_data.get('http_status', 'unknown'),
+                report_data.get("http_status", "unknown"),
                 str(resource),
-                report_data.get('client'),
-                proto_version)
+                report_data.get("client"),
+                proto_version,
+            )
             log.info(log_msg)
         total_reports = len(dataset)
         redis_pipeline = Datastores.redis.pipeline(transaction=False)
-        key = REDIS_KEYS['counters']['reports_per_minute'].format(current_time)
+        key = REDIS_KEYS["counters"]["reports_per_minute"].format(current_time)
         redis_pipeline.incr(key, total_reports)
         redis_pipeline.expire(key, 3600 * 24)
-        key = REDIS_KEYS['counters']['events_per_minute_per_user'].format(
-            resource.owner_user_id, current_time)
+        key = REDIS_KEYS["counters"]["events_per_minute_per_user"].format(
+            resource.owner_user_id, current_time
+        )
         redis_pipeline.incr(key, total_reports)
         redis_pipeline.expire(key, 3600)
-        key = REDIS_KEYS['counters']['reports_per_hour_per_app'].format(
-            resource_id, current_time.replace(minute=0))
+        key = REDIS_KEYS["counters"]["reports_per_hour_per_app"].format(
+            resource_id, current_time.replace(minute=0)
+        )
         redis_pipeline.incr(key, total_reports)
         redis_pipeline.expire(key, 3600 * 24 * 7)
         redis_pipeline.sadd(
-            REDIS_KEYS['apps_that_got_new_data_per_hour'].format(
-                current_time.replace(minute=0)), resource_id)
+            REDIS_KEYS["apps_that_got_new_data_per_hour"].format(
+                current_time.replace(minute=0)
+            ),
+            resource_id,
+        )
         redis_pipeline.execute()
 
         add_reports_es(es_report_group_docs, es_report_docs)
         add_reports_slow_calls_es(es_slow_calls_docs)
         add_reports_stats_rows_es(es_reports_stats_rows)
         return True
     except Exception as exc:
         print_traceback(log)
         if celery.conf["CELERY_EAGER_PROPAGATES_EXCEPTIONS"]:
             raise
         add_reports.retry(exc=exc)
 
 
 @celery.task(queue="es", default_retry_delay=600, max_retries=144)
 def add_reports_es(report_group_docs, report_docs):
     for k, v in report_group_docs.items():
-        to_update = {'_index': k, '_type': 'report_group'}
+        to_update = {"_index": k, "_type": "report_group"}
         [i.update(to_update) for i in v]
         elasticsearch.helpers.bulk(Datastores.es, v)
     for k, v in report_docs.items():
-        to_update = {'_index': k, '_type': 'report'}
+        to_update = {"_index": k, "_type": "report"}
         [i.update(to_update) for i in v]
         elasticsearch.helpers.bulk(Datastores.es, v)
 
 
 @celery.task(queue="es", default_retry_delay=600, max_retries=144)
 def add_reports_slow_calls_es(es_docs):
     for k, v in es_docs.items():
-        to_update = {'_index': k, '_type': 'log'}
+        to_update = {"_index": k, "_type": "log"}
         [i.update(to_update) for i in v]
         elasticsearch.helpers.bulk(Datastores.es, v)
 
 
 @celery.task(queue="es", default_retry_delay=600, max_retries=144)
 def add_reports_stats_rows_es(es_docs):
     for k, v in es_docs.items():
-        to_update = {'_index': k, '_type': 'log'}
+        to_update = {"_index": k, "_type": "log"}
         [i.update(to_update) for i in v]
         elasticsearch.helpers.bulk(Datastores.es, v)
 
 
 @celery.task(queue="logs", default_retry_delay=600, max_retries=144)
 def add_logs(resource_id, request_params, dataset, **kwargs):
-    proto_version = request_params.get('protocol_version')
+    proto_version = request_params.get("protocol_version")
     current_time = datetime.utcnow().replace(second=0, microsecond=0)
 
     try:
         es_docs = collections.defaultdict(list)
         resource = ApplicationService.by_id_cached()(resource_id)
         resource = DBSession.merge(resource, load=False)
         ns_pairs = []
         for entry in dataset:
             # gather pk and ns so we can remove older versions of row later
-            if entry['primary_key'] is not None:
-                ns_pairs.append({"pk": entry['primary_key'],
-                                 "ns": entry['namespace']})
+            if entry["primary_key"] is not None:
+                ns_pairs.append({"pk": entry["primary_key"], "ns": entry["namespace"]})
             log_entry = Log()
             log_entry.set_data(entry, resource=resource)
             log_entry._skip_ft_index = True
             resource.logs.append(log_entry)
             DBSession.flush()
             # insert non pk rows first
-            if entry['primary_key'] is None:
+            if entry["primary_key"] is None:
                 es_docs[log_entry.partition_id].append(log_entry.es_doc())
 
         # 2nd pass to delete all log entries from db foe same pk/ns pair
         if ns_pairs:
             ids_to_delete = []
             es_docs = collections.defaultdict(list)
             es_docs_to_delete = collections.defaultdict(list)
             found_pkey_logs = LogService.query_by_primary_key_and_namespace(
-                list_of_pairs=ns_pairs)
+                list_of_pairs=ns_pairs
+            )
             log_dict = {}
             for log_entry in found_pkey_logs:
                 log_key = (log_entry.primary_key, log_entry.namespace)
                 if log_key not in log_dict:
                     log_dict[log_key] = []
                 log_dict[log_key].append(log_entry)
 
             for ns, entry_list in log_dict.items():
                 entry_list = sorted(entry_list, key=lambda x: x.timestamp)
                 # newest row needs to be indexed in es
                 log_entry = entry_list[-1]
                 # delete everything from pg and ES, leave the last row in pg
                 for e in entry_list[:-1]:
                     ids_to_delete.append(e.log_id)
                     es_docs_to_delete[e.partition_id].append(e.delete_hash)
 
-                es_docs_to_delete[log_entry.partition_id].append(
-                    log_entry.delete_hash)
+                es_docs_to_delete[log_entry.partition_id].append(log_entry.delete_hash)
 
                 es_docs[log_entry.partition_id].append(log_entry.es_doc())
 
             if ids_to_delete:
-                query = DBSession.query(Log).filter(
-                    Log.log_id.in_(ids_to_delete))
+                query = DBSession.query(Log).filter(Log.log_id.in_(ids_to_delete))
                 query.delete(synchronize_session=False)
             if es_docs_to_delete:
                 # batch this to avoid problems with default ES bulk limits
                 for es_index in es_docs_to_delete.keys():
                     for batch in in_batches(es_docs_to_delete[es_index], 20):
-                        query = {"query": {'terms': {'delete_hash': batch}}}
+                        query = {"query": {"terms": {"delete_hash": batch}}}
 
                         try:
                             Datastores.es.transport.perform_request(
-                                "DELETE", '/{}/{}/_query'.format(es_index, 'log'), body=query)
+                                "DELETE",
+                                "/{}/{}/_query".format(es_index, "log"),
+                                body=query,
+                            )
                         except elasticsearch.exceptions.NotFoundError as exc:
-                            msg = 'skipping index {}'.format(es_index)
+                            msg = "skipping index {}".format(es_index)
                             log.info(msg)
 
         total_logs = len(dataset)
 
-        log_msg = 'LOG_NEW: %s, entries: %s, proto:%s' % (
+        log_msg = "LOG_NEW: %s, entries: %s, proto:%s" % (
             str(resource),
             total_logs,
-            proto_version)
+            proto_version,
+        )
         log.info(log_msg)
         # mark_changed(session)
         redis_pipeline = Datastores.redis.pipeline(transaction=False)
-        key = REDIS_KEYS['counters']['logs_per_minute'].format(current_time)
+        key = REDIS_KEYS["counters"]["logs_per_minute"].format(current_time)
         redis_pipeline.incr(key, total_logs)
         redis_pipeline.expire(key, 3600 * 24)
-        key = REDIS_KEYS['counters']['events_per_minute_per_user'].format(
-            resource.owner_user_id, current_time)
+        key = REDIS_KEYS["counters"]["events_per_minute_per_user"].format(
+            resource.owner_user_id, current_time
+        )
         redis_pipeline.incr(key, total_logs)
         redis_pipeline.expire(key, 3600)
-        key = REDIS_KEYS['counters']['logs_per_hour_per_app'].format(
-            resource_id, current_time.replace(minute=0))
+        key = REDIS_KEYS["counters"]["logs_per_hour_per_app"].format(
+            resource_id, current_time.replace(minute=0)
+        )
         redis_pipeline.incr(key, total_logs)
         redis_pipeline.expire(key, 3600 * 24 * 7)
         redis_pipeline.sadd(
-            REDIS_KEYS['apps_that_got_new_data_per_hour'].format(
-                current_time.replace(minute=0)), resource_id)
+            REDIS_KEYS["apps_that_got_new_data_per_hour"].format(
+                current_time.replace(minute=0)
+            ),
+            resource_id,
+        )
         redis_pipeline.execute()
         add_logs_es(es_docs)
         return True
     except Exception as exc:
         print_traceback(log)
         if celery.conf["CELERY_EAGER_PROPAGATES_EXCEPTIONS"]:
             raise
         add_logs.retry(exc=exc)
 
 
 @celery.task(queue="es", default_retry_delay=600, max_retries=144)
 def add_logs_es(es_docs):
     for k, v in es_docs.items():
-        to_update = {'_index': k, '_type': 'log'}
+        to_update = {"_index": k, "_type": "log"}
         [i.update(to_update) for i in v]
         elasticsearch.helpers.bulk(Datastores.es, v)
 
 
 @celery.task(queue="metrics", default_retry_delay=600, max_retries=144)
 def add_metrics(resource_id, request_params, dataset, proto_version):
     current_time = datetime.utcnow().replace(second=0, microsecond=0)
     try:
         resource = ApplicationService.by_id_cached()(resource_id)
         resource = DBSession.merge(resource, load=False)
         es_docs = []
         rows = []
         for metric in dataset:
-            tags = dict(metric['tags'])
-            server_n = tags.get('server_name', metric['server_name']).lower()
-            tags['server_name'] = server_n or 'unknown'
+            tags = dict(metric["tags"])
+            server_n = tags.get("server_name", metric["server_name"]).lower()
+            tags["server_name"] = server_n or "unknown"
             new_metric = Metric(
-                timestamp=metric['timestamp'],
+                timestamp=metric["timestamp"],
                 resource_id=resource.resource_id,
-                namespace=metric['namespace'],
-                tags=tags)
+                namespace=metric["namespace"],
+                tags=tags,
+            )
             rows.append(new_metric)
             es_docs.append(new_metric.es_doc())
         session = DBSession()
         session.bulk_save_objects(rows)
         session.flush()
 
-        action = 'METRICS'
-        metrics_msg = '%s: %s, metrics: %s, proto:%s' % (
+        action = "METRICS"
+        metrics_msg = "%s: %s, metrics: %s, proto:%s" % (
             action,
             str(resource),
             len(dataset),
-            proto_version
+            proto_version,
         )
         log.info(metrics_msg)
 
         mark_changed(session)
         redis_pipeline = Datastores.redis.pipeline(transaction=False)
-        key = REDIS_KEYS['counters']['metrics_per_minute'].format(current_time)
+        key = REDIS_KEYS["counters"]["metrics_per_minute"].format(current_time)
         redis_pipeline.incr(key, len(rows))
         redis_pipeline.expire(key, 3600 * 24)
-        key = REDIS_KEYS['counters']['events_per_minute_per_user'].format(
-            resource.owner_user_id, current_time)
+        key = REDIS_KEYS["counters"]["events_per_minute_per_user"].format(
+            resource.owner_user_id, current_time
+        )
         redis_pipeline.incr(key, len(rows))
         redis_pipeline.expire(key, 3600)
-        key = REDIS_KEYS['counters']['metrics_per_hour_per_app'].format(
-            resource_id, current_time.replace(minute=0))
+        key = REDIS_KEYS["counters"]["metrics_per_hour_per_app"].format(
+            resource_id, current_time.replace(minute=0)
+        )
         redis_pipeline.incr(key, len(rows))
         redis_pipeline.expire(key, 3600 * 24 * 7)
         redis_pipeline.sadd(
-            REDIS_KEYS['apps_that_got_new_data_per_hour'].format(
-                current_time.replace(minute=0)), resource_id)
+            REDIS_KEYS["apps_that_got_new_data_per_hour"].format(
+                current_time.replace(minute=0)
+            ),
+            resource_id,
+        )
         redis_pipeline.execute()
         add_metrics_es(es_docs)
         return True
     except Exception as exc:
         print_traceback(log)
         if celery.conf["CELERY_EAGER_PROPAGATES_EXCEPTIONS"]:
             raise
         add_metrics.retry(exc=exc)
 
 
 @celery.task(queue="es", default_retry_delay=600, max_retries=144)
 def add_metrics_es(es_docs):
     for doc in es_docs:
-        partition = 'rcae_m_%s' % doc['timestamp'].strftime('%Y_%m_%d')
-        Datastores.es.index(partition, 'log', doc)
+        partition = "rcae_m_%s" % doc["timestamp"].strftime("%Y_%m_%d")
+        Datastores.es.index(partition, "log", doc)
 
 
 @celery.task(queue="default", default_retry_delay=5, max_retries=2)
 def check_user_report_notifications(resource_id):
     since_when = datetime.utcnow()
     try:
         request = get_current_request()
         application = ApplicationService.by_id(resource_id)
         if not application:
             return
-        error_key = REDIS_KEYS['reports_to_notify_per_type_per_app'].format(
-            ReportType.error, resource_id)
-        slow_key = REDIS_KEYS['reports_to_notify_per_type_per_app'].format(
-            ReportType.slow, resource_id)
+        error_key = REDIS_KEYS["reports_to_notify_per_type_per_app"].format(
+            ReportType.error, resource_id
+        )
+        slow_key = REDIS_KEYS["reports_to_notify_per_type_per_app"].format(
+            ReportType.slow, resource_id
+        )
         error_group_ids = Datastores.redis.smembers(error_key)
         slow_group_ids = Datastores.redis.smembers(slow_key)
         Datastores.redis.delete(error_key)
         Datastores.redis.delete(slow_key)
         err_gids = [int(g_id) for g_id in error_group_ids]
         slow_gids = [int(g_id) for g_id in list(slow_group_ids)]
         group_ids = err_gids + slow_gids
         occurence_dict = {}
         for g_id in group_ids:
-            key = REDIS_KEYS['counters']['report_group_occurences'].format(
-                g_id)
+            key = REDIS_KEYS["counters"]["report_group_occurences"].format(g_id)
             val = Datastores.redis.get(key)
             Datastores.redis.delete(key)
             if val:
                 occurence_dict[g_id] = int(val)
             else:
                 occurence_dict[g_id] = 1
         report_groups = ReportGroupService.by_ids(group_ids)
         report_groups.options(sa.orm.joinedload(ReportGroup.last_report_ref))
 
         ApplicationService.check_for_groups_alert(
-            application, 'alert', report_groups=report_groups,
-            occurence_dict=occurence_dict)
-        users = set([p.user for p in ResourceService.users_for_perm(application, 'view')])
+            application,
+            "alert",
+            report_groups=report_groups,
+            occurence_dict=occurence_dict,
+        )
+        users = set(
+            [p.user for p in ResourceService.users_for_perm(application, "view")]
+        )
         report_groups = report_groups.all()
         for user in users:
-            UserService.report_notify(user, request, application,
-                                      report_groups=report_groups,
-                                      occurence_dict=occurence_dict)
+            UserService.report_notify(
+                user,
+                request,
+                application,
+                report_groups=report_groups,
+                occurence_dict=occurence_dict,
+            )
         for group in report_groups:
             # marks report_groups as notified
             if not group.notified:
                 group.notified = True
     except Exception as exc:
         print_traceback(log)
         raise
 
 
 @celery.task(queue="default", default_retry_delay=5, max_retries=2)
 def check_alerts(resource_id):
     since_when = datetime.utcnow()
     try:
         request = get_current_request()
         application = ApplicationService.by_id(resource_id)
         if not application:
             return
-        error_key = REDIS_KEYS[
-            'reports_to_notify_per_type_per_app_alerting'].format(
-            ReportType.error, resource_id)
-        slow_key = REDIS_KEYS[
-            'reports_to_notify_per_type_per_app_alerting'].format(
-            ReportType.slow, resource_id)
+        error_key = REDIS_KEYS["reports_to_notify_per_type_per_app_alerting"].format(
+            ReportType.error, resource_id
+        )
+        slow_key = REDIS_KEYS["reports_to_notify_per_type_per_app_alerting"].format(
+            ReportType.slow, resource_id
+        )
         error_group_ids = Datastores.redis.smembers(error_key)
         slow_group_ids = Datastores.redis.smembers(slow_key)
         Datastores.redis.delete(error_key)
         Datastores.redis.delete(slow_key)
         err_gids = [int(g_id) for g_id in error_group_ids]
         slow_gids = [int(g_id) for g_id in list(slow_group_ids)]
         group_ids = err_gids + slow_gids
         occurence_dict = {}
         for g_id in group_ids:
-            key = REDIS_KEYS['counters'][
-                'report_group_occurences_alerting'].format(
-                g_id)
+            key = REDIS_KEYS["counters"]["report_group_occurences_alerting"].format(
+                g_id
+            )
             val = Datastores.redis.get(key)
             Datastores.redis.delete(key)
             if val:
                 occurence_dict[g_id] = int(val)
             else:
                 occurence_dict[g_id] = 1
         report_groups = ReportGroupService.by_ids(group_ids)
         report_groups.options(sa.orm.joinedload(ReportGroup.last_report_ref))
 
         ApplicationService.check_for_groups_alert(
-            application, 'alert', report_groups=report_groups,
-            occurence_dict=occurence_dict, since_when=since_when)
+            application,
+            "alert",
+            report_groups=report_groups,
+            occurence_dict=occurence_dict,
+            since_when=since_when,
+        )
     except Exception as exc:
         print_traceback(log)
         raise
 
 
 @celery.task(queue="default", default_retry_delay=1, max_retries=2)
 def close_alerts():
-    log.warning('Checking alerts')
+    log.warning("Checking alerts")
     since_when = datetime.utcnow()
     try:
-        event_types = [Event.types['error_report_alert'],
-                       Event.types['slow_report_alert'], ]
-        statuses = [Event.statuses['active']]
+        event_types = [
+            Event.types["error_report_alert"],
+            Event.types["slow_report_alert"],
+        ]
+        statuses = [Event.statuses["active"]]
         # get events older than 5 min
         events = EventService.by_type_and_status(
-            event_types,
-            statuses,
-            older_than=(since_when - timedelta(minutes=5)))
+            event_types, statuses, older_than=(since_when - timedelta(minutes=5))
+        )
         for event in events:
             # see if we can close them
-            event.validate_or_close(
-                since_when=(since_when - timedelta(minutes=1)))
+            event.validate_or_close(since_when=(since_when - timedelta(minutes=1)))
     except Exception as exc:
         print_traceback(log)
         raise
 
 
 @celery.task(queue="default", default_retry_delay=600, max_retries=144)
 def update_tag_counter(tag_name, tag_value, count):
     try:
-        query = DBSession.query(Tag).filter(Tag.name == tag_name).filter(
-            sa.cast(Tag.value, sa.types.TEXT) == sa.cast(json.dumps(tag_value),
-                                                         sa.types.TEXT))
-        query.update({'times_seen': Tag.times_seen + count,
-                      'last_timestamp': datetime.utcnow()},
-                     synchronize_session=False)
+        query = (
+            DBSession.query(Tag)
+            .filter(Tag.name == tag_name)
+            .filter(
+                sa.cast(Tag.value, sa.types.TEXT)
+                == sa.cast(json.dumps(tag_value), sa.types.TEXT)
+            )
+        )
+        query.update(
+            {"times_seen": Tag.times_seen + count, "last_timestamp": datetime.utcnow()},
+            synchronize_session=False,
+        )
         session = DBSession()
         mark_changed(session)
         return True
     except Exception as exc:
         print_traceback(log)
         if celery.conf["CELERY_EAGER_PROPAGATES_EXCEPTIONS"]:
             raise
         update_tag_counter.retry(exc=exc)
 
 
 @celery.task(queue="default")
 def update_tag_counters():
     """
     Sets task to update counters for application tags
     """
-    tags = Datastores.redis.lrange(REDIS_KEYS['seen_tag_list'], 0, -1)
-    Datastores.redis.delete(REDIS_KEYS['seen_tag_list'])
+    tags = Datastores.redis.lrange(REDIS_KEYS["seen_tag_list"], 0, -1)
+    Datastores.redis.delete(REDIS_KEYS["seen_tag_list"])
     c = collections.Counter(tags)
     for t_json, count in c.items():
         tag_info = json.loads(t_json)
         update_tag_counter.delay(tag_info[0], tag_info[1], count)
 
 
 @celery.task(queue="default")
 def daily_digest():
     """
     Sends daily digest with top 50 error reports
     """
     request = get_current_request()
-    apps = Datastores.redis.smembers(REDIS_KEYS['apps_that_had_reports'])
-    Datastores.redis.delete(REDIS_KEYS['apps_that_had_reports'])
+    apps = Datastores.redis.smembers(REDIS_KEYS["apps_that_had_reports"])
+    Datastores.redis.delete(REDIS_KEYS["apps_that_had_reports"])
     since_when = datetime.utcnow() - timedelta(hours=8)
-    log.warning('Generating daily digests')
+    log.warning("Generating daily digests")
     for resource_id in apps:
-        resource_id = resource_id.decode('utf8')
+        resource_id = resource_id.decode("utf8")
         end_date = datetime.utcnow().replace(microsecond=0, second=0)
-        filter_settings = {'resource': [resource_id],
-                           'tags': [{'name': 'type',
-                                     'value': ['error'], 'op': None}],
-                           'type': 'error', 'start_date': since_when,
-                           'end_date': end_date}
+        filter_settings = {
+            "resource": [resource_id],
+            "tags": [{"name": "type", "value": ["error"], "op": None}],
+            "type": "error",
+            "start_date": since_when,
+            "end_date": end_date,
+        }
 
         reports = ReportGroupService.get_trending(
-            request, filter_settings=filter_settings, limit=50)
+            request, filter_settings=filter_settings, limit=50
+        )
 
         application = ApplicationService.by_id(resource_id)
         if application:
-            users = set([p.user for p in ResourceService.users_for_perm(application, 'view')])
+            users = set(
+                [p.user for p in ResourceService.users_for_perm(application, "view")]
+            )
             for user in users:
-                user.send_digest(request, application, reports=reports,
-                                 since_when=since_when)
+                user.send_digest(
+                    request, application, reports=reports, since_when=since_when
+                )
 
 
 @celery.task(queue="default")
 def notifications_reports():
     """
     Loop that checks redis for info and then issues new tasks to celery to
     issue notifications
     """
-    apps = Datastores.redis.smembers(REDIS_KEYS['apps_that_had_reports'])
-    Datastores.redis.delete(REDIS_KEYS['apps_that_had_reports'])
+    apps = Datastores.redis.smembers(REDIS_KEYS["apps_that_had_reports"])
+    Datastores.redis.delete(REDIS_KEYS["apps_that_had_reports"])
     for app in apps:
-        log.warning('Notify for app: %s' % app)
-        check_user_report_notifications.delay(app.decode('utf8'))
+        log.warning("Notify for app: %s" % app)
+        check_user_report_notifications.delay(app.decode("utf8"))
+
 
 @celery.task(queue="default")
 def alerting_reports():
     """
     Loop that checks redis for info and then issues new tasks to celery to
     perform the following:
     - which applications should have new alerts opened
     """
 
-    apps = Datastores.redis.smembers(REDIS_KEYS['apps_that_had_reports_alerting'])
-    Datastores.redis.delete(REDIS_KEYS['apps_that_had_reports_alerting'])
+    apps = Datastores.redis.smembers(REDIS_KEYS["apps_that_had_reports_alerting"])
+    Datastores.redis.delete(REDIS_KEYS["apps_that_had_reports_alerting"])
     for app in apps:
-        log.warning('Notify for app: %s' % app)
-        check_alerts.delay(app.decode('utf8'))
+        log.warning("Notify for app: %s" % app)
+        check_alerts.delay(app.decode("utf8"))
 
 
[e-mail](queue="default", soft_time_limit=3600 * 4,
-             hard_time_limit=3600 * 4, max_retries=144)
[e-mail](
+    queue="default", soft_time_limit=3600 * 4, hard_time_limit=3600 * 4, max_retries=144
+)
 def logs_cleanup(resource_id, filter_settings):
     request = get_current_request()
     request.tm.begin()
     es_query = {
         "query": {
-            "filtered": {
-                "filter": {
-                    "and": [{"term": {"resource_id": resource_id}}]
-                }
-            }
+            "filtered": {"filter": {"and": [{"term": {"resource_id": resource_id}}]}}
         }
     }
 
     query = DBSession.query(Log).filter(Log.resource_id == resource_id)
-    if filter_settings['namespace']:
-        query = query.filter(Log.namespace == filter_settings['namespace'][0])
-        es_query['query']['filtered']['filter']['and'].append(
-            {"term": {"namespace": filter_settings['namespace'][0]}}
+    if filter_settings["namespace"]:
+        query = query.filter(Log.namespace == filter_settings["namespace"][0])
+        es_query["query"]["filtered"]["filter"]["and"].append(
+            {"term": {"namespace": filter_settings["namespace"][0]}}
         )
     query.delete(synchronize_session=False)
     request.tm.commit()
-    Datastores.es.transport.perform_request("DELETE", '/{}/{}/_query'.format('rcae_l_*', 'log'), body=es_query)
+    Datastores.es.transport.perform_request(
+        "DELETE", "/{}/{}/_query".format("rcae_l_*", "log"), body=es_query
+    )

backend/src/appenlight/fil.py

@@ -1,19 +1,20 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 def filter_callable(structure, section=None):
-    structure['SOMEVAL'] = '***REMOVED***'
+    structure["SOMEVAL"] = "***REMOVED***"
     return structure

backend/src/appenlight/forms.py

@@ -1,896 +1,981 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import wtforms
 import formencode
 import re
 import pyramid.threadlocal
 import datetime
 import appenlight.lib.helpers as h
 
 from ziggurat_foundations.models.services.user import UserService
 from ziggurat_foundations.models.services.group import GroupService
 from appenlight.models import DBSession
 from appenlight.models.alert_channel import AlertChannel
 from appenlight.models.integrations import IntegrationException
 from appenlight.models.integrations.campfire import CampfireIntegration
 from appenlight.models.integrations.bitbucket import BitbucketIntegration
 from appenlight.models.integrations.github import GithubIntegration
 from appenlight.models.integrations.flowdock import FlowdockIntegration
 from appenlight.models.integrations.hipchat import HipchatIntegration
 from appenlight.models.integrations.jira import JiraClient
 from appenlight.models.integrations.slack import SlackIntegration
 from appenlight.lib.ext_json import json
 from wtforms.ext.csrf.form import SecureForm
 from wtforms.compat import iteritems
 from collections import defaultdict
 
 _ = str
 
 strip_filter = lambda x: x.strip() if x else None
 uppercase_filter = lambda x: x.upper() if x else None
 
-FALSE_VALUES = ('false', '', False, None)
+FALSE_VALUES = ("false", "", False, None)
 
 
 class CSRFException(Exception):
     pass
 
 
 class ReactorForm(SecureForm):
-    def __init__(self, formdata=None, obj=None, prefix='', csrf_context=None,
-                 **kwargs):
-        super(ReactorForm, self).__init__(formdata=formdata, obj=obj,
-                                          prefix=prefix,
-                                          csrf_context=csrf_context, **kwargs)
+    def __init__(self, formdata=None, obj=None, prefix="", csrf_context=None, **kwargs):
+        super(ReactorForm, self).__init__(
+            formdata=formdata,
+            obj=obj,
+            prefix=prefix,
+            csrf_context=csrf_context,
+            **kwargs
+        )
         self._csrf_context = csrf_context
 
     def generate_csrf_token(self, csrf_context):
         return csrf_context.session.get_csrf_token()
 
     def validate_csrf_token(self, field):
         request = self._csrf_context or pyramid.threadlocal.get_current_request()
-        is_from_auth_token = 'auth:auth_token' in request.effective_principals
+        is_from_auth_token = "auth:auth_token" in request.effective_principals
         if is_from_auth_token:
             return True
 
         if field.data != field.current_token:
             # try to save the day by using token from angular
-            if request.headers.get('X-XSRF-TOKEN') != field.current_token:
-                raise CSRFException('Invalid CSRF token')
+            if request.headers.get("X-XSRF-TOKEN") != field.current_token:
+                raise CSRFException("Invalid CSRF token")
 
     @property
     def errors_dict(self):
         r_dict = defaultdict(list)
         for k, errors in self.errors.items():
             r_dict[k].extend([str(e) for e in errors])
         return r_dict
 
     @property
     def errors_json(self):
         return json.dumps(self.errors_dict)
 
     def populate_obj(self, obj, ignore_none=False):
         """
         Populates the attributes of the passed `obj` with data from the form's
         fields.
 
         :note: This is a destructive operation; Any attribute with the same name
                as a field will be overridden. Use with caution.
         """
         if ignore_none:
             for name, field in iteritems(self._fields):
                 if field.data is not None:
                     field.populate_obj(obj, name)
         else:
             for name, field in iteritems(self._fields):
                 field.populate_obj(obj, name)
 
     css_classes = {}
     ignore_labels = {}
 
 
 class SignInForm(ReactorForm):
     came_from = wtforms.HiddenField()
-    sign_in_user_name = wtforms.StringField(_('User Name'))
-    sign_in_user_password = wtforms.PasswordField(_('Password'))
+    sign_in_user_name = wtforms.StringField(_("User Name"))
+    sign_in_user_password = wtforms.PasswordField(_("Password"))
 
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
 
-    html_attrs = {'sign_in_user_name': {'placeholder': 'Your login'},
-                  'sign_in_user_password': {
-                      'placeholder': 'Your password'}}
+    html_attrs = {
+        "sign_in_user_name": {"placeholder": "Your login"},
+        "sign_in_user_password": {"placeholder": "Your password"},
+    }
 
 
 from wtforms.widgets import html_params, HTMLString
 
 
-def select_multi_checkbox(field, ul_class='set', **kwargs):
+def select_multi_checkbox(field, ul_class="set", **kwargs):
     """Render a multi-checkbox widget"""
-    kwargs.setdefault('type', 'checkbox')
-    field_id = kwargs.pop('id', field.id)
-    html = ['<ul %s>' % html_params(id=field_id, class_=ul_class)]
+    kwargs.setdefault("type", "checkbox")
+    field_id = kwargs.pop("id", field.id)
+    html = ["<ul %s>" % html_params(id=field_id, class_=ul_class)]
     for value, label, checked in field.iter_choices():
-        choice_id = '%s-%s' % (field_id, value)
+        choice_id = "%s-%s" % (field_id, value)
         options = dict(kwargs, name=field.name, value=value, id=choice_id)
         if checked:
-            options['checked'] = 'checked'
-        html.append('<li><input %s /> ' % html_params(**options))
+            options["checked"] = "checked"
+        html.append("<li><input %s /> " % html_params(**options))
         html.append('<label for="%s">%s</label></li>' % (choice_id, label))
-    html.append('</ul>')
-    return HTMLString(''.join(html))
+    html.append("</ul>")
+    return HTMLString("".join(html))
 
 
-def button_widget(field, button_cls='ButtonField btn btn-default', **kwargs):
+def button_widget(field, button_cls="ButtonField btn btn-default", **kwargs):
     """Render a button widget"""
-    kwargs.setdefault('type', 'button')
-    field_id = kwargs.pop('id', field.id)
-    kwargs.setdefault('value', field.label.text)
-    html = ['<button %s>%s</button>' % (html_params(id=field_id,
-                                                     class_=button_cls),
-                                         kwargs['value'],)]
-    return HTMLString(''.join(html))
+    kwargs.setdefault("type", "button")
+    field_id = kwargs.pop("id", field.id)
+    kwargs.setdefault("value", field.label.text)
+    html = [
+        "<button %s>%s</button>"
+        % (html_params(id=field_id, class_=button_cls), kwargs["value"])
+    ]
+    return HTMLString("".join(html))
 
 
 def clean_whitespace(value):
     if value:
         return value.strip()
     return value
 
 
 def found_username_validator(form, field):
     user = UserService.by_user_name(field.data)
     # sets user to recover in email validator
     form.field_user = user
     if not user:
-        raise wtforms.ValidationError('This username does not exist')
+        raise wtforms.ValidationError("This username does not exist")
 
 
 def found_username_email_validator(form, field):
     user = UserService.by_email(field.data)
     if not user:
-        raise wtforms.ValidationError('Email is incorrect')
+        raise wtforms.ValidationError("Email is incorrect")
 
 
 def unique_username_validator(form, field):
     user = UserService.by_user_name(field.data)
     if user:
-        raise wtforms.ValidationError('This username already exists in system')
+        raise wtforms.ValidationError("This username already exists in system")
 
 
 def unique_groupname_validator(form, field):
     group = GroupService.by_group_name(field.data)
-    mod_group = getattr(form, '_modified_group', None)
+    mod_group = getattr(form, "_modified_group", None)
     if group and (not mod_group or mod_group.id != group.id):
-        raise wtforms.ValidationError(
-            'This group name already exists in system')
+        raise wtforms.ValidationError("This group name already exists in system")
 
 
 def unique_email_validator(form, field):
     user = UserService.by_email(field.data)
     if user:
-        raise wtforms.ValidationError('This email already exists in system')
+        raise wtforms.ValidationError("This email already exists in system")
 
 
 def email_validator(form, field):
     validator = formencode.validators.Email()
     try:
         validator.to_python(field.data)
     except formencode.Invalid as e:
         raise wtforms.ValidationError(e)
 
 
 def unique_alert_email_validator(form, field):
     q = DBSession.query(AlertChannel)
-    q = q.filter(AlertChannel.channel_name == 'email')
+    q = q.filter(AlertChannel.channel_name == "email")
     q = q.filter(AlertChannel.channel_value == field.data)
     email = q.first()
     if email:
-        raise wtforms.ValidationError(
-            'This email already exists in alert system')
+        raise wtforms.ValidationError("This email already exists in alert system")
 
 
 def blocked_email_validator(form, field):
     blocked_emails = [
-        'goood-mail.org',
-        'shoeonlineblog.com',
-        'louboutinemart.com',
-        'guccibagshere.com',
-        'nikeshoesoutletforsale.com'
+        "goood-mail.org",
+        "shoeonlineblog.com",
+        "louboutinemart.com",
+        "guccibagshere.com",
+        "nikeshoesoutletforsale.com",
     ]
-    data = field.data or ''
-    domain = data.split('@')[-1]
+    data = field.data or ""
+    domain = data.split("@")[-1]
     if domain in blocked_emails:
-        raise wtforms.ValidationError('Don\'t spam')
+        raise wtforms.ValidationError("Don't spam")
 
 
 def old_password_validator(form, field):
-    if not UserService.check_password(field.user, field.data or ''):
-        raise wtforms.ValidationError('You need to enter correct password')
+    if not UserService.check_password(field.user, field.data or ""):
+        raise wtforms.ValidationError("You need to enter correct password")
 
 
 class UserRegisterForm(ReactorForm):
     user_name = wtforms.StringField(
-        _('User Name'),
+        _("User Name"),
         filters=[strip_filter],
         validators=[
             wtforms.validators.Length(min=2, max=30),
             wtforms.validators.Regexp(
-                re.compile(r'^[\.\w-]+$', re.UNICODE),
-                message="Invalid characters used"),
+                re.compile(r"^[\.\w-]+$", re.UNICODE), message="Invalid characters used"
+            ),
             unique_username_validator,
-            wtforms.validators.DataRequired()
-        ])
+            wtforms.validators.DataRequired(),
+        ],
+    )
 
-    user_password = wtforms.PasswordField(_('User Password'),
-                                          filters=[strip_filter],
-                                          validators=[
-                                              wtforms.validators.Length(min=4),
-                                              wtforms.validators.DataRequired()
-                                          ])
+    user_password = wtforms.PasswordField(
+        _("User Password"),
+        filters=[strip_filter],
+        validators=[
+            wtforms.validators.Length(min=4),
+            wtforms.validators.DataRequired(),
+        ],
+    )
 
-    email = wtforms.StringField(_('Email Address'),
-                                filters=[strip_filter],
-                                validators=[email_validator,
-                                            unique_email_validator,
-                                            blocked_email_validator,
-                                            wtforms.validators.DataRequired()])
-    first_name = wtforms.HiddenField(_('First Name'))
-    last_name = wtforms.HiddenField(_('Last Name'))
+    email = wtforms.StringField(
+        _("Email Address"),
+        filters=[strip_filter],
+        validators=[
+            email_validator,
+            unique_email_validator,
+            blocked_email_validator,
+            wtforms.validators.DataRequired(),
+        ],
+    )
+    first_name = wtforms.HiddenField(_("First Name"))
+    last_name = wtforms.HiddenField(_("Last Name"))
 
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
 
-    html_attrs = {'user_name': {'placeholder': 'Your login'},
-                  'user_password': {'placeholder': 'Your password'},
-                  'email': {'placeholder': 'Your email'}}
+    html_attrs = {
+        "user_name": {"placeholder": "Your login"},
+        "user_password": {"placeholder": "Your password"},
+        "email": {"placeholder": "Your email"},
+    }
 
 
 class UserCreateForm(UserRegisterForm):
-    status = wtforms.BooleanField('User status',
-                                  false_values=FALSE_VALUES)
+    status = wtforms.BooleanField("User status", false_values=FALSE_VALUES)
 
 
 class UserUpdateForm(UserCreateForm):
     user_name = None
-    user_password = wtforms.PasswordField(_('User Password'),
-                                          filters=[strip_filter],
-                                          validators=[
-                                              wtforms.validators.Length(min=4),
-                                              wtforms.validators.Optional()
-                                          ])
-    email = wtforms.StringField(_('Email Address'),
-                                filters=[strip_filter],
-                                validators=[email_validator,
-                                            wtforms.validators.DataRequired()])
+    user_password = wtforms.PasswordField(
+        _("User Password"),
+        filters=[strip_filter],
+        validators=[wtforms.validators.Length(min=4), wtforms.validators.Optional()],
+    )
+    email = wtforms.StringField(
+        _("Email Address"),
+        filters=[strip_filter],
+        validators=[email_validator, wtforms.validators.DataRequired()],
+    )
 
 
 class LostPasswordForm(ReactorForm):
-    email = wtforms.StringField(_('Email Address'),
-                                filters=[strip_filter],
-                                validators=[email_validator,
-                                            found_username_email_validator,
-                                            wtforms.validators.DataRequired()])
+    email = wtforms.StringField(
+        _("Email Address"),
+        filters=[strip_filter],
+        validators=[
+            email_validator,
+            found_username_email_validator,
+            wtforms.validators.DataRequired(),
+        ],
+    )
 
-    submit = wtforms.SubmitField(_('Reset password'))
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
+    submit = wtforms.SubmitField(_("Reset password"))
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
 
 
 class ChangePasswordForm(ReactorForm):
     old_password = wtforms.PasswordField(
-        'Old Password',
+        "Old Password",
         filters=[strip_filter],
-        validators=[old_password_validator,
-                    wtforms.validators.DataRequired()])
+        validators=[old_password_validator, wtforms.validators.DataRequired()],
+    )
 
     new_password = wtforms.PasswordField(
-        'New Password',
+        "New Password",
         filters=[strip_filter],
-        validators=[wtforms.validators.Length(min=4),
-                    wtforms.validators.DataRequired()])
+        validators=[
+            wtforms.validators.Length(min=4),
+            wtforms.validators.DataRequired(),
+        ],
+    )
     new_password_confirm = wtforms.PasswordField(
-        'Confirm Password',
+        "Confirm Password",
         filters=[strip_filter],
-        validators=[wtforms.validators.EqualTo('new_password'),
-                    wtforms.validators.DataRequired()])
-    submit = wtforms.SubmitField('Change Password')
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
+        validators=[
+            wtforms.validators.EqualTo("new_password"),
+            wtforms.validators.DataRequired(),
+        ],
+    )
+    submit = wtforms.SubmitField("Change Password")
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
 
 
 class CheckPasswordForm(ReactorForm):
     password = wtforms.PasswordField(
-        'Password',
+        "Password",
         filters=[strip_filter],
-        validators=[old_password_validator,
-                    wtforms.validators.DataRequired()])
+        validators=[old_password_validator, wtforms.validators.DataRequired()],
+    )
 
 
 class NewPasswordForm(ReactorForm):
     new_password = wtforms.PasswordField(
-        'New Password',
+        "New Password",
         filters=[strip_filter],
-        validators=[wtforms.validators.Length(min=4),
-                    wtforms.validators.DataRequired()])
+        validators=[
+            wtforms.validators.Length(min=4),
+            wtforms.validators.DataRequired(),
+        ],
+    )
     new_password_confirm = wtforms.PasswordField(
-        'Confirm Password',
+        "Confirm Password",
         filters=[strip_filter],
-        validators=[wtforms.validators.EqualTo('new_password'),
-                    wtforms.validators.DataRequired()])
-    submit = wtforms.SubmitField('Set Password')
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
+        validators=[
+            wtforms.validators.EqualTo("new_password"),
+            wtforms.validators.DataRequired(),
+        ],
+    )
+    submit = wtforms.SubmitField("Set Password")
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
 
 
 class CORSTextAreaField(wtforms.StringField):
     """
     This field represents an HTML ``<textarea>`` and can be used to take
     multi-line input.
     """
+
     widget = wtforms.widgets.TextArea()
 
     def process_formdata(self, valuelist):
         self.data = []
         if valuelist:
-            data = [x.strip() for x in valuelist[0].split('\n')]
+            data = [x.strip() for x in valuelist[0].split("\n")]
             for d in data:
                 if not d:
                     continue
-                if d.startswith('www.'):
+                if d.startswith("www."):
                     d = d[4:]
                 if data:
                     self.data.append(d)
         else:
             self.data = []
-        self.data = '\n'.join(self.data)
+        self.data = "\n".join(self.data)
 
 
 class ApplicationCreateForm(ReactorForm):
     resource_name = wtforms.StringField(
-        _('Application name'),
+        _("Application name"),
         filters=[strip_filter],
-        validators=[wtforms.validators.Length(min=1),
-                    wtforms.validators.DataRequired()])
+        validators=[
+            wtforms.validators.Length(min=1),
+            wtforms.validators.DataRequired(),
+        ],
+    )
 
     domains = CORSTextAreaField(
-        _('Domain names for CORS headers '),
-        validators=[wtforms.validators.Length(min=1),
-                    wtforms.validators.Optional()],
-        description='Required for Javascript error '
-                    'tracking (one line one domain, skip http:// part)')
+        _("Domain names for CORS headers "),
+        validators=[wtforms.validators.Length(min=1), wtforms.validators.Optional()],
+        description="Required for Javascript error "
+        "tracking (one line one domain, skip http:// part)",
+    )
 
-    submit = wtforms.SubmitField(_('Create Application'))
+    submit = wtforms.SubmitField(_("Create Application"))
 
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
-    html_attrs = {'resource_name': {'placeholder': 'Application Name'},
-                  'uptime_url': {'placeholder': 'http://somedomain.com'}}
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
+    html_attrs = {
+        "resource_name": {"placeholder": "Application Name"},
+        "uptime_url": {"placeholder": "http://somedomain.com"},
+    }
 
 
 class ApplicationUpdateForm(ApplicationCreateForm):
     default_grouping = wtforms.SelectField(
-        _('Default grouping for errors'),
-        choices=[('url_type', 'Error Type + location',),
-                 ('url_traceback', 'Traceback + location',),
-                 ('traceback_server', 'Traceback + Server',)],
-        default='url_traceback')
+        _("Default grouping for errors"),
+        choices=[
+            ("url_type", "Error Type + location"),
+            ("url_traceback", "Traceback + location"),
+            ("traceback_server", "Traceback + Server"),
+        ],
+        default="url_traceback",
+    )
 
     error_report_threshold = wtforms.IntegerField(
-        _('Alert on error reports'),
+        _("Alert on error reports"),
         validators=[
             wtforms.validators.NumberRange(min=1),
-            wtforms.validators.DataRequired()
+            wtforms.validators.DataRequired(),
         ],
-        description='Application requires to send at least this amount of '
-                    'error reports per minute to open alert'
+        description="Application requires to send at least this amount of "
+        "error reports per minute to open alert",
     )
 
     slow_report_threshold = wtforms.IntegerField(
-        _('Alert on slow reports'),
-        validators=[wtforms.validators.NumberRange(min=1),
-                    wtforms.validators.DataRequired()],
-        description='Application requires to send at least this amount of '
-                    'slow reports per minute to open alert')
+        _("Alert on slow reports"),
+        validators=[
+            wtforms.validators.NumberRange(min=1),
+            wtforms.validators.DataRequired(),
+        ],
+        description="Application requires to send at least this amount of "
+        "slow reports per minute to open alert",
+    )
 
     allow_permanent_storage = wtforms.BooleanField(
-        _('Permanent logs'),
+        _("Permanent logs"),
         false_values=FALSE_VALUES,
-        description=_(
-            'Allow permanent storage of logs in separate DB partitions'))
+        description=_("Allow permanent storage of logs in separate DB partitions"),
+    )
 
-    submit = wtforms.SubmitField(_('Create Application'))
+    submit = wtforms.SubmitField(_("Create Application"))
 
 
 class UserSearchSchemaForm(ReactorForm):
-    user_name = wtforms.StringField('User Name',
-                                    filters=[strip_filter], )
+    user_name = wtforms.StringField("User Name", filters=[strip_filter])
 
-    submit = wtforms.SubmitField(_('Search User'))
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
+    submit = wtforms.SubmitField(_("Search User"))
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
 
     '<li class="user_exists"><span></span></li>'
 
 
 class YesNoForm(ReactorForm):
-    no = wtforms.SubmitField('No', default='')
-    yes = wtforms.SubmitField('Yes', default='')
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
+    no = wtforms.SubmitField("No", default="")
+    yes = wtforms.SubmitField("Yes", default="")
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
 
 
-status_codes = [('', 'All',), ('500', '500',), ('404', '404',)]
+status_codes = [("", "All"), ("500", "500"), ("404", "404")]
 
-priorities = [('', 'All',)]
+priorities = [("", "All")]
 for i in range(1, 11):
-    priorities.append((str(i), str(i),))
+    priorities.append((str(i), str(i)))
 
-report_status_choices = [('', 'All',),
-                         ('never_reviewed', 'Never revieved',),
-                         ('reviewed', 'Revieved',),
-                         ('public', 'Public',),
-                         ('fixed', 'Fixed',), ]
+report_status_choices = [
+    ("", "All"),
+    ("never_reviewed", "Never revieved"),
+    ("reviewed", "Revieved"),
+    ("public", "Public"),
+    ("fixed", "Fixed"),
+]
 
 
 class ReportBrowserForm(ReactorForm):
-    applications = wtforms.SelectMultipleField('Applications',
-                                               widget=select_multi_checkbox)
-    http_status = wtforms.SelectField('HTTP Status', choices=status_codes)
-    priority = wtforms.SelectField('Priority', choices=priorities, default='')
-    start_date = wtforms.DateField('Start Date')
-    end_date = wtforms.DateField('End Date')
-    error = wtforms.StringField('Error')
-    url_path = wtforms.StringField('URL Path')
-    url_domain = wtforms.StringField('URL Domain')
-    report_status = wtforms.SelectField('Report status',
-                                        choices=report_status_choices,
-                                        default='')
-    submit = wtforms.SubmitField('<span class="glyphicon glyphicon-search">'
-                                 '</span> Filter results',
-                                 widget=button_widget)
-
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
-
-
-slow_report_status_choices = [('', 'All',),
-                              ('never_reviewed', 'Never revieved',),
-                              ('reviewed', 'Revieved',),
-                              ('public', 'Public',), ]
+    applications = wtforms.SelectMultipleField(
+        "Applications", widget=select_multi_checkbox
+    )
+    http_status = wtforms.SelectField("HTTP Status", choices=status_codes)
+    priority = wtforms.SelectField("Priority", choices=priorities, default="")
+    start_date = wtforms.DateField("Start Date")
+    end_date = wtforms.DateField("End Date")
+    error = wtforms.StringField("Error")
+    url_path = wtforms.StringField("URL Path")
+    url_domain = wtforms.StringField("URL Domain")
+    report_status = wtforms.SelectField(
+        "Report status", choices=report_status_choices, default=""
+    )
+    submit = wtforms.SubmitField(
+        '<span class="glyphicon glyphicon-search">' "</span> Filter results",
+        widget=button_widget,
+    )
+
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
+
+
+slow_report_status_choices = [
+    ("", "All"),
+    ("never_reviewed", "Never revieved"),
+    ("reviewed", "Revieved"),
+    ("public", "Public"),
+]
 
 
 class BulkOperationForm(ReactorForm):
-    applications = wtforms.SelectField('Applications')
+    applications = wtforms.SelectField("Applications")
     start_date = wtforms.DateField(
-        'Start Date',
-        default=lambda: datetime.datetime.utcnow() - datetime.timedelta(
-            days=90))
-    end_date = wtforms.DateField('End Date')
+        "Start Date",
+        default=lambda: datetime.datetime.utcnow() - datetime.timedelta(days=90),
+    )
+    end_date = wtforms.DateField("End Date")
     confirm = wtforms.BooleanField(
-        'Confirm operation',
-        validators=[wtforms.validators.DataRequired()])
+        "Confirm operation", validators=[wtforms.validators.DataRequired()]
+    )
 
 
 class LogBrowserForm(ReactorForm):
-    applications = wtforms.SelectMultipleField('Applications',
-                                               widget=select_multi_checkbox)
-    start_date = wtforms.DateField('Start Date')
-    log_level = wtforms.StringField('Log level')
-    message = wtforms.StringField('Message')
-    namespace = wtforms.StringField('Namespace')
+    applications = wtforms.SelectMultipleField(
+        "Applications", widget=select_multi_checkbox
+    )
+    start_date = wtforms.DateField("Start Date")
+    log_level = wtforms.StringField("Log level")
+    message = wtforms.StringField("Message")
+    namespace = wtforms.StringField("Namespace")
     submit = wtforms.SubmitField(
         '<span class="glyphicon glyphicon-search"></span> Filter results',
-        widget=button_widget)
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
+        widget=button_widget,
+    )
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
 
 
 class CommentForm(ReactorForm):
-    body = wtforms.TextAreaField('Comment', validators=[
-        wtforms.validators.Length(min=1),
-        wtforms.validators.DataRequired()
-    ])
-    submit = wtforms.SubmitField('Comment', )
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
+    body = wtforms.TextAreaField(
+        "Comment",
+        validators=[
+            wtforms.validators.Length(min=1),
+            wtforms.validators.DataRequired(),
+        ],
+    )
+    submit = wtforms.SubmitField("Comment")
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
 
 
 class EmailChannelCreateForm(ReactorForm):
-    email = wtforms.StringField(_('Email Address'),
-                                filters=[strip_filter],
-                                validators=[email_validator,
-                                            unique_alert_email_validator,
-                                            wtforms.validators.DataRequired()])
-    submit = wtforms.SubmitField('Add email channel', )
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
+    email = wtforms.StringField(
+        _("Email Address"),
+        filters=[strip_filter],
+        validators=[
+            email_validator,
+            unique_alert_email_validator,
+            wtforms.validators.DataRequired(),
+        ],
+    )
+    submit = wtforms.SubmitField("Add email channel")
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
 
 
 def gen_user_profile_form():
     class UserProfileForm(ReactorForm):
         email = wtforms.StringField(
-            _('Email Address'),
-            validators=[email_validator, wtforms.validators.DataRequired()])
-        first_name = wtforms.StringField(_('First Name'))
-        last_name = wtforms.StringField(_('Last Name'))
-        company_name = wtforms.StringField(_('Company Name'))
-        company_address = wtforms.TextAreaField(_('Company Address'))
-        zip_code = wtforms.StringField(_('ZIP code'))
-        city = wtforms.StringField(_('City'))
-        notifications = wtforms.BooleanField('Account notifications',
-                                             false_values=FALSE_VALUES)
-        submit = wtforms.SubmitField(_('Update Account'))
-        ignore_labels = ['submit']
-        css_classes = {'submit': 'btn btn-primary'}
+            _("Email Address"),
+            validators=[email_validator, wtforms.validators.DataRequired()],
+        )
+        first_name = wtforms.StringField(_("First Name"))
+        last_name = wtforms.StringField(_("Last Name"))
+        company_name = wtforms.StringField(_("Company Name"))
+        company_address = wtforms.TextAreaField(_("Company Address"))
+        zip_code = wtforms.StringField(_("ZIP code"))
+        city = wtforms.StringField(_("City"))
+        notifications = wtforms.BooleanField(
+            "Account notifications", false_values=FALSE_VALUES
+        )
+        submit = wtforms.SubmitField(_("Update Account"))
+        ignore_labels = ["submit"]
+        css_classes = {"submit": "btn btn-primary"}
 
     return UserProfileForm
 
 
 class PurgeAppForm(ReactorForm):
     resource_id = wtforms.HiddenField(
-        'App Id',
-        validators=[wtforms.validators.DataRequired()])
-    days = wtforms.IntegerField(
-        'Days',
-        validators=[wtforms.validators.DataRequired()])
+        "App Id", validators=[wtforms.validators.DataRequired()]
+    )
+    days = wtforms.IntegerField("Days", validators=[wtforms.validators.DataRequired()])
     password = wtforms.PasswordField(
-        'Admin Password',
-        validators=[old_password_validator, wtforms.validators.DataRequired()])
-    submit = wtforms.SubmitField(_('Purge Data'))
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
+        "Admin Password",
+        validators=[old_password_validator, wtforms.validators.DataRequired()],
+    )
+    submit = wtforms.SubmitField(_("Purge Data"))
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
 
 
 class IntegrationRepoForm(ReactorForm):
-    host_name = wtforms.StringField("Service Host", default='')
+    host_name = wtforms.StringField("Service Host", default="")
     user_name = wtforms.StringField(
         "User Name",
         filters=[strip_filter],
-        validators=[wtforms.validators.DataRequired(),
-                    wtforms.validators.Length(min=1)])
+        validators=[
+            wtforms.validators.DataRequired(),
+            wtforms.validators.Length(min=1),
+        ],
+    )
     repo_name = wtforms.StringField(
         "Repo Name",
         filters=[strip_filter],
-        validators=[wtforms.validators.DataRequired(),
-                    wtforms.validators.Length(min=1)])
+        validators=[
+            wtforms.validators.DataRequired(),
+            wtforms.validators.Length(min=1),
+        ],
+    )
 
 
 class IntegrationBitbucketForm(IntegrationRepoForm):
-    host_name = wtforms.StringField("Service Host",
-                                    default='https://bitbucket.org')
+    host_name = wtforms.StringField("Service Host", default="https://bitbucket.org")
 
     def validate_user_name(self, field):
         try:
             request = pyramid.threadlocal.get_current_request()
             client = BitbucketIntegration.create_client(
-                request,
-                self.user_name.data,
-                self.repo_name.data)
+                request, self.user_name.data, self.repo_name.data
+            )
             client.get_assignees()
         except IntegrationException as e:
             raise wtforms.validators.ValidationError(str(e))
 
 
 class IntegrationGithubForm(IntegrationRepoForm):
-    host_name = wtforms.StringField("Service Host",
-                                    default='https://github.com')
+    host_name = wtforms.StringField("Service Host", default="https://github.com")
 
     def validate_user_name(self, field):
         try:
             request = pyramid.threadlocal.get_current_request()
             client = GithubIntegration.create_client(
-                request,
-                self.user_name.data,
-                self.repo_name.data)
+                request, self.user_name.data, self.repo_name.data
+            )
             client.get_assignees()
         except IntegrationException as e:
             raise wtforms.validators.ValidationError(str(e))
             raise wtforms.validators.ValidationError(str(e))
 
 
 def filter_rooms(data):
     if data is not None:
-        rooms = data.split(',')
-        return ','.join([r.strip() for r in rooms])
+        rooms = data.split(",")
+        return ",".join([r.strip() for r in rooms])
 
 
 class IntegrationCampfireForm(ReactorForm):
     account = wtforms.StringField(
-        'Account',
+        "Account",
         filters=[strip_filter],
-        validators=[wtforms.validators.DataRequired()])
+        validators=[wtforms.validators.DataRequired()],
+    )
     api_token = wtforms.StringField(
-        'Api Token',
+        "Api Token",
         filters=[strip_filter],
-        validators=[wtforms.validators.DataRequired()])
-    rooms = wtforms.StringField('Room ID list', filters=[filter_rooms])
+        validators=[wtforms.validators.DataRequired()],
+    )
+    rooms = wtforms.StringField("Room ID list", filters=[filter_rooms])
 
     def validate_api_token(self, field):
         try:
-            client = CampfireIntegration.create_client(self.api_token.data,
-                                                       self.account.data)
+            client = CampfireIntegration.create_client(
+                self.api_token.data, self.account.data
+            )
             client.get_account()
         except IntegrationException as e:
             raise wtforms.validators.ValidationError(str(e))
 
     def validate_rooms(self, field):
         if not field.data:
             return
-        client = CampfireIntegration.create_client(self.api_token.data,
-                                                   self.account.data)
+        client = CampfireIntegration.create_client(
+            self.api_token.data, self.account.data
+        )
 
         try:
-            room_list = [r['id'] for r in client.get_rooms()]
+            room_list = [r["id"] for r in client.get_rooms()]
         except IntegrationException as e:
             raise wtforms.validators.ValidationError(str(e))
 
-        rooms = field.data.split(',')
+        rooms = field.data.split(",")
         if len(rooms) > 3:
-            msg = 'You can use up to 3 room ids'
+            msg = "You can use up to 3 room ids"
             raise wtforms.validators.ValidationError(msg)
         if rooms:
             for room_id in rooms:
                 if int(room_id) not in room_list:
                     msg = "Room %s doesn't exist"
                     raise wtforms.validators.ValidationError(msg % room_id)
                 if not room_id.strip().isdigit():
-                    msg = 'You must use only integers for room ids'
+                    msg = "You must use only integers for room ids"
                     raise wtforms.validators.ValidationError(msg)
 
-    submit = wtforms.SubmitField(_('Connect to Campfire'))
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
+    submit = wtforms.SubmitField(_("Connect to Campfire"))
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
 
 
 def filter_rooms(data):
     if data is not None:
-        rooms = data.split(',')
-        return ','.join([r.strip() for r in rooms])
+        rooms = data.split(",")
+        return ",".join([r.strip() for r in rooms])
 
 
 class IntegrationHipchatForm(ReactorForm):
     api_token = wtforms.StringField(
-        'Api Token',
+        "Api Token",
         filters=[strip_filter],
-        validators=[wtforms.validators.DataRequired()])
+        validators=[wtforms.validators.DataRequired()],
+    )
     rooms = wtforms.StringField(
-        'Room ID list',
+        "Room ID list",
         filters=[filter_rooms],
-        validators=[wtforms.validators.DataRequired()])
+        validators=[wtforms.validators.DataRequired()],
+    )
 
     def validate_rooms(self, field):
         if not field.data:
             return
         client = HipchatIntegration.create_client(self.api_token.data)
-        rooms = field.data.split(',')
+        rooms = field.data.split(",")
         if len(rooms) > 3:
-            msg = 'You can use up to 3 room ids'
+            msg = "You can use up to 3 room ids"
             raise wtforms.validators.ValidationError(msg)
         if rooms:
             for room_id in rooms:
                 if not room_id.strip().isdigit():
-                    msg = 'You must use only integers for room ids'
+                    msg = "You must use only integers for room ids"
                     raise wtforms.validators.ValidationError(msg)
                 try:
-                    client.send({
-                        "message_format": 'text',
-                        "message": "testing for room existence",
-                        "from": "AppEnlight",
-                        "room_id": room_id,
-                        "color": "green"
-                    })
+                    client.send(
+                        {
+                            "message_format": "text",
+                            "message": "testing for room existence",
+                            "from": "AppEnlight",
+                            "room_id": room_id,
+                            "color": "green",
+                        }
+                    )
                 except IntegrationException as exc:
-                    msg = 'Room id: %s exception: %s'
-                    raise wtforms.validators.ValidationError(msg % (room_id,
-                                                                    exc))
+                    msg = "Room id: %s exception: %s"
+                    raise wtforms.validators.ValidationError(msg % (room_id, exc))
 
 
 class IntegrationFlowdockForm(ReactorForm):
-    api_token = wtforms.StringField('API Token',
-                                    filters=[strip_filter],
-                                    validators=[
-                                        wtforms.validators.DataRequired()
-                                    ], )
+    api_token = wtforms.StringField(
+        "API Token",
+        filters=[strip_filter],
+        validators=[wtforms.validators.DataRequired()],
+    )
 
     def validate_api_token(self, field):
         try:
             client = FlowdockIntegration.create_client(self.api_token.data)
             registry = pyramid.threadlocal.get_current_registry()
             payload = {
-                "source": registry.settings['mailing.from_name'],
-                "from_address": registry.settings['mailing.from_email'],
+                "source": registry.settings["mailing.from_name"],
+                "from_address": registry.settings["mailing.from_email"],
                 "subject": "Integration test",
                 "content": "If you can see this it was successful",
                 "tags": ["appenlight"],
-                "link": registry.settings['mailing.app_url']
+                "link": registry.settings["mailing.app_url"],
             }
             client.send_to_inbox(payload)
         except IntegrationException as e:
             raise wtforms.validators.ValidationError(str(e))
 
 
 class IntegrationSlackForm(ReactorForm):
     webhook_url = wtforms.StringField(
-        'Reports webhook',
+        "Reports webhook",
         filters=[strip_filter],
-        validators=[wtforms.validators.DataRequired()])
+        validators=[wtforms.validators.DataRequired()],
+    )
 
     def validate_webhook_url(self, field):
         registry = pyramid.threadlocal.get_current_registry()
         client = SlackIntegration.create_client(field.data)
-        link = "<%s|%s>" % (registry.settings['mailing.app_url'],
-                            registry.settings['mailing.from_name'])
+        link = "<%s|%s>" % (
+            registry.settings["mailing.app_url"],
+            registry.settings["mailing.from_name"],
+        )
         test_data = {
             "username": "AppEnlight",
             "icon_emoji": ":fire:",
             "attachments": [
-                {"fallback": "Testing integration channel: %s" % link,
-                 "pretext": "Testing integration channel:  %s" % link,
-                 "color": "good",
-                 "fields": [
-                     {
-                         "title": "Status",
-                         "value": "Integration is working fine",
-                         "short": False
-                     }
-                 ]}
-            ]
+                {
+                    "fallback": "Testing integration channel: %s" % link,
+                    "pretext": "Testing integration channel:  %s" % link,
+                    "color": "good",
+                    "fields": [
+                        {
+                            "title": "Status",
+                            "value": "Integration is working fine",
+                            "short": False,
+                        }
+                    ],
+                }
+            ],
         }
         try:
             client.make_request(data=test_data)
         except IntegrationException as exc:
             raise wtforms.validators.ValidationError(str(exc))
 
 
 class IntegrationWebhooksForm(ReactorForm):
     reports_webhook = wtforms.StringField(
-        'Reports webhook',
+        "Reports webhook",
         filters=[strip_filter],
-        validators=[wtforms.validators.DataRequired()])
+        validators=[wtforms.validators.DataRequired()],
+    )
     alerts_webhook = wtforms.StringField(
-        'Alerts webhook',
+        "Alerts webhook",
         filters=[strip_filter],
-        validators=[wtforms.validators.DataRequired()])
-    submit = wtforms.SubmitField(_('Setup webhooks'))
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
+        validators=[wtforms.validators.DataRequired()],
+    )
+    submit = wtforms.SubmitField(_("Setup webhooks"))
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
 
 
 class IntegrationJiraForm(ReactorForm):
     host_name = wtforms.StringField(
-        'Server URL',
+        "Server URL",
         filters=[strip_filter],
-        validators=[wtforms.validators.DataRequired()])
+        validators=[wtforms.validators.DataRequired()],
+    )
     user_name = wtforms.StringField(
-        'Username',
+        "Username",
         filters=[strip_filter],
-        validators=[wtforms.validators.DataRequired()])
+        validators=[wtforms.validators.DataRequired()],
+    )
     password = wtforms.PasswordField(
-        'Password',
+        "Password",
         filters=[strip_filter],
-        validators=[wtforms.validators.DataRequired()])
+        validators=[wtforms.validators.DataRequired()],
+    )
     project = wtforms.StringField(
-        'Project key',
+        "Project key",
         filters=[uppercase_filter, strip_filter],
-        validators=[wtforms.validators.DataRequired()])
+        validators=[wtforms.validators.DataRequired()],
+    )
 
     def validate_project(self, field):
         if not field.data:
             return
         try:
-            client = JiraClient(self.user_name.data,
-                                self.password.data,
-                                self.host_name.data,
-                                self.project.data)
+            client = JiraClient(
+                self.user_name.data,
+                self.password.data,
+                self.host_name.data,
+                self.project.data,
+            )
         except Exception as exc:
             raise wtforms.validators.ValidationError(str(exc))
 
         room_list = [r.key.upper() for r in client.get_projects()]
         if field.data.upper() not in room_list:
             msg = "Project %s doesn\t exist in your Jira Instance"
             raise wtforms.validators.ValidationError(msg % field.data)
 
 
 def get_deletion_form(resource):
     class F(ReactorForm):
         application_name = wtforms.StringField(
-            'Application Name',
+            "Application Name",
             filters=[strip_filter],
-            validators=[wtforms.validators.AnyOf([resource.resource_name])])
+            validators=[wtforms.validators.AnyOf([resource.resource_name])],
+        )
         resource_id = wtforms.HiddenField(default=resource.resource_id)
-        submit = wtforms.SubmitField(_('Delete my application'))
-        ignore_labels = ['submit']
-        css_classes = {'submit': 'btn btn-danger'}
+        submit = wtforms.SubmitField(_("Delete my application"))
+        ignore_labels = ["submit"]
+        css_classes = {"submit": "btn btn-danger"}
 
     return F
 
 
 class ChangeApplicationOwnerForm(ReactorForm):
     password = wtforms.PasswordField(
-        'Password',
+        "Password",
         filters=[strip_filter],
-        validators=[old_password_validator,
-                    wtforms.validators.DataRequired()])
+        validators=[old_password_validator, wtforms.validators.DataRequired()],
+    )
 
     user_name = wtforms.StringField(
-        'New owners username',
+        "New owners username",
         filters=[strip_filter],
-        validators=[found_username_validator,
-                    wtforms.validators.DataRequired()])
-    submit = wtforms.SubmitField(_('Transfer ownership of application'))
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-danger'}
+        validators=[found_username_validator, wtforms.validators.DataRequired()],
+    )
+    submit = wtforms.SubmitField(_("Transfer ownership of application"))
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-danger"}
 
 
 def default_filename():
-    return 'Invoice %s' % datetime.datetime.utcnow().strftime('%Y/%m')
+    return "Invoice %s" % datetime.datetime.utcnow().strftime("%Y/%m")
 
 
 class FileUploadForm(ReactorForm):
-    title = wtforms.StringField('File Title',
-                                default=default_filename,
-                                validators=[wtforms.validators.DataRequired()])
-    file = wtforms.FileField('File')
+    title = wtforms.StringField(
+        "File Title",
+        default=default_filename,
+        validators=[wtforms.validators.DataRequired()],
+    )
+    file = wtforms.FileField("File")
 
     def validate_file(self, field):
-        if not hasattr(field.data, 'file'):
-            raise wtforms.ValidationError('File is missing')
+        if not hasattr(field.data, "file"):
+            raise wtforms.ValidationError("File is missing")
 
-    submit = wtforms.SubmitField(_('Upload'))
+    submit = wtforms.SubmitField(_("Upload"))
 
 
 def get_partition_deletion_form(es_indices, pg_indices):
     class F(ReactorForm):
-        es_index = wtforms.SelectMultipleField('Elasticsearch',
-                                               choices=[(ix, '') for ix in
-                                                        es_indices])
-        pg_index = wtforms.SelectMultipleField('pg',
-                                               choices=[(ix, '') for ix in
-                                                        pg_indices])
-        confirm = wtforms.TextField('Confirm',
-                                    filters=[uppercase_filter, strip_filter],
-                                    validators=[
-                                        wtforms.validators.AnyOf(['CONFIRM']),
-                                        wtforms.validators.DataRequired()])
-        ignore_labels = ['submit']
-        css_classes = {'submit': 'btn btn-danger'}
+        es_index = wtforms.SelectMultipleField(
+            "Elasticsearch", choices=[(ix, "") for ix in es_indices]
+        )
+        pg_index = wtforms.SelectMultipleField(
+            "pg", choices=[(ix, "") for ix in pg_indices]
+        )
+        confirm = wtforms.TextField(
+            "Confirm",
+            filters=[uppercase_filter, strip_filter],
+            validators=[
+                wtforms.validators.AnyOf(["CONFIRM"]),
+                wtforms.validators.DataRequired(),
+            ],
+        )
+        ignore_labels = ["submit"]
+        css_classes = {"submit": "btn btn-danger"}
 
     return F
 
 
 class GroupCreateForm(ReactorForm):
     group_name = wtforms.StringField(
-        _('Group Name'),
+        _("Group Name"),
         filters=[strip_filter],
         validators=[
             wtforms.validators.Length(min=2, max=50),
             unique_groupname_validator,
-            wtforms.validators.DataRequired()
-        ])
-    description = wtforms.StringField(_('Group description'))
+            wtforms.validators.DataRequired(),
+        ],
+    )
+    description = wtforms.StringField(_("Group description"))
 
 
-time_choices = [(k, v['label'],) for k, v in h.time_deltas.items()]
+time_choices = [(k, v["label"]) for k, v in h.time_deltas.items()]
 
 
 class AuthTokenCreateForm(ReactorForm):
-    description = wtforms.StringField(_('Token description'))
-    expires = wtforms.SelectField('Expires',
-                                  coerce=lambda x: x,
-                                  choices=time_choices,
-                                  validators=[wtforms.validators.Optional()])
+    description = wtforms.StringField(_("Token description"))
+    expires = wtforms.SelectField(
+        "Expires",
+        coerce=lambda x: x,
+        choices=time_choices,
+        validators=[wtforms.validators.Optional()],
+    )

backend/src/appenlight/lib/__init__.py

@@ -1,50 +1,49 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 """Miscellaneous support packages for {{project}}.
 """
 import random
 import string
 import importlib
 
 from appenlight_client.exceptions import get_current_traceback
 
 
 def generate_random_string(chars=10):
-    return ''.join(random.sample(string.ascii_letters * 2 + string.digits,
-                                 chars))
+    return "".join(random.sample(string.ascii_letters * 2 + string.digits, chars))
 
 
 def to_integer_safe(input):
     try:
         return int(input)
-    except (TypeError, ValueError,):
+    except (TypeError, ValueError):
         return None
 
 
 def print_traceback(log):
-    traceback = get_current_traceback(skip=1, show_hidden_frames=True,
-                                      ignore_system_exceptions=True)
+    traceback = get_current_traceback(
+        skip=1, show_hidden_frames=True, ignore_system_exceptions=True
+    )
     exception_text = traceback.exception
     log.error(exception_text)
     log.error(traceback.plaintext)
     del traceback
 
 
 def get_callable(import_string):
-    import_module, indexer_callable = import_string.split(':')
-    return getattr(importlib.import_module(import_module),
-                   indexer_callable)
+    import_module, indexer_callable = import_string.split(":")
+    return getattr(importlib.import_module(import_module), indexer_callable)

backend/src/appenlight/lib/api.py

@@ -1,80 +1,80 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import datetime
 import logging
 
 from pyramid.httpexceptions import HTTPForbidden, HTTPTooManyRequests
 
 from appenlight.models.services.config import ConfigService
 from appenlight.lib.redis_keys import REDIS_KEYS
 
 log = logging.getLogger(__name__)
 
 
 def rate_limiting(request, resource, section, to_increment=1):
     tsample = datetime.datetime.utcnow().replace(second=0, microsecond=0)
-    key = REDIS_KEYS['rate_limits'][section].format(tsample,
-                                                    resource.resource_id)
+    key = REDIS_KEYS["rate_limits"][section].format(tsample, resource.resource_id)
     redis_pipeline = request.registry.redis_conn.pipeline()
     redis_pipeline.incr(key, to_increment)
     redis_pipeline.expire(key, 3600 * 24)
     results = redis_pipeline.execute()
     current_count = results[0]
-    config = ConfigService.by_key_and_section(section, 'global')
+    config = ConfigService.by_key_and_section(section, "global")
     limit = config.value if config else 1000
     if current_count > int(limit):
-        log.info('RATE LIMITING: {}: {}, {}'.format(
-            section, resource, current_count))
-        abort_msg = 'Rate limits are in effect for this application'
-        raise HTTPTooManyRequests(abort_msg,
-                                  headers={'X-AppEnlight': abort_msg})
+        log.info("RATE LIMITING: {}: {}, {}".format(section, resource, current_count))
+        abort_msg = "Rate limits are in effect for this application"
+        raise HTTPTooManyRequests(abort_msg, headers={"X-AppEnlight": abort_msg})
 
 
 def check_cors(request, application, should_return=True):
     """
     Performs a check and validation if request comes from authorized domain for
     application, otherwise return 403
     """
     origin_found = False
-    origin = request.headers.get('Origin')
+    origin = request.headers.get("Origin")
     if should_return:
-        log.info('CORS for %s' % origin)
+        log.info("CORS for %s" % origin)
     if not origin:
         return False
-    for domain in application.domains.split('\n'):
+    for domain in application.domains.split("\n"):
         if domain in origin:
             origin_found = True
     if origin_found:
-        request.response.headers.add('Access-Control-Allow-Origin', origin)
-        request.response.headers.add('XDomainRequestAllowed', '1')
-        request.response.headers.add('Access-Control-Allow-Methods',
-                                     'GET, POST, OPTIONS')
-        request.response.headers.add('Access-Control-Allow-Headers',
-                                     'Accept-Encoding, Accept-Language, '
-                                     'Content-Type, '
-                                     'Depth, User-Agent, X-File-Size, '
-                                     'X-Requested-With, If-Modified-Since, '
-                                     'X-File-Name, '
-                                     'Cache-Control, Host, Pragma, Accept, '
-                                     'Origin, Connection, '
-                                     'Referer, Cookie, '
-                                     'X-appenlight-public-api-key, '
-                                     'x-appenlight-public-api-key')
-        request.response.headers.add('Access-Control-Max-Age', '86400')
+        request.response.headers.add("Access-Control-Allow-Origin", origin)
+        request.response.headers.add("XDomainRequestAllowed", "1")
+        request.response.headers.add(
+            "Access-Control-Allow-Methods", "GET, POST, OPTIONS"
+        )
+        request.response.headers.add(
+            "Access-Control-Allow-Headers",
+            "Accept-Encoding, Accept-Language, "
+            "Content-Type, "
+            "Depth, User-Agent, X-File-Size, "
+            "X-Requested-With, If-Modified-Since, "
+            "X-File-Name, "
+            "Cache-Control, Host, Pragma, Accept, "
+            "Origin, Connection, "
+            "Referer, Cookie, "
+            "X-appenlight-public-api-key, "
+            "x-appenlight-public-api-key",
+        )
+        request.response.headers.add("Access-Control-Max-Age", "86400")
         return request.response
     else:
         return HTTPForbidden()

backend/src/appenlight/lib/cache_regions.py

@@ -1,184 +1,169 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import copy
 import hashlib
 import inspect
 
 from dogpile.cache import make_region
 from dogpile.cache.util import compat
 
 regions = None
 
 
 def key_mangler(key):
     return "appenlight:dogpile:{}".format(key)
 
 
 def hashgen(namespace, fn, to_str=compat.string_type):
     """Return a function that generates a string
     key, based on a given function as well as
     arguments to the returned function itself.
 
     This is used by :meth:`.CacheRegion.cache_on_arguments`
     to generate a cache key from a decorated function.
 
     It can be replaced using the ``function_key_generator``
     argument passed to :func:`.make_region`.
 
     """
 
     if namespace is None:
-        namespace = '%s:%s' % (fn.__module__, fn.__name__)
+        namespace = "%s:%s" % (fn.__module__, fn.__name__)
     else:
-        namespace = '%s:%s|%s' % (fn.__module__, fn.__name__, namespace)
+        namespace = "%s:%s|%s" % (fn.__module__, fn.__name__, namespace)
 
     args = inspect.getargspec(fn)
-    has_self = args[0] and args[0][0] in ('self', 'cls')
+    has_self = args[0] and args[0][0] in ("self", "cls")
 
     def generate_key(*args, **kw):
         if kw:
             raise ValueError(
                 "dogpile.cache's default key creation "
-                "function does not accept keyword arguments.")
+                "function does not accept keyword arguments."
+            )
         if has_self:
             args = args[1:]
 
-        return namespace + "|" + hashlib.sha1(
-            " ".join(map(to_str, args)).encode('utf8')).hexdigest()
+        return (
+            namespace
+            + "|"
+            + hashlib.sha1(" ".join(map(to_str, args)).encode("utf8")).hexdigest()
+        )
 
     return generate_key
 
 
 class CacheRegions(object):
     def __init__(self, settings):
         config_redis = {"arguments": settings}
 
         self.redis_min_1 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.redis",
-            expiration_time=60,
-            **copy.deepcopy(config_redis))
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure(
+            "dogpile.cache.redis", expiration_time=60, **copy.deepcopy(config_redis)
+        )
         self.redis_min_5 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.redis",
-            expiration_time=300,
-            **copy.deepcopy(config_redis))
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure(
+            "dogpile.cache.redis", expiration_time=300, **copy.deepcopy(config_redis)
+        )
 
         self.redis_min_10 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.redis",
-            expiration_time=60,
-            **copy.deepcopy(config_redis))
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure(
+            "dogpile.cache.redis", expiration_time=60, **copy.deepcopy(config_redis)
+        )
 
         self.redis_min_60 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.redis",
-            expiration_time=3600,
-            **copy.deepcopy(config_redis))
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure(
+            "dogpile.cache.redis", expiration_time=3600, **copy.deepcopy(config_redis)
+        )
 
         self.redis_sec_1 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.redis",
-            expiration_time=1,
-            **copy.deepcopy(config_redis))
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure(
+            "dogpile.cache.redis", expiration_time=1, **copy.deepcopy(config_redis)
+        )
 
         self.redis_sec_5 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.redis",
-            expiration_time=5,
-            **copy.deepcopy(config_redis))
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure(
+            "dogpile.cache.redis", expiration_time=5, **copy.deepcopy(config_redis)
+        )
 
         self.redis_sec_30 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.redis",
-            expiration_time=30,
-            **copy.deepcopy(config_redis))
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure(
+            "dogpile.cache.redis", expiration_time=30, **copy.deepcopy(config_redis)
+        )
 
         self.redis_day_1 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.redis",
-            expiration_time=86400,
-            **copy.deepcopy(config_redis))
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure(
+            "dogpile.cache.redis", expiration_time=86400, **copy.deepcopy(config_redis)
+        )
 
         self.redis_day_7 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure(
             "dogpile.cache.redis",
             expiration_time=86400 * 7,
-            **copy.deepcopy(config_redis))
+            **copy.deepcopy(config_redis)
+        )
 
         self.redis_day_30 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure(
             "dogpile.cache.redis",
             expiration_time=86400 * 30,
-            **copy.deepcopy(config_redis))
+            **copy.deepcopy(config_redis)
+        )
 
         self.memory_day_1 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.memory",
-            expiration_time=86400,
-            **copy.deepcopy(config_redis))
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure(
+            "dogpile.cache.memory", expiration_time=86400, **copy.deepcopy(config_redis)
+        )
 
         self.memory_sec_1 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.memory",
-            expiration_time=1)
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure("dogpile.cache.memory", expiration_time=1)
 
         self.memory_sec_5 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.memory",
-            expiration_time=5)
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure("dogpile.cache.memory", expiration_time=5)
 
         self.memory_min_1 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.memory",
-            expiration_time=60)
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure("dogpile.cache.memory", expiration_time=60)
 
         self.memory_min_5 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.memory",
-            expiration_time=300)
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure("dogpile.cache.memory", expiration_time=300)
 
         self.memory_min_10 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.memory",
-            expiration_time=600)
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure("dogpile.cache.memory", expiration_time=600)
 
         self.memory_min_60 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.memory",
-            expiration_time=3600)
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure("dogpile.cache.memory", expiration_time=3600)
 
 
 def get_region(region):
     return getattr(regions, region)

backend/src/appenlight/lib/configurator.py

@@ -1,92 +1,92 b''
 import inspect
 import logging
 
 from pyramid.config import Configurator
 
 log = logging.getLogger(__name__)
 
+
 class InspectProxy(object):
     """
     Proxy to the `inspect` module that allows us to use the pyramid include
     mechanism for cythonized modules without source file.
     """
 
     def _get_cyfunction_func_code(self, cyfunction):
         """
         Unpack the `func_code` attribute of a cython function.
         """
         if inspect.ismethod(cyfunction):
             cyfunction = cyfunction.im_func
-        return getattr(cyfunction, 'func_code')
+        return getattr(cyfunction, "func_code")
 
     def getmodule(self, *args, **kwds):
         """
         Simple proxy to `inspect.getmodule`.
         """
         return inspect.getmodule(*args, **kwds)
 
     def getsourcefile(self, obj):
         """
         Proxy to `inspect.getsourcefile` or `inspect.getfile` depending on if
         it's called to look up the source file that contains the magic pyramid
         `includeme` callable.
 
         For cythonized modules the source file may be deleted. Therefore we
         return the result of `inspect.getfile` instead. In the case of the
         `configurator.include` method this is OK, because the result is passed
         to `os.path.dirname` which strips the file name. So it doesn't matter
         if we return the path to the source file or another file in the same
         directory.
         """
         # Check if it's called to look up the source file that contains the
         # magic pyramid `includeme` callable.
-        if getattr(obj, '__name__') == 'includeme':
+        if getattr(obj, "__name__") == "includeme":
             try:
                 return inspect.getfile(obj)
             except TypeError as e:
                 # Cython functions are not recognized as functions by the
                 # inspect module. We have to unpack the func_code attribute
                 # ourself.
-                if 'cyfunction' in e.message:
+                if "cyfunction" in e.message:
                     obj = self._get_cyfunction_func_code(obj)
                     return inspect.getfile(obj)
                 raise
         else:
             return inspect.getsourcefile(obj)
 
 
 class CythonCompatConfigurator(Configurator):
     """
     Customized configurator to replace the inspect class attribute with
     a custom one that is cython compatible.
     """
+
     inspect = InspectProxy()
 
 
 def register_appenlight_plugin(config, plugin_name, plugin_config):
     def register():
-        log.warning('Registering plugin: {}'.format(plugin_name))
+        log.warning("Registering plugin: {}".format(plugin_name))
         if plugin_name not in config.registry.appenlight_plugins:
             config.registry.appenlight_plugins[plugin_name] = {
-                'javascript': None,
-                'static': None,
-                'css': None,
-                'celery_tasks': None,
-                'celery_beats': None,
-                'fulltext_indexer': None,
-                'sqlalchemy_migrations': None,
-                'default_values_setter': None,
-                'header_html': None,
-                'resource_types': [],
-                'url_gen': None
+                "javascript": None,
+                "static": None,
+                "css": None,
+                "celery_tasks": None,
+                "celery_beats": None,
+                "fulltext_indexer": None,
+                "sqlalchemy_migrations": None,
+                "default_values_setter": None,
+                "header_html": None,
+                "resource_types": [],
+                "url_gen": None,
             }
-        config.registry.appenlight_plugins[plugin_name].update(
-            plugin_config)
+        config.registry.appenlight_plugins[plugin_name].update(plugin_config)
         # inform AE what kind of resource types we have available
         # so we can avoid failing when a plugin is removed but data
         # is still present in the db
-        if plugin_config.get('resource_types'):
-            config.registry.resource_types.extend(
-                plugin_config['resource_types'])
+        if plugin_config.get("resource_types"):
+            config.registry.resource_types.extend(plugin_config["resource_types"])
 
-    config.action('appenlight_plugin={}'.format(plugin_name), register)
+    config.action("appenlight_plugin={}".format(plugin_name), register)

backend/src/appenlight/lib/encryption.py

@@ -1,58 +1,58 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 # this gets set on runtime
 from cryptography.fernet import Fernet
 
 ENCRYPTION_SECRET = None
 
 
 def encrypt_fernet(value):
     # avoid double encryption
     # not sure if this is needed but it won't hurt too much to have this
-    if value.startswith('enc$fernet$'):
+    if value.startswith("enc$fernet$"):
         return value
     f = Fernet(ENCRYPTION_SECRET)
-    return 'enc$fernet${}'.format(f.encrypt(value.encode('utf8')).decode('utf8'))
+    return "enc$fernet${}".format(f.encrypt(value.encode("utf8")).decode("utf8"))
 
 
 def decrypt_fernet(value):
-    parts = value.split('$', 3)
+    parts = value.split("$", 3)
     if not len(parts) == 3:
         # not encrypted values
         return value
     else:
         f = Fernet(ENCRYPTION_SECRET)
-        decrypted_data = f.decrypt(parts[2].encode('utf8')).decode('utf8')
+        decrypted_data = f.decrypt(parts[2].encode("utf8")).decode("utf8")
         return decrypted_data
 
 
 def encrypt_dictionary_keys(_dict, exclude_keys=None):
     if not exclude_keys:
         exclude_keys = []
     keys = [k for k in _dict.keys() if k not in exclude_keys]
     for k in keys:
         _dict[k] = encrypt_fernet(_dict[k])
     return _dict
 
 
 def decrypt_dictionary_keys(_dict, exclude_keys=None):
     if not exclude_keys:
         exclude_keys = []
     keys = [k for k in _dict.keys() if k not in exclude_keys]
     for k in keys:
         _dict[k] = decrypt_fernet(_dict[k])
     return _dict

backend/src/appenlight/lib/enums.py

@@ -1,88 +1,90 b''
 import collections
+
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 
 class StupidEnum(object):
     @classmethod
     def set_inverse(cls):
         cls._inverse_values = dict(
-            (y, x) for x, y in vars(cls).items() if
-            not x.startswith('_') and not callable(y)
+            (y, x)
+            for x, y in vars(cls).items()
+            if not x.startswith("_") and not callable(y)
         )
 
     @classmethod
     def key_from_value(cls, value):
-        if not hasattr(cls, '_inverse_values'):
+        if not hasattr(cls, "_inverse_values"):
             cls.set_inverse()
         return cls._inverse_values.get(value)
 
 
 class ReportType(StupidEnum):
     unknown = 0
     error = 1
     not_found = 2
     slow = 3
 
 
 class Language(StupidEnum):
     unknown = 0
     python = 1
     javascript = 2
     java = 3
     objectivec = 4
     swift = 5
     cpp = 6
     basic = 7
     csharp = 8
     php = 9
     perl = 10
     vb = 11
     vbnet = 12
     ruby = 13
     fsharp = 14
     actionscript = 15
     go = 16
     scala = 17
     haskell = 18
     erlang = 19
     haxe = 20
     scheme = 21
 
 
 class LogLevel(StupidEnum):
     UNKNOWN = 0
     DEBUG = 2
     TRACE = 4
     INFO = 6
     WARNING = 8
     ERROR = 10
     CRITICAL = 12
     FATAL = 14
 
 
 class LogLevelPython(StupidEnum):
     CRITICAL = 50
     ERROR = 40
     WARNING = 30
     INFO = 20
     DEBUG = 10
     NOTSET = 0
 
 
 class ParsedSentryEventType(StupidEnum):
     ERROR_REPORT = 1
     LOG = 2

backend/src/appenlight/lib/ext_json.py

@@ -1,148 +1,143 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 """
 ex-json borrowed from Marcin Kuzminski
 
 source: https://secure.rhodecode.org/ext-json
 
 """
 import datetime
 import functools
 import decimal
 import imp
 
-__all__ = ['json', 'simplejson', 'stdlibjson']
+__all__ = ["json", "simplejson", "stdlibjson"]
 
 
 def _is_aware(value):
     """
     Determines if a given datetime.time is aware.
 
     The logic is described in Python's docs:
     http://docs.python.org/library/datetime.html#datetime.tzinfo
     """
-    return (value.tzinfo is not None
-            and value.tzinfo.utcoffset(value) is not None)
+    return value.tzinfo is not None and value.tzinfo.utcoffset(value) is not None
 
 
 def _obj_dump(obj):
     """
     Custom function for dumping objects to JSON, if obj has __json__ attribute
     or method defined it will be used for serialization
 
     :param obj:
     """
 
     if isinstance(obj, complex):
         return [obj.real, obj.imag]
     # See "Date Time String Format" in the ECMA-262 specification.
     # some code borrowed from django 1.4
     elif isinstance(obj, datetime.datetime):
         r = obj.isoformat()
         # if obj.microsecond:
         #     r = r[:23] + r[26:]
-        if r.endswith('+00:00'):
-            r = r[:-6] + 'Z'
+        if r.endswith("+00:00"):
+            r = r[:-6] + "Z"
         return r
     elif isinstance(obj, datetime.date):
         return obj.isoformat()
     elif isinstance(obj, decimal.Decimal):
         return str(obj)
     elif isinstance(obj, datetime.time):
         if _is_aware(obj):
             raise ValueError("JSON can't represent timezone-aware times.")
         r = obj.isoformat()
         if obj.microsecond:
             r = r[:12]
         return r
     elif isinstance(obj, set):
         return list(obj)
-    elif hasattr(obj, '__json__'):
+    elif hasattr(obj, "__json__"):
         if callable(obj.__json__):
             return obj.__json__()
         else:
             return obj.__json__
     else:
         raise NotImplementedError
 
 
 # Import simplejson
 try:
     # import simplejson initially
-    _sj = imp.load_module('_sj', *imp.find_module('simplejson'))
-
+    _sj = imp.load_module("_sj", *imp.find_module("simplejson"))
 
     def extended_encode(obj):
         try:
             return _obj_dump(obj)
         except NotImplementedError:
             pass
         raise TypeError("%r is not JSON serializable" % (obj,))
 
-
     # we handle decimals our own it makes unified behavior of json vs
     # simplejson
-    sj_version = [int(x) for x in _sj.__version__.split('.')]
+    sj_version = [int(x) for x in _sj.__version__.split(".")]
     major, minor = sj_version[0], sj_version[1]
     if major < 2 or (major == 2 and minor < 1):
         # simplejson < 2.1 doesnt support use_decimal
-        _sj.dumps = functools.partial(
-            _sj.dumps, default=extended_encode)
-        _sj.dump = functools.partial(
-            _sj.dump, default=extended_encode)
+        _sj.dumps = functools.partial(_sj.dumps, default=extended_encode)
+        _sj.dump = functools.partial(_sj.dump, default=extended_encode)
     else:
         _sj.dumps = functools.partial(
-            _sj.dumps, default=extended_encode, use_decimal=False)
+            _sj.dumps, default=extended_encode, use_decimal=False
+        )
         _sj.dump = functools.partial(
-            _sj.dump, default=extended_encode, use_decimal=False)
+            _sj.dump, default=extended_encode, use_decimal=False
+        )
     simplejson = _sj
 
 except ImportError:
     # no simplejson set it to None
     simplejson = None
 
 try:
     # simplejson not found try out regular json module
-    _json = imp.load_module('_json', *imp.find_module('json'))
-
+    _json = imp.load_module("_json", *imp.find_module("json"))
 
     # extended JSON encoder for json
     class ExtendedEncoder(_json.JSONEncoder):
         def default(self, obj):
             try:
                 return _obj_dump(obj)
             except NotImplementedError:
                 pass
             raise TypeError("%r is not JSON serializable" % (obj,))
 
-
     # monkey-patch JSON encoder to use extended version
     _json.dumps = functools.partial(_json.dumps, cls=ExtendedEncoder)
     _json.dump = functools.partial(_json.dump, cls=ExtendedEncoder)
 
 except ImportError:
     json = None
 
 stdlibjson = _json
 
 # set all available json modules
 if simplejson:
     json = _sj
 elif _json:
     json = _json
 else:
-    raise ImportError('Could not find any json modules')
+    raise ImportError("Could not find any json modules")

backend/src/appenlight/lib/helpers.py

@@ -1,119 +1,160 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 """
 Helper functions
 """
 import copy
 import datetime
 
 from collections import namedtuple, OrderedDict
 
 _ = lambda x: x
 
 time_deltas = OrderedDict()
 
-time_deltas['1m'] = {'delta': datetime.timedelta(minutes=1),
-                     'label': '1 minute', 'minutes': 1}
-
-time_deltas['5m'] = {'delta': datetime.timedelta(minutes=5),
-                     'label': '5 minutes', 'minutes': 5}
-time_deltas['30m'] = {'delta': datetime.timedelta(minutes=30),
-                      'label': '30 minutes', 'minutes': 30}
-time_deltas['1h'] = {'delta': datetime.timedelta(hours=1),
-                     'label': '60 minutes', 'minutes': 60}
-time_deltas['4h'] = {'delta': datetime.timedelta(hours=4), 'label': '4 hours',
-                     'minutes': 60 * 4}
-time_deltas['12h'] = {'delta': datetime.timedelta(hours=12),
-                      'label': '12 hours', 'minutes': 60 * 12}
-time_deltas['24h'] = {'delta': datetime.timedelta(hours=24),
-                      'label': '24 hours', 'minutes': 60 * 24}
-time_deltas['3d'] = {'delta': datetime.timedelta(days=3), 'label': '3 days',
-                     'minutes': 60 * 24 * 3}
-time_deltas['1w'] = {'delta': datetime.timedelta(days=7), 'label': '7 days',
-                     'minutes': 60 * 24 * 7}
-time_deltas['2w'] = {'delta': datetime.timedelta(days=14), 'label': '14 days',
-                     'minutes': 60 * 24 * 14}
-time_deltas['1M'] = {'delta': datetime.timedelta(days=31), 'label': '31 days',
-                     'minutes': 60 * 24 * 31}
-time_deltas['3M'] = {'delta': datetime.timedelta(days=31 * 3),
-                     'label': '3 months',
-                     'minutes': 60 * 24 * 31 * 3}
-time_deltas['6M'] = {'delta': datetime.timedelta(days=31 * 6),
-                     'label': '6 months',
-                     'minutes': 60 * 24 * 31 * 6}
-time_deltas['12M'] = {'delta': datetime.timedelta(days=31 * 12),
-                      'label': '12 months',
-                      'minutes': 60 * 24 * 31 * 12}
+time_deltas["1m"] = {
+    "delta": datetime.timedelta(minutes=1),
+    "label": "1 minute",
+    "minutes": 1,
+}
+
+time_deltas["5m"] = {
+    "delta": datetime.timedelta(minutes=5),
+    "label": "5 minutes",
+    "minutes": 5,
+}
+time_deltas["30m"] = {
+    "delta": datetime.timedelta(minutes=30),
+    "label": "30 minutes",
+    "minutes": 30,
+}
+time_deltas["1h"] = {
+    "delta": datetime.timedelta(hours=1),
+    "label": "60 minutes",
+    "minutes": 60,
+}
+time_deltas["4h"] = {
+    "delta": datetime.timedelta(hours=4),
+    "label": "4 hours",
+    "minutes": 60 * 4,
+}
+time_deltas["12h"] = {
+    "delta": datetime.timedelta(hours=12),
+    "label": "12 hours",
+    "minutes": 60 * 12,
+}
+time_deltas["24h"] = {
+    "delta": datetime.timedelta(hours=24),
+    "label": "24 hours",
+    "minutes": 60 * 24,
+}
+time_deltas["3d"] = {
+    "delta": datetime.timedelta(days=3),
+    "label": "3 days",
+    "minutes": 60 * 24 * 3,
+}
+time_deltas["1w"] = {
+    "delta": datetime.timedelta(days=7),
+    "label": "7 days",
+    "minutes": 60 * 24 * 7,
+}
+time_deltas["2w"] = {
+    "delta": datetime.timedelta(days=14),
+    "label": "14 days",
+    "minutes": 60 * 24 * 14,
+}
+time_deltas["1M"] = {
+    "delta": datetime.timedelta(days=31),
+    "label": "31 days",
+    "minutes": 60 * 24 * 31,
+}
+time_deltas["3M"] = {
+    "delta": datetime.timedelta(days=31 * 3),
+    "label": "3 months",
+    "minutes": 60 * 24 * 31 * 3,
+}
+time_deltas["6M"] = {
+    "delta": datetime.timedelta(days=31 * 6),
+    "label": "6 months",
+    "minutes": 60 * 24 * 31 * 6,
+}
+time_deltas["12M"] = {
+    "delta": datetime.timedelta(days=31 * 12),
+    "label": "12 months",
+    "minutes": 60 * 24 * 31 * 12,
+}
 
 # used in json representation
-time_options = dict([(k, {'label': v['label'], 'minutes': v['minutes']})
-                     for k, v in time_deltas.items()])
-FlashMsg = namedtuple('FlashMsg', ['msg', 'level'])
+time_options = dict(
+    [
+        (k, {"label": v["label"], "minutes": v["minutes"]})
+        for k, v in time_deltas.items()
+    ]
+)
+FlashMsg = namedtuple("FlashMsg", ["msg", "level"])
 
 
 def get_flash(request):
     messages = []
     messages.extend(
-        [FlashMsg(msg, 'error')
-         for msg in request.session.peek_flash('error')])
-    messages.extend([FlashMsg(msg, 'warning')
-                     for msg in request.session.peek_flash('warning')])
+        [FlashMsg(msg, "error") for msg in request.session.peek_flash("error")]
+    )
     messages.extend(
-        [FlashMsg(msg, 'notice') for msg in request.session.peek_flash()])
+        [FlashMsg(msg, "warning") for msg in request.session.peek_flash("warning")]
+    )
+    messages.extend([FlashMsg(msg, "notice") for msg in request.session.peek_flash()])
     return messages
 
 
 def clear_flash(request):
-    request.session.pop_flash('error')
-    request.session.pop_flash('warning')
+    request.session.pop_flash("error")
+    request.session.pop_flash("warning")
     request.session.pop_flash()
 
 
 def get_type_formatted_flash(request):
-    return [{'msg': message.msg, 'type': message.level}
-            for message in get_flash(request)]
+    return [
+        {"msg": message.msg, "type": message.level} for message in get_flash(request)
+    ]
 
 
 def gen_pagination_headers(request, paginator):
     headers = {
-        'x-total-count': str(paginator.item_count),
-        'x-current-page': str(paginator.page),
-        'x-items-per-page': str(paginator.items_per_page)
+        "x-total-count": str(paginator.item_count),
+        "x-current-page": str(paginator.page),
+        "x-items-per-page": str(paginator.items_per_page),
     }
     params_dict = request.GET.dict_of_lists()
     last_page_params = copy.deepcopy(params_dict)
-    last_page_params['page'] = paginator.last_page or 1
+    last_page_params["page"] = paginator.last_page or 1
     first_page_params = copy.deepcopy(params_dict)
-    first_page_params.pop('page', None)
+    first_page_params.pop("page", None)
     next_page_params = copy.deepcopy(params_dict)
-    next_page_params['page'] = paginator.next_page or paginator.last_page or 1
+    next_page_params["page"] = paginator.next_page or paginator.last_page or 1
     prev_page_params = copy.deepcopy(params_dict)
-    prev_page_params['page'] = paginator.previous_page or 1
+    prev_page_params["page"] = paginator.previous_page or 1
     lp_url = request.current_route_url(_query=last_page_params)
     fp_url = request.current_route_url(_query=first_page_params)
-    links = [
-        'rel="last", <{}>'.format(lp_url),
-        'rel="first", <{}>'.format(fp_url),
-    ]
+    links = ['rel="last", <{}>'.format(lp_url), 'rel="first", <{}>'.format(fp_url)]
     if first_page_params != prev_page_params:
         prev_url = request.current_route_url(_query=prev_page_params)
         links.append('rel="prev", <{}>'.format(prev_url))
     if last_page_params != next_page_params:
         next_url = request.current_route_url(_query=next_page_params)
         links.append('rel="next", <{}>'.format(next_url))
-    headers['link'] = '; '.join(links)
+    headers["link"] = "; ".join(links)
     return headers

backend/src/appenlight/lib/jinja2_filters.py

@@ -1,46 +1,53 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import re
 from appenlight.lib.ext_json import json
 from jinja2 import Markup, escape, evalcontextfilter
 
-_paragraph_re = re.compile(r'(?:\r\n|\r|\n){2,}')
+_paragraph_re = re.compile(r"(?:\r\n|\r|\n){2,}")
 
 
 @evalcontextfilter
 def nl2br(eval_ctx, value):
     if eval_ctx.autoescape:
-        result = '\n\n'.join('<p>%s</p>' % p.replace('\n', Markup('<br>\n'))
-                              for p in _paragraph_re.split(escape(value)))
+        result = "\n\n".join(
+            "<p>%s</p>" % p.replace("\n", Markup("<br>\n"))
+            for p in _paragraph_re.split(escape(value))
+        )
     else:
-        result = '\n\n'.join('<p>%s</p>' % p.replace('\n', '<br>\n')
-                              for p in _paragraph_re.split(escape(value)))
+        result = "\n\n".join(
+            "<p>%s</p>" % p.replace("\n", "<br>\n")
+            for p in _paragraph_re.split(escape(value))
+        )
     if eval_ctx.autoescape:
         result = Markup(result)
     return result
 
 
 @evalcontextfilter
 def toJSONUnsafe(eval_ctx, value):
-    encoded = json.dumps(value).replace('&', '\\u0026') \
-        .replace('<', '\\u003c') \
-        .replace('>', '\\u003e') \
-        .replace('>', '\\u003e') \
-        .replace('"', '\\u0022') \
-        .replace("'", '\\u0027') \
-        .replace(r'\n', '/\\\n')
+    encoded = (
+        json.dumps(value)
+        .replace("&", "\\u0026")
+        .replace("<", "\\u003c")
+        .replace(">", "\\u003e")
+        .replace(">", "\\u003e")
+        .replace('"', "\\u0022")
+        .replace("'", "\\u0027")
+        .replace(r"\n", "/\\\n")
+    )
     return Markup("'%s'" % encoded)

backend/src/appenlight/lib/logging.py

@@ -1,64 +1,83 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import json
 import logging
 
-ignored_keys = ['args', 'asctime', 'created', 'exc_info', 'exc_text',
-                'filename', 'funcName', 'levelname', 'levelno', 'lineno',
-                'message', 'module', 'msecs', 'msg', 'name', 'pathname',
-                'process', 'processName', 'relativeCreated', 'stack_info',
-                'thread', 'threadName']
+ignored_keys = [
+    "args",
+    "asctime",
+    "created",
+    "exc_info",
+    "exc_text",
+    "filename",
+    "funcName",
+    "levelname",
+    "levelno",
+    "lineno",
+    "message",
+    "module",
+    "msecs",
+    "msg",
+    "name",
+    "pathname",
+    "process",
+    "processName",
+    "relativeCreated",
+    "stack_info",
+    "thread",
+    "threadName",
+]
 
 
 class JSONFormatter(logging.Formatter):
     def format(self, record):
         """
         Format the specified record as text.
 
         The record's attribute dictionary is used as the operand to a
         string formatting operation which yields the returned string.
         Before formatting the dictionary, a couple of preparatory steps
         are carried out. The message attribute of the record is computed
         using LogRecord.getMessage(). If the formatting string uses the
         time (as determined by a call to usesTime(), formatTime() is
         called to format the event time. If there is exception information,
         it is formatted using formatException() and appended to the message.
         """
         record.message = record.getMessage()
         log_dict = vars(record)
         keys = [k for k in log_dict.keys() if k not in ignored_keys]
-        payload = {'message': record.message}
+        payload = {"message": record.message}
         payload.update({k: log_dict[k] for k in keys})
         record.message = json.dumps(payload, default=lambda x: str(x))
 
         if self.usesTime():
             record.asctime = self.formatTime(record, self.datefmt)
         s = self.formatMessage(record)
         if record.exc_info:
             # Cache the traceback text to avoid converting it multiple times
             # (it's constant anyway)
             if not record.exc_text:
                 record.exc_text = self.formatException(record.exc_info)
         if record.exc_text:
             if s[-1:] != "\n":
                 s = s + "\n"
             s = s + record.exc_text
         if record.stack_info:
             if s[-1:] != "\n":
                 s = s + "\n"
             s = s + self.formatStack(record.stack_info)
         return s

backend/src/appenlight/lib/redis_keys.py

@@ -1,65 +1,69 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-BASE = 'appenlight:data:{}'
+BASE = "appenlight:data:{}"
 
 REDIS_KEYS = {
-    'tasks': {
-        'add_reports_lock': BASE.format('add_reports_lock:{}'),
-        'add_logs_lock': BASE.format('add_logs_lock:{}'),
+    "tasks": {
+        "add_reports_lock": BASE.format("add_reports_lock:{}"),
+        "add_logs_lock": BASE.format("add_logs_lock:{}"),
     },
-    'counters': {
-        'events_per_minute_per_user': BASE.format(
-            'events_per_minute_per_user:{}:{}'),
-        'reports_per_minute': BASE.format('reports_per_minute:{}'),
-        'reports_per_hour_per_app': BASE.format(
-            'reports_per_hour_per_app:{}:{}'),
-        'reports_per_type': BASE.format('reports_per_type:{}'),
-        'logs_per_minute': BASE.format('logs_per_minute:{}'),
-        'logs_per_hour_per_app': BASE.format(
-            'logs_per_hour_per_app:{}:{}'),
-        'metrics_per_minute': BASE.format('metrics_per_minute:{}'),
-        'metrics_per_hour_per_app': BASE.format(
-            'metrics_per_hour_per_app:{}:{}'),
-        'report_group_occurences': BASE.format('report_group_occurences:{}'),
-        'report_group_occurences_alerting': BASE.format(
-            'report_group_occurences_alerting:{}'),
-        'report_group_occurences_10th': BASE.format(
-            'report_group_occurences_10th:{}'),
-        'report_group_occurences_100th': BASE.format(
-            'report_group_occurences_100th:{}'),
+    "counters": {
+        "events_per_minute_per_user": BASE.format("events_per_minute_per_user:{}:{}"),
+        "reports_per_minute": BASE.format("reports_per_minute:{}"),
+        "reports_per_hour_per_app": BASE.format("reports_per_hour_per_app:{}:{}"),
+        "reports_per_type": BASE.format("reports_per_type:{}"),
+        "logs_per_minute": BASE.format("logs_per_minute:{}"),
+        "logs_per_hour_per_app": BASE.format("logs_per_hour_per_app:{}:{}"),
+        "metrics_per_minute": BASE.format("metrics_per_minute:{}"),
+        "metrics_per_hour_per_app": BASE.format("metrics_per_hour_per_app:{}:{}"),
+        "report_group_occurences": BASE.format("report_group_occurences:{}"),
+        "report_group_occurences_alerting": BASE.format(
+            "report_group_occurences_alerting:{}"
+        ),
+        "report_group_occurences_10th": BASE.format("report_group_occurences_10th:{}"),
+        "report_group_occurences_100th": BASE.format(
+            "report_group_occurences_100th:{}"
+        ),
     },
-    'rate_limits': {
-        'per_application_reports_rate_limit': BASE.format(
-            'per_application_reports_limit:{}:{}'),
-        'per_application_logs_rate_limit': BASE.format(
-            'per_application_logs_rate_limit:{}:{}'),
-        'per_application_metrics_rate_limit': BASE.format(
-            'per_application_metrics_rate_limit:{}:{}'),
+    "rate_limits": {
+        "per_application_reports_rate_limit": BASE.format(
+            "per_application_reports_limit:{}:{}"
+        ),
+        "per_application_logs_rate_limit": BASE.format(
+            "per_application_logs_rate_limit:{}:{}"
+        ),
+        "per_application_metrics_rate_limit": BASE.format(
+            "per_application_metrics_rate_limit:{}:{}"
+        ),
     },
-    'apps_that_got_new_data_per_hour': BASE.format('apps_that_got_new_data_per_hour:{}'),
-    'apps_that_had_reports': BASE.format('apps_that_had_reports'),
-    'apps_that_had_error_reports': BASE.format('apps_that_had_error_reports'),
-    'apps_that_had_reports_alerting': BASE.format(
-        'apps_that_had_reports_alerting'),
-    'apps_that_had_error_reports_alerting': BASE.format(
-        'apps_that_had_error_reports_alerting'),
-    'reports_to_notify_per_type_per_app': BASE.format(
-        'reports_to_notify_per_type_per_app:{}:{}'),
-    'reports_to_notify_per_type_per_app_alerting': BASE.format(
-        'reports_to_notify_per_type_per_app_alerting:{}:{}'),
-    'seen_tag_list': BASE.format('seen_tag_list')
+    "apps_that_got_new_data_per_hour": BASE.format(
+        "apps_that_got_new_data_per_hour:{}"
+    ),
+    "apps_that_had_reports": BASE.format("apps_that_had_reports"),
+    "apps_that_had_error_reports": BASE.format("apps_that_had_error_reports"),
+    "apps_that_had_reports_alerting": BASE.format("apps_that_had_reports_alerting"),
+    "apps_that_had_error_reports_alerting": BASE.format(
+        "apps_that_had_error_reports_alerting"
+    ),
+    "reports_to_notify_per_type_per_app": BASE.format(
+        "reports_to_notify_per_type_per_app:{}:{}"
+    ),
+    "reports_to_notify_per_type_per_app_alerting": BASE.format(
+        "reports_to_notify_per_type_per_app_alerting:{}:{}"
+    ),
+    "seen_tag_list": BASE.format("seen_tag_list"),
 }

backend/src/appenlight/lib/request.py

@@ -1,135 +1,131 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import json
 
 from pyramid.security import unauthenticated_userid
 
 import appenlight.lib.helpers as helpers
 
 from authomatic.providers import oauth2, oauth1
 from authomatic import Authomatic
 from ziggurat_foundations.models.services.user import UserService
 
 
 class CSRFException(Exception):
     pass
 
 
 class JSONException(Exception):
     pass
 
 
 def get_csrf_token(request):
     return request.session.get_csrf_token()
 
 
 def safe_json_body(request):
     """
     Returns None if json body is missing or erroneous
     """
     try:
         return request.json_body
     except ValueError:
         return None
 
 
 def unsafe_json_body(request):
     """
     Throws JSONException if json can't deserialize
     """
     try:
         return request.json_body
     except ValueError:
-        raise JSONException('Incorrect JSON')
+        raise JSONException("Incorrect JSON")
 
 
 def get_user(request):
-    if not request.path_info.startswith('/static'):
+    if not request.path_info.startswith("/static"):
         user_id = unauthenticated_userid(request)
         try:
             user_id = int(user_id)
         except Exception:
             return None
 
         if user_id:
             user = UserService.by_id(user_id)
             if user:
-                request.environ['appenlight.username'] = '%d:%s' % (
-                    user_id, user.user_name)
+                request.environ["appenlight.username"] = "%d:%s" % (
+                    user_id,
+                    user.user_name,
+                )
             return user
         else:
             return None
 
 
 def es_conn(request):
     return request.registry.es_conn
 
 
 def add_flash_to_headers(request, clear=True):
     """
     Adds pending flash messages to response, if clear is true clears out the
     flash queue
     """
     flash_msgs = helpers.get_type_formatted_flash(request)
-    request.response.headers['x-flash-messages'] = json.dumps(flash_msgs)
+    request.response.headers["x-flash-messages"] = json.dumps(flash_msgs)
     helpers.clear_flash(request)
 
 
 def get_authomatic(request):
     settings = request.registry.settings
     # authomatic social auth
     authomatic_conf = {
         # callback http://yourapp.com/social_auth/twitter
-        'twitter': {
-            'class_': oauth1.Twitter,
-            'consumer_key': settings.get('authomatic.pr.twitter.key', ''),
-            'consumer_secret': settings.get('authomatic.pr.twitter.secret',
-                                            ''),
+        "twitter": {
+            "class_": oauth1.Twitter,
+            "consumer_key": settings.get("authomatic.pr.twitter.key", ""),
+            "consumer_secret": settings.get("authomatic.pr.twitter.secret", ""),
         },
         # callback http://yourapp.com/social_auth/facebook
-        'facebook': {
-            'class_': oauth2.Facebook,
-            'consumer_key': settings.get('authomatic.pr.facebook.app_id', ''),
-            'consumer_secret': settings.get('authomatic.pr.facebook.secret',
-                                            ''),
-            'scope': ['email'],
+        "facebook": {
+            "class_": oauth2.Facebook,
+            "consumer_key": settings.get("authomatic.pr.facebook.app_id", ""),
+            "consumer_secret": settings.get("authomatic.pr.facebook.secret", ""),
+            "scope": ["email"],
         },
         # callback http://yourapp.com/social_auth/google
-        'google': {
-            'class_': oauth2.Google,
-            'consumer_key': settings.get('authomatic.pr.google.key', ''),
-            'consumer_secret': settings.get(
-                'authomatic.pr.google.secret', ''),
-            'scope': ['profile', 'email'],
+        "google": {
+            "class_": oauth2.Google,
+            "consumer_key": settings.get("authomatic.pr.google.key", ""),
+            "consumer_secret": settings.get("authomatic.pr.google.secret", ""),
+            "scope": ["profile", "email"],
         },
-        'github': {
-            'class_': oauth2.GitHub,
-            'consumer_key': settings.get('authomatic.pr.github.key', ''),
-            'consumer_secret': settings.get(
-                'authomatic.pr.github.secret', ''),
-            'scope': ['repo', 'public_repo', 'user:email'],
-            'access_headers': {'User-Agent': 'AppEnlight'},
+        "github": {
+            "class_": oauth2.GitHub,
+            "consumer_key": settings.get("authomatic.pr.github.key", ""),
+            "consumer_secret": settings.get("authomatic.pr.github.secret", ""),
+            "scope": ["repo", "public_repo", "user:email"],
+            "access_headers": {"User-Agent": "AppEnlight"},
+        },
+        "bitbucket": {
+            "class_": oauth1.Bitbucket,
+            "consumer_key": settings.get("authomatic.pr.bitbucket.key", ""),
+            "consumer_secret": settings.get("authomatic.pr.bitbucket.secret", ""),
         },
-        'bitbucket': {
-            'class_': oauth1.Bitbucket,
-            'consumer_key': settings.get('authomatic.pr.bitbucket.key', ''),
-            'consumer_secret': settings.get(
-                'authomatic.pr.bitbucket.secret', '')
-        }
     }
-    return Authomatic(
-        config=authomatic_conf, secret=settings['authomatic.secret'])
+    return Authomatic(config=authomatic_conf, secret=settings["authomatic.secret"])

backend/src/appenlight/lib/rule.py

@@ -1,298 +1,312 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 import operator
 
 log = logging.getLogger(__name__)
 
 
 class RuleException(Exception):
     pass
 
 
 class KeyNotFoundException(RuleException):
     pass
 
 
 class UnknownTypeException(RuleException):
     pass
 
 
 class BadConfigException(RuleException):
     pass
 
 
 class InvalidValueException(RuleException):
     pass
 
 
 class RuleBase(object):
     @classmethod
     def default_dict_struct_getter(cls, struct, field_name):
         """
         returns a key from dictionary based on field_name, if the name contains
         `:` then it means additional nesting levels should be checked for the
         key so `a:b:c` means return struct['a']['b']['c']
 
         :param struct:
         :param field_name:
         :return:
         """
-        parts = field_name.split(':') if field_name else []
+        parts = field_name.split(":") if field_name else []
         found = struct
         while parts:
             current_key = parts.pop(0)
             found = found.get(current_key)
             if not found and parts:
-                raise KeyNotFoundException('Key not found in structure')
+                raise KeyNotFoundException("Key not found in structure")
         return found
 
     @classmethod
     def default_obj_struct_getter(cls, struct, field_name):
         """
         returns a key from instance based on field_name, if the name contains
         `:` then it means additional nesting levels should be checked for the
         key so `a:b:c` means return struct.a.b.c
 
         :param struct:
         :param field_name:
         :return:
         """
-        parts = field_name.split(':')
+        parts = field_name.split(":")
         found = struct
         while parts:
             current_key = parts.pop(0)
             found = getattr(found, current_key, None)
             if not found and parts:
-                raise KeyNotFoundException('Key not found in structure')
+                raise KeyNotFoundException("Key not found in structure")
         return found
 
     def normalized_type(self, field, value):
         """
         Converts text values from self.conf_value based on type_matrix below
         check_matrix defines what kind of checks we can perform on a field
         value based on field name
         """
         f_type = self.type_matrix.get(field)
         if f_type:
-            cast_to = f_type['type']
+            cast_to = f_type["type"]
         else:
-            raise UnknownTypeException('Unknown type')
+            raise UnknownTypeException("Unknown type")
 
         if value is None:
             return None
 
         try:
-            if cast_to == 'int':
+            if cast_to == "int":
                 return int(value)
-            elif cast_to == 'float':
+            elif cast_to == "float":
                 return float(value)
-            elif cast_to == 'unicode':
+            elif cast_to == "unicode":
                 return str(value)
         except ValueError as exc:
             raise InvalidValueException(exc)
 
 
 class Rule(RuleBase):
-    def __init__(self, config, type_matrix,
-                 struct_getter=RuleBase.default_dict_struct_getter,
-                 config_manipulator=None):
+    def __init__(
+        self,
+        config,
+        type_matrix,
+        struct_getter=RuleBase.default_dict_struct_getter,
+        config_manipulator=None,
+    ):
         """
 
         :param config: dict - contains rule configuration
                               example::
                                     {
                                         "field": "__OR__",
                                         "rules": [
                                             {
                                                 "field": "__AND__",
                                                 "rules": [
                                                     {
                                                         "op": "ge",
                                                         "field": "occurences",
                                                         "value": "10"
                                                     },
                                                     {
                                                         "op": "ge",
                                                         "field": "priority",
                                                         "value": "4"
                                                     }
                                                 ]
                                             },
                                             {
                                                 "op": "eq",
                                                 "field": "http_status",
                                                 "value": "500"
                                             }
                                         ]
                                     }
         :param type_matrix: dict - contains map of type casts
                                    example::
                                         {
                                         'http_status': 'int',
                                         'priority': 'unicode',
                                         }
         :param struct_getter: callable - used to grab the value of field from
                                          the structure passed to match() based
                                          on key, default
 
         """
         self.type_matrix = type_matrix
         self.config = config
         self.struct_getter = struct_getter
         self.config_manipulator = config_manipulator
         if config_manipulator:
             config_manipulator(self)
 
     def subrule_check(self, rule_config, struct):
-        rule = Rule(rule_config, self.type_matrix,
-                    config_manipulator=self.config_manipulator)
+        rule = Rule(
+            rule_config, self.type_matrix, config_manipulator=self.config_manipulator
+        )
         return rule.match(struct)
 
     def match(self, struct):
         """
         Check if rule matched for this specific report
         First tries report value, then tests tags in not found, then finally
         report group
         """
-        field_name = self.config.get('field')
-        test_value = self.config.get('value')
+        field_name = self.config.get("field")
+        test_value = self.config.get("value")
 
         if not field_name:
             return False
 
-        if field_name == '__AND__':
-            rule = AND(self.config['rules'], self.type_matrix,
-                       config_manipulator=self.config_manipulator)
+        if field_name == "__AND__":
+            rule = AND(
+                self.config["rules"],
+                self.type_matrix,
+                config_manipulator=self.config_manipulator,
+            )
             return rule.match(struct)
-        elif field_name == '__OR__':
-            rule = OR(self.config['rules'], self.type_matrix,
-                      config_manipulator=self.config_manipulator)
+        elif field_name == "__OR__":
+            rule = OR(
+                self.config["rules"],
+                self.type_matrix,
+                config_manipulator=self.config_manipulator,
+            )
             return rule.match(struct)
-        elif field_name == '__NOT__':
-            rule = NOT(self.config['rules'], self.type_matrix,
-                      config_manipulator=self.config_manipulator)
+        elif field_name == "__NOT__":
+            rule = NOT(
+                self.config["rules"],
+                self.type_matrix,
+                config_manipulator=self.config_manipulator,
+            )
             return rule.match(struct)
 
         if test_value is None:
             return False
 
         try:
-            struct_value = self.normalized_type(field_name,
-                                                self.struct_getter(struct,
-                                                                   field_name))
+            struct_value = self.normalized_type(
+                field_name, self.struct_getter(struct, field_name)
+            )
         except (UnknownTypeException, InvalidValueException) as exc:
             log.error(str(exc))
             return False
 
         try:
             test_value = self.normalized_type(field_name, test_value)
         except (UnknownTypeException, InvalidValueException) as exc:
             log.error(str(exc))
             return False
 
-        if self.config['op'] not in ('startswith', 'endswith', 'contains'):
+        if self.config["op"] not in ("startswith", "endswith", "contains"):
             try:
-                return getattr(operator,
-                               self.config['op'])(struct_value, test_value)
+                return getattr(operator, self.config["op"])(struct_value, test_value)
             except TypeError:
                 return False
-        elif self.config['op'] == 'startswith':
+        elif self.config["op"] == "startswith":
             return struct_value.startswith(test_value)
-        elif self.config['op'] == 'endswith':
+        elif self.config["op"] == "endswith":
             return struct_value.endswith(test_value)
-        elif self.config['op'] == 'contains':
+        elif self.config["op"] == "contains":
             return test_value in struct_value
-        raise BadConfigException('Invalid configuration, '
-                                 'unknown operator: {}'.format(self.config))
+        raise BadConfigException(
+            "Invalid configuration, " "unknown operator: {}".format(self.config)
+        )
 
     def __repr__(self):
-        return '<Rule {} {}>'.format(self.config.get('field'),
-                                     self.config.get('value'))
+        return "<Rule {} {}>".format(self.config.get("field"), self.config.get("value"))
 
 
 class AND(Rule):
     def __init__(self, rules, *args, **kwargs):
         super(AND, self).__init__({}, *args, **kwargs)
         self.rules = rules
 
     def match(self, struct):
-        return all([self.subrule_check(r_conf, struct) for r_conf
-                    in self.rules])
+        return all([self.subrule_check(r_conf, struct) for r_conf in self.rules])
 
 
 class NOT(Rule):
     def __init__(self, rules, *args, **kwargs):
         super(NOT, self).__init__({}, *args, **kwargs)
         self.rules = rules
 
     def match(self, struct):
-        return all([not self.subrule_check(r_conf, struct) for r_conf
-                    in self.rules])
+        return all([not self.subrule_check(r_conf, struct) for r_conf in self.rules])
 
 
 class OR(Rule):
     def __init__(self, rules, *args, **kwargs):
         super(OR, self).__init__({}, *args, **kwargs)
         self.rules = rules
 
     def match(self, struct):
-        return any([self.subrule_check(r_conf, struct) for r_conf
-                    in self.rules])
+        return any([self.subrule_check(r_conf, struct) for r_conf in self.rules])
 
 
 class RuleService(object):
     @staticmethod
-    def rule_from_config(config, field_mappings, labels_dict,
-                         manipulator_func=None):
+    def rule_from_config(config, field_mappings, labels_dict, manipulator_func=None):
         """
         Returns modified rule with manipulator function
         By default manipulator function replaces field id from labels_dict
         with current field id proper for the rule from fields_mappings
 
         because label X_X id might be pointing different value on next request
         when new term is returned from elasticsearch - this ensures things
         are kept 1:1 all the time
         """
         rev_map = {}
         for k, v in labels_dict.items():
-            rev_map[(v['agg'], v['key'],)] = k
+            rev_map[(v["agg"], v["key"])] = k
 
         if manipulator_func is None:
+
             def label_rewriter_func(rule):
-                field = rule.config.get('field')
-                if not field or rule.config['field'] in ['__OR__',
-                                                         '__AND__', '__NOT__']:
+                field = rule.config.get("field")
+                if not field or rule.config["field"] in [
+                    "__OR__",
+                    "__AND__",
+                    "__NOT__",
+                ]:
                     return
 
-                to_map = field_mappings.get(rule.config['field'])
+                to_map = field_mappings.get(rule.config["field"])
 
                 # we need to replace series field with _AE_NOT_FOUND_ to not match
                 # accidently some other field which happens to have the series that
                 # was used when the alert was created
                 if to_map:
-                    to_replace = rev_map.get((to_map['agg'], to_map['key'],),
-                                             '_AE_NOT_FOUND_')
+                    to_replace = rev_map.get(
+                        (to_map["agg"], to_map["key"]), "_AE_NOT_FOUND_"
+                    )
                 else:
-                    to_replace = '_AE_NOT_FOUND_'
+                    to_replace = "_AE_NOT_FOUND_"
 
-                rule.config['field'] = to_replace
-                rule.type_matrix[to_replace] = {"type": 'float'}
+                rule.config["field"] = to_replace
+                rule.type_matrix[to_replace] = {"type": "float"}
 
             manipulator_func = label_rewriter_func
 
         return Rule(config, {}, config_manipulator=manipulator_func)

backend/src/appenlight/lib/social.py

@@ -1,60 +1,62 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ziggurat_foundations.models.services.external_identity import \
-    ExternalIdentityService
+from ziggurat_foundations.models.services.external_identity import (
+    ExternalIdentityService,
+)
 from appenlight.models.external_identity import ExternalIdentity
 
 
 def handle_social_data(request, user, social_data):
     social_data = social_data
     update_identity = False
 
     extng_id = ExternalIdentityService.by_external_id_and_provider(
-        social_data['user']['id'],
-        social_data['credentials'].provider_name
+        social_data["user"]["id"], social_data["credentials"].provider_name
     )
 
     # fix legacy accounts with wrong google ID
-    if not extng_id and social_data['credentials'].provider_name == 'google':
+    if not extng_id and social_data["credentials"].provider_name == "google":
         extng_id = ExternalIdentityService.by_external_id_and_provider(
-            social_data['user']['email'],
-            social_data['credentials'].provider_name
+            social_data["user"]["email"], social_data["credentials"].provider_name
         )
 
     if extng_id:
         extng_id.delete()
         update_identity = True
 
-    if not social_data['user']['id']:
+    if not social_data["user"]["id"]:
         request.session.flash(
-            'No external user id found? Perhaps permissions for '
-            'authentication are set incorrectly', 'error')
+            "No external user id found? Perhaps permissions for "
+            "authentication are set incorrectly",
+            "error",
+        )
         return False
 
     if not extng_id or update_identity:
         if not update_identity:
-            request.session.flash('Your external identity is now '
-                                  'connected with your account')
+            request.session.flash(
+                "Your external identity is now " "connected with your account"
+            )
         ex_identity = ExternalIdentity()
-        ex_identity.external_id = social_data['user']['id']
-        ex_identity.external_user_name = social_data['user']['user_name']
-        ex_identity.provider_name = social_data['credentials'].provider_name
-        ex_identity.access_token = social_data['credentials'].token
-        ex_identity.token_secret = social_data['credentials'].token_secret
-        ex_identity.alt_token = social_data['credentials'].refresh_token
+        ex_identity.external_id = social_data["user"]["id"]
+        ex_identity.external_user_name = social_data["user"]["user_name"]
+        ex_identity.provider_name = social_data["credentials"].provider_name
+        ex_identity.access_token = social_data["credentials"].token
+        ex_identity.token_secret = social_data["credentials"].token_secret
+        ex_identity.alt_token = social_data["credentials"].refresh_token
         user.external_identities.append(ex_identity)
-        request.session.pop('zigg.social_auth', None)
+        request.session.pop("zigg.social_auth", None)

backend/src/appenlight/lib/utils/__init__.py

@@ -1,491 +1,548 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 """
 Utility functions.
 """
 import logging
 import requests
 import hashlib
 import json
 import copy
 import uuid
 import appenlight.lib.helpers as h
 from collections import namedtuple
 from datetime import timedelta, datetime, date
 from dogpile.cache.api import NO_VALUE
 from appenlight.models import Datastores
-from appenlight.validators import (LogSearchSchema,
-                                   TagListSchema,
-                                   accepted_search_params)
+from appenlight.validators import LogSearchSchema, TagListSchema, accepted_search_params
 from itsdangerous import TimestampSigner
 from ziggurat_foundations.permissions import ALL_PERMISSIONS
 from ziggurat_foundations.models.services.user import UserService
 from dateutil.relativedelta import relativedelta
 from dateutil.rrule import rrule, MONTHLY, DAILY
 
 log = logging.getLogger(__name__)
 
 
-Stat = namedtuple('Stat', 'start_interval value')
+Stat = namedtuple("Stat", "start_interval value")
 
 
 def default_extractor(item):
     """
     :param item - item to extract date from
     """
-    if hasattr(item, 'start_interval'):
+    if hasattr(item, "start_interval"):
         return item.start_interval
-    return item['start_interval']
+    return item["start_interval"]
 
 
 # fast gap generator
-def gap_gen_default(start, step, itemiterator, end_time=None,
-                    iv_extractor=None):
+def gap_gen_default(start, step, itemiterator, end_time=None, iv_extractor=None):
     """ generates a list of time/value items based on step and itemiterator
         if there are entries missing from iterator time/None will be returned
         instead
     :param start - datetime - what time should we start generating our values
     :param step - timedelta - stepsize
     :param itemiterator - iterable - we will check this iterable for values
     corresponding to generated steps
     :param end_time - datetime - when last step is >= end_time stop iterating
     :param iv_extractor - extracts current step from iterable items
     """
 
     if not iv_extractor:
         iv_extractor = default_extractor
 
     next_step = start
     minutes = step.total_seconds() / 60.0
     while next_step.minute % minutes != 0:
         next_step = next_step.replace(minute=next_step.minute - 1)
     for item in itemiterator:
         item_start_interval = iv_extractor(item)
         # do we have a match for current time step in our data?
         # no gen a new tuple with 0 values
         while next_step < item_start_interval:
             yield Stat(next_step, None)
             next_step = next_step + step
         if next_step == item_start_interval:
             yield Stat(item_start_interval, item)
             next_step = next_step + step
     if end_time:
         while next_step < end_time:
             yield Stat(next_step, None)
             next_step = next_step + step
 
 
 class DateTimeEncoder(json.JSONEncoder):
     """ Simple datetime to ISO encoder for json serialization"""
 
     def default(self, obj):
         if isinstance(obj, date):
             return obj.isoformat()
         if isinstance(obj, datetime):
             return obj.isoformat()
         return json.JSONEncoder.default(self, obj)
 
 
-def channelstream_request(secret, endpoint, payload, throw_exceptions=False,
-                   servers=None):
+def channelstream_request(
+    secret, endpoint, payload, throw_exceptions=False, servers=None
+):
     responses = []
     if not servers:
         servers = []
 
     signer = TimestampSigner(secret)
     sig_for_server = signer.sign(endpoint)
-    for secret, server in [(s['secret'], s['server']) for s in servers]:
+    for secret, server in [(s["secret"], s["server"]) for s in servers]:
         response = {}
-        secret_headers = {'x-channelstream-secret': sig_for_server,
-                          'x-channelstream-endpoint': endpoint,
-                          'Content-Type': 'application/json'}
-        url = '%s%s' % (server, endpoint)
+        secret_headers = {
+            "x-channelstream-secret": sig_for_server,
+            "x-channelstream-endpoint": endpoint,
+            "Content-Type": "application/json",
+        }
+        url = "%s%s" % (server, endpoint)
         try:
-            response = requests.post(url,
-                                     data=json.dumps(payload,
-                                                     cls=DateTimeEncoder),
-                                     headers=secret_headers,
-                                     verify=False,
-                                     timeout=2).json()
+            response = requests.post(
+                url,
+                data=json.dumps(payload, cls=DateTimeEncoder),
+                headers=secret_headers,
+                verify=False,
+                timeout=2,
+            ).json()
         except requests.exceptions.RequestException as e:
             if throw_exceptions:
                 raise
         responses.append(response)
     return responses
 
 
 def add_cors_headers(response):
     # allow CORS
-    response.headers.add('Access-Control-Allow-Origin', '*')
-    response.headers.add('XDomainRequestAllowed', '1')
-    response.headers.add('Access-Control-Allow-Methods', 'GET, POST, OPTIONS')
+    response.headers.add("Access-Control-Allow-Origin", "*")
+    response.headers.add("XDomainRequestAllowed", "1")
+    response.headers.add("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
     # response.headers.add('Access-Control-Allow-Credentials', 'true')
-    response.headers.add('Access-Control-Allow-Headers',
-                         'Content-Type, Depth, User-Agent, X-File-Size, X-Requested-With, If-Modified-Since, X-File-Name, Cache-Control, Pragma, Origin, Connection, Referer, Cookie')
-    response.headers.add('Access-Control-Max-Age', '86400')
+    response.headers.add(
+        "Access-Control-Allow-Headers",
+        "Content-Type, Depth, User-Agent, X-File-Size, X-Requested-With, If-Modified-Since, X-File-Name, Cache-Control, Pragma, Origin, Connection, Referer, Cookie",
+    )
+    response.headers.add("Access-Control-Max-Age", "86400")
 
 
 from sqlalchemy.sql import compiler
 from psycopg2.extensions import adapt as sqlescape
 
 
 # or use the appropiate escape function from your db driver
 
+
 def compile_query(query):
     dialect = query.session.bind.dialect
     statement = query.statement
     comp = compiler.SQLCompiler(dialect, statement)
     comp.compile()
     enc = dialect.encoding
     params = {}
     for k, v in comp.params.items():
         if isinstance(v, str):
             v = v.encode(enc)
         params[k] = sqlescape(v)
     return (comp.string.encode(enc) % params).decode(enc)
 
 
 def convert_es_type(input_data):
     """
     This might need to convert some text or other types to corresponding ES types
     """
     return str(input_data)
 
 
-ProtoVersion = namedtuple('ProtoVersion', ['major', 'minor', 'patch'])
+ProtoVersion = namedtuple("ProtoVersion", ["major", "minor", "patch"])
 
 
 def parse_proto(input_data):
     try:
-        parts = [int(x) for x in input_data.split('.')]
+        parts = [int(x) for x in input_data.split(".")]
         while len(parts) < 3:
             parts.append(0)
         return ProtoVersion(*parts)
     except Exception as e:
-        log.info('Unknown protocol version: %s' % e)
+        log.info("Unknown protocol version: %s" % e)
     return ProtoVersion(99, 99, 99)
 
 
-def es_index_name_limiter(start_date=None, end_date=None, months_in_past=6,
-                          ixtypes=None):
+def es_index_name_limiter(
+    start_date=None, end_date=None, months_in_past=6, ixtypes=None
+):
     """
     This function limits the search to 6 months by default so we don't have to
     query 300 elasticsearch indices for 20 years of historical data for example
     """
 
     # should be cached later
     def get_possible_names():
-        return list(Datastores.es.indices.get_alias('*'))
+        return list(Datastores.es.indices.get_alias("*"))
 
     possible_names = get_possible_names()
     es_index_types = []
     if not ixtypes:
-        ixtypes = ['reports', 'metrics', 'logs']
+        ixtypes = ["reports", "metrics", "logs"]
     for t in ixtypes:
-        if t == 'reports':
-            es_index_types.append('rcae_r_%s')
-        elif t == 'logs':
-            es_index_types.append('rcae_l_%s')
-        elif t == 'metrics':
-            es_index_types.append('rcae_m_%s')
-        elif t == 'uptime':
-            es_index_types.append('rcae_u_%s')
-        elif t == 'slow_calls':
-            es_index_types.append('rcae_sc_%s')
+        if t == "reports":
+            es_index_types.append("rcae_r_%s")
+        elif t == "logs":
+            es_index_types.append("rcae_l_%s")
+        elif t == "metrics":
+            es_index_types.append("rcae_m_%s")
+        elif t == "uptime":
+            es_index_types.append("rcae_u_%s")
+        elif t == "slow_calls":
+            es_index_types.append("rcae_sc_%s")
 
     if start_date:
         start_date = copy.copy(start_date)
     else:
         if not end_date:
             end_date = datetime.utcnow()
         start_date = end_date + relativedelta(months=months_in_past * -1)
 
     if not end_date:
         end_date = start_date + relativedelta(months=months_in_past)
 
-    index_dates = list(rrule(MONTHLY,
-                             dtstart=start_date.date().replace(day=1),
-                             until=end_date.date(),
-                             count=36))
+    index_dates = list(
+        rrule(
+            MONTHLY,
+            dtstart=start_date.date().replace(day=1),
+            until=end_date.date(),
+            count=36,
+        )
+    )
     index_names = []
     for ix_type in es_index_types:
-        to_extend = [ix_type % d.strftime('%Y_%m') for d in index_dates
-                     if ix_type % d.strftime('%Y_%m') in possible_names]
+        to_extend = [
+            ix_type % d.strftime("%Y_%m")
+            for d in index_dates
+            if ix_type % d.strftime("%Y_%m") in possible_names
+        ]
         index_names.extend(to_extend)
-        for day in list(rrule(DAILY, dtstart=start_date.date(),
-                              until=end_date.date(), count=366)):
-            ix_name = ix_type % day.strftime('%Y_%m_%d')
+        for day in list(
+            rrule(DAILY, dtstart=start_date.date(), until=end_date.date(), count=366)
+        ):
+            ix_name = ix_type % day.strftime("%Y_%m_%d")
             if ix_name in possible_names:
                 index_names.append(ix_name)
     return index_names
 
 
 def build_filter_settings_from_query_dict(
-        request, params=None, override_app_ids=None,
-        resource_permissions=None):
+    request, params=None, override_app_ids=None, resource_permissions=None
+):
     """
     Builds list of normalized search terms for ES from query params
     ensuring application list is restricted to only applications user
     has access to
 
     :param params (dictionary)
     :param override_app_ids - list of application id's to use instead of
     applications user normally has access to
     """
     params = copy.deepcopy(params)
     applications = []
     if not resource_permissions:
-        resource_permissions = ['view']
+        resource_permissions = ["view"]
 
     if request.user:
         applications = UserService.resources_with_perms(
-            request.user, resource_permissions, resource_types=['application'])
+            request.user, resource_permissions, resource_types=["application"]
+        )
 
     # CRITICAL - this ensures our resultset is limited to only the ones
     # user has view permissions
     all_possible_app_ids = set([app.resource_id for app in applications])
 
     # if override is preset we force permission for app to be present
     # this allows users to see dashboards and applications they would
     # normally not be able to
 
     if override_app_ids:
         all_possible_app_ids = set(override_app_ids)
 
     schema = LogSearchSchema().bind(resources=all_possible_app_ids)
     tag_schema = TagListSchema()
     filter_settings = schema.deserialize(params)
     tag_list = []
     for k, v in list(filter_settings.items()):
         if k in accepted_search_params:
             continue
-        tag_list.append({"name": k, "value": v, "op": 'eq'})
+        tag_list.append({"name": k, "value": v, "op": "eq"})
         # remove the key from filter_settings
         filter_settings.pop(k, None)
     tags = tag_schema.deserialize(tag_list)
-    filter_settings['tags'] = tags
+    filter_settings["tags"] = tags
     return filter_settings
 
 
 def gen_uuid():
     return str(uuid.uuid4())
 
 
 def gen_uuid4_sha_hex():
     return hashlib.sha1(uuid.uuid4().bytes).hexdigest()
 
 
 def permission_tuple_to_dict(data):
     out = {
         "user_name": None,
         "perm_name": data.perm_name,
         "owner": data.owner,
         "type": data.type,
         "resource_name": None,
         "resource_type": None,
         "resource_id": None,
         "group_name": None,
-        "group_id": None
+        "group_id": None,
     }
     if data.user:
         out["user_name"] = data.user.user_name
     if data.perm_name == ALL_PERMISSIONS:
-        out['perm_name'] = '__all_permissions__'
+        out["perm_name"] = "__all_permissions__"
     if data.resource:
-        out['resource_name'] = data.resource.resource_name
-        out['resource_type'] = data.resource.resource_type
-        out['resource_id'] = data.resource.resource_id
+        out["resource_name"] = data.resource.resource_name
+        out["resource_type"] = data.resource.resource_type
+        out["resource_id"] = data.resource.resource_id
     if data.group:
-        out['group_name'] = data.group.group_name
-        out['group_id'] = data.group.id
+        out["group_name"] = data.group.group_name
+        out["group_id"] = data.group.id
     return out
 
 
-def get_cached_buckets(request, stats_since, end_time, fn, cache_key,
-                       gap_gen=None, db_session=None, step_interval=None,
-                       iv_extractor=None,
-                       rerange=False, *args, **kwargs):
+def get_cached_buckets(
+    request,
+    stats_since,
+    end_time,
+    fn,
+    cache_key,
+    gap_gen=None,
+    db_session=None,
+    step_interval=None,
+    iv_extractor=None,
+    rerange=False,
+    *args,
+    **kwargs
+):
     """ Takes "fn" that should return some data and tries to load the data
     dividing it into daily buckets - if the stats_since and end time give a
     delta bigger than 24hours, then only "todays" data is computed on the fly
 
     :param request: (request) request object
     :param stats_since: (datetime) start date of buckets range
     :param end_time: (datetime) end date of buckets range - utcnow() if None
     :param fn: (callable) callable to use to populate buckets should have
     following signature:
         def get_data(request, since_when, until, *args, **kwargs):
 
     :param cache_key: (string) cache key that will be used to build bucket
     caches
     :param gap_gen: (callable) gap generator - should return step intervals
     to use with out `fn` callable
     :param db_session: (Session) sqlalchemy session
     :param step_interval: (timedelta) optional step interval if we want to
     override the default determined from total start/end time delta
     :param iv_extractor: (callable) used to get step intervals from data
     returned by `fn` callable
     :param rerange: (bool) handy if we want to change ranges from hours to
     days when cached data is missing - will shorten execution time if `fn`
     callable supports that and we are working with multiple rows - like metrics
     :param args:
     :param kwargs:
 
     :return: iterable
     """
     if not end_time:
         end_time = datetime.utcnow().replace(second=0, microsecond=0)
     delta = end_time - stats_since
     # if smaller than 3 days we want to group by 5min else by 1h,
     # for 60 min group by min
     if not gap_gen:
         gap_gen = gap_gen_default
     if not iv_extractor:
         iv_extractor = default_extractor
 
     # do not use custom interval if total time range with new iv would exceed
     # end time
     if not step_interval or stats_since + step_interval >= end_time:
-        if delta < h.time_deltas.get('12h')['delta']:
+        if delta < h.time_deltas.get("12h")["delta"]:
             step_interval = timedelta(seconds=60)
-        elif delta < h.time_deltas.get('3d')['delta']:
+        elif delta < h.time_deltas.get("3d")["delta"]:
             step_interval = timedelta(seconds=60 * 5)
-        elif delta > h.time_deltas.get('2w')['delta']:
+        elif delta > h.time_deltas.get("2w")["delta"]:
             step_interval = timedelta(days=1)
         else:
             step_interval = timedelta(minutes=60)
 
     if step_interval >= timedelta(minutes=60):
-        log.info('cached_buckets:{}: adjusting start time '
-                 'for hourly or daily intervals'.format(cache_key))
+        log.info(
+            "cached_buckets:{}: adjusting start time "
+            "for hourly or daily intervals".format(cache_key)
+        )
         stats_since = stats_since.replace(hour=0, minute=0)
 
-    ranges = [i.start_interval for i in list(gap_gen(stats_since,
-                                                     step_interval, [],
-                                                     end_time=end_time))]
+    ranges = [
+        i.start_interval
+        for i in list(gap_gen(stats_since, step_interval, [], end_time=end_time))
+    ]
     buckets = {}
-    storage_key = 'buckets:' + cache_key + '{}|{}'
+    storage_key = "buckets:" + cache_key + "{}|{}"
     # this means we basicly cache per hour in 3-14 day intervals but i think
     # its fine at this point - will be faster than db access anyways
 
     if len(ranges) >= 1:
         last_ranges = [ranges[-1]]
     else:
         last_ranges = []
     if step_interval >= timedelta(minutes=60):
         for r in ranges:
             k = storage_key.format(step_interval.total_seconds(), r)
             value = request.registry.cache_regions.redis_day_30.get(k)
             # last buckets are never loaded from cache
-            is_last_result = (
-                r >= end_time - timedelta(hours=6) or r in last_ranges)
+            is_last_result = r >= end_time - timedelta(hours=6) or r in last_ranges
             if value is not NO_VALUE and not is_last_result:
-                log.info("cached_buckets:{}: "
-                         "loading range {} from cache".format(cache_key, r))
+                log.info(
+                    "cached_buckets:{}: "
+                    "loading range {} from cache".format(cache_key, r)
+                )
                 buckets[r] = value
             else:
-                log.info("cached_buckets:{}: "
-                         "loading range {} from storage".format(cache_key, r))
+                log.info(
+                    "cached_buckets:{}: "
+                    "loading range {} from storage".format(cache_key, r)
+                )
                 range_size = step_interval
-                if (step_interval == timedelta(minutes=60) and
-                        not is_last_result and rerange):
+                if (
+                    step_interval == timedelta(minutes=60)
+                    and not is_last_result
+                    and rerange
+                ):
                     range_size = timedelta(days=1)
                     r = r.replace(hour=0, minute=0)
-                    log.info("cached_buckets:{}: "
-                             "loading collapsed "
-                             "range {} {}".format(cache_key, r,
-                                                  r + range_size))
+                    log.info(
+                        "cached_buckets:{}: "
+                        "loading collapsed "
+                        "range {} {}".format(cache_key, r, r + range_size)
+                    )
                 bucket_data = fn(
-                    request, r, r + range_size, step_interval,
-                    gap_gen, bucket_count=len(ranges), *args, **kwargs)
+                    request,
+                    r,
+                    r + range_size,
+                    step_interval,
+                    gap_gen,
+                    bucket_count=len(ranges),
+                    *args,
+                    **kwargs
+                )
                 for b in bucket_data:
                     b_iv = iv_extractor(b)
                     buckets[b_iv] = b
-                    k2 = storage_key.format(
-                        step_interval.total_seconds(), b_iv)
+                    k2 = storage_key.format(step_interval.total_seconds(), b_iv)
                     request.registry.cache_regions.redis_day_30.set(k2, b)
         log.info("cached_buckets:{}: saving cache".format(cache_key))
     else:
         # bucket count is 1 for short time ranges <= 24h from now
-        bucket_data = fn(request, stats_since, end_time, step_interval,
-                         gap_gen, bucket_count=1, *args, **kwargs)
+        bucket_data = fn(
+            request,
+            stats_since,
+            end_time,
+            step_interval,
+            gap_gen,
+            bucket_count=1,
+            *args,
+            **kwargs
+        )
         for b in bucket_data:
             buckets[iv_extractor(b)] = b
     return buckets
 
 
-def get_cached_split_data(request, stats_since, end_time, fn, cache_key,
-                          db_session=None, *args, **kwargs):
+def get_cached_split_data(
+    request, stats_since, end_time, fn, cache_key, db_session=None, *args, **kwargs
+):
     """ Takes "fn" that should return some data and tries to load the data
     dividing it into 2 buckets - cached "since_from" bucket and "today"
     bucket - then the data can be reduced into single value
 
     Data is cached if the stats_since and end time give a delta bigger
     than 24hours - then only 24h is computed on the fly
     """
     if not end_time:
         end_time = datetime.utcnow().replace(second=0, microsecond=0)
     delta = end_time - stats_since
 
     if delta >= timedelta(minutes=60):
-        log.info('cached_split_data:{}: adjusting start time '
-                 'for hourly or daily intervals'.format(cache_key))
+        log.info(
+            "cached_split_data:{}: adjusting start time "
+            "for hourly or daily intervals".format(cache_key)
+        )
         stats_since = stats_since.replace(hour=0, minute=0)
 
-    storage_key = 'buckets_split_data:' + cache_key + ':{}|{}'
+    storage_key = "buckets_split_data:" + cache_key + ":{}|{}"
     old_end_time = end_time.replace(hour=0, minute=0)
 
-    final_storage_key = storage_key.format(delta.total_seconds(),
-                                           old_end_time)
+    final_storage_key = storage_key.format(delta.total_seconds(), old_end_time)
     older_data = None
 
-    cdata = request.registry.cache_regions.redis_day_7.get(
-        final_storage_key)
+    cdata = request.registry.cache_regions.redis_day_7.get(final_storage_key)
 
     if cdata:
-        log.info("cached_split_data:{}: found old "
-                 "bucket data".format(cache_key))
+        log.info("cached_split_data:{}: found old " "bucket data".format(cache_key))
         older_data = cdata
 
-    if (stats_since < end_time - h.time_deltas.get('24h')['delta'] and
-            not cdata):
-        log.info("cached_split_data:{}: didn't find the "
-                 "start bucket in cache so load older data".format(cache_key))
+    if stats_since < end_time - h.time_deltas.get("24h")["delta"] and not cdata:
+        log.info(
+            "cached_split_data:{}: didn't find the "
+            "start bucket in cache so load older data".format(cache_key)
+        )
         recent_stats_since = old_end_time
-        older_data = fn(request, stats_since, recent_stats_since,
-                        db_session=db_session, *args, **kwargs)
-        request.registry.cache_regions.redis_day_7.set(final_storage_key,
-                                                       older_data)
-    elif stats_since < end_time - h.time_deltas.get('24h')['delta']:
+        older_data = fn(
+            request,
+            stats_since,
+            recent_stats_since,
+            db_session=db_session,
+            *args,
+            **kwargs
+        )
+        request.registry.cache_regions.redis_day_7.set(final_storage_key, older_data)
+    elif stats_since < end_time - h.time_deltas.get("24h")["delta"]:
         recent_stats_since = old_end_time
     else:
         recent_stats_since = stats_since
 
-    log.info("cached_split_data:{}: loading fresh "
-             "data bucksts from last 24h ".format(cache_key))
-    todays_data = fn(request, recent_stats_since, end_time,
-                     db_session=db_session, *args, **kwargs)
+    log.info(
+        "cached_split_data:{}: loading fresh "
+        "data bucksts from last 24h ".format(cache_key)
+    )
+    todays_data = fn(
+        request, recent_stats_since, end_time, db_session=db_session, *args, **kwargs
+    )
     return older_data, todays_data
 
 
 def in_batches(seq, size):
     """
     Splits am iterable into batches of specified size
     :param seq (iterable)
     :param size integer
     """
-    return (seq[pos:pos + size] for pos in range(0, len(seq), size))
+    return (seq[pos : pos + size] for pos in range(0, len(seq), size))

backend/src/appenlight/lib/utils/airbrake.py

@@ -1,142 +1,161 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 import uuid
 
 from datetime import datetime
 
 log = logging.getLogger(__name__)
 
 
 def parse_airbrake_xml(request):
     root = request.context.airbrake_xml_etree
-    error = root.find('error')
-    notifier = root.find('notifier')
-    server_env = root.find('server-environment')
-    request_data = root.find('request')
-    user = root.find('current-user')
+    error = root.find("error")
+    notifier = root.find("notifier")
+    server_env = root.find("server-environment")
+    request_data = root.find("request")
+    user = root.find("current-user")
     if request_data is not None:
-        cgi_data = request_data.find('cgi-data')
+        cgi_data = request_data.find("cgi-data")
         if cgi_data is None:
             cgi_data = []
 
     error_dict = {
-        'class_name': error.findtext('class') or '',
-        'error': error.findtext('message') or '',
+        "class_name": error.findtext("class") or "",
+        "error": error.findtext("message") or "",
         "occurences": 1,
         "http_status": 500,
         "priority": 5,
-        "server": 'unknown',
-        'url': 'unknown', 'request': {}
+        "server": "unknown",
+        "url": "unknown",
+        "request": {},
     }
     if user is not None:
-        error_dict['username'] = user.findtext('username') or \
-                                 user.findtext('id')
+        error_dict["username"] = user.findtext("username") or user.findtext("id")
     if notifier is not None:
-        error_dict['client'] = notifier.findtext('name')
+        error_dict["client"] = notifier.findtext("name")
 
     if server_env is not None:
-        error_dict["server"] = server_env.findtext('hostname', 'unknown')
+        error_dict["server"] = server_env.findtext("hostname", "unknown")
 
-    whitelist_environ = ['REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME',
-                         'CONTENT_TYPE', 'HTTP_REFERER']
+    whitelist_environ = [
+        "REMOTE_USER",
+        "REMOTE_ADDR",
+        "SERVER_NAME",
+        "CONTENT_TYPE",
+        "HTTP_REFERER",
+    ]
 
     if request_data is not None:
-        error_dict['url'] = request_data.findtext('url', 'unknown')
-        component = request_data.findtext('component')
-        action = request_data.findtext('action')
+        error_dict["url"] = request_data.findtext("url", "unknown")
+        component = request_data.findtext("component")
+        action = request_data.findtext("action")
         if component and action:
-            error_dict['view_name'] = '%s:%s' % (component, action)
+            error_dict["view_name"] = "%s:%s" % (component, action)
         for node in cgi_data:
-            key = node.get('key')
-            if key.startswith('HTTP') or key in whitelist_environ:
-                error_dict['request'][key] = node.text
-            elif 'query_parameters' in key:
-                error_dict['request']['GET'] = {}
+            key = node.get("key")
+            if key.startswith("HTTP") or key in whitelist_environ:
+                error_dict["request"][key] = node.text
+            elif "query_parameters" in key:
+                error_dict["request"]["GET"] = {}
                 for x in node:
-                    error_dict['request']['GET'][x.get('key')] = x.text
-            elif 'request_parameters' in key:
-                error_dict['request']['POST'] = {}
+                    error_dict["request"]["GET"][x.get("key")] = x.text
+            elif "request_parameters" in key:
+                error_dict["request"]["POST"] = {}
                 for x in node:
-                    error_dict['request']['POST'][x.get('key')] = x.text
-            elif key.endswith('cookie'):
-                error_dict['request']['COOKIE'] = {}
+                    error_dict["request"]["POST"][x.get("key")] = x.text
+            elif key.endswith("cookie"):
+                error_dict["request"]["COOKIE"] = {}
                 for x in node:
-                    error_dict['request']['COOKIE'][x.get('key')] = x.text
-            elif key.endswith('request_id'):
-                error_dict['request_id'] = node.text
-            elif key.endswith('session'):
-                error_dict['request']['SESSION'] = {}
+                    error_dict["request"]["COOKIE"][x.get("key")] = x.text
+            elif key.endswith("request_id"):
+                error_dict["request_id"] = node.text
+            elif key.endswith("session"):
+                error_dict["request"]["SESSION"] = {}
                 for x in node:
-                    error_dict['request']['SESSION'][x.get('key')] = x.text
+                    error_dict["request"]["SESSION"][x.get("key")] = x.text
             else:
-                if key in ['rack.session.options']:
+                if key in ["rack.session.options"]:
                     # skip secret configs
                     continue
                 try:
                     if len(node):
-                        error_dict['request'][key] = dict(
-                            [(x.get('key'), x.text,) for x in node])
+                        error_dict["request"][key] = dict(
+                            [(x.get("key"), x.text) for x in node]
+                        )
                     else:
-                        error_dict['request'][key] = node.text
+                        error_dict["request"][key] = node.text
                 except Exception as e:
-                    log.warning('Airbrake integration exception: %s' % e)
+                    log.warning("Airbrake integration exception: %s" % e)
 
-            error_dict['request'].pop('HTTP_COOKIE', '')
+            error_dict["request"].pop("HTTP_COOKIE", "")
 
-        error_dict['ip'] = error_dict.pop('REMOTE_ADDR', '')
-        error_dict['user_agent'] = error_dict.pop('HTTP_USER_AGENT', '')
-    if 'request_id' not in error_dict:
-        error_dict['request_id'] = str(uuid.uuid4())
+        error_dict["ip"] = error_dict.pop("REMOTE_ADDR", "")
+        error_dict["user_agent"] = error_dict.pop("HTTP_USER_AGENT", "")
+    if "request_id" not in error_dict:
+        error_dict["request_id"] = str(uuid.uuid4())
     if request.context.possibly_public:
         # set ip for reports that come from airbrake js client
         error_dict["timestamp"] = datetime.utcnow()
         if request.environ.get("HTTP_X_FORWARDED_FOR"):
-            ip = request.environ.get("HTTP_X_FORWARDED_FOR", '')
-            first_ip = ip.split(',')[0]
+            ip = request.environ.get("HTTP_X_FORWARDED_FOR", "")
+            first_ip = ip.split(",")[0]
             remote_addr = first_ip.strip()
         else:
-            remote_addr = (request.environ.get("HTTP_X_REAL_IP") or
-                           request.environ.get('REMOTE_ADDR'))
+            remote_addr = request.environ.get("HTTP_X_REAL_IP") or request.environ.get(
+                "REMOTE_ADDR"
+            )
         error_dict["ip"] = remote_addr
 
-    blacklist = ['password', 'passwd', 'pwd', 'auth_tkt', 'secret', 'csrf',
-                 'session', 'test']
+    blacklist = [
+        "password",
+        "passwd",
+        "pwd",
+        "auth_tkt",
+        "secret",
+        "csrf",
+        "session",
+        "test",
+    ]
 
     lines = []
-    for l in error.find('backtrace'):
-        lines.append({'file': l.get("file", ""),
-                      'line': l.get("number", ""),
-                      'fn': l.get("method", ""),
-                      'module': l.get("module", ""),
-                      'cline': l.get("method", ""),
-                      'vars': {}})
-    error_dict['traceback'] = list(reversed(lines))
+    for l in error.find("backtrace"):
+        lines.append(
+            {
+                "file": l.get("file", ""),
+                "line": l.get("number", ""),
+                "fn": l.get("method", ""),
+                "module": l.get("module", ""),
+                "cline": l.get("method", ""),
+                "vars": {},
+            }
+        )
+    error_dict["traceback"] = list(reversed(lines))
     # filtering is not provided by airbrake
     keys_to_check = (
-        error_dict['request'].get('COOKIE'),
-        error_dict['request'].get('COOKIES'),
-        error_dict['request'].get('POST'),
-        error_dict['request'].get('SESSION'),
+        error_dict["request"].get("COOKIE"),
+        error_dict["request"].get("COOKIES"),
+        error_dict["request"].get("POST"),
+        error_dict["request"].get("SESSION"),
     )
     for source in [_f for _f in keys_to_check if _f]:
         for k in source.keys():
             for bad_key in blacklist:
                 if bad_key in k.lower():
-                    source[k] = '***'
+                    source[k] = "***"
 
     return error_dict

backend/src/appenlight/lib/utils/date_utils.py

@@ -1,56 +1,56 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from datetime import tzinfo, timedelta, datetime
 from dateutil.relativedelta import relativedelta
 import logging
 
 log = logging.getLogger(__name__)
 
 
 def to_relativedelta(time_delta):
-    return relativedelta(seconds=int(time_delta.total_seconds()),
-                         microseconds=time_delta.microseconds)
+    return relativedelta(
+        seconds=int(time_delta.total_seconds()), microseconds=time_delta.microseconds
+    )
 
 
-def convert_date(date_str, return_utcnow_if_wrong=True,
-                 normalize_future=False):
+def convert_date(date_str, return_utcnow_if_wrong=True, normalize_future=False):
     utcnow = datetime.utcnow()
     if isinstance(date_str, datetime):
         # get rid of tzinfo
         return date_str.replace(tzinfo=None)
     if not date_str and return_utcnow_if_wrong:
         return utcnow
     try:
         try:
-            if 'Z' in date_str:
-                date_str = date_str[:date_str.index('Z')]
-            if '.' in date_str:
-                date = datetime.strptime(date_str, '%Y-%m-%dT%H:%M:%S.%f')
+            if "Z" in date_str:
+                date_str = date_str[: date_str.index("Z")]
+            if "." in date_str:
+                date = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S.%f")
             else:
-                date = datetime.strptime(date_str, '%Y-%m-%dT%H:%M:%S')
+                date = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S")
         except Exception:
             # bw compat with old client
-            date = datetime.strptime(date_str, '%Y-%m-%d %H:%M:%S,%f')
+            date = datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S,%f")
     except Exception:
         if return_utcnow_if_wrong:
             date = utcnow
         else:
             date = None
     if normalize_future and date and date > (utcnow + timedelta(minutes=3)):
-        log.warning('time %s in future + 3 min, normalizing' % date)
+        log.warning("time %s in future + 3 min, normalizing" % date)
         return utcnow
     return date

backend/src/appenlight/lib/utils/sentry.py

@@ -1,296 +1,317 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from datetime import timedelta
 
 from appenlight.lib.enums import LogLevelPython, ParsedSentryEventType
 
 EXCLUDED_LOG_VARS = [
-    'args', 'asctime', 'created', 'exc_info', 'exc_text', 'filename',
-    'funcName', 'levelname', 'levelno', 'lineno', 'message', 'module', 'msecs',
-    'msg', 'name', 'pathname', 'process', 'processName', 'relativeCreated',
-    'thread', 'threadName']
+    "args",
+    "asctime",
+    "created",
+    "exc_info",
+    "exc_text",
+    "filename",
+    "funcName",
+    "levelname",
+    "levelno",
+    "lineno",
+    "message",
+    "module",
+    "msecs",
+    "msg",
+    "name",
+    "pathname",
+    "process",
+    "processName",
+    "relativeCreated",
+    "thread",
+    "threadName",
+]
 
 EXCLUDE_SENTRY_KEYS = [
-    'csp',
-    'culprit',
-    'event_id',
-    'exception',
-    'extra',
-    'level',
-    'logentry',
-    'logger',
-    'message',
-    'modules',
-    'platform',
-    'query',
-    'release',
-    'request',
-    'sentry.interfaces.Csp', 'sentry.interfaces.Exception',
-    'sentry.interfaces.Http', 'sentry.interfaces.Message',
-    'sentry.interfaces.Query',
-    'sentry.interfaces.Stacktrace',
-    'sentry.interfaces.Template', 'sentry.interfaces.User',
-    'sentry.interfaces.csp.Csp',
-    'sentry.interfaces.exception.Exception',
-    'sentry.interfaces.http.Http',
-    'sentry.interfaces.message.Message',
-    'sentry.interfaces.query.Query',
-    'sentry.interfaces.stacktrace.Stacktrace',
-    'sentry.interfaces.template.Template',
-    'sentry.interfaces.user.User', 'server_name',
-    'stacktrace',
-    'tags',
-    'template',
-    'time_spent',
-    'timestamp',
-    'user']
+    "csp",
+    "culprit",
+    "event_id",
+    "exception",
+    "extra",
+    "level",
+    "logentry",
+    "logger",
+    "message",
+    "modules",
+    "platform",
+    "query",
+    "release",
+    "request",
+    "sentry.interfaces.Csp",
+    "sentry.interfaces.Exception",
+    "sentry.interfaces.Http",
+    "sentry.interfaces.Message",
+    "sentry.interfaces.Query",
+    "sentry.interfaces.Stacktrace",
+    "sentry.interfaces.Template",
+    "sentry.interfaces.User",
+    "sentry.interfaces.csp.Csp",
+    "sentry.interfaces.exception.Exception",
+    "sentry.interfaces.http.Http",
+    "sentry.interfaces.message.Message",
+    "sentry.interfaces.query.Query",
+    "sentry.interfaces.stacktrace.Stacktrace",
+    "sentry.interfaces.template.Template",
+    "sentry.interfaces.user.User",
+    "server_name",
+    "stacktrace",
+    "tags",
+    "template",
+    "time_spent",
+    "timestamp",
+    "user",
+]
 
 
 def get_keys(list_of_keys, json_body):
     for k in list_of_keys:
         if k in json_body:
             return json_body[k]
 
 
 def get_logentry(json_body):
-    key_names = ['logentry',
-                 'sentry.interfaces.message.Message',
-                 'sentry.interfaces.Message'
-                 ]
+    key_names = [
+        "logentry",
+        "sentry.interfaces.message.Message",
+        "sentry.interfaces.Message",
+    ]
     logentry = get_keys(key_names, json_body)
     return logentry
 
 
 def get_exception(json_body):
     parsed_exception = {}
-    key_names = ['exception',
-                 'sentry.interfaces.exception.Exception',
-                 'sentry.interfaces.Exception'
-                 ]
+    key_names = [
+        "exception",
+        "sentry.interfaces.exception.Exception",
+        "sentry.interfaces.Exception",
+    ]
     exception = get_keys(key_names, json_body) or {}
     if exception:
         if isinstance(exception, dict):
-            exception = exception['values'][0]
+            exception = exception["values"][0]
         else:
             exception = exception[0]
 
-    parsed_exception['type'] = exception.get('type')
-    parsed_exception['value'] = exception.get('value')
-    parsed_exception['module'] = exception.get('module')
+    parsed_exception["type"] = exception.get("type")
+    parsed_exception["value"] = exception.get("value")
+    parsed_exception["module"] = exception.get("module")
     parsed_stacktrace = get_stacktrace(exception) or {}
     parsed_exception = exception or {}
     return parsed_exception, parsed_stacktrace
 
 
 def get_stacktrace(json_body):
     parsed_stacktrace = []
-    key_names = ['stacktrace',
-                 'sentry.interfaces.stacktrace.Stacktrace',
-                 'sentry.interfaces.Stacktrace'
-                 ]
+    key_names = [
+        "stacktrace",
+        "sentry.interfaces.stacktrace.Stacktrace",
+        "sentry.interfaces.Stacktrace",
+    ]
     stacktrace = get_keys(key_names, json_body)
     if stacktrace:
-        for frame in stacktrace['frames']:
+        for frame in stacktrace["frames"]:
             parsed_stacktrace.append(
-                {"cline": frame.get('context_line', ''),
-                 "file": frame.get('filename', ''),
-                 "module": frame.get('module', ''),
-                 "fn": frame.get('function', ''),
-                 "line": frame.get('lineno', ''),
-                 "vars": list(frame.get('vars', {}).items())
-                 }
+                {
+                    "cline": frame.get("context_line", ""),
+                    "file": frame.get("filename", ""),
+                    "module": frame.get("module", ""),
+                    "fn": frame.get("function", ""),
+                    "line": frame.get("lineno", ""),
+                    "vars": list(frame.get("vars", {}).items()),
+                }
             )
     return parsed_stacktrace
 
 
 def get_template(json_body):
     parsed_template = {}
-    key_names = ['template',
-                 'sentry.interfaces.template.Template',
-                 'sentry.interfaces.Template'
-                 ]
+    key_names = [
+        "template",
+        "sentry.interfaces.template.Template",
+        "sentry.interfaces.Template",
+    ]
     template = get_keys(key_names, json_body)
     if template:
-        for frame in template['frames']:
+        for frame in template["frames"]:
             parsed_template.append(
-                {"cline": frame.get('context_line', ''),
-                 "file": frame.get('filename', ''),
-                 "fn": '',
-                 "line": frame.get('lineno', ''),
-                 "vars": []
-                 }
+                {
+                    "cline": frame.get("context_line", ""),
+                    "file": frame.get("filename", ""),
+                    "fn": "",
+                    "line": frame.get("lineno", ""),
+                    "vars": [],
+                }
             )
 
     return parsed_template
 
 
 def get_request(json_body):
     parsed_http = {}
-    key_names = ['request',
-                 'sentry.interfaces.http.Http',
-                 'sentry.interfaces.Http'
-                 ]
+    key_names = ["request", "sentry.interfaces.http.Http", "sentry.interfaces.Http"]
     http = get_keys(key_names, json_body) or {}
     for k, v in http.items():
-        if k == 'headers':
-            parsed_http['headers'] = {}
-            for sk, sv in http['headers'].items():
-                parsed_http['headers'][sk.title()] = sv
+        if k == "headers":
+            parsed_http["headers"] = {}
+            for sk, sv in http["headers"].items():
+                parsed_http["headers"][sk.title()] = sv
         else:
             parsed_http[k.lower()] = v
     return parsed_http
 
 
 def get_user(json_body):
     parsed_user = {}
-    key_names = ['user',
-                 'sentry.interfaces.user.User',
-                 'sentry.interfaces.User'
-                 ]
+    key_names = ["user", "sentry.interfaces.user.User", "sentry.interfaces.User"]
     user = get_keys(key_names, json_body)
     if user:
-        parsed_user['id'] = user.get('id')
-        parsed_user['username'] = user.get('username')
-        parsed_user['email'] = user.get('email')
-        parsed_user['ip_address'] = user.get('ip_address')
+        parsed_user["id"] = user.get("id")
+        parsed_user["username"] = user.get("username")
+        parsed_user["email"] = user.get("email")
+        parsed_user["ip_address"] = user.get("ip_address")
 
     return parsed_user
 
 
 def get_query(json_body):
     query = None
-    key_name = ['query',
-                'sentry.interfaces.query.Query',
-                'sentry.interfaces.Query'
-                ]
+    key_name = ["query", "sentry.interfaces.query.Query", "sentry.interfaces.Query"]
     query = get_keys(key_name, json_body)
     return query
 
 
 def parse_sentry_event(json_body):
-    request_id = json_body.get('event_id')
+    request_id = json_body.get("event_id")
 
     # required
-    message = json_body.get('message')
-    log_timestamp = json_body.get('timestamp')
-    level = json_body.get('level')
+    message = json_body.get("message")
+    log_timestamp = json_body.get("timestamp")
+    level = json_body.get("level")
     if isinstance(level, int):
         level = LogLevelPython.key_from_value(level)
 
-    namespace = json_body.get('logger')
-    language = json_body.get('platform')
+    namespace = json_body.get("logger")
+    language = json_body.get("platform")
 
     # optional
-    server_name = json_body.get('server_name')
-    culprit = json_body.get('culprit')
-    release = json_body.get('release')
+    server_name = json_body.get("server_name")
+    culprit = json_body.get("culprit")
+    release = json_body.get("release")
 
-    tags = json_body.get('tags', {})
-    if hasattr(tags, 'items'):
+    tags = json_body.get("tags", {})
+    if hasattr(tags, "items"):
         tags = list(tags.items())
-    extra = json_body.get('extra', {})
-    if hasattr(extra, 'items'):
+    extra = json_body.get("extra", {})
+    if hasattr(extra, "items"):
         extra = list(extra.items())
 
     parsed_req = get_request(json_body)
     user = get_user(json_body)
     template = get_template(json_body)
     query = get_query(json_body)
 
     # other unidentified keys found
-    other_keys = [(k, json_body[k]) for k in json_body.keys()
-                  if k not in EXCLUDE_SENTRY_KEYS]
+    other_keys = [
+        (k, json_body[k]) for k in json_body.keys() if k not in EXCLUDE_SENTRY_KEYS
+    ]
 
     logentry = get_logentry(json_body)
     if logentry:
-        message = logentry['message']
+        message = logentry["message"]
 
     exception, stacktrace = get_exception(json_body)
 
     alt_stacktrace = get_stacktrace(json_body)
     event_type = None
     if not exception and not stacktrace and not alt_stacktrace and not template:
         event_type = ParsedSentryEventType.LOG
 
         event_dict = {
-            'log_level': level,
-            'message': message,
-            'namespace': namespace,
-            'request_id': request_id,
-            'server': server_name,
-            'date': log_timestamp,
-            'tags': tags
+            "log_level": level,
+            "message": message,
+            "namespace": namespace,
+            "request_id": request_id,
+            "server": server_name,
+            "date": log_timestamp,
+            "tags": tags,
         }
-        event_dict['tags'].extend(
-            [(k, v) for k, v in extra if k not in EXCLUDED_LOG_VARS])
+        event_dict["tags"].extend(
+            [(k, v) for k, v in extra if k not in EXCLUDED_LOG_VARS]
+        )
 
         # other keys can be various object types
-        event_dict['tags'].extend([(k, v) for k, v in other_keys
-                                   if isinstance(v, str)])
+        event_dict["tags"].extend([(k, v) for k, v in other_keys if isinstance(v, str)])
         if culprit:
-            event_dict['tags'].append(('sentry_culprit', culprit))
+            event_dict["tags"].append(("sentry_culprit", culprit))
         if language:
-            event_dict['tags'].append(('sentry_language', language))
+            event_dict["tags"].append(("sentry_language", language))
         if release:
-            event_dict['tags'].append(('sentry_release', release))
+            event_dict["tags"].append(("sentry_release", release))
 
     if exception or stacktrace or alt_stacktrace or template:
         event_type = ParsedSentryEventType.ERROR_REPORT
         event_dict = {
-            'client': 'sentry',
-            'error': message,
-            'namespace': namespace,
-            'request_id': request_id,
-            'server': server_name,
-            'start_time': log_timestamp,
-            'end_time': None,
-            'tags': tags,
-            'extra': extra,
-            'language': language,
-            'view_name': json_body.get('culprit'),
-            'http_status': None,
-            'username': None,
-            'url': parsed_req.get('url'),
-            'ip': None,
-            'user_agent': None,
-            'request': None,
-            'slow_calls': None,
-            'request_stats': None,
-            'traceback': None
+            "client": "sentry",
+            "error": message,
+            "namespace": namespace,
+            "request_id": request_id,
+            "server": server_name,
+            "start_time": log_timestamp,
+            "end_time": None,
+            "tags": tags,
+            "extra": extra,
+            "language": language,
+            "view_name": json_body.get("culprit"),
+            "http_status": None,
+            "username": None,
+            "url": parsed_req.get("url"),
+            "ip": None,
+            "user_agent": None,
+            "request": None,
+            "slow_calls": None,
+            "request_stats": None,
+            "traceback": None,
         }
 
-        event_dict['extra'].extend(other_keys)
+        event_dict["extra"].extend(other_keys)
         if release:
-            event_dict['tags'].append(('sentry_release', release))
-        event_dict['request'] = parsed_req
-        if 'headers' in parsed_req:
-            event_dict['user_agent'] = parsed_req['headers'].get('User-Agent')
-        if 'env' in parsed_req:
-            event_dict['ip'] = parsed_req['env'].get('REMOTE_ADDR')
-        ts_ms = int(json_body.get('time_spent') or 0)
+            event_dict["tags"].append(("sentry_release", release))
+        event_dict["request"] = parsed_req
+        if "headers" in parsed_req:
+            event_dict["user_agent"] = parsed_req["headers"].get("User-Agent")
+        if "env" in parsed_req:
+            event_dict["ip"] = parsed_req["env"].get("REMOTE_ADDR")
+        ts_ms = int(json_body.get("time_spent") or 0)
         if ts_ms > 0:
-            event_dict['end_time'] = event_dict['start_time'] + \
-                                     timedelta(milliseconds=ts_ms)
+            event_dict["end_time"] = event_dict["start_time"] + timedelta(
+                milliseconds=ts_ms
+            )
         if stacktrace or alt_stacktrace or template:
-            event_dict['traceback'] = stacktrace or alt_stacktrace or template
+            event_dict["traceback"] = stacktrace or alt_stacktrace or template
         for k in list(event_dict.keys()):
             if event_dict[k] is None:
                 del event_dict[k]
         if user:
-            event_dict['username'] = user['username'] or user['id'] \
-                                     or user['email']
+            event_dict["username"] = user["username"] or user["id"] or user["email"]
     return event_dict, event_type

backend/src/appenlight/migrations/__init__.py

@@ -1,17 +1,15 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-

backend/src/appenlight/migrations/env.py

@@ -1,98 +1,102 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from alembic import context
 from sqlalchemy import engine_from_config, pool, MetaData
 from logging.config import fileConfig
 from appenlight.models import NAMING_CONVENTION
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
 config = context.config
 
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.
 if config.config_file_name:
     fileConfig(config.config_file_name)
 
 # add your model's MetaData object here
 # for 'autogenerate' support
 # from myapp import mymodel
 # target_metadata = mymodel.Base.metadata
 
 
 target_metadata = MetaData(naming_convention=NAMING_CONVENTION)
 
 # other values from the config, defined by the needs of env.py,
 # can be acquired:
 # my_important_option = config.get_main_option("my_important_option")
 # ... etc.
 
-VERSION_TABLE_NAME = 'alembic_appenlight_version'
+VERSION_TABLE_NAME = "alembic_appenlight_version"
 
 
 def run_migrations_offline():
     """Run migrations in 'offline' mode.
 
     This configures the context with just a URL
     and not an Engine, though an Engine is acceptable
     here as well.  By skipping the Engine creation
     we don't even need a DBAPI to be available.
 
     Calls to context.execute() here emit the given string to the
     script output.
 
     """
     url = config.get_main_option("sqlalchemy.url")
-    context.configure(url=url, target_metadata=target_metadata,
-                      transaction_per_migration=True,
-                      version_table=VERSION_TABLE_NAME)
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        transaction_per_migration=True,
+        version_table=VERSION_TABLE_NAME,
+    )
 
     with context.begin_transaction():
         context.run_migrations()
 
 
 def run_migrations_online():
     """Run migrations in 'online' mode.
 
     In this scenario we need to create an Engine
     and associate a connection with the context.
 
     """
     engine = engine_from_config(
         config.get_section(config.config_ini_section),
-        prefix='sqlalchemy.',
-        poolclass=pool.NullPool)
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
 
     connection = engine.connect()
     context.configure(
         connection=connection,
         target_metadata=target_metadata,
         transaction_per_migration=True,
-        version_table=VERSION_TABLE_NAME
+        version_table=VERSION_TABLE_NAME,
     )
 
     try:
         with context.begin_transaction():
             context.run_migrations()
     finally:
         connection.close()
 
 
 if context.is_offline_mode():
     run_migrations_offline()
 else:
     run_migrations_online()

backend/src/appenlight/migrations/versions/55b6e612672f_initial_tables.py

@@ -1,624 +1,813 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 """initial tables
 
 Revision ID: 55b6e612672f
 Revises: None
 Create Date: 2014-10-13 23:47:38.295159
 
 """
 
 # revision identifiers, used by Alembic.
-revision = '55b6e612672f'
+revision = "55b6e612672f"
 down_revision = None
 
 from alembic import op
 import sqlalchemy as sa
 
 
 def upgrade():
-    op.add_column('users', sa.Column('first_name', sa.Unicode(25)))
-    op.add_column('users', sa.Column('last_name', sa.Unicode(50)))
-    op.add_column('users', sa.Column('company_name', sa.Unicode(255)))
-    op.add_column('users', sa.Column('company_address', sa.Unicode(255)))
-    op.add_column('users', sa.Column('phone1', sa.Unicode(25)))
-    op.add_column('users', sa.Column('phone2', sa.Unicode(25)))
-    op.add_column('users', sa.Column('zip_code', sa.Unicode(25)))
-    op.add_column('users', sa.Column('default_report_sort', sa.Unicode(20), nullable=False, server_default="newest"))
-    op.add_column('users', sa.Column('city', sa.Unicode(128)))
-    op.add_column('users', sa.Column('notes', sa.UnicodeText, server_default=''))
-    op.add_column('users', sa.Column('notifications', sa.Boolean(), nullable=False, server_default='true'))
-    op.add_column('users', sa.Column('registration_ip', sa.Unicode(40), nullable=False, server_default=''))
+    op.add_column("users", sa.Column("first_name", sa.Unicode(25)))
+    op.add_column("users", sa.Column("last_name", sa.Unicode(50)))
+    op.add_column("users", sa.Column("company_name", sa.Unicode(255)))
+    op.add_column("users", sa.Column("company_address", sa.Unicode(255)))
+    op.add_column("users", sa.Column("phone1", sa.Unicode(25)))
+    op.add_column("users", sa.Column("phone2", sa.Unicode(25)))
+    op.add_column("users", sa.Column("zip_code", sa.Unicode(25)))
+    op.add_column(
+        "users",
+        sa.Column(
+            "default_report_sort",
+            sa.Unicode(20),
+            nullable=False,
+            server_default="newest",
+        ),
+    )
+    op.add_column("users", sa.Column("city", sa.Unicode(128)))
+    op.add_column("users", sa.Column("notes", sa.UnicodeText, server_default=""))
+    op.add_column(
+        "users",
+        sa.Column("notifications", sa.Boolean(), nullable=False, server_default="true"),
+    )
+    op.add_column(
+        "users",
+        sa.Column("registration_ip", sa.Unicode(40), nullable=False, server_default=""),
+    )
 
     op.create_table(
-        'integrations',
-        sa.Column('id', sa.Integer(), primary_key=True),
-        sa.Column('resource_id', sa.Integer(),
-                  sa.ForeignKey('resources.resource_id', onupdate='cascade',
-                                ondelete='cascade')),
-        sa.Column('integration_name', sa.Unicode(64)),
-        sa.Column('config', sa.dialects.postgresql.JSON, nullable=False),
-        sa.Column('modified_date', sa.DateTime(), nullable=False, server_default=sa.func.now()),
-        sa.Column('external_id', sa.Unicode(255)),
-        sa.Column('external_id2', sa.Unicode(255))
+        "integrations",
+        sa.Column("id", sa.Integer(), primary_key=True),
+        sa.Column(
+            "resource_id",
+            sa.Integer(),
+            sa.ForeignKey(
+                "resources.resource_id", onupdate="cascade", ondelete="cascade"
+            ),
+        ),
+        sa.Column("integration_name", sa.Unicode(64)),
+        sa.Column("config", sa.dialects.postgresql.JSON, nullable=False),
+        sa.Column(
+            "modified_date", sa.DateTime(), nullable=False, server_default=sa.func.now()
+        ),
+        sa.Column("external_id", sa.Unicode(255)),
+        sa.Column("external_id2", sa.Unicode(255)),
     )
 
     op.create_table(
-        'alert_channels',
-        sa.Column('owner_id', sa.Integer(),
-                  sa.ForeignKey('users.id', onupdate='cascade',
-                                ondelete='cascade'), nullable=False),
-        sa.Column('channel_name', sa.Unicode(25), nullable=False),
-        sa.Column('channel_value', sa.Unicode(80), nullable=False),
-        sa.Column('channel_json_conf', sa.dialects.postgresql.JSON, nullable=False),
-        sa.Column('channel_validated', sa.Boolean, nullable=False, server_default='False'),
-        sa.Column('send_alerts', sa.Boolean, nullable=False, server_default='True'),
-        sa.Column('notify_only_first', sa.Boolean, nullable=False, server_default='False'),
-        sa.Column('daily_digest', sa.Boolean, nullable=False, server_default='True'),
-        sa.Column('pkey', sa.Integer(), primary_key=True),
-        sa.Column('integration_id', sa.Integer,
-                  sa.ForeignKey('integrations.id', onupdate='cascade',
-                                ondelete='cascade')),
-    )
-    op.create_unique_constraint('uq_alert_channels', 'alert_channels',
-                                ["owner_id", "channel_name", "channel_value"])
+        "alert_channels",
+        sa.Column(
+            "owner_id",
+            sa.Integer(),
+            sa.ForeignKey("users.id", onupdate="cascade", ondelete="cascade"),
+            nullable=False,
+        ),
+        sa.Column("channel_name", sa.Unicode(25), nullable=False),
+        sa.Column("channel_value", sa.Unicode(80), nullable=False),
+        sa.Column("channel_json_conf", sa.dialects.postgresql.JSON, nullable=False),
+        sa.Column(
+            "channel_validated", sa.Boolean, nullable=False, server_default="False"
+        ),
+        sa.Column("send_alerts", sa.Boolean, nullable=False, server_default="True"),
+        sa.Column(
+            "notify_only_first", sa.Boolean, nullable=False, server_default="False"
+        ),
+        sa.Column("daily_digest", sa.Boolean, nullable=False, server_default="True"),
+        sa.Column("pkey", sa.Integer(), primary_key=True),
+        sa.Column(
+            "integration_id",
+            sa.Integer,
+            sa.ForeignKey("integrations.id", onupdate="cascade", ondelete="cascade"),
+        ),
+    )
+    op.create_unique_constraint(
+        "uq_alert_channels",
+        "alert_channels",
+        ["owner_id", "channel_name", "channel_value"],
+    )
 
     op.create_table(
-        'alert_channels_actions',
-        sa.Column('owner_id', sa.Integer(), nullable=False),
-        sa.Column('resource_id', sa.Integer(),
-                  sa.ForeignKey('resources.resource_id', onupdate='cascade',
-                                ondelete='cascade')),
-        sa.Column('pkey', sa.Integer(), primary_key=True),
-        sa.Column('action', sa.Unicode(10), nullable=False, server_default='always'),
-        sa.Column('rule', sa.dialects.postgresql.JSON),
-        sa.Column('type', sa.Unicode(10), index=True),
-        sa.Column('other_id', sa.Unicode(40), index=True),
-        sa.Column('config', sa.dialects.postgresql.JSON),
-        sa.Column('name', sa.Unicode(255), server_default='')
+        "alert_channels_actions",
+        sa.Column("owner_id", sa.Integer(), nullable=False),
+        sa.Column(
+            "resource_id",
+            sa.Integer(),
+            sa.ForeignKey(
+                "resources.resource_id", onupdate="cascade", ondelete="cascade"
+            ),
+        ),
+        sa.Column("pkey", sa.Integer(), primary_key=True),
+        sa.Column("action", sa.Unicode(10), nullable=False, server_default="always"),
+        sa.Column("rule", sa.dialects.postgresql.JSON),
+        sa.Column("type", sa.Unicode(10), index=True),
+        sa.Column("other_id", sa.Unicode(40), index=True),
+        sa.Column("config", sa.dialects.postgresql.JSON),
+        sa.Column("name", sa.Unicode(255), server_default=""),
     )
 
-
     op.create_table(
-        'application_postprocess_conf',
-        sa.Column('pkey', sa.Integer(), primary_key=True),
-        sa.Column('do', sa.Unicode(25), nullable=False),
-        sa.Column('new_value', sa.UnicodeText(), nullable=False, server_default=''),
-        sa.Column('resource_id', sa.Integer(),
-                  sa.ForeignKey('resources.resource_id',
-                                onupdate='cascade',
-                                ondelete='cascade'), nullable=False),
-        sa.Column('rule', sa.dialects.postgresql.JSON),
+        "application_postprocess_conf",
+        sa.Column("pkey", sa.Integer(), primary_key=True),
+        sa.Column("do", sa.Unicode(25), nullable=False),
+        sa.Column("new_value", sa.UnicodeText(), nullable=False, server_default=""),
+        sa.Column(
+            "resource_id",
+            sa.Integer(),
+            sa.ForeignKey(
+                "resources.resource_id", onupdate="cascade", ondelete="cascade"
+            ),
+            nullable=False,
+        ),
+        sa.Column("rule", sa.dialects.postgresql.JSON),
     )
 
     op.create_table(
-        'applications',
-        sa.Column('resource_id', sa.Integer(),
-                  sa.ForeignKey('resources.resource_id', onupdate='cascade',
-                                ondelete='cascade'), nullable=False,
-                  primary_key=True, autoincrement=False),
-        sa.Column('domains', sa.UnicodeText, nullable=False),
-        sa.Column('api_key', sa.Unicode(32), nullable=False, index=True),
-        sa.Column('default_grouping', sa.Unicode(20), nullable=False, server_default='url_type'),
-        sa.Column('public_key', sa.Unicode(32), nullable=False, index=True),
-        sa.Column('error_report_threshold', sa.Integer(), server_default='10', nullable=False),
-        sa.Column('slow_report_threshold', sa.Integer(), server_default='10', nullable=False),
-        sa.Column('apdex_threshold', sa.Float(), server_default='0.7', nullable=False),
-        sa.Column('allow_permanent_storage', sa.Boolean(), server_default="false", nullable=False),
-    )
-    op.create_unique_constraint(None, 'applications',
-                                ["public_key"])
-    op.create_unique_constraint(None, 'applications',
-                                ["api_key"])
+        "applications",
+        sa.Column(
+            "resource_id",
+            sa.Integer(),
+            sa.ForeignKey(
+                "resources.resource_id", onupdate="cascade", ondelete="cascade"
+            ),
+            nullable=False,
+            primary_key=True,
+            autoincrement=False,
+        ),
+        sa.Column("domains", sa.UnicodeText, nullable=False),
+        sa.Column("api_key", sa.Unicode(32), nullable=False, index=True),
+        sa.Column(
+            "default_grouping",
+            sa.Unicode(20),
+            nullable=False,
+            server_default="url_type",
+        ),
+        sa.Column("public_key", sa.Unicode(32), nullable=False, index=True),
+        sa.Column(
+            "error_report_threshold", sa.Integer(), server_default="10", nullable=False
+        ),
+        sa.Column(
+            "slow_report_threshold", sa.Integer(), server_default="10", nullable=False
+        ),
+        sa.Column("apdex_threshold", sa.Float(), server_default="0.7", nullable=False),
+        sa.Column(
+            "allow_permanent_storage",
+            sa.Boolean(),
+            server_default="false",
+            nullable=False,
+        ),
+    )
+    op.create_unique_constraint(None, "applications", ["public_key"])
+    op.create_unique_constraint(None, "applications", ["api_key"])
 
     op.create_table(
-        'metrics',
-        sa.Column('pkey', sa.types.BigInteger, nullable=False, primary_key=True),
-        sa.Column('resource_id', sa.Integer(),
-                  sa.ForeignKey('resources.resource_id',
-                                onupdate='cascade',
-                                ondelete='cascade')),
-        sa.Column('timestamp', sa.DateTime),
-        sa.Column('namespace', sa.Unicode(255)),
-        sa.Column('tags', sa.dialects.postgresql.JSON, server_default="{}")
+        "metrics",
+        sa.Column("pkey", sa.types.BigInteger, nullable=False, primary_key=True),
+        sa.Column(
+            "resource_id",
+            sa.Integer(),
+            sa.ForeignKey(
+                "resources.resource_id", onupdate="cascade", ondelete="cascade"
+            ),
+        ),
+        sa.Column("timestamp", sa.DateTime),
+        sa.Column("namespace", sa.Unicode(255)),
+        sa.Column("tags", sa.dialects.postgresql.JSON, server_default="{}"),
     )
 
     op.create_table(
-        'events',
-        sa.Column('id', sa.Integer, nullable=False, primary_key=True),
-        sa.Column('start_date', sa.DateTime, nullable=False, index=True),
-        sa.Column('end_date', sa.DateTime),
-        sa.Column('status', sa.Integer(), nullable=False, index=True),
-        sa.Column('event_type', sa.Integer(), nullable=False, index=True),
-        sa.Column('origin_user_id', sa.Integer()),
-        sa.Column('target_user_id', sa.Integer()),
-        sa.Column('resource_id', sa.Integer(), index=True),
-        sa.Column('text', sa.UnicodeText, server_default=''),
-        sa.Column('values', sa.dialects.postgresql.JSON),
-        sa.Column('target_id', sa.Integer()),
-        sa.Column('target_uuid', sa.Unicode(40), index=True)
+        "events",
+        sa.Column("id", sa.Integer, nullable=False, primary_key=True),
+        sa.Column("start_date", sa.DateTime, nullable=False, index=True),
+        sa.Column("end_date", sa.DateTime),
+        sa.Column("status", sa.Integer(), nullable=False, index=True),
+        sa.Column("event_type", sa.Integer(), nullable=False, index=True),
+        sa.Column("origin_user_id", sa.Integer()),
+        sa.Column("target_user_id", sa.Integer()),
+        sa.Column("resource_id", sa.Integer(), index=True),
+        sa.Column("text", sa.UnicodeText, server_default=""),
+        sa.Column("values", sa.dialects.postgresql.JSON),
+        sa.Column("target_id", sa.Integer()),
+        sa.Column("target_uuid", sa.Unicode(40), index=True),
     )
 
     op.create_table(
-        'logs',
-        sa.Column('log_id', sa.types.BigInteger, nullable=False, primary_key=True),
-        sa.Column('resource_id', sa.Integer(),
-                  sa.ForeignKey('resources.resource_id',
-                                onupdate='cascade',
-                                ondelete='cascade')),
-        sa.Column('log_level', sa.SmallInteger(), nullable=False),
-        sa.Column('primary_key', sa.Unicode(128), nullable=True),
-        sa.Column('message', sa.UnicodeText, nullable=False, server_default=''),
-        sa.Column('timestamp', sa.DateTime),
-        sa.Column('namespace', sa.Unicode(255)),
-        sa.Column('request_id', sa.Unicode(40)),
-        sa.Column('tags', sa.dialects.postgresql.JSON, server_default="{}"),
-        sa.Column('permanent', sa.Boolean(), server_default="false",
-                  nullable=False)
+        "logs",
+        sa.Column("log_id", sa.types.BigInteger, nullable=False, primary_key=True),
+        sa.Column(
+            "resource_id",
+            sa.Integer(),
+            sa.ForeignKey(
+                "resources.resource_id", onupdate="cascade", ondelete="cascade"
+            ),
+        ),
+        sa.Column("log_level", sa.SmallInteger(), nullable=False),
+        sa.Column("primary_key", sa.Unicode(128), nullable=True),
+        sa.Column("message", sa.UnicodeText, nullable=False, server_default=""),
+        sa.Column("timestamp", sa.DateTime),
+        sa.Column("namespace", sa.Unicode(255)),
+        sa.Column("request_id", sa.Unicode(40)),
+        sa.Column("tags", sa.dialects.postgresql.JSON, server_default="{}"),
+        sa.Column("permanent", sa.Boolean(), server_default="false", nullable=False),
     )
 
     op.create_table(
-        'reports_groups',
-        sa.Column('id', sa.types.BigInteger, primary_key=True),
-        sa.Column('resource_id', sa.Integer,
-                  sa.ForeignKey('resources.resource_id', onupdate='cascade',
-                                ondelete='cascade'), nullable=False),
-        sa.Column('priority', sa.Integer, nullable=False, server_default="5"),
-        sa.Column('first_timestamp', sa.DateTime(), nullable=False, server_default=sa.func.now()),
-        sa.Column('last_timestamp', sa.DateTime()),
-        sa.Column('error', sa.UnicodeText, nullable=False, server_default=""),
-        sa.Column('grouping_hash', sa.Unicode(40), nullable=False, server_default=""),
-        sa.Column('triggered_postprocesses_ids', sa.dialects.postgresql.JSON, nullable=False, server_default="[]"),
-        sa.Column('report_type', sa.Integer, nullable=False, server_default="0"),
-        sa.Column('total_reports', sa.Integer, nullable=False, server_default="0"),
-        sa.Column('last_report', sa.Integer, nullable=False, server_default="0"),
-        sa.Column('occurences', sa.Integer, nullable=False, server_default="1"),
-        sa.Column('average_duration', sa.Float(), nullable=False, server_default="0"),
-        sa.Column('summed_duration', sa.Float(), nullable=False, server_default="0"),
-        sa.Column('notified', sa.Boolean, nullable=False, server_default="False"),
-        sa.Column('fixed', sa.Boolean, nullable=False, server_default="False"),
-        sa.Column('public', sa.Boolean, nullable=False, server_default="False"),
-        sa.Column('read', sa.Boolean, nullable=False, server_default="False"),
+        "reports_groups",
+        sa.Column("id", sa.types.BigInteger, primary_key=True),
+        sa.Column(
+            "resource_id",
+            sa.Integer,
+            sa.ForeignKey(
+                "resources.resource_id", onupdate="cascade", ondelete="cascade"
+            ),
+            nullable=False,
+        ),
+        sa.Column("priority", sa.Integer, nullable=False, server_default="5"),
+        sa.Column(
+            "first_timestamp",
+            sa.DateTime(),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.Column("last_timestamp", sa.DateTime()),
+        sa.Column("error", sa.UnicodeText, nullable=False, server_default=""),
+        sa.Column("grouping_hash", sa.Unicode(40), nullable=False, server_default=""),
+        sa.Column(
+            "triggered_postprocesses_ids",
+            sa.dialects.postgresql.JSON,
+            nullable=False,
+            server_default="[]",
+        ),
+        sa.Column("report_type", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("total_reports", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("last_report", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("occurences", sa.Integer, nullable=False, server_default="1"),
+        sa.Column("average_duration", sa.Float(), nullable=False, server_default="0"),
+        sa.Column("summed_duration", sa.Float(), nullable=False, server_default="0"),
+        sa.Column("notified", sa.Boolean, nullable=False, server_default="False"),
+        sa.Column("fixed", sa.Boolean, nullable=False, server_default="False"),
+        sa.Column("public", sa.Boolean, nullable=False, server_default="False"),
+        sa.Column("read", sa.Boolean, nullable=False, server_default="False"),
     )
 
     op.create_table(
-        'reports',
-        sa.Column('id', sa.types.BigInteger, primary_key=True),
-        sa.Column('group_id', sa.types.BigInteger,
-                  sa.ForeignKey('reports_groups.id', onupdate='cascade',
-                                ondelete='cascade'), nullable=False, index=True),
-        sa.Column('resource_id', sa.Integer, nullable=False, index=True),
-        sa.Column('report_type', sa.Integer, nullable=False, server_default="0"),
-        sa.Column('error', sa.UnicodeText, nullable=False, server_default=""),
-        sa.Column('extra', sa.dialects.postgresql.JSON, nullable=False, server_default="{}"),
-        sa.Column('request', sa.dialects.postgresql.JSON, nullable=False, server_default="{}"),
-        sa.Column('tags', sa.dialects.postgresql.JSON, nullable=False, server_default="{}"),
-        sa.Column('ip', sa.Unicode(39), nullable=False, server_default=""),
-        sa.Column('username', sa.Unicode(255), nullable=False, server_default=""),
-        sa.Column('user_agent', sa.Unicode(512), nullable=False, server_default=""),
-        sa.Column('url', sa.UnicodeText, nullable=False, server_default=""),
-        sa.Column('request_id', sa.Unicode(40), nullable=False, server_default=""),
-        sa.Column('request_stats', sa.dialects.postgresql.JSON, nullable=False, server_default="{}"),
-        sa.Column('traceback', sa.dialects.postgresql.JSON, nullable=False, server_default="{}"),
-        sa.Column('traceback_hash', sa.Unicode(40), nullable=False, server_default=""),
-        sa.Column('start_time', sa.DateTime(), nullable=False, server_default=sa.func.now()),
-        sa.Column('end_time', sa.DateTime()),
-        sa.Column('report_group_time', sa.DateTime, index=True, nullable=False, server_default=sa.func.now()),
-        sa.Column('duration', sa.Float(), nullable=False, server_default="0"),
-        sa.Column('http_status', sa.Integer, index=True),
-        sa.Column('url_domain', sa.Unicode(128)),
-        sa.Column('url_path', sa.UnicodeText),
-        sa.Column('language', sa.Integer, server_default="0"),
-    )
-    op.create_index(None, 'reports',
-                    [sa.text("(tags ->> 'server_name')")])
-    op.create_index(None, 'reports',
-                    [sa.text("(tags ->> 'view_name')")])
+        "reports",
+        sa.Column("id", sa.types.BigInteger, primary_key=True),
+        sa.Column(
+            "group_id",
+            sa.types.BigInteger,
+            sa.ForeignKey("reports_groups.id", onupdate="cascade", ondelete="cascade"),
+            nullable=False,
+            index=True,
+        ),
+        sa.Column("resource_id", sa.Integer, nullable=False, index=True),
+        sa.Column("report_type", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("error", sa.UnicodeText, nullable=False, server_default=""),
+        sa.Column(
+            "extra", sa.dialects.postgresql.JSON, nullable=False, server_default="{}"
+        ),
+        sa.Column(
+            "request", sa.dialects.postgresql.JSON, nullable=False, server_default="{}"
+        ),
+        sa.Column(
+            "tags", sa.dialects.postgresql.JSON, nullable=False, server_default="{}"
+        ),
+        sa.Column("ip", sa.Unicode(39), nullable=False, server_default=""),
+        sa.Column("username", sa.Unicode(255), nullable=False, server_default=""),
+        sa.Column("user_agent", sa.Unicode(512), nullable=False, server_default=""),
+        sa.Column("url", sa.UnicodeText, nullable=False, server_default=""),
+        sa.Column("request_id", sa.Unicode(40), nullable=False, server_default=""),
+        sa.Column(
+            "request_stats",
+            sa.dialects.postgresql.JSON,
+            nullable=False,
+            server_default="{}",
+        ),
+        sa.Column(
+            "traceback",
+            sa.dialects.postgresql.JSON,
+            nullable=False,
+            server_default="{}",
+        ),
+        sa.Column("traceback_hash", sa.Unicode(40), nullable=False, server_default=""),
+        sa.Column(
+            "start_time", sa.DateTime(), nullable=False, server_default=sa.func.now()
+        ),
+        sa.Column("end_time", sa.DateTime()),
+        sa.Column(
+            "report_group_time",
+            sa.DateTime,
+            index=True,
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.Column("duration", sa.Float(), nullable=False, server_default="0"),
+        sa.Column("http_status", sa.Integer, index=True),
+        sa.Column("url_domain", sa.Unicode(128)),
+        sa.Column("url_path", sa.UnicodeText),
+        sa.Column("language", sa.Integer, server_default="0"),
+    )
+    op.create_index(None, "reports", [sa.text("(tags ->> 'server_name')")])
+    op.create_index(None, "reports", [sa.text("(tags ->> 'view_name')")])
 
     op.create_table(
-        'reports_assignments',
-        sa.Column('group_id', sa.types.BigInteger, nullable=False, primary_key=True),
-        sa.Column('owner_id', sa.Integer,
-            sa.ForeignKey('users.id', onupdate='cascade',ondelete='cascade'),
-            nullable=False, primary_key=True),
-        sa.Column('report_time', sa.DateTime, nullable=False)
-        )
+        "reports_assignments",
+        sa.Column("group_id", sa.types.BigInteger, nullable=False, primary_key=True),
+        sa.Column(
+            "owner_id",
+            sa.Integer,
+            sa.ForeignKey("users.id", onupdate="cascade", ondelete="cascade"),
+            nullable=False,
+            primary_key=True,
+        ),
+        sa.Column("report_time", sa.DateTime, nullable=False),
+    )
 
     op.create_table(
-        'reports_comments',
-        sa.Column('comment_id', sa.Integer, primary_key=True),
-        sa.Column('body', sa.UnicodeText, nullable=False, server_default=''),
-        sa.Column('owner_id', sa.Integer,
-                  sa.ForeignKey('users.id', onupdate='cascade',
-                                ondelete='set null'), nullable=True),
-        sa.Column('created_timestamp', sa.DateTime, nullable=False, server_default=sa.func.now()),
-        sa.Column('report_time', sa.DateTime, nullable=False),
-        sa.Column('group_id', sa.types.BigInteger, nullable=False)
+        "reports_comments",
+        sa.Column("comment_id", sa.Integer, primary_key=True),
+        sa.Column("body", sa.UnicodeText, nullable=False, server_default=""),
+        sa.Column(
+            "owner_id",
+            sa.Integer,
+            sa.ForeignKey("users.id", onupdate="cascade", ondelete="set null"),
+            nullable=True,
+        ),
+        sa.Column(
+            "created_timestamp",
+            sa.DateTime,
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.Column("report_time", sa.DateTime, nullable=False),
+        sa.Column("group_id", sa.types.BigInteger, nullable=False),
     )
 
     op.create_table(
-        'reports_stats',
-        sa.Column('resource_id', sa.Integer, nullable=False, index=True),
-        sa.Column('start_interval', sa.DateTime, nullable=False, index=True),
-        sa.Column('group_id', sa.types.BigInteger, index=True),
-        sa.Column('occurences', sa.Integer, nullable=False, server_default='0', index=True),
-        sa.Column('owner_user_id', sa.Integer),
-        sa.Column('type', sa.Integer, index=True, nullable=False),
-        sa.Column('duration', sa.Float(), server_default='0'),
-        sa.Column('server_name', sa.Unicode(128),
-                  server_default=''),
-        sa.Column('view_name', sa.Unicode(128),
-                  server_default=''),
-        sa.Column('id', sa.BigInteger(), nullable=False, primary_key=True),
-    )
-    op.create_index('ix_reports_stats_start_interval_group_id', 'reports_stats',
-                    ["start_interval", "group_id"])
+        "reports_stats",
+        sa.Column("resource_id", sa.Integer, nullable=False, index=True),
+        sa.Column("start_interval", sa.DateTime, nullable=False, index=True),
+        sa.Column("group_id", sa.types.BigInteger, index=True),
+        sa.Column(
+            "occurences", sa.Integer, nullable=False, server_default="0", index=True
+        ),
+        sa.Column("owner_user_id", sa.Integer),
+        sa.Column("type", sa.Integer, index=True, nullable=False),
+        sa.Column("duration", sa.Float(), server_default="0"),
+        sa.Column("server_name", sa.Unicode(128), server_default=""),
+        sa.Column("view_name", sa.Unicode(128), server_default=""),
+        sa.Column("id", sa.BigInteger(), nullable=False, primary_key=True),
+    )
+    op.create_index(
+        "ix_reports_stats_start_interval_group_id",
+        "reports_stats",
+        ["start_interval", "group_id"],
+    )
 
     op.create_table(
-        'slow_calls',
-        sa.Column('id', sa.types.BigInteger, primary_key=True),
-        sa.Column('report_id', sa.types.BigInteger, sa.ForeignKey('reports.id', onupdate='cascade', ondelete='cascade'),
-                  nullable=False, index=True),
-        sa.Column('duration', sa.Float(), nullable=False, server_default="0", index=True),
-        sa.Column('timestamp', sa.DateTime, nullable=False, server_default=sa.func.now(), index=True),
-        sa.Column('report_group_time', sa.DateTime, index=True, nullable=False, server_default=sa.func.now()),
-        sa.Column('type', sa.Unicode(16), nullable=False, index=True),
-        sa.Column('statement', sa.UnicodeText, nullable=False, server_default=''),
-        sa.Column('parameters', sa.dialects.postgresql.JSON, nullable=False),
-        sa.Column('location', sa.UnicodeText, server_default=''),
-        sa.Column('subtype', sa.Unicode(16), nullable=False, index=True),
-        sa.Column('resource_id', sa.Integer, nullable=False, index=True),
-        sa.Column('statement_hash', sa.Unicode(60), index=True)
+        "slow_calls",
+        sa.Column("id", sa.types.BigInteger, primary_key=True),
+        sa.Column(
+            "report_id",
+            sa.types.BigInteger,
+            sa.ForeignKey("reports.id", onupdate="cascade", ondelete="cascade"),
+            nullable=False,
+            index=True,
+        ),
+        sa.Column(
+            "duration", sa.Float(), nullable=False, server_default="0", index=True
+        ),
+        sa.Column(
+            "timestamp",
+            sa.DateTime,
+            nullable=False,
+            server_default=sa.func.now(),
+            index=True,
+        ),
+        sa.Column(
+            "report_group_time",
+            sa.DateTime,
+            index=True,
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.Column("type", sa.Unicode(16), nullable=False, index=True),
+        sa.Column("statement", sa.UnicodeText, nullable=False, server_default=""),
+        sa.Column("parameters", sa.dialects.postgresql.JSON, nullable=False),
+        sa.Column("location", sa.UnicodeText, server_default=""),
+        sa.Column("subtype", sa.Unicode(16), nullable=False, index=True),
+        sa.Column("resource_id", sa.Integer, nullable=False, index=True),
+        sa.Column("statement_hash", sa.Unicode(60), index=True),
     )
 
     op.create_table(
-        'tags',
-        sa.Column('id', sa.types.BigInteger, primary_key=True),
-        sa.Column('resource_id', sa.Integer,
-                  sa.ForeignKey('resources.resource_id', onupdate='cascade',
-                                ondelete='cascade')),
-        sa.Column('first_timestamp', sa.DateTime, nullable=False, server_default=sa.func.now()),
-        sa.Column('last_timestamp', sa.DateTime, nullable=False, server_default=sa.func.now()),
-        sa.Column('name', sa.Unicode(32), nullable=False),
-        sa.Column('value', sa.dialects.postgresql.JSON, nullable=False),
-        sa.Column('times_seen', sa.Integer, nullable=False, server_default='1')
+        "tags",
+        sa.Column("id", sa.types.BigInteger, primary_key=True),
+        sa.Column(
+            "resource_id",
+            sa.Integer,
+            sa.ForeignKey(
+                "resources.resource_id", onupdate="cascade", ondelete="cascade"
+            ),
+        ),
+        sa.Column(
+            "first_timestamp", sa.DateTime, nullable=False, server_default=sa.func.now()
+        ),
+        sa.Column(
+            "last_timestamp", sa.DateTime, nullable=False, server_default=sa.func.now()
+        ),
+        sa.Column("name", sa.Unicode(32), nullable=False),
+        sa.Column("value", sa.dialects.postgresql.JSON, nullable=False),
+        sa.Column("times_seen", sa.Integer, nullable=False, server_default="1"),
     )
 
     op.create_table(
-        'auth_tokens',
-        sa.Column('id', sa.Integer, nullable=False, primary_key=True),
-        sa.Column('token', sa.Unicode),
-        sa.Column('creation_date', sa.DateTime, nullable=False, server_default=sa.func.now()),
-        sa.Column('expires', sa.DateTime),
-        sa.Column('owner_id', sa.Integer,
-                  sa.ForeignKey('users.id', onupdate='cascade',
-                                ondelete='cascade')),
-        sa.Column('description', sa.Unicode),
+        "auth_tokens",
+        sa.Column("id", sa.Integer, nullable=False, primary_key=True),
+        sa.Column("token", sa.Unicode),
+        sa.Column(
+            "creation_date", sa.DateTime, nullable=False, server_default=sa.func.now()
+        ),
+        sa.Column("expires", sa.DateTime),
+        sa.Column(
+            "owner_id",
+            sa.Integer,
+            sa.ForeignKey("users.id", onupdate="cascade", ondelete="cascade"),
+        ),
+        sa.Column("description", sa.Unicode),
     )
 
     op.create_table(
-        'channels_actions',
-        sa.Column('channel_pkey', sa.Integer,
-                  sa.ForeignKey('alert_channels.pkey',
-                                ondelete='CASCADE', onupdate='CASCADE')),
-        sa.Column('action_pkey', sa.Integer,
-                  sa.ForeignKey('alert_channels_actions.pkey',
-                                ondelete='CASCADE', onupdate='CASCADE'))
+        "channels_actions",
+        sa.Column(
+            "channel_pkey",
+            sa.Integer,
+            sa.ForeignKey(
+                "alert_channels.pkey", ondelete="CASCADE", onupdate="CASCADE"
+            ),
+        ),
+        sa.Column(
+            "action_pkey",
+            sa.Integer,
+            sa.ForeignKey(
+                "alert_channels_actions.pkey", ondelete="CASCADE", onupdate="CASCADE"
+            ),
+        ),
     )
 
     op.create_table(
-        'config',
-        sa.Column('key', sa.Unicode(128), primary_key=True),
-        sa.Column('section', sa.Unicode(128), primary_key=True),
-        sa.Column('value', sa.dialects.postgresql.JSON,
-                  server_default="{}")
+        "config",
+        sa.Column("key", sa.Unicode(128), primary_key=True),
+        sa.Column("section", sa.Unicode(128), primary_key=True),
+        sa.Column("value", sa.dialects.postgresql.JSON, server_default="{}"),
     )
 
     op.create_table(
-        'plugin_configs',
-        sa.Column('id', sa.Integer, primary_key=True),
-        sa.Column('plugin_name', sa.Unicode(128)),
-        sa.Column('section', sa.Unicode(128)),
-        sa.Column('config', sa.dialects.postgresql.JSON,
-                  server_default="{}"),
-        sa.Column('resource_id', sa.Integer(),
-                  sa.ForeignKey('resources.resource_id', onupdate='cascade',
-                                ondelete='cascade')),
-        sa.Column('owner_id', sa.Integer(),
-                  sa.ForeignKey('users.id', onupdate='cascade',
-                                ondelete='cascade')))
+        "plugin_configs",
+        sa.Column("id", sa.Integer, primary_key=True),
+        sa.Column("plugin_name", sa.Unicode(128)),
+        sa.Column("section", sa.Unicode(128)),
+        sa.Column("config", sa.dialects.postgresql.JSON, server_default="{}"),
+        sa.Column(
+            "resource_id",
+            sa.Integer(),
+            sa.ForeignKey(
+                "resources.resource_id", onupdate="cascade", ondelete="cascade"
+            ),
+        ),
+        sa.Column(
+            "owner_id",
+            sa.Integer(),
+            sa.ForeignKey("users.id", onupdate="cascade", ondelete="cascade"),
+        ),
+    )
 
     op.create_table(
-        'rc_versions',
-        sa.Column('name', sa.Unicode(40), primary_key=True),
-        sa.Column('value', sa.Unicode(40)),
+        "rc_versions",
+        sa.Column("name", sa.Unicode(40), primary_key=True),
+        sa.Column("value", sa.Unicode(40)),
+    )
+    version_table = sa.table(
+        "rc_versions",
+        sa.Column("name", sa.Unicode(40)),
+        sa.Column("value", sa.Unicode(40)),
     )
-    version_table = sa.table('rc_versions',
-                             sa.Column('name', sa.Unicode(40)),
-                             sa.Column('value', sa.Unicode(40)))
 
-    insert = version_table.insert().values(name='es_reports')
+    insert = version_table.insert().values(name="es_reports")
     op.execute(insert)
-    insert = version_table.insert().values(name='es_reports_groups')
+    insert = version_table.insert().values(name="es_reports_groups")
     op.execute(insert)
-    insert = version_table.insert().values(name='es_reports_stats')
+    insert = version_table.insert().values(name="es_reports_stats")
     op.execute(insert)
-    insert = version_table.insert().values(name='es_logs')
+    insert = version_table.insert().values(name="es_logs")
     op.execute(insert)
-    insert = version_table.insert().values(name='es_metrics')
+    insert = version_table.insert().values(name="es_metrics")
     op.execute(insert)
-    insert = version_table.insert().values(name='es_slow_calls')
+    insert = version_table.insert().values(name="es_slow_calls")
     op.execute(insert)
 
-
-    op.execute('''
+    op.execute(
+        """
         CREATE OR REPLACE FUNCTION floor_time_5min(timestamp without time zone)
           RETURNS timestamp without time zone AS
         $BODY$SELECT date_trunc('hour', $1) + INTERVAL '5 min' * FLOOR(date_part('minute', $1) / 5.0)$BODY$
           LANGUAGE sql VOLATILE;
-    ''')
+    """
+    )
 
-    op.execute('''
+    op.execute(
+        """
     CREATE OR REPLACE FUNCTION partition_logs() RETURNS trigger
         LANGUAGE plpgsql SECURITY DEFINER
         AS $$
     DECLARE
     main_table         varchar := 'logs';
     partitioned_table  varchar := '';
     BEGIN
 
     IF NEW.permanent THEN
         partitioned_table := main_table || '_p_' || date_part('year', NEW.timestamp)::TEXT || '_' || DATE_part('month', NEW.timestamp);
     ELSE
         partitioned_table := main_table || '_p_' || date_part('year', NEW.timestamp)::TEXT || '_' || DATE_part('month', NEW.timestamp) || '_' || DATE_part('day', NEW.timestamp);
     END IF;
 
     BEGIN
     EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
     EXCEPTION
     WHEN undefined_table THEN
         RAISE NOTICE 'A partition has been created %', partitioned_table;
         IF NEW.permanent THEN
         EXECUTE format('CREATE TABLE  IF NOT EXISTS %s ( CHECK( timestamp >= DATE %s AND timestamp < DATE %s)) INHERITS (%s)',
                 partitioned_table,
                 quote_literal(date_trunc('month', NEW.timestamp)::date) ,
                 quote_literal((date_trunc('month', NEW.timestamp)::date  + interval '1 month')::text),
                 main_table);
         EXECUTE format('ALTER TABLE %s ADD CONSTRAINT pk_%s PRIMARY KEY(log_id);', partitioned_table, partitioned_table);
         EXECUTE format('ALTER TABLE %s ADD CONSTRAINT fk_%s_resource_id FOREIGN KEY (resource_id) REFERENCES resources (resource_id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE;', partitioned_table, partitioned_table);
         EXECUTE format('CREATE INDEX ix_%s_timestamp ON %s (timestamp);', partitioned_table, partitioned_table);
         EXECUTE format('CREATE INDEX ix_%s_namespace_resource_id ON %s (namespace, resource_id);', partitioned_table, partitioned_table);
         EXECUTE format('CREATE INDEX ix_%s_resource_id ON %s (resource_id);', partitioned_table, partitioned_table);
         EXECUTE format('CREATE INDEX ix_%s_pkey_namespace ON %s (primary_key, namespace);', partitioned_table, partitioned_table);
         ELSE
         EXECUTE format('CREATE TABLE  IF NOT EXISTS %s ( CHECK( timestamp >= DATE %s AND timestamp < DATE %s)) INHERITS (%s)',
                 partitioned_table,
                 quote_literal(date_trunc('day', NEW.timestamp)::date) ,
                 quote_literal((date_trunc('day', NEW.timestamp)::date  + interval '1 day')::text),
                 main_table);
         EXECUTE format('ALTER TABLE %s ADD CONSTRAINT pk_%s_ PRIMARY KEY(log_id);', partitioned_table, partitioned_table);
         EXECUTE format('ALTER TABLE %s ADD CONSTRAINT fk_%s_resource_id FOREIGN KEY (resource_id) REFERENCES resources (resource_id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE;', partitioned_table, partitioned_table);
         EXECUTE format('CREATE INDEX ix_%s_timestamp ON %s (timestamp);', partitioned_table, partitioned_table);
         EXECUTE format('CREATE INDEX ix_%s_namespace_resource_id ON %s (namespace, resource_id);', partitioned_table, partitioned_table);
         EXECUTE format('CREATE INDEX ix_%s_resource_id ON %s (resource_id);', partitioned_table, partitioned_table);
         EXECUTE format('CREATE INDEX ix_%s_primary_key_namespace ON %s (primary_key,namespace);', partitioned_table, partitioned_table);
         END IF;
 
 
         EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
     END;
 
 
     RETURN NULL;
     END
     $$;
-    ''')
+    """
+    )
 
-    op.execute('''
+    op.execute(
+        """
     CREATE TRIGGER partition_logs BEFORE INSERT ON logs FOR EACH ROW EXECUTE PROCEDURE partition_logs();
-    ''')
+    """
+    )
 
-    op.execute('''
+    op.execute(
+        """
     CREATE OR REPLACE FUNCTION partition_metrics() RETURNS trigger
         LANGUAGE plpgsql SECURITY DEFINER
         AS $$
         DECLARE
         main_table         varchar := 'metrics';
         partitioned_table  varchar := '';
         BEGIN
 
         partitioned_table := main_table || '_p_' || date_part('year', NEW.timestamp)::TEXT || '_' || DATE_part('month', NEW.timestamp) || '_' || DATE_part('day', NEW.timestamp);
 
     BEGIN
     EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
     EXCEPTION
     WHEN undefined_table THEN
         RAISE NOTICE 'A partition has been created %', partitioned_table;
         EXECUTE format('CREATE TABLE  IF NOT EXISTS %s ( CHECK( timestamp >= DATE %s AND timestamp < DATE %s)) INHERITS (%s)',
                 partitioned_table,
                 quote_literal(date_trunc('day', NEW.timestamp)::date) ,
                 quote_literal((date_trunc('day', NEW.timestamp)::date  + interval '1 day')::text),
                 main_table);
         EXECUTE format('ALTER TABLE %s ADD CONSTRAINT pk_%s PRIMARY KEY(pkey);', partitioned_table, partitioned_table);
         EXECUTE format('ALTER TABLE %s ADD CONSTRAINT fk_%s_resource_id FOREIGN KEY (resource_id) REFERENCES resources (resource_id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE;', partitioned_table, partitioned_table);
         EXECUTE format('CREATE INDEX ix_%s_timestamp ON %s (timestamp);', partitioned_table, partitioned_table);
         EXECUTE format('CREATE INDEX ix_%s_resource_id ON %s (resource_id);', partitioned_table, partitioned_table);
         EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
     END;
 
     RETURN NULL;
     END
     $$;
-    ''')
+    """
+    )
 
-    op.execute('''
+    op.execute(
+        """
     CREATE TRIGGER partition_metrics BEFORE INSERT ON metrics FOR EACH ROW EXECUTE PROCEDURE partition_metrics();
-    ''')
+    """
+    )
 
-    op.execute('''
+    op.execute(
+        """
     CREATE FUNCTION partition_reports_stats() RETURNS trigger
         LANGUAGE plpgsql SECURITY DEFINER
         AS $$
         DECLARE
             main_table         varchar := 'reports_stats';
             partitioned_table  varchar := '';
         BEGIN
 
             partitioned_table := main_table || '_p_' || date_part('year', NEW.start_interval)::TEXT || '_' || DATE_part('month', NEW.start_interval);
 
             BEGIN
                 EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
             EXCEPTION
             WHEN undefined_table THEN
                 RAISE NOTICE 'A partition has been created %', partitioned_table;
                 EXECUTE format('CREATE TABLE  IF NOT EXISTS %s ( CHECK( start_interval >= DATE %s AND start_interval < DATE %s )) INHERITS (%s)',
                     partitioned_table,
                     quote_literal(date_trunc('month', NEW.start_interval)::date) ,
                     quote_literal((date_trunc('month', NEW.start_interval)::date  + interval '1 month')::text),
                     main_table);
                 EXECUTE format('ALTER TABLE %s ADD CONSTRAINT pk_%s PRIMARY KEY(id);', partitioned_table, partitioned_table);
                 EXECUTE format('CREATE INDEX ix_%s_start_interval ON %s USING btree (start_interval);', partitioned_table, partitioned_table);
                 EXECUTE format('CREATE INDEX ix_%s_type ON %s USING btree (type);', partitioned_table, partitioned_table);
                 EXECUTE format('CREATE INDEX ix_%s_resource_id ON %s USING btree (resource_id);', partitioned_table, partitioned_table);
                 EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
             END;
             RETURN NULL;
         END
         $$;
-    ''')
+    """
+    )
 
-    op.execute('''
+    op.execute(
+        """
     CREATE TRIGGER partition_reports_stats BEFORE INSERT ON reports_stats FOR EACH ROW EXECUTE PROCEDURE partition_reports_stats();
-    ''')
+    """
+    )
 
-    op.execute('''
+    op.execute(
+        """
     CREATE OR REPLACE FUNCTION partition_reports_groups() RETURNS trigger
         LANGUAGE plpgsql SECURITY DEFINER
         AS $$
     DECLARE
         main_table         varchar := 'reports_groups';
         partitioned_table  varchar := '';
     BEGIN
 
         partitioned_table := main_table || '_p_' || date_part('year', NEW.first_timestamp)::TEXT || '_' || DATE_part('month', NEW.first_timestamp);
 
         BEGIN
             EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
         EXCEPTION
         WHEN undefined_table THEN
             RAISE NOTICE 'A partition has been created %', partitioned_table;
             EXECUTE format('CREATE TABLE  IF NOT EXISTS %s ( CHECK( first_timestamp >= DATE %s AND first_timestamp < DATE %s )) INHERITS (%s)',
                 partitioned_table,
                 quote_literal(date_trunc('month', NEW.first_timestamp)::date) ,
                 quote_literal((date_trunc('month', NEW.first_timestamp)::date  + interval '1 month')::text),
                 main_table);
             EXECUTE format('ALTER TABLE %s ADD CONSTRAINT pk_%s PRIMARY KEY(id);', partitioned_table, partitioned_table);
             EXECUTE format('ALTER TABLE %s ADD CONSTRAINT fk_%s_resource_id FOREIGN KEY (resource_id) REFERENCES resources (resource_id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE;', partitioned_table, partitioned_table);
             EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
         END;
         RETURN NULL;
     END
     $$;
-    ''')
+    """
+    )
 
-    op.execute('''
+    op.execute(
+        """
     CREATE TRIGGER partition_reports_groups BEFORE INSERT ON reports_groups FOR EACH ROW EXECUTE PROCEDURE partition_reports_groups();
-    ''')
+    """
+    )
 
-    op.execute('''
+    op.execute(
+        """
     CREATE OR REPLACE FUNCTION partition_reports() RETURNS trigger
         LANGUAGE plpgsql SECURITY DEFINER
         AS $$
     DECLARE
         main_table         varchar := 'reports';
         partitioned_table  varchar := '';
         partitioned_parent_table  varchar := '';
     BEGIN
 
         partitioned_table := main_table || '_p_' || date_part('year', NEW.report_group_time)::TEXT || '_' || DATE_part('month', NEW.report_group_time);
         partitioned_parent_table := 'reports_groups_p_' || date_part('year', NEW.report_group_time)::TEXT || '_' || DATE_part('month', NEW.report_group_time);
 
         BEGIN
             EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
         EXCEPTION
         WHEN undefined_table THEN
             RAISE NOTICE 'A partition has been created %', partitioned_table;
             EXECUTE format('CREATE TABLE  IF NOT EXISTS %s ( CHECK( report_group_time >= DATE %s AND report_group_time < DATE %s )) INHERITS (%s)',
                 partitioned_table,
                 quote_literal(date_trunc('month', NEW.report_group_time)::date) ,
                 quote_literal((date_trunc('month', NEW.report_group_time)::date  + interval '1 month')::text),
                 main_table);
             EXECUTE format('ALTER TABLE %s ADD CONSTRAINT pk_%s PRIMARY KEY(id);', partitioned_table, partitioned_table);
             EXECUTE format('ALTER TABLE %s ADD CONSTRAINT fk_%s_resource_id FOREIGN KEY (resource_id) REFERENCES resources (resource_id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE;', partitioned_table, partitioned_table);
             EXECUTE format('ALTER TABLE %s ADD CONSTRAINT fk_%s_group_id FOREIGN KEY (group_id) REFERENCES %s (id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE;', partitioned_table, partitioned_table, partitioned_parent_table);
             EXECUTE format('CREATE INDEX ix_%s_report_group_time ON %s USING btree (report_group_time);', partitioned_table, partitioned_table);
             EXECUTE format('CREATE INDEX ix_%s_group_id ON %s USING btree (group_id);', partitioned_table, partitioned_table);
             EXECUTE format('CREATE INDEX ix_%s_resource_id ON %s USING btree (resource_id);', partitioned_table, partitioned_table);
             EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
         END;
         RETURN NULL;
     END
     $$;
-    ''')
+    """
+    )
 
-    op.execute('''
+    op.execute(
+        """
     CREATE TRIGGER partition_reports BEFORE INSERT ON reports FOR EACH ROW EXECUTE PROCEDURE partition_reports();
-    ''')
-
+    """
+    )
 
-    op.execute('''
+    op.execute(
+        """
     CREATE OR REPLACE FUNCTION partition_slow_calls() RETURNS trigger
         LANGUAGE plpgsql SECURITY DEFINER
         AS $$
     DECLARE
         main_table         varchar := 'slow_calls';
         partitioned_table  varchar := '';
         partitioned_parent_table  varchar := '';
     BEGIN
 
         partitioned_table := main_table || '_p_' || date_part('year', NEW.report_group_time)::TEXT || '_' || DATE_part('month', NEW.report_group_time);
         partitioned_parent_table := 'reports_p_' || date_part('year', NEW.report_group_time)::TEXT || '_' || DATE_part('month', NEW.report_group_time);
 
         BEGIN
             EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
         EXCEPTION
         WHEN undefined_table THEN
             RAISE NOTICE 'A partition has been created %', partitioned_table;
             EXECUTE format('CREATE TABLE  IF NOT EXISTS %s ( CHECK( report_group_time >= DATE %s AND report_group_time < DATE %s )) INHERITS (%s)',
                 partitioned_table,
                 quote_literal(date_trunc('month', NEW.report_group_time)::date) ,
                 quote_literal((date_trunc('month', NEW.report_group_time)::date  + interval '1 month')::text),
                 main_table);
             EXECUTE format('ALTER TABLE %s ADD CONSTRAINT pk_%s PRIMARY KEY(id);', partitioned_table, partitioned_table);
             EXECUTE format('ALTER TABLE %s ADD CONSTRAINT fk_%s_resource_id FOREIGN KEY (resource_id) REFERENCES resources (resource_id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE;', partitioned_table, partitioned_table);
             EXECUTE format('ALTER TABLE %s ADD CONSTRAINT fk_%s_report_id FOREIGN KEY (report_id) REFERENCES %s (id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE;', partitioned_table, partitioned_table, partitioned_parent_table);
             EXECUTE format('CREATE INDEX ix_%s_resource_id  ON %s USING btree (resource_id);', partitioned_table, partitioned_table);
             EXECUTE format('CREATE INDEX ix_%s_report_id ON %s USING btree (report_id);', partitioned_table, partitioned_table);
             EXECUTE format('CREATE INDEX ix_%s_timestamp ON %s USING btree (timestamp);', partitioned_table, partitioned_table);
             EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
         END;
         RETURN NULL;
     END
     $$;
-    ''')
+    """
+    )
 
-    op.execute('''
+    op.execute(
+        """
     CREATE TRIGGER partition_slow_calls BEFORE INSERT ON slow_calls FOR EACH ROW EXECUTE PROCEDURE partition_slow_calls();
-    ''')
+    """
+    )
+
 
 def downgrade():
     pass

backend/src/appenlight/migrations/versions/e9fcfbdd9498_connect_resources_to_alert_channels.py

@@ -1,32 +1,40 b''
 """connect resources to alert_channels
 
 Revision ID: e9fcfbdd9498
 Revises: 55b6e612672f
 Create Date: 2018-02-28 13:52:50.717217
 
 """
 
 # revision identifiers, used by Alembic.
-revision = 'e9fcfbdd9498'
-down_revision = '55b6e612672f'
+revision = "e9fcfbdd9498"
+down_revision = "55b6e612672f"
 
 from alembic import op
 import sqlalchemy as sa
 
 
 def upgrade():
     op.create_table(
-        'channels_resources',
-        sa.Column('channel_pkey', sa.Integer,
-                  sa.ForeignKey('alert_channels.pkey',
-                                ondelete='CASCADE', onupdate='CASCADE'),
-                  primary_key=True),
-        sa.Column('resource_id', sa.Integer,
-                  sa.ForeignKey('resources.resource_id',
-                                ondelete='CASCADE', onupdate='CASCADE'),
-                  primary_key=True)
+        "channels_resources",
+        sa.Column(
+            "channel_pkey",
+            sa.Integer,
+            sa.ForeignKey(
+                "alert_channels.pkey", ondelete="CASCADE", onupdate="CASCADE"
+            ),
+            primary_key=True,
+        ),
+        sa.Column(
+            "resource_id",
+            sa.Integer,
+            sa.ForeignKey(
+                "resources.resource_id", ondelete="CASCADE", onupdate="CASCADE"
+            ),
+            primary_key=True,
+        ),
     )
 
 
 def downgrade():
-    op.drop_table('channels_resources')
+    op.drop_table("channels_resources")

backend/src/appenlight/models/__init__.py

@@ -1,130 +1,139 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy import MetaData
 from sqlalchemy.orm import scoped_session
 from sqlalchemy.orm import sessionmaker
 from zope.sqlalchemy import ZopeTransactionExtension
 import ziggurat_foundations
 from ziggurat_foundations.models.base import get_db_session
 
 log = logging.getLogger(__name__)
 
 DBSession = scoped_session(sessionmaker(extension=ZopeTransactionExtension()))
 
 NAMING_CONVENTION = {
-    "ix": 'ix_%(column_0_label)s',
+    "ix": "ix_%(column_0_label)s",
     "uq": "uq_%(table_name)s_%(column_0_name)s",
     "ck": "ck_%(table_name)s_%(constraint_name)s",
     "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
-    "pk": "pk_%(table_name)s"
+    "pk": "pk_%(table_name)s",
 }
 
 metadata = MetaData(naming_convention=NAMING_CONVENTION)
 Base = declarative_base(metadata=metadata)
 
 # optional for request.db approach
 ziggurat_foundations.models.DBSession = DBSession
 
 
 class Datastores(object):
     redis = None
     es = None
 
 
 def register_datastores(es_conn, redis_conn, redis_lockmgr):
     Datastores.es = es_conn
     Datastores.redis = redis_conn
     Datastores.lockmgr = redis_lockmgr
 
 
 class SliceableESQuery(object):
     def __init__(self, query, sort_query=None, aggregations=False, **kwconfig):
         self.query = query
         self.sort_query = sort_query
         self.aggregations = aggregations
-        self.items_per_page = kwconfig.pop('items_per_page', 10)
-        self.page = kwconfig.pop('page', 1)
+        self.items_per_page = kwconfig.pop("items_per_page", 10)
+        self.page = kwconfig.pop("page", 1)
         self.kwconfig = kwconfig
         self.result = None
 
     def __getitem__(self, index):
         config = self.kwconfig.copy()
-        config['from_'] = index.start
+        config["from_"] = index.start
         query = self.query.copy()
         if self.sort_query:
             query.update(self.sort_query)
-        self.result = Datastores.es.search(body=query, size=self.items_per_page,
-                                           **config)
+        self.result = Datastores.es.search(
+            body=query, size=self.items_per_page, **config
+        )
         if self.aggregations:
-            self.items = self.result.get('aggregations')
+            self.items = self.result.get("aggregations")
         else:
-            self.items = self.result['hits']['hits']
+            self.items = self.result["hits"]["hits"]
 
         return self.items
 
     def __iter__(self):
         return self.result
 
     def __len__(self):
         config = self.kwconfig.copy()
         query = self.query.copy()
-        self.result = Datastores.es.search(body=query, size=self.items_per_page,
-                                           **config)
+        self.result = Datastores.es.search(
+            body=query, size=self.items_per_page, **config
+        )
         if self.aggregations:
-            self.items = self.result.get('aggregations')
+            self.items = self.result.get("aggregations")
         else:
-            self.items = self.result['hits']['hits']
+            self.items = self.result["hits"]["hits"]
 
-        count = int(self.result['hits']['total'])
+        count = int(self.result["hits"]["total"])
         return count if count < 5000 else 5000
 
 
 from appenlight.models.resource import Resource
 from appenlight.models.application import Application
 from appenlight.models.user import User
 from appenlight.models.alert_channel import AlertChannel
 from appenlight.models.alert_channel_action import AlertChannelAction
 from appenlight.models.metric import Metric
-from appenlight.models.application_postprocess_conf import \
-    ApplicationPostprocessConf
+from appenlight.models.application_postprocess_conf import ApplicationPostprocessConf
 from appenlight.models.auth_token import AuthToken
 from appenlight.models.event import Event
 from appenlight.models.external_identity import ExternalIdentity
 from appenlight.models.group import Group
 from appenlight.models.group_permission import GroupPermission
 from appenlight.models.group_resource_permission import GroupResourcePermission
 from appenlight.models.log import Log
 from appenlight.models.plugin_config import PluginConfig
 from appenlight.models.report import Report
 from appenlight.models.report_group import ReportGroup
 from appenlight.models.report_comment import ReportComment
 from appenlight.models.report_assignment import ReportAssignment
 from appenlight.models.report_stat import ReportStat
 from appenlight.models.slow_call import SlowCall
 from appenlight.models.tag import Tag
 from appenlight.models.user_group import UserGroup
 from appenlight.models.user_permission import UserPermission
 from appenlight.models.user_resource_permission import UserResourcePermission
 from ziggurat_foundations import ziggurat_model_init
 
-ziggurat_model_init(User, Group, UserGroup, GroupPermission, UserPermission,
-                    UserResourcePermission, GroupResourcePermission,
-                    Resource,
-                    ExternalIdentity, passwordmanager=None)
+ziggurat_model_init(
+    User,
+    Group,
+    UserGroup,
+    GroupPermission,
+    UserPermission,
+    UserResourcePermission,
+    GroupResourcePermission,
+    Resource,
+    ExternalIdentity,
+    passwordmanager=None,
+)

backend/src/appenlight/models/alert_channel.py

@@ -1,305 +1,298 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 import sqlalchemy as sa
 import urllib.request, urllib.parse, urllib.error
 from datetime import timedelta
 from appenlight.models import Base
 from appenlight.lib.utils.date_utils import convert_date
 from sqlalchemy.dialects.postgresql import JSON
 from ziggurat_foundations.models.base import BaseModel
 
 log = logging.getLogger(__name__)
 
 #
 channel_rules_m2m_table = sa.Table(
-    'channels_actions', Base.metadata,
-    sa.Column('channel_pkey', sa.Integer,
-              sa.ForeignKey('alert_channels.pkey')),
-    sa.Column('action_pkey', sa.Integer,
-              sa.ForeignKey('alert_channels_actions.pkey'))
+    "channels_actions",
+    Base.metadata,
+    sa.Column("channel_pkey", sa.Integer, sa.ForeignKey("alert_channels.pkey")),
+    sa.Column("action_pkey", sa.Integer, sa.ForeignKey("alert_channels_actions.pkey")),
 )
 
 channel_resources_m2m_table = sa.Table(
-    'channels_resources', Base.metadata,
-    sa.Column('channel_pkey', sa.Integer,
-              sa.ForeignKey('alert_channels.pkey')),
-    sa.Column('resource_id', sa.Integer,
-              sa.ForeignKey('resources.resource_id'))
+    "channels_resources",
+    Base.metadata,
+    sa.Column("channel_pkey", sa.Integer, sa.ForeignKey("alert_channels.pkey")),
+    sa.Column("resource_id", sa.Integer, sa.ForeignKey("resources.resource_id")),
 )
 
-DATE_FRMT = '%Y-%m-%dT%H:%M'
+DATE_FRMT = "%Y-%m-%dT%H:%M"
 
 
 class AlertChannel(Base, BaseModel):
     """
     Stores information about possible alerting options
     """
-    __tablename__ = 'alert_channels'
-    __possible_channel_names__ = ['email']
+
+    __tablename__ = "alert_channels"
+    __possible_channel_names__ = ["email"]
     __mapper_args__ = {
-        'polymorphic_on': 'channel_name',
-        'polymorphic_identity': 'integration'
+        "polymorphic_on": "channel_name",
+        "polymorphic_identity": "integration",
     }
 
-    owner_id = sa.Column(sa.Unicode(30),
-                         sa.ForeignKey('users.id', onupdate='CASCADE',
-                                       ondelete='CASCADE'))
+    owner_id = sa.Column(
+        sa.Unicode(30),
+        sa.ForeignKey("users.id", onupdate="CASCADE", ondelete="CASCADE"),
+    )
     channel_name = sa.Column(sa.Unicode(25), nullable=False)
-    channel_value = sa.Column(sa.Unicode(80), nullable=False, default='')
-    channel_json_conf = sa.Column(JSON(), nullable=False, default='')
-    channel_validated = sa.Column(sa.Boolean, nullable=False,
-                                  default=False)
-    send_alerts = sa.Column(sa.Boolean, nullable=False,
-                            default=True)
-    daily_digest = sa.Column(sa.Boolean, nullable=False,
-                             default=True)
-    integration_id = sa.Column(sa.Integer, sa.ForeignKey('integrations.id'),
-                               nullable=True)
+    channel_value = sa.Column(sa.Unicode(80), nullable=False, default="")
+    channel_json_conf = sa.Column(JSON(), nullable=False, default="")
+    channel_validated = sa.Column(sa.Boolean, nullable=False, default=False)
+    send_alerts = sa.Column(sa.Boolean, nullable=False, default=True)
+    daily_digest = sa.Column(sa.Boolean, nullable=False, default=True)
+    integration_id = sa.Column(
+        sa.Integer, sa.ForeignKey("integrations.id"), nullable=True
+    )
     pkey = sa.Column(sa.Integer(), nullable=False, primary_key=True)
 
-    channel_actions = sa.orm.relationship('AlertChannelAction',
-                                          cascade="all",
-                                          passive_deletes=True,
-                                          passive_updates=True,
-                                          secondary=channel_rules_m2m_table,
-                                          backref='channels')
-    resources = sa.orm.relationship('Resource',
-                                    cascade="all",
-                                    passive_deletes=True,
-                                    passive_updates=True,
-                                    secondary=channel_resources_m2m_table,
-                                    backref='resources')
+    channel_actions = sa.orm.relationship(
+        "AlertChannelAction",
+        cascade="all",
+        passive_deletes=True,
+        passive_updates=True,
+        secondary=channel_rules_m2m_table,
+        backref="channels",
+    )
+    resources = sa.orm.relationship(
+        "Resource",
+        cascade="all",
+        passive_deletes=True,
+        passive_updates=True,
+        secondary=channel_resources_m2m_table,
+        backref="resources",
+    )
 
     @property
     def channel_visible_value(self):
         if self.integration:
-            return '{}: {}'.format(
-                self.channel_name,
-                self.integration.resource.resource_name
+            return "{}: {}".format(
+                self.channel_name, self.integration.resource.resource_name
             )
 
-        return '{}: {}'.format(
-            self.channel_name,
-            self.channel_value
-        )
+        return "{}: {}".format(self.channel_name, self.channel_value)
 
-    def get_dict(self, exclude_keys=None, include_keys=None,
-                 extended_info=True):
+    def get_dict(self, exclude_keys=None, include_keys=None, extended_info=True):
         """
         Returns dictionary with required information that will be consumed by
         angular
         """
-        instance_dict = super(AlertChannel, self).get_dict(exclude_keys,
-                                                           include_keys)
+        instance_dict = super(AlertChannel, self).get_dict(exclude_keys, include_keys)
         exclude_keys_list = exclude_keys or []
         include_keys_list = include_keys or []
 
-        instance_dict['supports_report_alerting'] = True
-        instance_dict['channel_visible_value'] = self.channel_visible_value
+        instance_dict["supports_report_alerting"] = True
+        instance_dict["channel_visible_value"] = self.channel_visible_value
 
         if extended_info:
-            instance_dict['actions'] = [
-                rule.get_dict(extended_info=True) for
-                rule in self.channel_actions]
+            instance_dict["actions"] = [
+                rule.get_dict(extended_info=True) for rule in self.channel_actions
+            ]
 
-        del instance_dict['channel_json_conf']
+        del instance_dict["channel_json_conf"]
 
         if self.integration:
             instance_dict[
-                'supports_report_alerting'] = \
-                self.integration.supports_report_alerting
+                "supports_report_alerting"
+            ] = self.integration.supports_report_alerting
         d = {}
         for k in instance_dict.keys():
-            if (k not in exclude_keys_list and
-                    (k in include_keys_list or not include_keys)):
+            if k not in exclude_keys_list and (
+                k in include_keys_list or not include_keys
+            ):
                 d[k] = instance_dict[k]
         return d
 
     def __repr__(self):
-        return '<AlertChannel: (%s,%s), user:%s>' % (self.channel_name,
-                                                     self.channel_value,
-                                                     self.user_name,)
+        return "<AlertChannel: (%s,%s), user:%s>" % (
+            self.channel_name,
+            self.channel_value,
+            self.user_name,
+        )
 
     def send_digest(self, **kwargs):
         """
         This should implement daily top error report notifications
         """
-        log.warning('send_digest NOT IMPLEMENTED')
+        log.warning("send_digest NOT IMPLEMENTED")
 
     def notify_reports(self, **kwargs):
         """
         This should implement notification of reports that occured in 1 min
         interval
         """
-        log.warning('notify_reports NOT IMPLEMENTED')
+        log.warning("notify_reports NOT IMPLEMENTED")
 
     def notify_alert(self, **kwargs):
         """
         Notify user of report/uptime/chart threshold events based on events alert
         type
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
-        alert_name = kwargs['event'].unified_alert_name()
-        if alert_name in ['slow_report_alert', 'error_report_alert']:
+        alert_name = kwargs["event"].unified_alert_name()
+        if alert_name in ["slow_report_alert", "error_report_alert"]:
             self.notify_report_alert(**kwargs)
-        elif alert_name == 'uptime_alert':
+        elif alert_name == "uptime_alert":
             self.notify_uptime_alert(**kwargs)
-        elif alert_name == 'chart_alert':
+        elif alert_name == "chart_alert":
             self.notify_chart_alert(**kwargs)
 
     def notify_chart_alert(self, **kwargs):
         """
         This should implement report open/close alerts notifications
         """
-        log.warning('notify_chart_alert NOT IMPLEMENTED')
+        log.warning("notify_chart_alert NOT IMPLEMENTED")
 
     def notify_report_alert(self, **kwargs):
         """
         This should implement report open/close alerts notifications
         """
-        log.warning('notify_report_alert NOT IMPLEMENTED')
+        log.warning("notify_report_alert NOT IMPLEMENTED")
 
     def notify_uptime_alert(self, **kwargs):
         """
         This should implement uptime open/close alerts notifications
         """
-        log.warning('notify_uptime_alert NOT IMPLEMENTED')
+        log.warning("notify_uptime_alert NOT IMPLEMENTED")
 
     def get_notification_basic_vars(self, kwargs):
         """
         Sets most common variables used later for rendering notifications for
         channel
         """
-        if 'event' in kwargs:
-            kwargs['since_when'] = kwargs['event'].start_date
+        if "event" in kwargs:
+            kwargs["since_when"] = kwargs["event"].start_date
 
-        url_start_date = kwargs.get('since_when') - timedelta(minutes=1)
-        url_end_date = kwargs.get('since_when') + timedelta(minutes=4)
+        url_start_date = kwargs.get("since_when") - timedelta(minutes=1)
+        url_end_date = kwargs.get("since_when") + timedelta(minutes=4)
         tmpl_vars = {
-            "timestamp": kwargs['since_when'],
-            "user": kwargs['user'],
-            "since_when": kwargs.get('since_when'),
+            "timestamp": kwargs["since_when"],
+            "user": kwargs["user"],
+            "since_when": kwargs.get("since_when"),
             "url_start_date": url_start_date,
-            "url_end_date": url_end_date
+            "url_end_date": url_end_date,
         }
-        tmpl_vars["resource_name"] = kwargs['resource'].resource_name
-        tmpl_vars["resource"] = kwargs['resource']
+        tmpl_vars["resource_name"] = kwargs["resource"].resource_name
+        tmpl_vars["resource"] = kwargs["resource"]
 
-        if 'event' in kwargs:
-            tmpl_vars['event_values'] = kwargs['event'].values
-            tmpl_vars['alert_type'] = kwargs['event'].unified_alert_name()
-            tmpl_vars['alert_action'] = kwargs['event'].unified_alert_action()
+        if "event" in kwargs:
+            tmpl_vars["event_values"] = kwargs["event"].values
+            tmpl_vars["alert_type"] = kwargs["event"].unified_alert_name()
+            tmpl_vars["alert_action"] = kwargs["event"].unified_alert_action()
         return tmpl_vars
 
     def report_alert_notification_vars(self, kwargs):
         tmpl_vars = self.get_notification_basic_vars(kwargs)
-        reports = kwargs.get('reports', [])
+        reports = kwargs.get("reports", [])
         tmpl_vars["reports"] = reports
         tmpl_vars["confirmed_total"] = len(reports)
 
         tmpl_vars["report_type"] = "error reports"
-        tmpl_vars["url_report_type"] = 'report/list'
+        tmpl_vars["url_report_type"] = "report/list"
 
-        alert_type = tmpl_vars.get('alert_type', '')
-        if 'slow_report' in alert_type:
+        alert_type = tmpl_vars.get("alert_type", "")
+        if "slow_report" in alert_type:
             tmpl_vars["report_type"] = "slow reports"
-            tmpl_vars["url_report_type"] = 'report/list_slow'
+            tmpl_vars["url_report_type"] = "report/list_slow"
 
-        app_url = kwargs['request'].registry.settings['_mail_url']
+        app_url = kwargs["request"].registry.settings["_mail_url"]
 
-        destination_url = kwargs['request'].route_url('/',
-                                                      _app_url=app_url)
+        destination_url = kwargs["request"].route_url("/", _app_url=app_url)
         if alert_type:
-            destination_url += 'ui/{}?resource={}&start_date={}&end_date={}'.format(
+            destination_url += "ui/{}?resource={}&start_date={}&end_date={}".format(
                 tmpl_vars["url_report_type"],
-                tmpl_vars['resource'].resource_id,
-                tmpl_vars['url_start_date'].strftime(DATE_FRMT),
-                tmpl_vars['url_end_date'].strftime(DATE_FRMT)
+                tmpl_vars["resource"].resource_id,
+                tmpl_vars["url_start_date"].strftime(DATE_FRMT),
+                tmpl_vars["url_end_date"].strftime(DATE_FRMT),
             )
         else:
-            destination_url += 'ui/{}?resource={}'.format(
-                tmpl_vars["url_report_type"],
-                tmpl_vars['resource'].resource_id
+            destination_url += "ui/{}?resource={}".format(
+                tmpl_vars["url_report_type"], tmpl_vars["resource"].resource_id
             )
         tmpl_vars["destination_url"] = destination_url
 
         return tmpl_vars
 
     def uptime_alert_notification_vars(self, kwargs):
         tmpl_vars = self.get_notification_basic_vars(kwargs)
-        app_url = kwargs['request'].registry.settings['_mail_url']
-        destination_url = kwargs['request'].route_url('/', _app_url=app_url)
-        destination_url += 'ui/{}?resource={}'.format(
-            'uptime',
-            tmpl_vars['resource'].resource_id)
-        tmpl_vars['destination_url'] = destination_url
-
-        reason = ''
-        e_values = tmpl_vars.get('event_values')
-
-        if e_values and e_values.get('response_time') == 0:
-            reason += ' Response time was slower than 20 seconds.'
+        app_url = kwargs["request"].registry.settings["_mail_url"]
+        destination_url = kwargs["request"].route_url("/", _app_url=app_url)
+        destination_url += "ui/{}?resource={}".format(
+            "uptime", tmpl_vars["resource"].resource_id
+        )
+        tmpl_vars["destination_url"] = destination_url
+
+        reason = ""
+        e_values = tmpl_vars.get("event_values")
+
+        if e_values and e_values.get("response_time") == 0:
+            reason += " Response time was slower than 20 seconds."
         elif e_values:
-            code = e_values.get('status_code')
-            reason += ' Response status code: %s.' % code
+            code = e_values.get("status_code")
+            reason += " Response status code: %s." % code
 
-        tmpl_vars['reason'] = reason
+        tmpl_vars["reason"] = reason
         return tmpl_vars
 
     def chart_alert_notification_vars(self, kwargs):
         tmpl_vars = self.get_notification_basic_vars(kwargs)
-        tmpl_vars['chart_name'] = tmpl_vars['event_values']['chart_name']
-        tmpl_vars['action_name'] = tmpl_vars['event_values'].get(
-            'action_name') or ''
-        matched_values = tmpl_vars['event_values']['matched_step_values']
-        tmpl_vars['readable_values'] = []
-        for key, value in list(matched_values['values'].items()):
-            matched_label = matched_values['labels'].get(key)
+        tmpl_vars["chart_name"] = tmpl_vars["event_values"]["chart_name"]
+        tmpl_vars["action_name"] = tmpl_vars["event_values"].get("action_name") or ""
+        matched_values = tmpl_vars["event_values"]["matched_step_values"]
+        tmpl_vars["readable_values"] = []
+        for key, value in list(matched_values["values"].items()):
+            matched_label = matched_values["labels"].get(key)
             if matched_label:
-                tmpl_vars['readable_values'].append({
-                    'label': matched_label['human_label'],
-                    'value': value
-                })
-        tmpl_vars['readable_values'] = sorted(tmpl_vars['readable_values'],
-                                              key=lambda x: x['label'])
-        start_date = convert_date(tmpl_vars['event_values']['start_interval'])
+                tmpl_vars["readable_values"].append(
+                    {"label": matched_label["human_label"], "value": value}
+                )
+        tmpl_vars["readable_values"] = sorted(
+            tmpl_vars["readable_values"], key=lambda x: x["label"]
+        )
+        start_date = convert_date(tmpl_vars["event_values"]["start_interval"])
         end_date = None
-        if tmpl_vars['event_values'].get('end_interval'):
-            end_date = convert_date(tmpl_vars['event_values']['end_interval'])
+        if tmpl_vars["event_values"].get("end_interval"):
+            end_date = convert_date(tmpl_vars["event_values"]["end_interval"])
 
-        app_url = kwargs['request'].registry.settings['_mail_url']
-        destination_url = kwargs['request'].route_url('/', _app_url=app_url)
+        app_url = kwargs["request"].registry.settings["_mail_url"]
+        destination_url = kwargs["request"].route_url("/", _app_url=app_url)
         to_encode = {
-            'resource': tmpl_vars['event_values']['resource'],
-            'start_date': start_date.strftime(DATE_FRMT),
+            "resource": tmpl_vars["event_values"]["resource"],
+            "start_date": start_date.strftime(DATE_FRMT),
         }
         if end_date:
-            to_encode['end_date'] = end_date.strftime(DATE_FRMT)
+            to_encode["end_date"] = end_date.strftime(DATE_FRMT)
 
-        destination_url += 'ui/{}?{}'.format(
-            'logs',
-            urllib.parse.urlencode(to_encode)
-        )
-        tmpl_vars['destination_url'] = destination_url
+        destination_url += "ui/{}?{}".format("logs", urllib.parse.urlencode(to_encode))
+        tmpl_vars["destination_url"] = destination_url
         return tmpl_vars

backend/src/appenlight/models/alert_channel_action.py

@@ -1,79 +1,84 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import sqlalchemy as sa
 
 from ziggurat_foundations.models.services.resource import ResourceService
 from appenlight.models import Base, get_db_session
 from sqlalchemy.orm import validates
 from ziggurat_foundations.models.base import BaseModel
 
 
 class AlertChannelAction(Base, BaseModel):
     """
     Stores notifications conditions for user's alert channels
     This is later used for rule parsing like "alert if http_status == 500"
     """
-    __tablename__ = 'alert_channels_actions'
 
-    types = ['report', 'chart']
+    __tablename__ = "alert_channels_actions"
 
-    owner_id = sa.Column(sa.Integer,
-                         sa.ForeignKey('users.id', onupdate='CASCADE',
-                                       ondelete='CASCADE'))
+    types = ["report", "chart"]
+
+    owner_id = sa.Column(
+        sa.Integer, sa.ForeignKey("users.id", onupdate="CASCADE", ondelete="CASCADE")
+    )
     resource_id = sa.Column(sa.Integer())
-    action = sa.Column(sa.Unicode(10), nullable=False, default='always')
+    action = sa.Column(sa.Unicode(10), nullable=False, default="always")
     type = sa.Column(sa.Unicode(10), nullable=False)
     other_id = sa.Column(sa.Unicode(40))
     pkey = sa.Column(sa.Integer(), nullable=False, primary_key=True)
-    rule = sa.Column(sa.dialects.postgresql.JSON,
-                     nullable=False, default={'field': 'http_status',
-                                              "op": "ge", "value": "500"})
+    rule = sa.Column(
+        sa.dialects.postgresql.JSON,
+        nullable=False,
+        default={"field": "http_status", "op": "ge", "value": "500"},
+    )
     config = sa.Column(sa.dialects.postgresql.JSON)
     name = sa.Column(sa.Unicode(255))
 
-    @validates('notify_type')
+    @validates("notify_type")
     def validate_email(self, key, notify_type):
-        assert notify_type in ['always', 'only_first']
+        assert notify_type in ["always", "only_first"]
         return notify_type
 
     def resource_name(self, db_session=None):
         db_session = get_db_session(db_session)
         if self.resource_id:
             return ResourceService.by_resource_id(
-                self.resource_id, db_session=db_session).resource_name
+                self.resource_id, db_session=db_session
+            ).resource_name
         else:
-            return 'any resource'
+            return "any resource"
 
-    def get_dict(self, exclude_keys=None, include_keys=None,
-                 extended_info=False):
+    def get_dict(self, exclude_keys=None, include_keys=None, extended_info=False):
         """
         Returns dictionary with required information that will be consumed by
         angular
         """
         instance_dict = super(AlertChannelAction, self).get_dict()
         exclude_keys_list = exclude_keys or []
         include_keys_list = include_keys or []
         if extended_info:
-            instance_dict['channels'] = [
-                c.get_dict(extended_info=False) for c in self.channels]
+            instance_dict["channels"] = [
+                c.get_dict(extended_info=False) for c in self.channels
+            ]
 
         d = {}
         for k in instance_dict.keys():
-            if (k not in exclude_keys_list and
-                    (k in include_keys_list or not include_keys)):
+            if k not in exclude_keys_list and (
+                k in include_keys_list or not include_keys
+            ):
                 d[k] = instance_dict[k]
         return d

backend/src/appenlight/models/alert_channels/__init__.py

@@ -1,16 +1,15 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-

backend/src/appenlight/models/alert_channels/campfire.py

@@ -1,188 +1,191 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 from appenlight.models.alert_channel import AlertChannel
 from appenlight.models.integrations.campfire import CampfireIntegration
 from webhelpers2.text import truncate
 
 log = logging.getLogger(__name__)
 
 
 class CampfireAlertChannel(AlertChannel):
-    __mapper_args__ = {
-        'polymorphic_identity': 'campfire'
-    }
+    __mapper_args__ = {"polymorphic_identity": "campfire"}
 
     @property
     def client(self):
         client = CampfireIntegration.create_client(
-            self.integration.config['api_token'],
-            self.integration.config['account'])
+            self.integration.config["api_token"], self.integration.config["account"]
+        )
         return client
 
     def notify_reports(self, **kwargs):
         """
         Notify user of individual reports
 
         kwargs:
             application: application that the event applies for,
             user: user that should be notified
             request: request object
             since_when: reports are newer than this time value,
             reports: list of reports to render
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
 
-        app_url = kwargs['request'].registry.settings['_mail_url']
-        destination_url = kwargs['request'].route_url('/',
-                                                      app_url=app_url)
-        f_args = ('report',
-                  template_vars['resource'].resource_id,
-                  template_vars['url_start_date'].strftime('%Y-%m-%dT%H:%M'),
-                  template_vars['url_end_date'].strftime('%Y-%m-%dT%H:%M'))
-        destination_url += 'ui/{}?resource={}&start_date={}&end_date={}'.format(
-            *f_args)
-
-        if template_vars['confirmed_total'] > 1:
+        app_url = kwargs["request"].registry.settings["_mail_url"]
+        destination_url = kwargs["request"].route_url("/", app_url=app_url)
+        f_args = (
+            "report",
+            template_vars["resource"].resource_id,
+            template_vars["url_start_date"].strftime("%Y-%m-%dT%H:%M"),
+            template_vars["url_end_date"].strftime("%Y-%m-%dT%H:%M"),
+        )
+        destination_url += "ui/{}?resource={}&start_date={}&end_date={}".format(*f_args)
+
+        if template_vars["confirmed_total"] > 1:
             template_vars["title"] = "%s - %s reports" % (
-                template_vars['resource_name'],
-                template_vars['confirmed_total'],
+                template_vars["resource_name"],
+                template_vars["confirmed_total"],
             )
         else:
-            error_title = truncate(template_vars['reports'][0][1].error or
-                                   'slow report', 90)
+            error_title = truncate(
+                template_vars["reports"][0][1].error or "slow report", 90
+            )
             template_vars["title"] = "%s - '%s' report" % (
-                template_vars['resource_name'],
-                error_title)
+                template_vars["resource_name"],
+                error_title,
+            )
 
-        template_vars["title"] += ' ' + destination_url
+        template_vars["title"] += " " + destination_url
 
-        log_msg = 'NOTIFY  : %s via %s :: %s reports' % (
-            kwargs['user'].user_name,
+        log_msg = "NOTIFY  : %s via %s :: %s reports" % (
+            kwargs["user"].user_name,
             self.channel_visible_value,
-            template_vars['confirmed_total'])
+            template_vars["confirmed_total"],
+        )
         log.warning(log_msg)
 
-        for room in self.integration.config['rooms'].split(','):
+        for room in self.integration.config["rooms"].split(","):
             self.client.speak_to_room(room.strip(), template_vars["title"])
 
     def notify_report_alert(self, **kwargs):
         """
         Build and send report alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
 
-        if kwargs['event'].unified_alert_action() == 'OPEN':
-            title = 'ALERT %s: %s - %s %s %s' % (
-                template_vars['alert_action'],
-                template_vars['resource_name'],
-                kwargs['event'].values['reports'],
-                template_vars['report_type'],
-                template_vars['destination_url']
+        if kwargs["event"].unified_alert_action() == "OPEN":
+            title = "ALERT %s: %s - %s %s %s" % (
+                template_vars["alert_action"],
+                template_vars["resource_name"],
+                kwargs["event"].values["reports"],
+                template_vars["report_type"],
+                template_vars["destination_url"],
             )
 
         else:
-            title = 'ALERT %s: %s type: %s' % (
-                template_vars['alert_action'],
-                template_vars['resource_name'],
-                template_vars['alert_type'].replace('_', ' '),
+            title = "ALERT %s: %s type: %s" % (
+                template_vars["alert_action"],
+                template_vars["resource_name"],
+                template_vars["alert_type"].replace("_", " "),
             )
-        for room in self.integration.config['rooms'].split(','):
-            self.client.speak_to_room(room.strip(), title, sound='VUVUZELA')
+        for room in self.integration.config["rooms"].split(","):
+            self.client.speak_to_room(room.strip(), title, sound="VUVUZELA")
 
     def notify_uptime_alert(self, **kwargs):
         """
         Build and send uptime alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.uptime_alert_notification_vars(kwargs)
 
-        message = 'ALERT %s: %s has uptime issues %s\n\n' % (
-            template_vars['alert_action'],
-            template_vars['resource_name'],
-            template_vars['destination_url']
+        message = "ALERT %s: %s has uptime issues %s\n\n" % (
+            template_vars["alert_action"],
+            template_vars["resource_name"],
+            template_vars["destination_url"],
         )
-        message += template_vars['reason']
+        message += template_vars["reason"]
 
-        for room in self.integration.config['rooms'].split(','):
-            self.client.speak_to_room(room.strip(), message, sound='VUVUZELA')
+        for room in self.integration.config["rooms"].split(","):
+            self.client.speak_to_room(room.strip(), message, sound="VUVUZELA")
 
     def send_digest(self, **kwargs):
         """
         Build and send daily digest notification
 
         kwargs:
             application: application that the event applies for,
             user: user that should be notified
             request: request object
             since_when: reports are newer than this time value,
             reports: list of reports to render
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
-        f_args = (template_vars['resource_name'],
-                  template_vars['confirmed_total'],)
+        f_args = (template_vars["resource_name"], template_vars["confirmed_total"])
         message = "Daily report digest: %s - %s reports" % f_args
-        message += '{}\n'.format(template_vars['destination_url'])
-        for room in self.integration.config['rooms'].split(','):
+        message += "{}\n".format(template_vars["destination_url"])
+        for room in self.integration.config["rooms"].split(","):
             self.client.speak_to_room(room.strip(), message)
 
-        log_msg = 'DIGEST  : %s via %s :: %s reports' % (
-            kwargs['user'].user_name,
+        log_msg = "DIGEST  : %s via %s :: %s reports" % (
+            kwargs["user"].user_name,
             self.channel_visible_value,
-            template_vars['confirmed_total'])
+            template_vars["confirmed_total"],
+        )
         log.warning(log_msg)
 
     def notify_chart_alert(self, **kwargs):
         """
         Build and send chart alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.chart_alert_notification_vars(kwargs)
-        message = 'ALERT {}: value in "{}" chart: ' \
-                  'met alert "{}" criteria {} \n'.format(
-            template_vars['alert_action'],
-            template_vars['chart_name'],
-            template_vars['action_name'],
-            template_vars['destination_url']
+        message = (
+            'ALERT {}: value in "{}" chart: '
+            'met alert "{}" criteria {} \n'.format(
+                template_vars["alert_action"],
+                template_vars["chart_name"],
+                template_vars["action_name"],
+                template_vars["destination_url"],
+            )
         )
 
-        for item in template_vars['readable_values']:
-            message += '{}: {}\n'.format(item['label'], item['value'])
+        for item in template_vars["readable_values"]:
+            message += "{}: {}\n".format(item["label"], item["value"])
 
-        for room in self.integration.config['rooms'].split(','):
-            self.client.speak_to_room(room.strip(), message, sound='VUVUZELA')
+        for room in self.integration.config["rooms"].split(","):
+            self.client.speak_to_room(room.strip(), message, sound="VUVUZELA")

backend/src/appenlight/models/alert_channels/email.py

@@ -1,175 +1,192 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 from appenlight.models.alert_channel import AlertChannel
 from appenlight.models.services.user import UserService
 from webhelpers2.text import truncate
 
 log = logging.getLogger(__name__)
 
 
 class EmailAlertChannel(AlertChannel):
     """
     Default email alerting channel
     """
 
-    __mapper_args__ = {
-        'polymorphic_identity': 'email'
-    }
+    __mapper_args__ = {"polymorphic_identity": "email"}
 
     def notify_reports(self, **kwargs):
         """
         Notify user of individual reports
 
         kwargs:
             application: application that the event applies for,
             user: user that should be notified
             request: request object
             since_when: reports are newer than this time value,
             reports: list of reports to render
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
 
-        if template_vars['confirmed_total'] > 1:
+        if template_vars["confirmed_total"] > 1:
             template_vars["title"] = "AppEnlight :: %s - %s reports" % (
-                template_vars['resource_name'],
-                template_vars['confirmed_total'],
+                template_vars["resource_name"],
+                template_vars["confirmed_total"],
             )
         else:
-            error_title = truncate(template_vars['reports'][0][1].error or
-                                   'slow report', 20)
+            error_title = truncate(
+                template_vars["reports"][0][1].error or "slow report", 20
+            )
             template_vars["title"] = "AppEnlight :: %s - '%s' report" % (
-                template_vars['resource_name'],
-                error_title)
-        UserService.send_email(kwargs['request'],
-                               [self.channel_value],
-                               template_vars,
-                               '/email_templates/notify_reports.jinja2')
-        log_msg = 'NOTIFY  : %s via %s :: %s reports' % (
-            kwargs['user'].user_name,
+                template_vars["resource_name"],
+                error_title,
+            )
+        UserService.send_email(
+            kwargs["request"],
+            [self.channel_value],
+            template_vars,
+            "/email_templates/notify_reports.jinja2",
+        )
+        log_msg = "NOTIFY  : %s via %s :: %s reports" % (
+            kwargs["user"].user_name,
             self.channel_visible_value,
-            template_vars['confirmed_total'])
+            template_vars["confirmed_total"],
+        )
         log.warning(log_msg)
 
     def send_digest(self, **kwargs):
         """
         Build and send daily digest notification
 
         kwargs:
             application: application that the event applies for,
             user: user that should be notified
             request: request object
             since_when: reports are newer than this time value,
             reports: list of reports to render
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
         title = "AppEnlight :: Daily report digest: %s - %s reports"
         template_vars["email_title"] = title % (
-            template_vars['resource_name'],
-            template_vars['confirmed_total'],
+            template_vars["resource_name"],
+            template_vars["confirmed_total"],
         )
 
-        UserService.send_email(kwargs['request'],
-                               [self.channel_value],
-                               template_vars,
-                               '/email_templates/notify_reports.jinja2',
-                               immediately=True,
-                               silent=True)
-        log_msg = 'DIGEST  : %s via %s :: %s reports' % (
-            kwargs['user'].user_name,
+        UserService.send_email(
+            kwargs["request"],
+            [self.channel_value],
+            template_vars,
+            "/email_templates/notify_reports.jinja2",
+            immediately=True,
+            silent=True,
+        )
+        log_msg = "DIGEST  : %s via %s :: %s reports" % (
+            kwargs["user"].user_name,
             self.channel_visible_value,
-            template_vars['confirmed_total'])
+            template_vars["confirmed_total"],
+        )
         log.warning(log_msg)
 
     def notify_report_alert(self, **kwargs):
         """
         Build and send report alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
 
-        if kwargs['event'].unified_alert_action() == 'OPEN':
-            title = 'AppEnlight :: ALERT %s: %s - %s %s' % (
-                template_vars['alert_action'],
-                template_vars['resource_name'],
-                kwargs['event'].values['reports'],
-                template_vars['report_type'],
+        if kwargs["event"].unified_alert_action() == "OPEN":
+            title = "AppEnlight :: ALERT %s: %s - %s %s" % (
+                template_vars["alert_action"],
+                template_vars["resource_name"],
+                kwargs["event"].values["reports"],
+                template_vars["report_type"],
             )
         else:
-            title = 'AppEnlight :: ALERT %s: %s type: %s' % (
-                template_vars['alert_action'],
-                template_vars['resource_name'],
-                template_vars['alert_type'].replace('_', ' '),
+            title = "AppEnlight :: ALERT %s: %s type: %s" % (
+                template_vars["alert_action"],
+                template_vars["resource_name"],
+                template_vars["alert_type"].replace("_", " "),
             )
-        template_vars['email_title'] = title
-        UserService.send_email(kwargs['request'], [self.channel_value],
-                               template_vars,
-                               '/email_templates/alert_reports.jinja2')
+        template_vars["email_title"] = title
+        UserService.send_email(
+            kwargs["request"],
+            [self.channel_value],
+            template_vars,
+            "/email_templates/alert_reports.jinja2",
+        )
 
     def notify_uptime_alert(self, **kwargs):
         """
         Build and send uptime alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.uptime_alert_notification_vars(kwargs)
-        title = 'AppEnlight :: ALERT %s: %s has uptime issues' % (
-            template_vars['alert_action'],
-            template_vars['resource_name'],
+        title = "AppEnlight :: ALERT %s: %s has uptime issues" % (
+            template_vars["alert_action"],
+            template_vars["resource_name"],
         )
-        template_vars['email_title'] = title
+        template_vars["email_title"] = title
 
-        UserService.send_email(kwargs['request'], [self.channel_value],
-                               template_vars,
-                               '/email_templates/alert_uptime.jinja2')
+        UserService.send_email(
+            kwargs["request"],
+            [self.channel_value],
+            template_vars,
+            "/email_templates/alert_uptime.jinja2",
+        )
 
     def notify_chart_alert(self, **kwargs):
         """
         Build and send chart alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.chart_alert_notification_vars(kwargs)
 
-        title = 'AppEnlight :: ALERT {} value in "{}" chart' \
-                ' met alert "{}" criteria'.format(
-            template_vars['alert_action'],
-            template_vars['chart_name'],
-            template_vars['action_name'],
+        title = (
+            'AppEnlight :: ALERT {} value in "{}" chart'
+            ' met alert "{}" criteria'.format(
+                template_vars["alert_action"],
+                template_vars["chart_name"],
+                template_vars["action_name"],
+            )
+        )
+        template_vars["email_title"] = title
+        UserService.send_email(
+            kwargs["request"],
+            [self.channel_value],
+            template_vars,
+            "/email_templates/alert_chart.jinja2",
         )
-        template_vars['email_title'] = title
-        UserService.send_email(kwargs['request'], [self.channel_value],
-                               template_vars,
-                               '/email_templates/alert_chart.jinja2')

backend/src/appenlight/models/alert_channels/flowdock.py

@@ -1,233 +1,225 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 from appenlight.models.alert_channel import AlertChannel
 from appenlight.models.integrations.flowdock import FlowdockIntegration
 from webhelpers2.text import truncate
 
 log = logging.getLogger(__name__)
 
 
 class FlowdockAlertChannel(AlertChannel):
-    __mapper_args__ = {
-        'polymorphic_identity': 'flowdock'
-    }
+    __mapper_args__ = {"polymorphic_identity": "flowdock"}
 
     def notify_reports(self, **kwargs):
         """
         Notify user of individual reports
 
         kwargs:
             application: application that the event applies for,
             user: user that should be notified
             request: request object
             since_when: reports are newer than this time value,
             reports: list of reports to render
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
 
-        app_url = kwargs['request'].registry.settings['_mail_url']
-        destination_url = kwargs['request'].route_url('/',
-                                                      _app_url=app_url)
-        f_args = ('report',
-                  template_vars['resource'].resource_id,
-                  template_vars['url_start_date'].strftime('%Y-%m-%dT%H:%M'),
-                  template_vars['url_end_date'].strftime('%Y-%m-%dT%H:%M'))
-        destination_url += 'ui/{}?resource={}&start_date={}&end_date={}'.format(
-            *f_args)
-
-        if template_vars['confirmed_total'] > 1:
+        app_url = kwargs["request"].registry.settings["_mail_url"]
+        destination_url = kwargs["request"].route_url("/", _app_url=app_url)
+        f_args = (
+            "report",
+            template_vars["resource"].resource_id,
+            template_vars["url_start_date"].strftime("%Y-%m-%dT%H:%M"),
+            template_vars["url_end_date"].strftime("%Y-%m-%dT%H:%M"),
+        )
+        destination_url += "ui/{}?resource={}&start_date={}&end_date={}".format(*f_args)
+
+        if template_vars["confirmed_total"] > 1:
             template_vars["title"] = "%s - %s reports" % (
-                template_vars['resource_name'],
-                template_vars['confirmed_total'],
+                template_vars["resource_name"],
+                template_vars["confirmed_total"],
             )
         else:
-            error_title = truncate(template_vars['reports'][0][1].error or
-                                   'slow report', 90)
+            error_title = truncate(
+                template_vars["reports"][0][1].error or "slow report", 90
+            )
             template_vars["title"] = "%s - '%s' report" % (
-                template_vars['resource_name'],
-                error_title)
+                template_vars["resource_name"],
+                error_title,
+            )
 
-        log_msg = 'NOTIFY : %s via %s :: %s reports' % (
-            kwargs['user'].user_name,
+        log_msg = "NOTIFY : %s via %s :: %s reports" % (
+            kwargs["user"].user_name,
             self.channel_visible_value,
-            template_vars['confirmed_total'])
+            template_vars["confirmed_total"],
+        )
         log.warning(log_msg)
 
-        client = FlowdockIntegration.create_client(
-            self.integration.config['api_token'])
+        client = FlowdockIntegration.create_client(self.integration.config["api_token"])
         payload = {
             "source": "AppEnlight",
-            "from_address": kwargs['request'].registry.settings[
-                'mailing.from_email'],
+            "from_address": kwargs["request"].registry.settings["mailing.from_email"],
             "subject": template_vars["title"],
             "content": "New report present",
             "tags": ["appenlight"],
-            "link": destination_url
+            "link": destination_url,
         }
         client.send_to_inbox(payload)
 
     def notify_report_alert(self, **kwargs):
         """
         Build and send report alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
 
-        if kwargs['event'].unified_alert_action() == 'OPEN':
+        if kwargs["event"].unified_alert_action() == "OPEN":
 
-            title = 'ALERT %s: %s - %s %s' % (
-                template_vars['alert_action'],
-                template_vars['resource_name'],
-                kwargs['event'].values['reports'],
-                template_vars['report_type'],
+            title = "ALERT %s: %s - %s %s" % (
+                template_vars["alert_action"],
+                template_vars["resource_name"],
+                kwargs["event"].values["reports"],
+                template_vars["report_type"],
             )
 
         else:
-            title = 'ALERT %s: %s type: %s' % (
-                template_vars['alert_action'],
-                template_vars['resource_name'],
-                template_vars['alert_type'].replace('_', ' '),
+            title = "ALERT %s: %s type: %s" % (
+                template_vars["alert_action"],
+                template_vars["resource_name"],
+                template_vars["alert_type"].replace("_", " "),
             )
 
-        client = FlowdockIntegration.create_client(
-            self.integration.config['api_token'])
+        client = FlowdockIntegration.create_client(self.integration.config["api_token"])
         payload = {
             "source": "AppEnlight",
-            "from_address": kwargs['request'].registry.settings[
-                'mailing.from_email'],
+            "from_address": kwargs["request"].registry.settings["mailing.from_email"],
             "subject": title,
-            "content": 'Investigation required',
-            "tags": ["appenlight", "alert", template_vars['alert_type']],
-            "link": template_vars['destination_url']
+            "content": "Investigation required",
+            "tags": ["appenlight", "alert", template_vars["alert_type"]],
+            "link": template_vars["destination_url"],
         }
         client.send_to_inbox(payload)
 
     def notify_uptime_alert(self, **kwargs):
         """
         Build and send uptime alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.uptime_alert_notification_vars(kwargs)
 
-        message = 'ALERT %s: %s has uptime issues' % (
-            template_vars['alert_action'],
-            template_vars['resource_name'],
+        message = "ALERT %s: %s has uptime issues" % (
+            template_vars["alert_action"],
+            template_vars["resource_name"],
         )
-        submessage = 'Info: '
-        submessage += template_vars['reason']
+        submessage = "Info: "
+        submessage += template_vars["reason"]
 
-        client = FlowdockIntegration.create_client(
-            self.integration.config['api_token'])
+        client = FlowdockIntegration.create_client(self.integration.config["api_token"])
         payload = {
             "source": "AppEnlight",
-            "from_address": kwargs['request'].registry.settings[
-                'mailing.from_email'],
+            "from_address": kwargs["request"].registry.settings["mailing.from_email"],
             "subject": message,
             "content": submessage,
-            "tags": ["appenlight", "alert", 'uptime'],
-            "link": template_vars['destination_url']
+            "tags": ["appenlight", "alert", "uptime"],
+            "link": template_vars["destination_url"],
         }
         client.send_to_inbox(payload)
 
     def send_digest(self, **kwargs):
         """
         Build and send daily digest notification
 
         kwargs:
             application: application that the event applies for,
             user: user that should be notified
             request: request object
             since_when: reports are newer than this time value,
             reports: list of reports to render
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
         message = "Daily report digest: %s - %s reports" % (
-            template_vars['resource_name'], template_vars['confirmed_total'])
+            template_vars["resource_name"],
+            template_vars["confirmed_total"],
+        )
 
-        f_args = (template_vars['confirmed_total'],
-                  template_vars['timestamp'])
+        f_args = (template_vars["confirmed_total"], template_vars["timestamp"])
 
         payload = {
             "source": "AppEnlight",
-            "from_address": kwargs['request'].registry.settings[
-                'mailing.from_email'],
+            "from_address": kwargs["request"].registry.settings["mailing.from_email"],
             "subject": message,
-            "content": '%s reports in total since %s' % f_args,
+            "content": "%s reports in total since %s" % f_args,
             "tags": ["appenlight", "digest"],
-            "link": template_vars['destination_url']
+            "link": template_vars["destination_url"],
         }
 
-        client = FlowdockIntegration.create_client(
-            self.integration.config['api_token'])
+        client = FlowdockIntegration.create_client(self.integration.config["api_token"])
         client.send_to_inbox(payload)
 
-        log_msg = 'DIGEST  : %s via %s :: %s reports' % (
-            kwargs['user'].user_name,
+        log_msg = "DIGEST  : %s via %s :: %s reports" % (
+            kwargs["user"].user_name,
             self.channel_visible_value,
-            template_vars['confirmed_total'])
+            template_vars["confirmed_total"],
+        )
         log.warning(log_msg)
 
     def notify_chart_alert(self, **kwargs):
         """
         Build and send chart alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.chart_alert_notification_vars(kwargs)
 
-        message = 'ALERT {}: value in "{}" chart ' \
-                  'met alert "{}" criteria'.format(
-            template_vars['alert_action'],
-            template_vars['chart_name'],
-            template_vars['action_name'],
+        message = 'ALERT {}: value in "{}" chart ' 'met alert "{}" criteria'.format(
+            template_vars["alert_action"],
+            template_vars["chart_name"],
+            template_vars["action_name"],
         )
-        submessage = 'Info: '
-        for item in template_vars['readable_values']:
-            submessage += '{}: {}\n'.format(item['label'], item['value'])
+        submessage = "Info: "
+        for item in template_vars["readable_values"]:
+            submessage += "{}: {}\n".format(item["label"], item["value"])
 
-        client = FlowdockIntegration.create_client(
-            self.integration.config['api_token'])
+        client = FlowdockIntegration.create_client(self.integration.config["api_token"])
         payload = {
             "source": "AppEnlight",
-            "from_address": kwargs['request'].registry.settings[
-                'mailing.from_email'],
+            "from_address": kwargs["request"].registry.settings["mailing.from_email"],
             "subject": message,
             "content": submessage,
-            "tags": ["appenlight", "alert", 'chart'],
-            "link": template_vars['destination_url']
+            "tags": ["appenlight", "alert", "chart"],
+            "link": template_vars["destination_url"],
         }
         client.send_to_inbox(payload)

backend/src/appenlight/models/alert_channels/hipchat.py

@@ -1,229 +1,238 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 from appenlight.models.alert_channel import AlertChannel
 from appenlight.models.integrations.hipchat import HipchatIntegration
 from webhelpers2.text import truncate
 
 log = logging.getLogger(__name__)
 
 
 class HipchatAlertChannel(AlertChannel):
-    __mapper_args__ = {
-        'polymorphic_identity': 'hipchat'
-    }
+    __mapper_args__ = {"polymorphic_identity": "hipchat"}
 
     def notify_reports(self, **kwargs):
         """
         Notify user of individual reports
 
         kwargs:
             application: application that the event applies for,
             user: user that should be notified
             request: request object
             since_when: reports are newer than this time value,
             reports: list of reports to render
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
 
-        app_url = kwargs['request'].registry.settings['_mail_url']
-        destination_url = kwargs['request'].route_url('/',
-                                                      _app_url=app_url)
-        f_args = ('report',
-                  template_vars['resource'].resource_id,
-                  template_vars['url_start_date'].strftime('%Y-%m-%dT%H:%M'),
-                  template_vars['url_end_date'].strftime('%Y-%m-%dT%H:%M'))
-        destination_url += 'ui/{}?resource={}&start_date={}&end_date={}'.format(
-            *f_args)
-
-        if template_vars['confirmed_total'] > 1:
+        app_url = kwargs["request"].registry.settings["_mail_url"]
+        destination_url = kwargs["request"].route_url("/", _app_url=app_url)
+        f_args = (
+            "report",
+            template_vars["resource"].resource_id,
+            template_vars["url_start_date"].strftime("%Y-%m-%dT%H:%M"),
+            template_vars["url_end_date"].strftime("%Y-%m-%dT%H:%M"),
+        )
+        destination_url += "ui/{}?resource={}&start_date={}&end_date={}".format(*f_args)
+
+        if template_vars["confirmed_total"] > 1:
             template_vars["title"] = "%s - %s reports" % (
-                template_vars['resource_name'],
-                template_vars['confirmed_total'],
+                template_vars["resource_name"],
+                template_vars["confirmed_total"],
             )
         else:
-            error_title = truncate(template_vars['reports'][0][1].error or
-                                   'slow report', 90)
+            error_title = truncate(
+                template_vars["reports"][0][1].error or "slow report", 90
+            )
             template_vars["title"] = "%s - '%s' report" % (
-                template_vars['resource_name'],
-                error_title)
+                template_vars["resource_name"],
+                error_title,
+            )
 
-        template_vars["title"] += ' ' + destination_url
+        template_vars["title"] += " " + destination_url
 
-        log_msg = 'NOTIFY  : %s via %s :: %s reports' % (
-            kwargs['user'].user_name,
+        log_msg = "NOTIFY  : %s via %s :: %s reports" % (
+            kwargs["user"].user_name,
             self.channel_visible_value,
-            template_vars['confirmed_total'])
+            template_vars["confirmed_total"],
+        )
         log.warning(log_msg)
 
-        client = HipchatIntegration.create_client(
-            self.integration.config['api_token'])
-        for room in self.integration.config['rooms'].split(','):
-            client.send({
-                "message_format": 'text',
-                "message": template_vars["title"],
-                "from": "AppEnlight",
-                "room_id": room.strip(),
-                "color": "yellow"
-            })
+        client = HipchatIntegration.create_client(self.integration.config["api_token"])
+        for room in self.integration.config["rooms"].split(","):
+            client.send(
+                {
+                    "message_format": "text",
+                    "message": template_vars["title"],
+                    "from": "AppEnlight",
+                    "room_id": room.strip(),
+                    "color": "yellow",
+                }
+            )
 
     def notify_report_alert(self, **kwargs):
         """
         Build and send report alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
 
-        if kwargs['event'].unified_alert_action() == 'OPEN':
+        if kwargs["event"].unified_alert_action() == "OPEN":
 
-            title = 'ALERT %s: %s - %s %s' % (
-                template_vars['alert_action'],
-                template_vars['resource_name'],
-                kwargs['event'].values['reports'],
-                template_vars['report_type'],
+            title = "ALERT %s: %s - %s %s" % (
+                template_vars["alert_action"],
+                template_vars["resource_name"],
+                kwargs["event"].values["reports"],
+                template_vars["report_type"],
             )
 
         else:
-            title = 'ALERT %s: %s type: %s' % (
-                template_vars['alert_action'],
-                template_vars['resource_name'],
-                template_vars['alert_type'].replace('_', ' '),
+            title = "ALERT %s: %s type: %s" % (
+                template_vars["alert_action"],
+                template_vars["resource_name"],
+                template_vars["alert_type"].replace("_", " "),
             )
 
-        title += '\n ' + template_vars['destination_url']
+        title += "\n " + template_vars["destination_url"]
 
-        api_token = self.integration.config['api_token']
+        api_token = self.integration.config["api_token"]
         client = HipchatIntegration.create_client(api_token)
-        for room in self.integration.config['rooms'].split(','):
-            client.send({
-                "message_format": 'text',
-                "message": title,
-                "from": "AppEnlight",
-                "room_id": room.strip(),
-                "color": "red",
-                "notify": '1'
-            })
+        for room in self.integration.config["rooms"].split(","):
+            client.send(
+                {
+                    "message_format": "text",
+                    "message": title,
+                    "from": "AppEnlight",
+                    "room_id": room.strip(),
+                    "color": "red",
+                    "notify": "1",
+                }
+            )
 
     def notify_uptime_alert(self, **kwargs):
         """
         Build and send uptime alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.uptime_alert_notification_vars(kwargs)
 
-        message = 'ALERT %s: %s has uptime issues\n' % (
-            template_vars['alert_action'],
-            template_vars['resource_name'],
+        message = "ALERT %s: %s has uptime issues\n" % (
+            template_vars["alert_action"],
+            template_vars["resource_name"],
         )
-        message += template_vars['reason']
-        message += '\n{}'.format(template_vars['destination_url'])
+        message += template_vars["reason"]
+        message += "\n{}".format(template_vars["destination_url"])
 
-        api_token = self.integration.config['api_token']
+        api_token = self.integration.config["api_token"]
         client = HipchatIntegration.create_client(api_token)
-        for room in self.integration.config['rooms'].split(','):
-            client.send({
-                "message_format": 'text',
-                "message": message,
-                "from": "AppEnlight",
-                "room_id": room.strip(),
-                "color": "red",
-                "notify": '1'
-            })
+        for room in self.integration.config["rooms"].split(","):
+            client.send(
+                {
+                    "message_format": "text",
+                    "message": message,
+                    "from": "AppEnlight",
+                    "room_id": room.strip(),
+                    "color": "red",
+                    "notify": "1",
+                }
+            )
 
     def notify_chart_alert(self, **kwargs):
         """
         Build and send chart alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.chart_alert_notification_vars(kwargs)
-        message = 'ALERT {}: value in "{}" chart: ' \
-                  'met alert "{}" criteria\n'.format(
-            template_vars['alert_action'],
-            template_vars['chart_name'],
-            template_vars['action_name'],
+        message = 'ALERT {}: value in "{}" chart: ' 'met alert "{}" criteria\n'.format(
+            template_vars["alert_action"],
+            template_vars["chart_name"],
+            template_vars["action_name"],
         )
 
-        for item in template_vars['readable_values']:
-            message += '{}: {}\n'.format(item['label'], item['value'])
+        for item in template_vars["readable_values"]:
+            message += "{}: {}\n".format(item["label"], item["value"])
 
-        message += template_vars['destination_url']
+        message += template_vars["destination_url"]
 
-        api_token = self.integration.config['api_token']
+        api_token = self.integration.config["api_token"]
         client = HipchatIntegration.create_client(api_token)
-        for room in self.integration.config['rooms'].split(','):
-            client.send({
-                "message_format": 'text',
-                "message": message,
-                "from": "AppEnlight",
-                "room_id": room.strip(),
-                "color": "red",
-                "notify": '1'
-            })
+        for room in self.integration.config["rooms"].split(","):
+            client.send(
+                {
+                    "message_format": "text",
+                    "message": message,
+                    "from": "AppEnlight",
+                    "room_id": room.strip(),
+                    "color": "red",
+                    "notify": "1",
+                }
+            )
 
     def send_digest(self, **kwargs):
         """
         Build and send daily digest notification
 
         kwargs:
             application: application that the event applies for,
             user: user that should be notified
             request: request object
             since_when: reports are newer than this time value,
             reports: list of reports to render
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
-        f_args = (template_vars['resource_name'],
-                  template_vars['confirmed_total'],)
+        f_args = (template_vars["resource_name"], template_vars["confirmed_total"])
         message = "Daily report digest: %s - %s reports" % f_args
-        message += '\n{}'.format(template_vars['destination_url'])
-        api_token = self.integration.config['api_token']
+        message += "\n{}".format(template_vars["destination_url"])
+        api_token = self.integration.config["api_token"]
         client = HipchatIntegration.create_client(api_token)
-        for room in self.integration.config['rooms'].split(','):
-            client.send({
-                "message_format": 'text',
-                "message": message,
-                "from": "AppEnlight",
-                "room_id": room.strip(),
-                "color": "green",
-                "notify": '1'
-            })
-
-        log_msg = 'DIGEST  : %s via %s :: %s reports' % (
-            kwargs['user'].user_name,
+        for room in self.integration.config["rooms"].split(","):
+            client.send(
+                {
+                    "message_format": "text",
+                    "message": message,
+                    "from": "AppEnlight",
+                    "room_id": room.strip(),
+                    "color": "green",
+                    "notify": "1",
+                }
+            )
+
+        log_msg = "DIGEST  : %s via %s :: %s reports" % (
+            kwargs["user"].user_name,
             self.channel_visible_value,
-            template_vars['confirmed_total'])
+            template_vars["confirmed_total"],
+        )
         log.warning(log_msg)

backend/src/appenlight/models/alert_channels/slack.py

@@ -1,285 +1,270 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 from appenlight.models.alert_channel import AlertChannel
 from appenlight.models.integrations.slack import SlackIntegration
 from webhelpers2.text import truncate
 
 log = logging.getLogger(__name__)
 
 
 class SlackAlertChannel(AlertChannel):
-    __mapper_args__ = {
-        'polymorphic_identity': 'slack'
-    }
+    __mapper_args__ = {"polymorphic_identity": "slack"}
 
     def notify_reports(self, **kwargs):
         """
         Notify user of individual reports
 
         kwargs:
             application: application that the event applies for,
             user: user that should be notified
             request: request object
             since_when: reports are newer than this time value,
             reports: list of reports to render
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
-        template_vars["title"] = template_vars['resource_name']
+        template_vars["title"] = template_vars["resource_name"]
 
-        if template_vars['confirmed_total'] > 1:
-            template_vars['subtext'] = '%s reports' % template_vars[
-                'confirmed_total']
+        if template_vars["confirmed_total"] > 1:
+            template_vars["subtext"] = "%s reports" % template_vars["confirmed_total"]
         else:
-            error_title = truncate(template_vars['reports'][0][1].error or
-                                   'slow report', 90)
-            template_vars['subtext'] = error_title
+            error_title = truncate(
+                template_vars["reports"][0][1].error or "slow report", 90
+            )
+            template_vars["subtext"] = error_title
 
-        log_msg = 'NOTIFY  : %s via %s :: %s reports' % (
-            kwargs['user'].user_name,
+        log_msg = "NOTIFY  : %s via %s :: %s reports" % (
+            kwargs["user"].user_name,
             self.channel_visible_value,
-            template_vars['confirmed_total'])
+            template_vars["confirmed_total"],
+        )
         log.warning(log_msg)
 
-        client = SlackIntegration.create_client(
-            self.integration.config['webhook_url'])
+        client = SlackIntegration.create_client(self.integration.config["webhook_url"])
         report_data = {
             "username": "AppEnlight",
             "icon_emoji": ":fire:",
             "attachments": [
                 {
                     "mrkdwn_in": ["text", "pretext", "title", "fallback"],
-                    "fallback": "*%s* - <%s| Browse>" % (
-                        template_vars["title"],
-                        template_vars['destination_url']),
-                    "pretext": "*%s* - <%s| Browse>" % (
-                        template_vars["title"],
-                        template_vars['destination_url']),
+                    "fallback": "*%s* - <%s| Browse>"
+                    % (template_vars["title"], template_vars["destination_url"]),
+                    "pretext": "*%s* - <%s| Browse>"
+                    % (template_vars["title"], template_vars["destination_url"]),
                     "color": "warning",
                     "fields": [
-                        {
-                            "value": 'Info: %s' % template_vars['subtext'],
-                            "short": False
-                        }
-                    ]
+                        {"value": "Info: %s" % template_vars["subtext"], "short": False}
+                    ],
                 }
-            ]
+            ],
         }
         client.make_request(data=report_data)
 
     def notify_report_alert(self, **kwargs):
         """
         Build and send report alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
 
-        if kwargs['event'].unified_alert_action() == 'OPEN':
-            title = '*ALERT %s*: %s' % (
-                template_vars['alert_action'],
-                template_vars['resource_name']
+        if kwargs["event"].unified_alert_action() == "OPEN":
+            title = "*ALERT %s*: %s" % (
+                template_vars["alert_action"],
+                template_vars["resource_name"],
             )
 
-            template_vars['subtext'] = 'Got at least %s %s' % (
-                kwargs['event'].values['reports'],
-                template_vars['report_type']
+            template_vars["subtext"] = "Got at least %s %s" % (
+                kwargs["event"].values["reports"],
+                template_vars["report_type"],
             )
 
         else:
-            title = '*ALERT %s*: %s' % (
-                template_vars['alert_action'],
-                template_vars['resource_name'],
+            title = "*ALERT %s*: %s" % (
+                template_vars["alert_action"],
+                template_vars["resource_name"],
             )
 
-            template_vars['subtext'] = ''
+            template_vars["subtext"] = ""
 
-        alert_type = template_vars['alert_type'].replace('_', ' ')
-        alert_type = alert_type.replace('alert', '').capitalize()
+        alert_type = template_vars["alert_type"].replace("_", " ")
+        alert_type = alert_type.replace("alert", "").capitalize()
 
-        template_vars['type'] = "Type: %s" % alert_type
+        template_vars["type"] = "Type: %s" % alert_type
 
-        client = SlackIntegration.create_client(
-            self.integration.config['webhook_url']
-        )
+        client = SlackIntegration.create_client(self.integration.config["webhook_url"])
         report_data = {
             "username": "AppEnlight",
             "icon_emoji": ":rage:",
             "attachments": [
                 {
                     "mrkdwn_in": ["text", "pretext", "title", "fallback"],
-                    "fallback": "%s - <%s| Browse>" % (
-                        title, template_vars['destination_url']),
-                    "pretext": "%s - <%s| Browse>" % (
-                        title, template_vars['destination_url']),
+                    "fallback": "%s - <%s| Browse>"
+                    % (title, template_vars["destination_url"]),
+                    "pretext": "%s - <%s| Browse>"
+                    % (title, template_vars["destination_url"]),
                     "color": "danger",
                     "fields": [
                         {
-                            "title": template_vars['type'],
-                            "value": template_vars['subtext'],
-                            "short": False
+                            "title": template_vars["type"],
+                            "value": template_vars["subtext"],
+                            "short": False,
                         }
-                    ]
+                    ],
                 }
-            ]
+            ],
         }
         client.make_request(data=report_data)
 
     def notify_uptime_alert(self, **kwargs):
         """
         Build and send uptime alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.uptime_alert_notification_vars(kwargs)
 
-        title = '*ALERT %s*: %s' % (
-            template_vars['alert_action'],
-            template_vars['resource_name'],
-        )
-        client = SlackIntegration.create_client(
-            self.integration.config['webhook_url']
+        title = "*ALERT %s*: %s" % (
+            template_vars["alert_action"],
+            template_vars["resource_name"],
         )
+        client = SlackIntegration.create_client(self.integration.config["webhook_url"])
         report_data = {
             "username": "AppEnlight",
             "icon_emoji": ":rage:",
             "attachments": [
                 {
                     "mrkdwn_in": ["text", "pretext", "title", "fallback"],
                     "fallback": "{} - <{}| Browse>".format(
-                        title, template_vars['destination_url']),
+                        title, template_vars["destination_url"]
+                    ),
                     "pretext": "{} - <{}| Browse>".format(
-                        title, template_vars['destination_url']),
+                        title, template_vars["destination_url"]
+                    ),
                     "color": "danger",
                     "fields": [
                         {
                             "title": "Application has uptime issues",
-                            "value": template_vars['reason'],
-                            "short": False
+                            "value": template_vars["reason"],
+                            "short": False,
                         }
-                    ]
+                    ],
                 }
-            ]
+            ],
         }
         client.make_request(data=report_data)
 
     def notify_chart_alert(self, **kwargs):
         """
         Build and send chart alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.chart_alert_notification_vars(kwargs)
 
-        title = '*ALERT {}*: value in *"{}"* chart ' \
-                'met alert *"{}"* criteria'.format(
-            template_vars['alert_action'],
-            template_vars['chart_name'],
-            template_vars['action_name'],
+        title = '*ALERT {}*: value in *"{}"* chart ' 'met alert *"{}"* criteria'.format(
+            template_vars["alert_action"],
+            template_vars["chart_name"],
+            template_vars["action_name"],
         )
 
-        subtext = ''
-        for item in template_vars['readable_values']:
-            subtext += '{} - {}\n'.format(item['label'], item['value'])
+        subtext = ""
+        for item in template_vars["readable_values"]:
+            subtext += "{} - {}\n".format(item["label"], item["value"])
 
-        client = SlackIntegration.create_client(
-            self.integration.config['webhook_url']
-        )
+        client = SlackIntegration.create_client(self.integration.config["webhook_url"])
         report_data = {
             "username": "AppEnlight",
             "icon_emoji": ":rage:",
             "attachments": [
-                {"mrkdwn_in": ["text", "pretext", "title", "fallback"],
-                 "fallback": "{} - <{}| Browse>".format(
-                     title, template_vars['destination_url']),
-                 "pretext": "{} - <{}| Browse>".format(
-                     title, template_vars['destination_url']),
-                 "color": "danger",
-                 "fields": [
-                     {
-                         "title": "Following criteria were met:",
-                         "value": subtext,
-                         "short": False
-                     }
-                 ]
-                 }
-            ]
+                {
+                    "mrkdwn_in": ["text", "pretext", "title", "fallback"],
+                    "fallback": "{} - <{}| Browse>".format(
+                        title, template_vars["destination_url"]
+                    ),
+                    "pretext": "{} - <{}| Browse>".format(
+                        title, template_vars["destination_url"]
+                    ),
+                    "color": "danger",
+                    "fields": [
+                        {
+                            "title": "Following criteria were met:",
+                            "value": subtext,
+                            "short": False,
+                        }
+                    ],
+                }
+            ],
         }
         client.make_request(data=report_data)
 
     def send_digest(self, **kwargs):
         """
         Build and send daily digest notification
 
         kwargs:
             application: application that the event applies for,
             user: user that should be notified
             request: request object
             since_when: reports are newer than this time value,
             reports: list of reports to render
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
-        title = "*Daily report digest*: %s" % template_vars['resource_name']
+        title = "*Daily report digest*: %s" % template_vars["resource_name"]
 
-        subtext = '%s reports' % template_vars['confirmed_total']
+        subtext = "%s reports" % template_vars["confirmed_total"]
 
-        client = SlackIntegration.create_client(
-            self.integration.config['webhook_url']
-        )
+        client = SlackIntegration.create_client(self.integration.config["webhook_url"])
         report_data = {
             "username": "AppEnlight",
             "attachments": [
                 {
                     "mrkdwn_in": ["text", "pretext", "title", "fallback"],
-                    "fallback": "%s : <%s| Browse>" % (
-                        title, template_vars['destination_url']),
-                    "pretext": "%s: <%s| Browse>" % (
-                        title, template_vars['destination_url']),
+                    "fallback": "%s : <%s| Browse>"
+                    % (title, template_vars["destination_url"]),
+                    "pretext": "%s: <%s| Browse>"
+                    % (title, template_vars["destination_url"]),
                     "color": "good",
-                    "fields": [
-                        {
-                            "title": "Got at least: %s" % subtext,
-                            "short": False
-                        }
-                    ]
+                    "fields": [{"title": "Got at least: %s" % subtext, "short": False}],
                 }
-            ]
+            ],
         }
         client.make_request(data=report_data)
 
-        log_msg = 'DIGEST  : %s via %s :: %s reports' % (
-            kwargs['user'].user_name,
+        log_msg = "DIGEST  : %s via %s :: %s reports" % (
+            kwargs["user"].user_name,
             self.channel_visible_value,
-            template_vars['confirmed_total'])
+            template_vars["confirmed_total"],
+        )
         log.warning(log_msg)

backend/src/appenlight/models/application.py

@@ -1,104 +1,113 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import uuid
 import logging
 import sqlalchemy as sa
 from appenlight.models.resource import Resource
 from sqlalchemy.orm import aliased
 
 log = logging.getLogger(__name__)
 
 
 def generate_api_key():
-    uid = str(uuid.uuid4()).replace('-', '')
+    uid = str(uuid.uuid4()).replace("-", "")
     return uid[0:32]
 
 
 class Application(Resource):
     """
     Resource of application type
     """
 
-    __tablename__ = 'applications'
-    __mapper_args__ = {'polymorphic_identity': 'application'}
+    __tablename__ = "applications"
+    __mapper_args__ = {"polymorphic_identity": "application"}
 
     # lists configurable possible permissions for this resource type
-    __possible_permissions__ = ('view', 'update_reports')
-
-    resource_id = sa.Column(sa.Integer(),
-                            sa.ForeignKey('resources.resource_id',
-                                          onupdate='CASCADE',
-                                          ondelete='CASCADE', ),
-                            primary_key=True, )
-    domains = sa.Column(sa.UnicodeText(), nullable=False, default='')
-    api_key = sa.Column(sa.String(32), nullable=False, unique=True, index=True,
-                        default=generate_api_key)
-    public_key = sa.Column(sa.String(32), nullable=False, unique=True,
-                           index=True,
-                           default=generate_api_key)
-    default_grouping = sa.Column(sa.Unicode(20), nullable=False,
-                                 default='url_traceback')
+    __possible_permissions__ = ("view", "update_reports")
+
+    resource_id = sa.Column(
+        sa.Integer(),
+        sa.ForeignKey("resources.resource_id", onupdate="CASCADE", ondelete="CASCADE"),
+        primary_key=True,
+    )
+    domains = sa.Column(sa.UnicodeText(), nullable=False, default="")
+    api_key = sa.Column(
+        sa.String(32), nullable=False, unique=True, index=True, default=generate_api_key
+    )
+    public_key = sa.Column(
+        sa.String(32), nullable=False, unique=True, index=True, default=generate_api_key
+    )
+    default_grouping = sa.Column(
+        sa.Unicode(20), nullable=False, default="url_traceback"
+    )
     error_report_threshold = sa.Column(sa.Integer(), default=10)
     slow_report_threshold = sa.Column(sa.Integer(), default=10)
-    allow_permanent_storage = sa.Column(sa.Boolean(), default=False,
-                                        nullable=False)
+    allow_permanent_storage = sa.Column(sa.Boolean(), default=False, nullable=False)
 
-    @sa.orm.validates('default_grouping')
+    @sa.orm.validates("default_grouping")
     def validate_default_grouping(self, key, grouping):
         """ validate if resouce can have specific permission """
-        assert grouping in ['url_type', 'url_traceback', 'traceback_server']
+        assert grouping in ["url_type", "url_traceback", "traceback_server"]
         return grouping
 
-    report_groups = sa.orm.relationship('ReportGroup',
-                                        cascade="all, delete-orphan",
-                                        passive_deletes=True,
-                                        passive_updates=True,
-                                        lazy='dynamic',
-                                        backref=sa.orm.backref('application',
-                                                               lazy="joined"))
-
-    postprocess_conf = sa.orm.relationship('ApplicationPostprocessConf',
-                                           cascade="all, delete-orphan",
-                                           passive_deletes=True,
-                                           passive_updates=True,
-                                           backref='resource')
-
-    logs = sa.orm.relationship('Log',
-                               lazy='dynamic',
-                               backref='application',
-                               passive_deletes=True,
-                               passive_updates=True, )
-
-    integrations = sa.orm.relationship('IntegrationBase',
-                                       backref='resource',
-                                       cascade="all, delete-orphan",
-                                       passive_deletes=True,
-                                       passive_updates=True, )
+    report_groups = sa.orm.relationship(
+        "ReportGroup",
+        cascade="all, delete-orphan",
+        passive_deletes=True,
+        passive_updates=True,
+        lazy="dynamic",
+        backref=sa.orm.backref("application", lazy="joined"),
+    )
+
+    postprocess_conf = sa.orm.relationship(
+        "ApplicationPostprocessConf",
+        cascade="all, delete-orphan",
+        passive_deletes=True,
+        passive_updates=True,
+        backref="resource",
+    )
+
+    logs = sa.orm.relationship(
+        "Log",
+        lazy="dynamic",
+        backref="application",
+        passive_deletes=True,
+        passive_updates=True,
+    )
+
+    integrations = sa.orm.relationship(
+        "IntegrationBase",
+        backref="resource",
+        cascade="all, delete-orphan",
+        passive_deletes=True,
+        passive_updates=True,
+    )
 
     def generate_api_key(self):
         return generate_api_key()
 
 
 def after_update(mapper, connection, target):
     from appenlight.models.services.application import ApplicationService
-    log.info('clearing out ApplicationService cache')
+
+    log.info("clearing out ApplicationService cache")
     ApplicationService.by_id_cached().invalidate(target.resource_id)
     ApplicationService.by_api_key_cached().invalidate(target.api_key)
 
 
-sa.event.listen(Application, 'after_update', after_update)
-sa.event.listen(Application, 'after_delete', after_update)
+sa.event.listen(Application, "after_update", after_update)
+sa.event.listen(Application, "after_delete", after_update)

backend/src/appenlight/models/application_postprocess_conf.py

@@ -1,45 +1,47 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from ziggurat_foundations.models.base import BaseModel
 import sqlalchemy as sa
 
 from appenlight.models import Base
 from appenlight.models.report_group import ReportGroup
 
 
 class ApplicationPostprocessConf(Base, BaseModel):
     """
     Stores prioritizing conditions for reports
     This is later used for rule parsing like "if 10 occurences bump priority +1"
     """
 
-    __tablename__ = 'application_postprocess_conf'
+    __tablename__ = "application_postprocess_conf"
 
     pkey = sa.Column(sa.Integer(), nullable=False, primary_key=True)
-    resource_id = sa.Column(sa.Integer(),
-                            sa.ForeignKey('resources.resource_id',
-                                          onupdate='CASCADE',
-                                          ondelete='CASCADE'))
+    resource_id = sa.Column(
+        sa.Integer(),
+        sa.ForeignKey("resources.resource_id", onupdate="CASCADE", ondelete="CASCADE"),
+    )
     do = sa.Column(sa.Unicode(25), nullable=False)
-    new_value = sa.Column(sa.UnicodeText(), nullable=False, default='')
-    rule = sa.Column(sa.dialects.postgresql.JSON,
-                     nullable=False, default={'field': 'http_status',
-                                              "op": "ge", "value": "500"})
+    new_value = sa.Column(sa.UnicodeText(), nullable=False, default="")
+    rule = sa.Column(
+        sa.dialects.postgresql.JSON,
+        nullable=False,
+        default={"field": "http_status", "op": "ge", "value": "500"},
+    )
 
     def postprocess(self, item):
         new_value = int(self.new_value)
         item.priority = ReportGroup.priority + new_value

backend/src/appenlight/models/auth_token.py

@@ -1,52 +1,57 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 import sqlalchemy as sa
 
 from datetime import datetime
 from appenlight.models import Base
 from ziggurat_foundations.models.base import BaseModel
 from ziggurat_foundations.models.services.user import UserService
 
 log = logging.getLogger(__name__)
 
 
 class AuthToken(Base, BaseModel):
     """
     Stores information about possible alerting options
     """
-    __tablename__ = 'auth_tokens'
+
+    __tablename__ = "auth_tokens"
 
     id = sa.Column(sa.Integer, primary_key=True, nullable=False)
-    token = sa.Column(sa.Unicode(40), nullable=False,
-                      default=lambda x: UserService.generate_random_string(40))
-    owner_id = sa.Column(sa.Unicode(30),
-                         sa.ForeignKey('users.id', onupdate='CASCADE',
-                                       ondelete='CASCADE'))
+    token = sa.Column(
+        sa.Unicode(40),
+        nullable=False,
+        default=lambda x: UserService.generate_random_string(40),
+    )
+    owner_id = sa.Column(
+        sa.Unicode(30),
+        sa.ForeignKey("users.id", onupdate="CASCADE", ondelete="CASCADE"),
+    )
     creation_date = sa.Column(sa.DateTime, default=lambda x: datetime.utcnow())
     expires = sa.Column(sa.DateTime)
-    description = sa.Column(sa.Unicode, default='')
+    description = sa.Column(sa.Unicode, default="")
 
     @property
     def is_expired(self):
         if self.expires:
             return self.expires < datetime.utcnow()
         else:
             return False
 
     def __str__(self):
-        return '<AuthToken u:%s t:%s...>' % (self.owner_id, self.token[0:10])
+        return "<AuthToken u:%s t:%s...>" % (self.owner_id, self.token[0:10])

backend/src/appenlight/models/config.py

@@ -1,32 +1,32 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import sqlalchemy as sa
 from ziggurat_foundations.models.base import BaseModel
 from sqlalchemy.dialects.postgresql import JSON
 
 from . import Base
 
 
 class Config(Base, BaseModel):
-    __tablename__ = 'config'
+    __tablename__ = "config"
 
     key = sa.Column(sa.Unicode, primary_key=True)
     section = sa.Column(sa.Unicode, primary_key=True)
     value = sa.Column(JSON, nullable=False)
 
     def __json__(self, request):
         return self.get_dict()

backend/src/appenlight/models/event.py

@@ -1,165 +1,170 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import sqlalchemy as sa
 import logging
 
 from datetime import datetime
 from appenlight.models import Base, get_db_session
 from appenlight.models.services.report_stat import ReportStatService
 from appenlight.models.integrations import IntegrationException
 from pyramid.threadlocal import get_current_request
 from sqlalchemy.dialects.postgresql import JSON
 from ziggurat_foundations.models.base import BaseModel
 from ziggurat_foundations.models.services.resource import ResourceService
 
 log = logging.getLogger(__name__)
 
 
 class Event(Base, BaseModel):
-    __tablename__ = 'events'
+    __tablename__ = "events"
 
-    types = {'error_report_alert': 1,
-             'slow_report_alert': 3,
-             'comment': 5,
-             'assignment': 6,
-             'uptime_alert': 7,
-             'chart_alert': 9}
+    types = {
+        "error_report_alert": 1,
+        "slow_report_alert": 3,
+        "comment": 5,
+        "assignment": 6,
+        "uptime_alert": 7,
+        "chart_alert": 9,
+    }
 
-    statuses = {'active': 1,
-                'closed': 0}
+    statuses = {"active": 1, "closed": 0}
 
     id = sa.Column(sa.Integer, primary_key=True)
     start_date = sa.Column(sa.DateTime, default=datetime.utcnow)
     end_date = sa.Column(sa.DateTime)
     status = sa.Column(sa.Integer, default=1)
     event_type = sa.Column(sa.Integer, default=1)
-    origin_user_id = sa.Column(sa.Integer(), sa.ForeignKey('users.id'),
-                               nullable=True)
-    target_user_id = sa.Column(sa.Integer(), sa.ForeignKey('users.id'),
-                               nullable=True)
-    resource_id = sa.Column(sa.Integer(),
-                            sa.ForeignKey('resources.resource_id'),
-                            nullable=True)
+    origin_user_id = sa.Column(sa.Integer(), sa.ForeignKey("users.id"), nullable=True)
+    target_user_id = sa.Column(sa.Integer(), sa.ForeignKey("users.id"), nullable=True)
+    resource_id = sa.Column(
+        sa.Integer(), sa.ForeignKey("resources.resource_id"), nullable=True
+    )
     target_id = sa.Column(sa.Integer)
     target_uuid = sa.Column(sa.Unicode(40))
     text = sa.Column(sa.UnicodeText())
     values = sa.Column(JSON(), nullable=False, default=None)
 
     def __repr__(self):
-        return '<Event %s, app:%s, %s>' % (self.unified_alert_name(),
-                                            self.resource_id,
-                                            self.unified_alert_action())
+        return "<Event %s, app:%s, %s>" % (
+            self.unified_alert_name(),
+            self.resource_id,
+            self.unified_alert_action(),
+        )
 
     @property
     def reverse_types(self):
         return dict([(v, k) for k, v in self.types.items()])
 
     def unified_alert_name(self):
         return self.reverse_types[self.event_type]
 
     def unified_alert_action(self):
         event_name = self.reverse_types[self.event_type]
-        if self.status == Event.statuses['closed']:
+        if self.status == Event.statuses["closed"]:
             return "CLOSE"
-        if self.status != Event.statuses['closed']:
+        if self.status != Event.statuses["closed"]:
             return "OPEN"
         return event_name
 
     def send_alerts(self, request=None, resource=None, db_session=None):
         """" Sends alerts to applicable channels """
         db_session = get_db_session(db_session)
         db_session.flush()
         if not resource:
             resource = ResourceService.by_resource_id(self.resource_id)
         if not request:
             request = get_current_request()
         if not resource:
             return
-        users = set([p.user for p in ResourceService.users_for_perm(resource, 'view')])
+        users = set([p.user for p in ResourceService.users_for_perm(resource, "view")])
         for user in users:
             for channel in user.alert_channels:
-                matches_resource = not channel.resources or resource in [r.resource_id for r in channel.resources]
+                matches_resource = not channel.resources or resource in [
+                    r.resource_id for r in channel.resources
+                ]
                 if (
-                    not channel.channel_validated or
-                    not channel.send_alerts or
-                    not matches_resource
+                    not channel.channel_validated
+                    or not channel.send_alerts
+                    or not matches_resource
                 ):
                     continue
                 else:
                     try:
-                        channel.notify_alert(resource=resource,
-                                             event=self,
-                                             user=user,
-                                             request=request)
+                        channel.notify_alert(
+                            resource=resource, event=self, user=user, request=request
+                        )
                     except IntegrationException as e:
-                        log.warning('%s' % e)
+                        log.warning("%s" % e)
 
     def validate_or_close(self, since_when, db_session=None):
         """ Checks if alerts should stay open or it's time to close them.
             Generates close alert event if alerts get closed """
-        event_types = [Event.types['error_report_alert'],
-                       Event.types['slow_report_alert']]
+        event_types = [
+            Event.types["error_report_alert"],
+            Event.types["slow_report_alert"],
+        ]
         app = ResourceService.by_resource_id(self.resource_id)
         # if app was deleted close instantly
         if not app:
             self.close()
             return
 
         if self.event_type in event_types:
             total = ReportStatService.count_by_type(
-                self.event_type, self.resource_id, since_when)
-            if Event.types['error_report_alert'] == self.event_type:
+                self.event_type, self.resource_id, since_when
+            )
+            if Event.types["error_report_alert"] == self.event_type:
                 threshold = app.error_report_threshold
-            if Event.types['slow_report_alert'] == self.event_type:
+            if Event.types["slow_report_alert"] == self.event_type:
                 threshold = app.slow_report_threshold
 
             if total < threshold:
                 self.close()
 
     def close(self, db_session=None):
         """
         Closes an event and sends notification to affected users
         """
         self.end_date = datetime.utcnow()
-        self.status = Event.statuses['closed']
-        log.warning('ALERT: CLOSE: %s' % self)
+        self.status = Event.statuses["closed"]
+        log.warning("ALERT: CLOSE: %s" % self)
         self.send_alerts()
 
     def text_representation(self):
         alert_type = self.unified_alert_name()
-        text = ''
-        if 'slow_report' in alert_type:
-            text += 'Slow report alert'
-        if 'error_report' in alert_type:
-            text += 'Exception report alert'
-        if 'uptime_alert' in alert_type:
-            text += 'Uptime alert'
-        if 'chart_alert' in alert_type:
-            text += 'Metrics value alert'
+        text = ""
+        if "slow_report" in alert_type:
+            text += "Slow report alert"
+        if "error_report" in alert_type:
+            text += "Exception report alert"
+        if "uptime_alert" in alert_type:
+            text += "Uptime alert"
+        if "chart_alert" in alert_type:
+            text += "Metrics value alert"
 
         alert_action = self.unified_alert_action()
-        if alert_action == 'OPEN':
-            text += ' got opened.'
-        if alert_action == 'CLOSE':
-            text += ' got closed.'
+        if alert_action == "OPEN":
+            text += " got opened."
+        if alert_action == "CLOSE":
+            text += " got closed."
         return text
 
     def get_dict(self, request=None):
         dict_data = super(Event, self).get_dict()
-        dict_data['text'] = self.text_representation()
-        dict_data['resource_name'] = self.resource.resource_name
+        dict_data["text"] = self.text_representation()
+        dict_data["resource_name"] = self.resource.resource_name
         return dict_data

backend/src/appenlight/models/external_identity.py

@@ -1,36 +1,36 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import sqlalchemy as sa
 from sqlalchemy.ext.declarative import declared_attr
 from ziggurat_foundations.models.external_identity import ExternalIdentityMixin
 
 from appenlight.models import Base
 from appenlight.lib.sqlalchemy_fields import EncryptedUnicode
 
 
 class ExternalIdentity(ExternalIdentityMixin, Base):
     @declared_attr
     def access_token(self):
-        return sa.Column(EncryptedUnicode(255), default='')
+        return sa.Column(EncryptedUnicode(255), default="")
 
     @declared_attr
     def alt_token(self):
-        return sa.Column(EncryptedUnicode(255), default='')
+        return sa.Column(EncryptedUnicode(255), default="")
 
     @declared_attr
     def token_secret(self):
-        return sa.Column(EncryptedUnicode(255), default='')
+        return sa.Column(EncryptedUnicode(255), default="")

backend/src/appenlight/models/group.py

@@ -1,45 +1,46 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from ziggurat_foundations.models.group import GroupMixin
 from appenlight.models import Base
 
 
 class Group(GroupMixin, Base):
-    __possible_permissions__ = ('root_administration',
-                                'test_features',
-                                'admin_panel',
-                                'admin_users',
-                                'manage_partitions',)
+    __possible_permissions__ = (
+        "root_administration",
+        "test_features",
+        "admin_panel",
+        "admin_users",
+        "manage_partitions",
+    )
 
-    def get_dict(self, exclude_keys=None, include_keys=None,
-                 include_perms=False):
+    def get_dict(self, exclude_keys=None, include_keys=None, include_perms=False):
         result = super(Group, self).get_dict(exclude_keys, include_keys)
         if include_perms:
-            result['possible_permissions'] = self.__possible_permissions__
-            result['current_permissions'] = [p.perm_name for p in
-                                             self.permissions]
+            result["possible_permissions"] = self.__possible_permissions__
+            result["current_permissions"] = [p.perm_name for p in self.permissions]
         else:
-            result['possible_permissions'] = []
-            result['current_permissions'] = []
+            result["possible_permissions"] = []
+            result["current_permissions"] = []
         exclude_keys_list = exclude_keys or []
         include_keys_list = include_keys or []
         d = {}
         for k in result.keys():
-            if (k not in exclude_keys_list and
-                    (k in include_keys_list or not include_keys)):
+            if k not in exclude_keys_list and (
+                k in include_keys_list or not include_keys
+            ):
                 d[k] = result[k]
         return d

backend/src/appenlight/models/group_resource_permission.py

@@ -1,23 +1,24 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ziggurat_foundations.models.group_resource_permission import \
-    GroupResourcePermissionMixin
+from ziggurat_foundations.models.group_resource_permission import (
+    GroupResourcePermissionMixin,
+)
 from appenlight.models import Base
 
 
 class GroupResourcePermission(GroupResourcePermissionMixin, Base):
     pass

backend/src/appenlight/models/integrations/__init__.py

@@ -1,78 +1,80 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import sqlalchemy as sa
 from sqlalchemy.dialects.postgresql import JSON
 from sqlalchemy.ext.hybrid import hybrid_property
 from ziggurat_foundations.models.base import BaseModel
 
 from appenlight.lib.encryption import decrypt_dictionary_keys
 from appenlight.lib.encryption import encrypt_dictionary_keys
 from appenlight.models import Base, get_db_session
 
 
 class IntegrationException(Exception):
     pass
 
 
 class IntegrationBase(Base, BaseModel):
     """
     Model from which all integrations inherit using polymorphic approach
     """
-    __tablename__ = 'integrations'
+
+    __tablename__ = "integrations"
 
     front_visible = False
     as_alert_channel = False
     supports_report_alerting = False
 
     id = sa.Column(sa.Integer, primary_key=True)
-    resource_id = sa.Column(sa.Integer,
-                            sa.ForeignKey('applications.resource_id'))
+    resource_id = sa.Column(sa.Integer, sa.ForeignKey("applications.resource_id"))
     integration_name = sa.Column(sa.Unicode(64))
-    _config = sa.Column('config', JSON(), nullable=False, default='')
+    _config = sa.Column("config", JSON(), nullable=False, default="")
     modified_date = sa.Column(sa.DateTime)
 
-    channel = sa.orm.relationship('AlertChannel',
-                                  cascade="all,delete-orphan",
-                                  passive_deletes=True,
-                                  passive_updates=True,
-                                  uselist=False,
-                                  backref='integration')
+    channel = sa.orm.relationship(
+        "AlertChannel",
+        cascade="all,delete-orphan",
+        passive_deletes=True,
+        passive_updates=True,
+        uselist=False,
+        backref="integration",
+    )
 
     __mapper_args__ = {
-        'polymorphic_on': 'integration_name',
-        'polymorphic_identity': 'integration'
+        "polymorphic_on": "integration_name",
+        "polymorphic_identity": "integration",
     }
 
     @classmethod
-    def by_app_id_and_integration_name(cls, resource_id, integration_name,
-                                       db_session=None):
+    def by_app_id_and_integration_name(
+        cls, resource_id, integration_name, db_session=None
+    ):
         db_session = get_db_session(db_session)
         query = db_session.query(cls)
         query = query.filter(cls.integration_name == integration_name)
         query = query.filter(cls.resource_id == resource_id)
         return query.first()
 
     @hybrid_property
     def config(self):
         return decrypt_dictionary_keys(self._config)
 
     @config.setter
     def config(self, value):
-        if not hasattr(value, 'items'):
-            raise Exception('IntegrationBase.config only accepts '
-                            'flat dictionaries')
+        if not hasattr(value, "items"):
+            raise Exception("IntegrationBase.config only accepts " "flat dictionaries")
         self._config = encrypt_dictionary_keys(value)

backend/src/appenlight/models/integrations/bitbucket.py

@@ -1,163 +1,188 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import requests
 from requests_oauthlib import OAuth1
-from appenlight.models.integrations import (IntegrationBase,
-                                            IntegrationException)
+from appenlight.models.integrations import IntegrationBase, IntegrationException
 
 _ = str
 
 
 class NotFoundException(Exception):
     pass
 
 
 class BitbucketIntegration(IntegrationBase):
-    __mapper_args__ = {
-        'polymorphic_identity': 'bitbucket'
-    }
+    __mapper_args__ = {"polymorphic_identity": "bitbucket"}
     front_visible = True
     as_alert_channel = False
     supports_report_alerting = False
     action_notification = True
-    integration_action = 'Add issue to Bitbucket'
+    integration_action = "Add issue to Bitbucket"
 
     @classmethod
     def create_client(cls, request, user_name=None, repo_name=None):
         """
         Creates REST client that can authenticate to specific repo
         uses auth tokens for current request user
         """
         config = request.registry.settings
         token = None
         secret = None
         for identity in request.user.external_identities:
-            if identity.provider_name == 'bitbucket':
+            if identity.provider_name == "bitbucket":
                 token = identity.access_token
                 secret = identity.token_secret
                 break
         if not token:
-            raise IntegrationException(
-                'No valid auth token present for this service')
-        client = BitbucketClient(token, secret,
-                                 user_name,
-                                 repo_name,
-                                 config['authomatic.pr.bitbucket.key'],
-                                 config['authomatic.pr.bitbucket.secret'])
+            raise IntegrationException("No valid auth token present for this service")
+        client = BitbucketClient(
+            token,
+            secret,
+            user_name,
+            repo_name,
+            config["authomatic.pr.bitbucket.key"],
+            config["authomatic.pr.bitbucket.secret"],
+        )
         return client
 
 
 class BitbucketClient(object):
-    api_url = 'https://bitbucket.org/api/1.0'
-    repo_type = 'bitbucket'
-
-    def __init__(self, token, secret, owner, repo_name, bitbucket_consumer_key,
-                 bitbucket_consumer_secret):
+    api_url = "https://bitbucket.org/api/1.0"
+    repo_type = "bitbucket"
+
+    def __init__(
+        self,
+        token,
+        secret,
+        owner,
+        repo_name,
+        bitbucket_consumer_key,
+        bitbucket_consumer_secret,
+    ):
         self.access_token = token
         self.token_secret = secret
         self.owner = owner
         self.repo_name = repo_name
         self.bitbucket_consumer_key = bitbucket_consumer_key
         self.bitbucket_consumer_secret = bitbucket_consumer_secret
 
     possible_keys = {
-        'status': ['new', 'open', 'resolved', 'on hold', 'invalid',
-                   'duplicate', 'wontfix'],
-        'priority': ['trivial', 'minor', 'major', 'critical', 'blocker'],
-        'kind': ['bug', 'enhancement', 'proposal', 'task']
+        "status": [
+            "new",
+            "open",
+            "resolved",
+            "on hold",
+            "invalid",
+            "duplicate",
+            "wontfix",
+        ],
+        "priority": ["trivial", "minor", "major", "critical", "blocker"],
+        "kind": ["bug", "enhancement", "proposal", "task"],
     }
 
     def get_statuses(self):
         """Gets list of possible item statuses"""
-        return self.possible_keys['status']
+        return self.possible_keys["status"]
 
     def get_priorities(self):
         """Gets list of possible item statuses"""
-        return self.possible_keys['priority']
+        return self.possible_keys["priority"]
 
-    def make_request(self, url, method='get', data=None, headers=None):
+    def make_request(self, url, method="get", data=None, headers=None):
         """
         Performs HTTP request to bitbucket
         """
-        auth = OAuth1(self.bitbucket_consumer_key,
-                      self.bitbucket_consumer_secret,
-                      self.access_token, self.token_secret)
+        auth = OAuth1(
+            self.bitbucket_consumer_key,
+            self.bitbucket_consumer_secret,
+            self.access_token,
+            self.token_secret,
+        )
         try:
-            resp = getattr(requests, method)(url, data=data, auth=auth,
-                                             timeout=10)
+            resp = getattr(requests, method)(url, data=data, auth=auth, timeout=10)
         except Exception as e:
             raise IntegrationException(
-                _('Error communicating with Bitbucket: %s') % (e,))
+                _("Error communicating with Bitbucket: %s") % (e,)
+            )
         if resp.status_code == 401:
-            raise IntegrationException(
-                _('You are not authorized to access this repo'))
+            raise IntegrationException(_("You are not authorized to access this repo"))
         elif resp.status_code == 404:
-            raise IntegrationException(_('User or repo name are incorrect'))
+            raise IntegrationException(_("User or repo name are incorrect"))
         elif resp.status_code not in [200, 201]:
             raise IntegrationException(
-                _('Bitbucket response_code: %s') % resp.status_code)
+                _("Bitbucket response_code: %s") % resp.status_code
+            )
         try:
             return resp.json()
         except Exception as e:
             raise IntegrationException(
-                _('Error decoding response from Bitbucket: %s') % (e,))
+                _("Error decoding response from Bitbucket: %s") % (e,)
+            )
 
     def get_assignees(self):
         """Gets list of possible assignees"""
-        url = '%(api_url)s/privileges/%(owner)s/%(repo_name)s' % {
-            'api_url': self.api_url,
-            'owner': self.owner,
-            'repo_name': self.repo_name}
+        url = "%(api_url)s/privileges/%(owner)s/%(repo_name)s" % {
+            "api_url": self.api_url,
+            "owner": self.owner,
+            "repo_name": self.repo_name,
+        }
 
         data = self.make_request(url)
-        results = [{'user': self.owner, 'name': '(Repo owner)'}]
+        results = [{"user": self.owner, "name": "(Repo owner)"}]
         if data:
             for entry in data:
-                results.append({"user": entry['user']['username'],
-                                "name": entry['user'].get('display_name')})
+                results.append(
+                    {
+                        "user": entry["user"]["username"],
+                        "name": entry["user"].get("display_name"),
+                    }
+                )
         return results
 
     def create_issue(self, form_data):
         """
         Sends creates a new issue in tracker using REST call
         """
-        url = '%(api_url)s/repositories/%(owner)s/%(repo_name)s/issues/' % {
-            'api_url': self.api_url,
-            'owner': self.owner,
-            'repo_name': self.repo_name}
+        url = "%(api_url)s/repositories/%(owner)s/%(repo_name)s/issues/" % {
+            "api_url": self.api_url,
+            "owner": self.owner,
+            "repo_name": self.repo_name,
+        }
 
         payload = {
-            "title": form_data['title'],
-            "content": form_data['content'],
-            "kind": form_data['kind'],
-            "priority": form_data['priority'],
-            "responsible": form_data['responsible']
+            "title": form_data["title"],
+            "content": form_data["content"],
+            "kind": form_data["kind"],
+            "priority": form_data["priority"],
+            "responsible": form_data["responsible"],
         }
-        data = self.make_request(url, 'post', payload)
+        data = self.make_request(url, "post", payload)
         f_args = {
             "owner": self.owner,
             "repo_name": self.repo_name,
-            "issue_id": data['local_id']
+            "issue_id": data["local_id"],
         }
-        web_url = 'https://bitbucket.org/%(owner)s/%(repo_name)s' \
-                  '/issue/%(issue_id)s/issue-title' % f_args
+        web_url = (
+            "https://bitbucket.org/%(owner)s/%(repo_name)s"
+            "/issue/%(issue_id)s/issue-title" % f_args
+        )
         to_return = {
-            'id': data['local_id'],
-            'resource_url': data['resource_uri'],
-            'web_url': web_url
+            "id": data["local_id"],
+            "resource_url": data["resource_uri"],
+            "web_url": web_url,
         }
         return to_return

backend/src/appenlight/models/integrations/campfire.py

@@ -1,74 +1,71 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 
 from requests.exceptions import HTTPError, ConnectionError
 from camplight import Request, Campfire
 from camplight.exceptions import CamplightException
 
-from appenlight.models.integrations import (IntegrationBase,
-                                            IntegrationException)
+from appenlight.models.integrations import IntegrationBase, IntegrationException
 
 _ = str
 
 log = logging.getLogger(__name__)
 
 
 class NotFoundException(Exception):
     pass
 
 
 class CampfireIntegration(IntegrationBase):
-    __mapper_args__ = {
-        'polymorphic_identity': 'campfire'
-    }
+    __mapper_args__ = {"polymorphic_identity": "campfire"}
     front_visible = False
     as_alert_channel = True
     supports_report_alerting = True
     action_notification = True
-    integration_action = 'Message via Campfire'
+    integration_action = "Message via Campfire"
 
     @classmethod
     def create_client(cls, api_token, account):
         client = CampfireClient(api_token, account)
         return client
 
 
 class CampfireClient(object):
     def __init__(self, api_token, account):
-        request = Request('https://%s.campfirenow.com' % account, api_token)
+        request = Request("https://%s.campfirenow.com" % account, api_token)
         self.campfire = Campfire(request)
 
     def get_account(self):
         try:
             return self.campfire.account()
         except (HTTPError, CamplightException) as e:
             raise IntegrationException(str(e))
 
     def get_rooms(self):
         try:
             return self.campfire.rooms()
         except (HTTPError, CamplightException) as e:
             raise IntegrationException(str(e))
 
-    def speak_to_room(self, room, message, sound='RIMSHOT'):
+    def speak_to_room(self, room, message, sound="RIMSHOT"):
         try:
             room = self.campfire.room(room)
             room.join()
-            room.speak(message, type_='TextMessage')
+            room.speak(message, type_="TextMessage")
         except (HTTPError, CamplightException, ConnectionError) as e:
             raise IntegrationException(str(e))

backend/src/appenlight/models/integrations/flowdock.py

@@ -1,82 +1,83 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 
 import requests
 from requests.auth import HTTPBasicAuth
 import simplejson as json
 
-from appenlight.models.integrations import (IntegrationBase,
-                                            IntegrationException)
+from appenlight.models.integrations import IntegrationBase, IntegrationException
 
 _ = str
 
 log = logging.getLogger(__name__)
 
 
 class NotFoundException(Exception):
     pass
 
 
 class FlowdockIntegration(IntegrationBase):
-    __mapper_args__ = {
-        'polymorphic_identity': 'flowdock'
-    }
+    __mapper_args__ = {"polymorphic_identity": "flowdock"}
     front_visible = False
     as_alert_channel = True
     supports_report_alerting = True
     action_notification = True
-    integration_action = 'Message via Flowdock'
+    integration_action = "Message via Flowdock"
 
     @classmethod
     def create_client(cls, api_token):
         client = FlowdockClient(api_token)
         return client
 
 
 class FlowdockClient(object):
     def __init__(self, api_token):
-        self.auth = HTTPBasicAuth(api_token, '')
+        self.auth = HTTPBasicAuth(api_token, "")
         self.api_token = api_token
-        self.api_url = 'https://api.flowdock.com/v1/messages'
+        self.api_url = "https://api.flowdock.com/v1/messages"
 
-    def make_request(self, url, method='get', data=None):
+    def make_request(self, url, method="get", data=None):
         headers = {
-            'Content-Type': 'application/json',
-            'User-Agent': 'appenlight-flowdock',
+            "Content-Type": "application/json",
+            "User-Agent": "appenlight-flowdock",
         }
         try:
             if data:
                 data = json.dumps(data)
-            resp = getattr(requests, method)(url, data=data, headers=headers,
-                                             timeout=10)
+            resp = getattr(requests, method)(
+                url, data=data, headers=headers, timeout=10
+            )
         except Exception as e:
             raise IntegrationException(
-                _('Error communicating with Flowdock: %s') % (e,))
+                _("Error communicating with Flowdock: %s") % (e,)
+            )
         if resp.status_code > 299:
             raise IntegrationException(resp.text)
         return resp
 
     def send_to_chat(self, payload):
-        url = '%(api_url)s/chat/%(api_token)s' % {'api_url': self.api_url,
-                                                  'api_token': self.api_token}
-        return self.make_request(url, method='post', data=payload).json()
+        url = "%(api_url)s/chat/%(api_token)s" % {
+            "api_url": self.api_url,
+            "api_token": self.api_token,
+        }
+        return self.make_request(url, method="post", data=payload).json()
 
     def send_to_inbox(self, payload):
-        f_args = {'api_url': self.api_url, 'api_token': self.api_token}
-        url = '%(api_url)s/team_inbox/%(api_token)s' % f_args
-        return self.make_request(url, method='post', data=payload).json()
+        f_args = {"api_url": self.api_url, "api_token": self.api_token}
+        url = "%(api_url)s/team_inbox/%(api_token)s" % f_args
+        return self.make_request(url, method="post", data=payload).json()

backend/src/appenlight/models/integrations/github.py

@@ -1,156 +1,158 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import json
 import requests
 
 from . import IntegrationBase, IntegrationException
 
 _ = str
 
 
 class GithubAuthException(Exception):
     pass
 
 
 class GithubIntegration(IntegrationBase):
-    __mapper_args__ = {
-        'polymorphic_identity': 'github'
-    }
+    __mapper_args__ = {"polymorphic_identity": "github"}
     front_visible = True
     as_alert_channel = False
     supports_report_alerting = False
     action_notification = True
-    integration_action = 'Add issue to Github'
+    integration_action = "Add issue to Github"
 
     @classmethod
     def create_client(cls, request, user_name=None, repo_name=None):
         """
         Creates REST client that can authenticate to specific repo
         uses auth tokens for current request user
         """
         token = None
         secret = None
         for identity in request.user.external_identities:
-            if identity.provider_name == 'github':
+            if identity.provider_name == "github":
                 token = identity.access_token
                 secret = identity.token_secret
                 break
         if not token:
-            raise IntegrationException(
-                'No valid auth token present for this service')
+            raise IntegrationException("No valid auth token present for this service")
         client = GithubClient(token=token, owner=user_name, name=repo_name)
         return client
 
 
 class GithubClient(object):
-    api_url = 'https://api.github.com'
-    repo_type = 'github'
+    api_url = "https://api.github.com"
+    repo_type = "github"
 
     def __init__(self, token, owner, name):
         self.access_token = token
         self.owner = owner
         self.name = name
 
-    def make_request(self, url, method='get', data=None, headers=None):
-        req_headers = {'User-Agent': 'appenlight',
-                       'Content-Type': 'application/json',
-                       'Authorization': 'token %s' % self.access_token}
+    def make_request(self, url, method="get", data=None, headers=None):
+        req_headers = {
+            "User-Agent": "appenlight",
+            "Content-Type": "application/json",
+            "Authorization": "token %s" % self.access_token,
+        }
         try:
             if data:
                 data = json.dumps(data)
-            resp = getattr(requests, method)(url, data=data,
-                                             headers=req_headers,
-                                             timeout=10)
+            resp = getattr(requests, method)(
+                url, data=data, headers=req_headers, timeout=10
+            )
         except Exception as e:
-            msg = 'Error communicating with Github: %s'
+            msg = "Error communicating with Github: %s"
             raise IntegrationException(_(msg) % (e,))
 
         if resp.status_code == 404:
-            msg = 'User or repo name are incorrect'
+            msg = "User or repo name are incorrect"
             raise IntegrationException(_(msg))
         if resp.status_code == 401:
-            msg = 'You are not authorized to access this repo'
+            msg = "You are not authorized to access this repo"
             raise IntegrationException(_(msg))
         elif resp.status_code not in [200, 201]:
-            msg = 'Github response_code: %s'
+            msg = "Github response_code: %s"
             raise IntegrationException(_(msg) % resp.status_code)
         try:
             return resp.json()
         except Exception as e:
-            msg = 'Error decoding response from Github: %s'
+            msg = "Error decoding response from Github: %s"
             raise IntegrationException(_(msg) % (e,))
 
     def get_statuses(self):
         """Gets list of possible item statuses"""
-        url = '%(api_url)s/repos/%(owner)s/%(name)s/labels' % {
-            'api_url': self.api_url,
-            'owner': self.owner,
-            'name': self.name}
+        url = "%(api_url)s/repos/%(owner)s/%(name)s/labels" % {
+            "api_url": self.api_url,
+            "owner": self.owner,
+            "name": self.name,
+        }
 
         data = self.make_request(url)
 
         statuses = []
         for status in data:
-            statuses.append(status['name'])
+            statuses.append(status["name"])
         return statuses
 
     def get_repo(self):
         """Gets list of possible item statuses"""
-        url = '%(api_url)s/repos/%(owner)s/%(name)s' % {
-            'api_url': self.api_url,
-            'owner': self.owner,
-            'name': self.name}
+        url = "%(api_url)s/repos/%(owner)s/%(name)s" % {
+            "api_url": self.api_url,
+            "owner": self.owner,
+            "name": self.name,
+        }
 
         data = self.make_request(url)
         return data
 
     def get_assignees(self):
         """Gets list of possible assignees"""
-        url = '%(api_url)s/repos/%(owner)s/%(name)s/collaborators' % {
-            'api_url': self.api_url,
-            'owner': self.owner,
-            'name': self.name}
+        url = "%(api_url)s/repos/%(owner)s/%(name)s/collaborators" % {
+            "api_url": self.api_url,
+            "owner": self.owner,
+            "name": self.name,
+        }
         data = self.make_request(url)
         results = []
         for entry in data:
-            results.append({"user": entry['login'],
-                            "name": entry.get('name')})
+            results.append({"user": entry["login"], "name": entry.get("name")})
         return results
 
     def create_issue(self, form_data):
         """
         Make a REST call to create issue in Github's issue tracker
         """
-        url = '%(api_url)s/repos/%(owner)s/%(name)s/issues' % {
-            'api_url': self.api_url,
-            'owner': self.owner,
-            'name': self.name}
+        url = "%(api_url)s/repos/%(owner)s/%(name)s/issues" % {
+            "api_url": self.api_url,
+            "owner": self.owner,
+            "name": self.name,
+        }
 
         payload = {
-            "title": form_data['title'],
-            "body": form_data['content'],
+            "title": form_data["title"],
+            "body": form_data["content"],
             "labels": [],
-            "assignee": form_data['responsible']
+            "assignee": form_data["responsible"],
         }
-        payload['labels'].extend(form_data['kind'])
-        data = self.make_request(url, 'post', data=payload)
+        payload["labels"].extend(form_data["kind"])
+        data = self.make_request(url, "post", data=payload)
         to_return = {
-            'id': data['number'],
-            'resource_url': data['url'],
-            'web_url': data['html_url']
+            "id": data["number"],
+            "resource_url": data["url"],
+            "web_url": data["html_url"],
         }
         return to_return

backend/src/appenlight/models/integrations/hipchat.py

@@ -1,83 +1,75 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 
 import requests
 
 from . import IntegrationBase, IntegrationException
 
 _ = str
 
 log = logging.getLogger(__name__)
 
 
 class NotFoundException(Exception):
     pass
 
 
 class HipchatIntegration(IntegrationBase):
-    __mapper_args__ = {
-        'polymorphic_identity': 'hipchat'
-    }
+    __mapper_args__ = {"polymorphic_identity": "hipchat"}
     front_visible = False
     as_alert_channel = True
     supports_report_alerting = True
     action_notification = True
-    integration_action = 'Message via Hipchat'
+    integration_action = "Message via Hipchat"
 
     @classmethod
     def create_client(cls, api_token):
         client = HipchatClient(api_token)
         return client
 
 
 class HipchatClient(object):
     def __init__(self, api_token):
         self.api_token = api_token
-        self.api_url = 'https://api.hipchat.com/v1'
-
-    def make_request(self, endpoint, method='get', data=None):
-        headers = {
-            'User-Agent': 'appenlight-hipchat',
-        }
-        url = '%s%s' % (self.api_url, endpoint)
-        params = {
-            'format': 'json',
-            'auth_token': self.api_token
-        }
+        self.api_url = "https://api.hipchat.com/v1"
+
+    def make_request(self, endpoint, method="get", data=None):
+        headers = {"User-Agent": "appenlight-hipchat"}
+        url = "%s%s" % (self.api_url, endpoint)
+        params = {"format": "json", "auth_token": self.api_token}
         try:
-            resp = getattr(requests, method)(url, data=data, headers=headers,
-                                             params=params,
-                                             timeout=3)
+            resp = getattr(requests, method)(
+                url, data=data, headers=headers, params=params, timeout=3
+            )
         except Exception as e:
-            msg = 'Error communicating with Hipchat: %s'
+            msg = "Error communicating with Hipchat: %s"
             raise IntegrationException(_(msg) % (e,))
         if resp.status_code == 404:
-            msg = 'Error communicating with Hipchat - Room not found'
+            msg = "Error communicating with Hipchat - Room not found"
             raise IntegrationException(msg)
         elif resp.status_code != requests.codes.ok:
-            msg = 'Error communicating with Hipchat - status code: %s'
+            msg = "Error communicating with Hipchat - status code: %s"
             raise IntegrationException(msg % resp.status_code)
         return resp
 
     def get_rooms(self):
         # not used with notification api token
-        return self.make_request('/rooms/list')
+        return self.make_request("/rooms/list")
 
     def send(self, payload):
-        return self.make_request('/rooms/message', method='post',
-                                 data=payload).json()
+        return self.make_request("/rooms/message", method="post", data=payload).json()

backend/src/appenlight/models/integrations/jira.py

@@ -1,136 +1,137 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import jira
-from appenlight.models.integrations import (IntegrationBase,
-                                            IntegrationException)
+from appenlight.models.integrations import IntegrationBase, IntegrationException
 
 _ = str
 
 
 class NotFoundException(Exception):
     pass
 
 
 class JiraIntegration(IntegrationBase):
-    __mapper_args__ = {
-        'polymorphic_identity': 'jira'
-    }
+    __mapper_args__ = {"polymorphic_identity": "jira"}
     front_visible = True
     as_alert_channel = False
     supports_report_alerting = False
     action_notification = True
-    integration_action = 'Add issue to Jira'
+    integration_action = "Add issue to Jira"
 
 
 class JiraClient(object):
     def __init__(self, user_name, password, host_name, project, request=None):
         self.user_name = user_name
         self.password = password
         self.host_name = host_name
         self.project = project
         self.request = request
         try:
-            self.client = jira.client.JIRA(options={'server': host_name},
-                                           basic_auth=(user_name, password))
+            self.client = jira.client.JIRA(
+                options={"server": host_name}, basic_auth=(user_name, password)
+            )
         except jira.JIRAError as e:
             raise IntegrationException(
-                'Communication problem: HTTP_STATUS:%s, URL:%s ' % (
-                    e.status_code, e.url))
+                "Communication problem: HTTP_STATUS:%s, URL:%s "
+                % (e.status_code, e.url)
+            )
 
     def get_projects(self):
         projects = self.client.projects()
         return projects
 
     def get_assignees(self, request):
         """Gets list of possible assignees"""
         cache_region = request.registry.cache_regions.redis_sec_30
-        @cache_region.cache_on_arguments('JiraClient.get_assignees')
+
+        @cache_region.cache_on_arguments("JiraClient.get_assignees")
         def cached(project_name):
             users = self.client.search_assignable_users_for_issues(
-                None, project=project_name)
+                None, project=project_name
+            )
             results = []
             for user in users:
                 results.append({"id": user.name, "name": user.displayName})
             return results
+
         return cached(self.project)
 
     def get_issue_types(self, request):
         metadata = self.get_metadata(request)
         assignees = self.get_assignees(request)
         parsed_metadata = []
-        for entry in metadata['projects'][0]['issuetypes']:
-            issue = {"name": entry['name'],
-                     "id": entry['id'],
-                     "fields": []}
-            for i_id, field_i in entry['fields'].items():
+        for entry in metadata["projects"][0]["issuetypes"]:
+            issue = {"name": entry["name"], "id": entry["id"], "fields": []}
+            for i_id, field_i in entry["fields"].items():
                 field = {
-                    "name": field_i['name'],
+                    "name": field_i["name"],
                     "id": i_id,
-                    "required": field_i['required'],
+                    "required": field_i["required"],
                     "values": [],
-                    "type": field_i['schema'].get('type')
+                    "type": field_i["schema"].get("type"),
                 }
-                if field_i.get('allowedValues'):
-                    field['values'] = []
-                    for i in field_i['allowedValues']:
-                        field['values'].append(
-                            {'id': i['id'],
-                             'name': i.get('name', i.get('value', ''))
-                             })
-                if field['id'] == 'assignee':
-                    field['values'] = assignees
-                issue['fields'].append(field)
+                if field_i.get("allowedValues"):
+                    field["values"] = []
+                    for i in field_i["allowedValues"]:
+                        field["values"].append(
+                            {"id": i["id"], "name": i.get("name", i.get("value", ""))}
+                        )
+                if field["id"] == "assignee":
+                    field["values"] = assignees
+                issue["fields"].append(field)
             parsed_metadata.append(issue)
         return parsed_metadata
 
     def get_metadata(self, request):
         # cache_region = request.registry.cache_regions.redis_sec_30
         # @cache_region.cache_on_arguments('JiraClient.get_metadata')
         def cached(project_name):
             return self.client.createmeta(
-                projectKeys=project_name, expand='projects.issuetypes.fields')
+                projectKeys=project_name, expand="projects.issuetypes.fields"
+            )
+
         return cached(self.project)
 
     def create_issue(self, form_data, request):
         issue_types = self.get_issue_types(request)
         payload = {
-            'project': {'key': form_data['project']},
-            'summary': form_data['title'],
-            'description': form_data['content'],
-            'issuetype': {'id': form_data['issue_type']},
-            "priority": {'id': form_data['priority']},
-            "assignee": {'name': form_data['responsible']},
+            "project": {"key": form_data["project"]},
+            "summary": form_data["title"],
+            "description": form_data["content"],
+            "issuetype": {"id": form_data["issue_type"]},
+            "priority": {"id": form_data["priority"]},
+            "assignee": {"name": form_data["responsible"]},
         }
         for issue_type in issue_types:
-            if issue_type['id'] == form_data['issue_type']:
-                for field in issue_type['fields']:
+            if issue_type["id"] == form_data["issue_type"]:
+                for field in issue_type["fields"]:
                     # set some defaults for other required fields
-                    if field == 'reporter':
-                        payload["reporter"] = {'id': self.user_name}
-                    if field['required'] and field['id'] not in payload:
-                        if field['type'] == 'array':
-                            payload[field['id']] = [field['values'][0], ]
-                        elif field['type'] == 'string':
-                            payload[field['id']] = ''
+                    if field == "reporter":
+                        payload["reporter"] = {"id": self.user_name}
+                    if field["required"] and field["id"] not in payload:
+                        if field["type"] == "array":
+                            payload[field["id"]] = [field["values"][0]]
+                        elif field["type"] == "string":
+                            payload[field["id"]] = ""
         new_issue = self.client.create_issue(fields=payload)
-        web_url = self.host_name + '/browse/' + new_issue.key
+        web_url = self.host_name + "/browse/" + new_issue.key
         to_return = {
-            'id': new_issue.id,
-            'resource_url': new_issue.self,
-            'web_url': web_url
+            "id": new_issue.id,
+            "resource_url": new_issue.self,
+            "web_url": web_url,
         }
         return to_return

backend/src/appenlight/models/integrations/slack.py

@@ -1,74 +1,65 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 
 import requests
 
-from appenlight.models.integrations import (IntegrationBase,
-                                            IntegrationException)
+from appenlight.models.integrations import IntegrationBase, IntegrationException
 from appenlight.lib.ext_json import json
 
 _ = str
 
 log = logging.getLogger(__name__)
 
 
 class NotFoundException(Exception):
     pass
 
 
 class SlackIntegration(IntegrationBase):
-    __mapper_args__ = {
-        'polymorphic_identity': 'slack'
-    }
+    __mapper_args__ = {"polymorphic_identity": "slack"}
     front_visible = False
     as_alert_channel = True
     supports_report_alerting = True
     action_notification = True
-    integration_action = 'Message via Slack'
+    integration_action = "Message via Slack"
 
     @classmethod
     def create_client(cls, api_token):
         client = SlackClient(api_token)
         return client
 
 
 class SlackClient(object):
     def __init__(self, api_url):
         self.api_url = api_url
 
     def make_request(self, data=None):
-        headers = {
-            'User-Agent': 'appenlight-slack',
-            'Content-Type': 'application/json'
-        }
+        headers = {"User-Agent": "appenlight-slack", "Content-Type": "application/json"}
         try:
-            resp = getattr(requests, 'post')(self.api_url,
-                                             data=json.dumps(data),
-                                             headers=headers,
-                                             timeout=3)
+            resp = getattr(requests, "post")(
+                self.api_url, data=json.dumps(data), headers=headers, timeout=3
+            )
         except Exception as e:
-            raise IntegrationException(
-                _('Error communicating with Slack: %s') % (e,))
+            raise IntegrationException(_("Error communicating with Slack: %s") % (e,))
         if resp.status_code != requests.codes.ok:
-            msg = 'Error communicating with Slack - status code: %s'
+            msg = "Error communicating with Slack - status code: %s"
             raise IntegrationException(msg % resp.status_code)
         return resp
 
     def send(self, payload):
-        return self.make_request('/rooms/message', method='post',
-                                 data=payload).json()
+        return self.make_request("/rooms/message", method="post", data=payload).json()

backend/src/appenlight/models/integrations/webhooks.py

@@ -1,138 +1,145 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 
 import requests
 
-from appenlight.models.integrations import (IntegrationBase,
-                                            IntegrationException)
+from appenlight.models.integrations import IntegrationBase, IntegrationException
 from appenlight.models.alert_channel import AlertChannel
 from appenlight.lib.ext_json import json
 
 _ = str
 
 log = logging.getLogger(__name__)
 
 
 class NotFoundException(Exception):
     pass
 
 
 class WebhooksIntegration(IntegrationBase):
-    __mapper_args__ = {
-        'polymorphic_identity': 'webhooks'
-    }
+    __mapper_args__ = {"polymorphic_identity": "webhooks"}
     front_visible = False
     as_alert_channel = True
     supports_report_alerting = True
     action_notification = True
-    integration_action = 'Message via Webhooks'
+    integration_action = "Message via Webhooks"
 
     @classmethod
     def create_client(cls, url):
         client = WebhooksClient(url)
         return client
 
 
 class WebhooksClient(object):
     def __init__(self, url):
         self.api_url = url
 
-    def make_request(self, url, method='get', data=None):
+    def make_request(self, url, method="get", data=None):
         headers = {
-            'Content-Type': 'application/json',
-            'User-Agent': 'appenlight-webhooks',
+            "Content-Type": "application/json",
+            "User-Agent": "appenlight-webhooks",
         }
         try:
             if data:
                 data = json.dumps(data)
-            resp = getattr(requests, method)(url, data=data, headers=headers,
-                                             timeout=3)
+            resp = getattr(requests, method)(url, data=data, headers=headers, timeout=3)
         except Exception as e:
             raise IntegrationException(
-                _('Error communicating with Webhooks: {}').format(e))
+                _("Error communicating with Webhooks: {}").format(e)
+            )
         if resp.status_code > 299:
             raise IntegrationException(
-                'Error communicating with Webhooks - status code: {}'.format(
-                    resp.status_code))
+                "Error communicating with Webhooks - status code: {}".format(
+                    resp.status_code
+                )
+            )
         return resp
 
     def send_to_hook(self, payload):
-        return self.make_request(self.api_url, method='post',
-                                 data=payload).json()
+        return self.make_request(self.api_url, method="post", data=payload).json()
 
 
 class WebhooksAlertChannel(AlertChannel):
-    __mapper_args__ = {
-        'polymorphic_identity': 'webhooks'
-    }
+    __mapper_args__ = {"polymorphic_identity": "webhooks"}
 
     def notify_reports(self, **kwargs):
         """
         Notify user of individual reports
 
         kwargs:
             application: application that the event applies for,
             user: user that should be notified
             request: request object
             since_when: reports are newer than this time value,
             reports: list of reports to render
 
         """
         template_vars = self.get_notification_basic_vars(kwargs)
         payload = []
-        include_keys = ('id', 'http_status', 'report_type', 'resource_name',
-                        'front_url', 'resource_id', 'error', 'url_path',
-                        'tags', 'duration')
-
-        for occurences, report in kwargs['reports']:
-            r_dict = report.last_report_ref.get_dict(kwargs['request'],
-                                                     include_keys=include_keys)
-            r_dict['group']['occurences'] = occurences
+        include_keys = (
+            "id",
+            "http_status",
+            "report_type",
+            "resource_name",
+            "front_url",
+            "resource_id",
+            "error",
+            "url_path",
+            "tags",
+            "duration",
+        )
+
+        for occurences, report in kwargs["reports"]:
+            r_dict = report.last_report_ref.get_dict(
+                kwargs["request"], include_keys=include_keys
+            )
+            r_dict["group"]["occurences"] = occurences
             payload.append(r_dict)
         client = WebhooksIntegration.create_client(
-            self.integration.config['reports_webhook'])
+            self.integration.config["reports_webhook"]
+        )
         client.send_to_hook(payload)
 
     def notify_alert(self, **kwargs):
         """
         Notify user of report or uptime threshold events based on events alert type
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         payload = {
-            'alert_action': kwargs['event'].unified_alert_action(),
-            'alert_name': kwargs['event'].unified_alert_name(),
-            'event_time': kwargs['event'].end_date or kwargs[
-                'event'].start_date,
-            'resource_name': None,
-            'resource_id': None
+            "alert_action": kwargs["event"].unified_alert_action(),
+            "alert_name": kwargs["event"].unified_alert_name(),
+            "event_time": kwargs["event"].end_date or kwargs["event"].start_date,
+            "resource_name": None,
+            "resource_id": None,
         }
-        if kwargs['event'].values and kwargs['event'].values.get('reports'):
-            payload['reports'] = kwargs['event'].values.get('reports', [])
-        if 'application' in kwargs:
-            payload['resource_name'] = kwargs['application'].resource_name
-            payload['resource_id'] = kwargs['application'].resource_id
+        if kwargs["event"].values and kwargs["event"].values.get("reports"):
+            payload["reports"] = kwargs["event"].values.get("reports", [])
+        if "application" in kwargs:
+            payload["resource_name"] = kwargs["application"].resource_name
+            payload["resource_id"] = kwargs["application"].resource_id
 
         client = WebhooksIntegration.create_client(
-            self.integration.config['alerts_webhook'])
+            self.integration.config["alerts_webhook"]
+        )
         client.send_to_hook(payload)

backend/src/appenlight/models/log.py

@@ -1,130 +1,132 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import sqlalchemy as sa
 import logging
 import hashlib
 
 from datetime import datetime
 from appenlight.models import Base
 from appenlight.lib.utils import convert_es_type
 from appenlight.lib.enums import LogLevel
 from sqlalchemy.dialects.postgresql import JSON
 from ziggurat_foundations.models.base import BaseModel
 
 log = logging.getLogger(__name__)
 
 
 class Log(Base, BaseModel):
-    __tablename__ = 'logs'
-    __table_args__ = {'implicit_returning': False}
+    __tablename__ = "logs"
+    __table_args__ = {"implicit_returning": False}
 
     log_id = sa.Column(sa.BigInteger(), nullable=False, primary_key=True)
-    resource_id = sa.Column(sa.Integer(),
-                            sa.ForeignKey('applications.resource_id',
-                                          onupdate='CASCADE',
-                                          ondelete='CASCADE'),
-                            nullable=False,
-                            index=True)
-    log_level = sa.Column(sa.Unicode, nullable=False, index=True,
-                          default='INFO')
-    message = sa.Column(sa.UnicodeText(), default='')
-    timestamp = sa.Column(sa.DateTime(), default=datetime.utcnow,
-                          server_default=sa.func.now())
+    resource_id = sa.Column(
+        sa.Integer(),
+        sa.ForeignKey(
+            "applications.resource_id", onupdate="CASCADE", ondelete="CASCADE"
+        ),
+        nullable=False,
+        index=True,
+    )
+    log_level = sa.Column(sa.Unicode, nullable=False, index=True, default="INFO")
+    message = sa.Column(sa.UnicodeText(), default="")
+    timestamp = sa.Column(
+        sa.DateTime(), default=datetime.utcnow, server_default=sa.func.now()
+    )
     request_id = sa.Column(sa.Unicode())
     namespace = sa.Column(sa.Unicode())
     primary_key = sa.Column(sa.Unicode())
 
     tags = sa.Column(JSON(), default={})
     permanent = sa.Column(sa.Boolean(), nullable=False, default=False)
 
     def __str__(self):
-        return self.__unicode__().encode('utf8')
+        return self.__unicode__().encode("utf8")
 
     def __unicode__(self):
-        return '<Log id:%s, lv:%s, ns:%s >' % (
-            self.log_id, self.log_level, self.namespace)
+        return "<Log id:%s, lv:%s, ns:%s >" % (
+            self.log_id,
+            self.log_level,
+            self.namespace,
+        )
 
     def set_data(self, data, resource):
-        level = data.get('log_level').upper()
+        level = data.get("log_level").upper()
         self.log_level = getattr(LogLevel, level, LogLevel.UNKNOWN)
-        self.message = data.get('message', '')
-        server_name = data.get('server', '').lower() or 'unknown'
-        self.tags = {
-            'server_name': server_name
-        }
-        if data.get('tags'):
-            for tag_tuple in data['tags']:
+        self.message = data.get("message", "")
+        server_name = data.get("server", "").lower() or "unknown"
+        self.tags = {"server_name": server_name}
+        if data.get("tags"):
+            for tag_tuple in data["tags"]:
                 self.tags[tag_tuple[0]] = tag_tuple[1]
-        self.timestamp = data['date']
-        r_id = data.get('request_id', '')
+        self.timestamp = data["date"]
+        r_id = data.get("request_id", "")
         if not r_id:
-            r_id = ''
-        self.request_id = r_id.replace('-', '')
+            r_id = ""
+        self.request_id = r_id.replace("-", "")
         self.resource_id = resource.resource_id
-        self.namespace = data.get('namespace') or ''
-        self.permanent = data.get('permanent')
-        self.primary_key = data.get('primary_key')
+        self.namespace = data.get("namespace") or ""
+        self.permanent = data.get("permanent")
+        self.primary_key = data.get("primary_key")
         if self.primary_key is not None:
-            self.tags['appenlight_primary_key'] = self.primary_key
+            self.tags["appenlight_primary_key"] = self.primary_key
 
     def get_dict(self):
         instance_dict = super(Log, self).get_dict()
-        instance_dict['log_level'] = LogLevel.key_from_value(self.log_level)
-        instance_dict['resource_name'] = self.application.resource_name
+        instance_dict["log_level"] = LogLevel.key_from_value(self.log_level)
+        instance_dict["resource_name"] = self.application.resource_name
         return instance_dict
 
     @property
     def delete_hash(self):
         if not self.primary_key:
             return None
 
-        to_hash = '{}_{}_{}'.format(self.resource_id, self.primary_key,
-                                    self.namespace)
-        return hashlib.sha1(to_hash.encode('utf8')).hexdigest()
+        to_hash = "{}_{}_{}".format(self.resource_id, self.primary_key, self.namespace)
+        return hashlib.sha1(to_hash.encode("utf8")).hexdigest()
 
     def es_doc(self):
         tags = {}
         tag_list = []
         for name, value in self.tags.items():
             # replace dot in indexed tag name
-            name = name.replace('.', '_')
+            name = name.replace(".", "_")
             tag_list.append(name)
             tags[name] = {
                 "values": convert_es_type(value),
-                "numeric_values": value if (
-                    isinstance(value, (int, float)) and
-                    not isinstance(value, bool)) else None
+                "numeric_values": value
+                if (isinstance(value, (int, float)) and not isinstance(value, bool))
+                else None,
             }
         return {
-            'pg_id': str(self.log_id),
-            'delete_hash': self.delete_hash,
-            'resource_id': self.resource_id,
-            'request_id': self.request_id,
-            'log_level': LogLevel.key_from_value(self.log_level),
-            'timestamp': self.timestamp,
-            'message': self.message if self.message else '',
-            'namespace': self.namespace if self.namespace else '',
-            'tags': tags,
-            'tag_list': tag_list
+            "pg_id": str(self.log_id),
+            "delete_hash": self.delete_hash,
+            "resource_id": self.resource_id,
+            "request_id": self.request_id,
+            "log_level": LogLevel.key_from_value(self.log_level),
+            "timestamp": self.timestamp,
+            "message": self.message if self.message else "",
+            "namespace": self.namespace if self.namespace else "",
+            "tags": tags,
+            "tag_list": tag_list,
         }
 
     @property
     def partition_id(self):
         if self.permanent:
-            return 'rcae_l_%s' % self.timestamp.strftime('%Y_%m')
+            return "rcae_l_%s" % self.timestamp.strftime("%Y_%m")
         else:
-            return 'rcae_l_%s' % self.timestamp.strftime('%Y_%m_%d')
+            return "rcae_l_%s" % self.timestamp.strftime("%Y_%m_%d")

backend/src/appenlight/models/metric.py

@@ -1,64 +1,68 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from datetime import datetime
 
 import sqlalchemy as sa
 from sqlalchemy.dialects.postgresql import JSON
 
 from ziggurat_foundations.models.base import BaseModel
 from appenlight.lib.utils import convert_es_type
 from appenlight.models import Base
 
 
 class Metric(Base, BaseModel):
-    __tablename__ = 'metrics'
-    __table_args__ = {'implicit_returning': False}
+    __tablename__ = "metrics"
+    __table_args__ = {"implicit_returning": False}
 
     pkey = sa.Column(sa.BigInteger(), primary_key=True)
-    resource_id = sa.Column(sa.Integer(),
-                            sa.ForeignKey('applications.resource_id'),
-                            nullable=False, primary_key=True)
-    timestamp = sa.Column(sa.DateTime(), default=datetime.utcnow,
-                          server_default=sa.func.now())
+    resource_id = sa.Column(
+        sa.Integer(),
+        sa.ForeignKey("applications.resource_id"),
+        nullable=False,
+        primary_key=True,
+    )
+    timestamp = sa.Column(
+        sa.DateTime(), default=datetime.utcnow, server_default=sa.func.now()
+    )
     tags = sa.Column(JSON(), default={})
     namespace = sa.Column(sa.Unicode(255))
 
     @property
     def partition_id(self):
-        return 'rcae_m_%s' % self.timestamp.strftime('%Y_%m_%d')
+        return "rcae_m_%s" % self.timestamp.strftime("%Y_%m_%d")
 
     def es_doc(self):
         tags = {}
         tag_list = []
         for name, value in self.tags.items():
             # replace dot in indexed tag name
-            name = name.replace('.', '_')
+            name = name.replace(".", "_")
             tag_list.append(name)
             tags[name] = {
                 "values": convert_es_type(value),
-                "numeric_values": value if (
-                    isinstance(value, (int, float)) and
-                    not isinstance(value, bool)) else None
+                "numeric_values": value
+                if (isinstance(value, (int, float)) and not isinstance(value, bool))
+                else None,
             }
 
         return {
-            'resource_id': self.resource_id,
-            'timestamp': self.timestamp,
-            'namespace': self.namespace,
-            'tags': tags,
-            'tag_list': tag_list
+            "resource_id": self.resource_id,
+            "timestamp": self.timestamp,
+            "namespace": self.namespace,
+            "tags": tags,
+            "tag_list": tag_list,
         }

backend/src/appenlight/models/plugin_config.py

@@ -1,40 +1,40 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import sqlalchemy as sa
 from ziggurat_foundations.models.base import BaseModel
 from sqlalchemy.dialects.postgresql import JSON
 
 from . import Base
 
 
 class PluginConfig(Base, BaseModel):
-    __tablename__ = 'plugin_configs'
+    __tablename__ = "plugin_configs"
 
     id = sa.Column(sa.Integer, primary_key=True)
     plugin_name = sa.Column(sa.Unicode)
     section = sa.Column(sa.Unicode)
     config = sa.Column(JSON, nullable=False)
-    resource_id = sa.Column(sa.Integer(),
-                            sa.ForeignKey('resources.resource_id',
-                                          onupdate='cascade',
-                                          ondelete='cascade'))
-    owner_id = sa.Column(sa.Integer(),
-                         sa.ForeignKey('users.id', onupdate='cascade',
-                                       ondelete='cascade'))
+    resource_id = sa.Column(
+        sa.Integer(),
+        sa.ForeignKey("resources.resource_id", onupdate="cascade", ondelete="cascade"),
+    )
+    owner_id = sa.Column(
+        sa.Integer(), sa.ForeignKey("users.id", onupdate="cascade", ondelete="cascade")
+    )
 
     def __json__(self, request):
         return self.get_dict()