black: reformat source

backend/setup.py

@@ -1,81 +1,89 b''
 import os
 import re
 
 from setuptools import setup, find_packages
 
 here = os.path.abspath(os.path.dirname(__file__))
-README = open(os.path.join(here, 'README.rst')).read()
-CHANGES = open(os.path.join(here, 'CHANGELOG.rst')).read()
+README = open(os.path.join(here, "README.rst")).read()
+CHANGES = open(os.path.join(here, "CHANGELOG.rst")).read()
 
-REQUIREMENTS = open(os.path.join(here, 'requirements.txt')).readlines()
+REQUIREMENTS = open(os.path.join(here, "requirements.txt")).readlines()
 
-compiled = re.compile('([^=><]*).*')
+compiled = re.compile("([^=><]*).*")
 
 
 def parse_req(req):
     return compiled.search(req).group(1).strip()
 
 
 requires = [_f for _f in map(parse_req, REQUIREMENTS) if _f]
 
 
 def _get_meta_var(name, data, callback_handler=None):
     import re
-    matches = re.compile(r'(?:%s)\s*=\s*(.*)' % name).search(data)
+
+    matches = re.compile(r"(?:%s)\s*=\s*(.*)" % name).search(data)
     if matches:
         if not callable(callback_handler):
             callback_handler = lambda v: v
 
         return callback_handler(eval(matches.groups()[0]))
 
 
-with open(os.path.join(here, 'src', 'appenlight', '__init__.py'), 'r') as _meta:
+with open(os.path.join(here, "src", "appenlight", "__init__.py"), "r") as _meta:
     _metadata = _meta.read()
 
-with open(os.path.join(here, 'VERSION'), 'r') as _meta_version:
+with open(os.path.join(here, "VERSION"), "r") as _meta_version:
     __version__ = _meta_version.read().strip()
 
-__license__ = _get_meta_var('__license__', _metadata)
-__author__ = _get_meta_var('__author__', _metadata)
-__url__ = _get_meta_var('__url__', _metadata)
-
-found_packages = find_packages('src')
-found_packages.append('appenlight.migrations.versions')
-setup(name='appenlight',
-      description='appenlight',
-      long_description=README + '\n\n' + CHANGES,
-      classifiers=[
-          "Programming Language :: Python",
-          "Framework :: Pylons",
-          "Topic :: Internet :: WWW/HTTP",
-          "Topic :: Internet :: WWW/HTTP :: WSGI :: Application",
-      ],
-      version=__version__,
-      license=__license__,
-      author=__author__,
-      url=__url__,
-      keywords='web wsgi bfg pylons pyramid',
-      package_dir={'': 'src'},
-      packages=found_packages,
-      include_package_data=True,
-      zip_safe=False,
-      test_suite='appenlight',
-      install_requires=requires,
-      extras_require={
-          "dev": ["coverage", "pytest", "pyramid", "tox", "mock", "pytest-mock", "webtest"],
-          "lint": ["black"],
-      },
-      entry_points={
-          'paste.app_factory': [
-              'main = appenlight:main'
-          ],
-          'console_scripts': [
-              'appenlight-cleanup = appenlight.scripts.cleanup:main',
-              'appenlight-initializedb = appenlight.scripts.initialize_db:main',
-              'appenlight-migratedb = appenlight.scripts.migratedb:main',
-              'appenlight-reindex-elasticsearch = appenlight.scripts.reindex_elasticsearch:main',
-              'appenlight-static = appenlight.scripts.static:main',
-              'appenlight-make-config = appenlight.scripts.make_config:main',
-          ]
-      }
-      )
+__license__ = _get_meta_var("__license__", _metadata)
+__author__ = _get_meta_var("__author__", _metadata)
+__url__ = _get_meta_var("__url__", _metadata)
+
+found_packages = find_packages("src")
+found_packages.append("appenlight.migrations.versions")
+setup(
+    name="appenlight",
+    description="appenlight",
+    long_description=README + "\n\n" + CHANGES,
+    classifiers=[
+        "Programming Language :: Python",
+        "Framework :: Pylons",
+        "Topic :: Internet :: WWW/HTTP",
+        "Topic :: Internet :: WWW/HTTP :: WSGI :: Application",
+    ],
+    version=__version__,
+    license=__license__,
+    author=__author__,
+    url=__url__,
+    keywords="web wsgi bfg pylons pyramid",
+    package_dir={"": "src"},
+    packages=found_packages,
+    include_package_data=True,
+    zip_safe=False,
+    test_suite="appenlight",
+    install_requires=requires,
+    extras_require={
+        "dev": [
+            "coverage",
+            "pytest",
+            "pyramid",
+            "tox",
+            "mock",
+            "pytest-mock",
+            "webtest",
+        ],
+        "lint": ["black"],
+    },
+    entry_points={
+        "paste.app_factory": ["main = appenlight:main"],
+        "console_scripts": [
+            "appenlight-cleanup = appenlight.scripts.cleanup:main",
+            "appenlight-initializedb = appenlight.scripts.initialize_db:main",
+            "appenlight-migratedb = appenlight.scripts.migratedb:main",
+            "appenlight-reindex-elasticsearch = appenlight.scripts.reindex_elasticsearch:main",
+            "appenlight-static = appenlight.scripts.static:main",
+            "appenlight-make-config = appenlight.scripts.make_config:main",
+        ],
+    },
+)

backend/src/appenlight/__init__.py

@@ -1,225 +1,254 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import datetime
 import logging
 from elasticsearch import Elasticsearch
 import redis
 import os
 import pkg_resources
 from pkg_resources import iter_entry_points
 
 import appenlight.lib.jinja2_filters as jinja2_filters
 import appenlight.lib.encryption as encryption
 
 from pyramid.config import PHASE3_CONFIG
 from pyramid.authentication import AuthTktAuthenticationPolicy
 from pyramid.authorization import ACLAuthorizationPolicy
 from pyramid_mailer.interfaces import IMailer
 from pyramid.renderers import JSON
 from pyramid_redis_sessions import session_factory_from_settings
 from pyramid.settings import asbool, aslist
 from pyramid.security import AllPermissionsList
 from pyramid_authstack import AuthenticationStackPolicy
 from redlock import Redlock
 from sqlalchemy import engine_from_config
 
 from appenlight.celery import configure_celery
-from appenlight.lib.configurator import (CythonCompatConfigurator,
-                                         register_appenlight_plugin)
+from appenlight.lib.configurator import (
+    CythonCompatConfigurator,
+    register_appenlight_plugin,
+)
 from appenlight.lib import cache_regions
 from appenlight.lib.ext_json import json
 from appenlight.security import groupfinder, AuthTokenAuthenticationPolicy
 
-__license__ = 'Apache 2.0'
-__author__ = 'RhodeCode GmbH'
-__url__ = 'http://rhodecode.com'
+__license__ = "Apache 2.0"
+__author__ = "RhodeCode GmbH"
+__url__ = "http://rhodecode.com"
 __version__ = pkg_resources.get_distribution("appenlight").parsed_version
 
 json_renderer = JSON(serializer=json.dumps, indent=4)
 
 log = logging.getLogger(__name__)
 
 
 def datetime_adapter(obj, request):
     return obj.isoformat()
 
 
 def all_permissions_adapter(obj, request):
-    return '__all_permissions__'
+    return "__all_permissions__"
 
 
 json_renderer.add_adapter(datetime.datetime, datetime_adapter)
 json_renderer.add_adapter(AllPermissionsList, all_permissions_adapter)
 
 
 def main(global_config, **settings):
     """ This function returns a Pyramid WSGI application.
     """
     auth_tkt_policy = AuthTktAuthenticationPolicy(
-        settings['authtkt.secret'],
-        hashalg='sha512',
+        settings["authtkt.secret"],
+        hashalg="sha512",
         callback=groupfinder,
         max_age=2592000,
-        secure=asbool(settings.get('authtkt.secure', 'false')))
-    auth_token_policy = AuthTokenAuthenticationPolicy(
-        callback=groupfinder
+        secure=asbool(settings.get("authtkt.secure", "false")),
     )
+    auth_token_policy = AuthTokenAuthenticationPolicy(callback=groupfinder)
     authorization_policy = ACLAuthorizationPolicy()
     authentication_policy = AuthenticationStackPolicy()
-    authentication_policy.add_policy('auth_tkt', auth_tkt_policy)
-    authentication_policy.add_policy('auth_token', auth_token_policy)
+    authentication_policy.add_policy("auth_tkt", auth_tkt_policy)
+    authentication_policy.add_policy("auth_token", auth_token_policy)
     # set crypto key
-    encryption.ENCRYPTION_SECRET = settings.get('encryption_secret')
+    encryption.ENCRYPTION_SECRET = settings.get("encryption_secret")
     # import this later so encyption key can be monkeypatched
     from appenlight.models import DBSession, register_datastores
 
     # registration
-    settings['appenlight.disable_registration'] = asbool(
-        settings.get('appenlight.disable_registration'))
+    settings["appenlight.disable_registration"] = asbool(
+        settings.get("appenlight.disable_registration")
+    )
 
     # update config with cometd info
-    settings['cometd_servers'] = {'server': settings['cometd.server'],
-                                  'secret': settings['cometd.secret']}
+    settings["cometd_servers"] = {
+        "server": settings["cometd.server"],
+        "secret": settings["cometd.secret"],
+    }
 
     # Create the Pyramid Configurator.
-    settings['_mail_url'] = settings['mailing.app_url']
+    settings["_mail_url"] = settings["mailing.app_url"]
     config = CythonCompatConfigurator(
         settings=settings,
         authentication_policy=authentication_policy,
         authorization_policy=authorization_policy,
-        root_factory='appenlight.security.RootFactory',
-        default_permission='view')
+        root_factory="appenlight.security.RootFactory",
+        default_permission="view",
+    )
     # custom registry variables
 
     # resource type information
-    config.registry.resource_types = ['resource', 'application']
+    config.registry.resource_types = ["resource", "application"]
     # plugin information
     config.registry.appenlight_plugins = {}
 
-    config.set_default_csrf_options(require_csrf=True, header='X-XSRF-TOKEN')
-    config.add_view_deriver('appenlight.predicates.csrf_view',
-                            name='csrf_view')
+    config.set_default_csrf_options(require_csrf=True, header="X-XSRF-TOKEN")
+    config.add_view_deriver("appenlight.predicates.csrf_view", name="csrf_view")
 
     # later, when config is available
-    dogpile_config = {'url': settings['redis.url'],
-                      "redis_expiration_time": 86400,
-                      "redis_distributed_lock": True}
+    dogpile_config = {
+        "url": settings["redis.url"],
+        "redis_expiration_time": 86400,
+        "redis_distributed_lock": True,
+    }
     cache_regions.regions = cache_regions.CacheRegions(dogpile_config)
     config.registry.cache_regions = cache_regions.regions
-    engine = engine_from_config(settings, 'sqlalchemy.',
-                                json_serializer=json.dumps)
+    engine = engine_from_config(settings, "sqlalchemy.", json_serializer=json.dumps)
     DBSession.configure(bind=engine)
 
     # json rederer that serializes datetime
-    config.add_renderer('json', json_renderer)
-    config.add_request_method('appenlight.lib.request.es_conn', 'es_conn', property=True)
-    config.add_request_method('appenlight.lib.request.get_user', 'user',
-                                reify=True, property=True)
-    config.add_request_method('appenlight.lib.request.get_csrf_token',
-                                'csrf_token', reify=True, property=True)
-    config.add_request_method('appenlight.lib.request.safe_json_body',
-                                'safe_json_body', reify=True, property=True)
-    config.add_request_method('appenlight.lib.request.unsafe_json_body',
-                                'unsafe_json_body', reify=True, property=True)
-    config.add_request_method('appenlight.lib.request.add_flash_to_headers',
-                              'add_flash_to_headers')
-    config.add_request_method('appenlight.lib.request.get_authomatic',
-                              'authomatic', reify=True)
-
-    config.include('pyramid_redis_sessions')
-    config.include('pyramid_tm')
-    config.include('pyramid_jinja2')
-    config.include('pyramid_mailer')
-    config.include('appenlight_client.ext.pyramid_tween')
-    config.include('ziggurat_foundations.ext.pyramid.sign_in')
-    es_server_list = aslist(settings['elasticsearch.nodes'])
-    redis_url = settings['redis.url']
-    log.warning('Elasticsearch server list: {}'.format(es_server_list))
-    log.warning('Redis server: {}'.format(redis_url))
+    config.add_renderer("json", json_renderer)
+    config.add_request_method(
+        "appenlight.lib.request.es_conn", "es_conn", property=True
+    )
+    config.add_request_method(
+        "appenlight.lib.request.get_user", "user", reify=True, property=True
+    )
+    config.add_request_method(
+        "appenlight.lib.request.get_csrf_token", "csrf_token", reify=True, property=True
+    )
+    config.add_request_method(
+        "appenlight.lib.request.safe_json_body",
+        "safe_json_body",
+        reify=True,
+        property=True,
+    )
+    config.add_request_method(
+        "appenlight.lib.request.unsafe_json_body",
+        "unsafe_json_body",
+        reify=True,
+        property=True,
+    )
+    config.add_request_method(
+        "appenlight.lib.request.add_flash_to_headers", "add_flash_to_headers"
+    )
+    config.add_request_method(
+        "appenlight.lib.request.get_authomatic", "authomatic", reify=True
+    )
+
+    config.include("pyramid_redis_sessions")
+    config.include("pyramid_tm")
+    config.include("pyramid_jinja2")
+    config.include("pyramid_mailer")
+    config.include("appenlight_client.ext.pyramid_tween")
+    config.include("ziggurat_foundations.ext.pyramid.sign_in")
+    es_server_list = aslist(settings["elasticsearch.nodes"])
+    redis_url = settings["redis.url"]
+    log.warning("Elasticsearch server list: {}".format(es_server_list))
+    log.warning("Redis server: {}".format(redis_url))
     config.registry.es_conn = Elasticsearch(es_server_list)
     config.registry.redis_conn = redis.StrictRedis.from_url(redis_url)
 
-    config.registry.redis_lockmgr = Redlock([settings['redis.redlock.url'], ],
-                                            retry_count=0, retry_delay=0)
+    config.registry.redis_lockmgr = Redlock(
+        [settings["redis.redlock.url"]], retry_count=0, retry_delay=0
+    )
     # mailer bw compat
     config.registry.mailer = config.registry.getUtility(IMailer)
 
     # Configure sessions
     session_factory = session_factory_from_settings(settings)
     config.set_session_factory(session_factory)
 
     # Configure renderers and event subscribers
-    config.add_jinja2_extension('jinja2.ext.loopcontrols')
-    config.add_jinja2_search_path('appenlight:templates')
+    config.add_jinja2_extension("jinja2.ext.loopcontrols")
+    config.add_jinja2_search_path("appenlight:templates")
     # event subscribers
-    config.add_subscriber("appenlight.subscribers.application_created",
-                          "pyramid.events.ApplicationCreated")
-    config.add_subscriber("appenlight.subscribers.add_renderer_globals",
-                          "pyramid.events.BeforeRender")
-    config.add_subscriber('appenlight.subscribers.new_request',
-                          'pyramid.events.NewRequest')
-    config.add_view_predicate('context_type_class',
-                              'appenlight.predicates.contextTypeClass')
-
-    register_datastores(es_conn=config.registry.es_conn,
-                        redis_conn=config.registry.redis_conn,
-                        redis_lockmgr=config.registry.redis_lockmgr)
+    config.add_subscriber(
+        "appenlight.subscribers.application_created",
+        "pyramid.events.ApplicationCreated",
+    )
+    config.add_subscriber(
+        "appenlight.subscribers.add_renderer_globals", "pyramid.events.BeforeRender"
+    )
+    config.add_subscriber(
+        "appenlight.subscribers.new_request", "pyramid.events.NewRequest"
+    )
+    config.add_view_predicate(
+        "context_type_class", "appenlight.predicates.contextTypeClass"
+    )
+
+    register_datastores(
+        es_conn=config.registry.es_conn,
+        redis_conn=config.registry.redis_conn,
+        redis_lockmgr=config.registry.redis_lockmgr,
+    )
 
     # base stuff and scan
 
     # need to ensure webassets exists otherwise config.override_asset()
     # throws exception
-    if not os.path.exists(settings['webassets.dir']):
-        os.mkdir(settings['webassets.dir'])
-    config.add_static_view(path='appenlight:webassets',
-                           name='static', cache_max_age=3600)
-    config.override_asset(to_override='appenlight:webassets/',
-                          override_with=settings['webassets.dir'])
-
-    config.include('appenlight.views')
-    config.include('appenlight.views.admin')
-    config.scan(ignore=['appenlight.migrations', 'appenlight.scripts',
-                        'appenlight.tests'])
-
-    config.add_directive('register_appenlight_plugin',
-                         register_appenlight_plugin)
-
-    for entry_point in iter_entry_points(group='appenlight.plugins'):
+    if not os.path.exists(settings["webassets.dir"]):
+        os.mkdir(settings["webassets.dir"])
+    config.add_static_view(
+        path="appenlight:webassets", name="static", cache_max_age=3600
+    )
+    config.override_asset(
+        to_override="appenlight:webassets/", override_with=settings["webassets.dir"]
+    )
+
+    config.include("appenlight.views")
+    config.include("appenlight.views.admin")
+    config.scan(
+        ignore=["appenlight.migrations", "appenlight.scripts", "appenlight.tests"]
+    )
+
+    config.add_directive("register_appenlight_plugin", register_appenlight_plugin)
+
+    for entry_point in iter_entry_points(group="appenlight.plugins"):
         plugin = entry_point.load()
         plugin.includeme(config)
 
     # include other appenlight plugins explictly if needed
-    includes = aslist(settings.get('appenlight.includes', []))
+    includes = aslist(settings.get("appenlight.includes", []))
     for inc in includes:
         config.include(inc)
 
     # run this after everything registers in configurator
 
     def pre_commit():
         jinja_env = config.get_jinja2_environment()
-        jinja_env.filters['tojson'] = json.dumps
-        jinja_env.filters['toJSONUnsafe'] = jinja2_filters.toJSONUnsafe
+        jinja_env.filters["tojson"] = json.dumps
+        jinja_env.filters["toJSONUnsafe"] = jinja2_filters.toJSONUnsafe
 
     config.action(None, pre_commit, order=PHASE3_CONFIG + 999)
 
     def wrap_config_celery():
         configure_celery(config.registry)
 
     config.action(None, wrap_config_celery, order=PHASE3_CONFIG + 999)
 
     app = config.make_wsgi_app()
     return app

backend/src/appenlight/celery/__init__.py

@@ -1,171 +1,181 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 
 from datetime import timedelta
 from celery import Celery
 from celery.bin import Option
 from celery.schedules import crontab
 from celery.signals import worker_init, task_revoked, user_preload_options
 from celery.signals import task_prerun, task_retry, task_failure, task_success
 from kombu.serialization import register
 from pyramid.paster import bootstrap
 from pyramid.request import Request
 from pyramid.scripting import prepare
 from pyramid.settings import asbool
 from pyramid.threadlocal import get_current_request
 
 from appenlight.celery.encoders import json_dumps, json_loads
 from appenlight_client.ext.celery import register_signals
 
 log = logging.getLogger(__name__)
 
-register('date_json', json_dumps, json_loads,
-         content_type='application/x-date_json',
-         content_encoding='utf-8')
+register(
+    "date_json",
+    json_dumps,
+    json_loads,
+    content_type="application/x-date_json",
+    content_encoding="utf-8",
+)
 
 celery = Celery()
 
-celery.user_options['preload'].add(
-    Option('--ini', dest='ini', default=None,
-           help='Specifies pyramid configuration file location.')
+celery.user_options["preload"].add(
+    Option(
+        "--ini",
+        dest="ini",
+        default=None,
+        help="Specifies pyramid configuration file location.",
+    )
 )
 
 
 @user_preload_options.connect
 def on_preload_parsed(options, **kwargs):
     """
     This actually configures celery from pyramid config file
     """
-    celery.conf['INI_PYRAMID'] = options['ini']
+    celery.conf["INI_PYRAMID"] = options["ini"]
     import appenlight_client.client as e_client
-    ini_location = options['ini']
+
+    ini_location = options["ini"]
     if not ini_location:
-        raise Exception('You need to pass pyramid ini location using '
-                        '--ini=filename.ini argument to the worker')
+        raise Exception(
+            "You need to pass pyramid ini location using "
+            "--ini=filename.ini argument to the worker"
+        )
     env = bootstrap(ini_location[0])
-    api_key = env['request'].registry.settings['appenlight.api_key']
-    tr_config = env['request'].registry.settings.get(
-        'appenlight.transport_config')
-    CONFIG = e_client.get_config({'appenlight.api_key': api_key})
+    api_key = env["request"].registry.settings["appenlight.api_key"]
+    tr_config = env["request"].registry.settings.get("appenlight.transport_config")
+    CONFIG = e_client.get_config({"appenlight.api_key": api_key})
     if tr_config:
-        CONFIG['appenlight.transport_config'] = tr_config
+        CONFIG["appenlight.transport_config"] = tr_config
     APPENLIGHT_CLIENT = e_client.Client(CONFIG)
     # log.addHandler(APPENLIGHT_CLIENT.log_handler)
     register_signals(APPENLIGHT_CLIENT)
     celery.pyramid = env
 
 
 celery_config = {
-    'CELERY_IMPORTS': ["appenlight.celery.tasks", ],
-    'CELERYD_TASK_TIME_LIMIT': 60,
-    'CELERYD_MAX_TASKS_PER_CHILD': 1000,
-    'CELERY_IGNORE_RESULT': True,
-    'CELERY_ACCEPT_CONTENT': ['date_json'],
-    'CELERY_TASK_SERIALIZER': 'date_json',
-    'CELERY_RESULT_SERIALIZER': 'date_json',
-    'BROKER_URL': None,
-    'CELERYD_CONCURRENCY': None,
-    'CELERY_TIMEZONE': None,
-    'CELERYBEAT_SCHEDULE': {
-        'alerting_reports': {
-            'task': 'appenlight.celery.tasks.alerting_reports',
-            'schedule': timedelta(seconds=60)
+    "CELERY_IMPORTS": ["appenlight.celery.tasks"],
+    "CELERYD_TASK_TIME_LIMIT": 60,
+    "CELERYD_MAX_TASKS_PER_CHILD": 1000,
+    "CELERY_IGNORE_RESULT": True,
+    "CELERY_ACCEPT_CONTENT": ["date_json"],
+    "CELERY_TASK_SERIALIZER": "date_json",
+    "CELERY_RESULT_SERIALIZER": "date_json",
+    "BROKER_URL": None,
+    "CELERYD_CONCURRENCY": None,
+    "CELERY_TIMEZONE": None,
+    "CELERYBEAT_SCHEDULE": {
+        "alerting_reports": {
+            "task": "appenlight.celery.tasks.alerting_reports",
+            "schedule": timedelta(seconds=60),
         },
-        'close_alerts': {
-            'task': 'appenlight.celery.tasks.close_alerts',
-            'schedule': timedelta(seconds=60)
-        }
-    }
+        "close_alerts": {
+            "task": "appenlight.celery.tasks.close_alerts",
+            "schedule": timedelta(seconds=60),
+        },
+    },
 }
 celery.config_from_object(celery_config)
 
 
 def configure_celery(pyramid_registry):
     settings = pyramid_registry.settings
-    celery_config['BROKER_URL'] = settings['celery.broker_url']
-    celery_config['CELERYD_CONCURRENCY'] = settings['celery.concurrency']
-    celery_config['CELERY_TIMEZONE'] = settings['celery.timezone']
+    celery_config["BROKER_URL"] = settings["celery.broker_url"]
+    celery_config["CELERYD_CONCURRENCY"] = settings["celery.concurrency"]
+    celery_config["CELERY_TIMEZONE"] = settings["celery.timezone"]
 
-    notifications_seconds = int(settings.get('tasks.notifications_reports.interval', 60))
+    notifications_seconds = int(
+        settings.get("tasks.notifications_reports.interval", 60)
+    )
 
-    celery_config['CELERYBEAT_SCHEDULE']['notifications'] = {
-        'task': 'appenlight.celery.tasks.notifications_reports',
-        'schedule': timedelta(seconds=notifications_seconds)
+    celery_config["CELERYBEAT_SCHEDULE"]["notifications"] = {
+        "task": "appenlight.celery.tasks.notifications_reports",
+        "schedule": timedelta(seconds=notifications_seconds),
     }
 
-    celery_config['CELERYBEAT_SCHEDULE']['daily_digest'] = {
-        'task': 'appenlight.celery.tasks.daily_digest',
-        'schedule': crontab(minute=1, hour='4,12,20')
+    celery_config["CELERYBEAT_SCHEDULE"]["daily_digest"] = {
+        "task": "appenlight.celery.tasks.daily_digest",
+        "schedule": crontab(minute=1, hour="4,12,20"),
     }
 
-    if asbool(settings.get('celery.always_eager')):
-        celery_config['CELERY_ALWAYS_EAGER'] = True
-        celery_config['CELERY_EAGER_PROPAGATES_EXCEPTIONS'] = True
+    if asbool(settings.get("celery.always_eager")):
+        celery_config["CELERY_ALWAYS_EAGER"] = True
+        celery_config["CELERY_EAGER_PROPAGATES_EXCEPTIONS"] = True
 
     for plugin in pyramid_registry.appenlight_plugins.values():
-        if plugin.get('celery_tasks'):
-            celery_config['CELERY_IMPORTS'].extend(plugin['celery_tasks'])
-        if plugin.get('celery_beats'):
-            for name, config in plugin['celery_beats']:
-                celery_config['CELERYBEAT_SCHEDULE'][name] = config
+        if plugin.get("celery_tasks"):
+            celery_config["CELERY_IMPORTS"].extend(plugin["celery_tasks"])
+        if plugin.get("celery_beats"):
+            for name, config in plugin["celery_beats"]:
+                celery_config["CELERYBEAT_SCHEDULE"][name] = config
     celery.config_from_object(celery_config)
 
 
 @task_prerun.connect
 def task_prerun_signal(task_id, task, args, kwargs, **kwaargs):
-    if hasattr(celery, 'pyramid'):
+    if hasattr(celery, "pyramid"):
         env = celery.pyramid
-        env = prepare(registry=env['request'].registry)
-        proper_base_url = env['request'].registry.settings['mailing.app_url']
-        tmp_req = Request.blank('/', base_url=proper_base_url)
+        env = prepare(registry=env["request"].registry)
+        proper_base_url = env["request"].registry.settings["mailing.app_url"]
+        tmp_req = Request.blank("/", base_url=proper_base_url)
         # ensure tasks generate url for right domain from config
-        env['request'].environ['HTTP_HOST'] = tmp_req.environ['HTTP_HOST']
-        env['request'].environ['SERVER_PORT'] = tmp_req.environ['SERVER_PORT']
-        env['request'].environ['SERVER_NAME'] = tmp_req.environ['SERVER_NAME']
-        env['request'].environ['wsgi.url_scheme'] = \
-            tmp_req.environ['wsgi.url_scheme']
+        env["request"].environ["HTTP_HOST"] = tmp_req.environ["HTTP_HOST"]
+        env["request"].environ["SERVER_PORT"] = tmp_req.environ["SERVER_PORT"]
+        env["request"].environ["SERVER_NAME"] = tmp_req.environ["SERVER_NAME"]
+        env["request"].environ["wsgi.url_scheme"] = tmp_req.environ["wsgi.url_scheme"]
     get_current_request().tm.begin()
 
 
 @task_success.connect
 def task_success_signal(result, **kwargs):
     get_current_request().tm.commit()
-    if hasattr(celery, 'pyramid'):
+    if hasattr(celery, "pyramid"):
         celery.pyramid["closer"]()
 
 
 @task_retry.connect
 def task_retry_signal(request, reason, einfo, **kwargs):
     get_current_request().tm.abort()
-    if hasattr(celery, 'pyramid'):
+    if hasattr(celery, "pyramid"):
         celery.pyramid["closer"]()
 
 
 @task_failure.connect
-def task_failure_signal(task_id, exception, args, kwargs, traceback, einfo,
-                        **kwaargs):
+def task_failure_signal(task_id, exception, args, kwargs, traceback, einfo, **kwaargs):
     get_current_request().tm.abort()
-    if hasattr(celery, 'pyramid'):
+    if hasattr(celery, "pyramid"):
         celery.pyramid["closer"]()
 
 
 @task_revoked.connect
 def task_revoked_signal(request, terminated, signum, expired, **kwaargs):
     get_current_request().tm.abort()
-    if hasattr(celery, 'pyramid'):
+    if hasattr(celery, "pyramid"):
         celery.pyramid["closer"]()

backend/src/appenlight/celery/encoders.py

@@ -1,60 +1,51 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import json
 from datetime import datetime, date, timedelta
 
-DATE_FORMAT = '%Y-%m-%dT%H:%M:%S.%f'
+DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%f"
 
 
 class DateEncoder(json.JSONEncoder):
     def default(self, obj):
         if isinstance(obj, datetime):
-            return {
-                '__type__': '__datetime__',
-                'iso': obj.strftime(DATE_FORMAT)
-            }
+            return {"__type__": "__datetime__", "iso": obj.strftime(DATE_FORMAT)}
         elif isinstance(obj, date):
-            return {
-                '__type__': '__date__',
-                'iso': obj.strftime(DATE_FORMAT)
-            }
+            return {"__type__": "__date__", "iso": obj.strftime(DATE_FORMAT)}
         elif isinstance(obj, timedelta):
-            return {
-                '__type__': '__timedelta__',
-                'seconds': obj.total_seconds()
-            }
+            return {"__type__": "__timedelta__", "seconds": obj.total_seconds()}
         else:
             return json.JSONEncoder.default(self, obj)
 
 
 def date_decoder(dct):
-    if '__type__' in dct:
-        if dct['__type__'] == '__datetime__':
-            return datetime.strptime(dct['iso'], DATE_FORMAT)
-        elif dct['__type__'] == '__date__':
-            return datetime.strptime(dct['iso'], DATE_FORMAT).date()
-        elif dct['__type__'] == '__timedelta__':
-            return timedelta(seconds=dct['seconds'])
+    if "__type__" in dct:
+        if dct["__type__"] == "__datetime__":
+            return datetime.strptime(dct["iso"], DATE_FORMAT)
+        elif dct["__type__"] == "__date__":
+            return datetime.strptime(dct["iso"], DATE_FORMAT).date()
+        elif dct["__type__"] == "__timedelta__":
+            return timedelta(seconds=dct["seconds"])
     return dct
 
 
 def json_dumps(obj):
     return json.dumps(obj, cls=DateEncoder)
 
 
 def json_loads(obj):
-    return json.loads(obj.decode('utf8'), object_hook=date_decoder)
+    return json.loads(obj.decode("utf8"), object_hook=date_decoder)

backend/src/appenlight/celery/tasks.py

@@ -1,657 +1,708 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import bisect
 import collections
 import math
 from datetime import datetime, timedelta
 
 import sqlalchemy as sa
 import elasticsearch.exceptions
 import elasticsearch.helpers
 
 from celery.utils.log import get_task_logger
 from zope.sqlalchemy import mark_changed
 from pyramid.threadlocal import get_current_request, get_current_registry
 from ziggurat_foundations.models.services.resource import ResourceService
 
 from appenlight.celery import celery
 from appenlight.models.report_group import ReportGroup
 from appenlight.models import DBSession, Datastores
 from appenlight.models.report import Report
 from appenlight.models.log import Log
 from appenlight.models.metric import Metric
 from appenlight.models.event import Event
 
 from appenlight.models.services.application import ApplicationService
 from appenlight.models.services.event import EventService
 from appenlight.models.services.log import LogService
 from appenlight.models.services.report import ReportService
 from appenlight.models.services.report_group import ReportGroupService
 from appenlight.models.services.user import UserService
 from appenlight.models.tag import Tag
 from appenlight.lib import print_traceback
 from appenlight.lib.utils import parse_proto, in_batches
 from appenlight.lib.ext_json import json
 from appenlight.lib.redis_keys import REDIS_KEYS
 from appenlight.lib.enums import ReportType
 
 log = get_task_logger(__name__)
 
-sample_boundries = list(range(100, 1000, 100)) + \
-                   list(range(1000, 10000, 1000)) + \
-                   list(range(10000, 100000, 5000))
+sample_boundries = (
+    list(range(100, 1000, 100))
+    + list(range(1000, 10000, 1000))
+    + list(range(10000, 100000, 5000))
+)
 
 
 def pick_sample(total_occurences, report_type=None):
     every = 1.0
     position = bisect.bisect_left(sample_boundries, total_occurences)
     if position > 0:
         if report_type == ReportType.not_found:
             divide = 10.0
         else:
             divide = 100.0
         every = sample_boundries[position - 1] / divide
     return total_occurences % every == 0
 
 
 @celery.task(queue="default", default_retry_delay=1, max_retries=2)
 def test_exception_task():
-    log.error('test celery log', extra={'location': 'celery'})
-    log.warning('test celery log', extra={'location': 'celery'})
-    raise Exception('Celery exception test')
+    log.error("test celery log", extra={"location": "celery"})
+    log.warning("test celery log", extra={"location": "celery"})
+    raise Exception("Celery exception test")
 
 
 @celery.task(queue="default", default_retry_delay=1, max_retries=2)
 def test_retry_exception_task():
     try:
         import time
 
         time.sleep(1.3)
-        log.error('test retry celery log', extra={'location': 'celery'})
-        log.warning('test retry celery log', extra={'location': 'celery'})
-        raise Exception('Celery exception test')
+        log.error("test retry celery log", extra={"location": "celery"})
+        log.warning("test retry celery log", extra={"location": "celery"})
+        raise Exception("Celery exception test")
     except Exception as exc:
         if celery.conf["CELERY_EAGER_PROPAGATES_EXCEPTIONS"]:
             raise
         test_retry_exception_task.retry(exc=exc)
 
 
 @celery.task(queue="reports", default_retry_delay=600, max_retries=144)
 def add_reports(resource_id, request_params, dataset, **kwargs):
-    proto_version = parse_proto(request_params.get('protocol_version', ''))
+    proto_version = parse_proto(request_params.get("protocol_version", ""))
     current_time = datetime.utcnow().replace(second=0, microsecond=0)
     try:
         # we will store solr docs here for single insert
         es_report_docs = {}
         es_report_group_docs = {}
         resource = ApplicationService.by_id(resource_id)
 
         tags = []
         es_slow_calls_docs = {}
         es_reports_stats_rows = {}
         for report_data in dataset:
             # build report details for later
             added_details = 0
             report = Report()
             report.set_data(report_data, resource, proto_version)
             report._skip_ft_index = True
 
             # find latest group in this months partition
             report_group = ReportGroupService.by_hash_and_resource(
                 report.resource_id,
                 report.grouping_hash,
-                since_when=datetime.utcnow().date().replace(day=1)
+                since_when=datetime.utcnow().date().replace(day=1),
             )
-            occurences = report_data.get('occurences', 1)
+            occurences = report_data.get("occurences", 1)
             if not report_group:
                 # total reports will be +1 moment later
-                report_group = ReportGroup(grouping_hash=report.grouping_hash,
-                                           occurences=0, total_reports=0,
-                                           last_report=0,
-                                           priority=report.priority,
-                                           error=report.error,
-                                           first_timestamp=report.start_time)
+                report_group = ReportGroup(
+                    grouping_hash=report.grouping_hash,
+                    occurences=0,
+                    total_reports=0,
+                    last_report=0,
+                    priority=report.priority,
+                    error=report.error,
+                    first_timestamp=report.start_time,
+                )
                 report_group._skip_ft_index = True
                 report_group.report_type = report.report_type
             report.report_group_time = report_group.first_timestamp
-            add_sample = pick_sample(report_group.occurences,
-                                     report_type=report_group.report_type)
+            add_sample = pick_sample(
+                report_group.occurences, report_type=report_group.report_type
+            )
             if add_sample:
                 resource.report_groups.append(report_group)
                 report_group.reports.append(report)
                 added_details += 1
                 DBSession.flush()
                 if report.partition_id not in es_report_docs:
                     es_report_docs[report.partition_id] = []
                 es_report_docs[report.partition_id].append(report.es_doc())
                 tags.extend(list(report.tags.items()))
                 slow_calls = report.add_slow_calls(report_data, report_group)
                 DBSession.flush()
                 for s_call in slow_calls:
                     if s_call.partition_id not in es_slow_calls_docs:
                         es_slow_calls_docs[s_call.partition_id] = []
-                    es_slow_calls_docs[s_call.partition_id].append(
-                        s_call.es_doc())
+                    es_slow_calls_docs[s_call.partition_id].append(s_call.es_doc())
                     # try generating new stat rows if needed
             else:
                 # required for postprocessing to not fail later
                 report.report_group = report_group
 
-            stat_row = ReportService.generate_stat_rows(
-                report, resource, report_group)
+            stat_row = ReportService.generate_stat_rows(report, resource, report_group)
             if stat_row.partition_id not in es_reports_stats_rows:
                 es_reports_stats_rows[stat_row.partition_id] = []
-            es_reports_stats_rows[stat_row.partition_id].append(
-                stat_row.es_doc())
+            es_reports_stats_rows[stat_row.partition_id].append(stat_row.es_doc())
 
             # see if we should mark 10th occurence of report
             last_occurences_10 = int(math.floor(report_group.occurences / 10))
-            curr_occurences_10 = int(math.floor(
-                (report_group.occurences + report.occurences) / 10))
-            last_occurences_100 = int(
-                math.floor(report_group.occurences / 100))
-            curr_occurences_100 = int(math.floor(
-                (report_group.occurences + report.occurences) / 100))
+            curr_occurences_10 = int(
+                math.floor((report_group.occurences + report.occurences) / 10)
+            )
+            last_occurences_100 = int(math.floor(report_group.occurences / 100))
+            curr_occurences_100 = int(
+                math.floor((report_group.occurences + report.occurences) / 100)
+            )
             notify_occurences_10 = last_occurences_10 != curr_occurences_10
             notify_occurences_100 = last_occurences_100 != curr_occurences_100
             report_group.occurences = ReportGroup.occurences + occurences
             report_group.last_timestamp = report.start_time
             report_group.summed_duration = ReportGroup.summed_duration + report.duration
             summed_duration = ReportGroup.summed_duration + report.duration
             summed_occurences = ReportGroup.occurences + occurences
             report_group.average_duration = summed_duration / summed_occurences
             report_group.run_postprocessing(report)
             if added_details:
                 report_group.total_reports = ReportGroup.total_reports + 1
                 report_group.last_report = report.id
-            report_group.set_notification_info(notify_10=notify_occurences_10,
-                                               notify_100=notify_occurences_100)
+            report_group.set_notification_info(
+                notify_10=notify_occurences_10, notify_100=notify_occurences_100
+            )
             DBSession.flush()
             report_group.get_report().notify_channel(report_group)
             if report_group.partition_id not in es_report_group_docs:
                 es_report_group_docs[report_group.partition_id] = []
             es_report_group_docs[report_group.partition_id].append(
-                report_group.es_doc())
+                report_group.es_doc()
+            )
 
-            action = 'REPORT'
-            log_msg = '%s: %s %s, client: %s, proto: %s' % (
+            action = "REPORT"
+            log_msg = "%s: %s %s, client: %s, proto: %s" % (
                 action,
-                report_data.get('http_status', 'unknown'),
+                report_data.get("http_status", "unknown"),
                 str(resource),
-                report_data.get('client'),
-                proto_version)
+                report_data.get("client"),
+                proto_version,
+            )
             log.info(log_msg)
         total_reports = len(dataset)
         redis_pipeline = Datastores.redis.pipeline(transaction=False)
-        key = REDIS_KEYS['counters']['reports_per_minute'].format(current_time)
+        key = REDIS_KEYS["counters"]["reports_per_minute"].format(current_time)
         redis_pipeline.incr(key, total_reports)
         redis_pipeline.expire(key, 3600 * 24)
-        key = REDIS_KEYS['counters']['events_per_minute_per_user'].format(
-            resource.owner_user_id, current_time)
+        key = REDIS_KEYS["counters"]["events_per_minute_per_user"].format(
+            resource.owner_user_id, current_time
+        )
         redis_pipeline.incr(key, total_reports)
         redis_pipeline.expire(key, 3600)
-        key = REDIS_KEYS['counters']['reports_per_hour_per_app'].format(
-            resource_id, current_time.replace(minute=0))
+        key = REDIS_KEYS["counters"]["reports_per_hour_per_app"].format(
+            resource_id, current_time.replace(minute=0)
+        )
         redis_pipeline.incr(key, total_reports)
         redis_pipeline.expire(key, 3600 * 24 * 7)
         redis_pipeline.sadd(
-            REDIS_KEYS['apps_that_got_new_data_per_hour'].format(
-                current_time.replace(minute=0)), resource_id)
+            REDIS_KEYS["apps_that_got_new_data_per_hour"].format(
+                current_time.replace(minute=0)
+            ),
+            resource_id,
+        )
         redis_pipeline.execute()
 
         add_reports_es(es_report_group_docs, es_report_docs)
         add_reports_slow_calls_es(es_slow_calls_docs)
         add_reports_stats_rows_es(es_reports_stats_rows)
         return True
     except Exception as exc:
         print_traceback(log)
         if celery.conf["CELERY_EAGER_PROPAGATES_EXCEPTIONS"]:
             raise
         add_reports.retry(exc=exc)
 
 
 @celery.task(queue="es", default_retry_delay=600, max_retries=144)
 def add_reports_es(report_group_docs, report_docs):
     for k, v in report_group_docs.items():
-        to_update = {'_index': k, '_type': 'report_group'}
+        to_update = {"_index": k, "_type": "report_group"}
         [i.update(to_update) for i in v]
         elasticsearch.helpers.bulk(Datastores.es, v)
     for k, v in report_docs.items():
-        to_update = {'_index': k, '_type': 'report'}
+        to_update = {"_index": k, "_type": "report"}
         [i.update(to_update) for i in v]
         elasticsearch.helpers.bulk(Datastores.es, v)
 
 
 @celery.task(queue="es", default_retry_delay=600, max_retries=144)
 def add_reports_slow_calls_es(es_docs):
     for k, v in es_docs.items():
-        to_update = {'_index': k, '_type': 'log'}
+        to_update = {"_index": k, "_type": "log"}
         [i.update(to_update) for i in v]
         elasticsearch.helpers.bulk(Datastores.es, v)
 
 
 @celery.task(queue="es", default_retry_delay=600, max_retries=144)
 def add_reports_stats_rows_es(es_docs):
     for k, v in es_docs.items():
-        to_update = {'_index': k, '_type': 'log'}
+        to_update = {"_index": k, "_type": "log"}
         [i.update(to_update) for i in v]
         elasticsearch.helpers.bulk(Datastores.es, v)
 
 
 @celery.task(queue="logs", default_retry_delay=600, max_retries=144)
 def add_logs(resource_id, request_params, dataset, **kwargs):
-    proto_version = request_params.get('protocol_version')
+    proto_version = request_params.get("protocol_version")
     current_time = datetime.utcnow().replace(second=0, microsecond=0)
 
     try:
         es_docs = collections.defaultdict(list)
         resource = ApplicationService.by_id_cached()(resource_id)
         resource = DBSession.merge(resource, load=False)
         ns_pairs = []
         for entry in dataset:
             # gather pk and ns so we can remove older versions of row later
-            if entry['primary_key'] is not None:
-                ns_pairs.append({"pk": entry['primary_key'],
-                                 "ns": entry['namespace']})
+            if entry["primary_key"] is not None:
+                ns_pairs.append({"pk": entry["primary_key"], "ns": entry["namespace"]})
             log_entry = Log()
             log_entry.set_data(entry, resource=resource)
             log_entry._skip_ft_index = True
             resource.logs.append(log_entry)
             DBSession.flush()
             # insert non pk rows first
-            if entry['primary_key'] is None:
+            if entry["primary_key"] is None:
                 es_docs[log_entry.partition_id].append(log_entry.es_doc())
 
         # 2nd pass to delete all log entries from db foe same pk/ns pair
         if ns_pairs:
             ids_to_delete = []
             es_docs = collections.defaultdict(list)
             es_docs_to_delete = collections.defaultdict(list)
             found_pkey_logs = LogService.query_by_primary_key_and_namespace(
-                list_of_pairs=ns_pairs)
+                list_of_pairs=ns_pairs
+            )
             log_dict = {}
             for log_entry in found_pkey_logs:
                 log_key = (log_entry.primary_key, log_entry.namespace)
                 if log_key not in log_dict:
                     log_dict[log_key] = []
                 log_dict[log_key].append(log_entry)
 
             for ns, entry_list in log_dict.items():
                 entry_list = sorted(entry_list, key=lambda x: x.timestamp)
                 # newest row needs to be indexed in es
                 log_entry = entry_list[-1]
                 # delete everything from pg and ES, leave the last row in pg
                 for e in entry_list[:-1]:
                     ids_to_delete.append(e.log_id)
                     es_docs_to_delete[e.partition_id].append(e.delete_hash)
 
-                es_docs_to_delete[log_entry.partition_id].append(
-                    log_entry.delete_hash)
+                es_docs_to_delete[log_entry.partition_id].append(log_entry.delete_hash)
 
                 es_docs[log_entry.partition_id].append(log_entry.es_doc())
 
             if ids_to_delete:
-                query = DBSession.query(Log).filter(
-                    Log.log_id.in_(ids_to_delete))
+                query = DBSession.query(Log).filter(Log.log_id.in_(ids_to_delete))
                 query.delete(synchronize_session=False)
             if es_docs_to_delete:
                 # batch this to avoid problems with default ES bulk limits
                 for es_index in es_docs_to_delete.keys():
                     for batch in in_batches(es_docs_to_delete[es_index], 20):
-                        query = {"query": {'terms': {'delete_hash': batch}}}
+                        query = {"query": {"terms": {"delete_hash": batch}}}
 
                         try:
                             Datastores.es.transport.perform_request(
-                                "DELETE", '/{}/{}/_query'.format(es_index, 'log'), body=query)
+                                "DELETE",
+                                "/{}/{}/_query".format(es_index, "log"),
+                                body=query,
+                            )
                         except elasticsearch.exceptions.NotFoundError as exc:
-                            msg = 'skipping index {}'.format(es_index)
+                            msg = "skipping index {}".format(es_index)
                             log.info(msg)
 
         total_logs = len(dataset)
 
-        log_msg = 'LOG_NEW: %s, entries: %s, proto:%s' % (
+        log_msg = "LOG_NEW: %s, entries: %s, proto:%s" % (
             str(resource),
             total_logs,
-            proto_version)
+            proto_version,
+        )
         log.info(log_msg)
         # mark_changed(session)
         redis_pipeline = Datastores.redis.pipeline(transaction=False)
-        key = REDIS_KEYS['counters']['logs_per_minute'].format(current_time)
+        key = REDIS_KEYS["counters"]["logs_per_minute"].format(current_time)
         redis_pipeline.incr(key, total_logs)
         redis_pipeline.expire(key, 3600 * 24)
-        key = REDIS_KEYS['counters']['events_per_minute_per_user'].format(
-            resource.owner_user_id, current_time)
+        key = REDIS_KEYS["counters"]["events_per_minute_per_user"].format(
+            resource.owner_user_id, current_time
+        )
         redis_pipeline.incr(key, total_logs)
         redis_pipeline.expire(key, 3600)
-        key = REDIS_KEYS['counters']['logs_per_hour_per_app'].format(
-            resource_id, current_time.replace(minute=0))
+        key = REDIS_KEYS["counters"]["logs_per_hour_per_app"].format(
+            resource_id, current_time.replace(minute=0)
+        )
         redis_pipeline.incr(key, total_logs)
         redis_pipeline.expire(key, 3600 * 24 * 7)
         redis_pipeline.sadd(
-            REDIS_KEYS['apps_that_got_new_data_per_hour'].format(
-                current_time.replace(minute=0)), resource_id)
+            REDIS_KEYS["apps_that_got_new_data_per_hour"].format(
+                current_time.replace(minute=0)
+            ),
+            resource_id,
+        )
         redis_pipeline.execute()
         add_logs_es(es_docs)
         return True
     except Exception as exc:
         print_traceback(log)
         if celery.conf["CELERY_EAGER_PROPAGATES_EXCEPTIONS"]:
             raise
         add_logs.retry(exc=exc)
 
 
 @celery.task(queue="es", default_retry_delay=600, max_retries=144)
 def add_logs_es(es_docs):
     for k, v in es_docs.items():
-        to_update = {'_index': k, '_type': 'log'}
+        to_update = {"_index": k, "_type": "log"}
         [i.update(to_update) for i in v]
         elasticsearch.helpers.bulk(Datastores.es, v)
 
 
 @celery.task(queue="metrics", default_retry_delay=600, max_retries=144)
 def add_metrics(resource_id, request_params, dataset, proto_version):
     current_time = datetime.utcnow().replace(second=0, microsecond=0)
     try:
         resource = ApplicationService.by_id_cached()(resource_id)
         resource = DBSession.merge(resource, load=False)
         es_docs = []
         rows = []
         for metric in dataset:
-            tags = dict(metric['tags'])
-            server_n = tags.get('server_name', metric['server_name']).lower()
-            tags['server_name'] = server_n or 'unknown'
+            tags = dict(metric["tags"])
+            server_n = tags.get("server_name", metric["server_name"]).lower()
+            tags["server_name"] = server_n or "unknown"
             new_metric = Metric(
-                timestamp=metric['timestamp'],
+                timestamp=metric["timestamp"],
                 resource_id=resource.resource_id,
-                namespace=metric['namespace'],
-                tags=tags)
+                namespace=metric["namespace"],
+                tags=tags,
+            )
             rows.append(new_metric)
             es_docs.append(new_metric.es_doc())
         session = DBSession()
         session.bulk_save_objects(rows)
         session.flush()
 
-        action = 'METRICS'
-        metrics_msg = '%s: %s, metrics: %s, proto:%s' % (
+        action = "METRICS"
+        metrics_msg = "%s: %s, metrics: %s, proto:%s" % (
             action,
             str(resource),
             len(dataset),
-            proto_version
+            proto_version,
         )
         log.info(metrics_msg)
 
         mark_changed(session)
         redis_pipeline = Datastores.redis.pipeline(transaction=False)
-        key = REDIS_KEYS['counters']['metrics_per_minute'].format(current_time)
+        key = REDIS_KEYS["counters"]["metrics_per_minute"].format(current_time)
         redis_pipeline.incr(key, len(rows))
         redis_pipeline.expire(key, 3600 * 24)
-        key = REDIS_KEYS['counters']['events_per_minute_per_user'].format(
-            resource.owner_user_id, current_time)
+        key = REDIS_KEYS["counters"]["events_per_minute_per_user"].format(
+            resource.owner_user_id, current_time
+        )
         redis_pipeline.incr(key, len(rows))
         redis_pipeline.expire(key, 3600)
-        key = REDIS_KEYS['counters']['metrics_per_hour_per_app'].format(
-            resource_id, current_time.replace(minute=0))
+        key = REDIS_KEYS["counters"]["metrics_per_hour_per_app"].format(
+            resource_id, current_time.replace(minute=0)
+        )
         redis_pipeline.incr(key, len(rows))
         redis_pipeline.expire(key, 3600 * 24 * 7)
         redis_pipeline.sadd(
-            REDIS_KEYS['apps_that_got_new_data_per_hour'].format(
-                current_time.replace(minute=0)), resource_id)
+            REDIS_KEYS["apps_that_got_new_data_per_hour"].format(
+                current_time.replace(minute=0)
+            ),
+            resource_id,
+        )
         redis_pipeline.execute()
         add_metrics_es(es_docs)
         return True
     except Exception as exc:
         print_traceback(log)
         if celery.conf["CELERY_EAGER_PROPAGATES_EXCEPTIONS"]:
             raise
         add_metrics.retry(exc=exc)
 
 
 @celery.task(queue="es", default_retry_delay=600, max_retries=144)
 def add_metrics_es(es_docs):
     for doc in es_docs:
-        partition = 'rcae_m_%s' % doc['timestamp'].strftime('%Y_%m_%d')
-        Datastores.es.index(partition, 'log', doc)
+        partition = "rcae_m_%s" % doc["timestamp"].strftime("%Y_%m_%d")
+        Datastores.es.index(partition, "log", doc)
 
 
 @celery.task(queue="default", default_retry_delay=5, max_retries=2)
 def check_user_report_notifications(resource_id):
     since_when = datetime.utcnow()
     try:
         request = get_current_request()
         application = ApplicationService.by_id(resource_id)
         if not application:
             return
-        error_key = REDIS_KEYS['reports_to_notify_per_type_per_app'].format(
-            ReportType.error, resource_id)
-        slow_key = REDIS_KEYS['reports_to_notify_per_type_per_app'].format(
-            ReportType.slow, resource_id)
+        error_key = REDIS_KEYS["reports_to_notify_per_type_per_app"].format(
+            ReportType.error, resource_id
+        )
+        slow_key = REDIS_KEYS["reports_to_notify_per_type_per_app"].format(
+            ReportType.slow, resource_id
+        )
         error_group_ids = Datastores.redis.smembers(error_key)
         slow_group_ids = Datastores.redis.smembers(slow_key)
         Datastores.redis.delete(error_key)
         Datastores.redis.delete(slow_key)
         err_gids = [int(g_id) for g_id in error_group_ids]
         slow_gids = [int(g_id) for g_id in list(slow_group_ids)]
         group_ids = err_gids + slow_gids
         occurence_dict = {}
         for g_id in group_ids:
-            key = REDIS_KEYS['counters']['report_group_occurences'].format(
-                g_id)
+            key = REDIS_KEYS["counters"]["report_group_occurences"].format(g_id)
             val = Datastores.redis.get(key)
             Datastores.redis.delete(key)
             if val:
                 occurence_dict[g_id] = int(val)
             else:
                 occurence_dict[g_id] = 1
         report_groups = ReportGroupService.by_ids(group_ids)
         report_groups.options(sa.orm.joinedload(ReportGroup.last_report_ref))
 
         ApplicationService.check_for_groups_alert(
-            application, 'alert', report_groups=report_groups,
-            occurence_dict=occurence_dict)
-        users = set([p.user for p in ResourceService.users_for_perm(application, 'view')])
+            application,
+            "alert",
+            report_groups=report_groups,
+            occurence_dict=occurence_dict,
+        )
+        users = set(
+            [p.user for p in ResourceService.users_for_perm(application, "view")]
+        )
         report_groups = report_groups.all()
         for user in users:
-            UserService.report_notify(user, request, application,
-                                      report_groups=report_groups,
-                                      occurence_dict=occurence_dict)
+            UserService.report_notify(
+                user,
+                request,
+                application,
+                report_groups=report_groups,
+                occurence_dict=occurence_dict,
+            )
         for group in report_groups:
             # marks report_groups as notified
             if not group.notified:
                 group.notified = True
     except Exception as exc:
         print_traceback(log)
         raise
 
 
 @celery.task(queue="default", default_retry_delay=5, max_retries=2)
 def check_alerts(resource_id):
     since_when = datetime.utcnow()
     try:
         request = get_current_request()
         application = ApplicationService.by_id(resource_id)
         if not application:
             return
-        error_key = REDIS_KEYS[
-            'reports_to_notify_per_type_per_app_alerting'].format(
-            ReportType.error, resource_id)
-        slow_key = REDIS_KEYS[
-            'reports_to_notify_per_type_per_app_alerting'].format(
-            ReportType.slow, resource_id)
+        error_key = REDIS_KEYS["reports_to_notify_per_type_per_app_alerting"].format(
+            ReportType.error, resource_id
+        )
+        slow_key = REDIS_KEYS["reports_to_notify_per_type_per_app_alerting"].format(
+            ReportType.slow, resource_id
+        )
         error_group_ids = Datastores.redis.smembers(error_key)
         slow_group_ids = Datastores.redis.smembers(slow_key)
         Datastores.redis.delete(error_key)
         Datastores.redis.delete(slow_key)
         err_gids = [int(g_id) for g_id in error_group_ids]
         slow_gids = [int(g_id) for g_id in list(slow_group_ids)]
         group_ids = err_gids + slow_gids
         occurence_dict = {}
         for g_id in group_ids:
-            key = REDIS_KEYS['counters'][
-                'report_group_occurences_alerting'].format(
-                g_id)
+            key = REDIS_KEYS["counters"]["report_group_occurences_alerting"].format(
+                g_id
+            )
             val = Datastores.redis.get(key)
             Datastores.redis.delete(key)
             if val:
                 occurence_dict[g_id] = int(val)
             else:
                 occurence_dict[g_id] = 1
         report_groups = ReportGroupService.by_ids(group_ids)
         report_groups.options(sa.orm.joinedload(ReportGroup.last_report_ref))
 
         ApplicationService.check_for_groups_alert(
-            application, 'alert', report_groups=report_groups,
-            occurence_dict=occurence_dict, since_when=since_when)
+            application,
+            "alert",
+            report_groups=report_groups,
+            occurence_dict=occurence_dict,
+            since_when=since_when,
+        )
     except Exception as exc:
         print_traceback(log)
         raise
 
 
 @celery.task(queue="default", default_retry_delay=1, max_retries=2)
 def close_alerts():
-    log.warning('Checking alerts')
+    log.warning("Checking alerts")
     since_when = datetime.utcnow()
     try:
-        event_types = [Event.types['error_report_alert'],
-                       Event.types['slow_report_alert'], ]
-        statuses = [Event.statuses['active']]
+        event_types = [
+            Event.types["error_report_alert"],
+            Event.types["slow_report_alert"],
+        ]
+        statuses = [Event.statuses["active"]]
         # get events older than 5 min
         events = EventService.by_type_and_status(
-            event_types,
-            statuses,
-            older_than=(since_when - timedelta(minutes=5)))
+            event_types, statuses, older_than=(since_when - timedelta(minutes=5))
+        )
         for event in events:
             # see if we can close them
-            event.validate_or_close(
-                since_when=(since_when - timedelta(minutes=1)))
+            event.validate_or_close(since_when=(since_when - timedelta(minutes=1)))
     except Exception as exc:
         print_traceback(log)
         raise
 
 
 @celery.task(queue="default", default_retry_delay=600, max_retries=144)
 def update_tag_counter(tag_name, tag_value, count):
     try:
-        query = DBSession.query(Tag).filter(Tag.name == tag_name).filter(
-            sa.cast(Tag.value, sa.types.TEXT) == sa.cast(json.dumps(tag_value),
-                                                         sa.types.TEXT))
-        query.update({'times_seen': Tag.times_seen + count,
-                      'last_timestamp': datetime.utcnow()},
-                     synchronize_session=False)
+        query = (
+            DBSession.query(Tag)
+            .filter(Tag.name == tag_name)
+            .filter(
+                sa.cast(Tag.value, sa.types.TEXT)
+                == sa.cast(json.dumps(tag_value), sa.types.TEXT)
+            )
+        )
+        query.update(
+            {"times_seen": Tag.times_seen + count, "last_timestamp": datetime.utcnow()},
+            synchronize_session=False,
+        )
         session = DBSession()
         mark_changed(session)
         return True
     except Exception as exc:
         print_traceback(log)
         if celery.conf["CELERY_EAGER_PROPAGATES_EXCEPTIONS"]:
             raise
         update_tag_counter.retry(exc=exc)
 
 
 @celery.task(queue="default")
 def update_tag_counters():
     """
     Sets task to update counters for application tags
     """
-    tags = Datastores.redis.lrange(REDIS_KEYS['seen_tag_list'], 0, -1)
-    Datastores.redis.delete(REDIS_KEYS['seen_tag_list'])
+    tags = Datastores.redis.lrange(REDIS_KEYS["seen_tag_list"], 0, -1)
+    Datastores.redis.delete(REDIS_KEYS["seen_tag_list"])
     c = collections.Counter(tags)
     for t_json, count in c.items():
         tag_info = json.loads(t_json)
         update_tag_counter.delay(tag_info[0], tag_info[1], count)
 
 
 @celery.task(queue="default")
 def daily_digest():
     """
     Sends daily digest with top 50 error reports
     """
     request = get_current_request()
-    apps = Datastores.redis.smembers(REDIS_KEYS['apps_that_had_reports'])
-    Datastores.redis.delete(REDIS_KEYS['apps_that_had_reports'])
+    apps = Datastores.redis.smembers(REDIS_KEYS["apps_that_had_reports"])
+    Datastores.redis.delete(REDIS_KEYS["apps_that_had_reports"])
     since_when = datetime.utcnow() - timedelta(hours=8)
-    log.warning('Generating daily digests')
+    log.warning("Generating daily digests")
     for resource_id in apps:
-        resource_id = resource_id.decode('utf8')
+        resource_id = resource_id.decode("utf8")
         end_date = datetime.utcnow().replace(microsecond=0, second=0)
-        filter_settings = {'resource': [resource_id],
-                           'tags': [{'name': 'type',
-                                     'value': ['error'], 'op': None}],
-                           'type': 'error', 'start_date': since_when,
-                           'end_date': end_date}
+        filter_settings = {
+            "resource": [resource_id],
+            "tags": [{"name": "type", "value": ["error"], "op": None}],
+            "type": "error",
+            "start_date": since_when,
+            "end_date": end_date,
+        }
 
         reports = ReportGroupService.get_trending(
-            request, filter_settings=filter_settings, limit=50)
+            request, filter_settings=filter_settings, limit=50
+        )
 
         application = ApplicationService.by_id(resource_id)
         if application:
-            users = set([p.user for p in ResourceService.users_for_perm(application, 'view')])
+            users = set(
+                [p.user for p in ResourceService.users_for_perm(application, "view")]
+            )
             for user in users:
-                user.send_digest(request, application, reports=reports,
-                                 since_when=since_when)
+                user.send_digest(
+                    request, application, reports=reports, since_when=since_when
+                )
 
 
 @celery.task(queue="default")
 def notifications_reports():
     """
     Loop that checks redis for info and then issues new tasks to celery to
     issue notifications
     """
-    apps = Datastores.redis.smembers(REDIS_KEYS['apps_that_had_reports'])
-    Datastores.redis.delete(REDIS_KEYS['apps_that_had_reports'])
+    apps = Datastores.redis.smembers(REDIS_KEYS["apps_that_had_reports"])
+    Datastores.redis.delete(REDIS_KEYS["apps_that_had_reports"])
     for app in apps:
-        log.warning('Notify for app: %s' % app)
-        check_user_report_notifications.delay(app.decode('utf8'))
+        log.warning("Notify for app: %s" % app)
+        check_user_report_notifications.delay(app.decode("utf8"))
+
 
 @celery.task(queue="default")
 def alerting_reports():
     """
     Loop that checks redis for info and then issues new tasks to celery to
     perform the following:
     - which applications should have new alerts opened
     """
 
-    apps = Datastores.redis.smembers(REDIS_KEYS['apps_that_had_reports_alerting'])
-    Datastores.redis.delete(REDIS_KEYS['apps_that_had_reports_alerting'])
+    apps = Datastores.redis.smembers(REDIS_KEYS["apps_that_had_reports_alerting"])
+    Datastores.redis.delete(REDIS_KEYS["apps_that_had_reports_alerting"])
     for app in apps:
-        log.warning('Notify for app: %s' % app)
-        check_alerts.delay(app.decode('utf8'))
+        log.warning("Notify for app: %s" % app)
+        check_alerts.delay(app.decode("utf8"))
 
 
[e-mail](queue="default", soft_time_limit=3600 * 4,
-             hard_time_limit=3600 * 4, max_retries=144)
[e-mail](
+    queue="default", soft_time_limit=3600 * 4, hard_time_limit=3600 * 4, max_retries=144
+)
 def logs_cleanup(resource_id, filter_settings):
     request = get_current_request()
     request.tm.begin()
     es_query = {
         "query": {
-            "filtered": {
-                "filter": {
-                    "and": [{"term": {"resource_id": resource_id}}]
-                }
-            }
+            "filtered": {"filter": {"and": [{"term": {"resource_id": resource_id}}]}}
         }
     }
 
     query = DBSession.query(Log).filter(Log.resource_id == resource_id)
-    if filter_settings['namespace']:
-        query = query.filter(Log.namespace == filter_settings['namespace'][0])
-        es_query['query']['filtered']['filter']['and'].append(
-            {"term": {"namespace": filter_settings['namespace'][0]}}
+    if filter_settings["namespace"]:
+        query = query.filter(Log.namespace == filter_settings["namespace"][0])
+        es_query["query"]["filtered"]["filter"]["and"].append(
+            {"term": {"namespace": filter_settings["namespace"][0]}}
         )
     query.delete(synchronize_session=False)
     request.tm.commit()
-    Datastores.es.transport.perform_request("DELETE", '/{}/{}/_query'.format('rcae_l_*', 'log'), body=es_query)
+    Datastores.es.transport.perform_request(
+        "DELETE", "/{}/{}/_query".format("rcae_l_*", "log"), body=es_query
+    )

backend/src/appenlight/fil.py

@@ -1,19 +1,20 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 def filter_callable(structure, section=None):
-    structure['SOMEVAL'] = '***REMOVED***'
+    structure["SOMEVAL"] = "***REMOVED***"
     return structure

backend/src/appenlight/forms.py

@@ -1,896 +1,981 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import wtforms
 import formencode
 import re
 import pyramid.threadlocal
 import datetime
 import appenlight.lib.helpers as h
 
 from ziggurat_foundations.models.services.user import UserService
 from ziggurat_foundations.models.services.group import GroupService
 from appenlight.models import DBSession
 from appenlight.models.alert_channel import AlertChannel
 from appenlight.models.integrations import IntegrationException
 from appenlight.models.integrations.campfire import CampfireIntegration
 from appenlight.models.integrations.bitbucket import BitbucketIntegration
 from appenlight.models.integrations.github import GithubIntegration
 from appenlight.models.integrations.flowdock import FlowdockIntegration
 from appenlight.models.integrations.hipchat import HipchatIntegration
 from appenlight.models.integrations.jira import JiraClient
 from appenlight.models.integrations.slack import SlackIntegration
 from appenlight.lib.ext_json import json
 from wtforms.ext.csrf.form import SecureForm
 from wtforms.compat import iteritems
 from collections import defaultdict
 
 _ = str
 
 strip_filter = lambda x: x.strip() if x else None
 uppercase_filter = lambda x: x.upper() if x else None
 
-FALSE_VALUES = ('false', '', False, None)
+FALSE_VALUES = ("false", "", False, None)
 
 
 class CSRFException(Exception):
     pass
 
 
 class ReactorForm(SecureForm):
-    def __init__(self, formdata=None, obj=None, prefix='', csrf_context=None,
-                 **kwargs):
-        super(ReactorForm, self).__init__(formdata=formdata, obj=obj,
-                                          prefix=prefix,
-                                          csrf_context=csrf_context, **kwargs)
+    def __init__(self, formdata=None, obj=None, prefix="", csrf_context=None, **kwargs):
+        super(ReactorForm, self).__init__(
+            formdata=formdata,
+            obj=obj,
+            prefix=prefix,
+            csrf_context=csrf_context,
+            **kwargs
+        )
         self._csrf_context = csrf_context
 
     def generate_csrf_token(self, csrf_context):
         return csrf_context.session.get_csrf_token()
 
     def validate_csrf_token(self, field):
         request = self._csrf_context or pyramid.threadlocal.get_current_request()
-        is_from_auth_token = 'auth:auth_token' in request.effective_principals
+        is_from_auth_token = "auth:auth_token" in request.effective_principals
         if is_from_auth_token:
             return True
 
         if field.data != field.current_token:
             # try to save the day by using token from angular
-            if request.headers.get('X-XSRF-TOKEN') != field.current_token:
-                raise CSRFException('Invalid CSRF token')
+            if request.headers.get("X-XSRF-TOKEN") != field.current_token:
+                raise CSRFException("Invalid CSRF token")
 
     @property
     def errors_dict(self):
         r_dict = defaultdict(list)
         for k, errors in self.errors.items():
             r_dict[k].extend([str(e) for e in errors])
         return r_dict
 
     @property
     def errors_json(self):
         return json.dumps(self.errors_dict)
 
     def populate_obj(self, obj, ignore_none=False):
         """
         Populates the attributes of the passed `obj` with data from the form's
         fields.
 
         :note: This is a destructive operation; Any attribute with the same name
                as a field will be overridden. Use with caution.
         """
         if ignore_none:
             for name, field in iteritems(self._fields):
                 if field.data is not None:
                     field.populate_obj(obj, name)
         else:
             for name, field in iteritems(self._fields):
                 field.populate_obj(obj, name)
 
     css_classes = {}
     ignore_labels = {}
 
 
 class SignInForm(ReactorForm):
     came_from = wtforms.HiddenField()
-    sign_in_user_name = wtforms.StringField(_('User Name'))
-    sign_in_user_password = wtforms.PasswordField(_('Password'))
+    sign_in_user_name = wtforms.StringField(_("User Name"))
+    sign_in_user_password = wtforms.PasswordField(_("Password"))
 
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
 
-    html_attrs = {'sign_in_user_name': {'placeholder': 'Your login'},
-                  'sign_in_user_password': {
-                      'placeholder': 'Your password'}}
+    html_attrs = {
+        "sign_in_user_name": {"placeholder": "Your login"},
+        "sign_in_user_password": {"placeholder": "Your password"},
+    }
 
 
 from wtforms.widgets import html_params, HTMLString
 
 
-def select_multi_checkbox(field, ul_class='set', **kwargs):
+def select_multi_checkbox(field, ul_class="set", **kwargs):
     """Render a multi-checkbox widget"""
-    kwargs.setdefault('type', 'checkbox')
-    field_id = kwargs.pop('id', field.id)
-    html = ['<ul %s>' % html_params(id=field_id, class_=ul_class)]
+    kwargs.setdefault("type", "checkbox")
+    field_id = kwargs.pop("id", field.id)
+    html = ["<ul %s>" % html_params(id=field_id, class_=ul_class)]
     for value, label, checked in field.iter_choices():
-        choice_id = '%s-%s' % (field_id, value)
+        choice_id = "%s-%s" % (field_id, value)
         options = dict(kwargs, name=field.name, value=value, id=choice_id)
         if checked:
-            options['checked'] = 'checked'
-        html.append('<li><input %s /> ' % html_params(**options))
+            options["checked"] = "checked"
+        html.append("<li><input %s /> " % html_params(**options))
         html.append('<label for="%s">%s</label></li>' % (choice_id, label))
-    html.append('</ul>')
-    return HTMLString(''.join(html))
+    html.append("</ul>")
+    return HTMLString("".join(html))
 
 
-def button_widget(field, button_cls='ButtonField btn btn-default', **kwargs):
+def button_widget(field, button_cls="ButtonField btn btn-default", **kwargs):
     """Render a button widget"""
-    kwargs.setdefault('type', 'button')
-    field_id = kwargs.pop('id', field.id)
-    kwargs.setdefault('value', field.label.text)
-    html = ['<button %s>%s</button>' % (html_params(id=field_id,
-                                                     class_=button_cls),
-                                         kwargs['value'],)]
-    return HTMLString(''.join(html))
+    kwargs.setdefault("type", "button")
+    field_id = kwargs.pop("id", field.id)
+    kwargs.setdefault("value", field.label.text)
+    html = [
+        "<button %s>%s</button>"
+        % (html_params(id=field_id, class_=button_cls), kwargs["value"])
+    ]
+    return HTMLString("".join(html))
 
 
 def clean_whitespace(value):
     if value:
         return value.strip()
     return value
 
 
 def found_username_validator(form, field):
     user = UserService.by_user_name(field.data)
     # sets user to recover in email validator
     form.field_user = user
     if not user:
-        raise wtforms.ValidationError('This username does not exist')
+        raise wtforms.ValidationError("This username does not exist")
 
 
 def found_username_email_validator(form, field):
     user = UserService.by_email(field.data)
     if not user:
-        raise wtforms.ValidationError('Email is incorrect')
+        raise wtforms.ValidationError("Email is incorrect")
 
 
 def unique_username_validator(form, field):
     user = UserService.by_user_name(field.data)
     if user:
-        raise wtforms.ValidationError('This username already exists in system')
+        raise wtforms.ValidationError("This username already exists in system")
 
 
 def unique_groupname_validator(form, field):
     group = GroupService.by_group_name(field.data)
-    mod_group = getattr(form, '_modified_group', None)
+    mod_group = getattr(form, "_modified_group", None)
     if group and (not mod_group or mod_group.id != group.id):
-        raise wtforms.ValidationError(
-            'This group name already exists in system')
+        raise wtforms.ValidationError("This group name already exists in system")
 
 
 def unique_email_validator(form, field):
     user = UserService.by_email(field.data)
     if user:
-        raise wtforms.ValidationError('This email already exists in system')
+        raise wtforms.ValidationError("This email already exists in system")
 
 
 def email_validator(form, field):
     validator = formencode.validators.Email()
     try:
         validator.to_python(field.data)
     except formencode.Invalid as e:
         raise wtforms.ValidationError(e)
 
 
 def unique_alert_email_validator(form, field):
     q = DBSession.query(AlertChannel)
-    q = q.filter(AlertChannel.channel_name == 'email')
+    q = q.filter(AlertChannel.channel_name == "email")
     q = q.filter(AlertChannel.channel_value == field.data)
     email = q.first()
     if email:
-        raise wtforms.ValidationError(
-            'This email already exists in alert system')
+        raise wtforms.ValidationError("This email already exists in alert system")
 
 
 def blocked_email_validator(form, field):
     blocked_emails = [
-        'goood-mail.org',
-        'shoeonlineblog.com',
-        'louboutinemart.com',
-        'guccibagshere.com',
-        'nikeshoesoutletforsale.com'
+        "goood-mail.org",
+        "shoeonlineblog.com",
+        "louboutinemart.com",
+        "guccibagshere.com",
+        "nikeshoesoutletforsale.com",
     ]
-    data = field.data or ''
-    domain = data.split('@')[-1]
+    data = field.data or ""
+    domain = data.split("@")[-1]
     if domain in blocked_emails:
-        raise wtforms.ValidationError('Don\'t spam')
+        raise wtforms.ValidationError("Don't spam")
 
 
 def old_password_validator(form, field):
-    if not UserService.check_password(field.user, field.data or ''):
-        raise wtforms.ValidationError('You need to enter correct password')
+    if not UserService.check_password(field.user, field.data or ""):
+        raise wtforms.ValidationError("You need to enter correct password")
 
 
 class UserRegisterForm(ReactorForm):
     user_name = wtforms.StringField(
-        _('User Name'),
+        _("User Name"),
         filters=[strip_filter],
         validators=[
             wtforms.validators.Length(min=2, max=30),
             wtforms.validators.Regexp(
-                re.compile(r'^[\.\w-]+$', re.UNICODE),
-                message="Invalid characters used"),
+                re.compile(r"^[\.\w-]+$", re.UNICODE), message="Invalid characters used"
+            ),
             unique_username_validator,
-            wtforms.validators.DataRequired()
-        ])
+            wtforms.validators.DataRequired(),
+        ],
+    )
 
-    user_password = wtforms.PasswordField(_('User Password'),
-                                          filters=[strip_filter],
-                                          validators=[
-                                              wtforms.validators.Length(min=4),
-                                              wtforms.validators.DataRequired()
-                                          ])
+    user_password = wtforms.PasswordField(
+        _("User Password"),
+        filters=[strip_filter],
+        validators=[
+            wtforms.validators.Length(min=4),
+            wtforms.validators.DataRequired(),
+        ],
+    )
 
-    email = wtforms.StringField(_('Email Address'),
-                                filters=[strip_filter],
-                                validators=[email_validator,
-                                            unique_email_validator,
-                                            blocked_email_validator,
-                                            wtforms.validators.DataRequired()])
-    first_name = wtforms.HiddenField(_('First Name'))
-    last_name = wtforms.HiddenField(_('Last Name'))
+    email = wtforms.StringField(
+        _("Email Address"),
+        filters=[strip_filter],
+        validators=[
+            email_validator,
+            unique_email_validator,
+            blocked_email_validator,
+            wtforms.validators.DataRequired(),
+        ],
+    )
+    first_name = wtforms.HiddenField(_("First Name"))
+    last_name = wtforms.HiddenField(_("Last Name"))
 
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
 
-    html_attrs = {'user_name': {'placeholder': 'Your login'},
-                  'user_password': {'placeholder': 'Your password'},
-                  'email': {'placeholder': 'Your email'}}
+    html_attrs = {
+        "user_name": {"placeholder": "Your login"},
+        "user_password": {"placeholder": "Your password"},
+        "email": {"placeholder": "Your email"},
+    }
 
 
 class UserCreateForm(UserRegisterForm):
-    status = wtforms.BooleanField('User status',
-                                  false_values=FALSE_VALUES)
+    status = wtforms.BooleanField("User status", false_values=FALSE_VALUES)
 
 
 class UserUpdateForm(UserCreateForm):
     user_name = None
-    user_password = wtforms.PasswordField(_('User Password'),
-                                          filters=[strip_filter],
-                                          validators=[
-                                              wtforms.validators.Length(min=4),
-                                              wtforms.validators.Optional()
-                                          ])
-    email = wtforms.StringField(_('Email Address'),
-                                filters=[strip_filter],
-                                validators=[email_validator,
-                                            wtforms.validators.DataRequired()])
+    user_password = wtforms.PasswordField(
+        _("User Password"),
+        filters=[strip_filter],
+        validators=[wtforms.validators.Length(min=4), wtforms.validators.Optional()],
+    )
+    email = wtforms.StringField(
+        _("Email Address"),
+        filters=[strip_filter],
+        validators=[email_validator, wtforms.validators.DataRequired()],
+    )
 
 
 class LostPasswordForm(ReactorForm):
-    email = wtforms.StringField(_('Email Address'),
-                                filters=[strip_filter],
-                                validators=[email_validator,
-                                            found_username_email_validator,
-                                            wtforms.validators.DataRequired()])
+    email = wtforms.StringField(
+        _("Email Address"),
+        filters=[strip_filter],
+        validators=[
+            email_validator,
+            found_username_email_validator,
+            wtforms.validators.DataRequired(),
+        ],
+    )
 
-    submit = wtforms.SubmitField(_('Reset password'))
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
+    submit = wtforms.SubmitField(_("Reset password"))
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
 
 
 class ChangePasswordForm(ReactorForm):
     old_password = wtforms.PasswordField(
-        'Old Password',
+        "Old Password",
         filters=[strip_filter],
-        validators=[old_password_validator,
-                    wtforms.validators.DataRequired()])
+        validators=[old_password_validator, wtforms.validators.DataRequired()],
+    )
 
     new_password = wtforms.PasswordField(
-        'New Password',
+        "New Password",
         filters=[strip_filter],
-        validators=[wtforms.validators.Length(min=4),
-                    wtforms.validators.DataRequired()])
+        validators=[
+            wtforms.validators.Length(min=4),
+            wtforms.validators.DataRequired(),
+        ],
+    )
     new_password_confirm = wtforms.PasswordField(
-        'Confirm Password',
+        "Confirm Password",
         filters=[strip_filter],
-        validators=[wtforms.validators.EqualTo('new_password'),
-                    wtforms.validators.DataRequired()])
-    submit = wtforms.SubmitField('Change Password')
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
+        validators=[
+            wtforms.validators.EqualTo("new_password"),
+            wtforms.validators.DataRequired(),
+        ],
+    )
+    submit = wtforms.SubmitField("Change Password")
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
 
 
 class CheckPasswordForm(ReactorForm):
     password = wtforms.PasswordField(
-        'Password',
+        "Password",
         filters=[strip_filter],
-        validators=[old_password_validator,
-                    wtforms.validators.DataRequired()])
+        validators=[old_password_validator, wtforms.validators.DataRequired()],
+    )
 
 
 class NewPasswordForm(ReactorForm):
     new_password = wtforms.PasswordField(
-        'New Password',
+        "New Password",
         filters=[strip_filter],
-        validators=[wtforms.validators.Length(min=4),
-                    wtforms.validators.DataRequired()])
+        validators=[
+            wtforms.validators.Length(min=4),
+            wtforms.validators.DataRequired(),
+        ],
+    )
     new_password_confirm = wtforms.PasswordField(
-        'Confirm Password',
+        "Confirm Password",
         filters=[strip_filter],
-        validators=[wtforms.validators.EqualTo('new_password'),
-                    wtforms.validators.DataRequired()])
-    submit = wtforms.SubmitField('Set Password')
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
+        validators=[
+            wtforms.validators.EqualTo("new_password"),
+            wtforms.validators.DataRequired(),
+        ],
+    )
+    submit = wtforms.SubmitField("Set Password")
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
 
 
 class CORSTextAreaField(wtforms.StringField):
     """
     This field represents an HTML ``<textarea>`` and can be used to take
     multi-line input.
     """
+
     widget = wtforms.widgets.TextArea()
 
     def process_formdata(self, valuelist):
         self.data = []
         if valuelist:
-            data = [x.strip() for x in valuelist[0].split('\n')]
+            data = [x.strip() for x in valuelist[0].split("\n")]
             for d in data:
                 if not d:
                     continue
-                if d.startswith('www.'):
+                if d.startswith("www."):
                     d = d[4:]
                 if data:
                     self.data.append(d)
         else:
             self.data = []
-        self.data = '\n'.join(self.data)
+        self.data = "\n".join(self.data)
 
 
 class ApplicationCreateForm(ReactorForm):
     resource_name = wtforms.StringField(
-        _('Application name'),
+        _("Application name"),
         filters=[strip_filter],
-        validators=[wtforms.validators.Length(min=1),
-                    wtforms.validators.DataRequired()])
+        validators=[
+            wtforms.validators.Length(min=1),
+            wtforms.validators.DataRequired(),
+        ],
+    )
 
     domains = CORSTextAreaField(
-        _('Domain names for CORS headers '),
-        validators=[wtforms.validators.Length(min=1),
-                    wtforms.validators.Optional()],
-        description='Required for Javascript error '
-                    'tracking (one line one domain, skip http:// part)')
+        _("Domain names for CORS headers "),
+        validators=[wtforms.validators.Length(min=1), wtforms.validators.Optional()],
+        description="Required for Javascript error "
+        "tracking (one line one domain, skip http:// part)",
+    )
 
-    submit = wtforms.SubmitField(_('Create Application'))
+    submit = wtforms.SubmitField(_("Create Application"))
 
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
-    html_attrs = {'resource_name': {'placeholder': 'Application Name'},
-                  'uptime_url': {'placeholder': 'http://somedomain.com'}}
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
+    html_attrs = {
+        "resource_name": {"placeholder": "Application Name"},
+        "uptime_url": {"placeholder": "http://somedomain.com"},
+    }
 
 
 class ApplicationUpdateForm(ApplicationCreateForm):
     default_grouping = wtforms.SelectField(
-        _('Default grouping for errors'),
-        choices=[('url_type', 'Error Type + location',),
-                 ('url_traceback', 'Traceback + location',),
-                 ('traceback_server', 'Traceback + Server',)],
-        default='url_traceback')
+        _("Default grouping for errors"),
+        choices=[
+            ("url_type", "Error Type + location"),
+            ("url_traceback", "Traceback + location"),
+            ("traceback_server", "Traceback + Server"),
+        ],
+        default="url_traceback",
+    )
 
     error_report_threshold = wtforms.IntegerField(
-        _('Alert on error reports'),
+        _("Alert on error reports"),
         validators=[
             wtforms.validators.NumberRange(min=1),
-            wtforms.validators.DataRequired()
+            wtforms.validators.DataRequired(),
         ],
-        description='Application requires to send at least this amount of '
-                    'error reports per minute to open alert'
+        description="Application requires to send at least this amount of "
+        "error reports per minute to open alert",
     )
 
     slow_report_threshold = wtforms.IntegerField(
-        _('Alert on slow reports'),
-        validators=[wtforms.validators.NumberRange(min=1),
-                    wtforms.validators.DataRequired()],
-        description='Application requires to send at least this amount of '
-                    'slow reports per minute to open alert')
+        _("Alert on slow reports"),
+        validators=[
+            wtforms.validators.NumberRange(min=1),
+            wtforms.validators.DataRequired(),
+        ],
+        description="Application requires to send at least this amount of "
+        "slow reports per minute to open alert",
+    )
 
     allow_permanent_storage = wtforms.BooleanField(
-        _('Permanent logs'),
+        _("Permanent logs"),
         false_values=FALSE_VALUES,
-        description=_(
-            'Allow permanent storage of logs in separate DB partitions'))
+        description=_("Allow permanent storage of logs in separate DB partitions"),
+    )
 
-    submit = wtforms.SubmitField(_('Create Application'))
+    submit = wtforms.SubmitField(_("Create Application"))
 
 
 class UserSearchSchemaForm(ReactorForm):
-    user_name = wtforms.StringField('User Name',
-                                    filters=[strip_filter], )
+    user_name = wtforms.StringField("User Name", filters=[strip_filter])
 
-    submit = wtforms.SubmitField(_('Search User'))
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
+    submit = wtforms.SubmitField(_("Search User"))
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
 
     '<li class="user_exists"><span></span></li>'
 
 
 class YesNoForm(ReactorForm):
-    no = wtforms.SubmitField('No', default='')
-    yes = wtforms.SubmitField('Yes', default='')
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
+    no = wtforms.SubmitField("No", default="")
+    yes = wtforms.SubmitField("Yes", default="")
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
 
 
-status_codes = [('', 'All',), ('500', '500',), ('404', '404',)]
+status_codes = [("", "All"), ("500", "500"), ("404", "404")]
 
-priorities = [('', 'All',)]
+priorities = [("", "All")]
 for i in range(1, 11):
-    priorities.append((str(i), str(i),))
+    priorities.append((str(i), str(i)))
 
-report_status_choices = [('', 'All',),
-                         ('never_reviewed', 'Never revieved',),
-                         ('reviewed', 'Revieved',),
-                         ('public', 'Public',),
-                         ('fixed', 'Fixed',), ]
+report_status_choices = [
+    ("", "All"),
+    ("never_reviewed", "Never revieved"),
+    ("reviewed", "Revieved"),
+    ("public", "Public"),
+    ("fixed", "Fixed"),
+]
 
 
 class ReportBrowserForm(ReactorForm):
-    applications = wtforms.SelectMultipleField('Applications',
-                                               widget=select_multi_checkbox)
-    http_status = wtforms.SelectField('HTTP Status', choices=status_codes)
-    priority = wtforms.SelectField('Priority', choices=priorities, default='')
-    start_date = wtforms.DateField('Start Date')
-    end_date = wtforms.DateField('End Date')
-    error = wtforms.StringField('Error')
-    url_path = wtforms.StringField('URL Path')
-    url_domain = wtforms.StringField('URL Domain')
-    report_status = wtforms.SelectField('Report status',
-                                        choices=report_status_choices,
-                                        default='')
-    submit = wtforms.SubmitField('<span class="glyphicon glyphicon-search">'
-                                 '</span> Filter results',
-                                 widget=button_widget)
-
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
-
-
-slow_report_status_choices = [('', 'All',),
-                              ('never_reviewed', 'Never revieved',),
-                              ('reviewed', 'Revieved',),
-                              ('public', 'Public',), ]
+    applications = wtforms.SelectMultipleField(
+        "Applications", widget=select_multi_checkbox
+    )
+    http_status = wtforms.SelectField("HTTP Status", choices=status_codes)
+    priority = wtforms.SelectField("Priority", choices=priorities, default="")
+    start_date = wtforms.DateField("Start Date")
+    end_date = wtforms.DateField("End Date")
+    error = wtforms.StringField("Error")
+    url_path = wtforms.StringField("URL Path")
+    url_domain = wtforms.StringField("URL Domain")
+    report_status = wtforms.SelectField(
+        "Report status", choices=report_status_choices, default=""
+    )
+    submit = wtforms.SubmitField(
+        '<span class="glyphicon glyphicon-search">' "</span> Filter results",
+        widget=button_widget,
+    )
+
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
+
+
+slow_report_status_choices = [
+    ("", "All"),
+    ("never_reviewed", "Never revieved"),
+    ("reviewed", "Revieved"),
+    ("public", "Public"),
+]
 
 
 class BulkOperationForm(ReactorForm):
-    applications = wtforms.SelectField('Applications')
+    applications = wtforms.SelectField("Applications")
     start_date = wtforms.DateField(
-        'Start Date',
-        default=lambda: datetime.datetime.utcnow() - datetime.timedelta(
-            days=90))
-    end_date = wtforms.DateField('End Date')
+        "Start Date",
+        default=lambda: datetime.datetime.utcnow() - datetime.timedelta(days=90),
+    )
+    end_date = wtforms.DateField("End Date")
     confirm = wtforms.BooleanField(
-        'Confirm operation',
-        validators=[wtforms.validators.DataRequired()])
+        "Confirm operation", validators=[wtforms.validators.DataRequired()]
+    )
 
 
 class LogBrowserForm(ReactorForm):
-    applications = wtforms.SelectMultipleField('Applications',
-                                               widget=select_multi_checkbox)
-    start_date = wtforms.DateField('Start Date')
-    log_level = wtforms.StringField('Log level')
-    message = wtforms.StringField('Message')
-    namespace = wtforms.StringField('Namespace')
+    applications = wtforms.SelectMultipleField(
+        "Applications", widget=select_multi_checkbox
+    )
+    start_date = wtforms.DateField("Start Date")
+    log_level = wtforms.StringField("Log level")
+    message = wtforms.StringField("Message")
+    namespace = wtforms.StringField("Namespace")
     submit = wtforms.SubmitField(
         '<span class="glyphicon glyphicon-search"></span> Filter results',
-        widget=button_widget)
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
+        widget=button_widget,
+    )
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
 
 
 class CommentForm(ReactorForm):
-    body = wtforms.TextAreaField('Comment', validators=[
-        wtforms.validators.Length(min=1),
-        wtforms.validators.DataRequired()
-    ])
-    submit = wtforms.SubmitField('Comment', )
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
+    body = wtforms.TextAreaField(
+        "Comment",
+        validators=[
+            wtforms.validators.Length(min=1),
+            wtforms.validators.DataRequired(),
+        ],
+    )
+    submit = wtforms.SubmitField("Comment")
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
 
 
 class EmailChannelCreateForm(ReactorForm):
-    email = wtforms.StringField(_('Email Address'),
-                                filters=[strip_filter],
-                                validators=[email_validator,
-                                            unique_alert_email_validator,
-                                            wtforms.validators.DataRequired()])
-    submit = wtforms.SubmitField('Add email channel', )
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
+    email = wtforms.StringField(
+        _("Email Address"),
+        filters=[strip_filter],
+        validators=[
+            email_validator,
+            unique_alert_email_validator,
+            wtforms.validators.DataRequired(),
+        ],
+    )
+    submit = wtforms.SubmitField("Add email channel")
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
 
 
 def gen_user_profile_form():
     class UserProfileForm(ReactorForm):
         email = wtforms.StringField(
-            _('Email Address'),
-            validators=[email_validator, wtforms.validators.DataRequired()])
-        first_name = wtforms.StringField(_('First Name'))
-        last_name = wtforms.StringField(_('Last Name'))
-        company_name = wtforms.StringField(_('Company Name'))
-        company_address = wtforms.TextAreaField(_('Company Address'))
-        zip_code = wtforms.StringField(_('ZIP code'))
-        city = wtforms.StringField(_('City'))
-        notifications = wtforms.BooleanField('Account notifications',
-                                             false_values=FALSE_VALUES)
-        submit = wtforms.SubmitField(_('Update Account'))
-        ignore_labels = ['submit']
-        css_classes = {'submit': 'btn btn-primary'}
+            _("Email Address"),
+            validators=[email_validator, wtforms.validators.DataRequired()],
+        )
+        first_name = wtforms.StringField(_("First Name"))
+        last_name = wtforms.StringField(_("Last Name"))
+        company_name = wtforms.StringField(_("Company Name"))
+        company_address = wtforms.TextAreaField(_("Company Address"))
+        zip_code = wtforms.StringField(_("ZIP code"))
+        city = wtforms.StringField(_("City"))
+        notifications = wtforms.BooleanField(
+            "Account notifications", false_values=FALSE_VALUES
+        )
+        submit = wtforms.SubmitField(_("Update Account"))
+        ignore_labels = ["submit"]
+        css_classes = {"submit": "btn btn-primary"}
 
     return UserProfileForm
 
 
 class PurgeAppForm(ReactorForm):
     resource_id = wtforms.HiddenField(
-        'App Id',
-        validators=[wtforms.validators.DataRequired()])
-    days = wtforms.IntegerField(
-        'Days',
-        validators=[wtforms.validators.DataRequired()])
+        "App Id", validators=[wtforms.validators.DataRequired()]
+    )
+    days = wtforms.IntegerField("Days", validators=[wtforms.validators.DataRequired()])
     password = wtforms.PasswordField(
-        'Admin Password',
-        validators=[old_password_validator, wtforms.validators.DataRequired()])
-    submit = wtforms.SubmitField(_('Purge Data'))
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
+        "Admin Password",
+        validators=[old_password_validator, wtforms.validators.DataRequired()],
+    )
+    submit = wtforms.SubmitField(_("Purge Data"))
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
 
 
 class IntegrationRepoForm(ReactorForm):
-    host_name = wtforms.StringField("Service Host", default='')
+    host_name = wtforms.StringField("Service Host", default="")
     user_name = wtforms.StringField(
         "User Name",
         filters=[strip_filter],
-        validators=[wtforms.validators.DataRequired(),
-                    wtforms.validators.Length(min=1)])
+        validators=[
+            wtforms.validators.DataRequired(),
+            wtforms.validators.Length(min=1),
+        ],
+    )
     repo_name = wtforms.StringField(
         "Repo Name",
         filters=[strip_filter],
-        validators=[wtforms.validators.DataRequired(),
-                    wtforms.validators.Length(min=1)])
+        validators=[
+            wtforms.validators.DataRequired(),
+            wtforms.validators.Length(min=1),
+        ],
+    )
 
 
 class IntegrationBitbucketForm(IntegrationRepoForm):
-    host_name = wtforms.StringField("Service Host",
-                                    default='https://bitbucket.org')
+    host_name = wtforms.StringField("Service Host", default="https://bitbucket.org")
 
     def validate_user_name(self, field):
         try:
             request = pyramid.threadlocal.get_current_request()
             client = BitbucketIntegration.create_client(
-                request,
-                self.user_name.data,
-                self.repo_name.data)
+                request, self.user_name.data, self.repo_name.data
+            )
             client.get_assignees()
         except IntegrationException as e:
             raise wtforms.validators.ValidationError(str(e))
 
 
 class IntegrationGithubForm(IntegrationRepoForm):
-    host_name = wtforms.StringField("Service Host",
-                                    default='https://github.com')
+    host_name = wtforms.StringField("Service Host", default="https://github.com")
 
     def validate_user_name(self, field):
         try:
             request = pyramid.threadlocal.get_current_request()
             client = GithubIntegration.create_client(
-                request,
-                self.user_name.data,
-                self.repo_name.data)
+                request, self.user_name.data, self.repo_name.data
+            )
             client.get_assignees()
         except IntegrationException as e:
             raise wtforms.validators.ValidationError(str(e))
             raise wtforms.validators.ValidationError(str(e))
 
 
 def filter_rooms(data):
     if data is not None:
-        rooms = data.split(',')
-        return ','.join([r.strip() for r in rooms])
+        rooms = data.split(",")
+        return ",".join([r.strip() for r in rooms])
 
 
 class IntegrationCampfireForm(ReactorForm):
     account = wtforms.StringField(
-        'Account',
+        "Account",
         filters=[strip_filter],
-        validators=[wtforms.validators.DataRequired()])
+        validators=[wtforms.validators.DataRequired()],
+    )
     api_token = wtforms.StringField(
-        'Api Token',
+        "Api Token",
         filters=[strip_filter],
-        validators=[wtforms.validators.DataRequired()])
-    rooms = wtforms.StringField('Room ID list', filters=[filter_rooms])
+        validators=[wtforms.validators.DataRequired()],
+    )
+    rooms = wtforms.StringField("Room ID list", filters=[filter_rooms])
 
     def validate_api_token(self, field):
         try:
-            client = CampfireIntegration.create_client(self.api_token.data,
-                                                       self.account.data)
+            client = CampfireIntegration.create_client(
+                self.api_token.data, self.account.data
+            )
             client.get_account()
         except IntegrationException as e:
             raise wtforms.validators.ValidationError(str(e))
 
     def validate_rooms(self, field):
         if not field.data:
             return
-        client = CampfireIntegration.create_client(self.api_token.data,
-                                                   self.account.data)
+        client = CampfireIntegration.create_client(
+            self.api_token.data, self.account.data
+        )
 
         try:
-            room_list = [r['id'] for r in client.get_rooms()]
+            room_list = [r["id"] for r in client.get_rooms()]
         except IntegrationException as e:
             raise wtforms.validators.ValidationError(str(e))
 
-        rooms = field.data.split(',')
+        rooms = field.data.split(",")
         if len(rooms) > 3:
-            msg = 'You can use up to 3 room ids'
+            msg = "You can use up to 3 room ids"
             raise wtforms.validators.ValidationError(msg)
         if rooms:
             for room_id in rooms:
                 if int(room_id) not in room_list:
                     msg = "Room %s doesn't exist"
                     raise wtforms.validators.ValidationError(msg % room_id)
                 if not room_id.strip().isdigit():
-                    msg = 'You must use only integers for room ids'
+                    msg = "You must use only integers for room ids"
                     raise wtforms.validators.ValidationError(msg)
 
-    submit = wtforms.SubmitField(_('Connect to Campfire'))
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
+    submit = wtforms.SubmitField(_("Connect to Campfire"))
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
 
 
 def filter_rooms(data):
     if data is not None:
-        rooms = data.split(',')
-        return ','.join([r.strip() for r in rooms])
+        rooms = data.split(",")
+        return ",".join([r.strip() for r in rooms])
 
 
 class IntegrationHipchatForm(ReactorForm):
     api_token = wtforms.StringField(
-        'Api Token',
+        "Api Token",
         filters=[strip_filter],
-        validators=[wtforms.validators.DataRequired()])
+        validators=[wtforms.validators.DataRequired()],
+    )
     rooms = wtforms.StringField(
-        'Room ID list',
+        "Room ID list",
         filters=[filter_rooms],
-        validators=[wtforms.validators.DataRequired()])
+        validators=[wtforms.validators.DataRequired()],
+    )
 
     def validate_rooms(self, field):
         if not field.data:
             return
         client = HipchatIntegration.create_client(self.api_token.data)
-        rooms = field.data.split(',')
+        rooms = field.data.split(",")
         if len(rooms) > 3:
-            msg = 'You can use up to 3 room ids'
+            msg = "You can use up to 3 room ids"
             raise wtforms.validators.ValidationError(msg)
         if rooms:
             for room_id in rooms:
                 if not room_id.strip().isdigit():
-                    msg = 'You must use only integers for room ids'
+                    msg = "You must use only integers for room ids"
                     raise wtforms.validators.ValidationError(msg)
                 try:
-                    client.send({
-                        "message_format": 'text',
-                        "message": "testing for room existence",
-                        "from": "AppEnlight",
-                        "room_id": room_id,
-                        "color": "green"
-                    })
+                    client.send(
+                        {
+                            "message_format": "text",
+                            "message": "testing for room existence",
+                            "from": "AppEnlight",
+                            "room_id": room_id,
+                            "color": "green",
+                        }
+                    )
                 except IntegrationException as exc:
-                    msg = 'Room id: %s exception: %s'
-                    raise wtforms.validators.ValidationError(msg % (room_id,
-                                                                    exc))
+                    msg = "Room id: %s exception: %s"
+                    raise wtforms.validators.ValidationError(msg % (room_id, exc))
 
 
 class IntegrationFlowdockForm(ReactorForm):
-    api_token = wtforms.StringField('API Token',
-                                    filters=[strip_filter],
-                                    validators=[
-                                        wtforms.validators.DataRequired()
-                                    ], )
+    api_token = wtforms.StringField(
+        "API Token",
+        filters=[strip_filter],
+        validators=[wtforms.validators.DataRequired()],
+    )
 
     def validate_api_token(self, field):
         try:
             client = FlowdockIntegration.create_client(self.api_token.data)
             registry = pyramid.threadlocal.get_current_registry()
             payload = {
-                "source": registry.settings['mailing.from_name'],
-                "from_address": registry.settings['mailing.from_email'],
+                "source": registry.settings["mailing.from_name"],
+                "from_address": registry.settings["mailing.from_email"],
                 "subject": "Integration test",
                 "content": "If you can see this it was successful",
                 "tags": ["appenlight"],
-                "link": registry.settings['mailing.app_url']
+                "link": registry.settings["mailing.app_url"],
             }
             client.send_to_inbox(payload)
         except IntegrationException as e:
             raise wtforms.validators.ValidationError(str(e))
 
 
 class IntegrationSlackForm(ReactorForm):
     webhook_url = wtforms.StringField(
-        'Reports webhook',
+        "Reports webhook",
         filters=[strip_filter],
-        validators=[wtforms.validators.DataRequired()])
+        validators=[wtforms.validators.DataRequired()],
+    )
 
     def validate_webhook_url(self, field):
         registry = pyramid.threadlocal.get_current_registry()
         client = SlackIntegration.create_client(field.data)
-        link = "<%s|%s>" % (registry.settings['mailing.app_url'],
-                            registry.settings['mailing.from_name'])
+        link = "<%s|%s>" % (
+            registry.settings["mailing.app_url"],
+            registry.settings["mailing.from_name"],
+        )
         test_data = {
             "username": "AppEnlight",
             "icon_emoji": ":fire:",
             "attachments": [
-                {"fallback": "Testing integration channel: %s" % link,
-                 "pretext": "Testing integration channel:  %s" % link,
-                 "color": "good",
-                 "fields": [
-                     {
-                         "title": "Status",
-                         "value": "Integration is working fine",
-                         "short": False
-                     }
-                 ]}
-            ]
+                {
+                    "fallback": "Testing integration channel: %s" % link,
+                    "pretext": "Testing integration channel:  %s" % link,
+                    "color": "good",
+                    "fields": [
+                        {
+                            "title": "Status",
+                            "value": "Integration is working fine",
+                            "short": False,
+                        }
+                    ],
+                }
+            ],
         }
         try:
             client.make_request(data=test_data)
         except IntegrationException as exc:
             raise wtforms.validators.ValidationError(str(exc))
 
 
 class IntegrationWebhooksForm(ReactorForm):
     reports_webhook = wtforms.StringField(
-        'Reports webhook',
+        "Reports webhook",
         filters=[strip_filter],
-        validators=[wtforms.validators.DataRequired()])
+        validators=[wtforms.validators.DataRequired()],
+    )
     alerts_webhook = wtforms.StringField(
-        'Alerts webhook',
+        "Alerts webhook",
         filters=[strip_filter],
-        validators=[wtforms.validators.DataRequired()])
-    submit = wtforms.SubmitField(_('Setup webhooks'))
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-primary'}
+        validators=[wtforms.validators.DataRequired()],
+    )
+    submit = wtforms.SubmitField(_("Setup webhooks"))
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-primary"}
 
 
 class IntegrationJiraForm(ReactorForm):
     host_name = wtforms.StringField(
-        'Server URL',
+        "Server URL",
         filters=[strip_filter],
-        validators=[wtforms.validators.DataRequired()])
+        validators=[wtforms.validators.DataRequired()],
+    )
     user_name = wtforms.StringField(
-        'Username',
+        "Username",
         filters=[strip_filter],
-        validators=[wtforms.validators.DataRequired()])
+        validators=[wtforms.validators.DataRequired()],
+    )
     password = wtforms.PasswordField(
-        'Password',
+        "Password",
         filters=[strip_filter],
-        validators=[wtforms.validators.DataRequired()])
+        validators=[wtforms.validators.DataRequired()],
+    )
     project = wtforms.StringField(
-        'Project key',
+        "Project key",
         filters=[uppercase_filter, strip_filter],
-        validators=[wtforms.validators.DataRequired()])
+        validators=[wtforms.validators.DataRequired()],
+    )
 
     def validate_project(self, field):
         if not field.data:
             return
         try:
-            client = JiraClient(self.user_name.data,
-                                self.password.data,
-                                self.host_name.data,
-                                self.project.data)
+            client = JiraClient(
+                self.user_name.data,
+                self.password.data,
+                self.host_name.data,
+                self.project.data,
+            )
         except Exception as exc:
             raise wtforms.validators.ValidationError(str(exc))
 
         room_list = [r.key.upper() for r in client.get_projects()]
         if field.data.upper() not in room_list:
             msg = "Project %s doesn\t exist in your Jira Instance"
             raise wtforms.validators.ValidationError(msg % field.data)
 
 
 def get_deletion_form(resource):
     class F(ReactorForm):
         application_name = wtforms.StringField(
-            'Application Name',
+            "Application Name",
             filters=[strip_filter],
-            validators=[wtforms.validators.AnyOf([resource.resource_name])])
+            validators=[wtforms.validators.AnyOf([resource.resource_name])],
+        )
         resource_id = wtforms.HiddenField(default=resource.resource_id)
-        submit = wtforms.SubmitField(_('Delete my application'))
-        ignore_labels = ['submit']
-        css_classes = {'submit': 'btn btn-danger'}
+        submit = wtforms.SubmitField(_("Delete my application"))
+        ignore_labels = ["submit"]
+        css_classes = {"submit": "btn btn-danger"}
 
     return F
 
 
 class ChangeApplicationOwnerForm(ReactorForm):
     password = wtforms.PasswordField(
-        'Password',
+        "Password",
         filters=[strip_filter],
-        validators=[old_password_validator,
-                    wtforms.validators.DataRequired()])
+        validators=[old_password_validator, wtforms.validators.DataRequired()],
+    )
 
     user_name = wtforms.StringField(
-        'New owners username',
+        "New owners username",
         filters=[strip_filter],
-        validators=[found_username_validator,
-                    wtforms.validators.DataRequired()])
-    submit = wtforms.SubmitField(_('Transfer ownership of application'))
-    ignore_labels = ['submit']
-    css_classes = {'submit': 'btn btn-danger'}
+        validators=[found_username_validator, wtforms.validators.DataRequired()],
+    )
+    submit = wtforms.SubmitField(_("Transfer ownership of application"))
+    ignore_labels = ["submit"]
+    css_classes = {"submit": "btn btn-danger"}
 
 
 def default_filename():
-    return 'Invoice %s' % datetime.datetime.utcnow().strftime('%Y/%m')
+    return "Invoice %s" % datetime.datetime.utcnow().strftime("%Y/%m")
 
 
 class FileUploadForm(ReactorForm):
-    title = wtforms.StringField('File Title',
-                                default=default_filename,
-                                validators=[wtforms.validators.DataRequired()])
-    file = wtforms.FileField('File')
+    title = wtforms.StringField(
+        "File Title",
+        default=default_filename,
+        validators=[wtforms.validators.DataRequired()],
+    )
+    file = wtforms.FileField("File")
 
     def validate_file(self, field):
-        if not hasattr(field.data, 'file'):
-            raise wtforms.ValidationError('File is missing')
+        if not hasattr(field.data, "file"):
+            raise wtforms.ValidationError("File is missing")
 
-    submit = wtforms.SubmitField(_('Upload'))
+    submit = wtforms.SubmitField(_("Upload"))
 
 
 def get_partition_deletion_form(es_indices, pg_indices):
     class F(ReactorForm):
-        es_index = wtforms.SelectMultipleField('Elasticsearch',
-                                               choices=[(ix, '') for ix in
-                                                        es_indices])
-        pg_index = wtforms.SelectMultipleField('pg',
-                                               choices=[(ix, '') for ix in
-                                                        pg_indices])
-        confirm = wtforms.TextField('Confirm',
-                                    filters=[uppercase_filter, strip_filter],
-                                    validators=[
-                                        wtforms.validators.AnyOf(['CONFIRM']),
-                                        wtforms.validators.DataRequired()])
-        ignore_labels = ['submit']
-        css_classes = {'submit': 'btn btn-danger'}
+        es_index = wtforms.SelectMultipleField(
+            "Elasticsearch", choices=[(ix, "") for ix in es_indices]
+        )
+        pg_index = wtforms.SelectMultipleField(
+            "pg", choices=[(ix, "") for ix in pg_indices]
+        )
+        confirm = wtforms.TextField(
+            "Confirm",
+            filters=[uppercase_filter, strip_filter],
+            validators=[
+                wtforms.validators.AnyOf(["CONFIRM"]),
+                wtforms.validators.DataRequired(),
+            ],
+        )
+        ignore_labels = ["submit"]
+        css_classes = {"submit": "btn btn-danger"}
 
     return F
 
 
 class GroupCreateForm(ReactorForm):
     group_name = wtforms.StringField(
-        _('Group Name'),
+        _("Group Name"),
         filters=[strip_filter],
         validators=[
             wtforms.validators.Length(min=2, max=50),
             unique_groupname_validator,
-            wtforms.validators.DataRequired()
-        ])
-    description = wtforms.StringField(_('Group description'))
+            wtforms.validators.DataRequired(),
+        ],
+    )
+    description = wtforms.StringField(_("Group description"))
 
 
-time_choices = [(k, v['label'],) for k, v in h.time_deltas.items()]
+time_choices = [(k, v["label"]) for k, v in h.time_deltas.items()]
 
 
 class AuthTokenCreateForm(ReactorForm):
-    description = wtforms.StringField(_('Token description'))
-    expires = wtforms.SelectField('Expires',
-                                  coerce=lambda x: x,
-                                  choices=time_choices,
-                                  validators=[wtforms.validators.Optional()])
+    description = wtforms.StringField(_("Token description"))
+    expires = wtforms.SelectField(
+        "Expires",
+        coerce=lambda x: x,
+        choices=time_choices,
+        validators=[wtforms.validators.Optional()],
+    )

backend/src/appenlight/lib/__init__.py

@@ -1,50 +1,49 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 """Miscellaneous support packages for {{project}}.
 """
 import random
 import string
 import importlib
 
 from appenlight_client.exceptions import get_current_traceback
 
 
 def generate_random_string(chars=10):
-    return ''.join(random.sample(string.ascii_letters * 2 + string.digits,
-                                 chars))
+    return "".join(random.sample(string.ascii_letters * 2 + string.digits, chars))
 
 
 def to_integer_safe(input):
     try:
         return int(input)
-    except (TypeError, ValueError,):
+    except (TypeError, ValueError):
         return None
 
 
 def print_traceback(log):
-    traceback = get_current_traceback(skip=1, show_hidden_frames=True,
-                                      ignore_system_exceptions=True)
+    traceback = get_current_traceback(
+        skip=1, show_hidden_frames=True, ignore_system_exceptions=True
+    )
     exception_text = traceback.exception
     log.error(exception_text)
     log.error(traceback.plaintext)
     del traceback
 
 
 def get_callable(import_string):
-    import_module, indexer_callable = import_string.split(':')
-    return getattr(importlib.import_module(import_module),
-                   indexer_callable)
+    import_module, indexer_callable = import_string.split(":")
+    return getattr(importlib.import_module(import_module), indexer_callable)

backend/src/appenlight/lib/api.py

@@ -1,80 +1,80 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import datetime
 import logging
 
 from pyramid.httpexceptions import HTTPForbidden, HTTPTooManyRequests
 
 from appenlight.models.services.config import ConfigService
 from appenlight.lib.redis_keys import REDIS_KEYS
 
 log = logging.getLogger(__name__)
 
 
 def rate_limiting(request, resource, section, to_increment=1):
     tsample = datetime.datetime.utcnow().replace(second=0, microsecond=0)
-    key = REDIS_KEYS['rate_limits'][section].format(tsample,
-                                                    resource.resource_id)
+    key = REDIS_KEYS["rate_limits"][section].format(tsample, resource.resource_id)
     redis_pipeline = request.registry.redis_conn.pipeline()
     redis_pipeline.incr(key, to_increment)
     redis_pipeline.expire(key, 3600 * 24)
     results = redis_pipeline.execute()
     current_count = results[0]
-    config = ConfigService.by_key_and_section(section, 'global')
+    config = ConfigService.by_key_and_section(section, "global")
     limit = config.value if config else 1000
     if current_count > int(limit):
-        log.info('RATE LIMITING: {}: {}, {}'.format(
-            section, resource, current_count))
-        abort_msg = 'Rate limits are in effect for this application'
-        raise HTTPTooManyRequests(abort_msg,
-                                  headers={'X-AppEnlight': abort_msg})
+        log.info("RATE LIMITING: {}: {}, {}".format(section, resource, current_count))
+        abort_msg = "Rate limits are in effect for this application"
+        raise HTTPTooManyRequests(abort_msg, headers={"X-AppEnlight": abort_msg})
 
 
 def check_cors(request, application, should_return=True):
     """
     Performs a check and validation if request comes from authorized domain for
     application, otherwise return 403
     """
     origin_found = False
-    origin = request.headers.get('Origin')
+    origin = request.headers.get("Origin")
     if should_return:
-        log.info('CORS for %s' % origin)
+        log.info("CORS for %s" % origin)
     if not origin:
         return False
-    for domain in application.domains.split('\n'):
+    for domain in application.domains.split("\n"):
         if domain in origin:
             origin_found = True
     if origin_found:
-        request.response.headers.add('Access-Control-Allow-Origin', origin)
-        request.response.headers.add('XDomainRequestAllowed', '1')
-        request.response.headers.add('Access-Control-Allow-Methods',
-                                     'GET, POST, OPTIONS')
-        request.response.headers.add('Access-Control-Allow-Headers',
-                                     'Accept-Encoding, Accept-Language, '
-                                     'Content-Type, '
-                                     'Depth, User-Agent, X-File-Size, '
-                                     'X-Requested-With, If-Modified-Since, '
-                                     'X-File-Name, '
-                                     'Cache-Control, Host, Pragma, Accept, '
-                                     'Origin, Connection, '
-                                     'Referer, Cookie, '
-                                     'X-appenlight-public-api-key, '
-                                     'x-appenlight-public-api-key')
-        request.response.headers.add('Access-Control-Max-Age', '86400')
+        request.response.headers.add("Access-Control-Allow-Origin", origin)
+        request.response.headers.add("XDomainRequestAllowed", "1")
+        request.response.headers.add(
+            "Access-Control-Allow-Methods", "GET, POST, OPTIONS"
+        )
+        request.response.headers.add(
+            "Access-Control-Allow-Headers",
+            "Accept-Encoding, Accept-Language, "
+            "Content-Type, "
+            "Depth, User-Agent, X-File-Size, "
+            "X-Requested-With, If-Modified-Since, "
+            "X-File-Name, "
+            "Cache-Control, Host, Pragma, Accept, "
+            "Origin, Connection, "
+            "Referer, Cookie, "
+            "X-appenlight-public-api-key, "
+            "x-appenlight-public-api-key",
+        )
+        request.response.headers.add("Access-Control-Max-Age", "86400")
         return request.response
     else:
         return HTTPForbidden()

backend/src/appenlight/lib/cache_regions.py

@@ -1,184 +1,169 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import copy
 import hashlib
 import inspect
 
 from dogpile.cache import make_region
 from dogpile.cache.util import compat
 
 regions = None
 
 
 def key_mangler(key):
     return "appenlight:dogpile:{}".format(key)
 
 
 def hashgen(namespace, fn, to_str=compat.string_type):
     """Return a function that generates a string
     key, based on a given function as well as
     arguments to the returned function itself.
 
     This is used by :meth:`.CacheRegion.cache_on_arguments`
     to generate a cache key from a decorated function.
 
     It can be replaced using the ``function_key_generator``
     argument passed to :func:`.make_region`.
 
     """
 
     if namespace is None:
-        namespace = '%s:%s' % (fn.__module__, fn.__name__)
+        namespace = "%s:%s" % (fn.__module__, fn.__name__)
     else:
-        namespace = '%s:%s|%s' % (fn.__module__, fn.__name__, namespace)
+        namespace = "%s:%s|%s" % (fn.__module__, fn.__name__, namespace)
 
     args = inspect.getargspec(fn)
-    has_self = args[0] and args[0][0] in ('self', 'cls')
+    has_self = args[0] and args[0][0] in ("self", "cls")
 
     def generate_key(*args, **kw):
         if kw:
             raise ValueError(
                 "dogpile.cache's default key creation "
-                "function does not accept keyword arguments.")
+                "function does not accept keyword arguments."
+            )
         if has_self:
             args = args[1:]
 
-        return namespace + "|" + hashlib.sha1(
-            " ".join(map(to_str, args)).encode('utf8')).hexdigest()
+        return (
+            namespace
+            + "|"
+            + hashlib.sha1(" ".join(map(to_str, args)).encode("utf8")).hexdigest()
+        )
 
     return generate_key
 
 
 class CacheRegions(object):
     def __init__(self, settings):
         config_redis = {"arguments": settings}
 
         self.redis_min_1 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.redis",
-            expiration_time=60,
-            **copy.deepcopy(config_redis))
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure(
+            "dogpile.cache.redis", expiration_time=60, **copy.deepcopy(config_redis)
+        )
         self.redis_min_5 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.redis",
-            expiration_time=300,
-            **copy.deepcopy(config_redis))
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure(
+            "dogpile.cache.redis", expiration_time=300, **copy.deepcopy(config_redis)
+        )
 
         self.redis_min_10 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.redis",
-            expiration_time=60,
-            **copy.deepcopy(config_redis))
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure(
+            "dogpile.cache.redis", expiration_time=60, **copy.deepcopy(config_redis)
+        )
 
         self.redis_min_60 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.redis",
-            expiration_time=3600,
-            **copy.deepcopy(config_redis))
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure(
+            "dogpile.cache.redis", expiration_time=3600, **copy.deepcopy(config_redis)
+        )
 
         self.redis_sec_1 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.redis",
-            expiration_time=1,
-            **copy.deepcopy(config_redis))
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure(
+            "dogpile.cache.redis", expiration_time=1, **copy.deepcopy(config_redis)
+        )
 
         self.redis_sec_5 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.redis",
-            expiration_time=5,
-            **copy.deepcopy(config_redis))
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure(
+            "dogpile.cache.redis", expiration_time=5, **copy.deepcopy(config_redis)
+        )
 
         self.redis_sec_30 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.redis",
-            expiration_time=30,
-            **copy.deepcopy(config_redis))
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure(
+            "dogpile.cache.redis", expiration_time=30, **copy.deepcopy(config_redis)
+        )
 
         self.redis_day_1 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.redis",
-            expiration_time=86400,
-            **copy.deepcopy(config_redis))
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure(
+            "dogpile.cache.redis", expiration_time=86400, **copy.deepcopy(config_redis)
+        )
 
         self.redis_day_7 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure(
             "dogpile.cache.redis",
             expiration_time=86400 * 7,
-            **copy.deepcopy(config_redis))
+            **copy.deepcopy(config_redis)
+        )
 
         self.redis_day_30 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure(
             "dogpile.cache.redis",
             expiration_time=86400 * 30,
-            **copy.deepcopy(config_redis))
+            **copy.deepcopy(config_redis)
+        )
 
         self.memory_day_1 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.memory",
-            expiration_time=86400,
-            **copy.deepcopy(config_redis))
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure(
+            "dogpile.cache.memory", expiration_time=86400, **copy.deepcopy(config_redis)
+        )
 
         self.memory_sec_1 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.memory",
-            expiration_time=1)
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure("dogpile.cache.memory", expiration_time=1)
 
         self.memory_sec_5 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.memory",
-            expiration_time=5)
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure("dogpile.cache.memory", expiration_time=5)
 
         self.memory_min_1 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.memory",
-            expiration_time=60)
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure("dogpile.cache.memory", expiration_time=60)
 
         self.memory_min_5 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.memory",
-            expiration_time=300)
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure("dogpile.cache.memory", expiration_time=300)
 
         self.memory_min_10 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.memory",
-            expiration_time=600)
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure("dogpile.cache.memory", expiration_time=600)
 
         self.memory_min_60 = make_region(
-            function_key_generator=hashgen,
-            key_mangler=key_mangler).configure(
-            "dogpile.cache.memory",
-            expiration_time=3600)
+            function_key_generator=hashgen, key_mangler=key_mangler
+        ).configure("dogpile.cache.memory", expiration_time=3600)
 
 
 def get_region(region):
     return getattr(regions, region)

backend/src/appenlight/lib/configurator.py

@@ -1,92 +1,92 b''
 import inspect
 import logging
 
 from pyramid.config import Configurator
 
 log = logging.getLogger(__name__)
 
+
 class InspectProxy(object):
     """
     Proxy to the `inspect` module that allows us to use the pyramid include
     mechanism for cythonized modules without source file.
     """
 
     def _get_cyfunction_func_code(self, cyfunction):
         """
         Unpack the `func_code` attribute of a cython function.
         """
         if inspect.ismethod(cyfunction):
             cyfunction = cyfunction.im_func
-        return getattr(cyfunction, 'func_code')
+        return getattr(cyfunction, "func_code")
 
     def getmodule(self, *args, **kwds):
         """
         Simple proxy to `inspect.getmodule`.
         """
         return inspect.getmodule(*args, **kwds)
 
     def getsourcefile(self, obj):
         """
         Proxy to `inspect.getsourcefile` or `inspect.getfile` depending on if
         it's called to look up the source file that contains the magic pyramid
         `includeme` callable.
 
         For cythonized modules the source file may be deleted. Therefore we
         return the result of `inspect.getfile` instead. In the case of the
         `configurator.include` method this is OK, because the result is passed
         to `os.path.dirname` which strips the file name. So it doesn't matter
         if we return the path to the source file or another file in the same
         directory.
         """
         # Check if it's called to look up the source file that contains the
         # magic pyramid `includeme` callable.
-        if getattr(obj, '__name__') == 'includeme':
+        if getattr(obj, "__name__") == "includeme":
             try:
                 return inspect.getfile(obj)
             except TypeError as e:
                 # Cython functions are not recognized as functions by the
                 # inspect module. We have to unpack the func_code attribute
                 # ourself.
-                if 'cyfunction' in e.message:
+                if "cyfunction" in e.message:
                     obj = self._get_cyfunction_func_code(obj)
                     return inspect.getfile(obj)
                 raise
         else:
             return inspect.getsourcefile(obj)
 
 
 class CythonCompatConfigurator(Configurator):
     """
     Customized configurator to replace the inspect class attribute with
     a custom one that is cython compatible.
     """
+
     inspect = InspectProxy()
 
 
 def register_appenlight_plugin(config, plugin_name, plugin_config):
     def register():
-        log.warning('Registering plugin: {}'.format(plugin_name))
+        log.warning("Registering plugin: {}".format(plugin_name))
         if plugin_name not in config.registry.appenlight_plugins:
             config.registry.appenlight_plugins[plugin_name] = {
-                'javascript': None,
-                'static': None,
-                'css': None,
-                'celery_tasks': None,
-                'celery_beats': None,
-                'fulltext_indexer': None,
-                'sqlalchemy_migrations': None,
-                'default_values_setter': None,
-                'header_html': None,
-                'resource_types': [],
-                'url_gen': None
+                "javascript": None,
+                "static": None,
+                "css": None,
+                "celery_tasks": None,
+                "celery_beats": None,
+                "fulltext_indexer": None,
+                "sqlalchemy_migrations": None,
+                "default_values_setter": None,
+                "header_html": None,
+                "resource_types": [],
+                "url_gen": None,
             }
-        config.registry.appenlight_plugins[plugin_name].update(
-            plugin_config)
+        config.registry.appenlight_plugins[plugin_name].update(plugin_config)
         # inform AE what kind of resource types we have available
         # so we can avoid failing when a plugin is removed but data
         # is still present in the db
-        if plugin_config.get('resource_types'):
-            config.registry.resource_types.extend(
-                plugin_config['resource_types'])
+        if plugin_config.get("resource_types"):
+            config.registry.resource_types.extend(plugin_config["resource_types"])
 
-    config.action('appenlight_plugin={}'.format(plugin_name), register)
+    config.action("appenlight_plugin={}".format(plugin_name), register)

backend/src/appenlight/lib/encryption.py

@@ -1,58 +1,58 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 # this gets set on runtime
 from cryptography.fernet import Fernet
 
 ENCRYPTION_SECRET = None
 
 
 def encrypt_fernet(value):
     # avoid double encryption
     # not sure if this is needed but it won't hurt too much to have this
-    if value.startswith('enc$fernet$'):
+    if value.startswith("enc$fernet$"):
         return value
     f = Fernet(ENCRYPTION_SECRET)
-    return 'enc$fernet${}'.format(f.encrypt(value.encode('utf8')).decode('utf8'))
+    return "enc$fernet${}".format(f.encrypt(value.encode("utf8")).decode("utf8"))
 
 
 def decrypt_fernet(value):
-    parts = value.split('$', 3)
+    parts = value.split("$", 3)
     if not len(parts) == 3:
         # not encrypted values
         return value
     else:
         f = Fernet(ENCRYPTION_SECRET)
-        decrypted_data = f.decrypt(parts[2].encode('utf8')).decode('utf8')
+        decrypted_data = f.decrypt(parts[2].encode("utf8")).decode("utf8")
         return decrypted_data
 
 
 def encrypt_dictionary_keys(_dict, exclude_keys=None):
     if not exclude_keys:
         exclude_keys = []
     keys = [k for k in _dict.keys() if k not in exclude_keys]
     for k in keys:
         _dict[k] = encrypt_fernet(_dict[k])
     return _dict
 
 
 def decrypt_dictionary_keys(_dict, exclude_keys=None):
     if not exclude_keys:
         exclude_keys = []
     keys = [k for k in _dict.keys() if k not in exclude_keys]
     for k in keys:
         _dict[k] = decrypt_fernet(_dict[k])
     return _dict

backend/src/appenlight/lib/enums.py

@@ -1,88 +1,90 b''
 import collections
+
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 
 class StupidEnum(object):
     @classmethod
     def set_inverse(cls):
         cls._inverse_values = dict(
-            (y, x) for x, y in vars(cls).items() if
-            not x.startswith('_') and not callable(y)
+            (y, x)
+            for x, y in vars(cls).items()
+            if not x.startswith("_") and not callable(y)
         )
 
     @classmethod
     def key_from_value(cls, value):
-        if not hasattr(cls, '_inverse_values'):
+        if not hasattr(cls, "_inverse_values"):
             cls.set_inverse()
         return cls._inverse_values.get(value)
 
 
 class ReportType(StupidEnum):
     unknown = 0
     error = 1
     not_found = 2
     slow = 3
 
 
 class Language(StupidEnum):
     unknown = 0
     python = 1
     javascript = 2
     java = 3
     objectivec = 4
     swift = 5
     cpp = 6
     basic = 7
     csharp = 8
     php = 9
     perl = 10
     vb = 11
     vbnet = 12
     ruby = 13
     fsharp = 14
     actionscript = 15
     go = 16
     scala = 17
     haskell = 18
     erlang = 19
     haxe = 20
     scheme = 21
 
 
 class LogLevel(StupidEnum):
     UNKNOWN = 0
     DEBUG = 2
     TRACE = 4
     INFO = 6
     WARNING = 8
     ERROR = 10
     CRITICAL = 12
     FATAL = 14
 
 
 class LogLevelPython(StupidEnum):
     CRITICAL = 50
     ERROR = 40
     WARNING = 30
     INFO = 20
     DEBUG = 10
     NOTSET = 0
 
 
 class ParsedSentryEventType(StupidEnum):
     ERROR_REPORT = 1
     LOG = 2

backend/src/appenlight/lib/ext_json.py

@@ -1,148 +1,143 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 """
 ex-json borrowed from Marcin Kuzminski
 
 source: https://secure.rhodecode.org/ext-json
 
 """
 import datetime
 import functools
 import decimal
 import imp
 
-__all__ = ['json', 'simplejson', 'stdlibjson']
+__all__ = ["json", "simplejson", "stdlibjson"]
 
 
 def _is_aware(value):
     """
     Determines if a given datetime.time is aware.
 
     The logic is described in Python's docs:
     http://docs.python.org/library/datetime.html#datetime.tzinfo
     """
-    return (value.tzinfo is not None
-            and value.tzinfo.utcoffset(value) is not None)
+    return value.tzinfo is not None and value.tzinfo.utcoffset(value) is not None
 
 
 def _obj_dump(obj):
     """
     Custom function for dumping objects to JSON, if obj has __json__ attribute
     or method defined it will be used for serialization
 
     :param obj:
     """
 
     if isinstance(obj, complex):
         return [obj.real, obj.imag]
     # See "Date Time String Format" in the ECMA-262 specification.
     # some code borrowed from django 1.4
     elif isinstance(obj, datetime.datetime):
         r = obj.isoformat()
         # if obj.microsecond:
         #     r = r[:23] + r[26:]
-        if r.endswith('+00:00'):
-            r = r[:-6] + 'Z'
+        if r.endswith("+00:00"):
+            r = r[:-6] + "Z"
         return r
     elif isinstance(obj, datetime.date):
         return obj.isoformat()
     elif isinstance(obj, decimal.Decimal):
         return str(obj)
     elif isinstance(obj, datetime.time):
         if _is_aware(obj):
             raise ValueError("JSON can't represent timezone-aware times.")
         r = obj.isoformat()
         if obj.microsecond:
             r = r[:12]
         return r
     elif isinstance(obj, set):
         return list(obj)
-    elif hasattr(obj, '__json__'):
+    elif hasattr(obj, "__json__"):
         if callable(obj.__json__):
             return obj.__json__()
         else:
             return obj.__json__
     else:
         raise NotImplementedError
 
 
 # Import simplejson
 try:
     # import simplejson initially
-    _sj = imp.load_module('_sj', *imp.find_module('simplejson'))
-
+    _sj = imp.load_module("_sj", *imp.find_module("simplejson"))
 
     def extended_encode(obj):
         try:
             return _obj_dump(obj)
         except NotImplementedError:
             pass
         raise TypeError("%r is not JSON serializable" % (obj,))
 
-
     # we handle decimals our own it makes unified behavior of json vs
     # simplejson
-    sj_version = [int(x) for x in _sj.__version__.split('.')]
+    sj_version = [int(x) for x in _sj.__version__.split(".")]
     major, minor = sj_version[0], sj_version[1]
     if major < 2 or (major == 2 and minor < 1):
         # simplejson < 2.1 doesnt support use_decimal
-        _sj.dumps = functools.partial(
-            _sj.dumps, default=extended_encode)
-        _sj.dump = functools.partial(
-            _sj.dump, default=extended_encode)
+        _sj.dumps = functools.partial(_sj.dumps, default=extended_encode)
+        _sj.dump = functools.partial(_sj.dump, default=extended_encode)
     else:
         _sj.dumps = functools.partial(
-            _sj.dumps, default=extended_encode, use_decimal=False)
+            _sj.dumps, default=extended_encode, use_decimal=False
+        )
         _sj.dump = functools.partial(
-            _sj.dump, default=extended_encode, use_decimal=False)
+            _sj.dump, default=extended_encode, use_decimal=False
+        )
     simplejson = _sj
 
 except ImportError:
     # no simplejson set it to None
     simplejson = None
 
 try:
     # simplejson not found try out regular json module
-    _json = imp.load_module('_json', *imp.find_module('json'))
-
+    _json = imp.load_module("_json", *imp.find_module("json"))
 
     # extended JSON encoder for json
     class ExtendedEncoder(_json.JSONEncoder):
         def default(self, obj):
             try:
                 return _obj_dump(obj)
             except NotImplementedError:
                 pass
             raise TypeError("%r is not JSON serializable" % (obj,))
 
-
     # monkey-patch JSON encoder to use extended version
     _json.dumps = functools.partial(_json.dumps, cls=ExtendedEncoder)
     _json.dump = functools.partial(_json.dump, cls=ExtendedEncoder)
 
 except ImportError:
     json = None
 
 stdlibjson = _json
 
 # set all available json modules
 if simplejson:
     json = _sj
 elif _json:
     json = _json
 else:
-    raise ImportError('Could not find any json modules')
+    raise ImportError("Could not find any json modules")

backend/src/appenlight/lib/helpers.py

@@ -1,119 +1,160 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 """
 Helper functions
 """
 import copy
 import datetime
 
 from collections import namedtuple, OrderedDict
 
 _ = lambda x: x
 
 time_deltas = OrderedDict()
 
-time_deltas['1m'] = {'delta': datetime.timedelta(minutes=1),
-                     'label': '1 minute', 'minutes': 1}
-
-time_deltas['5m'] = {'delta': datetime.timedelta(minutes=5),
-                     'label': '5 minutes', 'minutes': 5}
-time_deltas['30m'] = {'delta': datetime.timedelta(minutes=30),
-                      'label': '30 minutes', 'minutes': 30}
-time_deltas['1h'] = {'delta': datetime.timedelta(hours=1),
-                     'label': '60 minutes', 'minutes': 60}
-time_deltas['4h'] = {'delta': datetime.timedelta(hours=4), 'label': '4 hours',
-                     'minutes': 60 * 4}
-time_deltas['12h'] = {'delta': datetime.timedelta(hours=12),
-                      'label': '12 hours', 'minutes': 60 * 12}
-time_deltas['24h'] = {'delta': datetime.timedelta(hours=24),
-                      'label': '24 hours', 'minutes': 60 * 24}
-time_deltas['3d'] = {'delta': datetime.timedelta(days=3), 'label': '3 days',
-                     'minutes': 60 * 24 * 3}
-time_deltas['1w'] = {'delta': datetime.timedelta(days=7), 'label': '7 days',
-                     'minutes': 60 * 24 * 7}
-time_deltas['2w'] = {'delta': datetime.timedelta(days=14), 'label': '14 days',
-                     'minutes': 60 * 24 * 14}
-time_deltas['1M'] = {'delta': datetime.timedelta(days=31), 'label': '31 days',
-                     'minutes': 60 * 24 * 31}
-time_deltas['3M'] = {'delta': datetime.timedelta(days=31 * 3),
-                     'label': '3 months',
-                     'minutes': 60 * 24 * 31 * 3}
-time_deltas['6M'] = {'delta': datetime.timedelta(days=31 * 6),
-                     'label': '6 months',
-                     'minutes': 60 * 24 * 31 * 6}
-time_deltas['12M'] = {'delta': datetime.timedelta(days=31 * 12),
-                      'label': '12 months',
-                      'minutes': 60 * 24 * 31 * 12}
+time_deltas["1m"] = {
+    "delta": datetime.timedelta(minutes=1),
+    "label": "1 minute",
+    "minutes": 1,
+}
+
+time_deltas["5m"] = {
+    "delta": datetime.timedelta(minutes=5),
+    "label": "5 minutes",
+    "minutes": 5,
+}
+time_deltas["30m"] = {
+    "delta": datetime.timedelta(minutes=30),
+    "label": "30 minutes",
+    "minutes": 30,
+}
+time_deltas["1h"] = {
+    "delta": datetime.timedelta(hours=1),
+    "label": "60 minutes",
+    "minutes": 60,
+}
+time_deltas["4h"] = {
+    "delta": datetime.timedelta(hours=4),
+    "label": "4 hours",
+    "minutes": 60 * 4,
+}
+time_deltas["12h"] = {
+    "delta": datetime.timedelta(hours=12),
+    "label": "12 hours",
+    "minutes": 60 * 12,
+}
+time_deltas["24h"] = {
+    "delta": datetime.timedelta(hours=24),
+    "label": "24 hours",
+    "minutes": 60 * 24,
+}
+time_deltas["3d"] = {
+    "delta": datetime.timedelta(days=3),
+    "label": "3 days",
+    "minutes": 60 * 24 * 3,
+}
+time_deltas["1w"] = {
+    "delta": datetime.timedelta(days=7),
+    "label": "7 days",
+    "minutes": 60 * 24 * 7,
+}
+time_deltas["2w"] = {
+    "delta": datetime.timedelta(days=14),
+    "label": "14 days",
+    "minutes": 60 * 24 * 14,
+}
+time_deltas["1M"] = {
+    "delta": datetime.timedelta(days=31),
+    "label": "31 days",
+    "minutes": 60 * 24 * 31,
+}
+time_deltas["3M"] = {
+    "delta": datetime.timedelta(days=31 * 3),
+    "label": "3 months",
+    "minutes": 60 * 24 * 31 * 3,
+}
+time_deltas["6M"] = {
+    "delta": datetime.timedelta(days=31 * 6),
+    "label": "6 months",
+    "minutes": 60 * 24 * 31 * 6,
+}
+time_deltas["12M"] = {
+    "delta": datetime.timedelta(days=31 * 12),
+    "label": "12 months",
+    "minutes": 60 * 24 * 31 * 12,
+}
 
 # used in json representation
-time_options = dict([(k, {'label': v['label'], 'minutes': v['minutes']})
-                     for k, v in time_deltas.items()])
-FlashMsg = namedtuple('FlashMsg', ['msg', 'level'])
+time_options = dict(
+    [
+        (k, {"label": v["label"], "minutes": v["minutes"]})
+        for k, v in time_deltas.items()
+    ]
+)
+FlashMsg = namedtuple("FlashMsg", ["msg", "level"])
 
 
 def get_flash(request):
     messages = []
     messages.extend(
-        [FlashMsg(msg, 'error')
-         for msg in request.session.peek_flash('error')])
-    messages.extend([FlashMsg(msg, 'warning')
-                     for msg in request.session.peek_flash('warning')])
+        [FlashMsg(msg, "error") for msg in request.session.peek_flash("error")]
+    )
     messages.extend(
-        [FlashMsg(msg, 'notice') for msg in request.session.peek_flash()])
+        [FlashMsg(msg, "warning") for msg in request.session.peek_flash("warning")]
+    )
+    messages.extend([FlashMsg(msg, "notice") for msg in request.session.peek_flash()])
     return messages
 
 
 def clear_flash(request):
-    request.session.pop_flash('error')
-    request.session.pop_flash('warning')
+    request.session.pop_flash("error")
+    request.session.pop_flash("warning")
     request.session.pop_flash()
 
 
 def get_type_formatted_flash(request):
-    return [{'msg': message.msg, 'type': message.level}
-            for message in get_flash(request)]
+    return [
+        {"msg": message.msg, "type": message.level} for message in get_flash(request)
+    ]
 
 
 def gen_pagination_headers(request, paginator):
     headers = {
-        'x-total-count': str(paginator.item_count),
-        'x-current-page': str(paginator.page),
-        'x-items-per-page': str(paginator.items_per_page)
+        "x-total-count": str(paginator.item_count),
+        "x-current-page": str(paginator.page),
+        "x-items-per-page": str(paginator.items_per_page),
     }
     params_dict = request.GET.dict_of_lists()
     last_page_params = copy.deepcopy(params_dict)
-    last_page_params['page'] = paginator.last_page or 1
+    last_page_params["page"] = paginator.last_page or 1
     first_page_params = copy.deepcopy(params_dict)
-    first_page_params.pop('page', None)
+    first_page_params.pop("page", None)
     next_page_params = copy.deepcopy(params_dict)
-    next_page_params['page'] = paginator.next_page or paginator.last_page or 1
+    next_page_params["page"] = paginator.next_page or paginator.last_page or 1
     prev_page_params = copy.deepcopy(params_dict)
-    prev_page_params['page'] = paginator.previous_page or 1
+    prev_page_params["page"] = paginator.previous_page or 1
     lp_url = request.current_route_url(_query=last_page_params)
     fp_url = request.current_route_url(_query=first_page_params)
-    links = [
-        'rel="last", <{}>'.format(lp_url),
-        'rel="first", <{}>'.format(fp_url),
-    ]
+    links = ['rel="last", <{}>'.format(lp_url), 'rel="first", <{}>'.format(fp_url)]
     if first_page_params != prev_page_params:
         prev_url = request.current_route_url(_query=prev_page_params)
         links.append('rel="prev", <{}>'.format(prev_url))
     if last_page_params != next_page_params:
         next_url = request.current_route_url(_query=next_page_params)
         links.append('rel="next", <{}>'.format(next_url))
-    headers['link'] = '; '.join(links)
+    headers["link"] = "; ".join(links)
     return headers

backend/src/appenlight/lib/jinja2_filters.py

@@ -1,46 +1,53 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import re
 from appenlight.lib.ext_json import json
 from jinja2 import Markup, escape, evalcontextfilter
 
-_paragraph_re = re.compile(r'(?:\r\n|\r|\n){2,}')
+_paragraph_re = re.compile(r"(?:\r\n|\r|\n){2,}")
 
 
 @evalcontextfilter
 def nl2br(eval_ctx, value):
     if eval_ctx.autoescape:
-        result = '\n\n'.join('<p>%s</p>' % p.replace('\n', Markup('<br>\n'))
-                              for p in _paragraph_re.split(escape(value)))
+        result = "\n\n".join(
+            "<p>%s</p>" % p.replace("\n", Markup("<br>\n"))
+            for p in _paragraph_re.split(escape(value))
+        )
     else:
-        result = '\n\n'.join('<p>%s</p>' % p.replace('\n', '<br>\n')
-                              for p in _paragraph_re.split(escape(value)))
+        result = "\n\n".join(
+            "<p>%s</p>" % p.replace("\n", "<br>\n")
+            for p in _paragraph_re.split(escape(value))
+        )
     if eval_ctx.autoescape:
         result = Markup(result)
     return result
 
 
 @evalcontextfilter
 def toJSONUnsafe(eval_ctx, value):
-    encoded = json.dumps(value).replace('&', '\\u0026') \
-        .replace('<', '\\u003c') \
-        .replace('>', '\\u003e') \
-        .replace('>', '\\u003e') \
-        .replace('"', '\\u0022') \
-        .replace("'", '\\u0027') \
-        .replace(r'\n', '/\\\n')
+    encoded = (
+        json.dumps(value)
+        .replace("&", "\\u0026")
+        .replace("<", "\\u003c")
+        .replace(">", "\\u003e")
+        .replace(">", "\\u003e")
+        .replace('"', "\\u0022")
+        .replace("'", "\\u0027")
+        .replace(r"\n", "/\\\n")
+    )
     return Markup("'%s'" % encoded)

backend/src/appenlight/lib/logging.py

@@ -1,64 +1,83 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import json
 import logging
 
-ignored_keys = ['args', 'asctime', 'created', 'exc_info', 'exc_text',
-                'filename', 'funcName', 'levelname', 'levelno', 'lineno',
-                'message', 'module', 'msecs', 'msg', 'name', 'pathname',
-                'process', 'processName', 'relativeCreated', 'stack_info',
-                'thread', 'threadName']
+ignored_keys = [
+    "args",
+    "asctime",
+    "created",
+    "exc_info",
+    "exc_text",
+    "filename",
+    "funcName",
+    "levelname",
+    "levelno",
+    "lineno",
+    "message",
+    "module",
+    "msecs",
+    "msg",
+    "name",
+    "pathname",
+    "process",
+    "processName",
+    "relativeCreated",
+    "stack_info",
+    "thread",
+    "threadName",
+]
 
 
 class JSONFormatter(logging.Formatter):
     def format(self, record):
         """
         Format the specified record as text.
 
         The record's attribute dictionary is used as the operand to a
         string formatting operation which yields the returned string.
         Before formatting the dictionary, a couple of preparatory steps
         are carried out. The message attribute of the record is computed
         using LogRecord.getMessage(). If the formatting string uses the
         time (as determined by a call to usesTime(), formatTime() is
         called to format the event time. If there is exception information,
         it is formatted using formatException() and appended to the message.
         """
         record.message = record.getMessage()
         log_dict = vars(record)
         keys = [k for k in log_dict.keys() if k not in ignored_keys]
-        payload = {'message': record.message}
+        payload = {"message": record.message}
         payload.update({k: log_dict[k] for k in keys})
         record.message = json.dumps(payload, default=lambda x: str(x))
 
         if self.usesTime():
             record.asctime = self.formatTime(record, self.datefmt)
         s = self.formatMessage(record)
         if record.exc_info:
             # Cache the traceback text to avoid converting it multiple times
             # (it's constant anyway)
             if not record.exc_text:
                 record.exc_text = self.formatException(record.exc_info)
         if record.exc_text:
             if s[-1:] != "\n":
                 s = s + "\n"
             s = s + record.exc_text
         if record.stack_info:
             if s[-1:] != "\n":
                 s = s + "\n"
             s = s + self.formatStack(record.stack_info)
         return s

backend/src/appenlight/lib/redis_keys.py

@@ -1,65 +1,69 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-BASE = 'appenlight:data:{}'
+BASE = "appenlight:data:{}"
 
 REDIS_KEYS = {
-    'tasks': {
-        'add_reports_lock': BASE.format('add_reports_lock:{}'),
-        'add_logs_lock': BASE.format('add_logs_lock:{}'),
+    "tasks": {
+        "add_reports_lock": BASE.format("add_reports_lock:{}"),
+        "add_logs_lock": BASE.format("add_logs_lock:{}"),
     },
-    'counters': {
-        'events_per_minute_per_user': BASE.format(
-            'events_per_minute_per_user:{}:{}'),
-        'reports_per_minute': BASE.format('reports_per_minute:{}'),
-        'reports_per_hour_per_app': BASE.format(
-            'reports_per_hour_per_app:{}:{}'),
-        'reports_per_type': BASE.format('reports_per_type:{}'),
-        'logs_per_minute': BASE.format('logs_per_minute:{}'),
-        'logs_per_hour_per_app': BASE.format(
-            'logs_per_hour_per_app:{}:{}'),
-        'metrics_per_minute': BASE.format('metrics_per_minute:{}'),
-        'metrics_per_hour_per_app': BASE.format(
-            'metrics_per_hour_per_app:{}:{}'),
-        'report_group_occurences': BASE.format('report_group_occurences:{}'),
-        'report_group_occurences_alerting': BASE.format(
-            'report_group_occurences_alerting:{}'),
-        'report_group_occurences_10th': BASE.format(
-            'report_group_occurences_10th:{}'),
-        'report_group_occurences_100th': BASE.format(
-            'report_group_occurences_100th:{}'),
+    "counters": {
+        "events_per_minute_per_user": BASE.format("events_per_minute_per_user:{}:{}"),
+        "reports_per_minute": BASE.format("reports_per_minute:{}"),
+        "reports_per_hour_per_app": BASE.format("reports_per_hour_per_app:{}:{}"),
+        "reports_per_type": BASE.format("reports_per_type:{}"),
+        "logs_per_minute": BASE.format("logs_per_minute:{}"),
+        "logs_per_hour_per_app": BASE.format("logs_per_hour_per_app:{}:{}"),
+        "metrics_per_minute": BASE.format("metrics_per_minute:{}"),
+        "metrics_per_hour_per_app": BASE.format("metrics_per_hour_per_app:{}:{}"),
+        "report_group_occurences": BASE.format("report_group_occurences:{}"),
+        "report_group_occurences_alerting": BASE.format(
+            "report_group_occurences_alerting:{}"
+        ),
+        "report_group_occurences_10th": BASE.format("report_group_occurences_10th:{}"),
+        "report_group_occurences_100th": BASE.format(
+            "report_group_occurences_100th:{}"
+        ),
     },
-    'rate_limits': {
-        'per_application_reports_rate_limit': BASE.format(
-            'per_application_reports_limit:{}:{}'),
-        'per_application_logs_rate_limit': BASE.format(
-            'per_application_logs_rate_limit:{}:{}'),
-        'per_application_metrics_rate_limit': BASE.format(
-            'per_application_metrics_rate_limit:{}:{}'),
+    "rate_limits": {
+        "per_application_reports_rate_limit": BASE.format(
+            "per_application_reports_limit:{}:{}"
+        ),
+        "per_application_logs_rate_limit": BASE.format(
+            "per_application_logs_rate_limit:{}:{}"
+        ),
+        "per_application_metrics_rate_limit": BASE.format(
+            "per_application_metrics_rate_limit:{}:{}"
+        ),
     },
-    'apps_that_got_new_data_per_hour': BASE.format('apps_that_got_new_data_per_hour:{}'),
-    'apps_that_had_reports': BASE.format('apps_that_had_reports'),
-    'apps_that_had_error_reports': BASE.format('apps_that_had_error_reports'),
-    'apps_that_had_reports_alerting': BASE.format(
-        'apps_that_had_reports_alerting'),
-    'apps_that_had_error_reports_alerting': BASE.format(
-        'apps_that_had_error_reports_alerting'),
-    'reports_to_notify_per_type_per_app': BASE.format(
-        'reports_to_notify_per_type_per_app:{}:{}'),
-    'reports_to_notify_per_type_per_app_alerting': BASE.format(
-        'reports_to_notify_per_type_per_app_alerting:{}:{}'),
-    'seen_tag_list': BASE.format('seen_tag_list')
+    "apps_that_got_new_data_per_hour": BASE.format(
+        "apps_that_got_new_data_per_hour:{}"
+    ),
+    "apps_that_had_reports": BASE.format("apps_that_had_reports"),
+    "apps_that_had_error_reports": BASE.format("apps_that_had_error_reports"),
+    "apps_that_had_reports_alerting": BASE.format("apps_that_had_reports_alerting"),
+    "apps_that_had_error_reports_alerting": BASE.format(
+        "apps_that_had_error_reports_alerting"
+    ),
+    "reports_to_notify_per_type_per_app": BASE.format(
+        "reports_to_notify_per_type_per_app:{}:{}"
+    ),
+    "reports_to_notify_per_type_per_app_alerting": BASE.format(
+        "reports_to_notify_per_type_per_app_alerting:{}:{}"
+    ),
+    "seen_tag_list": BASE.format("seen_tag_list"),
 }

backend/src/appenlight/lib/request.py

@@ -1,135 +1,131 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import json
 
 from pyramid.security import unauthenticated_userid
 
 import appenlight.lib.helpers as helpers
 
 from authomatic.providers import oauth2, oauth1
 from authomatic import Authomatic
 from ziggurat_foundations.models.services.user import UserService
 
 
 class CSRFException(Exception):
     pass
 
 
 class JSONException(Exception):
     pass
 
 
 def get_csrf_token(request):
     return request.session.get_csrf_token()
 
 
 def safe_json_body(request):
     """
     Returns None if json body is missing or erroneous
     """
     try:
         return request.json_body
     except ValueError:
         return None
 
 
 def unsafe_json_body(request):
     """
     Throws JSONException if json can't deserialize
     """
     try:
         return request.json_body
     except ValueError:
-        raise JSONException('Incorrect JSON')
+        raise JSONException("Incorrect JSON")
 
 
 def get_user(request):
-    if not request.path_info.startswith('/static'):
+    if not request.path_info.startswith("/static"):
         user_id = unauthenticated_userid(request)
         try:
             user_id = int(user_id)
         except Exception:
             return None
 
         if user_id:
             user = UserService.by_id(user_id)
             if user:
-                request.environ['appenlight.username'] = '%d:%s' % (
-                    user_id, user.user_name)
+                request.environ["appenlight.username"] = "%d:%s" % (
+                    user_id,
+                    user.user_name,
+                )
             return user
         else:
             return None
 
 
 def es_conn(request):
     return request.registry.es_conn
 
 
 def add_flash_to_headers(request, clear=True):
     """
     Adds pending flash messages to response, if clear is true clears out the
     flash queue
     """
     flash_msgs = helpers.get_type_formatted_flash(request)
-    request.response.headers['x-flash-messages'] = json.dumps(flash_msgs)
+    request.response.headers["x-flash-messages"] = json.dumps(flash_msgs)
     helpers.clear_flash(request)
 
 
 def get_authomatic(request):
     settings = request.registry.settings
     # authomatic social auth
     authomatic_conf = {
         # callback http://yourapp.com/social_auth/twitter
-        'twitter': {
-            'class_': oauth1.Twitter,
-            'consumer_key': settings.get('authomatic.pr.twitter.key', ''),
-            'consumer_secret': settings.get('authomatic.pr.twitter.secret',
-                                            ''),
+        "twitter": {
+            "class_": oauth1.Twitter,
+            "consumer_key": settings.get("authomatic.pr.twitter.key", ""),
+            "consumer_secret": settings.get("authomatic.pr.twitter.secret", ""),
         },
         # callback http://yourapp.com/social_auth/facebook
-        'facebook': {
-            'class_': oauth2.Facebook,
-            'consumer_key': settings.get('authomatic.pr.facebook.app_id', ''),
-            'consumer_secret': settings.get('authomatic.pr.facebook.secret',
-                                            ''),
-            'scope': ['email'],
+        "facebook": {
+            "class_": oauth2.Facebook,
+            "consumer_key": settings.get("authomatic.pr.facebook.app_id", ""),
+            "consumer_secret": settings.get("authomatic.pr.facebook.secret", ""),
+            "scope": ["email"],
         },
         # callback http://yourapp.com/social_auth/google
-        'google': {
-            'class_': oauth2.Google,
-            'consumer_key': settings.get('authomatic.pr.google.key', ''),
-            'consumer_secret': settings.get(
-                'authomatic.pr.google.secret', ''),
-            'scope': ['profile', 'email'],
+        "google": {
+            "class_": oauth2.Google,
+            "consumer_key": settings.get("authomatic.pr.google.key", ""),
+            "consumer_secret": settings.get("authomatic.pr.google.secret", ""),
+            "scope": ["profile", "email"],
         },
-        'github': {
-            'class_': oauth2.GitHub,
-            'consumer_key': settings.get('authomatic.pr.github.key', ''),
-            'consumer_secret': settings.get(
-                'authomatic.pr.github.secret', ''),
-            'scope': ['repo', 'public_repo', 'user:email'],
-            'access_headers': {'User-Agent': 'AppEnlight'},
+        "github": {
+            "class_": oauth2.GitHub,
+            "consumer_key": settings.get("authomatic.pr.github.key", ""),
+            "consumer_secret": settings.get("authomatic.pr.github.secret", ""),
+            "scope": ["repo", "public_repo", "user:email"],
+            "access_headers": {"User-Agent": "AppEnlight"},
+        },
+        "bitbucket": {
+            "class_": oauth1.Bitbucket,
+            "consumer_key": settings.get("authomatic.pr.bitbucket.key", ""),
+            "consumer_secret": settings.get("authomatic.pr.bitbucket.secret", ""),
         },
-        'bitbucket': {
-            'class_': oauth1.Bitbucket,
-            'consumer_key': settings.get('authomatic.pr.bitbucket.key', ''),
-            'consumer_secret': settings.get(
-                'authomatic.pr.bitbucket.secret', '')
-        }
     }
-    return Authomatic(
-        config=authomatic_conf, secret=settings['authomatic.secret'])
+    return Authomatic(config=authomatic_conf, secret=settings["authomatic.secret"])

backend/src/appenlight/lib/rule.py

@@ -1,298 +1,312 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 import operator
 
 log = logging.getLogger(__name__)
 
 
 class RuleException(Exception):
     pass
 
 
 class KeyNotFoundException(RuleException):
     pass
 
 
 class UnknownTypeException(RuleException):
     pass
 
 
 class BadConfigException(RuleException):
     pass
 
 
 class InvalidValueException(RuleException):
     pass
 
 
 class RuleBase(object):
     @classmethod
     def default_dict_struct_getter(cls, struct, field_name):
         """
         returns a key from dictionary based on field_name, if the name contains
         `:` then it means additional nesting levels should be checked for the
         key so `a:b:c` means return struct['a']['b']['c']
 
         :param struct:
         :param field_name:
         :return:
         """
-        parts = field_name.split(':') if field_name else []
+        parts = field_name.split(":") if field_name else []
         found = struct
         while parts:
             current_key = parts.pop(0)
             found = found.get(current_key)
             if not found and parts:
-                raise KeyNotFoundException('Key not found in structure')
+                raise KeyNotFoundException("Key not found in structure")
         return found
 
     @classmethod
     def default_obj_struct_getter(cls, struct, field_name):
         """
         returns a key from instance based on field_name, if the name contains
         `:` then it means additional nesting levels should be checked for the
         key so `a:b:c` means return struct.a.b.c
 
         :param struct:
         :param field_name:
         :return:
         """
-        parts = field_name.split(':')
+        parts = field_name.split(":")
         found = struct
         while parts:
             current_key = parts.pop(0)
             found = getattr(found, current_key, None)
             if not found and parts:
-                raise KeyNotFoundException('Key not found in structure')
+                raise KeyNotFoundException("Key not found in structure")
         return found
 
     def normalized_type(self, field, value):
         """
         Converts text values from self.conf_value based on type_matrix below
         check_matrix defines what kind of checks we can perform on a field
         value based on field name
         """
         f_type = self.type_matrix.get(field)
         if f_type:
-            cast_to = f_type['type']
+            cast_to = f_type["type"]
         else:
-            raise UnknownTypeException('Unknown type')
+            raise UnknownTypeException("Unknown type")
 
         if value is None:
             return None
 
         try:
-            if cast_to == 'int':
+            if cast_to == "int":
                 return int(value)
-            elif cast_to == 'float':
+            elif cast_to == "float":
                 return float(value)
-            elif cast_to == 'unicode':
+            elif cast_to == "unicode":
                 return str(value)
         except ValueError as exc:
             raise InvalidValueException(exc)
 
 
 class Rule(RuleBase):
-    def __init__(self, config, type_matrix,
-                 struct_getter=RuleBase.default_dict_struct_getter,
-                 config_manipulator=None):
+    def __init__(
+        self,
+        config,
+        type_matrix,
+        struct_getter=RuleBase.default_dict_struct_getter,
+        config_manipulator=None,
+    ):
         """
 
         :param config: dict - contains rule configuration
                               example::
                                     {
                                         "field": "__OR__",
                                         "rules": [
                                             {
                                                 "field": "__AND__",
                                                 "rules": [
                                                     {
                                                         "op": "ge",
                                                         "field": "occurences",
                                                         "value": "10"
                                                     },
                                                     {
                                                         "op": "ge",
                                                         "field": "priority",
                                                         "value": "4"
                                                     }
                                                 ]
                                             },
                                             {
                                                 "op": "eq",
                                                 "field": "http_status",
                                                 "value": "500"
                                             }
                                         ]
                                     }
         :param type_matrix: dict - contains map of type casts
                                    example::
                                         {
                                         'http_status': 'int',
                                         'priority': 'unicode',
                                         }
         :param struct_getter: callable - used to grab the value of field from
                                          the structure passed to match() based
                                          on key, default
 
         """
         self.type_matrix = type_matrix
         self.config = config
         self.struct_getter = struct_getter
         self.config_manipulator = config_manipulator
         if config_manipulator:
             config_manipulator(self)
 
     def subrule_check(self, rule_config, struct):
-        rule = Rule(rule_config, self.type_matrix,
-                    config_manipulator=self.config_manipulator)
+        rule = Rule(
+            rule_config, self.type_matrix, config_manipulator=self.config_manipulator
+        )
         return rule.match(struct)
 
     def match(self, struct):
         """
         Check if rule matched for this specific report
         First tries report value, then tests tags in not found, then finally
         report group
         """
-        field_name = self.config.get('field')
-        test_value = self.config.get('value')
+        field_name = self.config.get("field")
+        test_value = self.config.get("value")
 
         if not field_name:
             return False
 
-        if field_name == '__AND__':
-            rule = AND(self.config['rules'], self.type_matrix,
-                       config_manipulator=self.config_manipulator)
+        if field_name == "__AND__":
+            rule = AND(
+                self.config["rules"],
+                self.type_matrix,
+                config_manipulator=self.config_manipulator,
+            )
             return rule.match(struct)
-        elif field_name == '__OR__':
-            rule = OR(self.config['rules'], self.type_matrix,
-                      config_manipulator=self.config_manipulator)
+        elif field_name == "__OR__":
+            rule = OR(
+                self.config["rules"],
+                self.type_matrix,
+                config_manipulator=self.config_manipulator,
+            )
             return rule.match(struct)
-        elif field_name == '__NOT__':
-            rule = NOT(self.config['rules'], self.type_matrix,
-                      config_manipulator=self.config_manipulator)
+        elif field_name == "__NOT__":
+            rule = NOT(
+                self.config["rules"],
+                self.type_matrix,
+                config_manipulator=self.config_manipulator,
+            )
             return rule.match(struct)
 
         if test_value is None:
             return False
 
         try:
-            struct_value = self.normalized_type(field_name,
-                                                self.struct_getter(struct,
-                                                                   field_name))
+            struct_value = self.normalized_type(
+                field_name, self.struct_getter(struct, field_name)
+            )
         except (UnknownTypeException, InvalidValueException) as exc:
             log.error(str(exc))
             return False
 
         try:
             test_value = self.normalized_type(field_name, test_value)
         except (UnknownTypeException, InvalidValueException) as exc:
             log.error(str(exc))
             return False
 
-        if self.config['op'] not in ('startswith', 'endswith', 'contains'):
+        if self.config["op"] not in ("startswith", "endswith", "contains"):
             try:
-                return getattr(operator,
-                               self.config['op'])(struct_value, test_value)
+                return getattr(operator, self.config["op"])(struct_value, test_value)
             except TypeError:
                 return False
-        elif self.config['op'] == 'startswith':
+        elif self.config["op"] == "startswith":
             return struct_value.startswith(test_value)
-        elif self.config['op'] == 'endswith':
+        elif self.config["op"] == "endswith":
             return struct_value.endswith(test_value)
-        elif self.config['op'] == 'contains':
+        elif self.config["op"] == "contains":
             return test_value in struct_value
-        raise BadConfigException('Invalid configuration, '
-                                 'unknown operator: {}'.format(self.config))
+        raise BadConfigException(
+            "Invalid configuration, " "unknown operator: {}".format(self.config)
+        )
 
     def __repr__(self):
-        return '<Rule {} {}>'.format(self.config.get('field'),
-                                     self.config.get('value'))
+        return "<Rule {} {}>".format(self.config.get("field"), self.config.get("value"))
 
 
 class AND(Rule):
     def __init__(self, rules, *args, **kwargs):
         super(AND, self).__init__({}, *args, **kwargs)
         self.rules = rules
 
     def match(self, struct):
-        return all([self.subrule_check(r_conf, struct) for r_conf
-                    in self.rules])
+        return all([self.subrule_check(r_conf, struct) for r_conf in self.rules])
 
 
 class NOT(Rule):
     def __init__(self, rules, *args, **kwargs):
         super(NOT, self).__init__({}, *args, **kwargs)
         self.rules = rules
 
     def match(self, struct):
-        return all([not self.subrule_check(r_conf, struct) for r_conf
-                    in self.rules])
+        return all([not self.subrule_check(r_conf, struct) for r_conf in self.rules])
 
 
 class OR(Rule):
     def __init__(self, rules, *args, **kwargs):
         super(OR, self).__init__({}, *args, **kwargs)
         self.rules = rules
 
     def match(self, struct):
-        return any([self.subrule_check(r_conf, struct) for r_conf
-                    in self.rules])
+        return any([self.subrule_check(r_conf, struct) for r_conf in self.rules])
 
 
 class RuleService(object):
     @staticmethod
-    def rule_from_config(config, field_mappings, labels_dict,
-                         manipulator_func=None):
+    def rule_from_config(config, field_mappings, labels_dict, manipulator_func=None):
         """
         Returns modified rule with manipulator function
         By default manipulator function replaces field id from labels_dict
         with current field id proper for the rule from fields_mappings
 
         because label X_X id might be pointing different value on next request
         when new term is returned from elasticsearch - this ensures things
         are kept 1:1 all the time
         """
         rev_map = {}
         for k, v in labels_dict.items():
-            rev_map[(v['agg'], v['key'],)] = k
+            rev_map[(v["agg"], v["key"])] = k
 
         if manipulator_func is None:
+
             def label_rewriter_func(rule):
-                field = rule.config.get('field')
-                if not field or rule.config['field'] in ['__OR__',
-                                                         '__AND__', '__NOT__']:
+                field = rule.config.get("field")
+                if not field or rule.config["field"] in [
+                    "__OR__",
+                    "__AND__",
+                    "__NOT__",
+                ]:
                     return
 
-                to_map = field_mappings.get(rule.config['field'])
+                to_map = field_mappings.get(rule.config["field"])
 
                 # we need to replace series field with _AE_NOT_FOUND_ to not match
                 # accidently some other field which happens to have the series that
                 # was used when the alert was created
                 if to_map:
-                    to_replace = rev_map.get((to_map['agg'], to_map['key'],),
-                                             '_AE_NOT_FOUND_')
+                    to_replace = rev_map.get(
+                        (to_map["agg"], to_map["key"]), "_AE_NOT_FOUND_"
+                    )
                 else:
-                    to_replace = '_AE_NOT_FOUND_'
+                    to_replace = "_AE_NOT_FOUND_"
 
-                rule.config['field'] = to_replace
-                rule.type_matrix[to_replace] = {"type": 'float'}
+                rule.config["field"] = to_replace
+                rule.type_matrix[to_replace] = {"type": "float"}
 
             manipulator_func = label_rewriter_func
 
         return Rule(config, {}, config_manipulator=manipulator_func)

backend/src/appenlight/lib/social.py

@@ -1,60 +1,62 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ziggurat_foundations.models.services.external_identity import \
-    ExternalIdentityService
+from ziggurat_foundations.models.services.external_identity import (
+    ExternalIdentityService,
+)
 from appenlight.models.external_identity import ExternalIdentity
 
 
 def handle_social_data(request, user, social_data):
     social_data = social_data
     update_identity = False
 
     extng_id = ExternalIdentityService.by_external_id_and_provider(
-        social_data['user']['id'],
-        social_data['credentials'].provider_name
+        social_data["user"]["id"], social_data["credentials"].provider_name
     )
 
     # fix legacy accounts with wrong google ID
-    if not extng_id and social_data['credentials'].provider_name == 'google':
+    if not extng_id and social_data["credentials"].provider_name == "google":
         extng_id = ExternalIdentityService.by_external_id_and_provider(
-            social_data['user']['email'],
-            social_data['credentials'].provider_name
+            social_data["user"]["email"], social_data["credentials"].provider_name
         )
 
     if extng_id:
         extng_id.delete()
         update_identity = True
 
-    if not social_data['user']['id']:
+    if not social_data["user"]["id"]:
         request.session.flash(
-            'No external user id found? Perhaps permissions for '
-            'authentication are set incorrectly', 'error')
+            "No external user id found? Perhaps permissions for "
+            "authentication are set incorrectly",
+            "error",
+        )
         return False
 
     if not extng_id or update_identity:
         if not update_identity:
-            request.session.flash('Your external identity is now '
-                                  'connected with your account')
+            request.session.flash(
+                "Your external identity is now " "connected with your account"
+            )
         ex_identity = ExternalIdentity()
-        ex_identity.external_id = social_data['user']['id']
-        ex_identity.external_user_name = social_data['user']['user_name']
-        ex_identity.provider_name = social_data['credentials'].provider_name
-        ex_identity.access_token = social_data['credentials'].token
-        ex_identity.token_secret = social_data['credentials'].token_secret
-        ex_identity.alt_token = social_data['credentials'].refresh_token
+        ex_identity.external_id = social_data["user"]["id"]
+        ex_identity.external_user_name = social_data["user"]["user_name"]
+        ex_identity.provider_name = social_data["credentials"].provider_name
+        ex_identity.access_token = social_data["credentials"].token
+        ex_identity.token_secret = social_data["credentials"].token_secret
+        ex_identity.alt_token = social_data["credentials"].refresh_token
         user.external_identities.append(ex_identity)
-        request.session.pop('zigg.social_auth', None)
+        request.session.pop("zigg.social_auth", None)

backend/src/appenlight/lib/utils/__init__.py

@@ -1,491 +1,548 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 """
 Utility functions.
 """
 import logging
 import requests
 import hashlib
 import json
 import copy
 import uuid
 import appenlight.lib.helpers as h
 from collections import namedtuple
 from datetime import timedelta, datetime, date
 from dogpile.cache.api import NO_VALUE
 from appenlight.models import Datastores
-from appenlight.validators import (LogSearchSchema,
-                                   TagListSchema,
-                                   accepted_search_params)
+from appenlight.validators import LogSearchSchema, TagListSchema, accepted_search_params
 from itsdangerous import TimestampSigner
 from ziggurat_foundations.permissions import ALL_PERMISSIONS
 from ziggurat_foundations.models.services.user import UserService
 from dateutil.relativedelta import relativedelta
 from dateutil.rrule import rrule, MONTHLY, DAILY
 
 log = logging.getLogger(__name__)
 
 
-Stat = namedtuple('Stat', 'start_interval value')
+Stat = namedtuple("Stat", "start_interval value")
 
 
 def default_extractor(item):
     """
     :param item - item to extract date from
     """
-    if hasattr(item, 'start_interval'):
+    if hasattr(item, "start_interval"):
         return item.start_interval
-    return item['start_interval']
+    return item["start_interval"]
 
 
 # fast gap generator
-def gap_gen_default(start, step, itemiterator, end_time=None,
-                    iv_extractor=None):
+def gap_gen_default(start, step, itemiterator, end_time=None, iv_extractor=None):
     """ generates a list of time/value items based on step and itemiterator
         if there are entries missing from iterator time/None will be returned
         instead
     :param start - datetime - what time should we start generating our values
     :param step - timedelta - stepsize
     :param itemiterator - iterable - we will check this iterable for values
     corresponding to generated steps
     :param end_time - datetime - when last step is >= end_time stop iterating
     :param iv_extractor - extracts current step from iterable items
     """
 
     if not iv_extractor:
         iv_extractor = default_extractor
 
     next_step = start
     minutes = step.total_seconds() / 60.0
     while next_step.minute % minutes != 0:
         next_step = next_step.replace(minute=next_step.minute - 1)
     for item in itemiterator:
         item_start_interval = iv_extractor(item)
         # do we have a match for current time step in our data?
         # no gen a new tuple with 0 values
         while next_step < item_start_interval:
             yield Stat(next_step, None)
             next_step = next_step + step
         if next_step == item_start_interval:
             yield Stat(item_start_interval, item)
             next_step = next_step + step
     if end_time:
         while next_step < end_time:
             yield Stat(next_step, None)
             next_step = next_step + step
 
 
 class DateTimeEncoder(json.JSONEncoder):
     """ Simple datetime to ISO encoder for json serialization"""
 
     def default(self, obj):
         if isinstance(obj, date):
             return obj.isoformat()
         if isinstance(obj, datetime):
             return obj.isoformat()
         return json.JSONEncoder.default(self, obj)
 
 
-def channelstream_request(secret, endpoint, payload, throw_exceptions=False,
-                   servers=None):
+def channelstream_request(
+    secret, endpoint, payload, throw_exceptions=False, servers=None
+):
     responses = []
     if not servers:
         servers = []
 
     signer = TimestampSigner(secret)
     sig_for_server = signer.sign(endpoint)
-    for secret, server in [(s['secret'], s['server']) for s in servers]:
+    for secret, server in [(s["secret"], s["server"]) for s in servers]:
         response = {}
-        secret_headers = {'x-channelstream-secret': sig_for_server,
-                          'x-channelstream-endpoint': endpoint,
-                          'Content-Type': 'application/json'}
-        url = '%s%s' % (server, endpoint)
+        secret_headers = {
+            "x-channelstream-secret": sig_for_server,
+            "x-channelstream-endpoint": endpoint,
+            "Content-Type": "application/json",
+        }
+        url = "%s%s" % (server, endpoint)
         try:
-            response = requests.post(url,
-                                     data=json.dumps(payload,
-                                                     cls=DateTimeEncoder),
-                                     headers=secret_headers,
-                                     verify=False,
-                                     timeout=2).json()
+            response = requests.post(
+                url,
+                data=json.dumps(payload, cls=DateTimeEncoder),
+                headers=secret_headers,
+                verify=False,
+                timeout=2,
+            ).json()
         except requests.exceptions.RequestException as e:
             if throw_exceptions:
                 raise
         responses.append(response)
     return responses
 
 
 def add_cors_headers(response):
     # allow CORS
-    response.headers.add('Access-Control-Allow-Origin', '*')
-    response.headers.add('XDomainRequestAllowed', '1')
-    response.headers.add('Access-Control-Allow-Methods', 'GET, POST, OPTIONS')
+    response.headers.add("Access-Control-Allow-Origin", "*")
+    response.headers.add("XDomainRequestAllowed", "1")
+    response.headers.add("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
     # response.headers.add('Access-Control-Allow-Credentials', 'true')
-    response.headers.add('Access-Control-Allow-Headers',
-                         'Content-Type, Depth, User-Agent, X-File-Size, X-Requested-With, If-Modified-Since, X-File-Name, Cache-Control, Pragma, Origin, Connection, Referer, Cookie')
-    response.headers.add('Access-Control-Max-Age', '86400')
+    response.headers.add(
+        "Access-Control-Allow-Headers",
+        "Content-Type, Depth, User-Agent, X-File-Size, X-Requested-With, If-Modified-Since, X-File-Name, Cache-Control, Pragma, Origin, Connection, Referer, Cookie",
+    )
+    response.headers.add("Access-Control-Max-Age", "86400")
 
 
 from sqlalchemy.sql import compiler
 from psycopg2.extensions import adapt as sqlescape
 
 
 # or use the appropiate escape function from your db driver
 
+
 def compile_query(query):
     dialect = query.session.bind.dialect
     statement = query.statement
     comp = compiler.SQLCompiler(dialect, statement)
     comp.compile()
     enc = dialect.encoding
     params = {}
     for k, v in comp.params.items():
         if isinstance(v, str):
             v = v.encode(enc)
         params[k] = sqlescape(v)
     return (comp.string.encode(enc) % params).decode(enc)
 
 
 def convert_es_type(input_data):
     """
     This might need to convert some text or other types to corresponding ES types
     """
     return str(input_data)
 
 
-ProtoVersion = namedtuple('ProtoVersion', ['major', 'minor', 'patch'])
+ProtoVersion = namedtuple("ProtoVersion", ["major", "minor", "patch"])
 
 
 def parse_proto(input_data):
     try:
-        parts = [int(x) for x in input_data.split('.')]
+        parts = [int(x) for x in input_data.split(".")]
         while len(parts) < 3:
             parts.append(0)
         return ProtoVersion(*parts)
     except Exception as e:
-        log.info('Unknown protocol version: %s' % e)
+        log.info("Unknown protocol version: %s" % e)
     return ProtoVersion(99, 99, 99)
 
 
-def es_index_name_limiter(start_date=None, end_date=None, months_in_past=6,
-                          ixtypes=None):
+def es_index_name_limiter(
+    start_date=None, end_date=None, months_in_past=6, ixtypes=None
+):
     """
     This function limits the search to 6 months by default so we don't have to
     query 300 elasticsearch indices for 20 years of historical data for example
     """
 
     # should be cached later
     def get_possible_names():
-        return list(Datastores.es.indices.get_alias('*'))
+        return list(Datastores.es.indices.get_alias("*"))
 
     possible_names = get_possible_names()
     es_index_types = []
     if not ixtypes:
-        ixtypes = ['reports', 'metrics', 'logs']
+        ixtypes = ["reports", "metrics", "logs"]
     for t in ixtypes:
-        if t == 'reports':
-            es_index_types.append('rcae_r_%s')
-        elif t == 'logs':
-            es_index_types.append('rcae_l_%s')
-        elif t == 'metrics':
-            es_index_types.append('rcae_m_%s')
-        elif t == 'uptime':
-            es_index_types.append('rcae_u_%s')
-        elif t == 'slow_calls':
-            es_index_types.append('rcae_sc_%s')
+        if t == "reports":
+            es_index_types.append("rcae_r_%s")
+        elif t == "logs":
+            es_index_types.append("rcae_l_%s")
+        elif t == "metrics":
+            es_index_types.append("rcae_m_%s")
+        elif t == "uptime":
+            es_index_types.append("rcae_u_%s")
+        elif t == "slow_calls":
+            es_index_types.append("rcae_sc_%s")
 
     if start_date:
         start_date = copy.copy(start_date)
     else:
         if not end_date:
             end_date = datetime.utcnow()
         start_date = end_date + relativedelta(months=months_in_past * -1)
 
     if not end_date:
         end_date = start_date + relativedelta(months=months_in_past)
 
-    index_dates = list(rrule(MONTHLY,
-                             dtstart=start_date.date().replace(day=1),
-                             until=end_date.date(),
-                             count=36))
+    index_dates = list(
+        rrule(
+            MONTHLY,
+            dtstart=start_date.date().replace(day=1),
+            until=end_date.date(),
+            count=36,
+        )
+    )
     index_names = []
     for ix_type in es_index_types:
-        to_extend = [ix_type % d.strftime('%Y_%m') for d in index_dates
-                     if ix_type % d.strftime('%Y_%m') in possible_names]
+        to_extend = [
+            ix_type % d.strftime("%Y_%m")
+            for d in index_dates
+            if ix_type % d.strftime("%Y_%m") in possible_names
+        ]
         index_names.extend(to_extend)
-        for day in list(rrule(DAILY, dtstart=start_date.date(),
-                              until=end_date.date(), count=366)):
-            ix_name = ix_type % day.strftime('%Y_%m_%d')
+        for day in list(
+            rrule(DAILY, dtstart=start_date.date(), until=end_date.date(), count=366)
+        ):
+            ix_name = ix_type % day.strftime("%Y_%m_%d")
             if ix_name in possible_names:
                 index_names.append(ix_name)
     return index_names
 
 
 def build_filter_settings_from_query_dict(
-        request, params=None, override_app_ids=None,
-        resource_permissions=None):
+    request, params=None, override_app_ids=None, resource_permissions=None
+):
     """
     Builds list of normalized search terms for ES from query params
     ensuring application list is restricted to only applications user
     has access to
 
     :param params (dictionary)
     :param override_app_ids - list of application id's to use instead of
     applications user normally has access to
     """
     params = copy.deepcopy(params)
     applications = []
     if not resource_permissions:
-        resource_permissions = ['view']
+        resource_permissions = ["view"]
 
     if request.user:
         applications = UserService.resources_with_perms(
-            request.user, resource_permissions, resource_types=['application'])
+            request.user, resource_permissions, resource_types=["application"]
+        )
 
     # CRITICAL - this ensures our resultset is limited to only the ones
     # user has view permissions
     all_possible_app_ids = set([app.resource_id for app in applications])
 
     # if override is preset we force permission for app to be present
     # this allows users to see dashboards and applications they would
     # normally not be able to
 
     if override_app_ids:
         all_possible_app_ids = set(override_app_ids)
 
     schema = LogSearchSchema().bind(resources=all_possible_app_ids)
     tag_schema = TagListSchema()
     filter_settings = schema.deserialize(params)
     tag_list = []
     for k, v in list(filter_settings.items()):
         if k in accepted_search_params:
             continue
-        tag_list.append({"name": k, "value": v, "op": 'eq'})
+        tag_list.append({"name": k, "value": v, "op": "eq"})
         # remove the key from filter_settings
         filter_settings.pop(k, None)
     tags = tag_schema.deserialize(tag_list)
-    filter_settings['tags'] = tags
+    filter_settings["tags"] = tags
     return filter_settings
 
 
 def gen_uuid():
     return str(uuid.uuid4())
 
 
 def gen_uuid4_sha_hex():
     return hashlib.sha1(uuid.uuid4().bytes).hexdigest()
 
 
 def permission_tuple_to_dict(data):
     out = {
         "user_name": None,
         "perm_name": data.perm_name,
         "owner": data.owner,
         "type": data.type,
         "resource_name": None,
         "resource_type": None,
         "resource_id": None,
         "group_name": None,
-        "group_id": None
+        "group_id": None,
     }
     if data.user:
         out["user_name"] = data.user.user_name
     if data.perm_name == ALL_PERMISSIONS:
-        out['perm_name'] = '__all_permissions__'
+        out["perm_name"] = "__all_permissions__"
     if data.resource:
-        out['resource_name'] = data.resource.resource_name
-        out['resource_type'] = data.resource.resource_type
-        out['resource_id'] = data.resource.resource_id
+        out["resource_name"] = data.resource.resource_name
+        out["resource_type"] = data.resource.resource_type
+        out["resource_id"] = data.resource.resource_id
     if data.group:
-        out['group_name'] = data.group.group_name
-        out['group_id'] = data.group.id
+        out["group_name"] = data.group.group_name
+        out["group_id"] = data.group.id
     return out
 
 
-def get_cached_buckets(request, stats_since, end_time, fn, cache_key,
-                       gap_gen=None, db_session=None, step_interval=None,
-                       iv_extractor=None,
-                       rerange=False, *args, **kwargs):
+def get_cached_buckets(
+    request,
+    stats_since,
+    end_time,
+    fn,
+    cache_key,
+    gap_gen=None,
+    db_session=None,
+    step_interval=None,
+    iv_extractor=None,
+    rerange=False,
+    *args,
+    **kwargs
+):
     """ Takes "fn" that should return some data and tries to load the data
     dividing it into daily buckets - if the stats_since and end time give a
     delta bigger than 24hours, then only "todays" data is computed on the fly
 
     :param request: (request) request object
     :param stats_since: (datetime) start date of buckets range
     :param end_time: (datetime) end date of buckets range - utcnow() if None
     :param fn: (callable) callable to use to populate buckets should have
     following signature:
         def get_data(request, since_when, until, *args, **kwargs):
 
     :param cache_key: (string) cache key that will be used to build bucket
     caches
     :param gap_gen: (callable) gap generator - should return step intervals
     to use with out `fn` callable
     :param db_session: (Session) sqlalchemy session
     :param step_interval: (timedelta) optional step interval if we want to
     override the default determined from total start/end time delta
     :param iv_extractor: (callable) used to get step intervals from data
     returned by `fn` callable
     :param rerange: (bool) handy if we want to change ranges from hours to
     days when cached data is missing - will shorten execution time if `fn`
     callable supports that and we are working with multiple rows - like metrics
     :param args:
     :param kwargs:
 
     :return: iterable
     """
     if not end_time:
         end_time = datetime.utcnow().replace(second=0, microsecond=0)
     delta = end_time - stats_since
     # if smaller than 3 days we want to group by 5min else by 1h,
     # for 60 min group by min
     if not gap_gen:
         gap_gen = gap_gen_default
     if not iv_extractor:
         iv_extractor = default_extractor
 
     # do not use custom interval if total time range with new iv would exceed
     # end time
     if not step_interval or stats_since + step_interval >= end_time:
-        if delta < h.time_deltas.get('12h')['delta']:
+        if delta < h.time_deltas.get("12h")["delta"]:
             step_interval = timedelta(seconds=60)
-        elif delta < h.time_deltas.get('3d')['delta']:
+        elif delta < h.time_deltas.get("3d")["delta"]:
             step_interval = timedelta(seconds=60 * 5)
-        elif delta > h.time_deltas.get('2w')['delta']:
+        elif delta > h.time_deltas.get("2w")["delta"]:
             step_interval = timedelta(days=1)
         else:
             step_interval = timedelta(minutes=60)
 
     if step_interval >= timedelta(minutes=60):
-        log.info('cached_buckets:{}: adjusting start time '
-                 'for hourly or daily intervals'.format(cache_key))
+        log.info(
+            "cached_buckets:{}: adjusting start time "
+            "for hourly or daily intervals".format(cache_key)
+        )
         stats_since = stats_since.replace(hour=0, minute=0)
 
-    ranges = [i.start_interval for i in list(gap_gen(stats_since,
-                                                     step_interval, [],
-                                                     end_time=end_time))]
+    ranges = [
+        i.start_interval
+        for i in list(gap_gen(stats_since, step_interval, [], end_time=end_time))
+    ]
     buckets = {}
-    storage_key = 'buckets:' + cache_key + '{}|{}'
+    storage_key = "buckets:" + cache_key + "{}|{}"
     # this means we basicly cache per hour in 3-14 day intervals but i think
     # its fine at this point - will be faster than db access anyways
 
     if len(ranges) >= 1:
         last_ranges = [ranges[-1]]
     else:
         last_ranges = []
     if step_interval >= timedelta(minutes=60):
         for r in ranges:
             k = storage_key.format(step_interval.total_seconds(), r)
             value = request.registry.cache_regions.redis_day_30.get(k)
             # last buckets are never loaded from cache
-            is_last_result = (
-                r >= end_time - timedelta(hours=6) or r in last_ranges)
+            is_last_result = r >= end_time - timedelta(hours=6) or r in last_ranges
             if value is not NO_VALUE and not is_last_result:
-                log.info("cached_buckets:{}: "
-                         "loading range {} from cache".format(cache_key, r))
+                log.info(
+                    "cached_buckets:{}: "
+                    "loading range {} from cache".format(cache_key, r)
+                )
                 buckets[r] = value
             else:
-                log.info("cached_buckets:{}: "
-                         "loading range {} from storage".format(cache_key, r))
+                log.info(
+                    "cached_buckets:{}: "
+                    "loading range {} from storage".format(cache_key, r)
+                )
                 range_size = step_interval
-                if (step_interval == timedelta(minutes=60) and
-                        not is_last_result and rerange):
+                if (
+                    step_interval == timedelta(minutes=60)
+                    and not is_last_result
+                    and rerange
+                ):
                     range_size = timedelta(days=1)
                     r = r.replace(hour=0, minute=0)
-                    log.info("cached_buckets:{}: "
-                             "loading collapsed "
-                             "range {} {}".format(cache_key, r,
-                                                  r + range_size))
+                    log.info(
+                        "cached_buckets:{}: "
+                        "loading collapsed "
+                        "range {} {}".format(cache_key, r, r + range_size)
+                    )
                 bucket_data = fn(
-                    request, r, r + range_size, step_interval,
-                    gap_gen, bucket_count=len(ranges), *args, **kwargs)
+                    request,
+                    r,
+                    r + range_size,
+                    step_interval,
+                    gap_gen,
+                    bucket_count=len(ranges),
+                    *args,
+                    **kwargs
+                )
                 for b in bucket_data:
                     b_iv = iv_extractor(b)
                     buckets[b_iv] = b
-                    k2 = storage_key.format(
-                        step_interval.total_seconds(), b_iv)
+                    k2 = storage_key.format(step_interval.total_seconds(), b_iv)
                     request.registry.cache_regions.redis_day_30.set(k2, b)
         log.info("cached_buckets:{}: saving cache".format(cache_key))
     else:
         # bucket count is 1 for short time ranges <= 24h from now
-        bucket_data = fn(request, stats_since, end_time, step_interval,
-                         gap_gen, bucket_count=1, *args, **kwargs)
+        bucket_data = fn(
+            request,
+            stats_since,
+            end_time,
+            step_interval,
+            gap_gen,
+            bucket_count=1,
+            *args,
+            **kwargs
+        )
         for b in bucket_data:
             buckets[iv_extractor(b)] = b
     return buckets
 
 
-def get_cached_split_data(request, stats_since, end_time, fn, cache_key,
-                          db_session=None, *args, **kwargs):
+def get_cached_split_data(
+    request, stats_since, end_time, fn, cache_key, db_session=None, *args, **kwargs
+):
     """ Takes "fn" that should return some data and tries to load the data
     dividing it into 2 buckets - cached "since_from" bucket and "today"
     bucket - then the data can be reduced into single value
 
     Data is cached if the stats_since and end time give a delta bigger
     than 24hours - then only 24h is computed on the fly
     """
     if not end_time:
         end_time = datetime.utcnow().replace(second=0, microsecond=0)
     delta = end_time - stats_since
 
     if delta >= timedelta(minutes=60):
-        log.info('cached_split_data:{}: adjusting start time '
-                 'for hourly or daily intervals'.format(cache_key))
+        log.info(
+            "cached_split_data:{}: adjusting start time "
+            "for hourly or daily intervals".format(cache_key)
+        )
         stats_since = stats_since.replace(hour=0, minute=0)
 
-    storage_key = 'buckets_split_data:' + cache_key + ':{}|{}'
+    storage_key = "buckets_split_data:" + cache_key + ":{}|{}"
     old_end_time = end_time.replace(hour=0, minute=0)
 
-    final_storage_key = storage_key.format(delta.total_seconds(),
-                                           old_end_time)
+    final_storage_key = storage_key.format(delta.total_seconds(), old_end_time)
     older_data = None
 
-    cdata = request.registry.cache_regions.redis_day_7.get(
-        final_storage_key)
+    cdata = request.registry.cache_regions.redis_day_7.get(final_storage_key)
 
     if cdata:
-        log.info("cached_split_data:{}: found old "
-                 "bucket data".format(cache_key))
+        log.info("cached_split_data:{}: found old " "bucket data".format(cache_key))
         older_data = cdata
 
-    if (stats_since < end_time - h.time_deltas.get('24h')['delta'] and
-            not cdata):
-        log.info("cached_split_data:{}: didn't find the "
-                 "start bucket in cache so load older data".format(cache_key))
+    if stats_since < end_time - h.time_deltas.get("24h")["delta"] and not cdata:
+        log.info(
+            "cached_split_data:{}: didn't find the "
+            "start bucket in cache so load older data".format(cache_key)
+        )
         recent_stats_since = old_end_time
-        older_data = fn(request, stats_since, recent_stats_since,
-                        db_session=db_session, *args, **kwargs)
-        request.registry.cache_regions.redis_day_7.set(final_storage_key,
-                                                       older_data)
-    elif stats_since < end_time - h.time_deltas.get('24h')['delta']:
+        older_data = fn(
+            request,
+            stats_since,
+            recent_stats_since,
+            db_session=db_session,
+            *args,
+            **kwargs
+        )
+        request.registry.cache_regions.redis_day_7.set(final_storage_key, older_data)
+    elif stats_since < end_time - h.time_deltas.get("24h")["delta"]:
         recent_stats_since = old_end_time
     else:
         recent_stats_since = stats_since
 
-    log.info("cached_split_data:{}: loading fresh "
-             "data bucksts from last 24h ".format(cache_key))
-    todays_data = fn(request, recent_stats_since, end_time,
-                     db_session=db_session, *args, **kwargs)
+    log.info(
+        "cached_split_data:{}: loading fresh "
+        "data bucksts from last 24h ".format(cache_key)
+    )
+    todays_data = fn(
+        request, recent_stats_since, end_time, db_session=db_session, *args, **kwargs
+    )
     return older_data, todays_data
 
 
 def in_batches(seq, size):
     """
     Splits am iterable into batches of specified size
     :param seq (iterable)
     :param size integer
     """
-    return (seq[pos:pos + size] for pos in range(0, len(seq), size))
+    return (seq[pos : pos + size] for pos in range(0, len(seq), size))

backend/src/appenlight/lib/utils/airbrake.py

@@ -1,142 +1,161 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 import uuid
 
 from datetime import datetime
 
 log = logging.getLogger(__name__)
 
 
 def parse_airbrake_xml(request):
     root = request.context.airbrake_xml_etree
-    error = root.find('error')
-    notifier = root.find('notifier')
-    server_env = root.find('server-environment')
-    request_data = root.find('request')
-    user = root.find('current-user')
+    error = root.find("error")
+    notifier = root.find("notifier")
+    server_env = root.find("server-environment")
+    request_data = root.find("request")
+    user = root.find("current-user")
     if request_data is not None:
-        cgi_data = request_data.find('cgi-data')
+        cgi_data = request_data.find("cgi-data")
         if cgi_data is None:
             cgi_data = []
 
     error_dict = {
-        'class_name': error.findtext('class') or '',
-        'error': error.findtext('message') or '',
+        "class_name": error.findtext("class") or "",
+        "error": error.findtext("message") or "",
         "occurences": 1,
         "http_status": 500,
         "priority": 5,
-        "server": 'unknown',
-        'url': 'unknown', 'request': {}
+        "server": "unknown",
+        "url": "unknown",
+        "request": {},
     }
     if user is not None:
-        error_dict['username'] = user.findtext('username') or \
-                                 user.findtext('id')
+        error_dict["username"] = user.findtext("username") or user.findtext("id")
     if notifier is not None:
-        error_dict['client'] = notifier.findtext('name')
+        error_dict["client"] = notifier.findtext("name")
 
     if server_env is not None:
-        error_dict["server"] = server_env.findtext('hostname', 'unknown')
+        error_dict["server"] = server_env.findtext("hostname", "unknown")
 
-    whitelist_environ = ['REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME',
-                         'CONTENT_TYPE', 'HTTP_REFERER']
+    whitelist_environ = [
+        "REMOTE_USER",
+        "REMOTE_ADDR",
+        "SERVER_NAME",
+        "CONTENT_TYPE",
+        "HTTP_REFERER",
+    ]
 
     if request_data is not None:
-        error_dict['url'] = request_data.findtext('url', 'unknown')
-        component = request_data.findtext('component')
-        action = request_data.findtext('action')
+        error_dict["url"] = request_data.findtext("url", "unknown")
+        component = request_data.findtext("component")
+        action = request_data.findtext("action")
         if component and action:
-            error_dict['view_name'] = '%s:%s' % (component, action)
+            error_dict["view_name"] = "%s:%s" % (component, action)
         for node in cgi_data:
-            key = node.get('key')
-            if key.startswith('HTTP') or key in whitelist_environ:
-                error_dict['request'][key] = node.text
-            elif 'query_parameters' in key:
-                error_dict['request']['GET'] = {}
+            key = node.get("key")
+            if key.startswith("HTTP") or key in whitelist_environ:
+                error_dict["request"][key] = node.text
+            elif "query_parameters" in key:
+                error_dict["request"]["GET"] = {}
                 for x in node:
-                    error_dict['request']['GET'][x.get('key')] = x.text
-            elif 'request_parameters' in key:
-                error_dict['request']['POST'] = {}
+                    error_dict["request"]["GET"][x.get("key")] = x.text
+            elif "request_parameters" in key:
+                error_dict["request"]["POST"] = {}
                 for x in node:
-                    error_dict['request']['POST'][x.get('key')] = x.text
-            elif key.endswith('cookie'):
-                error_dict['request']['COOKIE'] = {}
+                    error_dict["request"]["POST"][x.get("key")] = x.text
+            elif key.endswith("cookie"):
+                error_dict["request"]["COOKIE"] = {}
                 for x in node:
-                    error_dict['request']['COOKIE'][x.get('key')] = x.text
-            elif key.endswith('request_id'):
-                error_dict['request_id'] = node.text
-            elif key.endswith('session'):
-                error_dict['request']['SESSION'] = {}
+                    error_dict["request"]["COOKIE"][x.get("key")] = x.text
+            elif key.endswith("request_id"):
+                error_dict["request_id"] = node.text
+            elif key.endswith("session"):
+                error_dict["request"]["SESSION"] = {}
                 for x in node:
-                    error_dict['request']['SESSION'][x.get('key')] = x.text
+                    error_dict["request"]["SESSION"][x.get("key")] = x.text
             else:
-                if key in ['rack.session.options']:
+                if key in ["rack.session.options"]:
                     # skip secret configs
                     continue
                 try:
                     if len(node):
-                        error_dict['request'][key] = dict(
-                            [(x.get('key'), x.text,) for x in node])
+                        error_dict["request"][key] = dict(
+                            [(x.get("key"), x.text) for x in node]
+                        )
                     else:
-                        error_dict['request'][key] = node.text
+                        error_dict["request"][key] = node.text
                 except Exception as e:
-                    log.warning('Airbrake integration exception: %s' % e)
+                    log.warning("Airbrake integration exception: %s" % e)
 
-            error_dict['request'].pop('HTTP_COOKIE', '')
+            error_dict["request"].pop("HTTP_COOKIE", "")
 
-        error_dict['ip'] = error_dict.pop('REMOTE_ADDR', '')
-        error_dict['user_agent'] = error_dict.pop('HTTP_USER_AGENT', '')
-    if 'request_id' not in error_dict:
-        error_dict['request_id'] = str(uuid.uuid4())
+        error_dict["ip"] = error_dict.pop("REMOTE_ADDR", "")
+        error_dict["user_agent"] = error_dict.pop("HTTP_USER_AGENT", "")
+    if "request_id" not in error_dict:
+        error_dict["request_id"] = str(uuid.uuid4())
     if request.context.possibly_public:
         # set ip for reports that come from airbrake js client
         error_dict["timestamp"] = datetime.utcnow()
         if request.environ.get("HTTP_X_FORWARDED_FOR"):
-            ip = request.environ.get("HTTP_X_FORWARDED_FOR", '')
-            first_ip = ip.split(',')[0]
+            ip = request.environ.get("HTTP_X_FORWARDED_FOR", "")
+            first_ip = ip.split(",")[0]
             remote_addr = first_ip.strip()
         else:
-            remote_addr = (request.environ.get("HTTP_X_REAL_IP") or
-                           request.environ.get('REMOTE_ADDR'))
+            remote_addr = request.environ.get("HTTP_X_REAL_IP") or request.environ.get(
+                "REMOTE_ADDR"
+            )
         error_dict["ip"] = remote_addr
 
-    blacklist = ['password', 'passwd', 'pwd', 'auth_tkt', 'secret', 'csrf',
-                 'session', 'test']
+    blacklist = [
+        "password",
+        "passwd",
+        "pwd",
+        "auth_tkt",
+        "secret",
+        "csrf",
+        "session",
+        "test",
+    ]
 
     lines = []
-    for l in error.find('backtrace'):
-        lines.append({'file': l.get("file", ""),
-                      'line': l.get("number", ""),
-                      'fn': l.get("method", ""),
-                      'module': l.get("module", ""),
-                      'cline': l.get("method", ""),
-                      'vars': {}})
-    error_dict['traceback'] = list(reversed(lines))
+    for l in error.find("backtrace"):
+        lines.append(
+            {
+                "file": l.get("file", ""),
+                "line": l.get("number", ""),
+                "fn": l.get("method", ""),
+                "module": l.get("module", ""),
+                "cline": l.get("method", ""),
+                "vars": {},
+            }
+        )
+    error_dict["traceback"] = list(reversed(lines))
     # filtering is not provided by airbrake
     keys_to_check = (
-        error_dict['request'].get('COOKIE'),
-        error_dict['request'].get('COOKIES'),
-        error_dict['request'].get('POST'),
-        error_dict['request'].get('SESSION'),
+        error_dict["request"].get("COOKIE"),
+        error_dict["request"].get("COOKIES"),
+        error_dict["request"].get("POST"),
+        error_dict["request"].get("SESSION"),
     )
     for source in [_f for _f in keys_to_check if _f]:
         for k in source.keys():
             for bad_key in blacklist:
                 if bad_key in k.lower():
-                    source[k] = '***'
+                    source[k] = "***"
 
     return error_dict

backend/src/appenlight/lib/utils/date_utils.py

@@ -1,56 +1,56 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from datetime import tzinfo, timedelta, datetime
 from dateutil.relativedelta import relativedelta
 import logging
 
 log = logging.getLogger(__name__)
 
 
 def to_relativedelta(time_delta):
-    return relativedelta(seconds=int(time_delta.total_seconds()),
-                         microseconds=time_delta.microseconds)
+    return relativedelta(
+        seconds=int(time_delta.total_seconds()), microseconds=time_delta.microseconds
+    )
 
 
-def convert_date(date_str, return_utcnow_if_wrong=True,
-                 normalize_future=False):
+def convert_date(date_str, return_utcnow_if_wrong=True, normalize_future=False):
     utcnow = datetime.utcnow()
     if isinstance(date_str, datetime):
         # get rid of tzinfo
         return date_str.replace(tzinfo=None)
     if not date_str and return_utcnow_if_wrong:
         return utcnow
     try:
         try:
-            if 'Z' in date_str:
-                date_str = date_str[:date_str.index('Z')]
-            if '.' in date_str:
-                date = datetime.strptime(date_str, '%Y-%m-%dT%H:%M:%S.%f')
+            if "Z" in date_str:
+                date_str = date_str[: date_str.index("Z")]
+            if "." in date_str:
+                date = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S.%f")
             else:
-                date = datetime.strptime(date_str, '%Y-%m-%dT%H:%M:%S')
+                date = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S")
         except Exception:
             # bw compat with old client
-            date = datetime.strptime(date_str, '%Y-%m-%d %H:%M:%S,%f')
+            date = datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S,%f")
     except Exception:
         if return_utcnow_if_wrong:
             date = utcnow
         else:
             date = None
     if normalize_future and date and date > (utcnow + timedelta(minutes=3)):
-        log.warning('time %s in future + 3 min, normalizing' % date)
+        log.warning("time %s in future + 3 min, normalizing" % date)
         return utcnow
     return date

backend/src/appenlight/lib/utils/sentry.py

@@ -1,296 +1,317 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from datetime import timedelta
 
 from appenlight.lib.enums import LogLevelPython, ParsedSentryEventType
 
 EXCLUDED_LOG_VARS = [
-    'args', 'asctime', 'created', 'exc_info', 'exc_text', 'filename',
-    'funcName', 'levelname', 'levelno', 'lineno', 'message', 'module', 'msecs',
-    'msg', 'name', 'pathname', 'process', 'processName', 'relativeCreated',
-    'thread', 'threadName']
+    "args",
+    "asctime",
+    "created",
+    "exc_info",
+    "exc_text",
+    "filename",
+    "funcName",
+    "levelname",
+    "levelno",
+    "lineno",
+    "message",
+    "module",
+    "msecs",
+    "msg",
+    "name",
+    "pathname",
+    "process",
+    "processName",
+    "relativeCreated",
+    "thread",
+    "threadName",
+]
 
 EXCLUDE_SENTRY_KEYS = [
-    'csp',
-    'culprit',
-    'event_id',
-    'exception',
-    'extra',
-    'level',
-    'logentry',
-    'logger',
-    'message',
-    'modules',
-    'platform',
-    'query',
-    'release',
-    'request',
-    'sentry.interfaces.Csp', 'sentry.interfaces.Exception',
-    'sentry.interfaces.Http', 'sentry.interfaces.Message',
-    'sentry.interfaces.Query',
-    'sentry.interfaces.Stacktrace',
-    'sentry.interfaces.Template', 'sentry.interfaces.User',
-    'sentry.interfaces.csp.Csp',
-    'sentry.interfaces.exception.Exception',
-    'sentry.interfaces.http.Http',
-    'sentry.interfaces.message.Message',
-    'sentry.interfaces.query.Query',
-    'sentry.interfaces.stacktrace.Stacktrace',
-    'sentry.interfaces.template.Template',
-    'sentry.interfaces.user.User', 'server_name',
-    'stacktrace',
-    'tags',
-    'template',
-    'time_spent',
-    'timestamp',
-    'user']
+    "csp",
+    "culprit",
+    "event_id",
+    "exception",
+    "extra",
+    "level",
+    "logentry",
+    "logger",
+    "message",
+    "modules",
+    "platform",
+    "query",
+    "release",
+    "request",
+    "sentry.interfaces.Csp",
+    "sentry.interfaces.Exception",
+    "sentry.interfaces.Http",
+    "sentry.interfaces.Message",
+    "sentry.interfaces.Query",
+    "sentry.interfaces.Stacktrace",
+    "sentry.interfaces.Template",
+    "sentry.interfaces.User",
+    "sentry.interfaces.csp.Csp",
+    "sentry.interfaces.exception.Exception",
+    "sentry.interfaces.http.Http",
+    "sentry.interfaces.message.Message",
+    "sentry.interfaces.query.Query",
+    "sentry.interfaces.stacktrace.Stacktrace",
+    "sentry.interfaces.template.Template",
+    "sentry.interfaces.user.User",
+    "server_name",
+    "stacktrace",
+    "tags",
+    "template",
+    "time_spent",
+    "timestamp",
+    "user",
+]
 
 
 def get_keys(list_of_keys, json_body):
     for k in list_of_keys:
         if k in json_body:
             return json_body[k]
 
 
 def get_logentry(json_body):
-    key_names = ['logentry',
-                 'sentry.interfaces.message.Message',
-                 'sentry.interfaces.Message'
-                 ]
+    key_names = [
+        "logentry",
+        "sentry.interfaces.message.Message",
+        "sentry.interfaces.Message",
+    ]
     logentry = get_keys(key_names, json_body)
     return logentry
 
 
 def get_exception(json_body):
     parsed_exception = {}
-    key_names = ['exception',
-                 'sentry.interfaces.exception.Exception',
-                 'sentry.interfaces.Exception'
-                 ]
+    key_names = [
+        "exception",
+        "sentry.interfaces.exception.Exception",
+        "sentry.interfaces.Exception",
+    ]
     exception = get_keys(key_names, json_body) or {}
     if exception:
         if isinstance(exception, dict):
-            exception = exception['values'][0]
+            exception = exception["values"][0]
         else:
             exception = exception[0]
 
-    parsed_exception['type'] = exception.get('type')
-    parsed_exception['value'] = exception.get('value')
-    parsed_exception['module'] = exception.get('module')
+    parsed_exception["type"] = exception.get("type")
+    parsed_exception["value"] = exception.get("value")
+    parsed_exception["module"] = exception.get("module")
     parsed_stacktrace = get_stacktrace(exception) or {}
     parsed_exception = exception or {}
     return parsed_exception, parsed_stacktrace
 
 
 def get_stacktrace(json_body):
     parsed_stacktrace = []
-    key_names = ['stacktrace',
-                 'sentry.interfaces.stacktrace.Stacktrace',
-                 'sentry.interfaces.Stacktrace'
-                 ]
+    key_names = [
+        "stacktrace",
+        "sentry.interfaces.stacktrace.Stacktrace",
+        "sentry.interfaces.Stacktrace",
+    ]
     stacktrace = get_keys(key_names, json_body)
     if stacktrace:
-        for frame in stacktrace['frames']:
+        for frame in stacktrace["frames"]:
             parsed_stacktrace.append(
-                {"cline": frame.get('context_line', ''),
-                 "file": frame.get('filename', ''),
-                 "module": frame.get('module', ''),
-                 "fn": frame.get('function', ''),
-                 "line": frame.get('lineno', ''),
-                 "vars": list(frame.get('vars', {}).items())
-                 }
+                {
+                    "cline": frame.get("context_line", ""),
+                    "file": frame.get("filename", ""),
+                    "module": frame.get("module", ""),
+                    "fn": frame.get("function", ""),
+                    "line": frame.get("lineno", ""),
+                    "vars": list(frame.get("vars", {}).items()),
+                }
             )
     return parsed_stacktrace
 
 
 def get_template(json_body):
     parsed_template = {}
-    key_names = ['template',
-                 'sentry.interfaces.template.Template',
-                 'sentry.interfaces.Template'
-                 ]
+    key_names = [
+        "template",
+        "sentry.interfaces.template.Template",
+        "sentry.interfaces.Template",
+    ]
     template = get_keys(key_names, json_body)
     if template:
-        for frame in template['frames']:
+        for frame in template["frames"]:
             parsed_template.append(
-                {"cline": frame.get('context_line', ''),
-                 "file": frame.get('filename', ''),
-                 "fn": '',
-                 "line": frame.get('lineno', ''),
-                 "vars": []
-                 }
+                {
+                    "cline": frame.get("context_line", ""),
+                    "file": frame.get("filename", ""),
+                    "fn": "",
+                    "line": frame.get("lineno", ""),
+                    "vars": [],
+                }
             )
 
     return parsed_template
 
 
 def get_request(json_body):
     parsed_http = {}
-    key_names = ['request',
-                 'sentry.interfaces.http.Http',
-                 'sentry.interfaces.Http'
-                 ]
+    key_names = ["request", "sentry.interfaces.http.Http", "sentry.interfaces.Http"]
     http = get_keys(key_names, json_body) or {}
     for k, v in http.items():
-        if k == 'headers':
-            parsed_http['headers'] = {}
-            for sk, sv in http['headers'].items():
-                parsed_http['headers'][sk.title()] = sv
+        if k == "headers":
+            parsed_http["headers"] = {}
+            for sk, sv in http["headers"].items():
+                parsed_http["headers"][sk.title()] = sv
         else:
             parsed_http[k.lower()] = v
     return parsed_http
 
 
 def get_user(json_body):
     parsed_user = {}
-    key_names = ['user',
-                 'sentry.interfaces.user.User',
-                 'sentry.interfaces.User'
-                 ]
+    key_names = ["user", "sentry.interfaces.user.User", "sentry.interfaces.User"]
     user = get_keys(key_names, json_body)
     if user:
-        parsed_user['id'] = user.get('id')
-        parsed_user['username'] = user.get('username')
-        parsed_user['email'] = user.get('email')
-        parsed_user['ip_address'] = user.get('ip_address')
+        parsed_user["id"] = user.get("id")
+        parsed_user["username"] = user.get("username")
+        parsed_user["email"] = user.get("email")
+        parsed_user["ip_address"] = user.get("ip_address")
 
     return parsed_user
 
 
 def get_query(json_body):
     query = None
-    key_name = ['query',
-                'sentry.interfaces.query.Query',
-                'sentry.interfaces.Query'
-                ]
+    key_name = ["query", "sentry.interfaces.query.Query", "sentry.interfaces.Query"]
     query = get_keys(key_name, json_body)
     return query
 
 
 def parse_sentry_event(json_body):
-    request_id = json_body.get('event_id')
+    request_id = json_body.get("event_id")
 
     # required
-    message = json_body.get('message')
-    log_timestamp = json_body.get('timestamp')
-    level = json_body.get('level')
+    message = json_body.get("message")
+    log_timestamp = json_body.get("timestamp")
+    level = json_body.get("level")
     if isinstance(level, int):
         level = LogLevelPython.key_from_value(level)
 
-    namespace = json_body.get('logger')
-    language = json_body.get('platform')
+    namespace = json_body.get("logger")
+    language = json_body.get("platform")
 
     # optional
-    server_name = json_body.get('server_name')
-    culprit = json_body.get('culprit')
-    release = json_body.get('release')
+    server_name = json_body.get("server_name")
+    culprit = json_body.get("culprit")
+    release = json_body.get("release")
 
-    tags = json_body.get('tags', {})
-    if hasattr(tags, 'items'):
+    tags = json_body.get("tags", {})
+    if hasattr(tags, "items"):
         tags = list(tags.items())
-    extra = json_body.get('extra', {})
-    if hasattr(extra, 'items'):
+    extra = json_body.get("extra", {})
+    if hasattr(extra, "items"):
         extra = list(extra.items())
 
     parsed_req = get_request(json_body)
     user = get_user(json_body)
     template = get_template(json_body)
     query = get_query(json_body)
 
     # other unidentified keys found
-    other_keys = [(k, json_body[k]) for k in json_body.keys()
-                  if k not in EXCLUDE_SENTRY_KEYS]
+    other_keys = [
+        (k, json_body[k]) for k in json_body.keys() if k not in EXCLUDE_SENTRY_KEYS
+    ]
 
     logentry = get_logentry(json_body)
     if logentry:
-        message = logentry['message']
+        message = logentry["message"]
 
     exception, stacktrace = get_exception(json_body)
 
     alt_stacktrace = get_stacktrace(json_body)
     event_type = None
     if not exception and not stacktrace and not alt_stacktrace and not template:
         event_type = ParsedSentryEventType.LOG
 
         event_dict = {
-            'log_level': level,
-            'message': message,
-            'namespace': namespace,
-            'request_id': request_id,
-            'server': server_name,
-            'date': log_timestamp,
-            'tags': tags
+            "log_level": level,
+            "message": message,
+            "namespace": namespace,
+            "request_id": request_id,
+            "server": server_name,
+            "date": log_timestamp,
+            "tags": tags,
         }
-        event_dict['tags'].extend(
-            [(k, v) for k, v in extra if k not in EXCLUDED_LOG_VARS])
+        event_dict["tags"].extend(
+            [(k, v) for k, v in extra if k not in EXCLUDED_LOG_VARS]
+        )
 
         # other keys can be various object types
-        event_dict['tags'].extend([(k, v) for k, v in other_keys
-                                   if isinstance(v, str)])
+        event_dict["tags"].extend([(k, v) for k, v in other_keys if isinstance(v, str)])
         if culprit:
-            event_dict['tags'].append(('sentry_culprit', culprit))
+            event_dict["tags"].append(("sentry_culprit", culprit))
         if language:
-            event_dict['tags'].append(('sentry_language', language))
+            event_dict["tags"].append(("sentry_language", language))
         if release:
-            event_dict['tags'].append(('sentry_release', release))
+            event_dict["tags"].append(("sentry_release", release))
 
     if exception or stacktrace or alt_stacktrace or template:
         event_type = ParsedSentryEventType.ERROR_REPORT
         event_dict = {
-            'client': 'sentry',
-            'error': message,
-            'namespace': namespace,
-            'request_id': request_id,
-            'server': server_name,
-            'start_time': log_timestamp,
-            'end_time': None,
-            'tags': tags,
-            'extra': extra,
-            'language': language,
-            'view_name': json_body.get('culprit'),
-            'http_status': None,
-            'username': None,
-            'url': parsed_req.get('url'),
-            'ip': None,
-            'user_agent': None,
-            'request': None,
-            'slow_calls': None,
-            'request_stats': None,
-            'traceback': None
+            "client": "sentry",
+            "error": message,
+            "namespace": namespace,
+            "request_id": request_id,
+            "server": server_name,
+            "start_time": log_timestamp,
+            "end_time": None,
+            "tags": tags,
+            "extra": extra,
+            "language": language,
+            "view_name": json_body.get("culprit"),
+            "http_status": None,
+            "username": None,
+            "url": parsed_req.get("url"),
+            "ip": None,
+            "user_agent": None,
+            "request": None,
+            "slow_calls": None,
+            "request_stats": None,
+            "traceback": None,
         }
 
-        event_dict['extra'].extend(other_keys)
+        event_dict["extra"].extend(other_keys)
         if release:
-            event_dict['tags'].append(('sentry_release', release))
-        event_dict['request'] = parsed_req
-        if 'headers' in parsed_req:
-            event_dict['user_agent'] = parsed_req['headers'].get('User-Agent')
-        if 'env' in parsed_req:
-            event_dict['ip'] = parsed_req['env'].get('REMOTE_ADDR')
-        ts_ms = int(json_body.get('time_spent') or 0)
+            event_dict["tags"].append(("sentry_release", release))
+        event_dict["request"] = parsed_req
+        if "headers" in parsed_req:
+            event_dict["user_agent"] = parsed_req["headers"].get("User-Agent")
+        if "env" in parsed_req:
+            event_dict["ip"] = parsed_req["env"].get("REMOTE_ADDR")
+        ts_ms = int(json_body.get("time_spent") or 0)
         if ts_ms > 0:
-            event_dict['end_time'] = event_dict['start_time'] + \
-                                     timedelta(milliseconds=ts_ms)
+            event_dict["end_time"] = event_dict["start_time"] + timedelta(
+                milliseconds=ts_ms
+            )
         if stacktrace or alt_stacktrace or template:
-            event_dict['traceback'] = stacktrace or alt_stacktrace or template
+            event_dict["traceback"] = stacktrace or alt_stacktrace or template
         for k in list(event_dict.keys()):
             if event_dict[k] is None:
                 del event_dict[k]
         if user:
-            event_dict['username'] = user['username'] or user['id'] \
-                                     or user['email']
+            event_dict["username"] = user["username"] or user["id"] or user["email"]
     return event_dict, event_type

backend/src/appenlight/migrations/__init__.py

@@ -1,17 +1,15 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-

backend/src/appenlight/migrations/env.py

@@ -1,98 +1,102 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from alembic import context
 from sqlalchemy import engine_from_config, pool, MetaData
 from logging.config import fileConfig
 from appenlight.models import NAMING_CONVENTION
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
 config = context.config
 
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.
 if config.config_file_name:
     fileConfig(config.config_file_name)
 
 # add your model's MetaData object here
 # for 'autogenerate' support
 # from myapp import mymodel
 # target_metadata = mymodel.Base.metadata
 
 
 target_metadata = MetaData(naming_convention=NAMING_CONVENTION)
 
 # other values from the config, defined by the needs of env.py,
 # can be acquired:
 # my_important_option = config.get_main_option("my_important_option")
 # ... etc.
 
-VERSION_TABLE_NAME = 'alembic_appenlight_version'
+VERSION_TABLE_NAME = "alembic_appenlight_version"
 
 
 def run_migrations_offline():
     """Run migrations in 'offline' mode.
 
     This configures the context with just a URL
     and not an Engine, though an Engine is acceptable
     here as well.  By skipping the Engine creation
     we don't even need a DBAPI to be available.
 
     Calls to context.execute() here emit the given string to the
     script output.
 
     """
     url = config.get_main_option("sqlalchemy.url")
-    context.configure(url=url, target_metadata=target_metadata,
-                      transaction_per_migration=True,
-                      version_table=VERSION_TABLE_NAME)
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        transaction_per_migration=True,
+        version_table=VERSION_TABLE_NAME,
+    )
 
     with context.begin_transaction():
         context.run_migrations()
 
 
 def run_migrations_online():
     """Run migrations in 'online' mode.
 
     In this scenario we need to create an Engine
     and associate a connection with the context.
 
     """
     engine = engine_from_config(
         config.get_section(config.config_ini_section),
-        prefix='sqlalchemy.',
-        poolclass=pool.NullPool)
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
 
     connection = engine.connect()
     context.configure(
         connection=connection,
         target_metadata=target_metadata,
         transaction_per_migration=True,
-        version_table=VERSION_TABLE_NAME
+        version_table=VERSION_TABLE_NAME,
     )
 
     try:
         with context.begin_transaction():
             context.run_migrations()
     finally:
         connection.close()
 
 
 if context.is_offline_mode():
     run_migrations_offline()
 else:
     run_migrations_online()

backend/src/appenlight/migrations/versions/55b6e612672f_initial_tables.py

@@ -1,624 +1,813 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 """initial tables
 
 Revision ID: 55b6e612672f
 Revises: None
 Create Date: 2014-10-13 23:47:38.295159
 
 """
 
 # revision identifiers, used by Alembic.
-revision = '55b6e612672f'
+revision = "55b6e612672f"
 down_revision = None
 
 from alembic import op
 import sqlalchemy as sa
 
 
 def upgrade():
-    op.add_column('users', sa.Column('first_name', sa.Unicode(25)))
-    op.add_column('users', sa.Column('last_name', sa.Unicode(50)))
-    op.add_column('users', sa.Column('company_name', sa.Unicode(255)))
-    op.add_column('users', sa.Column('company_address', sa.Unicode(255)))
-    op.add_column('users', sa.Column('phone1', sa.Unicode(25)))
-    op.add_column('users', sa.Column('phone2', sa.Unicode(25)))
-    op.add_column('users', sa.Column('zip_code', sa.Unicode(25)))
-    op.add_column('users', sa.Column('default_report_sort', sa.Unicode(20), nullable=False, server_default="newest"))
-    op.add_column('users', sa.Column('city', sa.Unicode(128)))
-    op.add_column('users', sa.Column('notes', sa.UnicodeText, server_default=''))
-    op.add_column('users', sa.Column('notifications', sa.Boolean(), nullable=False, server_default='true'))
-    op.add_column('users', sa.Column('registration_ip', sa.Unicode(40), nullable=False, server_default=''))
+    op.add_column("users", sa.Column("first_name", sa.Unicode(25)))
+    op.add_column("users", sa.Column("last_name", sa.Unicode(50)))
+    op.add_column("users", sa.Column("company_name", sa.Unicode(255)))
+    op.add_column("users", sa.Column("company_address", sa.Unicode(255)))
+    op.add_column("users", sa.Column("phone1", sa.Unicode(25)))
+    op.add_column("users", sa.Column("phone2", sa.Unicode(25)))
+    op.add_column("users", sa.Column("zip_code", sa.Unicode(25)))
+    op.add_column(
+        "users",
+        sa.Column(
+            "default_report_sort",
+            sa.Unicode(20),
+            nullable=False,
+            server_default="newest",
+        ),
+    )
+    op.add_column("users", sa.Column("city", sa.Unicode(128)))
+    op.add_column("users", sa.Column("notes", sa.UnicodeText, server_default=""))
+    op.add_column(
+        "users",
+        sa.Column("notifications", sa.Boolean(), nullable=False, server_default="true"),
+    )
+    op.add_column(
+        "users",
+        sa.Column("registration_ip", sa.Unicode(40), nullable=False, server_default=""),
+    )
 
     op.create_table(
-        'integrations',
-        sa.Column('id', sa.Integer(), primary_key=True),
-        sa.Column('resource_id', sa.Integer(),
-                  sa.ForeignKey('resources.resource_id', onupdate='cascade',
-                                ondelete='cascade')),
-        sa.Column('integration_name', sa.Unicode(64)),
-        sa.Column('config', sa.dialects.postgresql.JSON, nullable=False),
-        sa.Column('modified_date', sa.DateTime(), nullable=False, server_default=sa.func.now()),
-        sa.Column('external_id', sa.Unicode(255)),
-        sa.Column('external_id2', sa.Unicode(255))
+        "integrations",
+        sa.Column("id", sa.Integer(), primary_key=True),
+        sa.Column(
+            "resource_id",
+            sa.Integer(),
+            sa.ForeignKey(
+                "resources.resource_id", onupdate="cascade", ondelete="cascade"
+            ),
+        ),
+        sa.Column("integration_name", sa.Unicode(64)),
+        sa.Column("config", sa.dialects.postgresql.JSON, nullable=False),
+        sa.Column(
+            "modified_date", sa.DateTime(), nullable=False, server_default=sa.func.now()
+        ),
+        sa.Column("external_id", sa.Unicode(255)),
+        sa.Column("external_id2", sa.Unicode(255)),
     )
 
     op.create_table(
-        'alert_channels',
-        sa.Column('owner_id', sa.Integer(),
-                  sa.ForeignKey('users.id', onupdate='cascade',
-                                ondelete='cascade'), nullable=False),
-        sa.Column('channel_name', sa.Unicode(25), nullable=False),
-        sa.Column('channel_value', sa.Unicode(80), nullable=False),
-        sa.Column('channel_json_conf', sa.dialects.postgresql.JSON, nullable=False),
-        sa.Column('channel_validated', sa.Boolean, nullable=False, server_default='False'),
-        sa.Column('send_alerts', sa.Boolean, nullable=False, server_default='True'),
-        sa.Column('notify_only_first', sa.Boolean, nullable=False, server_default='False'),
-        sa.Column('daily_digest', sa.Boolean, nullable=False, server_default='True'),
-        sa.Column('pkey', sa.Integer(), primary_key=True),
-        sa.Column('integration_id', sa.Integer,
-                  sa.ForeignKey('integrations.id', onupdate='cascade',
-                                ondelete='cascade')),
-    )
-    op.create_unique_constraint('uq_alert_channels', 'alert_channels',
-                                ["owner_id", "channel_name", "channel_value"])
+        "alert_channels",
+        sa.Column(
+            "owner_id",
+            sa.Integer(),
+            sa.ForeignKey("users.id", onupdate="cascade", ondelete="cascade"),
+            nullable=False,
+        ),
+        sa.Column("channel_name", sa.Unicode(25), nullable=False),
+        sa.Column("channel_value", sa.Unicode(80), nullable=False),
+        sa.Column("channel_json_conf", sa.dialects.postgresql.JSON, nullable=False),
+        sa.Column(
+            "channel_validated", sa.Boolean, nullable=False, server_default="False"
+        ),
+        sa.Column("send_alerts", sa.Boolean, nullable=False, server_default="True"),
+        sa.Column(
+            "notify_only_first", sa.Boolean, nullable=False, server_default="False"
+        ),
+        sa.Column("daily_digest", sa.Boolean, nullable=False, server_default="True"),
+        sa.Column("pkey", sa.Integer(), primary_key=True),
+        sa.Column(
+            "integration_id",
+            sa.Integer,
+            sa.ForeignKey("integrations.id", onupdate="cascade", ondelete="cascade"),
+        ),
+    )
+    op.create_unique_constraint(
+        "uq_alert_channels",
+        "alert_channels",
+        ["owner_id", "channel_name", "channel_value"],
+    )
 
     op.create_table(
-        'alert_channels_actions',
-        sa.Column('owner_id', sa.Integer(), nullable=False),
-        sa.Column('resource_id', sa.Integer(),
-                  sa.ForeignKey('resources.resource_id', onupdate='cascade',
-                                ondelete='cascade')),
-        sa.Column('pkey', sa.Integer(), primary_key=True),
-        sa.Column('action', sa.Unicode(10), nullable=False, server_default='always'),
-        sa.Column('rule', sa.dialects.postgresql.JSON),
-        sa.Column('type', sa.Unicode(10), index=True),
-        sa.Column('other_id', sa.Unicode(40), index=True),
-        sa.Column('config', sa.dialects.postgresql.JSON),
-        sa.Column('name', sa.Unicode(255), server_default='')
+        "alert_channels_actions",
+        sa.Column("owner_id", sa.Integer(), nullable=False),
+        sa.Column(
+            "resource_id",
+            sa.Integer(),
+            sa.ForeignKey(
+                "resources.resource_id", onupdate="cascade", ondelete="cascade"
+            ),
+        ),
+        sa.Column("pkey", sa.Integer(), primary_key=True),
+        sa.Column("action", sa.Unicode(10), nullable=False, server_default="always"),
+        sa.Column("rule", sa.dialects.postgresql.JSON),
+        sa.Column("type", sa.Unicode(10), index=True),
+        sa.Column("other_id", sa.Unicode(40), index=True),
+        sa.Column("config", sa.dialects.postgresql.JSON),
+        sa.Column("name", sa.Unicode(255), server_default=""),
     )
 
-
     op.create_table(
-        'application_postprocess_conf',
-        sa.Column('pkey', sa.Integer(), primary_key=True),
-        sa.Column('do', sa.Unicode(25), nullable=False),
-        sa.Column('new_value', sa.UnicodeText(), nullable=False, server_default=''),
-        sa.Column('resource_id', sa.Integer(),
-                  sa.ForeignKey('resources.resource_id',
-                                onupdate='cascade',
-                                ondelete='cascade'), nullable=False),
-        sa.Column('rule', sa.dialects.postgresql.JSON),
+        "application_postprocess_conf",
+        sa.Column("pkey", sa.Integer(), primary_key=True),
+        sa.Column("do", sa.Unicode(25), nullable=False),
+        sa.Column("new_value", sa.UnicodeText(), nullable=False, server_default=""),
+        sa.Column(
+            "resource_id",
+            sa.Integer(),
+            sa.ForeignKey(
+                "resources.resource_id", onupdate="cascade", ondelete="cascade"
+            ),
+            nullable=False,
+        ),
+        sa.Column("rule", sa.dialects.postgresql.JSON),
     )
 
     op.create_table(
-        'applications',
-        sa.Column('resource_id', sa.Integer(),
-                  sa.ForeignKey('resources.resource_id', onupdate='cascade',
-                                ondelete='cascade'), nullable=False,
-                  primary_key=True, autoincrement=False),
-        sa.Column('domains', sa.UnicodeText, nullable=False),
-        sa.Column('api_key', sa.Unicode(32), nullable=False, index=True),
-        sa.Column('default_grouping', sa.Unicode(20), nullable=False, server_default='url_type'),
-        sa.Column('public_key', sa.Unicode(32), nullable=False, index=True),
-        sa.Column('error_report_threshold', sa.Integer(), server_default='10', nullable=False),
-        sa.Column('slow_report_threshold', sa.Integer(), server_default='10', nullable=False),
-        sa.Column('apdex_threshold', sa.Float(), server_default='0.7', nullable=False),
-        sa.Column('allow_permanent_storage', sa.Boolean(), server_default="false", nullable=False),
-    )
-    op.create_unique_constraint(None, 'applications',
-                                ["public_key"])
-    op.create_unique_constraint(None, 'applications',
-                                ["api_key"])
+        "applications",
+        sa.Column(
+            "resource_id",
+            sa.Integer(),
+            sa.ForeignKey(
+                "resources.resource_id", onupdate="cascade", ondelete="cascade"
+            ),
+            nullable=False,
+            primary_key=True,
+            autoincrement=False,
+        ),
+        sa.Column("domains", sa.UnicodeText, nullable=False),
+        sa.Column("api_key", sa.Unicode(32), nullable=False, index=True),
+        sa.Column(
+            "default_grouping",
+            sa.Unicode(20),
+            nullable=False,
+            server_default="url_type",
+        ),
+        sa.Column("public_key", sa.Unicode(32), nullable=False, index=True),
+        sa.Column(
+            "error_report_threshold", sa.Integer(), server_default="10", nullable=False
+        ),
+        sa.Column(
+            "slow_report_threshold", sa.Integer(), server_default="10", nullable=False
+        ),
+        sa.Column("apdex_threshold", sa.Float(), server_default="0.7", nullable=False),
+        sa.Column(
+            "allow_permanent_storage",
+            sa.Boolean(),
+            server_default="false",
+            nullable=False,
+        ),
+    )
+    op.create_unique_constraint(None, "applications", ["public_key"])
+    op.create_unique_constraint(None, "applications", ["api_key"])
 
     op.create_table(
-        'metrics',
-        sa.Column('pkey', sa.types.BigInteger, nullable=False, primary_key=True),
-        sa.Column('resource_id', sa.Integer(),
-                  sa.ForeignKey('resources.resource_id',
-                                onupdate='cascade',
-                                ondelete='cascade')),
-        sa.Column('timestamp', sa.DateTime),
-        sa.Column('namespace', sa.Unicode(255)),
-        sa.Column('tags', sa.dialects.postgresql.JSON, server_default="{}")
+        "metrics",
+        sa.Column("pkey", sa.types.BigInteger, nullable=False, primary_key=True),
+        sa.Column(
+            "resource_id",
+            sa.Integer(),
+            sa.ForeignKey(
+                "resources.resource_id", onupdate="cascade", ondelete="cascade"
+            ),
+        ),
+        sa.Column("timestamp", sa.DateTime),
+        sa.Column("namespace", sa.Unicode(255)),
+        sa.Column("tags", sa.dialects.postgresql.JSON, server_default="{}"),
     )
 
     op.create_table(
-        'events',
-        sa.Column('id', sa.Integer, nullable=False, primary_key=True),
-        sa.Column('start_date', sa.DateTime, nullable=False, index=True),
-        sa.Column('end_date', sa.DateTime),
-        sa.Column('status', sa.Integer(), nullable=False, index=True),
-        sa.Column('event_type', sa.Integer(), nullable=False, index=True),
-        sa.Column('origin_user_id', sa.Integer()),
-        sa.Column('target_user_id', sa.Integer()),
-        sa.Column('resource_id', sa.Integer(), index=True),
-        sa.Column('text', sa.UnicodeText, server_default=''),
-        sa.Column('values', sa.dialects.postgresql.JSON),
-        sa.Column('target_id', sa.Integer()),
-        sa.Column('target_uuid', sa.Unicode(40), index=True)
+        "events",
+        sa.Column("id", sa.Integer, nullable=False, primary_key=True),
+        sa.Column("start_date", sa.DateTime, nullable=False, index=True),
+        sa.Column("end_date", sa.DateTime),
+        sa.Column("status", sa.Integer(), nullable=False, index=True),
+        sa.Column("event_type", sa.Integer(), nullable=False, index=True),
+        sa.Column("origin_user_id", sa.Integer()),
+        sa.Column("target_user_id", sa.Integer()),
+        sa.Column("resource_id", sa.Integer(), index=True),
+        sa.Column("text", sa.UnicodeText, server_default=""),
+        sa.Column("values", sa.dialects.postgresql.JSON),
+        sa.Column("target_id", sa.Integer()),
+        sa.Column("target_uuid", sa.Unicode(40), index=True),
     )
 
     op.create_table(
-        'logs',
-        sa.Column('log_id', sa.types.BigInteger, nullable=False, primary_key=True),
-        sa.Column('resource_id', sa.Integer(),
-                  sa.ForeignKey('resources.resource_id',
-                                onupdate='cascade',
-                                ondelete='cascade')),
-        sa.Column('log_level', sa.SmallInteger(), nullable=False),
-        sa.Column('primary_key', sa.Unicode(128), nullable=True),
-        sa.Column('message', sa.UnicodeText, nullable=False, server_default=''),
-        sa.Column('timestamp', sa.DateTime),
-        sa.Column('namespace', sa.Unicode(255)),
-        sa.Column('request_id', sa.Unicode(40)),
-        sa.Column('tags', sa.dialects.postgresql.JSON, server_default="{}"),
-        sa.Column('permanent', sa.Boolean(), server_default="false",
-                  nullable=False)
+        "logs",
+        sa.Column("log_id", sa.types.BigInteger, nullable=False, primary_key=True),
+        sa.Column(
+            "resource_id",
+            sa.Integer(),
+            sa.ForeignKey(
+                "resources.resource_id", onupdate="cascade", ondelete="cascade"
+            ),
+        ),
+        sa.Column("log_level", sa.SmallInteger(), nullable=False),
+        sa.Column("primary_key", sa.Unicode(128), nullable=True),
+        sa.Column("message", sa.UnicodeText, nullable=False, server_default=""),
+        sa.Column("timestamp", sa.DateTime),
+        sa.Column("namespace", sa.Unicode(255)),
+        sa.Column("request_id", sa.Unicode(40)),
+        sa.Column("tags", sa.dialects.postgresql.JSON, server_default="{}"),
+        sa.Column("permanent", sa.Boolean(), server_default="false", nullable=False),
     )
 
     op.create_table(
-        'reports_groups',
-        sa.Column('id', sa.types.BigInteger, primary_key=True),
-        sa.Column('resource_id', sa.Integer,
-                  sa.ForeignKey('resources.resource_id', onupdate='cascade',
-                                ondelete='cascade'), nullable=False),
-        sa.Column('priority', sa.Integer, nullable=False, server_default="5"),
-        sa.Column('first_timestamp', sa.DateTime(), nullable=False, server_default=sa.func.now()),
-        sa.Column('last_timestamp', sa.DateTime()),
-        sa.Column('error', sa.UnicodeText, nullable=False, server_default=""),
-        sa.Column('grouping_hash', sa.Unicode(40), nullable=False, server_default=""),
-        sa.Column('triggered_postprocesses_ids', sa.dialects.postgresql.JSON, nullable=False, server_default="[]"),
-        sa.Column('report_type', sa.Integer, nullable=False, server_default="0"),
-        sa.Column('total_reports', sa.Integer, nullable=False, server_default="0"),
-        sa.Column('last_report', sa.Integer, nullable=False, server_default="0"),
-        sa.Column('occurences', sa.Integer, nullable=False, server_default="1"),
-        sa.Column('average_duration', sa.Float(), nullable=False, server_default="0"),
-        sa.Column('summed_duration', sa.Float(), nullable=False, server_default="0"),
-        sa.Column('notified', sa.Boolean, nullable=False, server_default="False"),
-        sa.Column('fixed', sa.Boolean, nullable=False, server_default="False"),
-        sa.Column('public', sa.Boolean, nullable=False, server_default="False"),
-        sa.Column('read', sa.Boolean, nullable=False, server_default="False"),
+        "reports_groups",
+        sa.Column("id", sa.types.BigInteger, primary_key=True),
+        sa.Column(
+            "resource_id",
+            sa.Integer,
+            sa.ForeignKey(
+                "resources.resource_id", onupdate="cascade", ondelete="cascade"
+            ),
+            nullable=False,
+        ),
+        sa.Column("priority", sa.Integer, nullable=False, server_default="5"),
+        sa.Column(
+            "first_timestamp",
+            sa.DateTime(),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.Column("last_timestamp", sa.DateTime()),
+        sa.Column("error", sa.UnicodeText, nullable=False, server_default=""),
+        sa.Column("grouping_hash", sa.Unicode(40), nullable=False, server_default=""),
+        sa.Column(
+            "triggered_postprocesses_ids",
+            sa.dialects.postgresql.JSON,
+            nullable=False,
+            server_default="[]",
+        ),
+        sa.Column("report_type", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("total_reports", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("last_report", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("occurences", sa.Integer, nullable=False, server_default="1"),
+        sa.Column("average_duration", sa.Float(), nullable=False, server_default="0"),
+        sa.Column("summed_duration", sa.Float(), nullable=False, server_default="0"),
+        sa.Column("notified", sa.Boolean, nullable=False, server_default="False"),
+        sa.Column("fixed", sa.Boolean, nullable=False, server_default="False"),
+        sa.Column("public", sa.Boolean, nullable=False, server_default="False"),
+        sa.Column("read", sa.Boolean, nullable=False, server_default="False"),
     )
 
     op.create_table(
-        'reports',
-        sa.Column('id', sa.types.BigInteger, primary_key=True),
-        sa.Column('group_id', sa.types.BigInteger,
-                  sa.ForeignKey('reports_groups.id', onupdate='cascade',
-                                ondelete='cascade'), nullable=False, index=True),
-        sa.Column('resource_id', sa.Integer, nullable=False, index=True),
-        sa.Column('report_type', sa.Integer, nullable=False, server_default="0"),
-        sa.Column('error', sa.UnicodeText, nullable=False, server_default=""),
-        sa.Column('extra', sa.dialects.postgresql.JSON, nullable=False, server_default="{}"),
-        sa.Column('request', sa.dialects.postgresql.JSON, nullable=False, server_default="{}"),
-        sa.Column('tags', sa.dialects.postgresql.JSON, nullable=False, server_default="{}"),
-        sa.Column('ip', sa.Unicode(39), nullable=False, server_default=""),
-        sa.Column('username', sa.Unicode(255), nullable=False, server_default=""),
-        sa.Column('user_agent', sa.Unicode(512), nullable=False, server_default=""),
-        sa.Column('url', sa.UnicodeText, nullable=False, server_default=""),
-        sa.Column('request_id', sa.Unicode(40), nullable=False, server_default=""),
-        sa.Column('request_stats', sa.dialects.postgresql.JSON, nullable=False, server_default="{}"),
-        sa.Column('traceback', sa.dialects.postgresql.JSON, nullable=False, server_default="{}"),
-        sa.Column('traceback_hash', sa.Unicode(40), nullable=False, server_default=""),
-        sa.Column('start_time', sa.DateTime(), nullable=False, server_default=sa.func.now()),
-        sa.Column('end_time', sa.DateTime()),
-        sa.Column('report_group_time', sa.DateTime, index=True, nullable=False, server_default=sa.func.now()),
-        sa.Column('duration', sa.Float(), nullable=False, server_default="0"),
-        sa.Column('http_status', sa.Integer, index=True),
-        sa.Column('url_domain', sa.Unicode(128)),
-        sa.Column('url_path', sa.UnicodeText),
-        sa.Column('language', sa.Integer, server_default="0"),
-    )
-    op.create_index(None, 'reports',
-                    [sa.text("(tags ->> 'server_name')")])
-    op.create_index(None, 'reports',
-                    [sa.text("(tags ->> 'view_name')")])
+        "reports",
+        sa.Column("id", sa.types.BigInteger, primary_key=True),
+        sa.Column(
+            "group_id",
+            sa.types.BigInteger,
+            sa.ForeignKey("reports_groups.id", onupdate="cascade", ondelete="cascade"),
+            nullable=False,
+            index=True,
+        ),
+        sa.Column("resource_id", sa.Integer, nullable=False, index=True),
+        sa.Column("report_type", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("error", sa.UnicodeText, nullable=False, server_default=""),
+        sa.Column(
+            "extra", sa.dialects.postgresql.JSON, nullable=False, server_default="{}"
+        ),
+        sa.Column(
+            "request", sa.dialects.postgresql.JSON, nullable=False, server_default="{}"
+        ),
+        sa.Column(
+            "tags", sa.dialects.postgresql.JSON, nullable=False, server_default="{}"
+        ),
+        sa.Column("ip", sa.Unicode(39), nullable=False, server_default=""),
+        sa.Column("username", sa.Unicode(255), nullable=False, server_default=""),
+        sa.Column("user_agent", sa.Unicode(512), nullable=False, server_default=""),
+        sa.Column("url", sa.UnicodeText, nullable=False, server_default=""),
+        sa.Column("request_id", sa.Unicode(40), nullable=False, server_default=""),
+        sa.Column(
+            "request_stats",
+            sa.dialects.postgresql.JSON,
+            nullable=False,
+            server_default="{}",
+        ),
+        sa.Column(
+            "traceback",
+            sa.dialects.postgresql.JSON,
+            nullable=False,
+            server_default="{}",
+        ),
+        sa.Column("traceback_hash", sa.Unicode(40), nullable=False, server_default=""),
+        sa.Column(
+            "start_time", sa.DateTime(), nullable=False, server_default=sa.func.now()
+        ),
+        sa.Column("end_time", sa.DateTime()),
+        sa.Column(
+            "report_group_time",
+            sa.DateTime,
+            index=True,
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.Column("duration", sa.Float(), nullable=False, server_default="0"),
+        sa.Column("http_status", sa.Integer, index=True),
+        sa.Column("url_domain", sa.Unicode(128)),
+        sa.Column("url_path", sa.UnicodeText),
+        sa.Column("language", sa.Integer, server_default="0"),
+    )
+    op.create_index(None, "reports", [sa.text("(tags ->> 'server_name')")])
+    op.create_index(None, "reports", [sa.text("(tags ->> 'view_name')")])
 
     op.create_table(
-        'reports_assignments',
-        sa.Column('group_id', sa.types.BigInteger, nullable=False, primary_key=True),
-        sa.Column('owner_id', sa.Integer,
-            sa.ForeignKey('users.id', onupdate='cascade',ondelete='cascade'),
-            nullable=False, primary_key=True),
-        sa.Column('report_time', sa.DateTime, nullable=False)
-        )
+        "reports_assignments",
+        sa.Column("group_id", sa.types.BigInteger, nullable=False, primary_key=True),
+        sa.Column(
+            "owner_id",
+            sa.Integer,
+            sa.ForeignKey("users.id", onupdate="cascade", ondelete="cascade"),
+            nullable=False,
+            primary_key=True,
+        ),
+        sa.Column("report_time", sa.DateTime, nullable=False),
+    )
 
     op.create_table(
-        'reports_comments',
-        sa.Column('comment_id', sa.Integer, primary_key=True),
-        sa.Column('body', sa.UnicodeText, nullable=False, server_default=''),
-        sa.Column('owner_id', sa.Integer,
-                  sa.ForeignKey('users.id', onupdate='cascade',
-                                ondelete='set null'), nullable=True),
-        sa.Column('created_timestamp', sa.DateTime, nullable=False, server_default=sa.func.now()),
-        sa.Column('report_time', sa.DateTime, nullable=False),
-        sa.Column('group_id', sa.types.BigInteger, nullable=False)
+        "reports_comments",
+        sa.Column("comment_id", sa.Integer, primary_key=True),
+        sa.Column("body", sa.UnicodeText, nullable=False, server_default=""),
+        sa.Column(
+            "owner_id",
+            sa.Integer,
+            sa.ForeignKey("users.id", onupdate="cascade", ondelete="set null"),
+            nullable=True,
+        ),
+        sa.Column(
+            "created_timestamp",
+            sa.DateTime,
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.Column("report_time", sa.DateTime, nullable=False),
+        sa.Column("group_id", sa.types.BigInteger, nullable=False),
     )
 
     op.create_table(
-        'reports_stats',
-        sa.Column('resource_id', sa.Integer, nullable=False, index=True),
-        sa.Column('start_interval', sa.DateTime, nullable=False, index=True),
-        sa.Column('group_id', sa.types.BigInteger, index=True),
-        sa.Column('occurences', sa.Integer, nullable=False, server_default='0', index=True),
-        sa.Column('owner_user_id', sa.Integer),
-        sa.Column('type', sa.Integer, index=True, nullable=False),
-        sa.Column('duration', sa.Float(), server_default='0'),
-        sa.Column('server_name', sa.Unicode(128),
-                  server_default=''),
-        sa.Column('view_name', sa.Unicode(128),
-                  server_default=''),
-        sa.Column('id', sa.BigInteger(), nullable=False, primary_key=True),
-    )
-    op.create_index('ix_reports_stats_start_interval_group_id', 'reports_stats',
-                    ["start_interval", "group_id"])
+        "reports_stats",
+        sa.Column("resource_id", sa.Integer, nullable=False, index=True),
+        sa.Column("start_interval", sa.DateTime, nullable=False, index=True),
+        sa.Column("group_id", sa.types.BigInteger, index=True),
+        sa.Column(
+            "occurences", sa.Integer, nullable=False, server_default="0", index=True
+        ),
+        sa.Column("owner_user_id", sa.Integer),
+        sa.Column("type", sa.Integer, index=True, nullable=False),
+        sa.Column("duration", sa.Float(), server_default="0"),
+        sa.Column("server_name", sa.Unicode(128), server_default=""),
+        sa.Column("view_name", sa.Unicode(128), server_default=""),
+        sa.Column("id", sa.BigInteger(), nullable=False, primary_key=True),
+    )
+    op.create_index(
+        "ix_reports_stats_start_interval_group_id",
+        "reports_stats",
+        ["start_interval", "group_id"],
+    )
 
     op.create_table(
-        'slow_calls',
-        sa.Column('id', sa.types.BigInteger, primary_key=True),
-        sa.Column('report_id', sa.types.BigInteger, sa.ForeignKey('reports.id', onupdate='cascade', ondelete='cascade'),
-                  nullable=False, index=True),
-        sa.Column('duration', sa.Float(), nullable=False, server_default="0", index=True),
-        sa.Column('timestamp', sa.DateTime, nullable=False, server_default=sa.func.now(), index=True),
-        sa.Column('report_group_time', sa.DateTime, index=True, nullable=False, server_default=sa.func.now()),
-        sa.Column('type', sa.Unicode(16), nullable=False, index=True),
-        sa.Column('statement', sa.UnicodeText, nullable=False, server_default=''),
-        sa.Column('parameters', sa.dialects.postgresql.JSON, nullable=False),
-        sa.Column('location', sa.UnicodeText, server_default=''),
-        sa.Column('subtype', sa.Unicode(16), nullable=False, index=True),
-        sa.Column('resource_id', sa.Integer, nullable=False, index=True),
-        sa.Column('statement_hash', sa.Unicode(60), index=True)
+        "slow_calls",
+        sa.Column("id", sa.types.BigInteger, primary_key=True),
+        sa.Column(
+            "report_id",
+            sa.types.BigInteger,
+            sa.ForeignKey("reports.id", onupdate="cascade", ondelete="cascade"),
+            nullable=False,
+            index=True,
+        ),
+        sa.Column(
+            "duration", sa.Float(), nullable=False, server_default="0", index=True
+        ),
+        sa.Column(
+            "timestamp",
+            sa.DateTime,
+            nullable=False,
+            server_default=sa.func.now(),
+            index=True,
+        ),
+        sa.Column(
+            "report_group_time",
+            sa.DateTime,
+            index=True,
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.Column("type", sa.Unicode(16), nullable=False, index=True),
+        sa.Column("statement", sa.UnicodeText, nullable=False, server_default=""),
+        sa.Column("parameters", sa.dialects.postgresql.JSON, nullable=False),
+        sa.Column("location", sa.UnicodeText, server_default=""),
+        sa.Column("subtype", sa.Unicode(16), nullable=False, index=True),
+        sa.Column("resource_id", sa.Integer, nullable=False, index=True),
+        sa.Column("statement_hash", sa.Unicode(60), index=True),
     )
 
     op.create_table(
-        'tags',
-        sa.Column('id', sa.types.BigInteger, primary_key=True),
-        sa.Column('resource_id', sa.Integer,
-                  sa.ForeignKey('resources.resource_id', onupdate='cascade',
-                                ondelete='cascade')),
-        sa.Column('first_timestamp', sa.DateTime, nullable=False, server_default=sa.func.now()),
-        sa.Column('last_timestamp', sa.DateTime, nullable=False, server_default=sa.func.now()),
-        sa.Column('name', sa.Unicode(32), nullable=False),
-        sa.Column('value', sa.dialects.postgresql.JSON, nullable=False),
-        sa.Column('times_seen', sa.Integer, nullable=False, server_default='1')
+        "tags",
+        sa.Column("id", sa.types.BigInteger, primary_key=True),
+        sa.Column(
+            "resource_id",
+            sa.Integer,
+            sa.ForeignKey(
+                "resources.resource_id", onupdate="cascade", ondelete="cascade"
+            ),
+        ),
+        sa.Column(
+            "first_timestamp", sa.DateTime, nullable=False, server_default=sa.func.now()
+        ),
+        sa.Column(
+            "last_timestamp", sa.DateTime, nullable=False, server_default=sa.func.now()
+        ),
+        sa.Column("name", sa.Unicode(32), nullable=False),
+        sa.Column("value", sa.dialects.postgresql.JSON, nullable=False),
+        sa.Column("times_seen", sa.Integer, nullable=False, server_default="1"),
     )
 
     op.create_table(
-        'auth_tokens',
-        sa.Column('id', sa.Integer, nullable=False, primary_key=True),
-        sa.Column('token', sa.Unicode),
-        sa.Column('creation_date', sa.DateTime, nullable=False, server_default=sa.func.now()),
-        sa.Column('expires', sa.DateTime),
-        sa.Column('owner_id', sa.Integer,
-                  sa.ForeignKey('users.id', onupdate='cascade',
-                                ondelete='cascade')),
-        sa.Column('description', sa.Unicode),
+        "auth_tokens",
+        sa.Column("id", sa.Integer, nullable=False, primary_key=True),
+        sa.Column("token", sa.Unicode),
+        sa.Column(
+            "creation_date", sa.DateTime, nullable=False, server_default=sa.func.now()
+        ),
+        sa.Column("expires", sa.DateTime),
+        sa.Column(
+            "owner_id",
+            sa.Integer,
+            sa.ForeignKey("users.id", onupdate="cascade", ondelete="cascade"),
+        ),
+        sa.Column("description", sa.Unicode),
     )
 
     op.create_table(
-        'channels_actions',
-        sa.Column('channel_pkey', sa.Integer,
-                  sa.ForeignKey('alert_channels.pkey',
-                                ondelete='CASCADE', onupdate='CASCADE')),
-        sa.Column('action_pkey', sa.Integer,
-                  sa.ForeignKey('alert_channels_actions.pkey',
-                                ondelete='CASCADE', onupdate='CASCADE'))
+        "channels_actions",
+        sa.Column(
+            "channel_pkey",
+            sa.Integer,
+            sa.ForeignKey(
+                "alert_channels.pkey", ondelete="CASCADE", onupdate="CASCADE"
+            ),
+        ),
+        sa.Column(
+            "action_pkey",
+            sa.Integer,
+            sa.ForeignKey(
+                "alert_channels_actions.pkey", ondelete="CASCADE", onupdate="CASCADE"
+            ),
+        ),
     )
 
     op.create_table(
-        'config',
-        sa.Column('key', sa.Unicode(128), primary_key=True),
-        sa.Column('section', sa.Unicode(128), primary_key=True),
-        sa.Column('value', sa.dialects.postgresql.JSON,
-                  server_default="{}")
+        "config",
+        sa.Column("key", sa.Unicode(128), primary_key=True),
+        sa.Column("section", sa.Unicode(128), primary_key=True),
+        sa.Column("value", sa.dialects.postgresql.JSON, server_default="{}"),
     )
 
     op.create_table(
-        'plugin_configs',
-        sa.Column('id', sa.Integer, primary_key=True),
-        sa.Column('plugin_name', sa.Unicode(128)),
-        sa.Column('section', sa.Unicode(128)),
-        sa.Column('config', sa.dialects.postgresql.JSON,
-                  server_default="{}"),
-        sa.Column('resource_id', sa.Integer(),
-                  sa.ForeignKey('resources.resource_id', onupdate='cascade',
-                                ondelete='cascade')),
-        sa.Column('owner_id', sa.Integer(),
-                  sa.ForeignKey('users.id', onupdate='cascade',
-                                ondelete='cascade')))
+        "plugin_configs",
+        sa.Column("id", sa.Integer, primary_key=True),
+        sa.Column("plugin_name", sa.Unicode(128)),
+        sa.Column("section", sa.Unicode(128)),
+        sa.Column("config", sa.dialects.postgresql.JSON, server_default="{}"),
+        sa.Column(
+            "resource_id",
+            sa.Integer(),
+            sa.ForeignKey(
+                "resources.resource_id", onupdate="cascade", ondelete="cascade"
+            ),
+        ),
+        sa.Column(
+            "owner_id",
+            sa.Integer(),
+            sa.ForeignKey("users.id", onupdate="cascade", ondelete="cascade"),
+        ),
+    )
 
     op.create_table(
-        'rc_versions',
-        sa.Column('name', sa.Unicode(40), primary_key=True),
-        sa.Column('value', sa.Unicode(40)),
+        "rc_versions",
+        sa.Column("name", sa.Unicode(40), primary_key=True),
+        sa.Column("value", sa.Unicode(40)),
+    )
+    version_table = sa.table(
+        "rc_versions",
+        sa.Column("name", sa.Unicode(40)),
+        sa.Column("value", sa.Unicode(40)),
     )
-    version_table = sa.table('rc_versions',
-                             sa.Column('name', sa.Unicode(40)),
-                             sa.Column('value', sa.Unicode(40)))
 
-    insert = version_table.insert().values(name='es_reports')
+    insert = version_table.insert().values(name="es_reports")
     op.execute(insert)
-    insert = version_table.insert().values(name='es_reports_groups')
+    insert = version_table.insert().values(name="es_reports_groups")
     op.execute(insert)
-    insert = version_table.insert().values(name='es_reports_stats')
+    insert = version_table.insert().values(name="es_reports_stats")
     op.execute(insert)
-    insert = version_table.insert().values(name='es_logs')
+    insert = version_table.insert().values(name="es_logs")
     op.execute(insert)
-    insert = version_table.insert().values(name='es_metrics')
+    insert = version_table.insert().values(name="es_metrics")
     op.execute(insert)
-    insert = version_table.insert().values(name='es_slow_calls')
+    insert = version_table.insert().values(name="es_slow_calls")
     op.execute(insert)
 
-
-    op.execute('''
+    op.execute(
+        """
         CREATE OR REPLACE FUNCTION floor_time_5min(timestamp without time zone)
           RETURNS timestamp without time zone AS
         $BODY$SELECT date_trunc('hour', $1) + INTERVAL '5 min' * FLOOR(date_part('minute', $1) / 5.0)$BODY$
           LANGUAGE sql VOLATILE;
-    ''')
+    """
+    )
 
-    op.execute('''
+    op.execute(
+        """
     CREATE OR REPLACE FUNCTION partition_logs() RETURNS trigger
         LANGUAGE plpgsql SECURITY DEFINER
         AS $$
     DECLARE
     main_table         varchar := 'logs';
     partitioned_table  varchar := '';
     BEGIN
 
     IF NEW.permanent THEN
         partitioned_table := main_table || '_p_' || date_part('year', NEW.timestamp)::TEXT || '_' || DATE_part('month', NEW.timestamp);
     ELSE
         partitioned_table := main_table || '_p_' || date_part('year', NEW.timestamp)::TEXT || '_' || DATE_part('month', NEW.timestamp) || '_' || DATE_part('day', NEW.timestamp);
     END IF;
 
     BEGIN
     EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
     EXCEPTION
     WHEN undefined_table THEN
         RAISE NOTICE 'A partition has been created %', partitioned_table;
         IF NEW.permanent THEN
         EXECUTE format('CREATE TABLE  IF NOT EXISTS %s ( CHECK( timestamp >= DATE %s AND timestamp < DATE %s)) INHERITS (%s)',
                 partitioned_table,
                 quote_literal(date_trunc('month', NEW.timestamp)::date) ,
                 quote_literal((date_trunc('month', NEW.timestamp)::date  + interval '1 month')::text),
                 main_table);
         EXECUTE format('ALTER TABLE %s ADD CONSTRAINT pk_%s PRIMARY KEY(log_id);', partitioned_table, partitioned_table);
         EXECUTE format('ALTER TABLE %s ADD CONSTRAINT fk_%s_resource_id FOREIGN KEY (resource_id) REFERENCES resources (resource_id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE;', partitioned_table, partitioned_table);
         EXECUTE format('CREATE INDEX ix_%s_timestamp ON %s (timestamp);', partitioned_table, partitioned_table);
         EXECUTE format('CREATE INDEX ix_%s_namespace_resource_id ON %s (namespace, resource_id);', partitioned_table, partitioned_table);
         EXECUTE format('CREATE INDEX ix_%s_resource_id ON %s (resource_id);', partitioned_table, partitioned_table);
         EXECUTE format('CREATE INDEX ix_%s_pkey_namespace ON %s (primary_key, namespace);', partitioned_table, partitioned_table);
         ELSE
         EXECUTE format('CREATE TABLE  IF NOT EXISTS %s ( CHECK( timestamp >= DATE %s AND timestamp < DATE %s)) INHERITS (%s)',
                 partitioned_table,
                 quote_literal(date_trunc('day', NEW.timestamp)::date) ,
                 quote_literal((date_trunc('day', NEW.timestamp)::date  + interval '1 day')::text),
                 main_table);
         EXECUTE format('ALTER TABLE %s ADD CONSTRAINT pk_%s_ PRIMARY KEY(log_id);', partitioned_table, partitioned_table);
         EXECUTE format('ALTER TABLE %s ADD CONSTRAINT fk_%s_resource_id FOREIGN KEY (resource_id) REFERENCES resources (resource_id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE;', partitioned_table, partitioned_table);
         EXECUTE format('CREATE INDEX ix_%s_timestamp ON %s (timestamp);', partitioned_table, partitioned_table);
         EXECUTE format('CREATE INDEX ix_%s_namespace_resource_id ON %s (namespace, resource_id);', partitioned_table, partitioned_table);
         EXECUTE format('CREATE INDEX ix_%s_resource_id ON %s (resource_id);', partitioned_table, partitioned_table);
         EXECUTE format('CREATE INDEX ix_%s_primary_key_namespace ON %s (primary_key,namespace);', partitioned_table, partitioned_table);
         END IF;
 
 
         EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
     END;
 
 
     RETURN NULL;
     END
     $$;
-    ''')
+    """
+    )
 
-    op.execute('''
+    op.execute(
+        """
     CREATE TRIGGER partition_logs BEFORE INSERT ON logs FOR EACH ROW EXECUTE PROCEDURE partition_logs();
-    ''')
+    """
+    )
 
-    op.execute('''
+    op.execute(
+        """
     CREATE OR REPLACE FUNCTION partition_metrics() RETURNS trigger
         LANGUAGE plpgsql SECURITY DEFINER
         AS $$
         DECLARE
         main_table         varchar := 'metrics';
         partitioned_table  varchar := '';
         BEGIN
 
         partitioned_table := main_table || '_p_' || date_part('year', NEW.timestamp)::TEXT || '_' || DATE_part('month', NEW.timestamp) || '_' || DATE_part('day', NEW.timestamp);
 
     BEGIN
     EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
     EXCEPTION
     WHEN undefined_table THEN
         RAISE NOTICE 'A partition has been created %', partitioned_table;
         EXECUTE format('CREATE TABLE  IF NOT EXISTS %s ( CHECK( timestamp >= DATE %s AND timestamp < DATE %s)) INHERITS (%s)',
                 partitioned_table,
                 quote_literal(date_trunc('day', NEW.timestamp)::date) ,
                 quote_literal((date_trunc('day', NEW.timestamp)::date  + interval '1 day')::text),
                 main_table);
         EXECUTE format('ALTER TABLE %s ADD CONSTRAINT pk_%s PRIMARY KEY(pkey);', partitioned_table, partitioned_table);
         EXECUTE format('ALTER TABLE %s ADD CONSTRAINT fk_%s_resource_id FOREIGN KEY (resource_id) REFERENCES resources (resource_id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE;', partitioned_table, partitioned_table);
         EXECUTE format('CREATE INDEX ix_%s_timestamp ON %s (timestamp);', partitioned_table, partitioned_table);
         EXECUTE format('CREATE INDEX ix_%s_resource_id ON %s (resource_id);', partitioned_table, partitioned_table);
         EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
     END;
 
     RETURN NULL;
     END
     $$;
-    ''')
+    """
+    )
 
-    op.execute('''
+    op.execute(
+        """
     CREATE TRIGGER partition_metrics BEFORE INSERT ON metrics FOR EACH ROW EXECUTE PROCEDURE partition_metrics();
-    ''')
+    """
+    )
 
-    op.execute('''
+    op.execute(
+        """
     CREATE FUNCTION partition_reports_stats() RETURNS trigger
         LANGUAGE plpgsql SECURITY DEFINER
         AS $$
         DECLARE
             main_table         varchar := 'reports_stats';
             partitioned_table  varchar := '';
         BEGIN
 
             partitioned_table := main_table || '_p_' || date_part('year', NEW.start_interval)::TEXT || '_' || DATE_part('month', NEW.start_interval);
 
             BEGIN
                 EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
             EXCEPTION
             WHEN undefined_table THEN
                 RAISE NOTICE 'A partition has been created %', partitioned_table;
                 EXECUTE format('CREATE TABLE  IF NOT EXISTS %s ( CHECK( start_interval >= DATE %s AND start_interval < DATE %s )) INHERITS (%s)',
                     partitioned_table,
                     quote_literal(date_trunc('month', NEW.start_interval)::date) ,
                     quote_literal((date_trunc('month', NEW.start_interval)::date  + interval '1 month')::text),
                     main_table);
                 EXECUTE format('ALTER TABLE %s ADD CONSTRAINT pk_%s PRIMARY KEY(id);', partitioned_table, partitioned_table);
                 EXECUTE format('CREATE INDEX ix_%s_start_interval ON %s USING btree (start_interval);', partitioned_table, partitioned_table);
                 EXECUTE format('CREATE INDEX ix_%s_type ON %s USING btree (type);', partitioned_table, partitioned_table);
                 EXECUTE format('CREATE INDEX ix_%s_resource_id ON %s USING btree (resource_id);', partitioned_table, partitioned_table);
                 EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
             END;
             RETURN NULL;
         END
         $$;
-    ''')
+    """
+    )
 
-    op.execute('''
+    op.execute(
+        """
     CREATE TRIGGER partition_reports_stats BEFORE INSERT ON reports_stats FOR EACH ROW EXECUTE PROCEDURE partition_reports_stats();
-    ''')
+    """
+    )
 
-    op.execute('''
+    op.execute(
+        """
     CREATE OR REPLACE FUNCTION partition_reports_groups() RETURNS trigger
         LANGUAGE plpgsql SECURITY DEFINER
         AS $$
     DECLARE
         main_table         varchar := 'reports_groups';
         partitioned_table  varchar := '';
     BEGIN
 
         partitioned_table := main_table || '_p_' || date_part('year', NEW.first_timestamp)::TEXT || '_' || DATE_part('month', NEW.first_timestamp);
 
         BEGIN
             EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
         EXCEPTION
         WHEN undefined_table THEN
             RAISE NOTICE 'A partition has been created %', partitioned_table;
             EXECUTE format('CREATE TABLE  IF NOT EXISTS %s ( CHECK( first_timestamp >= DATE %s AND first_timestamp < DATE %s )) INHERITS (%s)',
                 partitioned_table,
                 quote_literal(date_trunc('month', NEW.first_timestamp)::date) ,
                 quote_literal((date_trunc('month', NEW.first_timestamp)::date  + interval '1 month')::text),
                 main_table);
             EXECUTE format('ALTER TABLE %s ADD CONSTRAINT pk_%s PRIMARY KEY(id);', partitioned_table, partitioned_table);
             EXECUTE format('ALTER TABLE %s ADD CONSTRAINT fk_%s_resource_id FOREIGN KEY (resource_id) REFERENCES resources (resource_id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE;', partitioned_table, partitioned_table);
             EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
         END;
         RETURN NULL;
     END
     $$;
-    ''')
+    """
+    )
 
-    op.execute('''
+    op.execute(
+        """
     CREATE TRIGGER partition_reports_groups BEFORE INSERT ON reports_groups FOR EACH ROW EXECUTE PROCEDURE partition_reports_groups();
-    ''')
+    """
+    )
 
-    op.execute('''
+    op.execute(
+        """
     CREATE OR REPLACE FUNCTION partition_reports() RETURNS trigger
         LANGUAGE plpgsql SECURITY DEFINER
         AS $$
     DECLARE
         main_table         varchar := 'reports';
         partitioned_table  varchar := '';
         partitioned_parent_table  varchar := '';
     BEGIN
 
         partitioned_table := main_table || '_p_' || date_part('year', NEW.report_group_time)::TEXT || '_' || DATE_part('month', NEW.report_group_time);
         partitioned_parent_table := 'reports_groups_p_' || date_part('year', NEW.report_group_time)::TEXT || '_' || DATE_part('month', NEW.report_group_time);
 
         BEGIN
             EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
         EXCEPTION
         WHEN undefined_table THEN
             RAISE NOTICE 'A partition has been created %', partitioned_table;
             EXECUTE format('CREATE TABLE  IF NOT EXISTS %s ( CHECK( report_group_time >= DATE %s AND report_group_time < DATE %s )) INHERITS (%s)',
                 partitioned_table,
                 quote_literal(date_trunc('month', NEW.report_group_time)::date) ,
                 quote_literal((date_trunc('month', NEW.report_group_time)::date  + interval '1 month')::text),
                 main_table);
             EXECUTE format('ALTER TABLE %s ADD CONSTRAINT pk_%s PRIMARY KEY(id);', partitioned_table, partitioned_table);
             EXECUTE format('ALTER TABLE %s ADD CONSTRAINT fk_%s_resource_id FOREIGN KEY (resource_id) REFERENCES resources (resource_id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE;', partitioned_table, partitioned_table);
             EXECUTE format('ALTER TABLE %s ADD CONSTRAINT fk_%s_group_id FOREIGN KEY (group_id) REFERENCES %s (id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE;', partitioned_table, partitioned_table, partitioned_parent_table);
             EXECUTE format('CREATE INDEX ix_%s_report_group_time ON %s USING btree (report_group_time);', partitioned_table, partitioned_table);
             EXECUTE format('CREATE INDEX ix_%s_group_id ON %s USING btree (group_id);', partitioned_table, partitioned_table);
             EXECUTE format('CREATE INDEX ix_%s_resource_id ON %s USING btree (resource_id);', partitioned_table, partitioned_table);
             EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
         END;
         RETURN NULL;
     END
     $$;
-    ''')
+    """
+    )
 
-    op.execute('''
+    op.execute(
+        """
     CREATE TRIGGER partition_reports BEFORE INSERT ON reports FOR EACH ROW EXECUTE PROCEDURE partition_reports();
-    ''')
-
+    """
+    )
 
-    op.execute('''
+    op.execute(
+        """
     CREATE OR REPLACE FUNCTION partition_slow_calls() RETURNS trigger
         LANGUAGE plpgsql SECURITY DEFINER
         AS $$
     DECLARE
         main_table         varchar := 'slow_calls';
         partitioned_table  varchar := '';
         partitioned_parent_table  varchar := '';
     BEGIN
 
         partitioned_table := main_table || '_p_' || date_part('year', NEW.report_group_time)::TEXT || '_' || DATE_part('month', NEW.report_group_time);
         partitioned_parent_table := 'reports_p_' || date_part('year', NEW.report_group_time)::TEXT || '_' || DATE_part('month', NEW.report_group_time);
 
         BEGIN
             EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
         EXCEPTION
         WHEN undefined_table THEN
             RAISE NOTICE 'A partition has been created %', partitioned_table;
             EXECUTE format('CREATE TABLE  IF NOT EXISTS %s ( CHECK( report_group_time >= DATE %s AND report_group_time < DATE %s )) INHERITS (%s)',
                 partitioned_table,
                 quote_literal(date_trunc('month', NEW.report_group_time)::date) ,
                 quote_literal((date_trunc('month', NEW.report_group_time)::date  + interval '1 month')::text),
                 main_table);
             EXECUTE format('ALTER TABLE %s ADD CONSTRAINT pk_%s PRIMARY KEY(id);', partitioned_table, partitioned_table);
             EXECUTE format('ALTER TABLE %s ADD CONSTRAINT fk_%s_resource_id FOREIGN KEY (resource_id) REFERENCES resources (resource_id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE;', partitioned_table, partitioned_table);
             EXECUTE format('ALTER TABLE %s ADD CONSTRAINT fk_%s_report_id FOREIGN KEY (report_id) REFERENCES %s (id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE;', partitioned_table, partitioned_table, partitioned_parent_table);
             EXECUTE format('CREATE INDEX ix_%s_resource_id  ON %s USING btree (resource_id);', partitioned_table, partitioned_table);
             EXECUTE format('CREATE INDEX ix_%s_report_id ON %s USING btree (report_id);', partitioned_table, partitioned_table);
             EXECUTE format('CREATE INDEX ix_%s_timestamp ON %s USING btree (timestamp);', partitioned_table, partitioned_table);
             EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
         END;
         RETURN NULL;
     END
     $$;
-    ''')
+    """
+    )
 
-    op.execute('''
+    op.execute(
+        """
     CREATE TRIGGER partition_slow_calls BEFORE INSERT ON slow_calls FOR EACH ROW EXECUTE PROCEDURE partition_slow_calls();
-    ''')
+    """
+    )
+
 
 def downgrade():
     pass

backend/src/appenlight/migrations/versions/e9fcfbdd9498_connect_resources_to_alert_channels.py

@@ -1,32 +1,40 b''
 """connect resources to alert_channels
 
 Revision ID: e9fcfbdd9498
 Revises: 55b6e612672f
 Create Date: 2018-02-28 13:52:50.717217
 
 """
 
 # revision identifiers, used by Alembic.
-revision = 'e9fcfbdd9498'
-down_revision = '55b6e612672f'
+revision = "e9fcfbdd9498"
+down_revision = "55b6e612672f"
 
 from alembic import op
 import sqlalchemy as sa
 
 
 def upgrade():
     op.create_table(
-        'channels_resources',
-        sa.Column('channel_pkey', sa.Integer,
-                  sa.ForeignKey('alert_channels.pkey',
-                                ondelete='CASCADE', onupdate='CASCADE'),
-                  primary_key=True),
-        sa.Column('resource_id', sa.Integer,
-                  sa.ForeignKey('resources.resource_id',
-                                ondelete='CASCADE', onupdate='CASCADE'),
-                  primary_key=True)
+        "channels_resources",
+        sa.Column(
+            "channel_pkey",
+            sa.Integer,
+            sa.ForeignKey(
+                "alert_channels.pkey", ondelete="CASCADE", onupdate="CASCADE"
+            ),
+            primary_key=True,
+        ),
+        sa.Column(
+            "resource_id",
+            sa.Integer,
+            sa.ForeignKey(
+                "resources.resource_id", ondelete="CASCADE", onupdate="CASCADE"
+            ),
+            primary_key=True,
+        ),
     )
 
 
 def downgrade():
-    op.drop_table('channels_resources')
+    op.drop_table("channels_resources")

backend/src/appenlight/models/__init__.py

@@ -1,130 +1,139 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy import MetaData
 from sqlalchemy.orm import scoped_session
 from sqlalchemy.orm import sessionmaker
 from zope.sqlalchemy import ZopeTransactionExtension
 import ziggurat_foundations
 from ziggurat_foundations.models.base import get_db_session
 
 log = logging.getLogger(__name__)
 
 DBSession = scoped_session(sessionmaker(extension=ZopeTransactionExtension()))
 
 NAMING_CONVENTION = {
-    "ix": 'ix_%(column_0_label)s',
+    "ix": "ix_%(column_0_label)s",
     "uq": "uq_%(table_name)s_%(column_0_name)s",
     "ck": "ck_%(table_name)s_%(constraint_name)s",
     "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
-    "pk": "pk_%(table_name)s"
+    "pk": "pk_%(table_name)s",
 }
 
 metadata = MetaData(naming_convention=NAMING_CONVENTION)
 Base = declarative_base(metadata=metadata)
 
 # optional for request.db approach
 ziggurat_foundations.models.DBSession = DBSession
 
 
 class Datastores(object):
     redis = None
     es = None
 
 
 def register_datastores(es_conn, redis_conn, redis_lockmgr):
     Datastores.es = es_conn
     Datastores.redis = redis_conn
     Datastores.lockmgr = redis_lockmgr
 
 
 class SliceableESQuery(object):
     def __init__(self, query, sort_query=None, aggregations=False, **kwconfig):
         self.query = query
         self.sort_query = sort_query
         self.aggregations = aggregations
-        self.items_per_page = kwconfig.pop('items_per_page', 10)
-        self.page = kwconfig.pop('page', 1)
+        self.items_per_page = kwconfig.pop("items_per_page", 10)
+        self.page = kwconfig.pop("page", 1)
         self.kwconfig = kwconfig
         self.result = None
 
     def __getitem__(self, index):
         config = self.kwconfig.copy()
-        config['from_'] = index.start
+        config["from_"] = index.start
         query = self.query.copy()
         if self.sort_query:
             query.update(self.sort_query)
-        self.result = Datastores.es.search(body=query, size=self.items_per_page,
-                                           **config)
+        self.result = Datastores.es.search(
+            body=query, size=self.items_per_page, **config
+        )
         if self.aggregations:
-            self.items = self.result.get('aggregations')
+            self.items = self.result.get("aggregations")
         else:
-            self.items = self.result['hits']['hits']
+            self.items = self.result["hits"]["hits"]
 
         return self.items
 
     def __iter__(self):
         return self.result
 
     def __len__(self):
         config = self.kwconfig.copy()
         query = self.query.copy()
-        self.result = Datastores.es.search(body=query, size=self.items_per_page,
-                                           **config)
+        self.result = Datastores.es.search(
+            body=query, size=self.items_per_page, **config
+        )
         if self.aggregations:
-            self.items = self.result.get('aggregations')
+            self.items = self.result.get("aggregations")
         else:
-            self.items = self.result['hits']['hits']
+            self.items = self.result["hits"]["hits"]
 
-        count = int(self.result['hits']['total'])
+        count = int(self.result["hits"]["total"])
         return count if count < 5000 else 5000
 
 
 from appenlight.models.resource import Resource
 from appenlight.models.application import Application
 from appenlight.models.user import User
 from appenlight.models.alert_channel import AlertChannel
 from appenlight.models.alert_channel_action import AlertChannelAction
 from appenlight.models.metric import Metric
-from appenlight.models.application_postprocess_conf import \
-    ApplicationPostprocessConf
+from appenlight.models.application_postprocess_conf import ApplicationPostprocessConf
 from appenlight.models.auth_token import AuthToken
 from appenlight.models.event import Event
 from appenlight.models.external_identity import ExternalIdentity
 from appenlight.models.group import Group
 from appenlight.models.group_permission import GroupPermission
 from appenlight.models.group_resource_permission import GroupResourcePermission
 from appenlight.models.log import Log
 from appenlight.models.plugin_config import PluginConfig
 from appenlight.models.report import Report
 from appenlight.models.report_group import ReportGroup
 from appenlight.models.report_comment import ReportComment
 from appenlight.models.report_assignment import ReportAssignment
 from appenlight.models.report_stat import ReportStat
 from appenlight.models.slow_call import SlowCall
 from appenlight.models.tag import Tag
 from appenlight.models.user_group import UserGroup
 from appenlight.models.user_permission import UserPermission
 from appenlight.models.user_resource_permission import UserResourcePermission
 from ziggurat_foundations import ziggurat_model_init
 
-ziggurat_model_init(User, Group, UserGroup, GroupPermission, UserPermission,
-                    UserResourcePermission, GroupResourcePermission,
-                    Resource,
-                    ExternalIdentity, passwordmanager=None)
+ziggurat_model_init(
+    User,
+    Group,
+    UserGroup,
+    GroupPermission,
+    UserPermission,
+    UserResourcePermission,
+    GroupResourcePermission,
+    Resource,
+    ExternalIdentity,
+    passwordmanager=None,
+)

backend/src/appenlight/models/alert_channel.py

@@ -1,305 +1,298 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 import sqlalchemy as sa
 import urllib.request, urllib.parse, urllib.error
 from datetime import timedelta
 from appenlight.models import Base
 from appenlight.lib.utils.date_utils import convert_date
 from sqlalchemy.dialects.postgresql import JSON
 from ziggurat_foundations.models.base import BaseModel
 
 log = logging.getLogger(__name__)
 
 #
 channel_rules_m2m_table = sa.Table(
-    'channels_actions', Base.metadata,
-    sa.Column('channel_pkey', sa.Integer,
-              sa.ForeignKey('alert_channels.pkey')),
-    sa.Column('action_pkey', sa.Integer,
-              sa.ForeignKey('alert_channels_actions.pkey'))
+    "channels_actions",
+    Base.metadata,
+    sa.Column("channel_pkey", sa.Integer, sa.ForeignKey("alert_channels.pkey")),
+    sa.Column("action_pkey", sa.Integer, sa.ForeignKey("alert_channels_actions.pkey")),
 )
 
 channel_resources_m2m_table = sa.Table(
-    'channels_resources', Base.metadata,
-    sa.Column('channel_pkey', sa.Integer,
-              sa.ForeignKey('alert_channels.pkey')),
-    sa.Column('resource_id', sa.Integer,
-              sa.ForeignKey('resources.resource_id'))
+    "channels_resources",
+    Base.metadata,
+    sa.Column("channel_pkey", sa.Integer, sa.ForeignKey("alert_channels.pkey")),
+    sa.Column("resource_id", sa.Integer, sa.ForeignKey("resources.resource_id")),
 )
 
-DATE_FRMT = '%Y-%m-%dT%H:%M'
+DATE_FRMT = "%Y-%m-%dT%H:%M"
 
 
 class AlertChannel(Base, BaseModel):
     """
     Stores information about possible alerting options
     """
-    __tablename__ = 'alert_channels'
-    __possible_channel_names__ = ['email']
+
+    __tablename__ = "alert_channels"
+    __possible_channel_names__ = ["email"]
     __mapper_args__ = {
-        'polymorphic_on': 'channel_name',
-        'polymorphic_identity': 'integration'
+        "polymorphic_on": "channel_name",
+        "polymorphic_identity": "integration",
     }
 
-    owner_id = sa.Column(sa.Unicode(30),
-                         sa.ForeignKey('users.id', onupdate='CASCADE',
-                                       ondelete='CASCADE'))
+    owner_id = sa.Column(
+        sa.Unicode(30),
+        sa.ForeignKey("users.id", onupdate="CASCADE", ondelete="CASCADE"),
+    )
     channel_name = sa.Column(sa.Unicode(25), nullable=False)
-    channel_value = sa.Column(sa.Unicode(80), nullable=False, default='')
-    channel_json_conf = sa.Column(JSON(), nullable=False, default='')
-    channel_validated = sa.Column(sa.Boolean, nullable=False,
-                                  default=False)
-    send_alerts = sa.Column(sa.Boolean, nullable=False,
-                            default=True)
-    daily_digest = sa.Column(sa.Boolean, nullable=False,
-                             default=True)
-    integration_id = sa.Column(sa.Integer, sa.ForeignKey('integrations.id'),
-                               nullable=True)
+    channel_value = sa.Column(sa.Unicode(80), nullable=False, default="")
+    channel_json_conf = sa.Column(JSON(), nullable=False, default="")
+    channel_validated = sa.Column(sa.Boolean, nullable=False, default=False)
+    send_alerts = sa.Column(sa.Boolean, nullable=False, default=True)
+    daily_digest = sa.Column(sa.Boolean, nullable=False, default=True)
+    integration_id = sa.Column(
+        sa.Integer, sa.ForeignKey("integrations.id"), nullable=True
+    )
     pkey = sa.Column(sa.Integer(), nullable=False, primary_key=True)
 
-    channel_actions = sa.orm.relationship('AlertChannelAction',
-                                          cascade="all",
-                                          passive_deletes=True,
-                                          passive_updates=True,
-                                          secondary=channel_rules_m2m_table,
-                                          backref='channels')
-    resources = sa.orm.relationship('Resource',
-                                    cascade="all",
-                                    passive_deletes=True,
-                                    passive_updates=True,
-                                    secondary=channel_resources_m2m_table,
-                                    backref='resources')
+    channel_actions = sa.orm.relationship(
+        "AlertChannelAction",
+        cascade="all",
+        passive_deletes=True,
+        passive_updates=True,
+        secondary=channel_rules_m2m_table,
+        backref="channels",
+    )
+    resources = sa.orm.relationship(
+        "Resource",
+        cascade="all",
+        passive_deletes=True,
+        passive_updates=True,
+        secondary=channel_resources_m2m_table,
+        backref="resources",
+    )
 
     @property
     def channel_visible_value(self):
         if self.integration:
-            return '{}: {}'.format(
-                self.channel_name,
-                self.integration.resource.resource_name
+            return "{}: {}".format(
+                self.channel_name, self.integration.resource.resource_name
             )
 
-        return '{}: {}'.format(
-            self.channel_name,
-            self.channel_value
-        )
+        return "{}: {}".format(self.channel_name, self.channel_value)
 
-    def get_dict(self, exclude_keys=None, include_keys=None,
-                 extended_info=True):
+    def get_dict(self, exclude_keys=None, include_keys=None, extended_info=True):
         """
         Returns dictionary with required information that will be consumed by
         angular
         """
-        instance_dict = super(AlertChannel, self).get_dict(exclude_keys,
-                                                           include_keys)
+        instance_dict = super(AlertChannel, self).get_dict(exclude_keys, include_keys)
         exclude_keys_list = exclude_keys or []
         include_keys_list = include_keys or []
 
-        instance_dict['supports_report_alerting'] = True
-        instance_dict['channel_visible_value'] = self.channel_visible_value
+        instance_dict["supports_report_alerting"] = True
+        instance_dict["channel_visible_value"] = self.channel_visible_value
 
         if extended_info:
-            instance_dict['actions'] = [
-                rule.get_dict(extended_info=True) for
-                rule in self.channel_actions]
+            instance_dict["actions"] = [
+                rule.get_dict(extended_info=True) for rule in self.channel_actions
+            ]
 
-        del instance_dict['channel_json_conf']
+        del instance_dict["channel_json_conf"]
 
         if self.integration:
             instance_dict[
-                'supports_report_alerting'] = \
-                self.integration.supports_report_alerting
+                "supports_report_alerting"
+            ] = self.integration.supports_report_alerting
         d = {}
         for k in instance_dict.keys():
-            if (k not in exclude_keys_list and
-                    (k in include_keys_list or not include_keys)):
+            if k not in exclude_keys_list and (
+                k in include_keys_list or not include_keys
+            ):
                 d[k] = instance_dict[k]
         return d
 
     def __repr__(self):
-        return '<AlertChannel: (%s,%s), user:%s>' % (self.channel_name,
-                                                     self.channel_value,
-                                                     self.user_name,)
+        return "<AlertChannel: (%s,%s), user:%s>" % (
+            self.channel_name,
+            self.channel_value,
+            self.user_name,
+        )
 
     def send_digest(self, **kwargs):
         """
         This should implement daily top error report notifications
         """
-        log.warning('send_digest NOT IMPLEMENTED')
+        log.warning("send_digest NOT IMPLEMENTED")
 
     def notify_reports(self, **kwargs):
         """
         This should implement notification of reports that occured in 1 min
         interval
         """
-        log.warning('notify_reports NOT IMPLEMENTED')
+        log.warning("notify_reports NOT IMPLEMENTED")
 
     def notify_alert(self, **kwargs):
         """
         Notify user of report/uptime/chart threshold events based on events alert
         type
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
-        alert_name = kwargs['event'].unified_alert_name()
-        if alert_name in ['slow_report_alert', 'error_report_alert']:
+        alert_name = kwargs["event"].unified_alert_name()
+        if alert_name in ["slow_report_alert", "error_report_alert"]:
             self.notify_report_alert(**kwargs)
-        elif alert_name == 'uptime_alert':
+        elif alert_name == "uptime_alert":
             self.notify_uptime_alert(**kwargs)
-        elif alert_name == 'chart_alert':
+        elif alert_name == "chart_alert":
             self.notify_chart_alert(**kwargs)
 
     def notify_chart_alert(self, **kwargs):
         """
         This should implement report open/close alerts notifications
         """
-        log.warning('notify_chart_alert NOT IMPLEMENTED')
+        log.warning("notify_chart_alert NOT IMPLEMENTED")
 
     def notify_report_alert(self, **kwargs):
         """
         This should implement report open/close alerts notifications
         """
-        log.warning('notify_report_alert NOT IMPLEMENTED')
+        log.warning("notify_report_alert NOT IMPLEMENTED")
 
     def notify_uptime_alert(self, **kwargs):
         """
         This should implement uptime open/close alerts notifications
         """
-        log.warning('notify_uptime_alert NOT IMPLEMENTED')
+        log.warning("notify_uptime_alert NOT IMPLEMENTED")
 
     def get_notification_basic_vars(self, kwargs):
         """
         Sets most common variables used later for rendering notifications for
         channel
         """
-        if 'event' in kwargs:
-            kwargs['since_when'] = kwargs['event'].start_date
+        if "event" in kwargs:
+            kwargs["since_when"] = kwargs["event"].start_date
 
-        url_start_date = kwargs.get('since_when') - timedelta(minutes=1)
-        url_end_date = kwargs.get('since_when') + timedelta(minutes=4)
+        url_start_date = kwargs.get("since_when") - timedelta(minutes=1)
+        url_end_date = kwargs.get("since_when") + timedelta(minutes=4)
         tmpl_vars = {
-            "timestamp": kwargs['since_when'],
-            "user": kwargs['user'],
-            "since_when": kwargs.get('since_when'),
+            "timestamp": kwargs["since_when"],
+            "user": kwargs["user"],
+            "since_when": kwargs.get("since_when"),
             "url_start_date": url_start_date,
-            "url_end_date": url_end_date
+            "url_end_date": url_end_date,
         }
-        tmpl_vars["resource_name"] = kwargs['resource'].resource_name
-        tmpl_vars["resource"] = kwargs['resource']
+        tmpl_vars["resource_name"] = kwargs["resource"].resource_name
+        tmpl_vars["resource"] = kwargs["resource"]
 
-        if 'event' in kwargs:
-            tmpl_vars['event_values'] = kwargs['event'].values
-            tmpl_vars['alert_type'] = kwargs['event'].unified_alert_name()
-            tmpl_vars['alert_action'] = kwargs['event'].unified_alert_action()
+        if "event" in kwargs:
+            tmpl_vars["event_values"] = kwargs["event"].values
+            tmpl_vars["alert_type"] = kwargs["event"].unified_alert_name()
+            tmpl_vars["alert_action"] = kwargs["event"].unified_alert_action()
         return tmpl_vars
 
     def report_alert_notification_vars(self, kwargs):
         tmpl_vars = self.get_notification_basic_vars(kwargs)
-        reports = kwargs.get('reports', [])
+        reports = kwargs.get("reports", [])
         tmpl_vars["reports"] = reports
         tmpl_vars["confirmed_total"] = len(reports)
 
         tmpl_vars["report_type"] = "error reports"
-        tmpl_vars["url_report_type"] = 'report/list'
+        tmpl_vars["url_report_type"] = "report/list"
 
-        alert_type = tmpl_vars.get('alert_type', '')
-        if 'slow_report' in alert_type:
+        alert_type = tmpl_vars.get("alert_type", "")
+        if "slow_report" in alert_type:
             tmpl_vars["report_type"] = "slow reports"
-            tmpl_vars["url_report_type"] = 'report/list_slow'
+            tmpl_vars["url_report_type"] = "report/list_slow"
 
-        app_url = kwargs['request'].registry.settings['_mail_url']
+        app_url = kwargs["request"].registry.settings["_mail_url"]
 
-        destination_url = kwargs['request'].route_url('/',
-                                                      _app_url=app_url)
+        destination_url = kwargs["request"].route_url("/", _app_url=app_url)
         if alert_type:
-            destination_url += 'ui/{}?resource={}&start_date={}&end_date={}'.format(
+            destination_url += "ui/{}?resource={}&start_date={}&end_date={}".format(
                 tmpl_vars["url_report_type"],
-                tmpl_vars['resource'].resource_id,
-                tmpl_vars['url_start_date'].strftime(DATE_FRMT),
-                tmpl_vars['url_end_date'].strftime(DATE_FRMT)
+                tmpl_vars["resource"].resource_id,
+                tmpl_vars["url_start_date"].strftime(DATE_FRMT),
+                tmpl_vars["url_end_date"].strftime(DATE_FRMT),
             )
         else:
-            destination_url += 'ui/{}?resource={}'.format(
-                tmpl_vars["url_report_type"],
-                tmpl_vars['resource'].resource_id
+            destination_url += "ui/{}?resource={}".format(
+                tmpl_vars["url_report_type"], tmpl_vars["resource"].resource_id
             )
         tmpl_vars["destination_url"] = destination_url
 
         return tmpl_vars
 
     def uptime_alert_notification_vars(self, kwargs):
         tmpl_vars = self.get_notification_basic_vars(kwargs)
-        app_url = kwargs['request'].registry.settings['_mail_url']
-        destination_url = kwargs['request'].route_url('/', _app_url=app_url)
-        destination_url += 'ui/{}?resource={}'.format(
-            'uptime',
-            tmpl_vars['resource'].resource_id)
-        tmpl_vars['destination_url'] = destination_url
-
-        reason = ''
-        e_values = tmpl_vars.get('event_values')
-
-        if e_values and e_values.get('response_time') == 0:
-            reason += ' Response time was slower than 20 seconds.'
+        app_url = kwargs["request"].registry.settings["_mail_url"]
+        destination_url = kwargs["request"].route_url("/", _app_url=app_url)
+        destination_url += "ui/{}?resource={}".format(
+            "uptime", tmpl_vars["resource"].resource_id
+        )
+        tmpl_vars["destination_url"] = destination_url
+
+        reason = ""
+        e_values = tmpl_vars.get("event_values")
+
+        if e_values and e_values.get("response_time") == 0:
+            reason += " Response time was slower than 20 seconds."
         elif e_values:
-            code = e_values.get('status_code')
-            reason += ' Response status code: %s.' % code
+            code = e_values.get("status_code")
+            reason += " Response status code: %s." % code
 
-        tmpl_vars['reason'] = reason
+        tmpl_vars["reason"] = reason
         return tmpl_vars
 
     def chart_alert_notification_vars(self, kwargs):
         tmpl_vars = self.get_notification_basic_vars(kwargs)
-        tmpl_vars['chart_name'] = tmpl_vars['event_values']['chart_name']
-        tmpl_vars['action_name'] = tmpl_vars['event_values'].get(
-            'action_name') or ''
-        matched_values = tmpl_vars['event_values']['matched_step_values']
-        tmpl_vars['readable_values'] = []
-        for key, value in list(matched_values['values'].items()):
-            matched_label = matched_values['labels'].get(key)
+        tmpl_vars["chart_name"] = tmpl_vars["event_values"]["chart_name"]
+        tmpl_vars["action_name"] = tmpl_vars["event_values"].get("action_name") or ""
+        matched_values = tmpl_vars["event_values"]["matched_step_values"]
+        tmpl_vars["readable_values"] = []
+        for key, value in list(matched_values["values"].items()):
+            matched_label = matched_values["labels"].get(key)
             if matched_label:
-                tmpl_vars['readable_values'].append({
-                    'label': matched_label['human_label'],
-                    'value': value
-                })
-        tmpl_vars['readable_values'] = sorted(tmpl_vars['readable_values'],
-                                              key=lambda x: x['label'])
-        start_date = convert_date(tmpl_vars['event_values']['start_interval'])
+                tmpl_vars["readable_values"].append(
+                    {"label": matched_label["human_label"], "value": value}
+                )
+        tmpl_vars["readable_values"] = sorted(
+            tmpl_vars["readable_values"], key=lambda x: x["label"]
+        )
+        start_date = convert_date(tmpl_vars["event_values"]["start_interval"])
         end_date = None
-        if tmpl_vars['event_values'].get('end_interval'):
-            end_date = convert_date(tmpl_vars['event_values']['end_interval'])
+        if tmpl_vars["event_values"].get("end_interval"):
+            end_date = convert_date(tmpl_vars["event_values"]["end_interval"])
 
-        app_url = kwargs['request'].registry.settings['_mail_url']
-        destination_url = kwargs['request'].route_url('/', _app_url=app_url)
+        app_url = kwargs["request"].registry.settings["_mail_url"]
+        destination_url = kwargs["request"].route_url("/", _app_url=app_url)
         to_encode = {
-            'resource': tmpl_vars['event_values']['resource'],
-            'start_date': start_date.strftime(DATE_FRMT),
+            "resource": tmpl_vars["event_values"]["resource"],
+            "start_date": start_date.strftime(DATE_FRMT),
         }
         if end_date:
-            to_encode['end_date'] = end_date.strftime(DATE_FRMT)
+            to_encode["end_date"] = end_date.strftime(DATE_FRMT)
 
-        destination_url += 'ui/{}?{}'.format(
-            'logs',
-            urllib.parse.urlencode(to_encode)
-        )
-        tmpl_vars['destination_url'] = destination_url
+        destination_url += "ui/{}?{}".format("logs", urllib.parse.urlencode(to_encode))
+        tmpl_vars["destination_url"] = destination_url
         return tmpl_vars

backend/src/appenlight/models/alert_channel_action.py

@@ -1,79 +1,84 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import sqlalchemy as sa
 
 from ziggurat_foundations.models.services.resource import ResourceService
 from appenlight.models import Base, get_db_session
 from sqlalchemy.orm import validates
 from ziggurat_foundations.models.base import BaseModel
 
 
 class AlertChannelAction(Base, BaseModel):
     """
     Stores notifications conditions for user's alert channels
     This is later used for rule parsing like "alert if http_status == 500"
     """
-    __tablename__ = 'alert_channels_actions'
 
-    types = ['report', 'chart']
+    __tablename__ = "alert_channels_actions"
 
-    owner_id = sa.Column(sa.Integer,
-                         sa.ForeignKey('users.id', onupdate='CASCADE',
-                                       ondelete='CASCADE'))
+    types = ["report", "chart"]
+
+    owner_id = sa.Column(
+        sa.Integer, sa.ForeignKey("users.id", onupdate="CASCADE", ondelete="CASCADE")
+    )
     resource_id = sa.Column(sa.Integer())
-    action = sa.Column(sa.Unicode(10), nullable=False, default='always')
+    action = sa.Column(sa.Unicode(10), nullable=False, default="always")
     type = sa.Column(sa.Unicode(10), nullable=False)
     other_id = sa.Column(sa.Unicode(40))
     pkey = sa.Column(sa.Integer(), nullable=False, primary_key=True)
-    rule = sa.Column(sa.dialects.postgresql.JSON,
-                     nullable=False, default={'field': 'http_status',
-                                              "op": "ge", "value": "500"})
+    rule = sa.Column(
+        sa.dialects.postgresql.JSON,
+        nullable=False,
+        default={"field": "http_status", "op": "ge", "value": "500"},
+    )
     config = sa.Column(sa.dialects.postgresql.JSON)
     name = sa.Column(sa.Unicode(255))
 
-    @validates('notify_type')
+    @validates("notify_type")
     def validate_email(self, key, notify_type):
-        assert notify_type in ['always', 'only_first']
+        assert notify_type in ["always", "only_first"]
         return notify_type
 
     def resource_name(self, db_session=None):
         db_session = get_db_session(db_session)
         if self.resource_id:
             return ResourceService.by_resource_id(
-                self.resource_id, db_session=db_session).resource_name
+                self.resource_id, db_session=db_session
+            ).resource_name
         else:
-            return 'any resource'
+            return "any resource"
 
-    def get_dict(self, exclude_keys=None, include_keys=None,
-                 extended_info=False):
+    def get_dict(self, exclude_keys=None, include_keys=None, extended_info=False):
         """
         Returns dictionary with required information that will be consumed by
         angular
         """
         instance_dict = super(AlertChannelAction, self).get_dict()
         exclude_keys_list = exclude_keys or []
         include_keys_list = include_keys or []
         if extended_info:
-            instance_dict['channels'] = [
-                c.get_dict(extended_info=False) for c in self.channels]
+            instance_dict["channels"] = [
+                c.get_dict(extended_info=False) for c in self.channels
+            ]
 
         d = {}
         for k in instance_dict.keys():
-            if (k not in exclude_keys_list and
-                    (k in include_keys_list or not include_keys)):
+            if k not in exclude_keys_list and (
+                k in include_keys_list or not include_keys
+            ):
                 d[k] = instance_dict[k]
         return d

backend/src/appenlight/models/alert_channels/__init__.py

@@ -1,16 +1,15 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-

backend/src/appenlight/models/alert_channels/campfire.py

@@ -1,188 +1,191 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 from appenlight.models.alert_channel import AlertChannel
 from appenlight.models.integrations.campfire import CampfireIntegration
 from webhelpers2.text import truncate
 
 log = logging.getLogger(__name__)
 
 
 class CampfireAlertChannel(AlertChannel):
-    __mapper_args__ = {
-        'polymorphic_identity': 'campfire'
-    }
+    __mapper_args__ = {"polymorphic_identity": "campfire"}
 
     @property
     def client(self):
         client = CampfireIntegration.create_client(
-            self.integration.config['api_token'],
-            self.integration.config['account'])
+            self.integration.config["api_token"], self.integration.config["account"]
+        )
         return client
 
     def notify_reports(self, **kwargs):
         """
         Notify user of individual reports
 
         kwargs:
             application: application that the event applies for,
             user: user that should be notified
             request: request object
             since_when: reports are newer than this time value,
             reports: list of reports to render
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
 
-        app_url = kwargs['request'].registry.settings['_mail_url']
-        destination_url = kwargs['request'].route_url('/',
-                                                      app_url=app_url)
-        f_args = ('report',
-                  template_vars['resource'].resource_id,
-                  template_vars['url_start_date'].strftime('%Y-%m-%dT%H:%M'),
-                  template_vars['url_end_date'].strftime('%Y-%m-%dT%H:%M'))
-        destination_url += 'ui/{}?resource={}&start_date={}&end_date={}'.format(
-            *f_args)
-
-        if template_vars['confirmed_total'] > 1:
+        app_url = kwargs["request"].registry.settings["_mail_url"]
+        destination_url = kwargs["request"].route_url("/", app_url=app_url)
+        f_args = (
+            "report",
+            template_vars["resource"].resource_id,
+            template_vars["url_start_date"].strftime("%Y-%m-%dT%H:%M"),
+            template_vars["url_end_date"].strftime("%Y-%m-%dT%H:%M"),
+        )
+        destination_url += "ui/{}?resource={}&start_date={}&end_date={}".format(*f_args)
+
+        if template_vars["confirmed_total"] > 1:
             template_vars["title"] = "%s - %s reports" % (
-                template_vars['resource_name'],
-                template_vars['confirmed_total'],
+                template_vars["resource_name"],
+                template_vars["confirmed_total"],
             )
         else:
-            error_title = truncate(template_vars['reports'][0][1].error or
-                                   'slow report', 90)
+            error_title = truncate(
+                template_vars["reports"][0][1].error or "slow report", 90
+            )
             template_vars["title"] = "%s - '%s' report" % (
-                template_vars['resource_name'],
-                error_title)
+                template_vars["resource_name"],
+                error_title,
+            )
 
-        template_vars["title"] += ' ' + destination_url
+        template_vars["title"] += " " + destination_url
 
-        log_msg = 'NOTIFY  : %s via %s :: %s reports' % (
-            kwargs['user'].user_name,
+        log_msg = "NOTIFY  : %s via %s :: %s reports" % (
+            kwargs["user"].user_name,
             self.channel_visible_value,
-            template_vars['confirmed_total'])
+            template_vars["confirmed_total"],
+        )
         log.warning(log_msg)
 
-        for room in self.integration.config['rooms'].split(','):
+        for room in self.integration.config["rooms"].split(","):
             self.client.speak_to_room(room.strip(), template_vars["title"])
 
     def notify_report_alert(self, **kwargs):
         """
         Build and send report alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
 
-        if kwargs['event'].unified_alert_action() == 'OPEN':
-            title = 'ALERT %s: %s - %s %s %s' % (
-                template_vars['alert_action'],
-                template_vars['resource_name'],
-                kwargs['event'].values['reports'],
-                template_vars['report_type'],
-                template_vars['destination_url']
+        if kwargs["event"].unified_alert_action() == "OPEN":
+            title = "ALERT %s: %s - %s %s %s" % (
+                template_vars["alert_action"],
+                template_vars["resource_name"],
+                kwargs["event"].values["reports"],
+                template_vars["report_type"],
+                template_vars["destination_url"],
             )
 
         else:
-            title = 'ALERT %s: %s type: %s' % (
-                template_vars['alert_action'],
-                template_vars['resource_name'],
-                template_vars['alert_type'].replace('_', ' '),
+            title = "ALERT %s: %s type: %s" % (
+                template_vars["alert_action"],
+                template_vars["resource_name"],
+                template_vars["alert_type"].replace("_", " "),
             )
-        for room in self.integration.config['rooms'].split(','):
-            self.client.speak_to_room(room.strip(), title, sound='VUVUZELA')
+        for room in self.integration.config["rooms"].split(","):
+            self.client.speak_to_room(room.strip(), title, sound="VUVUZELA")
 
     def notify_uptime_alert(self, **kwargs):
         """
         Build and send uptime alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.uptime_alert_notification_vars(kwargs)
 
-        message = 'ALERT %s: %s has uptime issues %s\n\n' % (
-            template_vars['alert_action'],
-            template_vars['resource_name'],
-            template_vars['destination_url']
+        message = "ALERT %s: %s has uptime issues %s\n\n" % (
+            template_vars["alert_action"],
+            template_vars["resource_name"],
+            template_vars["destination_url"],
         )
-        message += template_vars['reason']
+        message += template_vars["reason"]
 
-        for room in self.integration.config['rooms'].split(','):
-            self.client.speak_to_room(room.strip(), message, sound='VUVUZELA')
+        for room in self.integration.config["rooms"].split(","):
+            self.client.speak_to_room(room.strip(), message, sound="VUVUZELA")
 
     def send_digest(self, **kwargs):
         """
         Build and send daily digest notification
 
         kwargs:
             application: application that the event applies for,
             user: user that should be notified
             request: request object
             since_when: reports are newer than this time value,
             reports: list of reports to render
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
-        f_args = (template_vars['resource_name'],
-                  template_vars['confirmed_total'],)
+        f_args = (template_vars["resource_name"], template_vars["confirmed_total"])
         message = "Daily report digest: %s - %s reports" % f_args
-        message += '{}\n'.format(template_vars['destination_url'])
-        for room in self.integration.config['rooms'].split(','):
+        message += "{}\n".format(template_vars["destination_url"])
+        for room in self.integration.config["rooms"].split(","):
             self.client.speak_to_room(room.strip(), message)
 
-        log_msg = 'DIGEST  : %s via %s :: %s reports' % (
-            kwargs['user'].user_name,
+        log_msg = "DIGEST  : %s via %s :: %s reports" % (
+            kwargs["user"].user_name,
             self.channel_visible_value,
-            template_vars['confirmed_total'])
+            template_vars["confirmed_total"],
+        )
         log.warning(log_msg)
 
     def notify_chart_alert(self, **kwargs):
         """
         Build and send chart alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.chart_alert_notification_vars(kwargs)
-        message = 'ALERT {}: value in "{}" chart: ' \
-                  'met alert "{}" criteria {} \n'.format(
-            template_vars['alert_action'],
-            template_vars['chart_name'],
-            template_vars['action_name'],
-            template_vars['destination_url']
+        message = (
+            'ALERT {}: value in "{}" chart: '
+            'met alert "{}" criteria {} \n'.format(
+                template_vars["alert_action"],
+                template_vars["chart_name"],
+                template_vars["action_name"],
+                template_vars["destination_url"],
+            )
         )
 
-        for item in template_vars['readable_values']:
-            message += '{}: {}\n'.format(item['label'], item['value'])
+        for item in template_vars["readable_values"]:
+            message += "{}: {}\n".format(item["label"], item["value"])
 
-        for room in self.integration.config['rooms'].split(','):
-            self.client.speak_to_room(room.strip(), message, sound='VUVUZELA')
+        for room in self.integration.config["rooms"].split(","):
+            self.client.speak_to_room(room.strip(), message, sound="VUVUZELA")

backend/src/appenlight/models/alert_channels/email.py

@@ -1,175 +1,192 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 from appenlight.models.alert_channel import AlertChannel
 from appenlight.models.services.user import UserService
 from webhelpers2.text import truncate
 
 log = logging.getLogger(__name__)
 
 
 class EmailAlertChannel(AlertChannel):
     """
     Default email alerting channel
     """
 
-    __mapper_args__ = {
-        'polymorphic_identity': 'email'
-    }
+    __mapper_args__ = {"polymorphic_identity": "email"}
 
     def notify_reports(self, **kwargs):
         """
         Notify user of individual reports
 
         kwargs:
             application: application that the event applies for,
             user: user that should be notified
             request: request object
             since_when: reports are newer than this time value,
             reports: list of reports to render
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
 
-        if template_vars['confirmed_total'] > 1:
+        if template_vars["confirmed_total"] > 1:
             template_vars["title"] = "AppEnlight :: %s - %s reports" % (
-                template_vars['resource_name'],
-                template_vars['confirmed_total'],
+                template_vars["resource_name"],
+                template_vars["confirmed_total"],
             )
         else:
-            error_title = truncate(template_vars['reports'][0][1].error or
-                                   'slow report', 20)
+            error_title = truncate(
+                template_vars["reports"][0][1].error or "slow report", 20
+            )
             template_vars["title"] = "AppEnlight :: %s - '%s' report" % (
-                template_vars['resource_name'],
-                error_title)
-        UserService.send_email(kwargs['request'],
-                               [self.channel_value],
-                               template_vars,
-                               '/email_templates/notify_reports.jinja2')
-        log_msg = 'NOTIFY  : %s via %s :: %s reports' % (
-            kwargs['user'].user_name,
+                template_vars["resource_name"],
+                error_title,
+            )
+        UserService.send_email(
+            kwargs["request"],
+            [self.channel_value],
+            template_vars,
+            "/email_templates/notify_reports.jinja2",
+        )
+        log_msg = "NOTIFY  : %s via %s :: %s reports" % (
+            kwargs["user"].user_name,
             self.channel_visible_value,
-            template_vars['confirmed_total'])
+            template_vars["confirmed_total"],
+        )
         log.warning(log_msg)
 
     def send_digest(self, **kwargs):
         """
         Build and send daily digest notification
 
         kwargs:
             application: application that the event applies for,
             user: user that should be notified
             request: request object
             since_when: reports are newer than this time value,
             reports: list of reports to render
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
         title = "AppEnlight :: Daily report digest: %s - %s reports"
         template_vars["email_title"] = title % (
-            template_vars['resource_name'],
-            template_vars['confirmed_total'],
+            template_vars["resource_name"],
+            template_vars["confirmed_total"],
         )
 
-        UserService.send_email(kwargs['request'],
-                               [self.channel_value],
-                               template_vars,
-                               '/email_templates/notify_reports.jinja2',
-                               immediately=True,
-                               silent=True)
-        log_msg = 'DIGEST  : %s via %s :: %s reports' % (
-            kwargs['user'].user_name,
+        UserService.send_email(
+            kwargs["request"],
+            [self.channel_value],
+            template_vars,
+            "/email_templates/notify_reports.jinja2",
+            immediately=True,
+            silent=True,
+        )
+        log_msg = "DIGEST  : %s via %s :: %s reports" % (
+            kwargs["user"].user_name,
             self.channel_visible_value,
-            template_vars['confirmed_total'])
+            template_vars["confirmed_total"],
+        )
         log.warning(log_msg)
 
     def notify_report_alert(self, **kwargs):
         """
         Build and send report alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
 
-        if kwargs['event'].unified_alert_action() == 'OPEN':
-            title = 'AppEnlight :: ALERT %s: %s - %s %s' % (
-                template_vars['alert_action'],
-                template_vars['resource_name'],
-                kwargs['event'].values['reports'],
-                template_vars['report_type'],
+        if kwargs["event"].unified_alert_action() == "OPEN":
+            title = "AppEnlight :: ALERT %s: %s - %s %s" % (
+                template_vars["alert_action"],
+                template_vars["resource_name"],
+                kwargs["event"].values["reports"],
+                template_vars["report_type"],
             )
         else:
-            title = 'AppEnlight :: ALERT %s: %s type: %s' % (
-                template_vars['alert_action'],
-                template_vars['resource_name'],
-                template_vars['alert_type'].replace('_', ' '),
+            title = "AppEnlight :: ALERT %s: %s type: %s" % (
+                template_vars["alert_action"],
+                template_vars["resource_name"],
+                template_vars["alert_type"].replace("_", " "),
             )
-        template_vars['email_title'] = title
-        UserService.send_email(kwargs['request'], [self.channel_value],
-                               template_vars,
-                               '/email_templates/alert_reports.jinja2')
+        template_vars["email_title"] = title
+        UserService.send_email(
+            kwargs["request"],
+            [self.channel_value],
+            template_vars,
+            "/email_templates/alert_reports.jinja2",
+        )
 
     def notify_uptime_alert(self, **kwargs):
         """
         Build and send uptime alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.uptime_alert_notification_vars(kwargs)
-        title = 'AppEnlight :: ALERT %s: %s has uptime issues' % (
-            template_vars['alert_action'],
-            template_vars['resource_name'],
+        title = "AppEnlight :: ALERT %s: %s has uptime issues" % (
+            template_vars["alert_action"],
+            template_vars["resource_name"],
         )
-        template_vars['email_title'] = title
+        template_vars["email_title"] = title
 
-        UserService.send_email(kwargs['request'], [self.channel_value],
-                               template_vars,
-                               '/email_templates/alert_uptime.jinja2')
+        UserService.send_email(
+            kwargs["request"],
+            [self.channel_value],
+            template_vars,
+            "/email_templates/alert_uptime.jinja2",
+        )
 
     def notify_chart_alert(self, **kwargs):
         """
         Build and send chart alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.chart_alert_notification_vars(kwargs)
 
-        title = 'AppEnlight :: ALERT {} value in "{}" chart' \
-                ' met alert "{}" criteria'.format(
-            template_vars['alert_action'],
-            template_vars['chart_name'],
-            template_vars['action_name'],
+        title = (
+            'AppEnlight :: ALERT {} value in "{}" chart'
+            ' met alert "{}" criteria'.format(
+                template_vars["alert_action"],
+                template_vars["chart_name"],
+                template_vars["action_name"],
+            )
+        )
+        template_vars["email_title"] = title
+        UserService.send_email(
+            kwargs["request"],
+            [self.channel_value],
+            template_vars,
+            "/email_templates/alert_chart.jinja2",
         )
-        template_vars['email_title'] = title
-        UserService.send_email(kwargs['request'], [self.channel_value],
-                               template_vars,
-                               '/email_templates/alert_chart.jinja2')

backend/src/appenlight/models/alert_channels/flowdock.py

@@ -1,233 +1,225 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 from appenlight.models.alert_channel import AlertChannel
 from appenlight.models.integrations.flowdock import FlowdockIntegration
 from webhelpers2.text import truncate
 
 log = logging.getLogger(__name__)
 
 
 class FlowdockAlertChannel(AlertChannel):
-    __mapper_args__ = {
-        'polymorphic_identity': 'flowdock'
-    }
+    __mapper_args__ = {"polymorphic_identity": "flowdock"}
 
     def notify_reports(self, **kwargs):
         """
         Notify user of individual reports
 
         kwargs:
             application: application that the event applies for,
             user: user that should be notified
             request: request object
             since_when: reports are newer than this time value,
             reports: list of reports to render
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
 
-        app_url = kwargs['request'].registry.settings['_mail_url']
-        destination_url = kwargs['request'].route_url('/',
-                                                      _app_url=app_url)
-        f_args = ('report',
-                  template_vars['resource'].resource_id,
-                  template_vars['url_start_date'].strftime('%Y-%m-%dT%H:%M'),
-                  template_vars['url_end_date'].strftime('%Y-%m-%dT%H:%M'))
-        destination_url += 'ui/{}?resource={}&start_date={}&end_date={}'.format(
-            *f_args)
-
-        if template_vars['confirmed_total'] > 1:
+        app_url = kwargs["request"].registry.settings["_mail_url"]
+        destination_url = kwargs["request"].route_url("/", _app_url=app_url)
+        f_args = (
+            "report",
+            template_vars["resource"].resource_id,
+            template_vars["url_start_date"].strftime("%Y-%m-%dT%H:%M"),
+            template_vars["url_end_date"].strftime("%Y-%m-%dT%H:%M"),
+        )
+        destination_url += "ui/{}?resource={}&start_date={}&end_date={}".format(*f_args)
+
+        if template_vars["confirmed_total"] > 1:
             template_vars["title"] = "%s - %s reports" % (
-                template_vars['resource_name'],
-                template_vars['confirmed_total'],
+                template_vars["resource_name"],
+                template_vars["confirmed_total"],
             )
         else:
-            error_title = truncate(template_vars['reports'][0][1].error or
-                                   'slow report', 90)
+            error_title = truncate(
+                template_vars["reports"][0][1].error or "slow report", 90
+            )
             template_vars["title"] = "%s - '%s' report" % (
-                template_vars['resource_name'],
-                error_title)
+                template_vars["resource_name"],
+                error_title,
+            )
 
-        log_msg = 'NOTIFY : %s via %s :: %s reports' % (
-            kwargs['user'].user_name,
+        log_msg = "NOTIFY : %s via %s :: %s reports" % (
+            kwargs["user"].user_name,
             self.channel_visible_value,
-            template_vars['confirmed_total'])
+            template_vars["confirmed_total"],
+        )
         log.warning(log_msg)
 
-        client = FlowdockIntegration.create_client(
-            self.integration.config['api_token'])
+        client = FlowdockIntegration.create_client(self.integration.config["api_token"])
         payload = {
             "source": "AppEnlight",
-            "from_address": kwargs['request'].registry.settings[
-                'mailing.from_email'],
+            "from_address": kwargs["request"].registry.settings["mailing.from_email"],
             "subject": template_vars["title"],
             "content": "New report present",
             "tags": ["appenlight"],
-            "link": destination_url
+            "link": destination_url,
         }
         client.send_to_inbox(payload)
 
     def notify_report_alert(self, **kwargs):
         """
         Build and send report alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
 
-        if kwargs['event'].unified_alert_action() == 'OPEN':
+        if kwargs["event"].unified_alert_action() == "OPEN":
 
-            title = 'ALERT %s: %s - %s %s' % (
-                template_vars['alert_action'],
-                template_vars['resource_name'],
-                kwargs['event'].values['reports'],
-                template_vars['report_type'],
+            title = "ALERT %s: %s - %s %s" % (
+                template_vars["alert_action"],
+                template_vars["resource_name"],
+                kwargs["event"].values["reports"],
+                template_vars["report_type"],
             )
 
         else:
-            title = 'ALERT %s: %s type: %s' % (
-                template_vars['alert_action'],
-                template_vars['resource_name'],
-                template_vars['alert_type'].replace('_', ' '),
+            title = "ALERT %s: %s type: %s" % (
+                template_vars["alert_action"],
+                template_vars["resource_name"],
+                template_vars["alert_type"].replace("_", " "),
             )
 
-        client = FlowdockIntegration.create_client(
-            self.integration.config['api_token'])
+        client = FlowdockIntegration.create_client(self.integration.config["api_token"])
         payload = {
             "source": "AppEnlight",
-            "from_address": kwargs['request'].registry.settings[
-                'mailing.from_email'],
+            "from_address": kwargs["request"].registry.settings["mailing.from_email"],
             "subject": title,
-            "content": 'Investigation required',
-            "tags": ["appenlight", "alert", template_vars['alert_type']],
-            "link": template_vars['destination_url']
+            "content": "Investigation required",
+            "tags": ["appenlight", "alert", template_vars["alert_type"]],
+            "link": template_vars["destination_url"],
         }
         client.send_to_inbox(payload)
 
     def notify_uptime_alert(self, **kwargs):
         """
         Build and send uptime alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.uptime_alert_notification_vars(kwargs)
 
-        message = 'ALERT %s: %s has uptime issues' % (
-            template_vars['alert_action'],
-            template_vars['resource_name'],
+        message = "ALERT %s: %s has uptime issues" % (
+            template_vars["alert_action"],
+            template_vars["resource_name"],
         )
-        submessage = 'Info: '
-        submessage += template_vars['reason']
+        submessage = "Info: "
+        submessage += template_vars["reason"]
 
-        client = FlowdockIntegration.create_client(
-            self.integration.config['api_token'])
+        client = FlowdockIntegration.create_client(self.integration.config["api_token"])
         payload = {
             "source": "AppEnlight",
-            "from_address": kwargs['request'].registry.settings[
-                'mailing.from_email'],
+            "from_address": kwargs["request"].registry.settings["mailing.from_email"],
             "subject": message,
             "content": submessage,
-            "tags": ["appenlight", "alert", 'uptime'],
-            "link": template_vars['destination_url']
+            "tags": ["appenlight", "alert", "uptime"],
+            "link": template_vars["destination_url"],
         }
         client.send_to_inbox(payload)
 
     def send_digest(self, **kwargs):
         """
         Build and send daily digest notification
 
         kwargs:
             application: application that the event applies for,
             user: user that should be notified
             request: request object
             since_when: reports are newer than this time value,
             reports: list of reports to render
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
         message = "Daily report digest: %s - %s reports" % (
-            template_vars['resource_name'], template_vars['confirmed_total'])
+            template_vars["resource_name"],
+            template_vars["confirmed_total"],
+        )
 
-        f_args = (template_vars['confirmed_total'],
-                  template_vars['timestamp'])
+        f_args = (template_vars["confirmed_total"], template_vars["timestamp"])
 
         payload = {
             "source": "AppEnlight",
-            "from_address": kwargs['request'].registry.settings[
-                'mailing.from_email'],
+            "from_address": kwargs["request"].registry.settings["mailing.from_email"],
             "subject": message,
-            "content": '%s reports in total since %s' % f_args,
+            "content": "%s reports in total since %s" % f_args,
             "tags": ["appenlight", "digest"],
-            "link": template_vars['destination_url']
+            "link": template_vars["destination_url"],
         }
 
-        client = FlowdockIntegration.create_client(
-            self.integration.config['api_token'])
+        client = FlowdockIntegration.create_client(self.integration.config["api_token"])
         client.send_to_inbox(payload)
 
-        log_msg = 'DIGEST  : %s via %s :: %s reports' % (
-            kwargs['user'].user_name,
+        log_msg = "DIGEST  : %s via %s :: %s reports" % (
+            kwargs["user"].user_name,
             self.channel_visible_value,
-            template_vars['confirmed_total'])
+            template_vars["confirmed_total"],
+        )
         log.warning(log_msg)
 
     def notify_chart_alert(self, **kwargs):
         """
         Build and send chart alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.chart_alert_notification_vars(kwargs)
 
-        message = 'ALERT {}: value in "{}" chart ' \
-                  'met alert "{}" criteria'.format(
-            template_vars['alert_action'],
-            template_vars['chart_name'],
-            template_vars['action_name'],
+        message = 'ALERT {}: value in "{}" chart ' 'met alert "{}" criteria'.format(
+            template_vars["alert_action"],
+            template_vars["chart_name"],
+            template_vars["action_name"],
         )
-        submessage = 'Info: '
-        for item in template_vars['readable_values']:
-            submessage += '{}: {}\n'.format(item['label'], item['value'])
+        submessage = "Info: "
+        for item in template_vars["readable_values"]:
+            submessage += "{}: {}\n".format(item["label"], item["value"])
 
-        client = FlowdockIntegration.create_client(
-            self.integration.config['api_token'])
+        client = FlowdockIntegration.create_client(self.integration.config["api_token"])
         payload = {
             "source": "AppEnlight",
-            "from_address": kwargs['request'].registry.settings[
-                'mailing.from_email'],
+            "from_address": kwargs["request"].registry.settings["mailing.from_email"],
             "subject": message,
             "content": submessage,
-            "tags": ["appenlight", "alert", 'chart'],
-            "link": template_vars['destination_url']
+            "tags": ["appenlight", "alert", "chart"],
+            "link": template_vars["destination_url"],
         }
         client.send_to_inbox(payload)

backend/src/appenlight/models/alert_channels/hipchat.py

@@ -1,229 +1,238 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 from appenlight.models.alert_channel import AlertChannel
 from appenlight.models.integrations.hipchat import HipchatIntegration
 from webhelpers2.text import truncate
 
 log = logging.getLogger(__name__)
 
 
 class HipchatAlertChannel(AlertChannel):
-    __mapper_args__ = {
-        'polymorphic_identity': 'hipchat'
-    }
+    __mapper_args__ = {"polymorphic_identity": "hipchat"}
 
     def notify_reports(self, **kwargs):
         """
         Notify user of individual reports
 
         kwargs:
             application: application that the event applies for,
             user: user that should be notified
             request: request object
             since_when: reports are newer than this time value,
             reports: list of reports to render
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
 
-        app_url = kwargs['request'].registry.settings['_mail_url']
-        destination_url = kwargs['request'].route_url('/',
-                                                      _app_url=app_url)
-        f_args = ('report',
-                  template_vars['resource'].resource_id,
-                  template_vars['url_start_date'].strftime('%Y-%m-%dT%H:%M'),
-                  template_vars['url_end_date'].strftime('%Y-%m-%dT%H:%M'))
-        destination_url += 'ui/{}?resource={}&start_date={}&end_date={}'.format(
-            *f_args)
-
-        if template_vars['confirmed_total'] > 1:
+        app_url = kwargs["request"].registry.settings["_mail_url"]
+        destination_url = kwargs["request"].route_url("/", _app_url=app_url)
+        f_args = (
+            "report",
+            template_vars["resource"].resource_id,
+            template_vars["url_start_date"].strftime("%Y-%m-%dT%H:%M"),
+            template_vars["url_end_date"].strftime("%Y-%m-%dT%H:%M"),
+        )
+        destination_url += "ui/{}?resource={}&start_date={}&end_date={}".format(*f_args)
+
+        if template_vars["confirmed_total"] > 1:
             template_vars["title"] = "%s - %s reports" % (
-                template_vars['resource_name'],
-                template_vars['confirmed_total'],
+                template_vars["resource_name"],
+                template_vars["confirmed_total"],
             )
         else:
-            error_title = truncate(template_vars['reports'][0][1].error or
-                                   'slow report', 90)
+            error_title = truncate(
+                template_vars["reports"][0][1].error or "slow report", 90
+            )
             template_vars["title"] = "%s - '%s' report" % (
-                template_vars['resource_name'],
-                error_title)
+                template_vars["resource_name"],
+                error_title,
+            )
 
-        template_vars["title"] += ' ' + destination_url
+        template_vars["title"] += " " + destination_url
 
-        log_msg = 'NOTIFY  : %s via %s :: %s reports' % (
-            kwargs['user'].user_name,
+        log_msg = "NOTIFY  : %s via %s :: %s reports" % (
+            kwargs["user"].user_name,
             self.channel_visible_value,
-            template_vars['confirmed_total'])
+            template_vars["confirmed_total"],
+        )
         log.warning(log_msg)
 
-        client = HipchatIntegration.create_client(
-            self.integration.config['api_token'])
-        for room in self.integration.config['rooms'].split(','):
-            client.send({
-                "message_format": 'text',
-                "message": template_vars["title"],
-                "from": "AppEnlight",
-                "room_id": room.strip(),
-                "color": "yellow"
-            })
+        client = HipchatIntegration.create_client(self.integration.config["api_token"])
+        for room in self.integration.config["rooms"].split(","):
+            client.send(
+                {
+                    "message_format": "text",
+                    "message": template_vars["title"],
+                    "from": "AppEnlight",
+                    "room_id": room.strip(),
+                    "color": "yellow",
+                }
+            )
 
     def notify_report_alert(self, **kwargs):
         """
         Build and send report alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
 
-        if kwargs['event'].unified_alert_action() == 'OPEN':
+        if kwargs["event"].unified_alert_action() == "OPEN":
 
-            title = 'ALERT %s: %s - %s %s' % (
-                template_vars['alert_action'],
-                template_vars['resource_name'],
-                kwargs['event'].values['reports'],
-                template_vars['report_type'],
+            title = "ALERT %s: %s - %s %s" % (
+                template_vars["alert_action"],
+                template_vars["resource_name"],
+                kwargs["event"].values["reports"],
+                template_vars["report_type"],
             )
 
         else:
-            title = 'ALERT %s: %s type: %s' % (
-                template_vars['alert_action'],
-                template_vars['resource_name'],
-                template_vars['alert_type'].replace('_', ' '),
+            title = "ALERT %s: %s type: %s" % (
+                template_vars["alert_action"],
+                template_vars["resource_name"],
+                template_vars["alert_type"].replace("_", " "),
             )
 
-        title += '\n ' + template_vars['destination_url']
+        title += "\n " + template_vars["destination_url"]
 
-        api_token = self.integration.config['api_token']
+        api_token = self.integration.config["api_token"]
         client = HipchatIntegration.create_client(api_token)
-        for room in self.integration.config['rooms'].split(','):
-            client.send({
-                "message_format": 'text',
-                "message": title,
-                "from": "AppEnlight",
-                "room_id": room.strip(),
-                "color": "red",
-                "notify": '1'
-            })
+        for room in self.integration.config["rooms"].split(","):
+            client.send(
+                {
+                    "message_format": "text",
+                    "message": title,
+                    "from": "AppEnlight",
+                    "room_id": room.strip(),
+                    "color": "red",
+                    "notify": "1",
+                }
+            )
 
     def notify_uptime_alert(self, **kwargs):
         """
         Build and send uptime alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.uptime_alert_notification_vars(kwargs)
 
-        message = 'ALERT %s: %s has uptime issues\n' % (
-            template_vars['alert_action'],
-            template_vars['resource_name'],
+        message = "ALERT %s: %s has uptime issues\n" % (
+            template_vars["alert_action"],
+            template_vars["resource_name"],
         )
-        message += template_vars['reason']
-        message += '\n{}'.format(template_vars['destination_url'])
+        message += template_vars["reason"]
+        message += "\n{}".format(template_vars["destination_url"])
 
-        api_token = self.integration.config['api_token']
+        api_token = self.integration.config["api_token"]
         client = HipchatIntegration.create_client(api_token)
-        for room in self.integration.config['rooms'].split(','):
-            client.send({
-                "message_format": 'text',
-                "message": message,
-                "from": "AppEnlight",
-                "room_id": room.strip(),
-                "color": "red",
-                "notify": '1'
-            })
+        for room in self.integration.config["rooms"].split(","):
+            client.send(
+                {
+                    "message_format": "text",
+                    "message": message,
+                    "from": "AppEnlight",
+                    "room_id": room.strip(),
+                    "color": "red",
+                    "notify": "1",
+                }
+            )
 
     def notify_chart_alert(self, **kwargs):
         """
         Build and send chart alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.chart_alert_notification_vars(kwargs)
-        message = 'ALERT {}: value in "{}" chart: ' \
-                  'met alert "{}" criteria\n'.format(
-            template_vars['alert_action'],
-            template_vars['chart_name'],
-            template_vars['action_name'],
+        message = 'ALERT {}: value in "{}" chart: ' 'met alert "{}" criteria\n'.format(
+            template_vars["alert_action"],
+            template_vars["chart_name"],
+            template_vars["action_name"],
         )
 
-        for item in template_vars['readable_values']:
-            message += '{}: {}\n'.format(item['label'], item['value'])
+        for item in template_vars["readable_values"]:
+            message += "{}: {}\n".format(item["label"], item["value"])
 
-        message += template_vars['destination_url']
+        message += template_vars["destination_url"]
 
-        api_token = self.integration.config['api_token']
+        api_token = self.integration.config["api_token"]
         client = HipchatIntegration.create_client(api_token)
-        for room in self.integration.config['rooms'].split(','):
-            client.send({
-                "message_format": 'text',
-                "message": message,
-                "from": "AppEnlight",
-                "room_id": room.strip(),
-                "color": "red",
-                "notify": '1'
-            })
+        for room in self.integration.config["rooms"].split(","):
+            client.send(
+                {
+                    "message_format": "text",
+                    "message": message,
+                    "from": "AppEnlight",
+                    "room_id": room.strip(),
+                    "color": "red",
+                    "notify": "1",
+                }
+            )
 
     def send_digest(self, **kwargs):
         """
         Build and send daily digest notification
 
         kwargs:
             application: application that the event applies for,
             user: user that should be notified
             request: request object
             since_when: reports are newer than this time value,
             reports: list of reports to render
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
-        f_args = (template_vars['resource_name'],
-                  template_vars['confirmed_total'],)
+        f_args = (template_vars["resource_name"], template_vars["confirmed_total"])
         message = "Daily report digest: %s - %s reports" % f_args
-        message += '\n{}'.format(template_vars['destination_url'])
-        api_token = self.integration.config['api_token']
+        message += "\n{}".format(template_vars["destination_url"])
+        api_token = self.integration.config["api_token"]
         client = HipchatIntegration.create_client(api_token)
-        for room in self.integration.config['rooms'].split(','):
-            client.send({
-                "message_format": 'text',
-                "message": message,
-                "from": "AppEnlight",
-                "room_id": room.strip(),
-                "color": "green",
-                "notify": '1'
-            })
-
-        log_msg = 'DIGEST  : %s via %s :: %s reports' % (
-            kwargs['user'].user_name,
+        for room in self.integration.config["rooms"].split(","):
+            client.send(
+                {
+                    "message_format": "text",
+                    "message": message,
+                    "from": "AppEnlight",
+                    "room_id": room.strip(),
+                    "color": "green",
+                    "notify": "1",
+                }
+            )
+
+        log_msg = "DIGEST  : %s via %s :: %s reports" % (
+            kwargs["user"].user_name,
             self.channel_visible_value,
-            template_vars['confirmed_total'])
+            template_vars["confirmed_total"],
+        )
         log.warning(log_msg)

backend/src/appenlight/models/alert_channels/slack.py

@@ -1,285 +1,270 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 from appenlight.models.alert_channel import AlertChannel
 from appenlight.models.integrations.slack import SlackIntegration
 from webhelpers2.text import truncate
 
 log = logging.getLogger(__name__)
 
 
 class SlackAlertChannel(AlertChannel):
-    __mapper_args__ = {
-        'polymorphic_identity': 'slack'
-    }
+    __mapper_args__ = {"polymorphic_identity": "slack"}
 
     def notify_reports(self, **kwargs):
         """
         Notify user of individual reports
 
         kwargs:
             application: application that the event applies for,
             user: user that should be notified
             request: request object
             since_when: reports are newer than this time value,
             reports: list of reports to render
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
-        template_vars["title"] = template_vars['resource_name']
+        template_vars["title"] = template_vars["resource_name"]
 
-        if template_vars['confirmed_total'] > 1:
-            template_vars['subtext'] = '%s reports' % template_vars[
-                'confirmed_total']
+        if template_vars["confirmed_total"] > 1:
+            template_vars["subtext"] = "%s reports" % template_vars["confirmed_total"]
         else:
-            error_title = truncate(template_vars['reports'][0][1].error or
-                                   'slow report', 90)
-            template_vars['subtext'] = error_title
+            error_title = truncate(
+                template_vars["reports"][0][1].error or "slow report", 90
+            )
+            template_vars["subtext"] = error_title
 
-        log_msg = 'NOTIFY  : %s via %s :: %s reports' % (
-            kwargs['user'].user_name,
+        log_msg = "NOTIFY  : %s via %s :: %s reports" % (
+            kwargs["user"].user_name,
             self.channel_visible_value,
-            template_vars['confirmed_total'])
+            template_vars["confirmed_total"],
+        )
         log.warning(log_msg)
 
-        client = SlackIntegration.create_client(
-            self.integration.config['webhook_url'])
+        client = SlackIntegration.create_client(self.integration.config["webhook_url"])
         report_data = {
             "username": "AppEnlight",
             "icon_emoji": ":fire:",
             "attachments": [
                 {
                     "mrkdwn_in": ["text", "pretext", "title", "fallback"],
-                    "fallback": "*%s* - <%s| Browse>" % (
-                        template_vars["title"],
-                        template_vars['destination_url']),
-                    "pretext": "*%s* - <%s| Browse>" % (
-                        template_vars["title"],
-                        template_vars['destination_url']),
+                    "fallback": "*%s* - <%s| Browse>"
+                    % (template_vars["title"], template_vars["destination_url"]),
+                    "pretext": "*%s* - <%s| Browse>"
+                    % (template_vars["title"], template_vars["destination_url"]),
                     "color": "warning",
                     "fields": [
-                        {
-                            "value": 'Info: %s' % template_vars['subtext'],
-                            "short": False
-                        }
-                    ]
+                        {"value": "Info: %s" % template_vars["subtext"], "short": False}
+                    ],
                 }
-            ]
+            ],
         }
         client.make_request(data=report_data)
 
     def notify_report_alert(self, **kwargs):
         """
         Build and send report alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
 
-        if kwargs['event'].unified_alert_action() == 'OPEN':
-            title = '*ALERT %s*: %s' % (
-                template_vars['alert_action'],
-                template_vars['resource_name']
+        if kwargs["event"].unified_alert_action() == "OPEN":
+            title = "*ALERT %s*: %s" % (
+                template_vars["alert_action"],
+                template_vars["resource_name"],
             )
 
-            template_vars['subtext'] = 'Got at least %s %s' % (
-                kwargs['event'].values['reports'],
-                template_vars['report_type']
+            template_vars["subtext"] = "Got at least %s %s" % (
+                kwargs["event"].values["reports"],
+                template_vars["report_type"],
             )
 
         else:
-            title = '*ALERT %s*: %s' % (
-                template_vars['alert_action'],
-                template_vars['resource_name'],
+            title = "*ALERT %s*: %s" % (
+                template_vars["alert_action"],
+                template_vars["resource_name"],
             )
 
-            template_vars['subtext'] = ''
+            template_vars["subtext"] = ""
 
-        alert_type = template_vars['alert_type'].replace('_', ' ')
-        alert_type = alert_type.replace('alert', '').capitalize()
+        alert_type = template_vars["alert_type"].replace("_", " ")
+        alert_type = alert_type.replace("alert", "").capitalize()
 
-        template_vars['type'] = "Type: %s" % alert_type
+        template_vars["type"] = "Type: %s" % alert_type
 
-        client = SlackIntegration.create_client(
-            self.integration.config['webhook_url']
-        )
+        client = SlackIntegration.create_client(self.integration.config["webhook_url"])
         report_data = {
             "username": "AppEnlight",
             "icon_emoji": ":rage:",
             "attachments": [
                 {
                     "mrkdwn_in": ["text", "pretext", "title", "fallback"],
-                    "fallback": "%s - <%s| Browse>" % (
-                        title, template_vars['destination_url']),
-                    "pretext": "%s - <%s| Browse>" % (
-                        title, template_vars['destination_url']),
+                    "fallback": "%s - <%s| Browse>"
+                    % (title, template_vars["destination_url"]),
+                    "pretext": "%s - <%s| Browse>"
+                    % (title, template_vars["destination_url"]),
                     "color": "danger",
                     "fields": [
                         {
-                            "title": template_vars['type'],
-                            "value": template_vars['subtext'],
-                            "short": False
+                            "title": template_vars["type"],
+                            "value": template_vars["subtext"],
+                            "short": False,
                         }
-                    ]
+                    ],
                 }
-            ]
+            ],
         }
         client.make_request(data=report_data)
 
     def notify_uptime_alert(self, **kwargs):
         """
         Build and send uptime alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.uptime_alert_notification_vars(kwargs)
 
-        title = '*ALERT %s*: %s' % (
-            template_vars['alert_action'],
-            template_vars['resource_name'],
-        )
-        client = SlackIntegration.create_client(
-            self.integration.config['webhook_url']
+        title = "*ALERT %s*: %s" % (
+            template_vars["alert_action"],
+            template_vars["resource_name"],
         )
+        client = SlackIntegration.create_client(self.integration.config["webhook_url"])
         report_data = {
             "username": "AppEnlight",
             "icon_emoji": ":rage:",
             "attachments": [
                 {
                     "mrkdwn_in": ["text", "pretext", "title", "fallback"],
                     "fallback": "{} - <{}| Browse>".format(
-                        title, template_vars['destination_url']),
+                        title, template_vars["destination_url"]
+                    ),
                     "pretext": "{} - <{}| Browse>".format(
-                        title, template_vars['destination_url']),
+                        title, template_vars["destination_url"]
+                    ),
                     "color": "danger",
                     "fields": [
                         {
                             "title": "Application has uptime issues",
-                            "value": template_vars['reason'],
-                            "short": False
+                            "value": template_vars["reason"],
+                            "short": False,
                         }
-                    ]
+                    ],
                 }
-            ]
+            ],
         }
         client.make_request(data=report_data)
 
     def notify_chart_alert(self, **kwargs):
         """
         Build and send chart alert notification
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         template_vars = self.chart_alert_notification_vars(kwargs)
 
-        title = '*ALERT {}*: value in *"{}"* chart ' \
-                'met alert *"{}"* criteria'.format(
-            template_vars['alert_action'],
-            template_vars['chart_name'],
-            template_vars['action_name'],
+        title = '*ALERT {}*: value in *"{}"* chart ' 'met alert *"{}"* criteria'.format(
+            template_vars["alert_action"],
+            template_vars["chart_name"],
+            template_vars["action_name"],
         )
 
-        subtext = ''
-        for item in template_vars['readable_values']:
-            subtext += '{} - {}\n'.format(item['label'], item['value'])
+        subtext = ""
+        for item in template_vars["readable_values"]:
+            subtext += "{} - {}\n".format(item["label"], item["value"])
 
-        client = SlackIntegration.create_client(
-            self.integration.config['webhook_url']
-        )
+        client = SlackIntegration.create_client(self.integration.config["webhook_url"])
         report_data = {
             "username": "AppEnlight",
             "icon_emoji": ":rage:",
             "attachments": [
-                {"mrkdwn_in": ["text", "pretext", "title", "fallback"],
-                 "fallback": "{} - <{}| Browse>".format(
-                     title, template_vars['destination_url']),
-                 "pretext": "{} - <{}| Browse>".format(
-                     title, template_vars['destination_url']),
-                 "color": "danger",
-                 "fields": [
-                     {
-                         "title": "Following criteria were met:",
-                         "value": subtext,
-                         "short": False
-                     }
-                 ]
-                 }
-            ]
+                {
+                    "mrkdwn_in": ["text", "pretext", "title", "fallback"],
+                    "fallback": "{} - <{}| Browse>".format(
+                        title, template_vars["destination_url"]
+                    ),
+                    "pretext": "{} - <{}| Browse>".format(
+                        title, template_vars["destination_url"]
+                    ),
+                    "color": "danger",
+                    "fields": [
+                        {
+                            "title": "Following criteria were met:",
+                            "value": subtext,
+                            "short": False,
+                        }
+                    ],
+                }
+            ],
         }
         client.make_request(data=report_data)
 
     def send_digest(self, **kwargs):
         """
         Build and send daily digest notification
 
         kwargs:
             application: application that the event applies for,
             user: user that should be notified
             request: request object
             since_when: reports are newer than this time value,
             reports: list of reports to render
 
         """
         template_vars = self.report_alert_notification_vars(kwargs)
-        title = "*Daily report digest*: %s" % template_vars['resource_name']
+        title = "*Daily report digest*: %s" % template_vars["resource_name"]
 
-        subtext = '%s reports' % template_vars['confirmed_total']
+        subtext = "%s reports" % template_vars["confirmed_total"]
 
-        client = SlackIntegration.create_client(
-            self.integration.config['webhook_url']
-        )
+        client = SlackIntegration.create_client(self.integration.config["webhook_url"])
         report_data = {
             "username": "AppEnlight",
             "attachments": [
                 {
                     "mrkdwn_in": ["text", "pretext", "title", "fallback"],
-                    "fallback": "%s : <%s| Browse>" % (
-                        title, template_vars['destination_url']),
-                    "pretext": "%s: <%s| Browse>" % (
-                        title, template_vars['destination_url']),
+                    "fallback": "%s : <%s| Browse>"
+                    % (title, template_vars["destination_url"]),
+                    "pretext": "%s: <%s| Browse>"
+                    % (title, template_vars["destination_url"]),
                     "color": "good",
-                    "fields": [
-                        {
-                            "title": "Got at least: %s" % subtext,
-                            "short": False
-                        }
-                    ]
+                    "fields": [{"title": "Got at least: %s" % subtext, "short": False}],
                 }
-            ]
+            ],
         }
         client.make_request(data=report_data)
 
-        log_msg = 'DIGEST  : %s via %s :: %s reports' % (
-            kwargs['user'].user_name,
+        log_msg = "DIGEST  : %s via %s :: %s reports" % (
+            kwargs["user"].user_name,
             self.channel_visible_value,
-            template_vars['confirmed_total'])
+            template_vars["confirmed_total"],
+        )
         log.warning(log_msg)

backend/src/appenlight/models/application.py

@@ -1,104 +1,113 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import uuid
 import logging
 import sqlalchemy as sa
 from appenlight.models.resource import Resource
 from sqlalchemy.orm import aliased
 
 log = logging.getLogger(__name__)
 
 
 def generate_api_key():
-    uid = str(uuid.uuid4()).replace('-', '')
+    uid = str(uuid.uuid4()).replace("-", "")
     return uid[0:32]
 
 
 class Application(Resource):
     """
     Resource of application type
     """
 
-    __tablename__ = 'applications'
-    __mapper_args__ = {'polymorphic_identity': 'application'}
+    __tablename__ = "applications"
+    __mapper_args__ = {"polymorphic_identity": "application"}
 
     # lists configurable possible permissions for this resource type
-    __possible_permissions__ = ('view', 'update_reports')
-
-    resource_id = sa.Column(sa.Integer(),
-                            sa.ForeignKey('resources.resource_id',
-                                          onupdate='CASCADE',
-                                          ondelete='CASCADE', ),
-                            primary_key=True, )
-    domains = sa.Column(sa.UnicodeText(), nullable=False, default='')
-    api_key = sa.Column(sa.String(32), nullable=False, unique=True, index=True,
-                        default=generate_api_key)
-    public_key = sa.Column(sa.String(32), nullable=False, unique=True,
-                           index=True,
-                           default=generate_api_key)
-    default_grouping = sa.Column(sa.Unicode(20), nullable=False,
-                                 default='url_traceback')
+    __possible_permissions__ = ("view", "update_reports")
+
+    resource_id = sa.Column(
+        sa.Integer(),
+        sa.ForeignKey("resources.resource_id", onupdate="CASCADE", ondelete="CASCADE"),
+        primary_key=True,
+    )
+    domains = sa.Column(sa.UnicodeText(), nullable=False, default="")
+    api_key = sa.Column(
+        sa.String(32), nullable=False, unique=True, index=True, default=generate_api_key
+    )
+    public_key = sa.Column(
+        sa.String(32), nullable=False, unique=True, index=True, default=generate_api_key
+    )
+    default_grouping = sa.Column(
+        sa.Unicode(20), nullable=False, default="url_traceback"
+    )
     error_report_threshold = sa.Column(sa.Integer(), default=10)
     slow_report_threshold = sa.Column(sa.Integer(), default=10)
-    allow_permanent_storage = sa.Column(sa.Boolean(), default=False,
-                                        nullable=False)
+    allow_permanent_storage = sa.Column(sa.Boolean(), default=False, nullable=False)
 
-    @sa.orm.validates('default_grouping')
+    @sa.orm.validates("default_grouping")
     def validate_default_grouping(self, key, grouping):
         """ validate if resouce can have specific permission """
-        assert grouping in ['url_type', 'url_traceback', 'traceback_server']
+        assert grouping in ["url_type", "url_traceback", "traceback_server"]
         return grouping
 
-    report_groups = sa.orm.relationship('ReportGroup',
-                                        cascade="all, delete-orphan",
-                                        passive_deletes=True,
-                                        passive_updates=True,
-                                        lazy='dynamic',
-                                        backref=sa.orm.backref('application',
-                                                               lazy="joined"))
-
-    postprocess_conf = sa.orm.relationship('ApplicationPostprocessConf',
-                                           cascade="all, delete-orphan",
-                                           passive_deletes=True,
-                                           passive_updates=True,
-                                           backref='resource')
-
-    logs = sa.orm.relationship('Log',
-                               lazy='dynamic',
-                               backref='application',
-                               passive_deletes=True,
-                               passive_updates=True, )
-
-    integrations = sa.orm.relationship('IntegrationBase',
-                                       backref='resource',
-                                       cascade="all, delete-orphan",
-                                       passive_deletes=True,
-                                       passive_updates=True, )
+    report_groups = sa.orm.relationship(
+        "ReportGroup",
+        cascade="all, delete-orphan",
+        passive_deletes=True,
+        passive_updates=True,
+        lazy="dynamic",
+        backref=sa.orm.backref("application", lazy="joined"),
+    )
+
+    postprocess_conf = sa.orm.relationship(
+        "ApplicationPostprocessConf",
+        cascade="all, delete-orphan",
+        passive_deletes=True,
+        passive_updates=True,
+        backref="resource",
+    )
+
+    logs = sa.orm.relationship(
+        "Log",
+        lazy="dynamic",
+        backref="application",
+        passive_deletes=True,
+        passive_updates=True,
+    )
+
+    integrations = sa.orm.relationship(
+        "IntegrationBase",
+        backref="resource",
+        cascade="all, delete-orphan",
+        passive_deletes=True,
+        passive_updates=True,
+    )
 
     def generate_api_key(self):
         return generate_api_key()
 
 
 def after_update(mapper, connection, target):
     from appenlight.models.services.application import ApplicationService
-    log.info('clearing out ApplicationService cache')
+
+    log.info("clearing out ApplicationService cache")
     ApplicationService.by_id_cached().invalidate(target.resource_id)
     ApplicationService.by_api_key_cached().invalidate(target.api_key)
 
 
-sa.event.listen(Application, 'after_update', after_update)
-sa.event.listen(Application, 'after_delete', after_update)
+sa.event.listen(Application, "after_update", after_update)
+sa.event.listen(Application, "after_delete", after_update)

backend/src/appenlight/models/application_postprocess_conf.py

@@ -1,45 +1,47 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from ziggurat_foundations.models.base import BaseModel
 import sqlalchemy as sa
 
 from appenlight.models import Base
 from appenlight.models.report_group import ReportGroup
 
 
 class ApplicationPostprocessConf(Base, BaseModel):
     """
     Stores prioritizing conditions for reports
     This is later used for rule parsing like "if 10 occurences bump priority +1"
     """
 
-    __tablename__ = 'application_postprocess_conf'
+    __tablename__ = "application_postprocess_conf"
 
     pkey = sa.Column(sa.Integer(), nullable=False, primary_key=True)
-    resource_id = sa.Column(sa.Integer(),
-                            sa.ForeignKey('resources.resource_id',
-                                          onupdate='CASCADE',
-                                          ondelete='CASCADE'))
+    resource_id = sa.Column(
+        sa.Integer(),
+        sa.ForeignKey("resources.resource_id", onupdate="CASCADE", ondelete="CASCADE"),
+    )
     do = sa.Column(sa.Unicode(25), nullable=False)
-    new_value = sa.Column(sa.UnicodeText(), nullable=False, default='')
-    rule = sa.Column(sa.dialects.postgresql.JSON,
-                     nullable=False, default={'field': 'http_status',
-                                              "op": "ge", "value": "500"})
+    new_value = sa.Column(sa.UnicodeText(), nullable=False, default="")
+    rule = sa.Column(
+        sa.dialects.postgresql.JSON,
+        nullable=False,
+        default={"field": "http_status", "op": "ge", "value": "500"},
+    )
 
     def postprocess(self, item):
         new_value = int(self.new_value)
         item.priority = ReportGroup.priority + new_value

backend/src/appenlight/models/auth_token.py

@@ -1,52 +1,57 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 import sqlalchemy as sa
 
 from datetime import datetime
 from appenlight.models import Base
 from ziggurat_foundations.models.base import BaseModel
 from ziggurat_foundations.models.services.user import UserService
 
 log = logging.getLogger(__name__)
 
 
 class AuthToken(Base, BaseModel):
     """
     Stores information about possible alerting options
     """
-    __tablename__ = 'auth_tokens'
+
+    __tablename__ = "auth_tokens"
 
     id = sa.Column(sa.Integer, primary_key=True, nullable=False)
-    token = sa.Column(sa.Unicode(40), nullable=False,
-                      default=lambda x: UserService.generate_random_string(40))
-    owner_id = sa.Column(sa.Unicode(30),
-                         sa.ForeignKey('users.id', onupdate='CASCADE',
-                                       ondelete='CASCADE'))
+    token = sa.Column(
+        sa.Unicode(40),
+        nullable=False,
+        default=lambda x: UserService.generate_random_string(40),
+    )
+    owner_id = sa.Column(
+        sa.Unicode(30),
+        sa.ForeignKey("users.id", onupdate="CASCADE", ondelete="CASCADE"),
+    )
     creation_date = sa.Column(sa.DateTime, default=lambda x: datetime.utcnow())
     expires = sa.Column(sa.DateTime)
-    description = sa.Column(sa.Unicode, default='')
+    description = sa.Column(sa.Unicode, default="")
 
     @property
     def is_expired(self):
         if self.expires:
             return self.expires < datetime.utcnow()
         else:
             return False
 
     def __str__(self):
-        return '<AuthToken u:%s t:%s...>' % (self.owner_id, self.token[0:10])
+        return "<AuthToken u:%s t:%s...>" % (self.owner_id, self.token[0:10])

backend/src/appenlight/models/config.py

@@ -1,32 +1,32 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import sqlalchemy as sa
 from ziggurat_foundations.models.base import BaseModel
 from sqlalchemy.dialects.postgresql import JSON
 
 from . import Base
 
 
 class Config(Base, BaseModel):
-    __tablename__ = 'config'
+    __tablename__ = "config"
 
     key = sa.Column(sa.Unicode, primary_key=True)
     section = sa.Column(sa.Unicode, primary_key=True)
     value = sa.Column(JSON, nullable=False)
 
     def __json__(self, request):
         return self.get_dict()

backend/src/appenlight/models/event.py

@@ -1,165 +1,170 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import sqlalchemy as sa
 import logging
 
 from datetime import datetime
 from appenlight.models import Base, get_db_session
 from appenlight.models.services.report_stat import ReportStatService
 from appenlight.models.integrations import IntegrationException
 from pyramid.threadlocal import get_current_request
 from sqlalchemy.dialects.postgresql import JSON
 from ziggurat_foundations.models.base import BaseModel
 from ziggurat_foundations.models.services.resource import ResourceService
 
 log = logging.getLogger(__name__)
 
 
 class Event(Base, BaseModel):
-    __tablename__ = 'events'
+    __tablename__ = "events"
 
-    types = {'error_report_alert': 1,
-             'slow_report_alert': 3,
-             'comment': 5,
-             'assignment': 6,
-             'uptime_alert': 7,
-             'chart_alert': 9}
+    types = {
+        "error_report_alert": 1,
+        "slow_report_alert": 3,
+        "comment": 5,
+        "assignment": 6,
+        "uptime_alert": 7,
+        "chart_alert": 9,
+    }
 
-    statuses = {'active': 1,
-                'closed': 0}
+    statuses = {"active": 1, "closed": 0}
 
     id = sa.Column(sa.Integer, primary_key=True)
     start_date = sa.Column(sa.DateTime, default=datetime.utcnow)
     end_date = sa.Column(sa.DateTime)
     status = sa.Column(sa.Integer, default=1)
     event_type = sa.Column(sa.Integer, default=1)
-    origin_user_id = sa.Column(sa.Integer(), sa.ForeignKey('users.id'),
-                               nullable=True)
-    target_user_id = sa.Column(sa.Integer(), sa.ForeignKey('users.id'),
-                               nullable=True)
-    resource_id = sa.Column(sa.Integer(),
-                            sa.ForeignKey('resources.resource_id'),
-                            nullable=True)
+    origin_user_id = sa.Column(sa.Integer(), sa.ForeignKey("users.id"), nullable=True)
+    target_user_id = sa.Column(sa.Integer(), sa.ForeignKey("users.id"), nullable=True)
+    resource_id = sa.Column(
+        sa.Integer(), sa.ForeignKey("resources.resource_id"), nullable=True
+    )
     target_id = sa.Column(sa.Integer)
     target_uuid = sa.Column(sa.Unicode(40))
     text = sa.Column(sa.UnicodeText())
     values = sa.Column(JSON(), nullable=False, default=None)
 
     def __repr__(self):
-        return '<Event %s, app:%s, %s>' % (self.unified_alert_name(),
-                                            self.resource_id,
-                                            self.unified_alert_action())
+        return "<Event %s, app:%s, %s>" % (
+            self.unified_alert_name(),
+            self.resource_id,
+            self.unified_alert_action(),
+        )
 
     @property
     def reverse_types(self):
         return dict([(v, k) for k, v in self.types.items()])
 
     def unified_alert_name(self):
         return self.reverse_types[self.event_type]
 
     def unified_alert_action(self):
         event_name = self.reverse_types[self.event_type]
-        if self.status == Event.statuses['closed']:
+        if self.status == Event.statuses["closed"]:
             return "CLOSE"
-        if self.status != Event.statuses['closed']:
+        if self.status != Event.statuses["closed"]:
             return "OPEN"
         return event_name
 
     def send_alerts(self, request=None, resource=None, db_session=None):
         """" Sends alerts to applicable channels """
         db_session = get_db_session(db_session)
         db_session.flush()
         if not resource:
             resource = ResourceService.by_resource_id(self.resource_id)
         if not request:
             request = get_current_request()
         if not resource:
             return
-        users = set([p.user for p in ResourceService.users_for_perm(resource, 'view')])
+        users = set([p.user for p in ResourceService.users_for_perm(resource, "view")])
         for user in users:
             for channel in user.alert_channels:
-                matches_resource = not channel.resources or resource in [r.resource_id for r in channel.resources]
+                matches_resource = not channel.resources or resource in [
+                    r.resource_id for r in channel.resources
+                ]
                 if (
-                    not channel.channel_validated or
-                    not channel.send_alerts or
-                    not matches_resource
+                    not channel.channel_validated
+                    or not channel.send_alerts
+                    or not matches_resource
                 ):
                     continue
                 else:
                     try:
-                        channel.notify_alert(resource=resource,
-                                             event=self,
-                                             user=user,
-                                             request=request)
+                        channel.notify_alert(
+                            resource=resource, event=self, user=user, request=request
+                        )
                     except IntegrationException as e:
-                        log.warning('%s' % e)
+                        log.warning("%s" % e)
 
     def validate_or_close(self, since_when, db_session=None):
         """ Checks if alerts should stay open or it's time to close them.
             Generates close alert event if alerts get closed """
-        event_types = [Event.types['error_report_alert'],
-                       Event.types['slow_report_alert']]
+        event_types = [
+            Event.types["error_report_alert"],
+            Event.types["slow_report_alert"],
+        ]
         app = ResourceService.by_resource_id(self.resource_id)
         # if app was deleted close instantly
         if not app:
             self.close()
             return
 
         if self.event_type in event_types:
             total = ReportStatService.count_by_type(
-                self.event_type, self.resource_id, since_when)
-            if Event.types['error_report_alert'] == self.event_type:
+                self.event_type, self.resource_id, since_when
+            )
+            if Event.types["error_report_alert"] == self.event_type:
                 threshold = app.error_report_threshold
-            if Event.types['slow_report_alert'] == self.event_type:
+            if Event.types["slow_report_alert"] == self.event_type:
                 threshold = app.slow_report_threshold
 
             if total < threshold:
                 self.close()
 
     def close(self, db_session=None):
         """
         Closes an event and sends notification to affected users
         """
         self.end_date = datetime.utcnow()
-        self.status = Event.statuses['closed']
-        log.warning('ALERT: CLOSE: %s' % self)
+        self.status = Event.statuses["closed"]
+        log.warning("ALERT: CLOSE: %s" % self)
         self.send_alerts()
 
     def text_representation(self):
         alert_type = self.unified_alert_name()
-        text = ''
-        if 'slow_report' in alert_type:
-            text += 'Slow report alert'
-        if 'error_report' in alert_type:
-            text += 'Exception report alert'
-        if 'uptime_alert' in alert_type:
-            text += 'Uptime alert'
-        if 'chart_alert' in alert_type:
-            text += 'Metrics value alert'
+        text = ""
+        if "slow_report" in alert_type:
+            text += "Slow report alert"
+        if "error_report" in alert_type:
+            text += "Exception report alert"
+        if "uptime_alert" in alert_type:
+            text += "Uptime alert"
+        if "chart_alert" in alert_type:
+            text += "Metrics value alert"
 
         alert_action = self.unified_alert_action()
-        if alert_action == 'OPEN':
-            text += ' got opened.'
-        if alert_action == 'CLOSE':
-            text += ' got closed.'
+        if alert_action == "OPEN":
+            text += " got opened."
+        if alert_action == "CLOSE":
+            text += " got closed."
         return text
 
     def get_dict(self, request=None):
         dict_data = super(Event, self).get_dict()
-        dict_data['text'] = self.text_representation()
-        dict_data['resource_name'] = self.resource.resource_name
+        dict_data["text"] = self.text_representation()
+        dict_data["resource_name"] = self.resource.resource_name
         return dict_data

backend/src/appenlight/models/external_identity.py

@@ -1,36 +1,36 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import sqlalchemy as sa
 from sqlalchemy.ext.declarative import declared_attr
 from ziggurat_foundations.models.external_identity import ExternalIdentityMixin
 
 from appenlight.models import Base
 from appenlight.lib.sqlalchemy_fields import EncryptedUnicode
 
 
 class ExternalIdentity(ExternalIdentityMixin, Base):
     @declared_attr
     def access_token(self):
-        return sa.Column(EncryptedUnicode(255), default='')
+        return sa.Column(EncryptedUnicode(255), default="")
 
     @declared_attr
     def alt_token(self):
-        return sa.Column(EncryptedUnicode(255), default='')
+        return sa.Column(EncryptedUnicode(255), default="")
 
     @declared_attr
     def token_secret(self):
-        return sa.Column(EncryptedUnicode(255), default='')
+        return sa.Column(EncryptedUnicode(255), default="")

backend/src/appenlight/models/group.py

@@ -1,45 +1,46 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from ziggurat_foundations.models.group import GroupMixin
 from appenlight.models import Base
 
 
 class Group(GroupMixin, Base):
-    __possible_permissions__ = ('root_administration',
-                                'test_features',
-                                'admin_panel',
-                                'admin_users',
-                                'manage_partitions',)
+    __possible_permissions__ = (
+        "root_administration",
+        "test_features",
+        "admin_panel",
+        "admin_users",
+        "manage_partitions",
+    )
 
-    def get_dict(self, exclude_keys=None, include_keys=None,
-                 include_perms=False):
+    def get_dict(self, exclude_keys=None, include_keys=None, include_perms=False):
         result = super(Group, self).get_dict(exclude_keys, include_keys)
         if include_perms:
-            result['possible_permissions'] = self.__possible_permissions__
-            result['current_permissions'] = [p.perm_name for p in
-                                             self.permissions]
+            result["possible_permissions"] = self.__possible_permissions__
+            result["current_permissions"] = [p.perm_name for p in self.permissions]
         else:
-            result['possible_permissions'] = []
-            result['current_permissions'] = []
+            result["possible_permissions"] = []
+            result["current_permissions"] = []
         exclude_keys_list = exclude_keys or []
         include_keys_list = include_keys or []
         d = {}
         for k in result.keys():
-            if (k not in exclude_keys_list and
-                    (k in include_keys_list or not include_keys)):
+            if k not in exclude_keys_list and (
+                k in include_keys_list or not include_keys
+            ):
                 d[k] = result[k]
         return d

backend/src/appenlight/models/group_resource_permission.py

@@ -1,23 +1,24 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ziggurat_foundations.models.group_resource_permission import \
-    GroupResourcePermissionMixin
+from ziggurat_foundations.models.group_resource_permission import (
+    GroupResourcePermissionMixin,
+)
 from appenlight.models import Base
 
 
 class GroupResourcePermission(GroupResourcePermissionMixin, Base):
     pass

backend/src/appenlight/models/integrations/__init__.py

@@ -1,78 +1,80 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import sqlalchemy as sa
 from sqlalchemy.dialects.postgresql import JSON
 from sqlalchemy.ext.hybrid import hybrid_property
 from ziggurat_foundations.models.base import BaseModel
 
 from appenlight.lib.encryption import decrypt_dictionary_keys
 from appenlight.lib.encryption import encrypt_dictionary_keys
 from appenlight.models import Base, get_db_session
 
 
 class IntegrationException(Exception):
     pass
 
 
 class IntegrationBase(Base, BaseModel):
     """
     Model from which all integrations inherit using polymorphic approach
     """
-    __tablename__ = 'integrations'
+
+    __tablename__ = "integrations"
 
     front_visible = False
     as_alert_channel = False
     supports_report_alerting = False
 
     id = sa.Column(sa.Integer, primary_key=True)
-    resource_id = sa.Column(sa.Integer,
-                            sa.ForeignKey('applications.resource_id'))
+    resource_id = sa.Column(sa.Integer, sa.ForeignKey("applications.resource_id"))
     integration_name = sa.Column(sa.Unicode(64))
-    _config = sa.Column('config', JSON(), nullable=False, default='')
+    _config = sa.Column("config", JSON(), nullable=False, default="")
     modified_date = sa.Column(sa.DateTime)
 
-    channel = sa.orm.relationship('AlertChannel',
-                                  cascade="all,delete-orphan",
-                                  passive_deletes=True,
-                                  passive_updates=True,
-                                  uselist=False,
-                                  backref='integration')
+    channel = sa.orm.relationship(
+        "AlertChannel",
+        cascade="all,delete-orphan",
+        passive_deletes=True,
+        passive_updates=True,
+        uselist=False,
+        backref="integration",
+    )
 
     __mapper_args__ = {
-        'polymorphic_on': 'integration_name',
-        'polymorphic_identity': 'integration'
+        "polymorphic_on": "integration_name",
+        "polymorphic_identity": "integration",
     }
 
     @classmethod
-    def by_app_id_and_integration_name(cls, resource_id, integration_name,
-                                       db_session=None):
+    def by_app_id_and_integration_name(
+        cls, resource_id, integration_name, db_session=None
+    ):
         db_session = get_db_session(db_session)
         query = db_session.query(cls)
         query = query.filter(cls.integration_name == integration_name)
         query = query.filter(cls.resource_id == resource_id)
         return query.first()
 
     @hybrid_property
     def config(self):
         return decrypt_dictionary_keys(self._config)
 
     @config.setter
     def config(self, value):
-        if not hasattr(value, 'items'):
-            raise Exception('IntegrationBase.config only accepts '
-                            'flat dictionaries')
+        if not hasattr(value, "items"):
+            raise Exception("IntegrationBase.config only accepts " "flat dictionaries")
         self._config = encrypt_dictionary_keys(value)

backend/src/appenlight/models/integrations/bitbucket.py

@@ -1,163 +1,188 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import requests
 from requests_oauthlib import OAuth1
-from appenlight.models.integrations import (IntegrationBase,
-                                            IntegrationException)
+from appenlight.models.integrations import IntegrationBase, IntegrationException
 
 _ = str
 
 
 class NotFoundException(Exception):
     pass
 
 
 class BitbucketIntegration(IntegrationBase):
-    __mapper_args__ = {
-        'polymorphic_identity': 'bitbucket'
-    }
+    __mapper_args__ = {"polymorphic_identity": "bitbucket"}
     front_visible = True
     as_alert_channel = False
     supports_report_alerting = False
     action_notification = True
-    integration_action = 'Add issue to Bitbucket'
+    integration_action = "Add issue to Bitbucket"
 
     @classmethod
     def create_client(cls, request, user_name=None, repo_name=None):
         """
         Creates REST client that can authenticate to specific repo
         uses auth tokens for current request user
         """
         config = request.registry.settings
         token = None
         secret = None
         for identity in request.user.external_identities:
-            if identity.provider_name == 'bitbucket':
+            if identity.provider_name == "bitbucket":
                 token = identity.access_token
                 secret = identity.token_secret
                 break
         if not token:
-            raise IntegrationException(
-                'No valid auth token present for this service')
-        client = BitbucketClient(token, secret,
-                                 user_name,
-                                 repo_name,
-                                 config['authomatic.pr.bitbucket.key'],
-                                 config['authomatic.pr.bitbucket.secret'])
+            raise IntegrationException("No valid auth token present for this service")
+        client = BitbucketClient(
+            token,
+            secret,
+            user_name,
+            repo_name,
+            config["authomatic.pr.bitbucket.key"],
+            config["authomatic.pr.bitbucket.secret"],
+        )
         return client
 
 
 class BitbucketClient(object):
-    api_url = 'https://bitbucket.org/api/1.0'
-    repo_type = 'bitbucket'
-
-    def __init__(self, token, secret, owner, repo_name, bitbucket_consumer_key,
-                 bitbucket_consumer_secret):
+    api_url = "https://bitbucket.org/api/1.0"
+    repo_type = "bitbucket"
+
+    def __init__(
+        self,
+        token,
+        secret,
+        owner,
+        repo_name,
+        bitbucket_consumer_key,
+        bitbucket_consumer_secret,
+    ):
         self.access_token = token
         self.token_secret = secret
         self.owner = owner
         self.repo_name = repo_name
         self.bitbucket_consumer_key = bitbucket_consumer_key
         self.bitbucket_consumer_secret = bitbucket_consumer_secret
 
     possible_keys = {
-        'status': ['new', 'open', 'resolved', 'on hold', 'invalid',
-                   'duplicate', 'wontfix'],
-        'priority': ['trivial', 'minor', 'major', 'critical', 'blocker'],
-        'kind': ['bug', 'enhancement', 'proposal', 'task']
+        "status": [
+            "new",
+            "open",
+            "resolved",
+            "on hold",
+            "invalid",
+            "duplicate",
+            "wontfix",
+        ],
+        "priority": ["trivial", "minor", "major", "critical", "blocker"],
+        "kind": ["bug", "enhancement", "proposal", "task"],
     }
 
     def get_statuses(self):
         """Gets list of possible item statuses"""
-        return self.possible_keys['status']
+        return self.possible_keys["status"]
 
     def get_priorities(self):
         """Gets list of possible item statuses"""
-        return self.possible_keys['priority']
+        return self.possible_keys["priority"]
 
-    def make_request(self, url, method='get', data=None, headers=None):
+    def make_request(self, url, method="get", data=None, headers=None):
         """
         Performs HTTP request to bitbucket
         """
-        auth = OAuth1(self.bitbucket_consumer_key,
-                      self.bitbucket_consumer_secret,
-                      self.access_token, self.token_secret)
+        auth = OAuth1(
+            self.bitbucket_consumer_key,
+            self.bitbucket_consumer_secret,
+            self.access_token,
+            self.token_secret,
+        )
         try:
-            resp = getattr(requests, method)(url, data=data, auth=auth,
-                                             timeout=10)
+            resp = getattr(requests, method)(url, data=data, auth=auth, timeout=10)
         except Exception as e:
             raise IntegrationException(
-                _('Error communicating with Bitbucket: %s') % (e,))
+                _("Error communicating with Bitbucket: %s") % (e,)
+            )
         if resp.status_code == 401:
-            raise IntegrationException(
-                _('You are not authorized to access this repo'))
+            raise IntegrationException(_("You are not authorized to access this repo"))
         elif resp.status_code == 404:
-            raise IntegrationException(_('User or repo name are incorrect'))
+            raise IntegrationException(_("User or repo name are incorrect"))
         elif resp.status_code not in [200, 201]:
             raise IntegrationException(
-                _('Bitbucket response_code: %s') % resp.status_code)
+                _("Bitbucket response_code: %s") % resp.status_code
+            )
         try:
             return resp.json()
         except Exception as e:
             raise IntegrationException(
-                _('Error decoding response from Bitbucket: %s') % (e,))
+                _("Error decoding response from Bitbucket: %s") % (e,)
+            )
 
     def get_assignees(self):
         """Gets list of possible assignees"""
-        url = '%(api_url)s/privileges/%(owner)s/%(repo_name)s' % {
-            'api_url': self.api_url,
-            'owner': self.owner,
-            'repo_name': self.repo_name}
+        url = "%(api_url)s/privileges/%(owner)s/%(repo_name)s" % {
+            "api_url": self.api_url,
+            "owner": self.owner,
+            "repo_name": self.repo_name,
+        }
 
         data = self.make_request(url)
-        results = [{'user': self.owner, 'name': '(Repo owner)'}]
+        results = [{"user": self.owner, "name": "(Repo owner)"}]
         if data:
             for entry in data:
-                results.append({"user": entry['user']['username'],
-                                "name": entry['user'].get('display_name')})
+                results.append(
+                    {
+                        "user": entry["user"]["username"],
+                        "name": entry["user"].get("display_name"),
+                    }
+                )
         return results
 
     def create_issue(self, form_data):
         """
         Sends creates a new issue in tracker using REST call
         """
-        url = '%(api_url)s/repositories/%(owner)s/%(repo_name)s/issues/' % {
-            'api_url': self.api_url,
-            'owner': self.owner,
-            'repo_name': self.repo_name}
+        url = "%(api_url)s/repositories/%(owner)s/%(repo_name)s/issues/" % {
+            "api_url": self.api_url,
+            "owner": self.owner,
+            "repo_name": self.repo_name,
+        }
 
         payload = {
-            "title": form_data['title'],
-            "content": form_data['content'],
-            "kind": form_data['kind'],
-            "priority": form_data['priority'],
-            "responsible": form_data['responsible']
+            "title": form_data["title"],
+            "content": form_data["content"],
+            "kind": form_data["kind"],
+            "priority": form_data["priority"],
+            "responsible": form_data["responsible"],
         }
-        data = self.make_request(url, 'post', payload)
+        data = self.make_request(url, "post", payload)
         f_args = {
             "owner": self.owner,
             "repo_name": self.repo_name,
-            "issue_id": data['local_id']
+            "issue_id": data["local_id"],
         }
-        web_url = 'https://bitbucket.org/%(owner)s/%(repo_name)s' \
-                  '/issue/%(issue_id)s/issue-title' % f_args
+        web_url = (
+            "https://bitbucket.org/%(owner)s/%(repo_name)s"
+            "/issue/%(issue_id)s/issue-title" % f_args
+        )
         to_return = {
-            'id': data['local_id'],
-            'resource_url': data['resource_uri'],
-            'web_url': web_url
+            "id": data["local_id"],
+            "resource_url": data["resource_uri"],
+            "web_url": web_url,
         }
         return to_return

backend/src/appenlight/models/integrations/campfire.py

@@ -1,74 +1,71 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 
 from requests.exceptions import HTTPError, ConnectionError
 from camplight import Request, Campfire
 from camplight.exceptions import CamplightException
 
-from appenlight.models.integrations import (IntegrationBase,
-                                            IntegrationException)
+from appenlight.models.integrations import IntegrationBase, IntegrationException
 
 _ = str
 
 log = logging.getLogger(__name__)
 
 
 class NotFoundException(Exception):
     pass
 
 
 class CampfireIntegration(IntegrationBase):
-    __mapper_args__ = {
-        'polymorphic_identity': 'campfire'
-    }
+    __mapper_args__ = {"polymorphic_identity": "campfire"}
     front_visible = False
     as_alert_channel = True
     supports_report_alerting = True
     action_notification = True
-    integration_action = 'Message via Campfire'
+    integration_action = "Message via Campfire"
 
     @classmethod
     def create_client(cls, api_token, account):
         client = CampfireClient(api_token, account)
         return client
 
 
 class CampfireClient(object):
     def __init__(self, api_token, account):
-        request = Request('https://%s.campfirenow.com' % account, api_token)
+        request = Request("https://%s.campfirenow.com" % account, api_token)
         self.campfire = Campfire(request)
 
     def get_account(self):
         try:
             return self.campfire.account()
         except (HTTPError, CamplightException) as e:
             raise IntegrationException(str(e))
 
     def get_rooms(self):
         try:
             return self.campfire.rooms()
         except (HTTPError, CamplightException) as e:
             raise IntegrationException(str(e))
 
-    def speak_to_room(self, room, message, sound='RIMSHOT'):
+    def speak_to_room(self, room, message, sound="RIMSHOT"):
         try:
             room = self.campfire.room(room)
             room.join()
-            room.speak(message, type_='TextMessage')
+            room.speak(message, type_="TextMessage")
         except (HTTPError, CamplightException, ConnectionError) as e:
             raise IntegrationException(str(e))

backend/src/appenlight/models/integrations/flowdock.py

@@ -1,82 +1,83 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 
 import requests
 from requests.auth import HTTPBasicAuth
 import simplejson as json
 
-from appenlight.models.integrations import (IntegrationBase,
-                                            IntegrationException)
+from appenlight.models.integrations import IntegrationBase, IntegrationException
 
 _ = str
 
 log = logging.getLogger(__name__)
 
 
 class NotFoundException(Exception):
     pass
 
 
 class FlowdockIntegration(IntegrationBase):
-    __mapper_args__ = {
-        'polymorphic_identity': 'flowdock'
-    }
+    __mapper_args__ = {"polymorphic_identity": "flowdock"}
     front_visible = False
     as_alert_channel = True
     supports_report_alerting = True
     action_notification = True
-    integration_action = 'Message via Flowdock'
+    integration_action = "Message via Flowdock"
 
     @classmethod
     def create_client(cls, api_token):
         client = FlowdockClient(api_token)
         return client
 
 
 class FlowdockClient(object):
     def __init__(self, api_token):
-        self.auth = HTTPBasicAuth(api_token, '')
+        self.auth = HTTPBasicAuth(api_token, "")
         self.api_token = api_token
-        self.api_url = 'https://api.flowdock.com/v1/messages'
+        self.api_url = "https://api.flowdock.com/v1/messages"
 
-    def make_request(self, url, method='get', data=None):
+    def make_request(self, url, method="get", data=None):
         headers = {
-            'Content-Type': 'application/json',
-            'User-Agent': 'appenlight-flowdock',
+            "Content-Type": "application/json",
+            "User-Agent": "appenlight-flowdock",
         }
         try:
             if data:
                 data = json.dumps(data)
-            resp = getattr(requests, method)(url, data=data, headers=headers,
-                                             timeout=10)
+            resp = getattr(requests, method)(
+                url, data=data, headers=headers, timeout=10
+            )
         except Exception as e:
             raise IntegrationException(
-                _('Error communicating with Flowdock: %s') % (e,))
+                _("Error communicating with Flowdock: %s") % (e,)
+            )
         if resp.status_code > 299:
             raise IntegrationException(resp.text)
         return resp
 
     def send_to_chat(self, payload):
-        url = '%(api_url)s/chat/%(api_token)s' % {'api_url': self.api_url,
-                                                  'api_token': self.api_token}
-        return self.make_request(url, method='post', data=payload).json()
+        url = "%(api_url)s/chat/%(api_token)s" % {
+            "api_url": self.api_url,
+            "api_token": self.api_token,
+        }
+        return self.make_request(url, method="post", data=payload).json()
 
     def send_to_inbox(self, payload):
-        f_args = {'api_url': self.api_url, 'api_token': self.api_token}
-        url = '%(api_url)s/team_inbox/%(api_token)s' % f_args
-        return self.make_request(url, method='post', data=payload).json()
+        f_args = {"api_url": self.api_url, "api_token": self.api_token}
+        url = "%(api_url)s/team_inbox/%(api_token)s" % f_args
+        return self.make_request(url, method="post", data=payload).json()

backend/src/appenlight/models/integrations/github.py

@@ -1,156 +1,158 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import json
 import requests
 
 from . import IntegrationBase, IntegrationException
 
 _ = str
 
 
 class GithubAuthException(Exception):
     pass
 
 
 class GithubIntegration(IntegrationBase):
-    __mapper_args__ = {
-        'polymorphic_identity': 'github'
-    }
+    __mapper_args__ = {"polymorphic_identity": "github"}
     front_visible = True
     as_alert_channel = False
     supports_report_alerting = False
     action_notification = True
-    integration_action = 'Add issue to Github'
+    integration_action = "Add issue to Github"
 
     @classmethod
     def create_client(cls, request, user_name=None, repo_name=None):
         """
         Creates REST client that can authenticate to specific repo
         uses auth tokens for current request user
         """
         token = None
         secret = None
         for identity in request.user.external_identities:
-            if identity.provider_name == 'github':
+            if identity.provider_name == "github":
                 token = identity.access_token
                 secret = identity.token_secret
                 break
         if not token:
-            raise IntegrationException(
-                'No valid auth token present for this service')
+            raise IntegrationException("No valid auth token present for this service")
         client = GithubClient(token=token, owner=user_name, name=repo_name)
         return client
 
 
 class GithubClient(object):
-    api_url = 'https://api.github.com'
-    repo_type = 'github'
+    api_url = "https://api.github.com"
+    repo_type = "github"
 
     def __init__(self, token, owner, name):
         self.access_token = token
         self.owner = owner
         self.name = name
 
-    def make_request(self, url, method='get', data=None, headers=None):
-        req_headers = {'User-Agent': 'appenlight',
-                       'Content-Type': 'application/json',
-                       'Authorization': 'token %s' % self.access_token}
+    def make_request(self, url, method="get", data=None, headers=None):
+        req_headers = {
+            "User-Agent": "appenlight",
+            "Content-Type": "application/json",
+            "Authorization": "token %s" % self.access_token,
+        }
         try:
             if data:
                 data = json.dumps(data)
-            resp = getattr(requests, method)(url, data=data,
-                                             headers=req_headers,
-                                             timeout=10)
+            resp = getattr(requests, method)(
+                url, data=data, headers=req_headers, timeout=10
+            )
         except Exception as e:
-            msg = 'Error communicating with Github: %s'
+            msg = "Error communicating with Github: %s"
             raise IntegrationException(_(msg) % (e,))
 
         if resp.status_code == 404:
-            msg = 'User or repo name are incorrect'
+            msg = "User or repo name are incorrect"
             raise IntegrationException(_(msg))
         if resp.status_code == 401:
-            msg = 'You are not authorized to access this repo'
+            msg = "You are not authorized to access this repo"
             raise IntegrationException(_(msg))
         elif resp.status_code not in [200, 201]:
-            msg = 'Github response_code: %s'
+            msg = "Github response_code: %s"
             raise IntegrationException(_(msg) % resp.status_code)
         try:
             return resp.json()
         except Exception as e:
-            msg = 'Error decoding response from Github: %s'
+            msg = "Error decoding response from Github: %s"
             raise IntegrationException(_(msg) % (e,))
 
     def get_statuses(self):
         """Gets list of possible item statuses"""
-        url = '%(api_url)s/repos/%(owner)s/%(name)s/labels' % {
-            'api_url': self.api_url,
-            'owner': self.owner,
-            'name': self.name}
+        url = "%(api_url)s/repos/%(owner)s/%(name)s/labels" % {
+            "api_url": self.api_url,
+            "owner": self.owner,
+            "name": self.name,
+        }
 
         data = self.make_request(url)
 
         statuses = []
         for status in data:
-            statuses.append(status['name'])
+            statuses.append(status["name"])
         return statuses
 
     def get_repo(self):
         """Gets list of possible item statuses"""
-        url = '%(api_url)s/repos/%(owner)s/%(name)s' % {
-            'api_url': self.api_url,
-            'owner': self.owner,
-            'name': self.name}
+        url = "%(api_url)s/repos/%(owner)s/%(name)s" % {
+            "api_url": self.api_url,
+            "owner": self.owner,
+            "name": self.name,
+        }
 
         data = self.make_request(url)
         return data
 
     def get_assignees(self):
         """Gets list of possible assignees"""
-        url = '%(api_url)s/repos/%(owner)s/%(name)s/collaborators' % {
-            'api_url': self.api_url,
-            'owner': self.owner,
-            'name': self.name}
+        url = "%(api_url)s/repos/%(owner)s/%(name)s/collaborators" % {
+            "api_url": self.api_url,
+            "owner": self.owner,
+            "name": self.name,
+        }
         data = self.make_request(url)
         results = []
         for entry in data:
-            results.append({"user": entry['login'],
-                            "name": entry.get('name')})
+            results.append({"user": entry["login"], "name": entry.get("name")})
         return results
 
     def create_issue(self, form_data):
         """
         Make a REST call to create issue in Github's issue tracker
         """
-        url = '%(api_url)s/repos/%(owner)s/%(name)s/issues' % {
-            'api_url': self.api_url,
-            'owner': self.owner,
-            'name': self.name}
+        url = "%(api_url)s/repos/%(owner)s/%(name)s/issues" % {
+            "api_url": self.api_url,
+            "owner": self.owner,
+            "name": self.name,
+        }
 
         payload = {
-            "title": form_data['title'],
-            "body": form_data['content'],
+            "title": form_data["title"],
+            "body": form_data["content"],
             "labels": [],
-            "assignee": form_data['responsible']
+            "assignee": form_data["responsible"],
         }
-        payload['labels'].extend(form_data['kind'])
-        data = self.make_request(url, 'post', data=payload)
+        payload["labels"].extend(form_data["kind"])
+        data = self.make_request(url, "post", data=payload)
         to_return = {
-            'id': data['number'],
-            'resource_url': data['url'],
-            'web_url': data['html_url']
+            "id": data["number"],
+            "resource_url": data["url"],
+            "web_url": data["html_url"],
         }
         return to_return

backend/src/appenlight/models/integrations/hipchat.py

@@ -1,83 +1,75 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 
 import requests
 
 from . import IntegrationBase, IntegrationException
 
 _ = str
 
 log = logging.getLogger(__name__)
 
 
 class NotFoundException(Exception):
     pass
 
 
 class HipchatIntegration(IntegrationBase):
-    __mapper_args__ = {
-        'polymorphic_identity': 'hipchat'
-    }
+    __mapper_args__ = {"polymorphic_identity": "hipchat"}
     front_visible = False
     as_alert_channel = True
     supports_report_alerting = True
     action_notification = True
-    integration_action = 'Message via Hipchat'
+    integration_action = "Message via Hipchat"
 
     @classmethod
     def create_client(cls, api_token):
         client = HipchatClient(api_token)
         return client
 
 
 class HipchatClient(object):
     def __init__(self, api_token):
         self.api_token = api_token
-        self.api_url = 'https://api.hipchat.com/v1'
-
-    def make_request(self, endpoint, method='get', data=None):
-        headers = {
-            'User-Agent': 'appenlight-hipchat',
-        }
-        url = '%s%s' % (self.api_url, endpoint)
-        params = {
-            'format': 'json',
-            'auth_token': self.api_token
-        }
+        self.api_url = "https://api.hipchat.com/v1"
+
+    def make_request(self, endpoint, method="get", data=None):
+        headers = {"User-Agent": "appenlight-hipchat"}
+        url = "%s%s" % (self.api_url, endpoint)
+        params = {"format": "json", "auth_token": self.api_token}
         try:
-            resp = getattr(requests, method)(url, data=data, headers=headers,
-                                             params=params,
-                                             timeout=3)
+            resp = getattr(requests, method)(
+                url, data=data, headers=headers, params=params, timeout=3
+            )
         except Exception as e:
-            msg = 'Error communicating with Hipchat: %s'
+            msg = "Error communicating with Hipchat: %s"
             raise IntegrationException(_(msg) % (e,))
         if resp.status_code == 404:
-            msg = 'Error communicating with Hipchat - Room not found'
+            msg = "Error communicating with Hipchat - Room not found"
             raise IntegrationException(msg)
         elif resp.status_code != requests.codes.ok:
-            msg = 'Error communicating with Hipchat - status code: %s'
+            msg = "Error communicating with Hipchat - status code: %s"
             raise IntegrationException(msg % resp.status_code)
         return resp
 
     def get_rooms(self):
         # not used with notification api token
-        return self.make_request('/rooms/list')
+        return self.make_request("/rooms/list")
 
     def send(self, payload):
-        return self.make_request('/rooms/message', method='post',
-                                 data=payload).json()
+        return self.make_request("/rooms/message", method="post", data=payload).json()

backend/src/appenlight/models/integrations/jira.py

@@ -1,136 +1,137 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import jira
-from appenlight.models.integrations import (IntegrationBase,
-                                            IntegrationException)
+from appenlight.models.integrations import IntegrationBase, IntegrationException
 
 _ = str
 
 
 class NotFoundException(Exception):
     pass
 
 
 class JiraIntegration(IntegrationBase):
-    __mapper_args__ = {
-        'polymorphic_identity': 'jira'
-    }
+    __mapper_args__ = {"polymorphic_identity": "jira"}
     front_visible = True
     as_alert_channel = False
     supports_report_alerting = False
     action_notification = True
-    integration_action = 'Add issue to Jira'
+    integration_action = "Add issue to Jira"
 
 
 class JiraClient(object):
     def __init__(self, user_name, password, host_name, project, request=None):
         self.user_name = user_name
         self.password = password
         self.host_name = host_name
         self.project = project
         self.request = request
         try:
-            self.client = jira.client.JIRA(options={'server': host_name},
-                                           basic_auth=(user_name, password))
+            self.client = jira.client.JIRA(
+                options={"server": host_name}, basic_auth=(user_name, password)
+            )
         except jira.JIRAError as e:
             raise IntegrationException(
-                'Communication problem: HTTP_STATUS:%s, URL:%s ' % (
-                    e.status_code, e.url))
+                "Communication problem: HTTP_STATUS:%s, URL:%s "
+                % (e.status_code, e.url)
+            )
 
     def get_projects(self):
         projects = self.client.projects()
         return projects
 
     def get_assignees(self, request):
         """Gets list of possible assignees"""
         cache_region = request.registry.cache_regions.redis_sec_30
-        @cache_region.cache_on_arguments('JiraClient.get_assignees')
+
+        @cache_region.cache_on_arguments("JiraClient.get_assignees")
         def cached(project_name):
             users = self.client.search_assignable_users_for_issues(
-                None, project=project_name)
+                None, project=project_name
+            )
             results = []
             for user in users:
                 results.append({"id": user.name, "name": user.displayName})
             return results
+
         return cached(self.project)
 
     def get_issue_types(self, request):
         metadata = self.get_metadata(request)
         assignees = self.get_assignees(request)
         parsed_metadata = []
-        for entry in metadata['projects'][0]['issuetypes']:
-            issue = {"name": entry['name'],
-                     "id": entry['id'],
-                     "fields": []}
-            for i_id, field_i in entry['fields'].items():
+        for entry in metadata["projects"][0]["issuetypes"]:
+            issue = {"name": entry["name"], "id": entry["id"], "fields": []}
+            for i_id, field_i in entry["fields"].items():
                 field = {
-                    "name": field_i['name'],
+                    "name": field_i["name"],
                     "id": i_id,
-                    "required": field_i['required'],
+                    "required": field_i["required"],
                     "values": [],
-                    "type": field_i['schema'].get('type')
+                    "type": field_i["schema"].get("type"),
                 }
-                if field_i.get('allowedValues'):
-                    field['values'] = []
-                    for i in field_i['allowedValues']:
-                        field['values'].append(
-                            {'id': i['id'],
-                             'name': i.get('name', i.get('value', ''))
-                             })
-                if field['id'] == 'assignee':
-                    field['values'] = assignees
-                issue['fields'].append(field)
+                if field_i.get("allowedValues"):
+                    field["values"] = []
+                    for i in field_i["allowedValues"]:
+                        field["values"].append(
+                            {"id": i["id"], "name": i.get("name", i.get("value", ""))}
+                        )
+                if field["id"] == "assignee":
+                    field["values"] = assignees
+                issue["fields"].append(field)
             parsed_metadata.append(issue)
         return parsed_metadata
 
     def get_metadata(self, request):
         # cache_region = request.registry.cache_regions.redis_sec_30
         # @cache_region.cache_on_arguments('JiraClient.get_metadata')
         def cached(project_name):
             return self.client.createmeta(
-                projectKeys=project_name, expand='projects.issuetypes.fields')
+                projectKeys=project_name, expand="projects.issuetypes.fields"
+            )
+
         return cached(self.project)
 
     def create_issue(self, form_data, request):
         issue_types = self.get_issue_types(request)
         payload = {
-            'project': {'key': form_data['project']},
-            'summary': form_data['title'],
-            'description': form_data['content'],
-            'issuetype': {'id': form_data['issue_type']},
-            "priority": {'id': form_data['priority']},
-            "assignee": {'name': form_data['responsible']},
+            "project": {"key": form_data["project"]},
+            "summary": form_data["title"],
+            "description": form_data["content"],
+            "issuetype": {"id": form_data["issue_type"]},
+            "priority": {"id": form_data["priority"]},
+            "assignee": {"name": form_data["responsible"]},
         }
         for issue_type in issue_types:
-            if issue_type['id'] == form_data['issue_type']:
-                for field in issue_type['fields']:
+            if issue_type["id"] == form_data["issue_type"]:
+                for field in issue_type["fields"]:
                     # set some defaults for other required fields
-                    if field == 'reporter':
-                        payload["reporter"] = {'id': self.user_name}
-                    if field['required'] and field['id'] not in payload:
-                        if field['type'] == 'array':
-                            payload[field['id']] = [field['values'][0], ]
-                        elif field['type'] == 'string':
-                            payload[field['id']] = ''
+                    if field == "reporter":
+                        payload["reporter"] = {"id": self.user_name}
+                    if field["required"] and field["id"] not in payload:
+                        if field["type"] == "array":
+                            payload[field["id"]] = [field["values"][0]]
+                        elif field["type"] == "string":
+                            payload[field["id"]] = ""
         new_issue = self.client.create_issue(fields=payload)
-        web_url = self.host_name + '/browse/' + new_issue.key
+        web_url = self.host_name + "/browse/" + new_issue.key
         to_return = {
-            'id': new_issue.id,
-            'resource_url': new_issue.self,
-            'web_url': web_url
+            "id": new_issue.id,
+            "resource_url": new_issue.self,
+            "web_url": web_url,
         }
         return to_return

backend/src/appenlight/models/integrations/slack.py

@@ -1,74 +1,65 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 
 import requests
 
-from appenlight.models.integrations import (IntegrationBase,
-                                            IntegrationException)
+from appenlight.models.integrations import IntegrationBase, IntegrationException
 from appenlight.lib.ext_json import json
 
 _ = str
 
 log = logging.getLogger(__name__)
 
 
 class NotFoundException(Exception):
     pass
 
 
 class SlackIntegration(IntegrationBase):
-    __mapper_args__ = {
-        'polymorphic_identity': 'slack'
-    }
+    __mapper_args__ = {"polymorphic_identity": "slack"}
     front_visible = False
     as_alert_channel = True
     supports_report_alerting = True
     action_notification = True
-    integration_action = 'Message via Slack'
+    integration_action = "Message via Slack"
 
     @classmethod
     def create_client(cls, api_token):
         client = SlackClient(api_token)
         return client
 
 
 class SlackClient(object):
     def __init__(self, api_url):
         self.api_url = api_url
 
     def make_request(self, data=None):
-        headers = {
-            'User-Agent': 'appenlight-slack',
-            'Content-Type': 'application/json'
-        }
+        headers = {"User-Agent": "appenlight-slack", "Content-Type": "application/json"}
         try:
-            resp = getattr(requests, 'post')(self.api_url,
-                                             data=json.dumps(data),
-                                             headers=headers,
-                                             timeout=3)
+            resp = getattr(requests, "post")(
+                self.api_url, data=json.dumps(data), headers=headers, timeout=3
+            )
         except Exception as e:
-            raise IntegrationException(
-                _('Error communicating with Slack: %s') % (e,))
+            raise IntegrationException(_("Error communicating with Slack: %s") % (e,))
         if resp.status_code != requests.codes.ok:
-            msg = 'Error communicating with Slack - status code: %s'
+            msg = "Error communicating with Slack - status code: %s"
             raise IntegrationException(msg % resp.status_code)
         return resp
 
     def send(self, payload):
-        return self.make_request('/rooms/message', method='post',
-                                 data=payload).json()
+        return self.make_request("/rooms/message", method="post", data=payload).json()

backend/src/appenlight/models/integrations/webhooks.py

@@ -1,138 +1,145 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 
 import requests
 
-from appenlight.models.integrations import (IntegrationBase,
-                                            IntegrationException)
+from appenlight.models.integrations import IntegrationBase, IntegrationException
 from appenlight.models.alert_channel import AlertChannel
 from appenlight.lib.ext_json import json
 
 _ = str
 
 log = logging.getLogger(__name__)
 
 
 class NotFoundException(Exception):
     pass
 
 
 class WebhooksIntegration(IntegrationBase):
-    __mapper_args__ = {
-        'polymorphic_identity': 'webhooks'
-    }
+    __mapper_args__ = {"polymorphic_identity": "webhooks"}
     front_visible = False
     as_alert_channel = True
     supports_report_alerting = True
     action_notification = True
-    integration_action = 'Message via Webhooks'
+    integration_action = "Message via Webhooks"
 
     @classmethod
     def create_client(cls, url):
         client = WebhooksClient(url)
         return client
 
 
 class WebhooksClient(object):
     def __init__(self, url):
         self.api_url = url
 
-    def make_request(self, url, method='get', data=None):
+    def make_request(self, url, method="get", data=None):
         headers = {
-            'Content-Type': 'application/json',
-            'User-Agent': 'appenlight-webhooks',
+            "Content-Type": "application/json",
+            "User-Agent": "appenlight-webhooks",
         }
         try:
             if data:
                 data = json.dumps(data)
-            resp = getattr(requests, method)(url, data=data, headers=headers,
-                                             timeout=3)
+            resp = getattr(requests, method)(url, data=data, headers=headers, timeout=3)
         except Exception as e:
             raise IntegrationException(
-                _('Error communicating with Webhooks: {}').format(e))
+                _("Error communicating with Webhooks: {}").format(e)
+            )
         if resp.status_code > 299:
             raise IntegrationException(
-                'Error communicating with Webhooks - status code: {}'.format(
-                    resp.status_code))
+                "Error communicating with Webhooks - status code: {}".format(
+                    resp.status_code
+                )
+            )
         return resp
 
     def send_to_hook(self, payload):
-        return self.make_request(self.api_url, method='post',
-                                 data=payload).json()
+        return self.make_request(self.api_url, method="post", data=payload).json()
 
 
 class WebhooksAlertChannel(AlertChannel):
-    __mapper_args__ = {
-        'polymorphic_identity': 'webhooks'
-    }
+    __mapper_args__ = {"polymorphic_identity": "webhooks"}
 
     def notify_reports(self, **kwargs):
         """
         Notify user of individual reports
 
         kwargs:
             application: application that the event applies for,
             user: user that should be notified
             request: request object
             since_when: reports are newer than this time value,
             reports: list of reports to render
 
         """
         template_vars = self.get_notification_basic_vars(kwargs)
         payload = []
-        include_keys = ('id', 'http_status', 'report_type', 'resource_name',
-                        'front_url', 'resource_id', 'error', 'url_path',
-                        'tags', 'duration')
-
-        for occurences, report in kwargs['reports']:
-            r_dict = report.last_report_ref.get_dict(kwargs['request'],
-                                                     include_keys=include_keys)
-            r_dict['group']['occurences'] = occurences
+        include_keys = (
+            "id",
+            "http_status",
+            "report_type",
+            "resource_name",
+            "front_url",
+            "resource_id",
+            "error",
+            "url_path",
+            "tags",
+            "duration",
+        )
+
+        for occurences, report in kwargs["reports"]:
+            r_dict = report.last_report_ref.get_dict(
+                kwargs["request"], include_keys=include_keys
+            )
+            r_dict["group"]["occurences"] = occurences
             payload.append(r_dict)
         client = WebhooksIntegration.create_client(
-            self.integration.config['reports_webhook'])
+            self.integration.config["reports_webhook"]
+        )
         client.send_to_hook(payload)
 
     def notify_alert(self, **kwargs):
         """
         Notify user of report or uptime threshold events based on events alert type
 
         Kwargs:
             application: application that the event applies for,
             event: event that is notified,
             user: user that should be notified
             request: request object
 
         """
         payload = {
-            'alert_action': kwargs['event'].unified_alert_action(),
-            'alert_name': kwargs['event'].unified_alert_name(),
-            'event_time': kwargs['event'].end_date or kwargs[
-                'event'].start_date,
-            'resource_name': None,
-            'resource_id': None
+            "alert_action": kwargs["event"].unified_alert_action(),
+            "alert_name": kwargs["event"].unified_alert_name(),
+            "event_time": kwargs["event"].end_date or kwargs["event"].start_date,
+            "resource_name": None,
+            "resource_id": None,
         }
-        if kwargs['event'].values and kwargs['event'].values.get('reports'):
-            payload['reports'] = kwargs['event'].values.get('reports', [])
-        if 'application' in kwargs:
-            payload['resource_name'] = kwargs['application'].resource_name
-            payload['resource_id'] = kwargs['application'].resource_id
+        if kwargs["event"].values and kwargs["event"].values.get("reports"):
+            payload["reports"] = kwargs["event"].values.get("reports", [])
+        if "application" in kwargs:
+            payload["resource_name"] = kwargs["application"].resource_name
+            payload["resource_id"] = kwargs["application"].resource_id
 
         client = WebhooksIntegration.create_client(
-            self.integration.config['alerts_webhook'])
+            self.integration.config["alerts_webhook"]
+        )
         client.send_to_hook(payload)

backend/src/appenlight/models/log.py

@@ -1,130 +1,132 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import sqlalchemy as sa
 import logging
 import hashlib
 
 from datetime import datetime
 from appenlight.models import Base
 from appenlight.lib.utils import convert_es_type
 from appenlight.lib.enums import LogLevel
 from sqlalchemy.dialects.postgresql import JSON
 from ziggurat_foundations.models.base import BaseModel
 
 log = logging.getLogger(__name__)
 
 
 class Log(Base, BaseModel):
-    __tablename__ = 'logs'
-    __table_args__ = {'implicit_returning': False}
+    __tablename__ = "logs"
+    __table_args__ = {"implicit_returning": False}
 
     log_id = sa.Column(sa.BigInteger(), nullable=False, primary_key=True)
-    resource_id = sa.Column(sa.Integer(),
-                            sa.ForeignKey('applications.resource_id',
-                                          onupdate='CASCADE',
-                                          ondelete='CASCADE'),
-                            nullable=False,
-                            index=True)
-    log_level = sa.Column(sa.Unicode, nullable=False, index=True,
-                          default='INFO')
-    message = sa.Column(sa.UnicodeText(), default='')
-    timestamp = sa.Column(sa.DateTime(), default=datetime.utcnow,
-                          server_default=sa.func.now())
+    resource_id = sa.Column(
+        sa.Integer(),
+        sa.ForeignKey(
+            "applications.resource_id", onupdate="CASCADE", ondelete="CASCADE"
+        ),
+        nullable=False,
+        index=True,
+    )
+    log_level = sa.Column(sa.Unicode, nullable=False, index=True, default="INFO")
+    message = sa.Column(sa.UnicodeText(), default="")
+    timestamp = sa.Column(
+        sa.DateTime(), default=datetime.utcnow, server_default=sa.func.now()
+    )
     request_id = sa.Column(sa.Unicode())
     namespace = sa.Column(sa.Unicode())
     primary_key = sa.Column(sa.Unicode())
 
     tags = sa.Column(JSON(), default={})
     permanent = sa.Column(sa.Boolean(), nullable=False, default=False)
 
     def __str__(self):
-        return self.__unicode__().encode('utf8')
+        return self.__unicode__().encode("utf8")
 
     def __unicode__(self):
-        return '<Log id:%s, lv:%s, ns:%s >' % (
-            self.log_id, self.log_level, self.namespace)
+        return "<Log id:%s, lv:%s, ns:%s >" % (
+            self.log_id,
+            self.log_level,
+            self.namespace,
+        )
 
     def set_data(self, data, resource):
-        level = data.get('log_level').upper()
+        level = data.get("log_level").upper()
         self.log_level = getattr(LogLevel, level, LogLevel.UNKNOWN)
-        self.message = data.get('message', '')
-        server_name = data.get('server', '').lower() or 'unknown'
-        self.tags = {
-            'server_name': server_name
-        }
-        if data.get('tags'):
-            for tag_tuple in data['tags']:
+        self.message = data.get("message", "")
+        server_name = data.get("server", "").lower() or "unknown"
+        self.tags = {"server_name": server_name}
+        if data.get("tags"):
+            for tag_tuple in data["tags"]:
                 self.tags[tag_tuple[0]] = tag_tuple[1]
-        self.timestamp = data['date']
-        r_id = data.get('request_id', '')
+        self.timestamp = data["date"]
+        r_id = data.get("request_id", "")
         if not r_id:
-            r_id = ''
-        self.request_id = r_id.replace('-', '')
+            r_id = ""
+        self.request_id = r_id.replace("-", "")
         self.resource_id = resource.resource_id
-        self.namespace = data.get('namespace') or ''
-        self.permanent = data.get('permanent')
-        self.primary_key = data.get('primary_key')
+        self.namespace = data.get("namespace") or ""
+        self.permanent = data.get("permanent")
+        self.primary_key = data.get("primary_key")
         if self.primary_key is not None:
-            self.tags['appenlight_primary_key'] = self.primary_key
+            self.tags["appenlight_primary_key"] = self.primary_key
 
     def get_dict(self):
         instance_dict = super(Log, self).get_dict()
-        instance_dict['log_level'] = LogLevel.key_from_value(self.log_level)
-        instance_dict['resource_name'] = self.application.resource_name
+        instance_dict["log_level"] = LogLevel.key_from_value(self.log_level)
+        instance_dict["resource_name"] = self.application.resource_name
         return instance_dict
 
     @property
     def delete_hash(self):
         if not self.primary_key:
             return None
 
-        to_hash = '{}_{}_{}'.format(self.resource_id, self.primary_key,
-                                    self.namespace)
-        return hashlib.sha1(to_hash.encode('utf8')).hexdigest()
+        to_hash = "{}_{}_{}".format(self.resource_id, self.primary_key, self.namespace)
+        return hashlib.sha1(to_hash.encode("utf8")).hexdigest()
 
     def es_doc(self):
         tags = {}
         tag_list = []
         for name, value in self.tags.items():
             # replace dot in indexed tag name
-            name = name.replace('.', '_')
+            name = name.replace(".", "_")
             tag_list.append(name)
             tags[name] = {
                 "values": convert_es_type(value),
-                "numeric_values": value if (
-                    isinstance(value, (int, float)) and
-                    not isinstance(value, bool)) else None
+                "numeric_values": value
+                if (isinstance(value, (int, float)) and not isinstance(value, bool))
+                else None,
             }
         return {
-            'pg_id': str(self.log_id),
-            'delete_hash': self.delete_hash,
-            'resource_id': self.resource_id,
-            'request_id': self.request_id,
-            'log_level': LogLevel.key_from_value(self.log_level),
-            'timestamp': self.timestamp,
-            'message': self.message if self.message else '',
-            'namespace': self.namespace if self.namespace else '',
-            'tags': tags,
-            'tag_list': tag_list
+            "pg_id": str(self.log_id),
+            "delete_hash": self.delete_hash,
+            "resource_id": self.resource_id,
+            "request_id": self.request_id,
+            "log_level": LogLevel.key_from_value(self.log_level),
+            "timestamp": self.timestamp,
+            "message": self.message if self.message else "",
+            "namespace": self.namespace if self.namespace else "",
+            "tags": tags,
+            "tag_list": tag_list,
         }
 
     @property
     def partition_id(self):
         if self.permanent:
-            return 'rcae_l_%s' % self.timestamp.strftime('%Y_%m')
+            return "rcae_l_%s" % self.timestamp.strftime("%Y_%m")
         else:
-            return 'rcae_l_%s' % self.timestamp.strftime('%Y_%m_%d')
+            return "rcae_l_%s" % self.timestamp.strftime("%Y_%m_%d")

backend/src/appenlight/models/metric.py

@@ -1,64 +1,68 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from datetime import datetime
 
 import sqlalchemy as sa
 from sqlalchemy.dialects.postgresql import JSON
 
 from ziggurat_foundations.models.base import BaseModel
 from appenlight.lib.utils import convert_es_type
 from appenlight.models import Base
 
 
 class Metric(Base, BaseModel):
-    __tablename__ = 'metrics'
-    __table_args__ = {'implicit_returning': False}
+    __tablename__ = "metrics"
+    __table_args__ = {"implicit_returning": False}
 
     pkey = sa.Column(sa.BigInteger(), primary_key=True)
-    resource_id = sa.Column(sa.Integer(),
-                            sa.ForeignKey('applications.resource_id'),
-                            nullable=False, primary_key=True)
-    timestamp = sa.Column(sa.DateTime(), default=datetime.utcnow,
-                          server_default=sa.func.now())
+    resource_id = sa.Column(
+        sa.Integer(),
+        sa.ForeignKey("applications.resource_id"),
+        nullable=False,
+        primary_key=True,
+    )
+    timestamp = sa.Column(
+        sa.DateTime(), default=datetime.utcnow, server_default=sa.func.now()
+    )
     tags = sa.Column(JSON(), default={})
     namespace = sa.Column(sa.Unicode(255))
 
     @property
     def partition_id(self):
-        return 'rcae_m_%s' % self.timestamp.strftime('%Y_%m_%d')
+        return "rcae_m_%s" % self.timestamp.strftime("%Y_%m_%d")
 
     def es_doc(self):
         tags = {}
         tag_list = []
         for name, value in self.tags.items():
             # replace dot in indexed tag name
-            name = name.replace('.', '_')
+            name = name.replace(".", "_")
             tag_list.append(name)
             tags[name] = {
                 "values": convert_es_type(value),
-                "numeric_values": value if (
-                    isinstance(value, (int, float)) and
-                    not isinstance(value, bool)) else None
+                "numeric_values": value
+                if (isinstance(value, (int, float)) and not isinstance(value, bool))
+                else None,
             }
 
         return {
-            'resource_id': self.resource_id,
-            'timestamp': self.timestamp,
-            'namespace': self.namespace,
-            'tags': tags,
-            'tag_list': tag_list
+            "resource_id": self.resource_id,
+            "timestamp": self.timestamp,
+            "namespace": self.namespace,
+            "tags": tags,
+            "tag_list": tag_list,
         }

backend/src/appenlight/models/plugin_config.py

@@ -1,40 +1,40 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import sqlalchemy as sa
 from ziggurat_foundations.models.base import BaseModel
 from sqlalchemy.dialects.postgresql import JSON
 
 from . import Base
 
 
 class PluginConfig(Base, BaseModel):
-    __tablename__ = 'plugin_configs'
+    __tablename__ = "plugin_configs"
 
     id = sa.Column(sa.Integer, primary_key=True)
     plugin_name = sa.Column(sa.Unicode)
     section = sa.Column(sa.Unicode)
     config = sa.Column(JSON, nullable=False)
-    resource_id = sa.Column(sa.Integer(),
-                            sa.ForeignKey('resources.resource_id',
-                                          onupdate='cascade',
-                                          ondelete='cascade'))
-    owner_id = sa.Column(sa.Integer(),
-                         sa.ForeignKey('users.id', onupdate='cascade',
-                                       ondelete='cascade'))
+    resource_id = sa.Column(
+        sa.Integer(),
+        sa.ForeignKey("resources.resource_id", onupdate="cascade", ondelete="cascade"),
+    )
+    owner_id = sa.Column(
+        sa.Integer(), sa.ForeignKey("users.id", onupdate="cascade", ondelete="cascade")
+    )
 
     def __json__(self, request):
         return self.get_dict()

backend/src/appenlight/models/report.py

@@ -1,514 +1,533 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from datetime import datetime, timedelta
 import math
 import uuid
 import hashlib
 import copy
 import urllib.parse
 import logging
 import sqlalchemy as sa
 
 from appenlight.models import Base, Datastores
 from appenlight.lib.utils.date_utils import convert_date
 from appenlight.lib.utils import convert_es_type
 from appenlight.models.slow_call import SlowCall
 from appenlight.lib.utils import channelstream_request
 from appenlight.lib.enums import ReportType, Language
 from pyramid.threadlocal import get_current_registry, get_current_request
 from sqlalchemy.dialects.postgresql import JSON
 from ziggurat_foundations.models.base import BaseModel
 
 log = logging.getLogger(__name__)
 
 REPORT_TYPE_MATRIX = {
-    'http_status': {"type": 'int',
-                    "ops": ('eq', 'ne', 'ge', 'le',)},
-    'group:priority': {"type": 'int',
-                       "ops": ('eq', 'ne', 'ge', 'le',)},
-    'duration': {"type": 'float',
-                 "ops": ('ge', 'le',)},
-    'url_domain': {"type": 'unicode',
-                   "ops": ('eq', 'ne', 'startswith', 'endswith', 'contains',)},
-    'url_path': {"type": 'unicode',
-                 "ops": ('eq', 'ne', 'startswith', 'endswith', 'contains',)},
-    'error': {"type": 'unicode',
-              "ops": ('eq', 'ne', 'startswith', 'endswith', 'contains',)},
-    'tags:server_name': {"type": 'unicode',
-                         "ops": ('eq', 'ne', 'startswith', 'endswith',
-                                 'contains',)},
-    'traceback': {"type": 'unicode',
-                  "ops": ('contains',)},
-    'group:occurences': {"type": 'int',
-                         "ops": ('eq', 'ne', 'ge', 'le',)}
+    "http_status": {"type": "int", "ops": ("eq", "ne", "ge", "le")},
+    "group:priority": {"type": "int", "ops": ("eq", "ne", "ge", "le")},
+    "duration": {"type": "float", "ops": ("ge", "le")},
+    "url_domain": {
+        "type": "unicode",
+        "ops": ("eq", "ne", "startswith", "endswith", "contains"),
+    },
+    "url_path": {
+        "type": "unicode",
+        "ops": ("eq", "ne", "startswith", "endswith", "contains"),
+    },
+    "error": {
+        "type": "unicode",
+        "ops": ("eq", "ne", "startswith", "endswith", "contains"),
+    },
+    "tags:server_name": {
+        "type": "unicode",
+        "ops": ("eq", "ne", "startswith", "endswith", "contains"),
+    },
+    "traceback": {"type": "unicode", "ops": ("contains",)},
+    "group:occurences": {"type": "int", "ops": ("eq", "ne", "ge", "le")},
 }
 
 
 class Report(Base, BaseModel):
-    __tablename__ = 'reports'
-    __table_args__ = {'implicit_returning': False}
+    __tablename__ = "reports"
+    __table_args__ = {"implicit_returning": False}
 
     id = sa.Column(sa.Integer, nullable=False, primary_key=True)
-    group_id = sa.Column(sa.BigInteger,
-                         sa.ForeignKey('reports_groups.id', ondelete='cascade',
-                                       onupdate='cascade'))
+    group_id = sa.Column(
+        sa.BigInteger,
+        sa.ForeignKey("reports_groups.id", ondelete="cascade", onupdate="cascade"),
+    )
     resource_id = sa.Column(sa.Integer(), nullable=False, index=True)
     report_type = sa.Column(sa.Integer(), nullable=False, index=True)
     error = sa.Column(sa.UnicodeText(), index=True)
     extra = sa.Column(JSON(), default={})
     request = sa.Column(JSON(), nullable=False, default={})
-    ip = sa.Column(sa.String(39), index=True, default='')
-    username = sa.Column(sa.Unicode(255), default='')
-    user_agent = sa.Column(sa.Unicode(255), default='')
+    ip = sa.Column(sa.String(39), index=True, default="")
+    username = sa.Column(sa.Unicode(255), default="")
+    user_agent = sa.Column(sa.Unicode(255), default="")
     url = sa.Column(sa.UnicodeText(), index=True)
     request_id = sa.Column(sa.Text())
     request_stats = sa.Column(JSON(), nullable=False, default={})
     traceback = sa.Column(JSON(), nullable=False, default=None)
     traceback_hash = sa.Column(sa.Text())
-    start_time = sa.Column(sa.DateTime(), default=datetime.utcnow,
-                           server_default=sa.func.now())
+    start_time = sa.Column(
+        sa.DateTime(), default=datetime.utcnow, server_default=sa.func.now()
+    )
     end_time = sa.Column(sa.DateTime())
     duration = sa.Column(sa.Float, default=0)
     http_status = sa.Column(sa.Integer, index=True)
     url_domain = sa.Column(sa.Unicode(100), index=True)
     url_path = sa.Column(sa.Unicode(255), index=True)
     tags = sa.Column(JSON(), nullable=False, default={})
     language = sa.Column(sa.Integer(), default=0)
     # this is used to determine partition for the report
-    report_group_time = sa.Column(sa.DateTime(), default=datetime.utcnow,
-                                  server_default=sa.func.now())
+    report_group_time = sa.Column(
+        sa.DateTime(), default=datetime.utcnow, server_default=sa.func.now()
+    )
 
     logs = sa.orm.relationship(
-        'Log',
-        lazy='dynamic',
+        "Log",
+        lazy="dynamic",
         passive_deletes=True,
         passive_updates=True,
         primaryjoin="and_(Report.request_id==Log.request_id, "
-                    "Log.request_id != None, Log.request_id != '')",
-        foreign_keys='[Log.request_id]')
-
-    slow_calls = sa.orm.relationship('SlowCall',
-                                     backref='detail',
-                                     cascade="all, delete-orphan",
-                                     passive_deletes=True,
-                                     passive_updates=True,
-                                     order_by='SlowCall.timestamp')
+        "Log.request_id != None, Log.request_id != '')",
+        foreign_keys="[Log.request_id]",
+    )
+
+    slow_calls = sa.orm.relationship(
+        "SlowCall",
+        backref="detail",
+        cascade="all, delete-orphan",
+        passive_deletes=True,
+        passive_updates=True,
+        order_by="SlowCall.timestamp",
+    )
 
     def set_data(self, data, resource, protocol_version=None):
-        self.http_status = data['http_status']
-        self.priority = data['priority']
-        self.error = data['error']
-        report_language = data.get('language', '').lower()
+        self.http_status = data["http_status"]
+        self.priority = data["priority"]
+        self.error = data["error"]
+        report_language = data.get("language", "").lower()
         self.language = getattr(Language, report_language, Language.unknown)
         # we need temp holder here to decide later
         # if we want to to commit the tags if report is marked for creation
-        self.tags = {
-            'server_name': data['server'],
-            'view_name': data['view_name']
-        }
-        if data.get('tags'):
-            for tag_tuple in data['tags']:
+        self.tags = {"server_name": data["server"], "view_name": data["view_name"]}
+        if data.get("tags"):
+            for tag_tuple in data["tags"]:
                 self.tags[tag_tuple[0]] = tag_tuple[1]
-        self.traceback = data['traceback']
+        self.traceback = data["traceback"]
         stripped_traceback = self.stripped_traceback()
-        tb_repr = repr(stripped_traceback).encode('utf8')
+        tb_repr = repr(stripped_traceback).encode("utf8")
         self.traceback_hash = hashlib.sha1(tb_repr).hexdigest()
-        url_info = urllib.parse.urlsplit(
-            data.get('url', ''), allow_fragments=False)
+        url_info = urllib.parse.urlsplit(data.get("url", ""), allow_fragments=False)
         self.url_domain = url_info.netloc[:128]
         self.url_path = url_info.path[:2048]
-        self.occurences = data['occurences']
+        self.occurences = data["occurences"]
         if self.error:
             self.report_type = ReportType.error
         else:
             self.report_type = ReportType.slow
 
         # but if its status 404 its 404 type
-        if self.http_status in [404, '404'] or self.error == '404 Not Found':
+        if self.http_status in [404, "404"] or self.error == "404 Not Found":
             self.report_type = ReportType.not_found
-            self.error = ''
+            self.error = ""
 
-        self.generate_grouping_hash(data.get('appenlight.group_string',
-                                             data.get('group_string')),
-                                    resource.default_grouping,
-                                    protocol_version)
+        self.generate_grouping_hash(
+            data.get("appenlight.group_string", data.get("group_string")),
+            resource.default_grouping,
+            protocol_version,
+        )
 
         # details
-        if data['http_status'] in [404, '404']:
-            data = {"username": data["username"],
-                    "ip": data["ip"],
-                    "url": data["url"],
-                    "user_agent": data["user_agent"]}
-            if data.get('HTTP_REFERER') or data.get('http_referer'):
-                data['HTTP_REFERER'] = data.get(
-                    'HTTP_REFERER', '') or data.get('http_referer', '')
+        if data["http_status"] in [404, "404"]:
+            data = {
+                "username": data["username"],
+                "ip": data["ip"],
+                "url": data["url"],
+                "user_agent": data["user_agent"],
+            }
+            if data.get("HTTP_REFERER") or data.get("http_referer"):
+                data["HTTP_REFERER"] = data.get("HTTP_REFERER", "") or data.get(
+                    "http_referer", ""
+                )
 
         self.resource_id = resource.resource_id
-        self.username = data['username']
-        self.user_agent = data['user_agent']
-        self.ip = data['ip']
+        self.username = data["username"]
+        self.user_agent = data["user_agent"]
+        self.ip = data["ip"]
         self.extra = {}
-        if data.get('extra'):
-            for extra_tuple in data['extra']:
+        if data.get("extra"):
+            for extra_tuple in data["extra"]:
                 self.extra[extra_tuple[0]] = extra_tuple[1]
 
-        self.url = data['url']
-        self.request_id = data.get('request_id', '').replace('-', '') or str(
-            uuid.uuid4())
-        request_data = data.get('request', {})
+        self.url = data["url"]
+        self.request_id = data.get("request_id", "").replace("-", "") or str(
+            uuid.uuid4()
+        )
+        request_data = data.get("request", {})
 
         self.request = request_data
-        self.request_stats = data.get('request_stats', {})
-        traceback = data.get('traceback')
+        self.request_stats = data.get("request_stats", {})
+        traceback = data.get("traceback")
         if not traceback:
-            traceback = data.get('frameinfo')
+            traceback = data.get("frameinfo")
         self.traceback = traceback
-        start_date = convert_date(data.get('start_time'))
+        start_date = convert_date(data.get("start_time"))
         if not self.start_time or self.start_time < start_date:
             self.start_time = start_date
 
-        self.end_time = convert_date(data.get('end_time'), False)
+        self.end_time = convert_date(data.get("end_time"), False)
         self.duration = 0
 
         if self.start_time and self.end_time:
             d = self.end_time - self.start_time
             self.duration = d.total_seconds()
 
         # update tags with other vars
         if self.username:
-            self.tags['user_name'] = self.username
-        self.tags['report_language'] = Language.key_from_value(self.language)
+            self.tags["user_name"] = self.username
+        self.tags["report_language"] = Language.key_from_value(self.language)
 
     def add_slow_calls(self, data, report_group):
         slow_calls = []
-        for call in data.get('slow_calls', []):
+        for call in data.get("slow_calls", []):
             sc_inst = SlowCall()
-            sc_inst.set_data(call, resource_id=self.resource_id,
-                             report_group=report_group)
+            sc_inst.set_data(
+                call, resource_id=self.resource_id, report_group=report_group
+            )
             slow_calls.append(sc_inst)
             self.slow_calls.extend(slow_calls)
         return slow_calls
 
-    def get_dict(self, request, details=False, exclude_keys=None,
-                 include_keys=None):
+    def get_dict(self, request, details=False, exclude_keys=None, include_keys=None):
         from appenlight.models.services.report_group import ReportGroupService
+
         instance_dict = super(Report, self).get_dict()
-        instance_dict['req_stats'] = self.req_stats()
-        instance_dict['group'] = {}
-        instance_dict['group']['id'] = self.report_group.id
-        instance_dict['group'][
-            'total_reports'] = self.report_group.total_reports
-        instance_dict['group']['last_report'] = self.report_group.last_report
-        instance_dict['group']['priority'] = self.report_group.priority
-        instance_dict['group']['occurences'] = self.report_group.occurences
-        instance_dict['group'][
-            'last_timestamp'] = self.report_group.last_timestamp
-        instance_dict['group'][
-            'first_timestamp'] = self.report_group.first_timestamp
-        instance_dict['group']['public'] = self.report_group.public
-        instance_dict['group']['fixed'] = self.report_group.fixed
-        instance_dict['group']['read'] = self.report_group.read
-        instance_dict['group'][
-            'average_duration'] = self.report_group.average_duration
-
-        instance_dict[
-            'resource_name'] = self.report_group.application.resource_name
-        instance_dict['report_type'] = self.report_type
-
-        if instance_dict['http_status'] == 404 and not instance_dict['error']:
-            instance_dict['error'] = '404 Not Found'
+        instance_dict["req_stats"] = self.req_stats()
+        instance_dict["group"] = {}
+        instance_dict["group"]["id"] = self.report_group.id
+        instance_dict["group"]["total_reports"] = self.report_group.total_reports
+        instance_dict["group"]["last_report"] = self.report_group.last_report
+        instance_dict["group"]["priority"] = self.report_group.priority
+        instance_dict["group"]["occurences"] = self.report_group.occurences
+        instance_dict["group"]["last_timestamp"] = self.report_group.last_timestamp
+        instance_dict["group"]["first_timestamp"] = self.report_group.first_timestamp
+        instance_dict["group"]["public"] = self.report_group.public
+        instance_dict["group"]["fixed"] = self.report_group.fixed
+        instance_dict["group"]["read"] = self.report_group.read
+        instance_dict["group"]["average_duration"] = self.report_group.average_duration
+
+        instance_dict["resource_name"] = self.report_group.application.resource_name
+        instance_dict["report_type"] = self.report_type
+
+        if instance_dict["http_status"] == 404 and not instance_dict["error"]:
+            instance_dict["error"] = "404 Not Found"
 
         if details:
-            instance_dict['affected_users_count'] = \
-                ReportGroupService.affected_users_count(self.report_group)
-            instance_dict['top_affected_users'] = [
-                {'username': u.username, 'count': u.count} for u in
-                ReportGroupService.top_affected_users(self.report_group)]
-            instance_dict['application'] = {'integrations': []}
+            instance_dict[
+                "affected_users_count"
+            ] = ReportGroupService.affected_users_count(self.report_group)
+            instance_dict["top_affected_users"] = [
+                {"username": u.username, "count": u.count}
+                for u in ReportGroupService.top_affected_users(self.report_group)
+            ]
+            instance_dict["application"] = {"integrations": []}
             for integration in self.report_group.application.integrations:
                 if integration.front_visible:
-                    instance_dict['application']['integrations'].append(
-                        {'name': integration.integration_name,
-                         'action': integration.integration_action})
-            instance_dict['comments'] = [c.get_dict() for c in
-                                         self.report_group.comments]
-
-            instance_dict['group']['next_report'] = None
-            instance_dict['group']['previous_report'] = None
+                    instance_dict["application"]["integrations"].append(
+                        {
+                            "name": integration.integration_name,
+                            "action": integration.integration_action,
+                        }
+                    )
+            instance_dict["comments"] = [
+                c.get_dict() for c in self.report_group.comments
+            ]
+
+            instance_dict["group"]["next_report"] = None
+            instance_dict["group"]["previous_report"] = None
             next_in_group = self.get_next_in_group(request)
             previous_in_group = self.get_previous_in_group(request)
             if next_in_group:
-                instance_dict['group']['next_report'] = next_in_group
+                instance_dict["group"]["next_report"] = next_in_group
             if previous_in_group:
-                instance_dict['group']['previous_report'] = previous_in_group
+                instance_dict["group"]["previous_report"] = previous_in_group
 
             # slow call ordering
             def find_parent(row, data):
                 for r in reversed(data):
                     try:
-                        if (row['timestamp'] > r['timestamp'] and
-                                    row['end_time'] < r['end_time']):
+                        if (
+                            row["timestamp"] > r["timestamp"]
+                            and row["end_time"] < r["end_time"]
+                        ):
                             return r
                     except TypeError as e:
-                        log.warning('reports_view.find_parent: %s' % e)
+                        log.warning("reports_view.find_parent: %s" % e)
                 return None
 
             new_calls = []
             calls = [c.get_dict() for c in self.slow_calls]
             while calls:
                 # start from end
                 for x in range(len(calls) - 1, -1, -1):
                     parent = find_parent(calls[x], calls)
                     if parent:
-                        parent['children'].append(calls[x])
+                        parent["children"].append(calls[x])
                     else:
                         # no parent at all? append to new calls anyways
                         new_calls.append(calls[x])
                         # print 'append', calls[x]
                     del calls[x]
                     break
-            instance_dict['slow_calls'] = new_calls
+            instance_dict["slow_calls"] = new_calls
 
-        instance_dict['front_url'] = self.get_public_url(request)
+        instance_dict["front_url"] = self.get_public_url(request)
 
         exclude_keys_list = exclude_keys or []
         include_keys_list = include_keys or []
         for k in list(instance_dict.keys()):
-            if k == 'group':
+            if k == "group":
                 continue
-            if (k in exclude_keys_list or
-                    (k not in include_keys_list and include_keys)):
+            if k in exclude_keys_list or (k not in include_keys_list and include_keys):
                 del instance_dict[k]
         return instance_dict
 
     def get_previous_in_group(self, request):
         query = {
             "size": 1,
             "query": {
                 "filtered": {
                     "filter": {
-                        "and": [{"term": {"group_id": self.group_id}},
-                                {"range": {"pg_id": {"lt": self.id}}}]
+                        "and": [
+                            {"term": {"group_id": self.group_id}},
+                            {"range": {"pg_id": {"lt": self.id}}},
+                        ]
                     }
                 }
             },
-            "sort": [
-                {"_doc": {"order": "desc"}},
-            ],
+            "sort": [{"_doc": {"order": "desc"}}],
         }
-        result = request.es_conn.search(body=query, index=self.partition_id,
-                                        doc_type='report')
-        if result['hits']['total']:
-            return result['hits']['hits'][0]['_source']['pg_id']
+        result = request.es_conn.search(
+            body=query, index=self.partition_id, doc_type="report"
+        )
+        if result["hits"]["total"]:
+            return result["hits"]["hits"][0]["_source"]["pg_id"]
 
     def get_next_in_group(self, request):
         query = {
             "size": 1,
             "query": {
                 "filtered": {
                     "filter": {
-                        "and": [{"term": {"group_id": self.group_id}},
-                                {"range": {"pg_id": {"gt": self.id}}}]
+                        "and": [
+                            {"term": {"group_id": self.group_id}},
+                            {"range": {"pg_id": {"gt": self.id}}},
+                        ]
                     }
                 }
             },
-            "sort": [
-                {"_doc": {"order": "asc"}},
-            ],
+            "sort": [{"_doc": {"order": "asc"}}],
         }
-        result = request.es_conn.search(body=query, index=self.partition_id,
-                                        doc_type='report')
-        if result['hits']['total']:
-            return result['hits']['hits'][0]['_source']['pg_id']
+        result = request.es_conn.search(
+            body=query, index=self.partition_id, doc_type="report"
+        )
+        if result["hits"]["total"]:
+            return result["hits"]["hits"][0]["_source"]["pg_id"]
 
     def get_public_url(self, request=None, report_group=None, _app_url=None):
         """
         Returns url that user can use to visit specific report
         """
         if not request:
             request = get_current_request()
-        url = request.route_url('/', _app_url=_app_url)
+        url = request.route_url("/", _app_url=_app_url)
         if report_group:
-            return (url + 'ui/report/%s/%s') % (report_group.id, self.id)
-        return (url + 'ui/report/%s/%s') % (self.group_id, self.id)
+            return (url + "ui/report/%s/%s") % (report_group.id, self.id)
+        return (url + "ui/report/%s/%s") % (self.group_id, self.id)
 
     def req_stats(self):
         stats = self.request_stats.copy()
-        stats['percentages'] = {}
-        stats['percentages']['main'] = 100.0
-        main = stats.get('main', 0.0)
+        stats["percentages"] = {}
+        stats["percentages"]["main"] = 100.0
+        main = stats.get("main", 0.0)
         if not main:
             return None
         for name, call_time in stats.items():
-            if ('calls' not in name and 'main' not in name and
-                        'percentages' not in name):
-                stats['main'] -= call_time
-                stats['percentages'][name] = math.floor(
-                    (call_time / main * 100.0))
-                stats['percentages']['main'] -= stats['percentages'][name]
-        if stats['percentages']['main'] < 0.0:
-            stats['percentages']['main'] = 0.0
-            stats['main'] = 0.0
+            if "calls" not in name and "main" not in name and "percentages" not in name:
+                stats["main"] -= call_time
+                stats["percentages"][name] = math.floor((call_time / main * 100.0))
+                stats["percentages"]["main"] -= stats["percentages"][name]
+        if stats["percentages"]["main"] < 0.0:
+            stats["percentages"]["main"] = 0.0
+            stats["main"] = 0.0
         return stats
 
-    def generate_grouping_hash(self, hash_string=None, default_grouping=None,
-                               protocol_version=None):
+    def generate_grouping_hash(
+        self, hash_string=None, default_grouping=None, protocol_version=None
+    ):
         """
         Generates SHA1 hash that will be used to group reports together
         """
         if not hash_string:
-            location = self.tags.get('view_name') or self.url_path;
-            server_name = self.tags.get('server_name') or ''
-            if default_grouping == 'url_traceback':
-                hash_string = '%s_%s_%s' % (self.traceback_hash, location,
-                                            self.error)
+            location = self.tags.get("view_name") or self.url_path
+            server_name = self.tags.get("server_name") or ""
+            if default_grouping == "url_traceback":
+                hash_string = "%s_%s_%s" % (self.traceback_hash, location, self.error)
                 if self.language == Language.javascript:
-                    hash_string = '%s_%s' % (self.traceback_hash, self.error)
+                    hash_string = "%s_%s" % (self.traceback_hash, self.error)
 
-            elif default_grouping == 'traceback_server':
-                hash_string = '%s_%s' % (self.traceback_hash, server_name)
+            elif default_grouping == "traceback_server":
+                hash_string = "%s_%s" % (self.traceback_hash, server_name)
                 if self.language == Language.javascript:
-                    hash_string = '%s_%s' % (self.traceback_hash, server_name)
+                    hash_string = "%s_%s" % (self.traceback_hash, server_name)
             else:
-                hash_string = '%s_%s' % (self.error, location)
+                hash_string = "%s_%s" % (self.error, location)
         month = datetime.utcnow().date().replace(day=1)
-        hash_string = '{}_{}'.format(month, hash_string)
-        binary_string = hash_string.encode('utf8')
+        hash_string = "{}_{}".format(month, hash_string)
+        binary_string = hash_string.encode("utf8")
         self.grouping_hash = hashlib.sha1(binary_string).hexdigest()
         return self.grouping_hash
 
     def stripped_traceback(self):
         """
         Traceback without local vars
         """
         stripped_traceback = copy.deepcopy(self.traceback)
 
         if isinstance(stripped_traceback, list):
             for row in stripped_traceback:
-                row.pop('vars', None)
+                row.pop("vars", None)
         return stripped_traceback
 
     def notify_channel(self, report_group):
         """
         Sends notification to websocket channel
         """
         settings = get_current_registry().settings
-        log.info('notify channelstream')
+        log.info("notify channelstream")
         if self.report_type != ReportType.error:
             return
         payload = {
-            'type': 'message',
-            "user": '__system__',
-            "channel": 'app_%s' % self.resource_id,
-            'message': {
-                'topic': 'front_dashboard.new_topic',
-                'report': {
-                    'group': {
-                        'priority': report_group.priority,
-                        'first_timestamp': report_group.first_timestamp,
-                        'last_timestamp': report_group.last_timestamp,
-                        'average_duration': report_group.average_duration,
-                        'occurences': report_group.occurences
+            "type": "message",
+            "user": "__system__",
+            "channel": "app_%s" % self.resource_id,
+            "message": {
+                "topic": "front_dashboard.new_topic",
+                "report": {
+                    "group": {
+                        "priority": report_group.priority,
+                        "first_timestamp": report_group.first_timestamp,
+                        "last_timestamp": report_group.last_timestamp,
+                        "average_duration": report_group.average_duration,
+                        "occurences": report_group.occurences,
                     },
-                    'report_id': self.id,
-                    'group_id': self.group_id,
-                    'resource_id': self.resource_id,
-                    'http_status': self.http_status,
-                    'url_domain': self.url_domain,
-                    'url_path': self.url_path,
-                    'error': self.error or '',
-                    'server': self.tags.get('server_name'),
-                    'view_name': self.tags.get('view_name'),
-                    'front_url': self.get_public_url(),
-                }
-            }
-
+                    "report_id": self.id,
+                    "group_id": self.group_id,
+                    "resource_id": self.resource_id,
+                    "http_status": self.http_status,
+                    "url_domain": self.url_domain,
+                    "url_path": self.url_path,
+                    "error": self.error or "",
+                    "server": self.tags.get("server_name"),
+                    "view_name": self.tags.get("view_name"),
+                    "front_url": self.get_public_url(),
+                },
+            },
         }
-        channelstream_request(settings['cometd.secret'], '/message', [payload],
-                              servers=[settings['cometd_servers']])
+        channelstream_request(
+            settings["cometd.secret"],
+            "/message",
+            [payload],
+            servers=[settings["cometd_servers"]],
+        )
 
     def es_doc(self):
         tags = {}
         tag_list = []
         for name, value in self.tags.items():
-            name = name.replace('.', '_')
+            name = name.replace(".", "_")
             tag_list.append(name)
             tags[name] = {
                 "values": convert_es_type(value),
-                "numeric_values": value if (
-                    isinstance(value, (int, float)) and
-                    not isinstance(value, bool)) else None}
+                "numeric_values": value
+                if (isinstance(value, (int, float)) and not isinstance(value, bool))
+                else None,
+            }
 
-        if 'user_name' not in self.tags and self.username:
-            tags["user_name"] = {"value": [self.username],
-                                 "numeric_value": None}
+        if "user_name" not in self.tags and self.username:
+            tags["user_name"] = {"value": [self.username], "numeric_value": None}
         return {
-            '_id': str(self.id),
-            'pg_id': str(self.id),
-            'resource_id': self.resource_id,
-            'http_status': self.http_status or '',
-            'start_time': self.start_time,
-            'end_time': self.end_time,
-            'url_domain': self.url_domain if self.url_domain else '',
-            'url_path': self.url_path if self.url_path else '',
-            'duration': self.duration,
-            'error': self.error if self.error else '',
-            'report_type': self.report_type,
-            'request_id': self.request_id,
-            'ip': self.ip,
-            'group_id': str(self.group_id),
-            '_parent': str(self.group_id),
-            'tags': tags,
-            'tag_list': tag_list
+            "_id": str(self.id),
+            "pg_id": str(self.id),
+            "resource_id": self.resource_id,
+            "http_status": self.http_status or "",
+            "start_time": self.start_time,
+            "end_time": self.end_time,
+            "url_domain": self.url_domain if self.url_domain else "",
+            "url_path": self.url_path if self.url_path else "",
+            "duration": self.duration,
+            "error": self.error if self.error else "",
+            "report_type": self.report_type,
+            "request_id": self.request_id,
+            "ip": self.ip,
+            "group_id": str(self.group_id),
+            "_parent": str(self.group_id),
+            "tags": tags,
+            "tag_list": tag_list,
         }
 
     @property
     def partition_id(self):
-        return 'rcae_r_%s' % self.report_group_time.strftime('%Y_%m')
+        return "rcae_r_%s" % self.report_group_time.strftime("%Y_%m")
 
     def partition_range(self):
         start_date = self.report_group_time.date().replace(day=1)
         end_date = start_date + timedelta(days=40)
         end_date = end_date.replace(day=1)
         return start_date, end_date
 
 
 def after_insert(mapper, connection, target):
-    if not hasattr(target, '_skip_ft_index'):
+    if not hasattr(target, "_skip_ft_index"):
         data = target.es_doc()
-        data.pop('_id', None)
-        Datastores.es.index(target.partition_id, 'report', data,
-                            parent=target.group_id, id=target.id)
+        data.pop("_id", None)
+        Datastores.es.index(
+            target.partition_id, "report", data, parent=target.group_id, id=target.id
+        )
 
 
 def after_update(mapper, connection, target):
-    if not hasattr(target, '_skip_ft_index'):
+    if not hasattr(target, "_skip_ft_index"):
         data = target.es_doc()
-        data.pop('_id', None)
-        Datastores.es.index(target.partition_id, 'report', data,
-                            parent=target.group_id, id=target.id)
+        data.pop("_id", None)
+        Datastores.es.index(
+            target.partition_id, "report", data, parent=target.group_id, id=target.id
+        )
 
 
 def after_delete(mapper, connection, target):
-    if not hasattr(target, '_skip_ft_index'):
-        query = {"query":{'term': {'pg_id': target.id}}}
-        Datastores.es.transport.perform_request("DELETE", '/{}/{}/_query'.format(target.partition_id, 'report'), body=query)
+    if not hasattr(target, "_skip_ft_index"):
+        query = {"query": {"term": {"pg_id": target.id}}}
+        Datastores.es.transport.perform_request(
+            "DELETE", "/{}/{}/_query".format(target.partition_id, "report"), body=query
+        )
 
 
-sa.event.listen(Report, 'after_insert', after_insert)
-sa.event.listen(Report, 'after_update', after_update)
-sa.event.listen(Report, 'after_delete', after_delete)
+sa.event.listen(Report, "after_insert", after_insert)
+sa.event.listen(Report, "after_update", after_update)
+sa.event.listen(Report, "after_delete", after_delete)

backend/src/appenlight/models/report_assignment.py

@@ -1,32 +1,35 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from ziggurat_foundations.models.base import BaseModel
 from appenlight.models import Base
 import sqlalchemy as sa
 
 
 class ReportAssignment(Base, BaseModel):
-    __tablename__ = 'reports_assignments'
+    __tablename__ = "reports_assignments"
 
-    group_id = sa.Column(sa.BigInteger,
-                         sa.ForeignKey('reports_groups.id', ondelete='cascade',
-                                       onupdate='cascade'),
-                         primary_key=True)
-    owner_id = sa.Column(sa.Integer,
-                          sa.ForeignKey('users.id', onupdate='CASCADE',
-                                        ondelete='CASCADE'), primary_key=True)
+    group_id = sa.Column(
+        sa.BigInteger,
+        sa.ForeignKey("reports_groups.id", ondelete="cascade", onupdate="cascade"),
+        primary_key=True,
+    )
+    owner_id = sa.Column(
+        sa.Integer,
+        sa.ForeignKey("users.id", onupdate="CASCADE", ondelete="CASCADE"),
+        primary_key=True,
+    )
     report_time = sa.Column(sa.DateTime(), nullable=False)

backend/src/appenlight/models/report_comment.py

@@ -1,50 +1,50 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import sqlalchemy as sa
 
 from datetime import datetime
 from appenlight.models import Base
 from ziggurat_foundations.models.base import BaseModel
 
 
 class ReportComment(Base, BaseModel):
-    __tablename__ = 'reports_comments'
+    __tablename__ = "reports_comments"
 
     comment_id = sa.Column(sa.Integer, nullable=False, primary_key=True)
-    group_id = sa.Column(sa.BigInteger,
-                         sa.ForeignKey('reports_groups.id', ondelete='cascade',
-                                       onupdate='cascade'))
-    body = sa.Column(sa.UnicodeText(), default='')
-    owner_id = sa.Column(sa.Integer,
-                         sa.ForeignKey('users.id', onupdate='CASCADE',
-                                       ondelete='CASCADE'))
-    created_timestamp = sa.Column(sa.DateTime(),
-                                  default=datetime.utcnow,
-                                  server_default=sa.func.now())
+    group_id = sa.Column(
+        sa.BigInteger,
+        sa.ForeignKey("reports_groups.id", ondelete="cascade", onupdate="cascade"),
+    )
+    body = sa.Column(sa.UnicodeText(), default="")
+    owner_id = sa.Column(
+        sa.Integer, sa.ForeignKey("users.id", onupdate="CASCADE", ondelete="CASCADE")
+    )
+    created_timestamp = sa.Column(
+        sa.DateTime(), default=datetime.utcnow, server_default=sa.func.now()
+    )
     report_time = sa.Column(sa.DateTime(), nullable=False)
 
-    owner = sa.orm.relationship('User',
-                                lazy='joined')
+    owner = sa.orm.relationship("User", lazy="joined")
 
     @property
     def processed_body(self):
         return self.body
 
     def get_dict(self):
         instance_dict = super(ReportComment, self).get_dict()
-        instance_dict['user_name'] = self.owner.user_name
+        instance_dict["user_name"] = self.owner.user_name
         return instance_dict

backend/src/appenlight/models/report_group.py

@@ -1,268 +1,287 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 import sqlalchemy as sa
 
 from datetime import datetime, timedelta
 
 from pyramid.threadlocal import get_current_request
 from sqlalchemy.dialects.postgresql import JSON
 from ziggurat_foundations.models.base import BaseModel
 
 from appenlight.models import Base, get_db_session, Datastores
 from appenlight.lib.enums import ReportType
 from appenlight.lib.rule import Rule
 from appenlight.lib.redis_keys import REDIS_KEYS
 from appenlight.models.report import REPORT_TYPE_MATRIX
 
 log = logging.getLogger(__name__)
 
 
 class ReportGroup(Base, BaseModel):
-    __tablename__ = 'reports_groups'
-    __table_args__ = {'implicit_returning': False}
+    __tablename__ = "reports_groups"
+    __table_args__ = {"implicit_returning": False}
 
     id = sa.Column(sa.BigInteger(), nullable=False, primary_key=True)
-    resource_id = sa.Column(sa.Integer(),
-                            sa.ForeignKey('applications.resource_id',
-                                          onupdate='CASCADE',
-                                          ondelete='CASCADE'),
-                            nullable=False,
-                            index=True)
-    priority = sa.Column(sa.Integer, nullable=False, index=True, default=5,
-                         server_default='5')
-    first_timestamp = sa.Column(sa.DateTime(), default=datetime.utcnow,
-                                server_default=sa.func.now())
-    last_timestamp = sa.Column(sa.DateTime(), default=datetime.utcnow,
-                               server_default=sa.func.now())
+    resource_id = sa.Column(
+        sa.Integer(),
+        sa.ForeignKey(
+            "applications.resource_id", onupdate="CASCADE", ondelete="CASCADE"
+        ),
+        nullable=False,
+        index=True,
+    )
+    priority = sa.Column(
+        sa.Integer, nullable=False, index=True, default=5, server_default="5"
+    )
+    first_timestamp = sa.Column(
+        sa.DateTime(), default=datetime.utcnow, server_default=sa.func.now()
+    )
+    last_timestamp = sa.Column(
+        sa.DateTime(), default=datetime.utcnow, server_default=sa.func.now()
+    )
     error = sa.Column(sa.UnicodeText(), index=True)
-    grouping_hash = sa.Column(sa.String(40), default='')
-    triggered_postprocesses_ids = sa.Column(JSON(), nullable=False,
-                                            default=list)
+    grouping_hash = sa.Column(sa.String(40), default="")
+    triggered_postprocesses_ids = sa.Column(JSON(), nullable=False, default=list)
     report_type = sa.Column(sa.Integer, default=1)
     total_reports = sa.Column(sa.Integer, default=1)
     last_report = sa.Column(sa.Integer)
     occurences = sa.Column(sa.Integer, default=1)
     average_duration = sa.Column(sa.Float, default=0)
     summed_duration = sa.Column(sa.Float, default=0)
     read = sa.Column(sa.Boolean(), index=True, default=False)
     fixed = sa.Column(sa.Boolean(), index=True, default=False)
     notified = sa.Column(sa.Boolean(), index=True, default=False)
     public = sa.Column(sa.Boolean(), index=True, default=False)
 
-    reports = sa.orm.relationship('Report',
-                                  lazy='dynamic',
-                                  backref='report_group',
-                                  cascade="all, delete-orphan",
-                                  passive_deletes=True,
-                                  passive_updates=True, )
-
-    comments = sa.orm.relationship('ReportComment',
-                                   lazy='dynamic',
-                                   backref='report',
-                                   cascade="all, delete-orphan",
-                                   passive_deletes=True,
-                                   passive_updates=True,
-                                   order_by="ReportComment.comment_id")
-
-    assigned_users = sa.orm.relationship('User',
-                                         backref=sa.orm.backref(
-                                             'assigned_reports_relation',
-                                             lazy='dynamic',
-                                             order_by=sa.desc(
-                                                 sa.text("reports_groups.id"))
-                                         ),
-                                         passive_deletes=True,
-                                         passive_updates=True,
-                                         secondary='reports_assignments',
-                                         order_by="User.user_name")
-
-    stats = sa.orm.relationship('ReportStat',
-                                lazy='dynamic',
-                                backref='report',
-                                passive_deletes=True,
-                                passive_updates=True, )
-
-    last_report_ref = sa.orm.relationship('Report',
-                                          uselist=False,
-                                          primaryjoin="ReportGroup.last_report "
-                                                      "== Report.id",
-                                          foreign_keys="Report.id",
-                                          cascade="all, delete-orphan",
-                                          passive_deletes=True,
-                                          passive_updates=True, )
+    reports = sa.orm.relationship(
+        "Report",
+        lazy="dynamic",
+        backref="report_group",
+        cascade="all, delete-orphan",
+        passive_deletes=True,
+        passive_updates=True,
+    )
+
+    comments = sa.orm.relationship(
+        "ReportComment",
+        lazy="dynamic",
+        backref="report",
+        cascade="all, delete-orphan",
+        passive_deletes=True,
+        passive_updates=True,
+        order_by="ReportComment.comment_id",
+    )
+
+    assigned_users = sa.orm.relationship(
+        "User",
+        backref=sa.orm.backref(
+            "assigned_reports_relation",
+            lazy="dynamic",
+            order_by=sa.desc(sa.text("reports_groups.id")),
+        ),
+        passive_deletes=True,
+        passive_updates=True,
+        secondary="reports_assignments",
+        order_by="User.user_name",
+    )
+
+    stats = sa.orm.relationship(
+        "ReportStat",
+        lazy="dynamic",
+        backref="report",
+        passive_deletes=True,
+        passive_updates=True,
+    )
+
+    last_report_ref = sa.orm.relationship(
+        "Report",
+        uselist=False,
+        primaryjoin="ReportGroup.last_report " "== Report.id",
+        foreign_keys="Report.id",
+        cascade="all, delete-orphan",
+        passive_deletes=True,
+        passive_updates=True,
+    )
 
     def __repr__(self):
-        return '<ReportGroup id:{}>'.format(self.id)
+        return "<ReportGroup id:{}>".format(self.id)
 
     def get_report(self, report_id=None, public=False):
         """
         Gets report with specific id or latest report if id was not specified
         """
         from .report import Report
 
         if not report_id:
             return self.last_report_ref
         else:
             return self.reports.filter(Report.id == report_id).first()
 
     def get_public_url(self, request, _app_url=None):
-        url = request.route_url('/', _app_url=_app_url)
-        return (url + 'ui/report/%s') % self.id
+        url = request.route_url("/", _app_url=_app_url)
+        return (url + "ui/report/%s") % self.id
 
     def run_postprocessing(self, report):
         """
         Alters report group priority based on postprocessing configuration
         """
         request = get_current_request()
         get_db_session(None, self).flush()
         for action in self.application.postprocess_conf:
             get_db_session(None, self).flush()
             rule_obj = Rule(action.rule, REPORT_TYPE_MATRIX)
             report_dict = report.get_dict(request)
             # if was not processed yet
-            if (rule_obj.match(report_dict) and
-                        action.pkey not in self.triggered_postprocesses_ids):
+            if (
+                rule_obj.match(report_dict)
+                and action.pkey not in self.triggered_postprocesses_ids
+            ):
                 action.postprocess(self)
                 # this way sqla can track mutation of list
-                self.triggered_postprocesses_ids = \
-                    self.triggered_postprocesses_ids + [action.pkey]
+                self.triggered_postprocesses_ids = self.triggered_postprocesses_ids + [
+                    action.pkey
+                ]
 
         get_db_session(None, self).flush()
         # do not go out of bounds
         if self.priority < 1:
             self.priority = 1
         if self.priority > 10:
             self.priority = 10
 
     def get_dict(self, request):
         instance_dict = super(ReportGroup, self).get_dict()
-        instance_dict['server_name'] = self.get_report().tags.get(
-            'server_name')
-        instance_dict['view_name'] = self.get_report().tags.get('view_name')
-        instance_dict['resource_name'] = self.application.resource_name
-        instance_dict['report_type'] = self.get_report().report_type
-        instance_dict['url_path'] = self.get_report().url_path
-        instance_dict['front_url'] = self.get_report().get_public_url(request)
-        del instance_dict['triggered_postprocesses_ids']
+        instance_dict["server_name"] = self.get_report().tags.get("server_name")
+        instance_dict["view_name"] = self.get_report().tags.get("view_name")
+        instance_dict["resource_name"] = self.application.resource_name
+        instance_dict["report_type"] = self.get_report().report_type
+        instance_dict["url_path"] = self.get_report().url_path
+        instance_dict["front_url"] = self.get_report().get_public_url(request)
+        del instance_dict["triggered_postprocesses_ids"]
         return instance_dict
 
     def es_doc(self):
         return {
-            '_id': str(self.id),
-            'pg_id': str(self.id),
-            'resource_id': self.resource_id,
-            'error': self.error,
-            'fixed': self.fixed,
-            'public': self.public,
-            'read': self.read,
-            'priority': self.priority,
-            'occurences': self.occurences,
-            'average_duration': self.average_duration,
-            'summed_duration': self.summed_duration,
-            'first_timestamp': self.first_timestamp,
-            'last_timestamp': self.last_timestamp
+            "_id": str(self.id),
+            "pg_id": str(self.id),
+            "resource_id": self.resource_id,
+            "error": self.error,
+            "fixed": self.fixed,
+            "public": self.public,
+            "read": self.read,
+            "priority": self.priority,
+            "occurences": self.occurences,
+            "average_duration": self.average_duration,
+            "summed_duration": self.summed_duration,
+            "first_timestamp": self.first_timestamp,
+            "last_timestamp": self.last_timestamp,
         }
 
     def set_notification_info(self, notify_10=False, notify_100=False):
         """
         Update redis notification maps for notification job
         """
         current_time = datetime.utcnow().replace(second=0, microsecond=0)
         # global app counter
-        key = REDIS_KEYS['counters']['reports_per_type'].format(
-            self.report_type, current_time)
+        key = REDIS_KEYS["counters"]["reports_per_type"].format(
+            self.report_type, current_time
+        )
         redis_pipeline = Datastores.redis.pipeline()
         redis_pipeline.incr(key)
         redis_pipeline.expire(key, 3600 * 24)
         # detailed app notification for alerts and notifications
+        redis_pipeline.sadd(REDIS_KEYS["apps_that_had_reports"], self.resource_id)
         redis_pipeline.sadd(
-            REDIS_KEYS['apps_that_had_reports'], self.resource_id)
-        redis_pipeline.sadd(
-            REDIS_KEYS['apps_that_had_reports_alerting'], self.resource_id)
+            REDIS_KEYS["apps_that_had_reports_alerting"], self.resource_id
+        )
         # only notify for exceptions here
         if self.report_type == ReportType.error:
+            redis_pipeline.sadd(REDIS_KEYS["apps_that_had_reports"], self.resource_id)
             redis_pipeline.sadd(
-                REDIS_KEYS['apps_that_had_reports'], self.resource_id)
-            redis_pipeline.sadd(
-                REDIS_KEYS['apps_that_had_error_reports_alerting'],
-                self.resource_id)
-        key = REDIS_KEYS['counters']['report_group_occurences'].format(self.id)
+                REDIS_KEYS["apps_that_had_error_reports_alerting"], self.resource_id
+            )
+        key = REDIS_KEYS["counters"]["report_group_occurences"].format(self.id)
         redis_pipeline.incr(key)
         redis_pipeline.expire(key, 3600 * 24)
-        key = REDIS_KEYS['counters']['report_group_occurences_alerting'].format(
-            self.id)
+        key = REDIS_KEYS["counters"]["report_group_occurences_alerting"].format(self.id)
         redis_pipeline.incr(key)
         redis_pipeline.expire(key, 3600 * 24)
 
         if notify_10:
-            key = REDIS_KEYS['counters'][
-                'report_group_occurences_10th'].format(self.id)
+            key = REDIS_KEYS["counters"]["report_group_occurences_10th"].format(self.id)
             redis_pipeline.setex(key, 3600 * 24, 1)
         if notify_100:
-            key = REDIS_KEYS['counters'][
-                'report_group_occurences_100th'].format(self.id)
+            key = REDIS_KEYS["counters"]["report_group_occurences_100th"].format(
+                self.id
+            )
             redis_pipeline.setex(key, 3600 * 24, 1)
 
-        key = REDIS_KEYS['reports_to_notify_per_type_per_app'].format(
-            self.report_type, self.resource_id)
+        key = REDIS_KEYS["reports_to_notify_per_type_per_app"].format(
+            self.report_type, self.resource_id
+        )
         redis_pipeline.sadd(key, self.id)
         redis_pipeline.expire(key, 3600 * 24)
-        key = REDIS_KEYS['reports_to_notify_per_type_per_app_alerting'].format(
-            self.report_type, self.resource_id)
+        key = REDIS_KEYS["reports_to_notify_per_type_per_app_alerting"].format(
+            self.report_type, self.resource_id
+        )
         redis_pipeline.sadd(key, self.id)
         redis_pipeline.expire(key, 3600 * 24)
         redis_pipeline.execute()
 
     @property
     def partition_id(self):
-        return 'rcae_r_%s' % self.first_timestamp.strftime('%Y_%m')
+        return "rcae_r_%s" % self.first_timestamp.strftime("%Y_%m")
 
     def partition_range(self):
         start_date = self.first_timestamp.date().replace(day=1)
         end_date = start_date + timedelta(days=40)
         end_date = end_date.replace(day=1)
         return start_date, end_date
 
 
 def after_insert(mapper, connection, target):
-    if not hasattr(target, '_skip_ft_index'):
+    if not hasattr(target, "_skip_ft_index"):
         data = target.es_doc()
-        data.pop('_id', None)
-        Datastores.es.index(target.partition_id, 'report_group',
-                            data, id=target.id)
+        data.pop("_id", None)
+        Datastores.es.index(target.partition_id, "report_group", data, id=target.id)
 
 
 def after_update(mapper, connection, target):
-    if not hasattr(target, '_skip_ft_index'):
+    if not hasattr(target, "_skip_ft_index"):
         data = target.es_doc()
-        data.pop('_id', None)
-        Datastores.es.index(target.partition_id, 'report_group',
-                            data, id=target.id)
+        data.pop("_id", None)
+        Datastores.es.index(target.partition_id, "report_group", data, id=target.id)
 
 
 def after_delete(mapper, connection, target):
-    query = {"query": {'term': {'group_id': target.id}}}
+    query = {"query": {"term": {"group_id": target.id}}}
     # delete by query
-    Datastores.es.transport.perform_request("DELETE", '/{}/{}/_query'.format(target.partition_id, 'report'), body=query)
-    query = {"query": {'term': {'pg_id': target.id}}}
-    Datastores.es.transport.perform_request("DELETE", '/{}/{}/_query'.format(target.partition_id, 'report_group'), body=query)
-
-
-sa.event.listen(ReportGroup, 'after_insert', after_insert)
-sa.event.listen(ReportGroup, 'after_update', after_update)
-sa.event.listen(ReportGroup, 'after_delete', after_delete)
+    Datastores.es.transport.perform_request(
+        "DELETE", "/{}/{}/_query".format(target.partition_id, "report"), body=query
+    )
+    query = {"query": {"term": {"pg_id": target.id}}}
+    Datastores.es.transport.perform_request(
+        "DELETE",
+        "/{}/{}/_query".format(target.partition_id, "report_group"),
+        body=query,
+    )
+
+
+sa.event.listen(ReportGroup, "after_insert", after_insert)
+sa.event.listen(ReportGroup, "after_update", after_update)
+sa.event.listen(ReportGroup, "after_delete", after_delete)

backend/src/appenlight/models/report_stat.py

@@ -1,74 +1,79 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import sqlalchemy as sa
 
 from appenlight.lib.enums import ReportType
 from appenlight.models import Base
 from ziggurat_foundations.models.base import BaseModel
 
 
 class ReportStat(Base, BaseModel):
-    __tablename__ = 'reports_stats'
-    __table_args__ = {'implicit_returning': False}
+    __tablename__ = "reports_stats"
+    __table_args__ = {"implicit_returning": False}
 
-    group_id = sa.Column(sa.BigInteger(),
-                         sa.ForeignKey('reports_groups.id'),
-                         nullable=False)
-    resource_id = sa.Column(sa.Integer(),
-                            sa.ForeignKey('applications.resource_id'),
-                            nullable=False)
+    group_id = sa.Column(
+        sa.BigInteger(), sa.ForeignKey("reports_groups.id"), nullable=False
+    )
+    resource_id = sa.Column(
+        sa.Integer(), sa.ForeignKey("applications.resource_id"), nullable=False
+    )
     start_interval = sa.Column(sa.DateTime(), nullable=False)
     occurences = sa.Column(sa.Integer, nullable=True, default=0)
-    owner_user_id = sa.Column(sa.Integer(), sa.ForeignKey('users.id'),
-                              nullable=True)
+    owner_user_id = sa.Column(sa.Integer(), sa.ForeignKey("users.id"), nullable=True)
     type = sa.Column(sa.Integer, nullable=True, default=0)
     duration = sa.Column(sa.Float, nullable=True, default=0)
     id = sa.Column(sa.BigInteger, nullable=False, primary_key=True)
-    server_name = sa.Column(sa.Unicode(128), nullable=False, default='')
-    view_name = sa.Column(sa.Unicode(128), nullable=False, default='')
+    server_name = sa.Column(sa.Unicode(128), nullable=False, default="")
+    view_name = sa.Column(sa.Unicode(128), nullable=False, default="")
 
     @property
     def partition_id(self):
-        return 'rcae_r_%s' % self.start_interval.strftime('%Y_%m')
+        return "rcae_r_%s" % self.start_interval.strftime("%Y_%m")
 
     def es_doc(self):
         return {
-            'resource_id': self.resource_id,
-            'timestamp': self.start_interval,
-            'pg_id': str(self.id),
-            'permanent': True,
-            'request_id': None,
-            'log_level': 'ERROR',
-            'message': None,
-            'namespace': 'appenlight.error',
-            'tags': {
-                'duration': {'values': self.duration,
-                             'numeric_values': self.duration},
-                'occurences': {'values': self.occurences,
-                               'numeric_values': self.occurences},
-                'group_id': {'values': self.group_id,
-                             'numeric_values': self.group_id},
-                'type': {'values': ReportType.key_from_value(self.type),
-                         'numeric_values': self.type},
-                'server_name': {'values': self.server_name,
-                                'numeric_values': None},
-                'view_name': {'values': self.view_name,
-                              'numeric_values': None},
+            "resource_id": self.resource_id,
+            "timestamp": self.start_interval,
+            "pg_id": str(self.id),
+            "permanent": True,
+            "request_id": None,
+            "log_level": "ERROR",
+            "message": None,
+            "namespace": "appenlight.error",
+            "tags": {
+                "duration": {"values": self.duration, "numeric_values": self.duration},
+                "occurences": {
+                    "values": self.occurences,
+                    "numeric_values": self.occurences,
+                },
+                "group_id": {"values": self.group_id, "numeric_values": self.group_id},
+                "type": {
+                    "values": ReportType.key_from_value(self.type),
+                    "numeric_values": self.type,
+                },
+                "server_name": {"values": self.server_name, "numeric_values": None},
+                "view_name": {"values": self.view_name, "numeric_values": None},
             },
-            'tag_list': ['duration', 'occurences', 'group_id', 'type',
-                         'server_name', 'view_name']
+            "tag_list": [
+                "duration",
+                "occurences",
+                "group_id",
+                "type",
+                "server_name",
+                "view_name",
+            ],
         }

backend/src/appenlight/models/resource.py

@@ -1,84 +1,96 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import sqlalchemy as sa
 from appenlight.models import Base
 from appenlight.lib.utils import permission_tuple_to_dict
 from pyramid.security import Allow, ALL_PERMISSIONS
 from ziggurat_foundations.models.resource import ResourceMixin
 from ziggurat_foundations.models.services.resource import ResourceService
 
 
 class Resource(ResourceMixin, Base):
-    events = sa.orm.relationship('Event',
-                                 lazy='dynamic',
-                                 backref='resource',
-                                 passive_deletes=True,
-                                 passive_updates=True)
+    events = sa.orm.relationship(
+        "Event",
+        lazy="dynamic",
+        backref="resource",
+        passive_deletes=True,
+        passive_updates=True,
+    )
 
     @property
     def owner_user_name(self):
         if self.owner:
             return self.owner.user_name
 
     @property
     def owner_group_name(self):
         if self.owner_group:
             return self.owner_group.group_name
 
-    def get_dict(self, exclude_keys=None, include_keys=None,
-                 include_perms=False, include_processing_rules=False):
+    def get_dict(
+        self,
+        exclude_keys=None,
+        include_keys=None,
+        include_perms=False,
+        include_processing_rules=False,
+    ):
         result = super(Resource, self).get_dict(exclude_keys, include_keys)
-        result['possible_permissions'] = self.__possible_permissions__
+        result["possible_permissions"] = self.__possible_permissions__
         if include_perms:
-            result['current_permissions'] = self.user_permissions_list
+            result["current_permissions"] = self.user_permissions_list
         else:
-            result['current_permissions'] = []
+            result["current_permissions"] = []
         if include_processing_rules:
-            result["postprocessing_rules"] = [rule.get_dict() for rule
-                                              in self.postprocess_conf]
+            result["postprocessing_rules"] = [
+                rule.get_dict() for rule in self.postprocess_conf
+            ]
         else:
             result["postprocessing_rules"] = []
         exclude_keys_list = exclude_keys or []
         include_keys_list = include_keys or []
         d = {}
         for k in result.keys():
-            if (k not in exclude_keys_list and
-                    (k in include_keys_list or not include_keys)):
+            if k not in exclude_keys_list and (
+                k in include_keys_list or not include_keys
+            ):
                 d[k] = result[k]
-        for k in ['owner_user_name', 'owner_group_name']:
-            if (k not in exclude_keys_list and
-                    (k in include_keys_list or not include_keys)):
+        for k in ["owner_user_name", "owner_group_name"]:
+            if k not in exclude_keys_list and (
+                k in include_keys_list or not include_keys
+            ):
                 d[k] = getattr(self, k)
         return d
 
     @property
     def user_permissions_list(self):
-        return [permission_tuple_to_dict(perm) for perm in
-                ResourceService.users_for_perm(
-                    self, '__any_permission__', limit_group_permissions=True)]
+        return [
+            permission_tuple_to_dict(perm)
+            for perm in ResourceService.users_for_perm(
+                self, "__any_permission__", limit_group_permissions=True
+            )
+        ]
 
     @property
     def __acl__(self):
         acls = []
 
         if self.owner_user_id:
-            acls.extend([(Allow, self.owner_user_id, ALL_PERMISSIONS,), ])
+            acls.extend([(Allow, self.owner_user_id, ALL_PERMISSIONS)])
 
         if self.owner_group_id:
-            acls.extend([(Allow, "group:%s" % self.owner_group_id,
-                          ALL_PERMISSIONS,), ])
+            acls.extend([(Allow, "group:%s" % self.owner_group_id, ALL_PERMISSIONS)])
         return acls

backend/src/appenlight/models/services/__init__.py

@@ -1,16 +1,15 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-

backend/src/appenlight/models/services/application.py

@@ -1,193 +1,200 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 import urllib.parse
 
 import sqlalchemy as sa
 from pyramid.threadlocal import get_current_request
 
 from appenlight.lib.cache_regions import get_region
 from appenlight.lib.enums import ReportType
 from appenlight.models import get_db_session
 from appenlight.models.report_group import ReportGroup
 from appenlight.models.application import Application
 from appenlight.models.event import Event
 from appenlight.models.services.base import BaseService
 from appenlight.models.services.event import EventService
 
 log = logging.getLogger(__name__)
 
 
 class ApplicationService(BaseService):
-
     @classmethod
     def all(cls, db_session=None):
         db_session = get_db_session(db_session)
         q = db_session.query(Application)
         return q
 
     @classmethod
     def by_api_key(cls, api_key, db_session=None):
         db_session = get_db_session(db_session)
         q = db_session.query(Application)
         q = q.filter(Application.api_key == api_key)
         q = q.options(sa.orm.eagerload(Application.owner))
         return q.first()
 
     @classmethod
     def by_api_key_cached(cls, db_session=None):
         db_session = get_db_session(db_session)
-        cache_region = get_region('redis_min_1')
+        cache_region = get_region("redis_min_1")
 
-        @cache_region.cache_on_arguments('ApplicationService.by_api_key')
+        @cache_region.cache_on_arguments("ApplicationService.by_api_key")
         def cached(*args, **kwargs):
             app = cls.by_api_key(*args, db_session=db_session, **kwargs)
             if app:
                 db_session.expunge(app)
             return app
 
         return cached
 
     @classmethod
-    def by_public_api_key(cls, api_key, db_session=None, from_cache=False,
-                          request=None):
+    def by_public_api_key(
+        cls, api_key, db_session=None, from_cache=False, request=None
+    ):
         db_session = get_db_session(db_session)
-        cache_region = get_region('redis_min_1')
+        cache_region = get_region("redis_min_1")
 
         def uncached(api_key):
             q = db_session.query(Application)
             q = q.filter(Application.public_key == api_key)
             q = q.options(sa.orm.eagerload(Application.owner))
             return q.first()
 
         if from_cache:
-            @cache_region.cache_on_arguments(
-                'ApplicationService.by_public_api_key')
+
+            @cache_region.cache_on_arguments("ApplicationService.by_public_api_key")
             def cached(api_key):
                 app = uncached(api_key)
                 if app:
                     db_session.expunge(app)
                 return app
 
             app = cached(api_key)
         else:
             app = uncached(api_key)
         return app
 
     @classmethod
     def by_id(cls, db_id, db_session=None):
         db_session = get_db_session(db_session)
         q = db_session.query(Application)
         q = q.filter(Application.resource_id == db_id)
         return q.first()
 
     @classmethod
     def by_id_cached(cls, db_session=None):
         db_session = get_db_session(db_session)
-        cache_region = get_region('redis_min_1')
+        cache_region = get_region("redis_min_1")
 
-        @cache_region.cache_on_arguments('ApplicationService.by_id')
+        @cache_region.cache_on_arguments("ApplicationService.by_id")
         def cached(*args, **kwargs):
             app = cls.by_id(*args, db_session=db_session, **kwargs)
             if app:
                 db_session.expunge(app)
             return app
 
         return cached
 
     @classmethod
     def by_ids(cls, db_ids, db_session=None):
         db_session = get_db_session(db_session)
         query = db_session.query(Application)
         query = query.filter(Application.resource_id.in_(db_ids))
         return query
 
     @classmethod
     def by_http_referer(cls, referer_string, db_session=None):
         db_session = get_db_session(db_session)
-        domain = urllib.parse.urlsplit(
-            referer_string, allow_fragments=False).netloc
+        domain = urllib.parse.urlsplit(referer_string, allow_fragments=False).netloc
         if domain:
-            if domain.startswith('www.'):
+            if domain.startswith("www."):
                 domain = domain[4:]
         q = db_session.query(Application).filter(Application.domain == domain)
         return q.first()
 
     @classmethod
     def last_updated(cls, since_when, exclude_status=None, db_session=None):
         db_session = get_db_session(db_session)
         q = db_session.query(Application)
         q2 = ReportGroup.last_updated(
-            since_when, exclude_status=exclude_status, db_session=db_session)
+            since_when, exclude_status=exclude_status, db_session=db_session
+        )
         q2 = q2.from_self(ReportGroup.resource_id)
         q2 = q2.group_by(ReportGroup.resource_id)
         q = q.filter(Application.resource_id.in_(q2))
         return q
 
     @classmethod
     def check_for_groups_alert(cls, resource, event_type, *args, **kwargs):
         """ Check for open alerts depending on group type.
         Create new one if nothing is found and send alerts """
-        db_session = get_db_session(kwargs.get('db_session'))
+        db_session = get_db_session(kwargs.get("db_session"))
         request = get_current_request()
-        report_groups = kwargs['report_groups']
-        occurence_dict = kwargs['occurence_dict']
+        report_groups = kwargs["report_groups"]
+        occurence_dict = kwargs["occurence_dict"]
 
         error_reports = 0
         slow_reports = 0
         for group in report_groups:
             occurences = occurence_dict.get(group.id, 1)
             if group.get_report().report_type == ReportType.error:
                 error_reports += occurences
             elif group.get_report().report_type == ReportType.slow:
                 slow_reports += occurences
 
-        log_msg = 'LIMIT INFO: %s : %s error reports. %s slow_reports' % (
+        log_msg = "LIMIT INFO: %s : %s error reports. %s slow_reports" % (
             resource,
             error_reports,
-            slow_reports)
+            slow_reports,
+        )
         logging.warning(log_msg)
         threshold = 10
-        for event_type in ['error_report_alert', 'slow_report_alert']:
-            if (error_reports < resource.error_report_threshold and
-                        event_type == 'error_report_alert'):
+        for event_type in ["error_report_alert", "slow_report_alert"]:
+            if (
+                error_reports < resource.error_report_threshold
+                and event_type == "error_report_alert"
+            ):
                 continue
-            elif (slow_reports <= resource.slow_report_threshold and
-                          event_type == 'slow_report_alert'):
+            elif (
+                slow_reports <= resource.slow_report_threshold
+                and event_type == "slow_report_alert"
+            ):
                 continue
-            if event_type == 'error_report_alert':
+            if event_type == "error_report_alert":
                 amount = error_reports
                 threshold = resource.error_report_threshold
-            elif event_type == 'slow_report_alert':
+            elif event_type == "slow_report_alert":
                 amount = slow_reports
                 threshold = resource.slow_report_threshold
 
-            event = EventService.for_resource([resource.resource_id],
-                                              event_type=Event.types[
-                                                  event_type],
-                                              status=Event.statuses['active'])
+            event = EventService.for_resource(
+                [resource.resource_id],
+                event_type=Event.types[event_type],
+                status=Event.statuses["active"],
+            )
             if event.first():
-                log.info('ALERT: PROGRESS: %s %s' % (event_type, resource))
+                log.info("ALERT: PROGRESS: %s %s" % (event_type, resource))
             else:
-                log.warning('ALERT: OPEN: %s %s' % (event_type, resource))
-                new_event = Event(resource_id=resource.resource_id,
-                                  event_type=Event.types[event_type],
-                                  status=Event.statuses['active'],
-                                  values={'reports': amount,
-                                          'threshold': threshold})
+                log.warning("ALERT: OPEN: %s %s" % (event_type, resource))
+                new_event = Event(
+                    resource_id=resource.resource_id,
+                    event_type=Event.types[event_type],
+                    status=Event.statuses["active"],
+                    values={"reports": amount, "threshold": threshold},
+                )
                 db_session.add(new_event)
                 new_event.send_alerts(request=request, resource=resource)

backend/src/appenlight/models/services/application_postprocess_conf.py

@@ -1,36 +1,35 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 
 from appenlight.models import get_db_session
 from appenlight.models.application_postprocess_conf import ApplicationPostprocessConf
 from appenlight.models.services.base import BaseService
 
 
 class ApplicationPostprocessConfService(BaseService):
-
     @classmethod
     def by_pkey(cls, pkey, db_session=None):
         db_session = get_db_session(db_session)
         query = db_session.query(ApplicationPostprocessConf)
         return query.filter(ApplicationPostprocessConf.pkey == pkey).first()
 
     @classmethod
     def by_pkey_and_resource_id(cls, pkey, resource_id, db_session=None):
         db_session = get_db_session(db_session)
         query = db_session.query(ApplicationPostprocessConf)
         query = query.filter(ApplicationPostprocessConf.resource_id == resource_id)
         return query.filter(ApplicationPostprocessConf.pkey == pkey).first()

backend/src/appenlight/models/services/config.py

@@ -1,100 +1,116 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 
 import sqlalchemy as sa
 from pyramid.threadlocal import get_current_registry
 
 from appenlight.models.config import Config
 from appenlight.models.services.base import BaseService
 from appenlight.models import get_db_session
 
 log = logging.getLogger(__name__)
 
 
 class ConfigService(BaseService):
     @classmethod
     def all(cls, db_session=None):
         db_session = get_db_session(db_session)
         query = db_session.query(Config)
         return query
 
     @classmethod
     def filtered_key_and_section(cls, pairs=None, db_session=None):
         db_session = get_db_session(db_session)
         query = db_session.query(Config)
         if pairs:
             conditions = []
             for pair in pairs:
-                conditions.append(sa.and_(
-                    Config.key == pair['key'],
-                    Config.section == pair['section'])
+                conditions.append(
+                    sa.and_(
+                        Config.key == pair["key"], Config.section == pair["section"]
+                    )
                 )
 
             query = query.filter(sa.or_(*conditions))
         return query
 
     @classmethod
     def create_config(cls, key, section, value=None, db_session=None):
         config = Config(key=key, section=section, value=value)
         db_session = get_db_session(db_session)
         db_session.add(config)
         db_session.flush()
         return config
 
     @classmethod
-    def by_key_and_section(cls, key, section, auto_create=False,
-                           default_value=None, db_session=None):
+    def by_key_and_section(
+        cls, key, section, auto_create=False, default_value=None, db_session=None
+    ):
         db_session = get_db_session(db_session)
         registry = get_current_registry()
 
         @registry.cache_regions.memory_min_1.cache_on_arguments(
-            namespace='ConfigService.by_key_and_section')
+            namespace="ConfigService.by_key_and_section"
+        )
         def cached(key, section):
             query = db_session.query(Config).filter(Config.key == key)
             query = query.filter(Config.section == section)
             config = query.first()
             if config:
                 db_session.expunge(config)
             return config
 
         config = cached(key, section)
         if config:
             config = db_session.merge(config, load=False)
         if config is None and auto_create:
-            config = ConfigService.create_config(key, section,
-                                                 value=default_value)
+            config = ConfigService.create_config(key, section, value=default_value)
             cached.invalidate(key, section)
         return config
 
     @classmethod
     def setup_default_values(self):
         """
         Will add fresh default config values to database if no keys are found
         :return:
         """
-        log.info('Checking/setting default values')
-        self.by_key_and_section('template_footer_html', 'global',
-                                default_value='', auto_create=True)
-        self.by_key_and_section('list_groups_to_non_admins', 'global',
-                                default_value=True, auto_create=True)
-        self.by_key_and_section('per_application_reports_rate_limit', 'global',
-                                default_value=2000, auto_create=True)
-        self.by_key_and_section('per_application_logs_rate_limit', 'global',
-                                default_value=100000, auto_create=True)
-        self.by_key_and_section('per_application_metrics_rate_limit', 'global',
-                                default_value=100000, auto_create=True)
+        log.info("Checking/setting default values")
+        self.by_key_and_section(
+            "template_footer_html", "global", default_value="", auto_create=True
+        )
+        self.by_key_and_section(
+            "list_groups_to_non_admins", "global", default_value=True, auto_create=True
+        )
+        self.by_key_and_section(
+            "per_application_reports_rate_limit",
+            "global",
+            default_value=2000,
+            auto_create=True,
+        )
+        self.by_key_and_section(
+            "per_application_logs_rate_limit",
+            "global",
+            default_value=100000,
+            auto_create=True,
+        )
+        self.by_key_and_section(
+            "per_application_metrics_rate_limit",
+            "global",
+            default_value=100000,
+            auto_create=True,
+        )

backend/src/appenlight/models/services/event.py

@@ -1,109 +1,136 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import sqlalchemy as sa
 from pyramid.threadlocal import get_current_registry
 from paginate_sqlalchemy import SqlalchemyOrmPage
 from ziggurat_foundations.models.services.user import UserService
 
 from appenlight.models import get_db_session
 from appenlight.models.event import Event
 from appenlight.models.services.base import BaseService
 
 
 class EventService(BaseService):
     @classmethod
-    def for_resource(cls, resource_ids, event_type=None, status=None,
-                     since_when=None, limit=20, event_id=None,
-                     target_uuid=None, order_by=None, or_target_user_id=None,
-                     db_session=None):
+    def for_resource(
+        cls,
+        resource_ids,
+        event_type=None,
+        status=None,
+        since_when=None,
+        limit=20,
+        event_id=None,
+        target_uuid=None,
+        order_by=None,
+        or_target_user_id=None,
+        db_session=None,
+    ):
         """
         Fetches events including based on passed params OR if target_user_id
         is present include events that just target this user
         """
         db_session = get_db_session(db_session)
         query = db_session.query(Event)
         query = query.options(sa.orm.joinedload(Event.resource))
         and_cond = [Event.resource_id.in_(resource_ids)]
         if not resource_ids:
             and_cond = [Event.resource_id == -999]
 
         if event_type:
             and_cond.append(Event.event_type == event_type)
         if status:
             and_cond.append(Event.status == status)
         if since_when:
             and_cond.append(Event.start_date >= since_when)
         if event_id:
             and_cond.append(Event.id == event_id)
         if target_uuid:
             and_cond.append(Event.target_uuid == target_uuid)
 
         or_cond = []
 
         if or_target_user_id:
             or_cond.append(sa.or_(Event.target_user_id == or_target_user_id))
 
-        query = query.filter(sa.or_(sa.and_(*and_cond),
-                                    *or_cond))
+        query = query.filter(sa.or_(sa.and_(*and_cond), *or_cond))
         if not order_by:
             query = query.order_by(sa.desc(Event.start_date))
         if limit:
             query = query.limit(limit)
 
         return query
 
     @classmethod
-    def by_type_and_status(cls, event_types, status_types, since_when=None,
-                           older_than=None, db_session=None, app_ids=None):
+    def by_type_and_status(
+        cls,
+        event_types,
+        status_types,
+        since_when=None,
+        older_than=None,
+        db_session=None,
+        app_ids=None,
+    ):
         db_session = get_db_session(db_session)
         query = db_session.query(Event)
         query = query.filter(Event.event_type.in_(event_types))
         query = query.filter(Event.status.in_(status_types))
         if since_when:
             query = query.filter(Event.start_date >= since_when)
         if older_than:
             query = query.filter(Event.start_date <= older_than)
         if app_ids:
             query = query.filter(Event.resource_id.in_(app_ids))
         return query
 
     @classmethod
     def latest_for_user(cls, user, db_session=None):
         registry = get_current_registry()
-        resources = UserService.resources_with_perms(user, ['view'], resource_types=registry.resource_types)
+        resources = UserService.resources_with_perms(
+            user, ["view"], resource_types=registry.resource_types
+        )
         resource_ids = [r.resource_id for r in resources]
         db_session = get_db_session(db_session)
         return EventService.for_resource(
-            resource_ids, or_target_user_id=user.id, limit=10,
-            db_session=db_session)
+            resource_ids, or_target_user_id=user.id, limit=10, db_session=db_session
+        )
 
     @classmethod
-    def get_paginator(cls, user, page=1, item_count=None, items_per_page=50,
-                      order_by=None, filter_settings=None, db_session=None):
+    def get_paginator(
+        cls,
+        user,
+        page=1,
+        item_count=None,
+        items_per_page=50,
+        order_by=None,
+        filter_settings=None,
+        db_session=None,
+    ):
         if not filter_settings:
             filter_settings = {}
         registry = get_current_registry()
-        resources = UserService.resources_with_perms(user, ['view'], resource_types=registry.resource_types)
+        resources = UserService.resources_with_perms(
+            user, ["view"], resource_types=registry.resource_types
+        )
         resource_ids = [r.resource_id for r in resources]
         query = EventService.for_resource(
-            resource_ids, or_target_user_id=user.id, limit=100,
-            db_session=db_session)
+            resource_ids, or_target_user_id=user.id, limit=100, db_session=db_session
+        )
 
-        paginator = SqlalchemyOrmPage(query, page=page,
-                                      items_per_page=items_per_page,
-                                      **filter_settings)
+        paginator = SqlalchemyOrmPage(
+            query, page=page, items_per_page=items_per_page, **filter_settings
+        )
         return paginator

backend/src/appenlight/models/services/group_resource_permission.py

@@ -1,33 +1,35 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from appenlight.models.group_resource_permission import GroupResourcePermission
 from appenlight.models import get_db_session
-from ziggurat_foundations.models.services.group_resource_permission import GroupResourcePermissionService
+from ziggurat_foundations.models.services.group_resource_permission import (
+    GroupResourcePermissionService,
+)
 
 
 class GroupResourcePermissionService(GroupResourcePermissionService):
     @classmethod
-    def by_resource_group_and_perm(cls, group_id, perm_name, resource_id,
-                                   db_session=None):
+    def by_resource_group_and_perm(
+        cls, group_id, perm_name, resource_id, db_session=None
+    ):
         """ return all instances by user name, perm name and resource id """
         db_session = get_db_session(db_session)
         query = db_session.query(GroupResourcePermission)
         query = query.filter(GroupResourcePermission.group_id == group_id)
-        query = query.filter(
-            GroupResourcePermission.resource_id == resource_id)
+        query = query.filter(GroupResourcePermission.resource_id == resource_id)
         query = query.filter(GroupResourcePermission.perm_name == perm_name)
         return query.first()

backend/src/appenlight/models/services/log.py

@@ -1,211 +1,222 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import paginate
 import logging
 import sqlalchemy as sa
 
 from appenlight.models.log import Log
 from appenlight.models import get_db_session, Datastores
 from appenlight.models.services.base import BaseService
 from appenlight.lib.utils import es_index_name_limiter
 
 log = logging.getLogger(__name__)
 
 
 class LogService(BaseService):
     @classmethod
-    def get_logs(cls, resource_ids=None, filter_settings=None,
-                 db_session=None):
+    def get_logs(cls, resource_ids=None, filter_settings=None, db_session=None):
         # ensure we always have id's passed
         if not resource_ids:
             # raise Exception('No App ID passed')
             return []
         db_session = get_db_session(db_session)
         q = db_session.query(Log)
         q = q.filter(Log.resource_id.in_(resource_ids))
-        if filter_settings.get('start_date'):
-            q = q.filter(Log.timestamp >= filter_settings.get('start_date'))
-        if filter_settings.get('end_date'):
-            q = q.filter(Log.timestamp <= filter_settings.get('end_date'))
-        if filter_settings.get('log_level'):
-            q = q.filter(
-                Log.log_level == filter_settings.get('log_level').upper())
-        if filter_settings.get('request_id'):
-            request_id = filter_settings.get('request_id', '')
-            q = q.filter(Log.request_id == request_id.replace('-', ''))
-        if filter_settings.get('namespace'):
-            q = q.filter(Log.namespace == filter_settings.get('namespace'))
+        if filter_settings.get("start_date"):
+            q = q.filter(Log.timestamp >= filter_settings.get("start_date"))
+        if filter_settings.get("end_date"):
+            q = q.filter(Log.timestamp <= filter_settings.get("end_date"))
+        if filter_settings.get("log_level"):
+            q = q.filter(Log.log_level == filter_settings.get("log_level").upper())
+        if filter_settings.get("request_id"):
+            request_id = filter_settings.get("request_id", "")
+            q = q.filter(Log.request_id == request_id.replace("-", ""))
+        if filter_settings.get("namespace"):
+            q = q.filter(Log.namespace == filter_settings.get("namespace"))
         q = q.order_by(sa.desc(Log.timestamp))
         return q
 
     @classmethod
     def es_query_builder(cls, app_ids, filter_settings):
         if not filter_settings:
             filter_settings = {}
 
         query = {
             "query": {
                 "filtered": {
-                    "filter": {
-                        "and": [{"terms": {"resource_id": list(app_ids)}}]
-                    }
+                    "filter": {"and": [{"terms": {"resource_id": list(app_ids)}}]}
                 }
             }
         }
 
-        start_date = filter_settings.get('start_date')
-        end_date = filter_settings.get('end_date')
-        filter_part = query['query']['filtered']['filter']['and']
+        start_date = filter_settings.get("start_date")
+        end_date = filter_settings.get("end_date")
+        filter_part = query["query"]["filtered"]["filter"]["and"]
 
-        for tag in filter_settings.get('tags', []):
-            tag_values = [v.lower() for v in tag['value']]
-            key = "tags.%s.values" % tag['name'].replace('.', '_')
+        for tag in filter_settings.get("tags", []):
+            tag_values = [v.lower() for v in tag["value"]]
+            key = "tags.%s.values" % tag["name"].replace(".", "_")
             filter_part.append({"terms": {key: tag_values}})
 
         date_range = {"range": {"timestamp": {}}}
         if start_date:
             date_range["range"]["timestamp"]["gte"] = start_date
         if end_date:
             date_range["range"]["timestamp"]["lte"] = end_date
         if start_date or end_date:
             filter_part.append(date_range)
 
-        levels = filter_settings.get('level')
+        levels = filter_settings.get("level")
         if levels:
-            filter_part.append({"terms": {'log_level': levels}})
-        namespaces = filter_settings.get('namespace')
+            filter_part.append({"terms": {"log_level": levels}})
+        namespaces = filter_settings.get("namespace")
         if namespaces:
-            filter_part.append({"terms": {'namespace': namespaces}})
+            filter_part.append({"terms": {"namespace": namespaces}})
 
-        request_ids = filter_settings.get('request_id')
+        request_ids = filter_settings.get("request_id")
         if request_ids:
-            filter_part.append({"terms": {'request_id': request_ids}})
+            filter_part.append({"terms": {"request_id": request_ids}})
 
-        messages = filter_settings.get('message')
+        messages = filter_settings.get("message")
         if messages:
-            query['query']['filtered']['query'] = {
-                'match': {
-                    'message': {
-                        'query': ' '.join(messages),
-                        'operator': 'and'
-                    }
-                }
+            query["query"]["filtered"]["query"] = {
+                "match": {"message": {"query": " ".join(messages), "operator": "and"}}
             }
         return query
 
     @classmethod
     def get_time_series_aggregate(cls, app_ids=None, filter_settings=None):
         if not app_ids:
             return {}
         es_query = cls.es_query_builder(app_ids, filter_settings)
         es_query["aggs"] = {
             "events_over_time": {
                 "date_histogram": {
                     "field": "timestamp",
                     "interval": "1h",
                     "min_doc_count": 0,
-                    'extended_bounds': {
-                        'max': filter_settings.get('end_date'),
-                        'min': filter_settings.get('start_date')}
+                    "extended_bounds": {
+                        "max": filter_settings.get("end_date"),
+                        "min": filter_settings.get("start_date"),
+                    },
                 }
             }
         }
         log.debug(es_query)
-        index_names = es_index_name_limiter(filter_settings.get('start_date'),
-                                            filter_settings.get('end_date'),
-                                            ixtypes=['logs'])
+        index_names = es_index_name_limiter(
+            filter_settings.get("start_date"),
+            filter_settings.get("end_date"),
+            ixtypes=["logs"],
+        )
         if index_names:
             results = Datastores.es.search(
-                body=es_query, index=index_names, doc_type='log', size=0)
+                body=es_query, index=index_names, doc_type="log", size=0
+            )
         else:
             results = []
         return results
 
     @classmethod
-    def get_search_iterator(cls, app_ids=None, page=1, items_per_page=50,
-                            order_by=None, filter_settings=None, limit=None):
+    def get_search_iterator(
+        cls,
+        app_ids=None,
+        page=1,
+        items_per_page=50,
+        order_by=None,
+        filter_settings=None,
+        limit=None,
+    ):
         if not app_ids:
             return {}, 0
 
         es_query = cls.es_query_builder(app_ids, filter_settings)
-        sort_query = {
-            "sort": [
-                {"timestamp": {"order": "desc"}}
-            ]
-        }
+        sort_query = {"sort": [{"timestamp": {"order": "desc"}}]}
         es_query.update(sort_query)
         log.debug(es_query)
         es_from = (page - 1) * items_per_page
-        index_names = es_index_name_limiter(filter_settings.get('start_date'),
-                                            filter_settings.get('end_date'),
-                                            ixtypes=['logs'])
+        index_names = es_index_name_limiter(
+            filter_settings.get("start_date"),
+            filter_settings.get("end_date"),
+            ixtypes=["logs"],
+        )
         if not index_names:
             return {}, 0
 
-        results = Datastores.es.search(body=es_query, index=index_names,
-                                       doc_type='log', size=items_per_page,
-                                       from_=es_from)
-        if results['hits']['total'] > 5000:
+        results = Datastores.es.search(
+            body=es_query,
+            index=index_names,
+            doc_type="log",
+            size=items_per_page,
+            from_=es_from,
+        )
+        if results["hits"]["total"] > 5000:
             count = 5000
         else:
-            count = results['hits']['total']
-        return results['hits'], count
+            count = results["hits"]["total"]
+        return results["hits"], count
 
     @classmethod
-    def get_paginator_by_app_ids(cls, app_ids=None, page=1, item_count=None,
-                                 items_per_page=50, order_by=None,
-                                 filter_settings=None,
-                                 exclude_columns=None, db_session=None):
+    def get_paginator_by_app_ids(
+        cls,
+        app_ids=None,
+        page=1,
+        item_count=None,
+        items_per_page=50,
+        order_by=None,
+        filter_settings=None,
+        exclude_columns=None,
+        db_session=None,
+    ):
         if not filter_settings:
             filter_settings = {}
-        results, item_count = cls.get_search_iterator(app_ids, page,
-                                                      items_per_page, order_by,
-                                                      filter_settings)
-        paginator = paginate.Page([],
-                                  item_count=item_count,
-                                  items_per_page=items_per_page,
-                                  **filter_settings)
-        ordered_ids = tuple(item['_source']['pg_id']
-                            for item in results.get('hits', []))
+        results, item_count = cls.get_search_iterator(
+            app_ids, page, items_per_page, order_by, filter_settings
+        )
+        paginator = paginate.Page(
+            [], item_count=item_count, items_per_page=items_per_page, **filter_settings
+        )
+        ordered_ids = tuple(
+            item["_source"]["pg_id"] for item in results.get("hits", [])
+        )
 
         sorted_instance_list = []
         if ordered_ids:
             db_session = get_db_session(db_session)
             query = db_session.query(Log)
             query = query.filter(Log.log_id.in_(ordered_ids))
-            query = query.order_by(sa.desc('timestamp'))
+            query = query.order_by(sa.desc("timestamp"))
             sa_items = query.all()
             # resort by score
             for i_id in ordered_ids:
                 for item in sa_items:
                     if str(item.log_id) == str(i_id):
                         sorted_instance_list.append(item)
         paginator.sa_items = sorted_instance_list
         return paginator
 
     @classmethod
-    def query_by_primary_key_and_namespace(cls, list_of_pairs,
-                                           db_session=None):
+    def query_by_primary_key_and_namespace(cls, list_of_pairs, db_session=None):
         db_session = get_db_session(db_session)
         list_of_conditions = []
         query = db_session.query(Log)
         for pair in list_of_pairs:
-            list_of_conditions.append(sa.and_(
-                Log.primary_key == pair['pk'], Log.namespace == pair['ns']))
+            list_of_conditions.append(
+                sa.and_(Log.primary_key == pair["pk"], Log.namespace == pair["ns"])
+            )
         query = query.filter(sa.or_(*list_of_conditions))
         query = query.order_by(sa.asc(Log.timestamp), sa.asc(Log.log_id))
         return query

backend/src/appenlight/models/services/plugin_config.py

@@ -1,52 +1,53 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 
 from appenlight.models.plugin_config import PluginConfig
 from appenlight.models.services.base import BaseService
 from appenlight.models import get_db_session
 
 log = logging.getLogger(__name__)
 
 
 class PluginConfigService(BaseService):
     @classmethod
     def all(cls, db_session=None):
         db_session = get_db_session(db_session)
         query = db_session.query(PluginConfig)
         return query
 
     @classmethod
     def by_id(cls, plugin_id, db_session=None):
         db_session = get_db_session(db_session)
         query = db_session.query(PluginConfig)
         query = query.filter(PluginConfig.id == plugin_id)
         return query.first()
 
     @classmethod
-    def by_query(cls, resource_id=None, plugin_name=None,
-                 section=None, db_session=None):
+    def by_query(
+        cls, resource_id=None, plugin_name=None, section=None, db_session=None
+    ):
         db_session = get_db_session(db_session)
 
         query = db_session.query(PluginConfig)
         if resource_id:
             query = query.filter(PluginConfig.resource_id == resource_id)
         if plugin_name:
             query = query.filter(PluginConfig.plugin_name == plugin_name)
         if section:
             query = query.filter(PluginConfig.section == section)
         return query

backend/src/appenlight/models/services/report.py

@@ -1,57 +1,60 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 import sqlalchemy as sa
 
 from appenlight.models import get_db_session
 from appenlight.models.report import Report
 from appenlight.models.report_stat import ReportStat
 from appenlight.models.services.base import BaseService
 
 log = logging.getLogger(__name__)
 
 
 class ReportService(BaseService):
     @classmethod
     def by_app_ids(cls, app_ids=None, order_by=True, db_session=None):
         db_session = get_db_session(db_session)
         q = db_session.query(Report)
         if app_ids:
             q = q.filter(Report.resource_id.in_(app_ids))
         if order_by:
             q = q.order_by(sa.desc(Report.id))
         return q
 
     @classmethod
-    def generate_stat_rows(cls, report, resource, report_group, occurences=1,
-                           db_session=None):
+    def generate_stat_rows(
+        cls, report, resource, report_group, occurences=1, db_session=None
+    ):
         """
         Generates timeseries for this report's group
         """
         db_session = get_db_session(db_session)
-        stats = ReportStat(resource_id=report.resource_id,
-                           group_id=report_group.id,
-                           start_interval=report.start_time,
-                           owner_user_id=resource.owner_user_id,
-                           server_name=report.tags.get('server_name'),
-                           view_name=report.tags.get('view_name'),
-                           type=report.report_type,
-                           occurences=occurences,
-                           duration=report.duration)
+        stats = ReportStat(
+            resource_id=report.resource_id,
+            group_id=report_group.id,
+            start_interval=report.start_time,
+            owner_user_id=resource.owner_user_id,
+            server_name=report.tags.get("server_name"),
+            view_name=report.tags.get("view_name"),
+            type=report.report_type,
+            occurences=occurences,
+            duration=report.duration,
+        )
         db_session.add(stats)
         db_session.flush()
         return stats

backend/src/appenlight/models/services/report_group.py

@@ -1,451 +1,521 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 import paginate
 import sqlalchemy as sa
 import appenlight.lib.helpers as h
 
 from datetime import datetime
 
 from appenlight.models import get_db_session, Datastores
 from appenlight.models.report import Report
 from appenlight.models.report_group import ReportGroup
 from appenlight.models.report_comment import ReportComment
 from appenlight.models.user import User
 from appenlight.models.services.base import BaseService
 from appenlight.lib.enums import ReportType
 from appenlight.lib.utils import es_index_name_limiter
 
 log = logging.getLogger(__name__)
 
 
 class ReportGroupService(BaseService):
     @classmethod
-    def get_trending(cls, request, filter_settings, limit=15,
-                     db_session=None):
+    def get_trending(cls, request, filter_settings, limit=15, db_session=None):
         """
         Returns report groups trending for specific time interval
         """
         db_session = get_db_session(db_session)
 
         tags = []
-        if filter_settings.get('tags'):
-            for tag in filter_settings['tags']:
+        if filter_settings.get("tags"):
+            for tag in filter_settings["tags"]:
                 tags.append(
-                    {'terms': {
-                        'tags.{}.values'.format(tag['name']): tag['value']}})
+                    {"terms": {"tags.{}.values".format(tag["name"]): tag["value"]}}
+                )
 
         index_names = es_index_name_limiter(
-            start_date=filter_settings['start_date'],
-            end_date=filter_settings['end_date'],
-            ixtypes=['reports'])
+            start_date=filter_settings["start_date"],
+            end_date=filter_settings["end_date"],
+            ixtypes=["reports"],
+        )
 
-        if not index_names or not filter_settings['resource']:
+        if not index_names or not filter_settings["resource"]:
             return []
 
         es_query = {
-            'aggs': {'parent_agg': {'aggs': {'groups': {'aggs': {
-                'sub_agg': {
-                    'value_count': {'field': 'tags.group_id.values'}}},
-                'filter': {'exists': {'field': 'tags.group_id.values'}}}},
-                'terms': {'field': 'tags.group_id.values', 'size': limit}}},
-            'query': {'filtered': {
-                'filter': {'and': [
-                    {'terms': {
-                        'resource_id': [filter_settings['resource'][0]]}
+            "aggs": {
+                "parent_agg": {
+                    "aggs": {
+                        "groups": {
+                            "aggs": {
+                                "sub_agg": {
+                                    "value_count": {"field": "tags.group_id.values"}
+                                }
+                            },
+                            "filter": {"exists": {"field": "tags.group_id.values"}},
+                        }
                     },
-                    {'range': {'timestamp': {
-                        'gte': filter_settings['start_date'],
-                        'lte': filter_settings['end_date']}}}]
+                    "terms": {"field": "tags.group_id.values", "size": limit},
                 }
-            }}
+            },
+            "query": {
+                "filtered": {
+                    "filter": {
+                        "and": [
+                            {
+                                "terms": {
+                                    "resource_id": [filter_settings["resource"][0]]
+                                }
+                            },
+                            {
+                                "range": {
+                                    "timestamp": {
+                                        "gte": filter_settings["start_date"],
+                                        "lte": filter_settings["end_date"],
+                                    }
+                                }
+                            },
+                        ]
+                    }
+                }
+            },
         }
         if tags:
-            es_query['query']['filtered']['filter']['and'].extend(tags)
+            es_query["query"]["filtered"]["filter"]["and"].extend(tags)
 
         result = Datastores.es.search(
-            body=es_query, index=index_names, doc_type='log', size=0)
+            body=es_query, index=index_names, doc_type="log", size=0
+        )
         series = []
-        for bucket in result['aggregations']['parent_agg']['buckets']:
-            series.append({
-                'key': bucket['key'],
-                'groups': bucket['groups']['sub_agg']['value']
-            })
+        for bucket in result["aggregations"]["parent_agg"]["buckets"]:
+            series.append(
+                {"key": bucket["key"], "groups": bucket["groups"]["sub_agg"]["value"]}
+            )
 
         report_groups_d = {}
         for g in series:
-            report_groups_d[int(g['key'])] = g['groups'] or 0
+            report_groups_d[int(g["key"])] = g["groups"] or 0
 
         query = db_session.query(ReportGroup)
         query = query.filter(ReportGroup.id.in_(list(report_groups_d.keys())))
-        query = query.options(
-            sa.orm.joinedload(ReportGroup.last_report_ref))
-        results = [(report_groups_d[group.id], group,) for group in query]
-        return sorted(results, reverse=True, key=lambda x:x[0])
+        query = query.options(sa.orm.joinedload(ReportGroup.last_report_ref))
+        results = [(report_groups_d[group.id], group) for group in query]
+        return sorted(results, reverse=True, key=lambda x: x[0])
 
     @classmethod
-    def get_search_iterator(cls, app_ids=None, page=1, items_per_page=50,
-                            order_by=None, filter_settings=None, limit=None):
+    def get_search_iterator(
+        cls,
+        app_ids=None,
+        page=1,
+        items_per_page=50,
+        order_by=None,
+        filter_settings=None,
+        limit=None,
+    ):
         if not app_ids:
             return {}
         if not filter_settings:
             filter_settings = {}
 
         query = {
             "size": 0,
             "query": {
                 "filtered": {
-                    "filter": {
-                        "and": [{"terms": {"resource_id": list(app_ids)}}]
-                    }
+                    "filter": {"and": [{"terms": {"resource_id": list(app_ids)}}]}
                 }
             },
-
             "aggs": {
                 "top_groups": {
                     "terms": {
                         "size": 5000,
                         "field": "_parent",
-                        "order": {
-                            "newest": "desc"
-                        }
+                        "order": {"newest": "desc"},
                     },
                     "aggs": {
                         "top_reports_hits": {
-                            "top_hits": {"size": 1,
-                                         "sort": {"start_time": "desc"}
-                                         }
+                            "top_hits": {"size": 1, "sort": {"start_time": "desc"}}
                         },
-                        "newest": {
-                            "max": {"field": "start_time"}
-                        }
-                    }
+                        "newest": {"max": {"field": "start_time"}},
+                    },
                 }
-            }
+            },
         }
 
-        start_date = filter_settings.get('start_date')
-        end_date = filter_settings.get('end_date')
-        filter_part = query['query']['filtered']['filter']['and']
+        start_date = filter_settings.get("start_date")
+        end_date = filter_settings.get("end_date")
+        filter_part = query["query"]["filtered"]["filter"]["and"]
         date_range = {"range": {"start_time": {}}}
         if start_date:
             date_range["range"]["start_time"]["gte"] = start_date
         if end_date:
             date_range["range"]["start_time"]["lte"] = end_date
         if start_date or end_date:
             filter_part.append(date_range)
 
-        priorities = filter_settings.get('priority')
+        priorities = filter_settings.get("priority")
 
-        for tag in filter_settings.get('tags', []):
-            tag_values = [v.lower() for v in tag['value']]
-            key = "tags.%s.values" % tag['name'].replace('.', '_')
+        for tag in filter_settings.get("tags", []):
+            tag_values = [v.lower() for v in tag["value"]]
+            key = "tags.%s.values" % tag["name"].replace(".", "_")
             filter_part.append({"terms": {key: tag_values}})
 
         if priorities:
-            filter_part.append({"has_parent": {
-                "parent_type": "report_group",
-                "query": {
-                    "terms": {'priority': priorities}
-                }}})
+            filter_part.append(
+                {
+                    "has_parent": {
+                        "parent_type": "report_group",
+                        "query": {"terms": {"priority": priorities}},
+                    }
+                }
+            )
 
-        min_occurences = filter_settings.get('min_occurences')
+        min_occurences = filter_settings.get("min_occurences")
         if min_occurences:
-            filter_part.append({"has_parent": {
-                "parent_type": "report_group",
-                "query": {
-                    "range": {'occurences': {"gte": min_occurences[0]}}
-                }}})
+            filter_part.append(
+                {
+                    "has_parent": {
+                        "parent_type": "report_group",
+                        "query": {"range": {"occurences": {"gte": min_occurences[0]}}},
+                    }
+                }
+            )
 
-        min_duration = filter_settings.get('min_duration')
-        max_duration = filter_settings.get('max_duration')
+        min_duration = filter_settings.get("min_duration")
+        max_duration = filter_settings.get("max_duration")
 
-        request_ids = filter_settings.get('request_id')
+        request_ids = filter_settings.get("request_id")
         if request_ids:
-            filter_part.append({"terms": {'request_id': request_ids}})
+            filter_part.append({"terms": {"request_id": request_ids}})
 
         duration_range = {"range": {"average_duration": {}}}
         if min_duration:
-            duration_range["range"]["average_duration"]["gte"] = \
-                min_duration[0]
+            duration_range["range"]["average_duration"]["gte"] = min_duration[0]
         if max_duration:
-            duration_range["range"]["average_duration"]["lte"] = \
-                max_duration[0]
+            duration_range["range"]["average_duration"]["lte"] = max_duration[0]
         if min_duration or max_duration:
-            filter_part.append({"has_parent": {
-                "parent_type": "report_group",
-                "query": duration_range}})
+            filter_part.append(
+                {"has_parent": {"parent_type": "report_group", "query": duration_range}}
+            )
 
-        http_status = filter_settings.get('http_status')
-        report_type = filter_settings.get('report_type', [ReportType.error])
+        http_status = filter_settings.get("http_status")
+        report_type = filter_settings.get("report_type", [ReportType.error])
         # set error report type if http status is not found
         # and we are dealing with slow reports
         if not http_status or ReportType.slow in report_type:
-            filter_part.append({"terms": {'report_type': report_type}})
+            filter_part.append({"terms": {"report_type": report_type}})
         if http_status:
-            filter_part.append({"terms": {'http_status': http_status}})
+            filter_part.append({"terms": {"http_status": http_status}})
 
-        messages = filter_settings.get('message')
+        messages = filter_settings.get("message")
         if messages:
-            condition = {'match': {"message": ' '.join(messages)}}
-            query['query']['filtered']['query'] = condition
-        errors = filter_settings.get('error')
+            condition = {"match": {"message": " ".join(messages)}}
+            query["query"]["filtered"]["query"] = condition
+        errors = filter_settings.get("error")
         if errors:
-            condition = {'match': {"error": ' '.join(errors)}}
-            query['query']['filtered']['query'] = condition
-        url_domains = filter_settings.get('url_domain')
+            condition = {"match": {"error": " ".join(errors)}}
+            query["query"]["filtered"]["query"] = condition
+        url_domains = filter_settings.get("url_domain")
         if url_domains:
-            condition = {'terms': {"url_domain": url_domains}}
-            query['query']['filtered']['query'] = condition
-        url_paths = filter_settings.get('url_path')
+            condition = {"terms": {"url_domain": url_domains}}
+            query["query"]["filtered"]["query"] = condition
+        url_paths = filter_settings.get("url_path")
         if url_paths:
-            condition = {'terms': {"url_path": url_paths}}
-            query['query']['filtered']['query'] = condition
-
-        if filter_settings.get('report_status'):
-            for status in filter_settings.get('report_status'):
-                if status == 'never_reviewed':
-                    filter_part.append({"has_parent": {
-                        "parent_type": "report_group",
-                        "query": {
-                            "term": {"read": False}
-                        }}})
-                elif status == 'reviewed':
-                    filter_part.append({"has_parent": {
-                        "parent_type": "report_group",
-                        "query": {
-                            "term": {"read": True}
-                        }}})
-                elif status == 'public':
-                    filter_part.append({"has_parent": {
-                        "parent_type": "report_group",
-                        "query": {
-                            "term": {"public": True}
-                        }}})
-                elif status == 'fixed':
-                    filter_part.append({"has_parent": {
-                        "parent_type": "report_group",
-                        "query": {
-                            "term": {"fixed": True}
-                        }}})
+            condition = {"terms": {"url_path": url_paths}}
+            query["query"]["filtered"]["query"] = condition
+
+        if filter_settings.get("report_status"):
+            for status in filter_settings.get("report_status"):
+                if status == "never_reviewed":
+                    filter_part.append(
+                        {
+                            "has_parent": {
+                                "parent_type": "report_group",
+                                "query": {"term": {"read": False}},
+                            }
+                        }
+                    )
+                elif status == "reviewed":
+                    filter_part.append(
+                        {
+                            "has_parent": {
+                                "parent_type": "report_group",
+                                "query": {"term": {"read": True}},
+                            }
+                        }
+                    )
+                elif status == "public":
+                    filter_part.append(
+                        {
+                            "has_parent": {
+                                "parent_type": "report_group",
+                                "query": {"term": {"public": True}},
+                            }
+                        }
+                    )
+                elif status == "fixed":
+                    filter_part.append(
+                        {
+                            "has_parent": {
+                                "parent_type": "report_group",
+                                "query": {"term": {"fixed": True}},
+                            }
+                        }
+                    )
 
         # logging.getLogger('pyelasticsearch').setLevel(logging.DEBUG)
-        index_names = es_index_name_limiter(filter_settings.get('start_date'),
-                                            filter_settings.get('end_date'),
-                                            ixtypes=['reports'])
+        index_names = es_index_name_limiter(
+            filter_settings.get("start_date"),
+            filter_settings.get("end_date"),
+            ixtypes=["reports"],
+        )
         if index_names:
             results = Datastores.es.search(
-                body=query, index=index_names, doc_type=["report", "report_group"],
-                size=0)
+                body=query,
+                index=index_names,
+                doc_type=["report", "report_group"],
+                size=0,
+            )
         else:
             return []
-        return results['aggregations']
+        return results["aggregations"]
 
     @classmethod
-    def get_paginator_by_app_ids(cls, app_ids=None, page=1, item_count=None,
-                                 items_per_page=50, order_by=None,
-                                 filter_settings=None,
-                                 exclude_columns=None, db_session=None):
+    def get_paginator_by_app_ids(
+        cls,
+        app_ids=None,
+        page=1,
+        item_count=None,
+        items_per_page=50,
+        order_by=None,
+        filter_settings=None,
+        exclude_columns=None,
+        db_session=None,
+    ):
         if not filter_settings:
             filter_settings = {}
-        results = cls.get_search_iterator(app_ids, page, items_per_page,
-                                          order_by, filter_settings)
+        results = cls.get_search_iterator(
+            app_ids, page, items_per_page, order_by, filter_settings
+        )
 
         ordered_ids = []
         if results:
-            for item in results['top_groups']['buckets']:
-                pg_id = item['top_reports_hits']['hits']['hits'][0]['_source'][
-                    'pg_id']
+            for item in results["top_groups"]["buckets"]:
+                pg_id = item["top_reports_hits"]["hits"]["hits"][0]["_source"]["pg_id"]
                 ordered_ids.append(pg_id)
         log.info(filter_settings)
-        paginator = paginate.Page(ordered_ids, items_per_page=items_per_page,
-                                  **filter_settings)
+        paginator = paginate.Page(
+            ordered_ids, items_per_page=items_per_page, **filter_settings
+        )
         sa_items = ()
         if paginator.items:
             db_session = get_db_session(db_session)
             # latest report detail
             query = db_session.query(Report)
             query = query.options(sa.orm.joinedload(Report.report_group))
             query = query.filter(Report.id.in_(paginator.items))
-            if filter_settings.get('order_col'):
-                order_col = filter_settings.get('order_col')
-                if filter_settings.get('order_dir') == 'dsc':
-                    sort_on = 'desc'
+            if filter_settings.get("order_col"):
+                order_col = filter_settings.get("order_col")
+                if filter_settings.get("order_dir") == "dsc":
+                    sort_on = "desc"
                 else:
-                    sort_on = 'asc'
-                if order_col == 'when':
-                    order_col = 'last_timestamp'
-                query = query.order_by(getattr(sa, sort_on)(
-                    getattr(ReportGroup, order_col)))
+                    sort_on = "asc"
+                if order_col == "when":
+                    order_col = "last_timestamp"
+                query = query.order_by(
+                    getattr(sa, sort_on)(getattr(ReportGroup, order_col))
+                )
             sa_items = query.all()
         sorted_instance_list = []
         for i_id in ordered_ids:
             for report in sa_items:
-                if (str(report.id) == i_id and
-                            report not in sorted_instance_list):
+                if str(report.id) == i_id and report not in sorted_instance_list:
                     sorted_instance_list.append(report)
         paginator.sa_items = sorted_instance_list
         return paginator
 
     @classmethod
     def by_app_ids(cls, app_ids=None, order_by=True, db_session=None):
         db_session = get_db_session(db_session)
         q = db_session.query(ReportGroup)
         if app_ids:
             q = q.filter(ReportGroup.resource_id.in_(app_ids))
         if order_by:
             q = q.order_by(sa.desc(ReportGroup.id))
         return q
 
     @classmethod
     def by_id(cls, group_id, app_ids=None, db_session=None):
         db_session = get_db_session(db_session)
-        q = db_session.query(ReportGroup).filter(
-            ReportGroup.id == int(group_id))
+        q = db_session.query(ReportGroup).filter(ReportGroup.id == int(group_id))
         if app_ids:
             q = q.filter(ReportGroup.resource_id.in_(app_ids))
         return q.first()
 
     @classmethod
     def by_ids(cls, group_ids=None, db_session=None):
         db_session = get_db_session(db_session)
         query = db_session.query(ReportGroup)
         query = query.filter(ReportGroup.id.in_(group_ids))
         return query
 
     @classmethod
-    def by_hash_and_resource(cls, resource_id, grouping_hash, since_when=None,
-                             db_session=None):
+    def by_hash_and_resource(
+        cls, resource_id, grouping_hash, since_when=None, db_session=None
+    ):
         db_session = get_db_session(db_session)
         q = db_session.query(ReportGroup)
         q = q.filter(ReportGroup.resource_id == resource_id)
         q = q.filter(ReportGroup.grouping_hash == grouping_hash)
         q = q.filter(ReportGroup.fixed == False)
         if since_when:
             q = q.filter(ReportGroup.first_timestamp >= since_when)
         return q.first()
 
     @classmethod
-    def users_commenting(cls, report_group, exclude_user_id=None,
-                         db_session=None):
+    def users_commenting(cls, report_group, exclude_user_id=None, db_session=None):
         db_session = get_db_session(None, report_group)
         query = db_session.query(User).distinct()
         query = query.filter(User.id == ReportComment.owner_id)
         query = query.filter(ReportComment.group_id == report_group.id)
         if exclude_user_id:
             query = query.filter(ReportComment.owner_id != exclude_user_id)
         return query
 
     @classmethod
     def affected_users_count(cls, report_group, db_session=None):
         db_session = get_db_session(db_session)
         query = db_session.query(sa.func.count(Report.username))
         query = query.filter(Report.group_id == report_group.id)
-        query = query.filter(Report.username != '')
+        query = query.filter(Report.username != "")
         query = query.filter(Report.username != None)
         query = query.group_by(Report.username)
         return query.count()
 
     @classmethod
     def top_affected_users(cls, report_group, db_session=None):
         db_session = get_db_session(db_session)
-        count_label = sa.func.count(Report.username).label('count')
+        count_label = sa.func.count(Report.username).label("count")
         query = db_session.query(Report.username, count_label)
         query = query.filter(Report.group_id == report_group.id)
         query = query.filter(Report.username != None)
-        query = query.filter(Report.username != '')
+        query = query.filter(Report.username != "")
         query = query.group_by(Report.username)
         query = query.order_by(sa.desc(count_label))
         query = query.limit(50)
         return query
 
     @classmethod
     def get_report_stats(cls, request, filter_settings):
         """
         Gets report dashboard graphs
         Returns information for BAR charts with occurences/interval information
         detailed means version that returns time intervals - non detailed
         returns total sum
         """
-        delta = filter_settings['end_date'] - filter_settings['start_date']
-        if delta < h.time_deltas.get('12h')['delta']:
-            interval = '1m'
-        elif delta <= h.time_deltas.get('3d')['delta']:
-            interval = '5m'
-        elif delta >= h.time_deltas.get('2w')['delta']:
-            interval = '24h'
+        delta = filter_settings["end_date"] - filter_settings["start_date"]
+        if delta < h.time_deltas.get("12h")["delta"]:
+            interval = "1m"
+        elif delta <= h.time_deltas.get("3d")["delta"]:
+            interval = "5m"
+        elif delta >= h.time_deltas.get("2w")["delta"]:
+            interval = "24h"
         else:
-            interval = '1h'
+            interval = "1h"
 
-        group_id = filter_settings.get('group_id')
+        group_id = filter_settings.get("group_id")
 
         es_query = {
-            'aggs': {'parent_agg': {'aggs': {'types': {
-                'aggs': {'sub_agg': {'terms': {'field': 'tags.type.values'}}},
-                'filter': {
-                    'and': [{'exists': {'field': 'tags.type.values'}}]}
-            }},
-                'date_histogram': {'extended_bounds': {
-                    'max': filter_settings['end_date'],
-                    'min': filter_settings['start_date']},
-                    'field': 'timestamp',
-                    'interval': interval,
-                    'min_doc_count': 0}}},
-            'query': {'filtered': {
-                'filter': {'and': [
-                    {'terms': {
-                        'resource_id': [filter_settings['resource'][0]]}},
-                    {'range': {'timestamp': {
-                        'gte': filter_settings['start_date'],
-                        'lte': filter_settings['end_date']}}}]
+            "aggs": {
+                "parent_agg": {
+                    "aggs": {
+                        "types": {
+                            "aggs": {
+                                "sub_agg": {"terms": {"field": "tags.type.values"}}
+                            },
+                            "filter": {
+                                "and": [{"exists": {"field": "tags.type.values"}}]
+                            },
+                        }
+                    },
+                    "date_histogram": {
+                        "extended_bounds": {
+                            "max": filter_settings["end_date"],
+                            "min": filter_settings["start_date"],
+                        },
+                        "field": "timestamp",
+                        "interval": interval,
+                        "min_doc_count": 0,
+                    },
+                }
+            },
+            "query": {
+                "filtered": {
+                    "filter": {
+                        "and": [
+                            {
+                                "terms": {
+                                    "resource_id": [filter_settings["resource"][0]]
+                                }
+                            },
+                            {
+                                "range": {
+                                    "timestamp": {
+                                        "gte": filter_settings["start_date"],
+                                        "lte": filter_settings["end_date"],
+                                    }
+                                }
+                            },
+                        ]
+                    }
                 }
-            }}
+            },
         }
         if group_id:
-            parent_agg = es_query['aggs']['parent_agg']
-            filters = parent_agg['aggs']['types']['filter']['and']
-            filters.append({'terms': {'tags.group_id.values': [group_id]}})
+            parent_agg = es_query["aggs"]["parent_agg"]
+            filters = parent_agg["aggs"]["types"]["filter"]["and"]
+            filters.append({"terms": {"tags.group_id.values": [group_id]}})
 
         index_names = es_index_name_limiter(
-            start_date=filter_settings['start_date'],
-            end_date=filter_settings['end_date'],
-            ixtypes=['reports'])
+            start_date=filter_settings["start_date"],
+            end_date=filter_settings["end_date"],
+            ixtypes=["reports"],
+        )
 
         if not index_names:
             return []
 
-        result = Datastores.es.search(body=es_query,
-                                      index=index_names,
-                                      doc_type='log',
-                                      size=0)
+        result = Datastores.es.search(
+            body=es_query, index=index_names, doc_type="log", size=0
+        )
         series = []
-        for bucket in result['aggregations']['parent_agg']['buckets']:
+        for bucket in result["aggregations"]["parent_agg"]["buckets"]:
             point = {
-                'x': datetime.utcfromtimestamp(int(bucket['key']) / 1000),
-                'report': 0,
-                'not_found': 0,
-                'slow_report': 0
+                "x": datetime.utcfromtimestamp(int(bucket["key"]) / 1000),
+                "report": 0,
+                "not_found": 0,
+                "slow_report": 0,
             }
-            for subbucket in bucket['types']['sub_agg']['buckets']:
-                if subbucket['key'] == 'slow':
-                    point['slow_report'] = subbucket['doc_count']
-                elif subbucket['key'] == 'error':
-                    point['report'] = subbucket['doc_count']
-                elif subbucket['key'] == 'not_found':
-                    point['not_found'] = subbucket['doc_count']
+            for subbucket in bucket["types"]["sub_agg"]["buckets"]:
+                if subbucket["key"] == "slow":
+                    point["slow_report"] = subbucket["doc_count"]
+                elif subbucket["key"] == "error":
+                    point["report"] = subbucket["doc_count"]
+                elif subbucket["key"] == "not_found":
+                    point["not_found"] = subbucket["doc_count"]
             series.append(point)
         return series

backend/src/appenlight/models/services/report_stat.py

@@ -1,50 +1,63 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from appenlight.models import Datastores
 from appenlight.models.services.base import BaseService
 from appenlight.lib.enums import ReportType
 from appenlight.lib.utils import es_index_name_limiter
 
 
 class ReportStatService(BaseService):
     @classmethod
     def count_by_type(cls, report_type, resource_id, since_when):
         report_type = ReportType.key_from_value(report_type)
 
-        index_names = es_index_name_limiter(start_date=since_when,
-                              ixtypes=['reports'])
+        index_names = es_index_name_limiter(start_date=since_when, ixtypes=["reports"])
 
         es_query = {
-            'aggs': {'reports': {'aggs': {
-                'sub_agg': {'value_count': {'field': 'tags.group_id.values'}}},
-                'filter': {'and': [{'terms': {'resource_id': [resource_id]}},
-                                   {'exists': {
-                                       'field': 'tags.group_id.values'}}]}}},
-            'query': {'filtered': {'filter': {
-                'and': [{'terms': {'resource_id': [resource_id]}},
-                        {'terms': {'tags.type.values': [report_type]}},
-                        {'range': {'timestamp': {
-                            'gte': since_when}}}]}}}}
+            "aggs": {
+                "reports": {
+                    "aggs": {
+                        "sub_agg": {"value_count": {"field": "tags.group_id.values"}}
+                    },
+                    "filter": {
+                        "and": [
+                            {"terms": {"resource_id": [resource_id]}},
+                            {"exists": {"field": "tags.group_id.values"}},
+                        ]
+                    },
+                }
+            },
+            "query": {
+                "filtered": {
+                    "filter": {
+                        "and": [
+                            {"terms": {"resource_id": [resource_id]}},
+                            {"terms": {"tags.type.values": [report_type]}},
+                            {"range": {"timestamp": {"gte": since_when}}},
+                        ]
+                    }
+                }
+            },
+        }
 
         if index_names:
-            result = Datastores.es.search(body=es_query,
-                                      index=index_names,
-                                      doc_type='log',
-                                      size=0)
-            return result['aggregations']['reports']['sub_agg']['value']
+            result = Datastores.es.search(
+                body=es_query, index=index_names, doc_type="log", size=0
+            )
+            return result["aggregations"]["reports"]["sub_agg"]["value"]
         else:
             return 0

backend/src/appenlight/models/services/request_metric.py

@@ -1,443 +1,617 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from datetime import datetime
 
 import appenlight.lib.helpers as h
 from appenlight.models import get_db_session, Datastores
 from appenlight.models.services.base import BaseService
 from appenlight.lib.enums import ReportType
 from appenlight.lib.utils import es_index_name_limiter
 
 try:
-    from ae_uptime_ce.models.services.uptime_metric import \
-        UptimeMetricService
+    from ae_uptime_ce.models.services.uptime_metric import UptimeMetricService
 except ImportError:
     UptimeMetricService = None
 
 
 def check_key(key, stats, uptime, total_seconds):
     if key not in stats:
-        stats[key] = {'name': key,
-                      'requests': 0,
-                      'errors': 0,
-                      'tolerated_requests': 0,
-                      'frustrating_requests': 0,
-                      'satisfying_requests': 0,
-                      'total_minutes': total_seconds / 60.0,
-                      'uptime': uptime,
-                      'apdex': 0,
-                      'rpm': 0,
-                      'response_time': 0,
-                      'avg_response_time': 0}
+        stats[key] = {
+            "name": key,
+            "requests": 0,
+            "errors": 0,
+            "tolerated_requests": 0,
+            "frustrating_requests": 0,
+            "satisfying_requests": 0,
+            "total_minutes": total_seconds / 60.0,
+            "uptime": uptime,
+            "apdex": 0,
+            "rpm": 0,
+            "response_time": 0,
+            "avg_response_time": 0,
+        }
 
 
 class RequestMetricService(BaseService):
     @classmethod
     def get_metrics_stats(cls, request, filter_settings, db_session=None):
-        delta = filter_settings['end_date'] - filter_settings['start_date']
-        if delta < h.time_deltas.get('12h')['delta']:
-            interval = '1m'
-        elif delta <= h.time_deltas.get('3d')['delta']:
-            interval = '5m'
-        elif delta >= h.time_deltas.get('2w')['delta']:
-            interval = '24h'
+        delta = filter_settings["end_date"] - filter_settings["start_date"]
+        if delta < h.time_deltas.get("12h")["delta"]:
+            interval = "1m"
+        elif delta <= h.time_deltas.get("3d")["delta"]:
+            interval = "5m"
+        elif delta >= h.time_deltas.get("2w")["delta"]:
+            interval = "24h"
         else:
-            interval = '1h'
+            interval = "1h"
 
-        filter_settings['namespace'] = ['appenlight.request_metric']
+        filter_settings["namespace"] = ["appenlight.request_metric"]
 
         es_query = {
-            'aggs': {
-                'parent_agg': {
-                    'aggs': {'custom': {'aggs': {'sub_agg': {
-                        'sum': {'field': 'tags.custom.numeric_values'}}},
-                        'filter': {'exists': {
-                            'field': 'tags.custom.numeric_values'}}},
-                        'main': {'aggs': {'sub_agg': {'sum': {
-                            'field': 'tags.main.numeric_values'}}},
-                            'filter': {'exists': {
-                                'field': 'tags.main.numeric_values'}}},
-                        'nosql': {'aggs': {'sub_agg': {'sum': {
-                            'field': 'tags.nosql.numeric_values'}}},
-                            'filter': {'exists': {
-                                'field': 'tags.nosql.numeric_values'}}},
-                        'remote': {'aggs': {'sub_agg': {'sum': {
-                            'field': 'tags.remote.numeric_values'}}},
-                            'filter': {'exists': {
-                                'field': 'tags.remote.numeric_values'}}},
-                        'requests': {'aggs': {'sub_agg': {'sum': {
-                            'field': 'tags.requests.numeric_values'}}},
-                            'filter': {'exists': {
-                                'field': 'tags.requests.numeric_values'}}},
-                        'sql': {'aggs': {'sub_agg': {
-                            'sum': {'field': 'tags.sql.numeric_values'}}},
-                            'filter': {'exists': {
-                                'field': 'tags.sql.numeric_values'}}},
-                        'tmpl': {'aggs': {'sub_agg': {'sum': {
-                            'field': 'tags.tmpl.numeric_values'}}},
-                            'filter': {'exists': {
-                                'field': 'tags.tmpl.numeric_values'}}}},
-                    'date_histogram': {'extended_bounds': {
-                        'max': filter_settings['end_date'],
-                        'min': filter_settings['start_date']},
-                        'field': 'timestamp',
-                        'interval': interval,
-                        'min_doc_count': 0}}},
-            'query': {'filtered': {
-                'filter': {'and': [{'terms': {
-                    'resource_id': [filter_settings['resource'][0]]}},
-                    {'range': {'timestamp': {
-                        'gte': filter_settings['start_date'],
-                        'lte': filter_settings['end_date']}}},
-                    {'terms': {'namespace': [
-                        'appenlight.request_metric']}}]}}}}
+            "aggs": {
+                "parent_agg": {
+                    "aggs": {
+                        "custom": {
+                            "aggs": {
+                                "sub_agg": {
+                                    "sum": {"field": "tags.custom.numeric_values"}
+                                }
+                            },
+                            "filter": {
+                                "exists": {"field": "tags.custom.numeric_values"}
+                            },
+                        },
+                        "main": {
+                            "aggs": {
+                                "sub_agg": {
+                                    "sum": {"field": "tags.main.numeric_values"}
+                                }
+                            },
+                            "filter": {"exists": {"field": "tags.main.numeric_values"}},
+                        },
+                        "nosql": {
+                            "aggs": {
+                                "sub_agg": {
+                                    "sum": {"field": "tags.nosql.numeric_values"}
+                                }
+                            },
+                            "filter": {
+                                "exists": {"field": "tags.nosql.numeric_values"}
+                            },
+                        },
+                        "remote": {
+                            "aggs": {
+                                "sub_agg": {
+                                    "sum": {"field": "tags.remote.numeric_values"}
+                                }
+                            },
+                            "filter": {
+                                "exists": {"field": "tags.remote.numeric_values"}
+                            },
+                        },
+                        "requests": {
+                            "aggs": {
+                                "sub_agg": {
+                                    "sum": {"field": "tags.requests.numeric_values"}
+                                }
+                            },
+                            "filter": {
+                                "exists": {"field": "tags.requests.numeric_values"}
+                            },
+                        },
+                        "sql": {
+                            "aggs": {
+                                "sub_agg": {"sum": {"field": "tags.sql.numeric_values"}}
+                            },
+                            "filter": {"exists": {"field": "tags.sql.numeric_values"}},
+                        },
+                        "tmpl": {
+                            "aggs": {
+                                "sub_agg": {
+                                    "sum": {"field": "tags.tmpl.numeric_values"}
+                                }
+                            },
+                            "filter": {"exists": {"field": "tags.tmpl.numeric_values"}},
+                        },
+                    },
+                    "date_histogram": {
+                        "extended_bounds": {
+                            "max": filter_settings["end_date"],
+                            "min": filter_settings["start_date"],
+                        },
+                        "field": "timestamp",
+                        "interval": interval,
+                        "min_doc_count": 0,
+                    },
+                }
+            },
+            "query": {
+                "filtered": {
+                    "filter": {
+                        "and": [
+                            {
+                                "terms": {
+                                    "resource_id": [filter_settings["resource"][0]]
+                                }
+                            },
+                            {
+                                "range": {
+                                    "timestamp": {
+                                        "gte": filter_settings["start_date"],
+                                        "lte": filter_settings["end_date"],
+                                    }
+                                }
+                            },
+                            {"terms": {"namespace": ["appenlight.request_metric"]}},
+                        ]
+                    }
+                }
+            },
+        }
 
         index_names = es_index_name_limiter(
-            start_date=filter_settings['start_date'],
-            end_date=filter_settings['end_date'],
-            ixtypes=['metrics'])
+            start_date=filter_settings["start_date"],
+            end_date=filter_settings["end_date"],
+            ixtypes=["metrics"],
+        )
         if not index_names:
             return []
 
-        result = Datastores.es.search(body=es_query,
-                                      index=index_names,
-                                      doc_type='log',
-                                      size=0)
+        result = Datastores.es.search(
+            body=es_query, index=index_names, doc_type="log", size=0
+        )
 
         plot_data = []
-        for item in result['aggregations']['parent_agg']['buckets']:
-            x_time = datetime.utcfromtimestamp(int(item['key']) / 1000)
+        for item in result["aggregations"]["parent_agg"]["buckets"]:
+            x_time = datetime.utcfromtimestamp(int(item["key"]) / 1000)
             point = {"x": x_time}
-            for key in ['custom', 'main', 'nosql', 'remote',
-                        'requests', 'sql', 'tmpl']:
-                value = item[key]['sub_agg']['value']
+            for key in ["custom", "main", "nosql", "remote", "requests", "sql", "tmpl"]:
+                value = item[key]["sub_agg"]["value"]
                 point[key] = round(value, 3) if value else 0
             plot_data.append(point)
 
         return plot_data
 
     @classmethod
-    def get_requests_breakdown(cls, request, filter_settings,
-                               db_session=None):
+    def get_requests_breakdown(cls, request, filter_settings, db_session=None):
         db_session = get_db_session(db_session)
 
         # fetch total time of all requests in this time range
         index_names = es_index_name_limiter(
-            start_date=filter_settings['start_date'],
-            end_date=filter_settings['end_date'],
-            ixtypes=['metrics'])
+            start_date=filter_settings["start_date"],
+            end_date=filter_settings["end_date"],
+            ixtypes=["metrics"],
+        )
 
-        if index_names and filter_settings['resource']:
+        if index_names and filter_settings["resource"]:
             es_query = {
-                'aggs': {'main': {'aggs': {
-                    'sub_agg': {'sum': {'field': 'tags.main.numeric_values'}}},
-                    'filter': {'exists': {
-                        'field': 'tags.main.numeric_values'}}}},
-                'query': {'filtered': {
-                    'filter': {'and': [
-                        {'terms': {
-                            'resource_id': [filter_settings['resource'][0]]}},
-                        {'range': {'timestamp': {
-                            'gte': filter_settings['start_date'],
-                            'lte': filter_settings['end_date']}}},
-                        {'terms': {'namespace': [
-                            'appenlight.request_metric']}}]}}}}
-            result = Datastores.es.search(body=es_query,
-                                          index=index_names,
-                                          doc_type='log',
-                                          size=0)
-            total_time_spent = result['aggregations']['main']['sub_agg'][
-                'value']
+                "aggs": {
+                    "main": {
+                        "aggs": {
+                            "sub_agg": {"sum": {"field": "tags.main.numeric_values"}}
+                        },
+                        "filter": {"exists": {"field": "tags.main.numeric_values"}},
+                    }
+                },
+                "query": {
+                    "filtered": {
+                        "filter": {
+                            "and": [
+                                {
+                                    "terms": {
+                                        "resource_id": [filter_settings["resource"][0]]
+                                    }
+                                },
+                                {
+                                    "range": {
+                                        "timestamp": {
+                                            "gte": filter_settings["start_date"],
+                                            "lte": filter_settings["end_date"],
+                                        }
+                                    }
+                                },
+                                {"terms": {"namespace": ["appenlight.request_metric"]}},
+                            ]
+                        }
+                    }
+                },
+            }
+            result = Datastores.es.search(
+                body=es_query, index=index_names, doc_type="log", size=0
+            )
+            total_time_spent = result["aggregations"]["main"]["sub_agg"]["value"]
         else:
             total_time_spent = 0
         script_text = "doc['tags.main.numeric_values'].value / {}".format(
-            total_time_spent)
+            total_time_spent
+        )
 
-        if index_names and filter_settings['resource']:
+        if index_names and filter_settings["resource"]:
             es_query = {
-                'aggs': {
-                    'parent_agg': {
-                        'aggs': {'main': {'aggs': {
-                            'sub_agg': {
-                                'sum': {'field': 'tags.main.numeric_values'}}},
-                            'filter': {
-                                'exists': {
-                                    'field': 'tags.main.numeric_values'}}},
-                            'percentage': {
-                                'aggs': {'sub_agg': {
-                                    'sum': {
-                                        'lang': 'expression',
-                                        'script': script_text}}},
-                                'filter': {
-                                    'exists': {
-                                        'field': 'tags.main.numeric_values'}}},
-                            'requests': {'aggs': {'sub_agg': {
-                                'sum': {
-                                    'field': 'tags.requests.numeric_values'}}},
-                                'filter': {'exists': {
-                                    'field': 'tags.requests.numeric_values'}}}},
-                        'terms': {'field': 'tags.view_name.values',
-                                  'order': {'percentage>sub_agg': 'desc'},
-                                  'size': 15}}},
-                'query': {'filtered': {'filter': {'and': [
-                    {'terms': {
-                        'resource_id': [filter_settings['resource'][0]]}},
-                    {'range': {
-                        'timestamp': {'gte': filter_settings['start_date'],
-                                      'lte': filter_settings['end_date']
-                                      }
+                "aggs": {
+                    "parent_agg": {
+                        "aggs": {
+                            "main": {
+                                "aggs": {
+                                    "sub_agg": {
+                                        "sum": {"field": "tags.main.numeric_values"}
+                                    }
+                                },
+                                "filter": {
+                                    "exists": {"field": "tags.main.numeric_values"}
+                                },
+                            },
+                            "percentage": {
+                                "aggs": {
+                                    "sub_agg": {
+                                        "sum": {
+                                            "lang": "expression",
+                                            "script": script_text,
+                                        }
+                                    }
+                                },
+                                "filter": {
+                                    "exists": {"field": "tags.main.numeric_values"}
+                                },
+                            },
+                            "requests": {
+                                "aggs": {
+                                    "sub_agg": {
+                                        "sum": {"field": "tags.requests.numeric_values"}
+                                    }
+                                },
+                                "filter": {
+                                    "exists": {"field": "tags.requests.numeric_values"}
+                                },
+                            },
+                        },
+                        "terms": {
+                            "field": "tags.view_name.values",
+                            "order": {"percentage>sub_agg": "desc"},
+                            "size": 15,
+                        },
                     }
+                },
+                "query": {
+                    "filtered": {
+                        "filter": {
+                            "and": [
+                                {
+                                    "terms": {
+                                        "resource_id": [filter_settings["resource"][0]]
+                                    }
+                                },
+                                {
+                                    "range": {
+                                        "timestamp": {
+                                            "gte": filter_settings["start_date"],
+                                            "lte": filter_settings["end_date"],
+                                        }
+                                    }
+                                },
+                            ]
+                        }
                     }
-                ]}
-                }}
+                },
             }
-            result = Datastores.es.search(body=es_query,
-                                          index=index_names,
-                                          doc_type='log',
-                                          size=0)
-            series = result['aggregations']['parent_agg']['buckets']
+            result = Datastores.es.search(
+                body=es_query, index=index_names, doc_type="log", size=0
+            )
+            series = result["aggregations"]["parent_agg"]["buckets"]
         else:
             series = []
 
         and_part = [
-            {"term": {"resource_id": filter_settings['resource'][0]}},
-            {"terms": {"tags.view_name.values": [row['key'] for
-                                                 row in series]}},
-            {"term": {"report_type": str(ReportType.slow)}}
+            {"term": {"resource_id": filter_settings["resource"][0]}},
+            {"terms": {"tags.view_name.values": [row["key"] for row in series]}},
+            {"term": {"report_type": str(ReportType.slow)}},
         ]
         query = {
             "aggs": {
                 "top_reports": {
-                    "terms": {
-                        "field": "tags.view_name.values",
-                        "size": len(series)
-                    },
+                    "terms": {"field": "tags.view_name.values", "size": len(series)},
                     "aggs": {
                         "top_calls_hits": {
-                            "top_hits": {
-                                "sort": {"start_time": "desc"},
-                                "size": 5
-                            }
+                            "top_hits": {"sort": {"start_time": "desc"}, "size": 5}
                         }
-                    }
+                    },
                 }
             },
-
-            "query": {
-                "filtered": {
-                    "filter": {
-                        "and": and_part
-                    }
-                }
-            }
+            "query": {"filtered": {"filter": {"and": and_part}}},
         }
         details = {}
-        index_names = es_index_name_limiter(ixtypes=['reports'])
+        index_names = es_index_name_limiter(ixtypes=["reports"])
         if index_names and series:
             result = Datastores.es.search(
-                body=query, doc_type='report', size=0, index=index_names)
-            for bucket in result['aggregations']['top_reports']['buckets']:
-                details[bucket['key']] = []
-
-                for hit in bucket['top_calls_hits']['hits']['hits']:
-                    details[bucket['key']].append(
-                        {'report_id': hit['_source']['pg_id'],
-                         'group_id': hit['_source']['group_id']}
+                body=query, doc_type="report", size=0, index=index_names
+            )
+            for bucket in result["aggregations"]["top_reports"]["buckets"]:
+                details[bucket["key"]] = []
+
+                for hit in bucket["top_calls_hits"]["hits"]["hits"]:
+                    details[bucket["key"]].append(
+                        {
+                            "report_id": hit["_source"]["pg_id"],
+                            "group_id": hit["_source"]["group_id"],
+                        }
                     )
 
         results = []
         for row in series:
             result = {
-                'key': row['key'],
-                'main': row['main']['sub_agg']['value'],
-                'requests': row['requests']['sub_agg']['value']
+                "key": row["key"],
+                "main": row["main"]["sub_agg"]["value"],
+                "requests": row["requests"]["sub_agg"]["value"],
             }
             # es can return 'infinity'
             try:
-                result['percentage'] = float(
-                    row['percentage']['sub_agg']['value'])
+                result["percentage"] = float(row["percentage"]["sub_agg"]["value"])
             except ValueError:
-                result['percentage'] = 0
+                result["percentage"] = 0
 
-            result['latest_details'] = details.get(row['key']) or []
+            result["latest_details"] = details.get(row["key"]) or []
             results.append(result)
 
         return results
 
     @classmethod
-    def get_apdex_stats(cls, request, filter_settings,
-                        threshold=1, db_session=None):
+    def get_apdex_stats(cls, request, filter_settings, threshold=1, db_session=None):
         """
         Returns information and calculates APDEX score per server for dashboard
         server information (upper right stats boxes)
         """
         # Apdex t = (Satisfied Count + Tolerated Count / 2) / Total Samples
         db_session = get_db_session(db_session)
         index_names = es_index_name_limiter(
-            start_date=filter_settings['start_date'],
-            end_date=filter_settings['end_date'], ixtypes=['metrics'])
+            start_date=filter_settings["start_date"],
+            end_date=filter_settings["end_date"],
+            ixtypes=["metrics"],
+        )
 
         requests_series = []
 
-        if index_names and filter_settings['resource']:
+        if index_names and filter_settings["resource"]:
             es_query = {
-                'aggs': {
-                    'parent_agg': {'aggs': {
-                        'frustrating': {'aggs': {'sub_agg': {
-                            'sum': {'field': 'tags.requests.numeric_values'}}},
-                            'filter': {'and': [
-                                {'range': {
-                                    'tags.main.numeric_values': {'gte': '4'}}},
-                                {'exists': {
-                                    'field': 'tags.requests.numeric_values'}}]
-                            }
+                "aggs": {
+                    "parent_agg": {
+                        "aggs": {
+                            "frustrating": {
+                                "aggs": {
+                                    "sub_agg": {
+                                        "sum": {"field": "tags.requests.numeric_values"}
+                                    }
+                                },
+                                "filter": {
+                                    "and": [
+                                        {
+                                            "range": {
+                                                "tags.main.numeric_values": {"gte": "4"}
+                                            }
+                                        },
+                                        {
+                                            "exists": {
+                                                "field": "tags.requests.numeric_values"
+                                            }
+                                        },
+                                    ]
+                                },
+                            },
+                            "main": {
+                                "aggs": {
+                                    "sub_agg": {
+                                        "sum": {"field": "tags.main.numeric_values"}
+                                    }
+                                },
+                                "filter": {
+                                    "exists": {"field": "tags.main.numeric_values"}
+                                },
+                            },
+                            "requests": {
+                                "aggs": {
+                                    "sub_agg": {
+                                        "sum": {"field": "tags.requests.numeric_values"}
+                                    }
+                                },
+                                "filter": {
+                                    "exists": {"field": "tags.requests.numeric_values"}
+                                },
+                            },
+                            "tolerated": {
+                                "aggs": {
+                                    "sub_agg": {
+                                        "sum": {"field": "tags.requests.numeric_values"}
+                                    }
+                                },
+                                "filter": {
+                                    "and": [
+                                        {
+                                            "range": {
+                                                "tags.main.numeric_values": {"gte": "1"}
+                                            }
+                                        },
+                                        {
+                                            "range": {
+                                                "tags.main.numeric_values": {"lt": "4"}
+                                            }
+                                        },
+                                        {
+                                            "exists": {
+                                                "field": "tags.requests.numeric_values"
+                                            }
+                                        },
+                                    ]
+                                },
+                            },
                         },
-                        'main': {'aggs': {'sub_agg': {'sum': {
-                            'field': 'tags.main.numeric_values'}}},
-                            'filter': {'exists': {
-                                'field': 'tags.main.numeric_values'}}},
-                        'requests': {'aggs': {'sub_agg': {
-                            'sum': {
-                                'field': 'tags.requests.numeric_values'}}},
-                            'filter': {'exists': {
-                                'field': 'tags.requests.numeric_values'}}},
-                        'tolerated': {'aggs': {'sub_agg': {
-                            'sum': {
-                                'field': 'tags.requests.numeric_values'}}},
-                            'filter': {'and': [
-                                {'range': {
-                                    'tags.main.numeric_values': {'gte': '1'}}},
-                                {'range': {
-                                    'tags.main.numeric_values': {'lt': '4'}}},
-                                {'exists': {
-                                    'field': 'tags.requests.numeric_values'}}]}
+                        "terms": {"field": "tags.server_name.values", "size": 999999},
+                    }
+                },
+                "query": {
+                    "filtered": {
+                        "filter": {
+                            "and": [
+                                {
+                                    "terms": {
+                                        "resource_id": [filter_settings["resource"][0]]
+                                    }
+                                },
+                                {
+                                    "range": {
+                                        "timestamp": {
+                                            "gte": filter_settings["start_date"],
+                                            "lte": filter_settings["end_date"],
+                                        }
+                                    }
+                                },
+                                {"terms": {"namespace": ["appenlight.request_metric"]}},
+                            ]
                         }
-                    },
-                        'terms': {'field': 'tags.server_name.values',
-                                  'size': 999999}}},
-                'query': {
-                    'filtered': {
-                        'filter': {'and': [{'terms': {
-                            'resource_id': [
-                                filter_settings['resource'][0]]}},
-                            {'range': {'timestamp': {
-                                'gte': filter_settings['start_date'],
-                                'lte': filter_settings['end_date']}}},
-                            {'terms': {'namespace': [
-                                'appenlight.request_metric']}}]}}}}
-
-            result = Datastores.es.search(body=es_query,
-                                          index=index_names,
-                                          doc_type='log',
-                                          size=0)
-            for bucket in result['aggregations']['parent_agg']['buckets']:
-                requests_series.append({
-                    'frustrating': bucket['frustrating']['sub_agg']['value'],
-                    'main': bucket['main']['sub_agg']['value'],
-                    'requests': bucket['requests']['sub_agg']['value'],
-                    'tolerated': bucket['tolerated']['sub_agg']['value'],
-                    'key': bucket['key']
-                })
-
-        since_when = filter_settings['start_date']
-        until = filter_settings['end_date']
+                    }
+                },
+            }
+
+            result = Datastores.es.search(
+                body=es_query, index=index_names, doc_type="log", size=0
+            )
+            for bucket in result["aggregations"]["parent_agg"]["buckets"]:
+                requests_series.append(
+                    {
+                        "frustrating": bucket["frustrating"]["sub_agg"]["value"],
+                        "main": bucket["main"]["sub_agg"]["value"],
+                        "requests": bucket["requests"]["sub_agg"]["value"],
+                        "tolerated": bucket["tolerated"]["sub_agg"]["value"],
+                        "key": bucket["key"],
+                    }
+                )
+
+        since_when = filter_settings["start_date"]
+        until = filter_settings["end_date"]
 
         # total errors
 
         index_names = es_index_name_limiter(
-            start_date=filter_settings['start_date'],
-            end_date=filter_settings['end_date'], ixtypes=['reports'])
+            start_date=filter_settings["start_date"],
+            end_date=filter_settings["end_date"],
+            ixtypes=["reports"],
+        )
 
         report_series = []
-        if index_names and filter_settings['resource']:
+        if index_names and filter_settings["resource"]:
             report_type = ReportType.key_from_value(ReportType.error)
             es_query = {
-                'aggs': {
-                    'parent_agg': {'aggs': {'errors': {'aggs': {'sub_agg': {
-                        'sum': {
-                            'field': 'tags.occurences.numeric_values'}}},
-                        'filter': {'and': [
-                            {'terms': {
-                                'tags.type.values': [report_type]}},
-                            {'exists': {
-                                'field': 'tags.occurences.numeric_values'}}]
-                        }
-                    }},
-                        'terms': {'field': 'tags.server_name.values',
-                                  'size': 999999}}},
-                'query': {'filtered': {
-                    'filter': {'and': [
-                        {'terms': {
-                            'resource_id': [filter_settings['resource'][0]]}},
-                        {'range': {
-                            'timestamp': {'gte': filter_settings['start_date'],
-                                          'lte': filter_settings['end_date']}}
+                "aggs": {
+                    "parent_agg": {
+                        "aggs": {
+                            "errors": {
+                                "aggs": {
+                                    "sub_agg": {
+                                        "sum": {
+                                            "field": "tags.occurences.numeric_values"
+                                        }
+                                    }
+                                },
+                                "filter": {
+                                    "and": [
+                                        {"terms": {"tags.type.values": [report_type]}},
+                                        {
+                                            "exists": {
+                                                "field": "tags.occurences.numeric_values"
+                                            }
+                                        },
+                                    ]
+                                },
+                            }
                         },
-                        {'terms': {'namespace': ['appenlight.error']}}]
+                        "terms": {"field": "tags.server_name.values", "size": 999999},
+                    }
+                },
+                "query": {
+                    "filtered": {
+                        "filter": {
+                            "and": [
+                                {
+                                    "terms": {
+                                        "resource_id": [filter_settings["resource"][0]]
+                                    }
+                                },
+                                {
+                                    "range": {
+                                        "timestamp": {
+                                            "gte": filter_settings["start_date"],
+                                            "lte": filter_settings["end_date"],
+                                        }
+                                    }
+                                },
+                                {"terms": {"namespace": ["appenlight.error"]}},
+                            ]
+                        }
                     }
-                }}
+                },
             }
-            result = Datastores.es.search(body=es_query,
-                                          index=index_names,
-                                          doc_type='log',
-                                          size=0)
-            for bucket in result['aggregations']['parent_agg']['buckets']:
+            result = Datastores.es.search(
+                body=es_query, index=index_names, doc_type="log", size=0
+            )
+            for bucket in result["aggregations"]["parent_agg"]["buckets"]:
                 report_series.append(
-                    {'key': bucket['key'],
-                     'errors': bucket['errors']['sub_agg']['value']
+                    {
+                        "key": bucket["key"],
+                        "errors": bucket["errors"]["sub_agg"]["value"],
                     }
                 )
 
         stats = {}
         if UptimeMetricService is not None:
             uptime = UptimeMetricService.get_uptime_by_app(
-                filter_settings['resource'][0],
-                since_when=since_when, until=until)
+                filter_settings["resource"][0], since_when=since_when, until=until
+            )
         else:
             uptime = 0
 
         total_seconds = (until - since_when).total_seconds()
 
         for stat in requests_series:
-            check_key(stat['key'], stats, uptime, total_seconds)
-            stats[stat['key']]['requests'] = int(stat['requests'])
-            stats[stat['key']]['response_time'] = stat['main']
-            stats[stat['key']]['tolerated_requests'] = stat['tolerated']
-            stats[stat['key']]['frustrating_requests'] = stat['frustrating']
+            check_key(stat["key"], stats, uptime, total_seconds)
+            stats[stat["key"]]["requests"] = int(stat["requests"])
+            stats[stat["key"]]["response_time"] = stat["main"]
+            stats[stat["key"]]["tolerated_requests"] = stat["tolerated"]
+            stats[stat["key"]]["frustrating_requests"] = stat["frustrating"]
         for server in report_series:
-            check_key(server['key'], stats, uptime, total_seconds)
-            stats[server['key']]['errors'] = server['errors']
+            check_key(server["key"], stats, uptime, total_seconds)
+            stats[server["key"]]["errors"] = server["errors"]
 
         server_stats = list(stats.values())
         for stat in server_stats:
-            stat['satisfying_requests'] = stat['requests'] - stat['errors'] \
-                                          - stat['frustrating_requests'] - \
-                                          stat['tolerated_requests']
-            if stat['satisfying_requests'] < 0:
-                stat['satisfying_requests'] = 0
-
-            if stat['requests']:
-                stat['avg_response_time'] = round(stat['response_time'] /
-                                                  stat['requests'], 3)
-                qual_requests = stat['satisfying_requests'] + \
-                                stat['tolerated_requests'] / 2.0
-                stat['apdex'] = round((qual_requests / stat['requests']) * 100,
-                                      2)
-                stat['rpm'] = round(stat['requests'] / stat['total_minutes'],
-                                    2)
-
-        return sorted(server_stats, key=lambda x: x['name'])
+            stat["satisfying_requests"] = (
+                stat["requests"]
+                - stat["errors"]
+                - stat["frustrating_requests"]
+                - stat["tolerated_requests"]
+            )
+            if stat["satisfying_requests"] < 0:
+                stat["satisfying_requests"] = 0
+
+            if stat["requests"]:
+                stat["avg_response_time"] = round(
+                    stat["response_time"] / stat["requests"], 3
+                )
+                qual_requests = (
+                    stat["satisfying_requests"] + stat["tolerated_requests"] / 2.0
+                )
+                stat["apdex"] = round((qual_requests / stat["requests"]) * 100, 2)
+                stat["rpm"] = round(stat["requests"] / stat["total_minutes"], 2)
+
+        return sorted(server_stats, key=lambda x: x["name"])

backend/src/appenlight/models/services/slow_call.py

@@ -1,169 +1,186 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from appenlight.models import get_db_session, Datastores
 from appenlight.models.report import Report
 from appenlight.models.services.base import BaseService
 from appenlight.lib.utils import es_index_name_limiter
 
 
 class SlowCallService(BaseService):
     @classmethod
-    def get_time_consuming_calls(cls, request, filter_settings,
-                                 db_session=None):
+    def get_time_consuming_calls(cls, request, filter_settings, db_session=None):
         db_session = get_db_session(db_session)
         # get slow calls from older partitions too
         index_names = es_index_name_limiter(
-            start_date=filter_settings['start_date'],
-            end_date=filter_settings['end_date'],
-            ixtypes=['slow_calls'])
-        if index_names and filter_settings['resource']:
+            start_date=filter_settings["start_date"],
+            end_date=filter_settings["end_date"],
+            ixtypes=["slow_calls"],
+        )
+        if index_names and filter_settings["resource"]:
             # get longest time taking hashes
             es_query = {
-                'aggs': {
-                    'parent_agg': {
-                        'aggs': {
-                            'duration': {
-                                'aggs': {'sub_agg': {
-                                    'sum': {
-                                        'field': 'tags.duration.numeric_values'}
-                                }},
-                                'filter': {'exists': {
-                                    'field': 'tags.duration.numeric_values'}}},
-                            'total': {
-                                'aggs': {'sub_agg': {'value_count': {
-                                    'field': 'tags.statement_hash.values'}}},
-                                'filter': {'exists': {
-                                    'field': 'tags.statement_hash.values'}}}},
-                        'terms': {'field': 'tags.statement_hash.values',
-                                  'order': {'duration>sub_agg': 'desc'},
-                                  'size': 15}}},
-                'query': {'filtered': {
-                    'filter': {'and': [
-                        {'terms': {
-                            'resource_id': [filter_settings['resource'][0]]
-                        }},
-                        {'range': {'timestamp': {
-                            'gte': filter_settings['start_date'],
-                            'lte': filter_settings['end_date']}
-                        }}]
+                "aggs": {
+                    "parent_agg": {
+                        "aggs": {
+                            "duration": {
+                                "aggs": {
+                                    "sub_agg": {
+                                        "sum": {"field": "tags.duration.numeric_values"}
+                                    }
+                                },
+                                "filter": {
+                                    "exists": {"field": "tags.duration.numeric_values"}
+                                },
+                            },
+                            "total": {
+                                "aggs": {
+                                    "sub_agg": {
+                                        "value_count": {
+                                            "field": "tags.statement_hash.values"
+                                        }
+                                    }
+                                },
+                                "filter": {
+                                    "exists": {"field": "tags.statement_hash.values"}
+                                },
+                            },
+                        },
+                        "terms": {
+                            "field": "tags.statement_hash.values",
+                            "order": {"duration>sub_agg": "desc"},
+                            "size": 15,
+                        },
                     }
-                }
-                }
+                },
+                "query": {
+                    "filtered": {
+                        "filter": {
+                            "and": [
+                                {
+                                    "terms": {
+                                        "resource_id": [filter_settings["resource"][0]]
+                                    }
+                                },
+                                {
+                                    "range": {
+                                        "timestamp": {
+                                            "gte": filter_settings["start_date"],
+                                            "lte": filter_settings["end_date"],
+                                        }
+                                    }
+                                },
+                            ]
+                        }
+                    }
+                },
             }
             result = Datastores.es.search(
-                body=es_query, index=index_names, doc_type='log', size=0)
-            results = result['aggregations']['parent_agg']['buckets']
+                body=es_query, index=index_names, doc_type="log", size=0
+            )
+            results = result["aggregations"]["parent_agg"]["buckets"]
         else:
             return []
-        hashes = [i['key'] for i in results]
+        hashes = [i["key"] for i in results]
 
         # get queries associated with hashes
         calls_query = {
             "aggs": {
                 "top_calls": {
-                    "terms": {
-                        "field": "tags.statement_hash.values",
-                        "size": 15
-                    },
+                    "terms": {"field": "tags.statement_hash.values", "size": 15},
                     "aggs": {
                         "top_calls_hits": {
-                            "top_hits": {
-                                "sort": {"timestamp": "desc"},
-                                "size": 5
-                            }
+                            "top_hits": {"sort": {"timestamp": "desc"}, "size": 5}
                         }
-                    }
+                    },
                 }
             },
             "query": {
                 "filtered": {
                     "filter": {
                         "and": [
                             {
                                 "terms": {
-                                    "resource_id": [
-                                        filter_settings['resource'][0]
-                                    ]
-                                }
-                            },
-                            {
-                                "terms": {
-                                    "tags.statement_hash.values": hashes
+                                    "resource_id": [filter_settings["resource"][0]]
                                 }
                             },
+                            {"terms": {"tags.statement_hash.values": hashes}},
                             {
                                 "range": {
                                     "timestamp": {
-                                        "gte": filter_settings['start_date'],
-                                        "lte": filter_settings['end_date']
+                                        "gte": filter_settings["start_date"],
+                                        "lte": filter_settings["end_date"],
                                     }
                                 }
-                            }
+                            },
                         ]
                     }
                 }
-            }
+            },
         }
-        calls = Datastores.es.search(body=calls_query,
-                                     index=index_names,
-                                     doc_type='log',
-                                     size=0)
+        calls = Datastores.es.search(
+            body=calls_query, index=index_names, doc_type="log", size=0
+        )
         call_results = {}
         report_ids = []
-        for call in calls['aggregations']['top_calls']['buckets']:
-            hits = call['top_calls_hits']['hits']['hits']
-            call_results[call['key']] = [i['_source'] for i in hits]
-            report_ids.extend([i['_source']['tags']['report_id']['values']
-                               for i in hits])
+        for call in calls["aggregations"]["top_calls"]["buckets"]:
+            hits = call["top_calls_hits"]["hits"]["hits"]
+            call_results[call["key"]] = [i["_source"] for i in hits]
+            report_ids.extend(
+                [i["_source"]["tags"]["report_id"]["values"] for i in hits]
+            )
         if report_ids:
             r_query = db_session.query(Report.group_id, Report.id)
             r_query = r_query.filter(Report.id.in_(report_ids))
-            r_query = r_query.filter(
-                Report.start_time >= filter_settings['start_date'])
+            r_query = r_query.filter(Report.start_time >= filter_settings["start_date"])
         else:
             r_query = []
         reports_reversed = {}
         for report in r_query:
             reports_reversed[report.id] = report.group_id
 
         final_results = []
         for item in results:
-            if item['key'] not in call_results:
+            if item["key"] not in call_results:
                 continue
-            call = call_results[item['key']][0]
-            row = {'occurences': item['total']['sub_agg']['value'],
-                   'total_duration': round(
-                       item['duration']['sub_agg']['value']),
-                   'statement': call['message'],
-                   'statement_type': call['tags']['type']['values'],
-                   'statement_subtype': call['tags']['subtype']['values'],
-                   'statement_hash': item['key'],
-                   'latest_details': []}
-            if row['statement_type'] in ['tmpl', ' remote']:
-                params = call['tags']['parameters']['values'] \
-                    if 'parameters' in call['tags'] else ''
-                row['statement'] = '{} ({})'.format(call['message'], params)
-            for call in call_results[item['key']]:
-                report_id = call['tags']['report_id']['values']
+            call = call_results[item["key"]][0]
+            row = {
+                "occurences": item["total"]["sub_agg"]["value"],
+                "total_duration": round(item["duration"]["sub_agg"]["value"]),
+                "statement": call["message"],
+                "statement_type": call["tags"]["type"]["values"],
+                "statement_subtype": call["tags"]["subtype"]["values"],
+                "statement_hash": item["key"],
+                "latest_details": [],
+            }
+            if row["statement_type"] in ["tmpl", " remote"]:
+                params = (
+                    call["tags"]["parameters"]["values"]
+                    if "parameters" in call["tags"]
+                    else ""
+                )
+                row["statement"] = "{} ({})".format(call["message"], params)
+            for call in call_results[item["key"]]:
+                report_id = call["tags"]["report_id"]["values"]
                 group_id = reports_reversed.get(report_id)
                 if group_id:
-                    row['latest_details'].append(
-                        {'group_id': group_id, 'report_id': report_id})
+                    row["latest_details"].append(
+                        {"group_id": group_id, "report_id": report_id}
+                    )
 
             final_results.append(row)
 
         return final_results

backend/src/appenlight/models/services/tag.py

@@ -1,97 +1,102 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import sqlalchemy as sa
 from appenlight.lib.ext_json import json
 from pyramid.threadlocal import get_current_registry
 from appenlight.models.tag import Tag
 from appenlight.models.services.base import BaseService
 from appenlight.models import get_db_session
 
 
 class TagService(BaseService):
     @classmethod
     def cut_name(cls, tag_name):
         return tag_name[:32]
 
     @classmethod
     def cut_value(cls, value):
         if isinstance(value, str):
             value = value[:128]
         return value
 
     @classmethod
-    def by_resource_id_and_value(cls, resource_id, tag_name, value,
-                                 db_session=None, create_missing=True):
+    def by_resource_id_and_value(
+        cls, resource_id, tag_name, value, db_session=None, create_missing=True
+    ):
         """
         Fetches tag and creates a new one if missing
         """
         db_session = get_db_session(db_session)
         registry = get_current_registry()
 
         @registry.cache_regions.redis_min_10.cache_on_arguments(
-            namespace='TagService.by_resource_id_and_value')
+            namespace="TagService.by_resource_id_and_value"
+        )
         def cached(resource_id, tag_name, value):
-            reduced_name = cls.cut_name(tag_name.decode('utf8'))
-            reduced_value = cls.cut_value(value.decode('utf8'))
+            reduced_name = cls.cut_name(tag_name.decode("utf8"))
+            reduced_value = cls.cut_value(value.decode("utf8"))
 
             query = db_session.query(Tag)
             query = query.filter(Tag.resource_id == resource_id)
             query = query.filter(Tag.name == reduced_name)
-            query = query.filter(sa.cast(Tag.value, sa.types.TEXT) ==
-                                 sa.cast(json.dumps(reduced_value),
-                                         sa.types.TEXT))
+            query = query.filter(
+                sa.cast(Tag.value, sa.types.TEXT)
+                == sa.cast(json.dumps(reduced_value), sa.types.TEXT)
+            )
             tag = query.first()
             if tag:
                 db_session.expunge(tag)
             return tag
 
-        view = cached(resource_id, tag_name.encode('utf8'),
-                      value.encode('utf8'))
+        view = cached(resource_id, tag_name.encode("utf8"), value.encode("utf8"))
         if not view and create_missing:
-            view = cls.create_tag(resource_id,
-                                  cls.cut_name(tag_name),
-                                  cls.cut_value(value),
-                                  db_session)
-            cached.invalidate(resource_id, tag_name.encode('utf8'),
-                              value.encode('utf8'))
+            view = cls.create_tag(
+                resource_id, cls.cut_name(tag_name), cls.cut_value(value), db_session
+            )
+            cached.invalidate(
+                resource_id, tag_name.encode("utf8"), value.encode("utf8")
+            )
         return view
 
     @classmethod
     def create_tag(cls, resource_id, tag_name, value, db_session=None):
 
-        tag = Tag(resource_id=resource_id,
-                  name=cls.cut_name(tag_name),
-                  value=cls.cut_value(value))
+        tag = Tag(
+            resource_id=resource_id,
+            name=cls.cut_name(tag_name),
+            value=cls.cut_value(value),
+        )
         db_session = get_db_session(db_session)
         db_session.add(tag)
         db_session.flush()
         return tag
 
     @classmethod
     def by_tag_id(cls, tag_id, db_session=None):
         db_session = get_db_session(db_session)
         registry = get_current_registry()
 
         @registry.cache_regions.redis_min_10.cache_on_arguments(
-            namespace='TagService.by_tag_id')
+            namespace="TagService.by_tag_id"
+        )
         def cached(tag_id):
             tag = db_session.query(Tag).filter(Tag.id == tag_id).first()
             if tag:
                 db_session.expunge(tag)
             return tag
 
         return cached(tag_id)

backend/src/appenlight/models/services/user.py

@@ -1,152 +1,166 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 import pyramid_mailer
 import pyramid.renderers
 import sqlalchemy as sa
 
 from collections import namedtuple
 from datetime import datetime
 
 from ziggurat_foundations.models.services.user import UserService
 
 from appenlight.lib.rule import Rule
 from appenlight.models import get_db_session
 from appenlight.models.integrations import IntegrationException
 from appenlight.models.report import REPORT_TYPE_MATRIX
 from appenlight.models.user import User
 from paginate_sqlalchemy import SqlalchemyOrmPage
 from pyramid.threadlocal import get_current_registry
 
 log = logging.getLogger(__name__)
 
-GroupOccurence = namedtuple('GroupOccurence', ['occurences', 'group'])
+GroupOccurence = namedtuple("GroupOccurence", ["occurences", "group"])
 
 
 class UserService(UserService):
     @classmethod
     def all(cls, db_session=None):
         return get_db_session(db_session).query(User).order_by(User.user_name)
 
     @classmethod
-    def send_email(cls, request, recipients, variables, template,
-                   immediately=False, silent=False):
+    def send_email(
+        cls, request, recipients, variables, template, immediately=False, silent=False
+    ):
         html = pyramid.renderers.render(template, variables, request)
-        title = variables.get('email_title',
-                              variables.get('title', "No Title"))
-        title = title.replace('\r', '').replace('\n', '')
+        title = variables.get("email_title", variables.get("title", "No Title"))
+        title = title.replace("\r", "").replace("\n", "")
         sender = "{} <{}>".format(
-            request.registry.settings['mailing.from_name'],
-            request.registry.settings['mailing.from_email'])
+            request.registry.settings["mailing.from_name"],
+            request.registry.settings["mailing.from_email"],
+        )
         message = pyramid_mailer.message.Message(
-            subject=title, sender=sender, recipients=recipients, html=html)
+            subject=title, sender=sender, recipients=recipients, html=html
+        )
         if immediately:
             try:
                 request.registry.mailer.send_immediately(message)
             except Exception as e:
-                log.warning('Exception %s' % e)
+                log.warning("Exception %s" % e)
                 if not silent:
                     raise
         else:
             request.registry.mailer.send(message)
 
     @classmethod
-    def get_paginator(cls, page=1, item_count=None, items_per_page=50,
-                      order_by=None, filter_settings=None,
-                      exclude_columns=None, db_session=None):
+    def get_paginator(
+        cls,
+        page=1,
+        item_count=None,
+        items_per_page=50,
+        order_by=None,
+        filter_settings=None,
+        exclude_columns=None,
+        db_session=None,
+    ):
         registry = get_current_registry()
         if not exclude_columns:
             exclude_columns = []
         if not filter_settings:
             filter_settings = {}
         db_session = get_db_session(db_session)
         q = db_session.query(User)
-        if filter_settings.get('order_col'):
-            order_col = filter_settings.get('order_col')
-            if filter_settings.get('order_dir') == 'dsc':
-                sort_on = 'desc'
+        if filter_settings.get("order_col"):
+            order_col = filter_settings.get("order_col")
+            if filter_settings.get("order_dir") == "dsc":
+                sort_on = "desc"
             else:
-                sort_on = 'asc'
+                sort_on = "asc"
             q = q.order_by(getattr(sa, sort_on)(getattr(User, order_col)))
         else:
             q = q.order_by(sa.desc(User.registered_date))
             # remove urlgen or it never caches count
         cache_params = dict(filter_settings)
-        cache_params.pop('url', None)
-        cache_params.pop('url_maker', None)
+        cache_params.pop("url", None)
+        cache_params.pop("url_maker", None)
 
         @registry.cache_regions.redis_min_5.cache_on_arguments()
         def estimate_users(cache_key):
             o_q = q.order_by(False)
             return o_q.count()
 
         item_count = estimate_users(cache_params)
         # if the number of pages is low we may want to invalidate the count to
         # provide 'real time' update - use case -
         # errors just started to flow in
         if item_count < 1000:
             item_count = estimate_users.refresh(cache_params)
-        paginator = SqlalchemyOrmPage(q, page=page,
-                                      item_count=item_count,
-                                      items_per_page=items_per_page,
-                                      **filter_settings)
+        paginator = SqlalchemyOrmPage(
+            q,
+            page=page,
+            item_count=item_count,
+            items_per_page=items_per_page,
+            **filter_settings
+        )
         return paginator
 
     @classmethod
     def get_valid_channels(cls, user):
-        return [channel for channel in user.alert_channels
-                if channel.channel_validated]
+        return [channel for channel in user.alert_channels if channel.channel_validated]
 
     @classmethod
-    def report_notify(cls, user, request, application, report_groups,
-                      occurence_dict, db_session=None):
+    def report_notify(
+        cls, user, request, application, report_groups, occurence_dict, db_session=None
+    ):
         db_session = get_db_session(db_session)
         if not report_groups:
             return True
         since_when = datetime.utcnow()
         for channel in cls.get_valid_channels(user):
             confirmed_groups = []
 
             for group in report_groups:
                 occurences = occurence_dict.get(group.id, 1)
                 for action in channel.channel_actions:
                     not_matched = (
-                        action.resource_id and action.resource_id !=
-                        application.resource_id)
-                    if action.type != 'report' or not_matched:
+                        action.resource_id
+                        and action.resource_id != application.resource_id
+                    )
+                    if action.type != "report" or not_matched:
                         continue
-                    should_notify = (action.action == 'always' or
-                                     not group.notified)
+                    should_notify = action.action == "always" or not group.notified
                     rule_obj = Rule(action.rule, REPORT_TYPE_MATRIX)
                     report_dict = group.get_report().get_dict(request)
                     if rule_obj.match(report_dict) and should_notify:
                         item = GroupOccurence(occurences, group)
                         if item not in confirmed_groups:
                             confirmed_groups.append(item)
 
             # send individual reports
             total_confirmed = len(confirmed_groups)
             if not total_confirmed:
                 continue
             try:
-                channel.notify_reports(resource=application,
-                                       user=user,
-                                       request=request,
-                                       since_when=since_when,
-                                       reports=confirmed_groups)
+                channel.notify_reports(
+                    resource=application,
+                    user=user,
+                    request=request,
+                    since_when=since_when,
+                    reports=confirmed_groups,
+                )
             except IntegrationException as e:
-                log.warning('%s' % e)
+                log.warning("%s" % e)

backend/src/appenlight/models/slow_call.py

@@ -1,115 +1,127 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import sqlalchemy as sa
 import hashlib
 
 from datetime import datetime, timedelta
 from appenlight.models import Base
 from sqlalchemy.dialects.postgresql import JSON
 from ziggurat_foundations.models.base import BaseModel
 
 
 class SlowCall(Base, BaseModel):
-    __tablename__ = 'slow_calls'
-    __table_args__ = {'implicit_returning': False}
+    __tablename__ = "slow_calls"
+    __table_args__ = {"implicit_returning": False}
 
     resource_id = sa.Column(sa.Integer(), nullable=False, index=True)
     id = sa.Column(sa.Integer, nullable=False, primary_key=True)
-    report_id = sa.Column(sa.BigInteger,
-                          sa.ForeignKey('reports.id',
-                                        ondelete='cascade',
-                                        onupdate='cascade'),
-                          primary_key=True)
+    report_id = sa.Column(
+        sa.BigInteger,
+        sa.ForeignKey("reports.id", ondelete="cascade", onupdate="cascade"),
+        primary_key=True,
+    )
     duration = sa.Column(sa.Float(), default=0)
-    statement = sa.Column(sa.UnicodeText(), default='')
-    statement_hash = sa.Column(sa.Unicode(60), default='')
+    statement = sa.Column(sa.UnicodeText(), default="")
+    statement_hash = sa.Column(sa.Unicode(60), default="")
     parameters = sa.Column(JSON(), nullable=False, default=dict)
-    type = sa.Column(sa.Unicode(16), default='')
+    type = sa.Column(sa.Unicode(16), default="")
     subtype = sa.Column(sa.Unicode(16), default=None)
-    location = sa.Column(sa.Unicode(255), default='')
-    timestamp = sa.Column(sa.DateTime(), default=datetime.utcnow,
-                          server_default=sa.func.now())
-    report_group_time = sa.Column(sa.DateTime(), default=datetime.utcnow,
-                                  server_default=sa.func.now())
+    location = sa.Column(sa.Unicode(255), default="")
+    timestamp = sa.Column(
+        sa.DateTime(), default=datetime.utcnow, server_default=sa.func.now()
+    )
+    report_group_time = sa.Column(
+        sa.DateTime(), default=datetime.utcnow, server_default=sa.func.now()
+    )
 
-    def set_data(self, data, protocol_version=None, resource_id=None,
-                 report_group=None):
+    def set_data(
+        self, data, protocol_version=None, resource_id=None, report_group=None
+    ):
         self.resource_id = resource_id
-        if data.get('start') and data.get('end'):
-            self.timestamp = data.get('start')
-            d = data.get('end') - data.get('start')
+        if data.get("start") and data.get("end"):
+            self.timestamp = data.get("start")
+            d = data.get("end") - data.get("start")
             self.duration = d.total_seconds()
-        self.statement = data.get('statement', '')
-        self.type = data.get('type', 'unknown')[:16]
-        self.parameters = data.get('parameters', {})
-        self.location = data.get('location', '')[:255]
+        self.statement = data.get("statement", "")
+        self.type = data.get("type", "unknown")[:16]
+        self.parameters = data.get("parameters", {})
+        self.location = data.get("location", "")[:255]
         self.report_group_time = report_group.first_timestamp
-        if 'subtype' in data:
-            self.subtype = data.get('subtype', 'unknown')[:16]
-        if self.type == 'tmpl':
-            self.set_hash('{} {}'.format(self.statement, self.parameters))
+        if "subtype" in data:
+            self.subtype = data.get("subtype", "unknown")[:16]
+        if self.type == "tmpl":
+            self.set_hash("{} {}".format(self.statement, self.parameters))
         else:
             self.set_hash()
 
     def set_hash(self, custom_statement=None):
         statement = custom_statement or self.statement
-        self.statement_hash = hashlib.sha1(
-            statement.encode('utf8')).hexdigest()
+        self.statement_hash = hashlib.sha1(statement.encode("utf8")).hexdigest()
 
     @property
     def end_time(self):
         if self.duration and self.timestamp:
             return self.timestamp + timedelta(seconds=self.duration)
         return None
 
     def get_dict(self):
         instance_dict = super(SlowCall, self).get_dict()
-        instance_dict['children'] = []
-        instance_dict['end_time'] = self.end_time
+        instance_dict["children"] = []
+        instance_dict["end_time"] = self.end_time
         return instance_dict
 
     def es_doc(self):
         doc = {
-            'resource_id': self.resource_id,
-            'timestamp': self.timestamp,
-            'pg_id': str(self.id),
-            'permanent': False,
-            'request_id': None,
-            'log_level': 'UNKNOWN',
-            'message': self.statement,
-            'namespace': 'appenlight.slow_call',
-            'tags': {
-                'report_id': {'values': self.report_id,
-                              'numeric_values': self.report_id},
-                'duration': {'values': None, 'numeric_values': self.duration},
-                'statement_hash': {'values': self.statement_hash,
-                                   'numeric_values': None},
-                'type': {'values': self.type, 'numeric_values': None},
-                'subtype': {'values': self.subtype, 'numeric_values': None},
-                'location': {'values': self.location, 'numeric_values': None},
-                'parameters': {'values': None, 'numeric_values': None}
+            "resource_id": self.resource_id,
+            "timestamp": self.timestamp,
+            "pg_id": str(self.id),
+            "permanent": False,
+            "request_id": None,
+            "log_level": "UNKNOWN",
+            "message": self.statement,
+            "namespace": "appenlight.slow_call",
+            "tags": {
+                "report_id": {
+                    "values": self.report_id,
+                    "numeric_values": self.report_id,
+                },
+                "duration": {"values": None, "numeric_values": self.duration},
+                "statement_hash": {
+                    "values": self.statement_hash,
+                    "numeric_values": None,
+                },
+                "type": {"values": self.type, "numeric_values": None},
+                "subtype": {"values": self.subtype, "numeric_values": None},
+                "location": {"values": self.location, "numeric_values": None},
+                "parameters": {"values": None, "numeric_values": None},
             },
-            'tag_list': ['report_id', 'duration', 'statement_hash', 'type',
-                         'subtype', 'location']
+            "tag_list": [
+                "report_id",
+                "duration",
+                "statement_hash",
+                "type",
+                "subtype",
+                "location",
+            ],
         }
         if isinstance(self.parameters, str):
-            doc['tags']['parameters']['values'] = self.parameters[:255]
+            doc["tags"]["parameters"]["values"] = self.parameters[:255]
         return doc
 
     @property
     def partition_id(self):
-        return 'rcae_sc_%s' % self.report_group_time.strftime('%Y_%m')
+        return "rcae_sc_%s" % self.report_group_time.strftime("%Y_%m")

backend/src/appenlight/models/tag.py

@@ -1,37 +1,38 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import sqlalchemy as sa
 from datetime import datetime
 from ziggurat_foundations.models.base import BaseModel
 from sqlalchemy.dialects.postgresql import JSON
 
 from . import Base
 
 
 class Tag(Base, BaseModel):
-    __tablename__ = 'tags'
+    __tablename__ = "tags"
 
     id = sa.Column(sa.Integer, primary_key=True)
-    resource_id = sa.Column(sa.Integer,
-                            sa.ForeignKey('resources.resource_id'))
+    resource_id = sa.Column(sa.Integer, sa.ForeignKey("resources.resource_id"))
     name = sa.Column(sa.Unicode(512), nullable=False)
     value = sa.Column(JSON, nullable=False)
-    first_timestamp = sa.Column(sa.DateTime(), default=datetime.utcnow,
-                                server_default=sa.func.now())
-    last_timestamp = sa.Column(sa.DateTime(), default=datetime.utcnow,
-                               server_default=sa.func.now())
+    first_timestamp = sa.Column(
+        sa.DateTime(), default=datetime.utcnow, server_default=sa.func.now()
+    )
+    last_timestamp = sa.Column(
+        sa.DateTime(), default=datetime.utcnow, server_default=sa.func.now()
+    )
     times_seen = sa.Column(sa.Integer, nullable=False, default=0)

backend/src/appenlight/models/user.py

@@ -1,133 +1,148 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 import sqlalchemy as sa
 from datetime import datetime
 from appenlight.models import Base, get_db_session
 from appenlight.models.services.event import EventService
 from appenlight.models.integrations import IntegrationException
 from pyramid.threadlocal import get_current_request
 from ziggurat_foundations.models.user import UserMixin
 from ziggurat_foundations.models.services.user import UserService
 
 log = logging.getLogger(__name__)
 
 
 class User(UserMixin, Base):
     __possible_permissions__ = []
 
     first_name = sa.Column(sa.Unicode(25))
     last_name = sa.Column(sa.Unicode(25))
-    company_name = sa.Column(sa.Unicode(255), default='')
-    company_address = sa.Column(sa.Unicode(255), default='')
-    zip_code = sa.Column(sa.Unicode(25), default='')
-    city = sa.Column(sa.Unicode(50), default='')
-    default_report_sort = sa.Column(sa.Unicode(25), default='newest')
-    notes = sa.Column(sa.UnicodeText, default='')
+    company_name = sa.Column(sa.Unicode(255), default="")
+    company_address = sa.Column(sa.Unicode(255), default="")
+    zip_code = sa.Column(sa.Unicode(25), default="")
+    city = sa.Column(sa.Unicode(50), default="")
+    default_report_sort = sa.Column(sa.Unicode(25), default="newest")
+    notes = sa.Column(sa.UnicodeText, default="")
     notifications = sa.Column(sa.Boolean(), default=True)
-    registration_ip = sa.Column(sa.UnicodeText(), default='')
-    alert_channels = sa.orm.relationship('AlertChannel',
-                                         cascade="all,delete-orphan",
-                                         passive_deletes=True,
-                                         passive_updates=True,
-                                         backref='owner',
-                                         order_by='AlertChannel.channel_name, '
-                                                  'AlertChannel.channel_value')
-
-    alert_actions = sa.orm.relationship('AlertChannelAction',
-                                        cascade="all,delete-orphan",
-                                        passive_deletes=True,
-                                        passive_updates=True,
-                                        backref='owner',
-                                        order_by='AlertChannelAction.pkey')
-
-    auth_tokens = sa.orm.relationship('AuthToken',
-                                      cascade="all,delete-orphan",
-                                      passive_deletes=True,
-                                      passive_updates=True,
-                                      backref='owner',
-                                      order_by='AuthToken.creation_date')
-
-    def get_dict(self, exclude_keys=None, include_keys=None,
-                 extended_info=False):
+    registration_ip = sa.Column(sa.UnicodeText(), default="")
+    alert_channels = sa.orm.relationship(
+        "AlertChannel",
+        cascade="all,delete-orphan",
+        passive_deletes=True,
+        passive_updates=True,
+        backref="owner",
+        order_by="AlertChannel.channel_name, " "AlertChannel.channel_value",
+    )
+
+    alert_actions = sa.orm.relationship(
+        "AlertChannelAction",
+        cascade="all,delete-orphan",
+        passive_deletes=True,
+        passive_updates=True,
+        backref="owner",
+        order_by="AlertChannelAction.pkey",
+    )
+
+    auth_tokens = sa.orm.relationship(
+        "AuthToken",
+        cascade="all,delete-orphan",
+        passive_deletes=True,
+        passive_updates=True,
+        backref="owner",
+        order_by="AuthToken.creation_date",
+    )
+
+    def get_dict(self, exclude_keys=None, include_keys=None, extended_info=False):
         result = super(User, self).get_dict(exclude_keys, include_keys)
         if extended_info:
-            result['groups'] = [g.group_name for g in self.groups]
-            result['permissions'] = [p.perm_name for p in UserService.permissions(self)]
+            result["groups"] = [g.group_name for g in self.groups]
+            result["permissions"] = [p.perm_name for p in UserService.permissions(self)]
             request = get_current_request()
-            apps = UserService.resources_with_perms(self,
-                ['view'], resource_types=['application'])
-            result['applications'] = sorted(
-                [{'resource_id': a.resource_id,
-                  'resource_name': a.resource_name}
-                 for a in apps.all()],
-                key=lambda x: x['resource_name'].lower())
-            result['assigned_reports'] = [r.get_dict(request) for r
-                                          in self.assigned_report_groups]
-            result['latest_events'] = [ev.get_dict(request) for ev
-                                       in self.latest_events()]
+            apps = UserService.resources_with_perms(
+                self, ["view"], resource_types=["application"]
+            )
+            result["applications"] = sorted(
+                [
+                    {"resource_id": a.resource_id, "resource_name": a.resource_name}
+                    for a in apps.all()
+                ],
+                key=lambda x: x["resource_name"].lower(),
+            )
+            result["assigned_reports"] = [
+                r.get_dict(request) for r in self.assigned_report_groups
+            ]
+            result["latest_events"] = [
+                ev.get_dict(request) for ev in self.latest_events()
+            ]
 
         exclude_keys_list = exclude_keys or []
         include_keys_list = include_keys or []
         d = {}
         for k in result.keys():
-            if (k not in exclude_keys_list and
-                    (k in include_keys_list or not include_keys)):
+            if k not in exclude_keys_list and (
+                k in include_keys_list or not include_keys
+            ):
                 d[k] = result[k]
         return d
 
     def __repr__(self):
-        return '<User: %s, id: %s>' % (self.user_name, self.id)
+        return "<User: %s, id: %s>" % (self.user_name, self.id)
 
     @property
     def assigned_report_groups(self):
         from appenlight.models.report_group import ReportGroup
 
-        resources = UserService.resources_with_perms(self, ['view'], resource_types=['application'])
+        resources = UserService.resources_with_perms(
+            self, ["view"], resource_types=["application"]
+        )
         query = self.assigned_reports_relation
         rid_list = [r.resource_id for r in resources]
         query = query.filter(ReportGroup.resource_id.in_(rid_list))
         query = query.limit(50)
         return query
 
     def feed_report(self, report):
         """ """
-        if not hasattr(self, 'current_reports'):
+        if not hasattr(self, "current_reports"):
             self.current_reports = []
         self.current_reports.append(report)
 
-    def send_digest(self, request, application, reports, since_when=None,
-                    db_session=None):
+    def send_digest(
+        self, request, application, reports, since_when=None, db_session=None
+    ):
         db_session = get_db_session(db_session)
         if not reports:
             return True
         if not since_when:
             since_when = datetime.utcnow()
         for channel in self.alert_channels:
             if not channel.channel_validated or not channel.daily_digest:
                 continue
             try:
-                channel.send_digest(resource=application,
-                                    user=self,
-                                    request=request,
-                                    since_when=since_when,
-                                    reports=reports)
+                channel.send_digest(
+                    resource=application,
+                    user=self,
+                    request=request,
+                    since_when=since_when,
+                    reports=reports,
+                )
             except IntegrationException as e:
-                log.warning('%s' % e)
+                log.warning("%s" % e)
 
     def latest_events(self):
         return EventService.latest_for_user(self)

backend/src/appenlight/models/user_resource_permission.py

@@ -1,22 +1,24 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ziggurat_foundations.models.user_resource_permission import UserResourcePermissionMixin
+from ziggurat_foundations.models.user_resource_permission import (
+    UserResourcePermissionMixin,
+)
 from appenlight.models import Base
 
 
 class UserResourcePermission(UserResourcePermissionMixin, Base):
     pass

backend/src/appenlight/predicates.py

@@ -1,143 +1,135 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 from appenlight.forms import CSRFException
 
 log = logging.getLogger(__name__)
 
 from pyramid.interfaces import IDefaultCSRFOptions
-from pyramid.session import (
-    check_csrf_origin,
-    check_csrf_token,
-)
+from pyramid.session import check_csrf_origin, check_csrf_token
 
 # taken directly from pyramid 1.7
 # pyramid/viewderivers.py
 # the difference is this deriver will ignore csrf_check when auth token
 # policy is in effect
 
+
 def csrf_view(view, info):
-    explicit_val = info.options.get('require_csrf')
+    explicit_val = info.options.get("require_csrf")
     defaults = info.registry.queryUtility(IDefaultCSRFOptions)
     if defaults is None:
         default_val = False
-        token = 'csrf_token'
-        header = 'X-CSRF-Token'
+        token = "csrf_token"
+        header = "X-CSRF-Token"
         safe_methods = frozenset(["GET", "HEAD", "OPTIONS", "TRACE"])
     else:
         default_val = defaults.require_csrf
         token = defaults.token
         header = defaults.header
         safe_methods = defaults.safe_methods
-    enabled = (
-        explicit_val is True or
-        (explicit_val is not False and default_val)
-    )
+    enabled = explicit_val is True or (explicit_val is not False and default_val)
     # disable if both header and token are disabled
     enabled = enabled and (token or header)
     wrapped_view = view
     if enabled:
+
         def csrf_view(context, request):
-            is_from_auth_token = 'auth:auth_token' in \
-                                 request.effective_principals
+            is_from_auth_token = "auth:auth_token" in request.effective_principals
             if is_from_auth_token:
-                log.debug('ignoring CSRF check, auth token used')
-            elif (
-                request.method not in safe_methods and
-                (
+                log.debug("ignoring CSRF check, auth token used")
+            elif request.method not in safe_methods and (
                 # skip exception views unless value is explicitly defined
-                        getattr(request, 'exception', None) is None or
-                        explicit_val is not None
-                )
+                getattr(request, "exception", None) is None
+                or explicit_val is not None
             ):
                 check_csrf_origin(request, raises=True)
                 check_csrf_token(request, token, header, raises=True)
             return view(context, request)
 
         wrapped_view = csrf_view
     return wrapped_view
 
-csrf_view.options = ('require_csrf',)
+
+csrf_view.options = ("require_csrf",)
 
 
 class PublicReportGroup(object):
     def __init__(self, val, config):
         self.val = val
 
     def text(self):
-        return 'public_report_group = %s' % (self.val,)
+        return "public_report_group = %s" % (self.val,)
 
     phash = text
 
     def __call__(self, context, request):
-        report_group = getattr(context, 'report_group', None)
+        report_group = getattr(context, "report_group", None)
         if report_group:
             return context.report_group.public == self.val
 
 
 class contextTypeClass(object):
     def __init__(self, context_property, config):
         self.context_property = context_property[0]
         self.cls = context_property[1]
 
     def text(self):
-        return 'context_type_class = %s, %s' % (
-            self.context_property, self.cls)
+        return "context_type_class = %s, %s" % (self.context_property, self.cls)
 
     phash = text
 
     def __call__(self, context, request):
         to_check = getattr(context, self.context_property, None)
         return isinstance(to_check, self.cls)
 
 
 def unauthed_report_predicate(context, request):
     """
     This allows the user to access the view if context object public
     flag is True
     """
     if context.public:
         return True
 
 
 def unauthed_report_predicate_inv(context, request):
     """
     This allows the user to access the view if context object public
     flag is NOT True
     """
     if context.public:
         return False
     return True
 
 
 def unauthed_report_predicate(context, request):
     """
     This allows the user to access the view if context object public
     flag is True
     """
     if context.public:
         return True
 
 
 def unauthed_report_predicate_inv(context, request):
     """
     This allows the user to access the view if context object public
     flag is NOT True
     """
     if context.public:
         return False
     return True

backend/src/appenlight/scripts/__init__.py

@@ -1,16 +1,15 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-

backend/src/appenlight/scripts/cleanup.py

@@ -1,63 +1,66 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 import argparse
 from pyramid.paster import setup_logging
 from pyramid.paster import bootstrap
 from appenlight.celery.tasks import logs_cleanup
 
 log = logging.getLogger(__name__)
 
 
 def main():
-    choices = ['logs']
-
-    parser = argparse.ArgumentParser(description='Cleanup AppEnlight logs')
-    parser.add_argument('-c', '--config', required=True,
-                        help='Configuration ini file of application')
-    parser.add_argument('-t', '--types', choices=choices,
-                        default='logs',
-                        help='Which parts of database should get cleared')
-    parser.add_argument('-r', '--resource', required=True, help='Resource id')
-    parser.add_argument('-n', '--namespace', help='Limit to Namespace')
+    choices = ["logs"]
+
+    parser = argparse.ArgumentParser(description="Cleanup AppEnlight logs")
+    parser.add_argument(
+        "-c", "--config", required=True, help="Configuration ini file of application"
+    )
+    parser.add_argument(
+        "-t",
+        "--types",
+        choices=choices,
+        default="logs",
+        help="Which parts of database should get cleared",
+    )
+    parser.add_argument("-r", "--resource", required=True, help="Resource id")
+    parser.add_argument("-n", "--namespace", help="Limit to Namespace")
     args = parser.parse_args()
 
     config_uri = args.config
     setup_logging(config_uri)
     log.setLevel(logging.INFO)
     env = bootstrap(config_uri)
 
     config = {
-        'types': args.types,
-        'namespace': args.namespace,
-        'resource': int(args.resource),
+        "types": args.types,
+        "namespace": args.namespace,
+        "resource": int(args.resource),
     }
 
     action_cleanup_logs(config)
 
 
 def action_cleanup_logs(config):
-    filter_settings = {
-        'namespace': []
-    }
-    if config['namespace']:
-        filter_settings['namespace'].append(config['namespace'])
-    logs_cleanup(config['resource'], filter_settings)
+    filter_settings = {"namespace": []}
+    if config["namespace"]:
+        filter_settings["namespace"].append(config["namespace"])
+    logs_cleanup(config["resource"], filter_settings)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

backend/src/appenlight/scripts/initialize_db.py

@@ -1,155 +1,149 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import argparse
 import getpass
 import logging
 
 from pyramid.paster import setup_logging, bootstrap
 from pyramid.threadlocal import get_current_request
 from ziggurat_foundations.models.services.user import UserService
 
 
 from appenlight.forms import UserRegisterForm
 from appenlight.lib.ext_json import json
-from appenlight.models import (
-    DBSession,
-    Group,
-    GroupPermission,
-    User,
-    AuthToken
-)
+from appenlight.models import DBSession, Group, GroupPermission, User, AuthToken
 from appenlight.models.services.group import GroupService
 
 log = logging.getLogger(__name__)
 
 _ = str
 
 
 def is_yes(input_data):
-    return input_data in ['y', 'yes']
+    return input_data in ["y", "yes"]
 
 
 def is_no(input_data):
-    return input_data in ['n', 'no']
+    return input_data in ["n", "no"]
 
 
 def main():
     parser = argparse.ArgumentParser(
-        description='Populate AppEnlight database',
-        add_help=False)
-    parser.add_argument('-c', '--config', required=True,
-                        help='Configuration ini file of application')
-    parser.add_argument('--username', default=None,
-                        help='User  to create')
-    parser.add_argument('--password', default=None,
-                        help='Password for created user')
-    parser.add_argument('--email', default=None,
-                        help='Email for created user')
-    parser.add_argument('--auth-token', default=None,
-                        help='Auth token for created user')
+        description="Populate AppEnlight database", add_help=False
+    )
+    parser.add_argument(
+        "-c", "--config", required=True, help="Configuration ini file of application"
+    )
+    parser.add_argument("--username", default=None, help="User  to create")
+    parser.add_argument("--password", default=None, help="Password for created user")
+    parser.add_argument("--email", default=None, help="Email for created user")
+    parser.add_argument(
+        "--auth-token", default=None, help="Auth token for created user"
+    )
     args = parser.parse_args()
     config_uri = args.config
 
     setup_logging(config_uri)
     env = bootstrap(config_uri)
-    request = env['request']
+    request = env["request"]
     with get_current_request().tm:
         group = GroupService.by_id(1)
         if not group:
-            group = Group(id=1, group_name='Administrators',
-                           description="Top level permission owners")
+            group = Group(
+                id=1,
+                group_name="Administrators",
+                description="Top level permission owners",
+            )
             DBSession.add(group)
-            permission = GroupPermission(perm_name='root_administration')
+            permission = GroupPermission(perm_name="root_administration")
             group.permissions.append(permission)
 
     create_user = True if args.username else None
     while create_user is None:
-        response = input(
-            'Do you want to create a new admin? (n)\n').lower()
+        response = input("Do you want to create a new admin? (n)\n").lower()
 
-        if is_yes(response or 'n'):
+        if is_yes(response or "n"):
             create_user = True
-        elif is_no(response or 'n'):
+        elif is_no(response or "n"):
             create_user = False
 
     if create_user:
         csrf_token = request.session.get_csrf_token()
         user_name = args.username
-        print('*********************************************************')
+        print("*********************************************************")
         while user_name is None:
-            response = input('What is the username of new admin?\n')
+            response = input("What is the username of new admin?\n")
             form = UserRegisterForm(
-                user_name=response, csrf_token=csrf_token,
-                csrf_context=request)
+                user_name=response, csrf_token=csrf_token, csrf_context=request
+            )
             form.validate()
             if form.user_name.errors:
                 print(form.user_name.errors[0])
             else:
                 user_name = response
                 print('The admin username is "{}"\n'.format(user_name))
-        print('*********************************************************')
+        print("*********************************************************")
         email = args.email
         while email is None:
-            response = input('What is the email of admin account?\n')
+            response = input("What is the email of admin account?\n")
             form = UserRegisterForm(
-                email=response, csrf_token=csrf_token,
-                csrf_context=request)
+                email=response, csrf_token=csrf_token, csrf_context=request
+            )
             form.validate()
             if form.email.errors:
                 print(form.email.errors[0])
             else:
                 email = response
                 print('The admin email is "{}"\n'.format(email))
-        print('*********************************************************')
+        print("*********************************************************")
         user_password = args.password
         confirmed_password = args.password
         while user_password is None or confirmed_password is None:
-            response = getpass.getpass(
-                'What is the password for admin account?\n')
+            response = getpass.getpass("What is the password for admin account?\n")
             form = UserRegisterForm(
-                user_password=response, csrf_token=csrf_token,
-                csrf_context=request)
+                user_password=response, csrf_token=csrf_token, csrf_context=request
+            )
             form.validate()
             if form.user_password.errors:
                 print(form.user_password.errors[0])
             else:
                 user_password = response
 
-            response = getpass.getpass('Please confirm the password.\n')
+            response = getpass.getpass("Please confirm the password.\n")
             if user_password == response:
                 confirmed_password = response
             else:
-                print('Passwords do not match. Please try again')
-        print('*********************************************************')
+                print("Passwords do not match. Please try again")
+        print("*********************************************************")
 
     with get_current_request().tm:
         if create_user:
             group = GroupService.by_id(1)
             user = User(user_name=user_name, email=email, status=1)
             UserService.regenerate_security_code(user)
             UserService.set_password(user, user_password)
             DBSession.add(user)
             token = AuthToken(description="Uptime monitoring token")
             if args.auth_token:
                 token.token = args.auth_token
             user.auth_tokens.append(token)
             group.users.append(user)
-            print('USER CREATED')
+            print("USER CREATED")
             print(json.dumps(user.get_dict()))
-            print('*********************************************************')
-            print('AUTH TOKEN')
+            print("*********************************************************")
+            print("AUTH TOKEN")
             print(json.dumps(user.auth_tokens[0].get_dict()))

backend/src/appenlight/scripts/make_config.py

@@ -1,57 +1,60 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import argparse
 import logging
 import os
 import pkg_resources
 import jinja2
 from cryptography.fernet import Fernet
 
 
 log = logging.getLogger(__name__)
 
 
 def gen_secret():
-    return Fernet.generate_key().decode('utf8')
+    return Fernet.generate_key().decode("utf8")
 
 
 def main():
     parser = argparse.ArgumentParser(
-        description='Generate AppEnlight static resources',
-        add_help=False)
-    parser.add_argument('config', help='Name of generated file')
+        description="Generate AppEnlight static resources", add_help=False
+    )
+    parser.add_argument("config", help="Name of generated file")
     parser.add_argument(
-        '--domain',
-        default='appenlight-rhodecode.local',
-        help='Domain which will be used to serve the application')
+        "--domain",
+        default="appenlight-rhodecode.local",
+        help="Domain which will be used to serve the application",
+    )
     parser.add_argument(
-        '--dbstring',
-        default='postgresql://appenlight:test@127.0.0.1:5432/appenlight',
-        help='Domain which will be used to serve the application')
+        "--dbstring",
+        default="postgresql://appenlight:test@127.0.0.1:5432/appenlight",
+        help="Domain which will be used to serve the application",
+    )
     args = parser.parse_args()
-    ini_path = os.path.join('templates', 'ini', 'production.ini.jinja2')
-    template_str = pkg_resources.resource_string('appenlight', ini_path)
-    template = jinja2.Template(template_str.decode('utf8'))
-    template_vars = {'appenlight_encryption_secret': gen_secret(),
-                     'appenlight_authtkt_secret': gen_secret(),
-                     'appenlight_redis_session_secret': gen_secret(),
-                     'appenlight_domain': args.domain,
-                     'appenlight_dbstring': args.dbstring,
-                     }
+    ini_path = os.path.join("templates", "ini", "production.ini.jinja2")
+    template_str = pkg_resources.resource_string("appenlight", ini_path)
+    template = jinja2.Template(template_str.decode("utf8"))
+    template_vars = {
+        "appenlight_encryption_secret": gen_secret(),
+        "appenlight_authtkt_secret": gen_secret(),
+        "appenlight_redis_session_secret": gen_secret(),
+        "appenlight_domain": args.domain,
+        "appenlight_dbstring": args.dbstring,
+    }
     compiled = template.render(**template_vars)
-    with open(args.config, 'w') as f:
+    with open(args.config, "w") as f:
         f.write(compiled)

backend/src/appenlight/scripts/migratedb.py

@@ -1,73 +1,73 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import argparse
 import logging
 import sys
 
 from alembic.config import Config
 from alembic import command
 from pyramid.paster import setup_logging, bootstrap
 from pyramid.threadlocal import get_current_registry, get_current_request
 
 from appenlight.lib import get_callable
 from appenlight.models.services.config import ConfigService
 
 log = logging.getLogger(__name__)
 
 
 def main(argv=sys.argv):
     parser = argparse.ArgumentParser(
-        description='Migrate AppEnlight database to latest version',
-        add_help=False)
-    parser.add_argument('-c', '--config', required=True,
-                        help='Configuration ini file of application')
+        description="Migrate AppEnlight database to latest version", add_help=False
+    )
+    parser.add_argument(
+        "-c", "--config", required=True, help="Configuration ini file of application"
+    )
     args = parser.parse_args()
     config_uri = args.config
 
     setup_logging(config_uri)
     bootstrap(config_uri)
     registry = get_current_registry()
     alembic_cfg = Config()
-    alembic_cfg.set_main_option("sqlalchemy.echo", 'true')
-    alembic_cfg.set_main_option("script_location",
-                                "ziggurat_foundations:migrations")
-    alembic_cfg.set_main_option("sqlalchemy.url",
-                                registry.settings["sqlalchemy.url"])
+    alembic_cfg.set_main_option("sqlalchemy.echo", "true")
+    alembic_cfg.set_main_option("script_location", "ziggurat_foundations:migrations")
+    alembic_cfg.set_main_option("sqlalchemy.url", registry.settings["sqlalchemy.url"])
     command.upgrade(alembic_cfg, "head")
     alembic_cfg = Config()
-    alembic_cfg.set_main_option("sqlalchemy.echo", 'true')
+    alembic_cfg.set_main_option("sqlalchemy.echo", "true")
     alembic_cfg.set_main_option("script_location", "appenlight:migrations")
-    alembic_cfg.set_main_option("sqlalchemy.url",
-                                registry.settings["sqlalchemy.url"])
+    alembic_cfg.set_main_option("sqlalchemy.url", registry.settings["sqlalchemy.url"])
     command.upgrade(alembic_cfg, "head")
 
     for plugin_name, config in registry.appenlight_plugins.items():
-        if config['sqlalchemy_migrations']:
+        if config["sqlalchemy_migrations"]:
             alembic_cfg = Config()
-            alembic_cfg.set_main_option("script_location",
-                                        config['sqlalchemy_migrations'])
-            alembic_cfg.set_main_option("sqlalchemy.url",
-                                        registry.settings["sqlalchemy.url"])
-            alembic_cfg.set_main_option("sqlalchemy.echo", 'true')
+            alembic_cfg.set_main_option(
+                "script_location", config["sqlalchemy_migrations"]
+            )
+            alembic_cfg.set_main_option(
+                "sqlalchemy.url", registry.settings["sqlalchemy.url"]
+            )
+            alembic_cfg.set_main_option("sqlalchemy.echo", "true")
             command.upgrade(alembic_cfg, "head")
 
     with get_current_request().tm:
         ConfigService.setup_default_values()
 
         for plugin_name, config in registry.appenlight_plugins.items():
-            if config['default_values_setter']:
-                get_callable(config['default_values_setter'])()
+            if config["default_values_setter"]:
+                get_callable(config["default_values_setter"])()

backend/src/appenlight/scripts/reindex_elasticsearch.py

@@ -1,443 +1,437 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import argparse
 import datetime
 import logging
 
 import sqlalchemy as sa
 import elasticsearch.exceptions
 import elasticsearch.helpers
 
 from collections import defaultdict
 from pyramid.paster import setup_logging
 from pyramid.paster import bootstrap
-from appenlight.models import (
-    DBSession,
-    Datastores,
-    metadata
-)
+from appenlight.models import DBSession, Datastores, metadata
 from appenlight.lib import get_callable
 from appenlight.models.report_group import ReportGroup
 from appenlight.models.report import Report
 from appenlight.models.report_stat import ReportStat
 from appenlight.models.log import Log
 from appenlight.models.slow_call import SlowCall
 from appenlight.models.metric import Metric
 
 
 log = logging.getLogger(__name__)
 
 tables = {
-    'slow_calls_p_': [],
-    'reports_stats_p_': [],
-    'reports_p_': [],
-    'reports_groups_p_': [],
-    'logs_p_': [],
-    'metrics_p_': [],
+    "slow_calls_p_": [],
+    "reports_stats_p_": [],
+    "reports_p_": [],
+    "reports_groups_p_": [],
+    "logs_p_": [],
+    "metrics_p_": [],
 }
 
+
 def detect_tables(table_prefix):
     found_tables = []
-    db_tables_query = '''
+    db_tables_query = """
     SELECT tablename FROM pg_tables WHERE tablename NOT LIKE 'pg_%' AND
-    tablename NOT LIKE 'sql_%' ORDER BY tablename ASC;'''
+    tablename NOT LIKE 'sql_%' ORDER BY tablename ASC;"""
 
     for table in DBSession.execute(db_tables_query).fetchall():
         tablename = table.tablename
         if tablename.startswith(table_prefix):
-            t = sa.Table(tablename, metadata, autoload=True,
-                         autoload_with=DBSession.bind.engine)
+            t = sa.Table(
+                tablename, metadata, autoload=True, autoload_with=DBSession.bind.engine
+            )
             found_tables.append(t)
     return found_tables
 
 
 def main():
     """
     Recreates Elasticsearch indexes
     Performs reindex of whole db to Elasticsearch
 
     """
 
     # need parser twice because we first need to load ini file
     # bootstrap pyramid and then load plugins
     pre_parser = argparse.ArgumentParser(
-        description='Reindex AppEnlight data',
-        add_help=False)
-    pre_parser.add_argument('-c', '--config', required=True,
-                            help='Configuration ini file of application')
-    pre_parser.add_argument('-h', '--help', help='Show help', nargs='?')
-    pre_parser.add_argument('-t', '--types', nargs='+',
-                            help='Which parts of database should get reindexed')
+        description="Reindex AppEnlight data", add_help=False
+    )
+    pre_parser.add_argument(
+        "-c", "--config", required=True, help="Configuration ini file of application"
+    )
+    pre_parser.add_argument("-h", "--help", help="Show help", nargs="?")
+    pre_parser.add_argument(
+        "-t", "--types", nargs="+", help="Which parts of database should get reindexed"
+    )
     args = pre_parser.parse_args()
 
     config_uri = args.config
     setup_logging(config_uri)
     log.setLevel(logging.INFO)
     env = bootstrap(config_uri)
-    parser = argparse.ArgumentParser(description='Reindex AppEnlight data')
+    parser = argparse.ArgumentParser(description="Reindex AppEnlight data")
     choices = {
-        'reports': 'appenlight.scripts.reindex_elasticsearch:reindex_reports',
-        'logs': 'appenlight.scripts.reindex_elasticsearch:reindex_logs',
-        'metrics': 'appenlight.scripts.reindex_elasticsearch:reindex_metrics',
-        'slow_calls': 'appenlight.scripts.reindex_elasticsearch:reindex_slow_calls',
-        'template': 'appenlight.scripts.reindex_elasticsearch:update_template'
+        "reports": "appenlight.scripts.reindex_elasticsearch:reindex_reports",
+        "logs": "appenlight.scripts.reindex_elasticsearch:reindex_logs",
+        "metrics": "appenlight.scripts.reindex_elasticsearch:reindex_metrics",
+        "slow_calls": "appenlight.scripts.reindex_elasticsearch:reindex_slow_calls",
+        "template": "appenlight.scripts.reindex_elasticsearch:update_template",
     }
-    for k, v in env['registry'].appenlight_plugins.items():
-        if v.get('fulltext_indexer'):
-            choices[k] = v['fulltext_indexer']
-    parser.add_argument('-t', '--types', nargs='*',
-                        choices=['all'] + list(choices.keys()), default=[],
-                        help='Which parts of database should get reindexed')
-    parser.add_argument('-c', '--config', required=True,
-                        help='Configuration ini file of application')
+    for k, v in env["registry"].appenlight_plugins.items():
+        if v.get("fulltext_indexer"):
+            choices[k] = v["fulltext_indexer"]
+    parser.add_argument(
+        "-t",
+        "--types",
+        nargs="*",
+        choices=["all"] + list(choices.keys()),
+        default=[],
+        help="Which parts of database should get reindexed",
+    )
+    parser.add_argument(
+        "-c", "--config", required=True, help="Configuration ini file of application"
+    )
     args = parser.parse_args()
 
-
-    if 'all' in args.types:
+    if "all" in args.types:
         args.types = list(choices.keys())
 
     print("Selected types to reindex: {}".format(args.types))
 
-    log.info('settings {}'.format(args.types))
+    log.info("settings {}".format(args.types))
 
-    if 'template' in args.types:
-        get_callable(choices['template'])()
-        args.types.remove('template')
+    if "template" in args.types:
+        get_callable(choices["template"])()
+        args.types.remove("template")
     for selected in args.types:
         get_callable(choices[selected])()
 
 
 def update_template():
     try:
-        Datastores.es.indices.delete_template('rcae')
+        Datastores.es.indices.delete_template("rcae")
     except elasticsearch.exceptions.NotFoundError as e:
         log.error(e)
-    log.info('updating elasticsearch template')
+    log.info("updating elasticsearch template")
     tag_templates = [
-        {"values": {
-            "path_match": "tags.*",
-            "mapping": {
-                "type": "object",
-                "properties": {
-                    "values": {"type": "string", "analyzer": "tag_value"},
-                    "numeric_values": {"type": "float"}
-                }
+        {
+            "values": {
+                "path_match": "tags.*",
+                "mapping": {
+                    "type": "object",
+                    "properties": {
+                        "values": {"type": "string", "analyzer": "tag_value"},
+                        "numeric_values": {"type": "float"},
+                    },
+                },
             }
-        }}
+        }
     ]
 
     template_schema = {
         "template": "rcae_*",
         "settings": {
             "index": {
                 "refresh_interval": "5s",
-                "translog": {"sync_interval": "5s",
-                             "durability": "async"}
+                "translog": {"sync_interval": "5s", "durability": "async"},
             },
             "number_of_shards": 5,
             "analysis": {
                 "analyzer": {
                     "url_path": {
                         "type": "custom",
                         "char_filter": [],
                         "tokenizer": "path_hierarchy",
-                        "filter": []
+                        "filter": [],
                     },
                     "tag_value": {
                         "type": "custom",
                         "char_filter": [],
                         "tokenizer": "keyword",
-                        "filter": ["lowercase"]
+                        "filter": ["lowercase"],
                     },
                 }
             },
         },
         "mappings": {
             "report_group": {
                 "_all": {"enabled": False},
                 "dynamic_templates": tag_templates,
                 "properties": {
                     "pg_id": {"type": "string", "index": "not_analyzed"},
                     "resource_id": {"type": "integer"},
                     "priority": {"type": "integer"},
                     "error": {"type": "string", "analyzer": "simple"},
                     "read": {"type": "boolean"},
                     "occurences": {"type": "integer"},
                     "fixed": {"type": "boolean"},
                     "first_timestamp": {"type": "date"},
                     "last_timestamp": {"type": "date"},
                     "average_duration": {"type": "float"},
                     "summed_duration": {"type": "float"},
-                    "public": {"type": "boolean"}
-                }
+                    "public": {"type": "boolean"},
+                },
             },
             "report": {
                 "_all": {"enabled": False},
                 "dynamic_templates": tag_templates,
                 "properties": {
                     "pg_id": {"type": "string", "index": "not_analyzed"},
                     "resource_id": {"type": "integer"},
                     "group_id": {"type": "string"},
                     "http_status": {"type": "integer"},
                     "ip": {"type": "string", "index": "not_analyzed"},
                     "url_domain": {"type": "string", "analyzer": "simple"},
                     "url_path": {"type": "string", "analyzer": "url_path"},
                     "error": {"type": "string", "analyzer": "simple"},
                     "report_type": {"type": "integer"},
                     "start_time": {"type": "date"},
                     "request_id": {"type": "string", "index": "not_analyzed"},
                     "end_time": {"type": "date"},
                     "duration": {"type": "float"},
-                    "tags": {
-                        "type": "object"
-                    },
+                    "tags": {"type": "object"},
                     "tag_list": {"type": "string", "analyzer": "tag_value"},
-                    "extra": {
-                        "type": "object"
-                    },
+                    "extra": {"type": "object"},
                 },
-                "_parent": {"type": "report_group"}
+                "_parent": {"type": "report_group"},
             },
             "log": {
                 "_all": {"enabled": False},
                 "dynamic_templates": tag_templates,
                 "properties": {
                     "pg_id": {"type": "string", "index": "not_analyzed"},
                     "delete_hash": {"type": "string", "index": "not_analyzed"},
                     "resource_id": {"type": "integer"},
                     "timestamp": {"type": "date"},
                     "permanent": {"type": "boolean"},
                     "request_id": {"type": "string", "index": "not_analyzed"},
                     "log_level": {"type": "string", "analyzer": "simple"},
                     "message": {"type": "string", "analyzer": "simple"},
                     "namespace": {"type": "string", "index": "not_analyzed"},
-                    "tags": {
-                        "type": "object"
-                    },
-                    "tag_list": {"type": "string", "analyzer": "tag_value"}
-                }
-            }
-        }
+                    "tags": {"type": "object"},
+                    "tag_list": {"type": "string", "analyzer": "tag_value"},
+                },
+            },
+        },
     }
 
-    Datastores.es.indices.put_template('rcae', body=template_schema)
+    Datastores.es.indices.put_template("rcae", body=template_schema)
 
 
 def reindex_reports():
-    reports_groups_tables = detect_tables('reports_groups_p_')
+    reports_groups_tables = detect_tables("reports_groups_p_")
     try:
-        Datastores.es.indices.delete('rcae_r*')
+        Datastores.es.indices.delete("rcae_r*")
     except elasticsearch.exceptions.NotFoundError as e:
         log.error(e)
 
-    log.info('reindexing report groups')
+    log.info("reindexing report groups")
     i = 0
     task_start = datetime.datetime.now()
     for partition_table in reports_groups_tables:
         conn = DBSession.connection().execution_options(stream_results=True)
         result = conn.execute(partition_table.select())
         while True:
             chunk = result.fetchmany(2000)
             if not chunk:
                 break
             es_docs = defaultdict(list)
             for row in chunk:
                 i += 1
                 item = ReportGroup(**dict(list(row.items())))
                 d_range = item.partition_id
                 es_docs[d_range].append(item.es_doc())
             if es_docs:
                 name = partition_table.name
-                log.info('round {}, {}'.format(i, name))
+                log.info("round {}, {}".format(i, name))
                 for k, v in es_docs.items():
-                    to_update = {'_index': k, '_type': 'report_group'}
+                    to_update = {"_index": k, "_type": "report_group"}
                     [i.update(to_update) for i in v]
                     elasticsearch.helpers.bulk(Datastores.es, v)
 
-    log.info(
-        'total docs {} {}'.format(i, datetime.datetime.now() - task_start))
+    log.info("total docs {} {}".format(i, datetime.datetime.now() - task_start))
 
     i = 0
-    log.info('reindexing reports')
+    log.info("reindexing reports")
     task_start = datetime.datetime.now()
-    reports_tables = detect_tables('reports_p_')
+    reports_tables = detect_tables("reports_p_")
     for partition_table in reports_tables:
         conn = DBSession.connection().execution_options(stream_results=True)
         result = conn.execute(partition_table.select())
         while True:
             chunk = result.fetchmany(2000)
             if not chunk:
                 break
             es_docs = defaultdict(list)
             for row in chunk:
                 i += 1
                 item = Report(**dict(list(row.items())))
                 d_range = item.partition_id
                 es_docs[d_range].append(item.es_doc())
             if es_docs:
                 name = partition_table.name
-                log.info('round {}, {}'.format(i, name))
+                log.info("round {}, {}".format(i, name))
                 for k, v in es_docs.items():
-                    to_update = {'_index': k, '_type': 'report'}
+                    to_update = {"_index": k, "_type": "report"}
                     [i.update(to_update) for i in v]
                     elasticsearch.helpers.bulk(Datastores.es, v)
 
-    log.info(
-        'total docs {} {}'.format(i, datetime.datetime.now() - task_start))
+    log.info("total docs {} {}".format(i, datetime.datetime.now() - task_start))
 
-    log.info('reindexing reports stats')
+    log.info("reindexing reports stats")
     i = 0
     task_start = datetime.datetime.now()
-    reports_stats_tables = detect_tables('reports_stats_p_')
+    reports_stats_tables = detect_tables("reports_stats_p_")
     for partition_table in reports_stats_tables:
         conn = DBSession.connection().execution_options(stream_results=True)
         result = conn.execute(partition_table.select())
         while True:
             chunk = result.fetchmany(2000)
             if not chunk:
                 break
             es_docs = defaultdict(list)
             for row in chunk:
                 rd = dict(list(row.items()))
                 # remove legacy columns
                 # TODO: remove the column later
-                rd.pop('size', None)
+                rd.pop("size", None)
                 item = ReportStat(**rd)
                 i += 1
                 d_range = item.partition_id
                 es_docs[d_range].append(item.es_doc())
             if es_docs:
                 name = partition_table.name
-                log.info('round  {}, {}'.format(i, name))
+                log.info("round  {}, {}".format(i, name))
                 for k, v in es_docs.items():
-                    to_update = {'_index': k, '_type': 'log'}
+                    to_update = {"_index": k, "_type": "log"}
                     [i.update(to_update) for i in v]
                     elasticsearch.helpers.bulk(Datastores.es, v)
 
-    log.info(
-        'total docs {} {}'.format(i, datetime.datetime.now() - task_start))
+    log.info("total docs {} {}".format(i, datetime.datetime.now() - task_start))
 
 
 def reindex_logs():
     try:
-        Datastores.es.indices.delete('rcae_l*')
+        Datastores.es.indices.delete("rcae_l*")
     except elasticsearch.exceptions.NotFoundError as e:
         log.error(e)
 
     # logs
-    log.info('reindexing logs')
+    log.info("reindexing logs")
     i = 0
     task_start = datetime.datetime.now()
-    log_tables = detect_tables('logs_p_')
+    log_tables = detect_tables("logs_p_")
     for partition_table in log_tables:
         conn = DBSession.connection().execution_options(stream_results=True)
         result = conn.execute(partition_table.select())
         while True:
             chunk = result.fetchmany(2000)
             if not chunk:
                 break
             es_docs = defaultdict(list)
 
             for row in chunk:
                 i += 1
                 item = Log(**dict(list(row.items())))
                 d_range = item.partition_id
                 es_docs[d_range].append(item.es_doc())
             if es_docs:
                 name = partition_table.name
-                log.info('round  {}, {}'.format(i, name))
+                log.info("round  {}, {}".format(i, name))
                 for k, v in es_docs.items():
-                    to_update = {'_index': k, '_type': 'log'}
+                    to_update = {"_index": k, "_type": "log"}
                     [i.update(to_update) for i in v]
                     elasticsearch.helpers.bulk(Datastores.es, v)
 
-    log.info(
-        'total docs {} {}'.format(i, datetime.datetime.now() - task_start))
+    log.info("total docs {} {}".format(i, datetime.datetime.now() - task_start))
 
 
 def reindex_metrics():
     try:
-        Datastores.es.indices.delete('rcae_m*')
+        Datastores.es.indices.delete("rcae_m*")
     except elasticsearch.exceptions.NotFoundError as e:
         log.error(e)
 
-    log.info('reindexing applications metrics')
+    log.info("reindexing applications metrics")
     i = 0
     task_start = datetime.datetime.now()
-    metric_tables = detect_tables('metrics_p_')
+    metric_tables = detect_tables("metrics_p_")
     for partition_table in metric_tables:
         conn = DBSession.connection().execution_options(stream_results=True)
         result = conn.execute(partition_table.select())
         while True:
             chunk = result.fetchmany(2000)
             if not chunk:
                 break
             es_docs = defaultdict(list)
             for row in chunk:
                 i += 1
                 item = Metric(**dict(list(row.items())))
                 d_range = item.partition_id
                 es_docs[d_range].append(item.es_doc())
             if es_docs:
                 name = partition_table.name
-                log.info('round  {}, {}'.format(i, name))
+                log.info("round  {}, {}".format(i, name))
                 for k, v in es_docs.items():
-                    to_update = {'_index': k, '_type': 'log'}
+                    to_update = {"_index": k, "_type": "log"}
                     [i.update(to_update) for i in v]
                     elasticsearch.helpers.bulk(Datastores.es, v)
 
-    log.info(
-        'total docs {} {}'.format(i, datetime.datetime.now() - task_start))
+    log.info("total docs {} {}".format(i, datetime.datetime.now() - task_start))
 
 
 def reindex_slow_calls():
     try:
-        Datastores.es.indices.delete('rcae_sc*')
+        Datastores.es.indices.delete("rcae_sc*")
     except elasticsearch.exceptions.NotFoundError as e:
         log.error(e)
 
-    log.info('reindexing slow calls')
+    log.info("reindexing slow calls")
     i = 0
     task_start = datetime.datetime.now()
-    slow_calls_tables = detect_tables('slow_calls_p_')
+    slow_calls_tables = detect_tables("slow_calls_p_")
     for partition_table in slow_calls_tables:
         conn = DBSession.connection().execution_options(stream_results=True)
         result = conn.execute(partition_table.select())
         while True:
             chunk = result.fetchmany(2000)
             if not chunk:
                 break
             es_docs = defaultdict(list)
             for row in chunk:
                 i += 1
                 item = SlowCall(**dict(list(row.items())))
                 d_range = item.partition_id
                 es_docs[d_range].append(item.es_doc())
             if es_docs:
                 name = partition_table.name
-                log.info('round  {}, {}'.format(i, name))
+                log.info("round  {}, {}".format(i, name))
                 for k, v in es_docs.items():
-                    to_update = {'_index': k, '_type': 'log'}
+                    to_update = {"_index": k, "_type": "log"}
                     [i.update(to_update) for i in v]
                     elasticsearch.helpers.bulk(Datastores.es, v)
 
-    log.info(
-        'total docs {} {}'.format(i, datetime.datetime.now() - task_start))
+    log.info("total docs {} {}".format(i, datetime.datetime.now() - task_start))
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

backend/src/appenlight/scripts/static.py

@@ -1,58 +1,58 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import argparse
 import logging
 import os
 import pkg_resources
 import shutil
 
 from pyramid.paster import setup_logging
 from pyramid.paster import bootstrap
 
 log = logging.getLogger(__name__)
 
 
 def main():
     parser = argparse.ArgumentParser(
-        description='Generate AppEnlight static resources',
-        add_help=False)
-    parser.add_argument('-c', '--config', required=True,
-                        help='Configuration ini file of application')
+        description="Generate AppEnlight static resources", add_help=False
+    )
+    parser.add_argument(
+        "-c", "--config", required=True, help="Configuration ini file of application"
+    )
     args = parser.parse_args()
     config_uri = args.config
     setup_logging(config_uri)
     env = bootstrap(config_uri)
-    registry = env['registry']
+    registry = env["registry"]
     settings = registry.settings
-    if os.path.exists(settings['webassets.dir']):
-        shutil.rmtree(settings['webassets.dir'])
-    os.mkdir(settings['webassets.dir'])
-    ae_basedir = pkg_resources.resource_filename('appenlight', 'static')
-    shutil.copytree(ae_basedir,
-                    os.path.join(settings['webassets.dir'], 'appenlight'))
+    if os.path.exists(settings["webassets.dir"]):
+        shutil.rmtree(settings["webassets.dir"])
+    os.mkdir(settings["webassets.dir"])
+    ae_basedir = pkg_resources.resource_filename("appenlight", "static")
+    shutil.copytree(ae_basedir, os.path.join(settings["webassets.dir"], "appenlight"))
 
     for plugin_name, config in registry.appenlight_plugins.items():
-        if config['static']:
-            shutil.copytree(config['static'],
-                            os.path.join(settings['webassets.dir'],
-                                         plugin_name))
+        if config["static"]:
+            shutil.copytree(
+                config["static"], os.path.join(settings["webassets.dir"], plugin_name)
+            )
 
-    for root, dirs, files in os.walk(settings['webassets.dir']):
+    for root, dirs, files in os.walk(settings["webassets.dir"]):
         for item in dirs:
             os.chmod(os.path.join(root, item), 0o775)
         for item in files:
             os.chmod(os.path.join(root, item), 0o664)

backend/src/appenlight/security.py

@@ -1,348 +1,352 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from pyramid.security import Allow, Everyone, Authenticated, ALL_PERMISSIONS
 from pyramid.authentication import CallbackAuthenticationPolicy
 import appenlight.models.resource
 from appenlight.models.services.auth_token import AuthTokenService
 from appenlight.models.services.application import ApplicationService
 from appenlight.models.services.report_group import ReportGroupService
 from appenlight.models.services.plugin_config import PluginConfigService
 from appenlight.lib import to_integer_safe
 from pyramid.httpexceptions import HTTPNotFound, HTTPBadRequest
 from ziggurat_foundations.permissions import permission_to_04_acls
 from ziggurat_foundations.models.services.user import UserService
 from ziggurat_foundations.models.services.resource import ResourceService
 import defusedxml.ElementTree as ElementTree
 import urllib.request, urllib.error, urllib.parse
 import logging
 import re
 from xml.sax.saxutils import quoteattr
 
 log = logging.getLogger(__name__)
 
 
 def groupfinder(userid, request):
-    if userid and hasattr(request, 'user') and request.user:
-        groups = ['group:%s' % g.id for g in request.user.groups]
+    if userid and hasattr(request, "user") and request.user:
+        groups = ["group:%s" % g.id for g in request.user.groups]
         return groups
     return []
 
 
 class AuthTokenAuthenticationPolicy(CallbackAuthenticationPolicy):
     def __init__(self, callback=None):
         self.callback = callback
 
     def remember(self, request, principal, **kw):
         return []
 
     def forget(self, request):
         return []
 
     def unauthenticated_userid(self, request):
-        token = request.headers.get('x-appenlight-auth-token')
+        token = request.headers.get("x-appenlight-auth-token")
         if token:
             auth_token = AuthTokenService.by_token(token)
             if auth_token and not auth_token.is_expired:
-                log.info('%s is valid' % auth_token)
+                log.info("%s is valid" % auth_token)
                 return auth_token.owner_id
             elif auth_token:
-                log.warning('%s is expired' % auth_token)
+                log.warning("%s is expired" % auth_token)
             else:
-                log.warning('token: %s is not found' % token)
+                log.warning("token: %s is not found" % token)
 
     def authenticated_userid(self, request):
         return self.unauthenticated_userid(request)
 
 
 def rewrite_root_perm(perm_user, perm_name):
     """
     Translates root_administration into ALL_PERMISSIONS object
     """
-    if perm_name == 'root_administration':
-        return (Allow, perm_user, ALL_PERMISSIONS,)
+    if perm_name == "root_administration":
+        return (Allow, perm_user, ALL_PERMISSIONS)
     else:
-        return (Allow, perm_user, perm_name,)
+        return (Allow, perm_user, perm_name)
 
 
 def add_root_superperm(request, context):
     """
     Adds ALL_PERMISSIONS to every resource if user somehow has 'root_permission'
     non-resource permission
     """
-    if hasattr(request, 'user') and request.user:
+    if hasattr(request, "user") and request.user:
         acls = permission_to_04_acls(UserService.permissions(request.user))
         for perm_user, perm_name in acls:
-            if perm_name == 'root_administration':
+            if perm_name == "root_administration":
                 context.__acl__.append(rewrite_root_perm(perm_user, perm_name))
 
 
 class RootFactory(object):
     """
     General factory for non-resource/report specific pages
     """
 
     def __init__(self, request):
-        self.__acl__ = [(Allow, Authenticated, 'authenticated'),
-                        (Allow, Authenticated, 'create_resources')]
+        self.__acl__ = [
+            (Allow, Authenticated, "authenticated"),
+            (Allow, Authenticated, "create_resources"),
+        ]
         # general page factory - append custom non resource permissions
-        if hasattr(request, 'user') and request.user:
+        if hasattr(request, "user") and request.user:
             acls = permission_to_04_acls(UserService.permissions(request.user))
             for perm_user, perm_name in acls:
                 self.__acl__.append(rewrite_root_perm(perm_user, perm_name))
 
+
 class ResourceFactory(object):
     """
     Checks permissions to specific resource based on user permissions or
     API key headers
     """
 
     def __init__(self, request):
         Resource = appenlight.models.resource.Resource
 
         self.__acl__ = []
-        resource_id = request.matchdict.get("resource_id",
-                                            request.GET.get("resource_id"))
+        resource_id = request.matchdict.get(
+            "resource_id", request.GET.get("resource_id")
+        )
         resource_id = to_integer_safe(resource_id)
-        self.resource = ResourceService.by_resource_id(resource_id) \
-            if resource_id else None
+        self.resource = (
+            ResourceService.by_resource_id(resource_id) if resource_id else None
+        )
         if self.resource and request.user:
             self.__acl__ = self.resource.__acl__
             permissions = ResourceService.perms_for_user(self.resource, request.user)
             for perm_user, perm_name in permission_to_04_acls(permissions):
                 self.__acl__.append(rewrite_root_perm(perm_user, perm_name))
         add_root_superperm(request, self)
 
 
 class ResourceReportFactory(object):
     """
     Checks permissions to specific resource based on user permissions or
     API key headers
     Resource is fetched based on report group information
     """
 
     def __init__(self, request):
         Resource = appenlight.models.resource.Resource
 
         self.__acl__ = []
-        group_id = request.matchdict.get("group_id",
-                                         request.params.get("group_id"))
+        group_id = request.matchdict.get("group_id", request.params.get("group_id"))
         group_id = to_integer_safe(group_id)
-        self.report_group = ReportGroupService.by_id(
-            group_id) if group_id else None
+        self.report_group = ReportGroupService.by_id(group_id) if group_id else None
         if not self.report_group:
             raise HTTPNotFound()
 
         self.public = self.report_group.public
-        self.resource = ResourceService.by_resource_id(self.report_group.resource_id) \
-            if self.report_group else None
+        self.resource = (
+            ResourceService.by_resource_id(self.report_group.resource_id)
+            if self.report_group
+            else None
+        )
 
         if self.resource:
             self.__acl__ = self.resource.__acl__
         if request.user:
             permissions = ResourceService.perms_for_user(self.resource, request.user)
             for perm_user, perm_name in permission_to_04_acls(permissions):
                 self.__acl__.append(rewrite_root_perm(perm_user, perm_name))
         if self.public:
-            self.__acl__.append((Allow, Everyone, 'view',))
+            self.__acl__.append((Allow, Everyone, "view"))
         if not request.user:
             # unauthed users need to visit using both group and report pair
-            report_id = request.params.get('reportId',
-                                           request.params.get('report_id', -1))
+            report_id = request.params.get(
+                "reportId", request.params.get("report_id", -1)
+            )
             report = self.report_group.get_report(report_id, public=True)
             if not report:
                 raise HTTPNotFound()
         add_root_superperm(request, self)
 
 
 class APIFactory(object):
     """
     Checks permissions to perform client API actions based on keys
     """
 
     def __init__(self, request):
         self.__acl__ = []
         self.possibly_public = False
         private_api_key = request.headers.get(
-            'x-appenlight-api-key',
-            request.params.get('api_key')
+            "x-appenlight-api-key", request.params.get("api_key")
         )
         log.debug("private key: %s" % private_api_key)
         if private_api_key:
-            self.resource = ApplicationService.by_api_key_cached()(
-                private_api_key)
+            self.resource = ApplicationService.by_api_key_cached()(private_api_key)
         # then try public key
         else:
             public_api_key = request.headers.get(
-                'x-appenlight-public-api-key',
-                request.GET.get('public_api_key'))
+                "x-appenlight-public-api-key", request.GET.get("public_api_key")
+            )
             log.debug("public key: %s" % public_api_key)
             self.resource = ApplicationService.by_public_api_key(
-                public_api_key, from_cache=True, request=request)
+                public_api_key, from_cache=True, request=request
+            )
             self.possibly_public = True
         if self.resource:
-            self.__acl__.append((Allow, Everyone, 'create',))
+            self.__acl__.append((Allow, Everyone, "create"))
 
 
 class AirbrakeV2APIFactory(object):
     """
     Check permission based on Airbrake XML report
     """
 
     def __init__(self, request):
         self.__acl__ = []
         self.possibly_public = False
-        fixed_xml_data = ''
+        fixed_xml_data = ""
         try:
-            data = request.GET.get('data')
+            data = request.GET.get("data")
             if data:
                 self.possibly_public = True
         except (UnicodeDecodeError, UnicodeEncodeError) as exc:
-            log.warning(
-                'Problem parsing Airbrake data: %s, failed decoding' % exc)
+            log.warning("Problem parsing Airbrake data: %s, failed decoding" % exc)
             raise HTTPBadRequest()
         try:
             if not data:
                 data = request.body
                 # fix shitty airbrake js client not escaping line method attr
 
             def repl(input):
-                return 'line method=%s file' % quoteattr(input.group(1))
+                return "line method=%s file" % quoteattr(input.group(1))
 
             fixed_xml_data = re.sub('line method="(.*?)" file', repl, data)
             root = ElementTree.fromstring(fixed_xml_data)
         except Exception as exc:
-            log.info(
-                'Problem parsing Airbrake '
-                'data: %s, trying unquoting' % exc)
+            log.info("Problem parsing Airbrake " "data: %s, trying unquoting" % exc)
             self.possibly_public = True
             try:
                 root = ElementTree.fromstring(urllib.parse.unquote(fixed_xml_data))
             except Exception as exc:
-                log.warning('Problem parsing Airbrake '
-                            'data: %s, failed completly' % exc)
+                log.warning(
+                    "Problem parsing Airbrake " "data: %s, failed completly" % exc
+                )
                 raise HTTPBadRequest()
         self.airbrake_xml_etree = root
-        api_key = root.findtext('api-key', '')
+        api_key = root.findtext("api-key", "")
 
         self.resource = ApplicationService.by_api_key_cached()(api_key)
         if not self.resource:
-            self.resource = ApplicationService.by_public_api_key(api_key,
-                                                                 from_cache=True,
-                                                                 request=request)
+            self.resource = ApplicationService.by_public_api_key(
+                api_key, from_cache=True, request=request
+            )
             if self.resource:
                 self.possibly_public = True
 
         if self.resource:
-            self.__acl__.append((Allow, Everyone, 'create',))
+            self.__acl__.append((Allow, Everyone, "create"))
 
 
 def parse_sentry_header(header):
-    parsed = header.split(' ', 1)[1].split(',') or []
-    return dict([x.strip().split('=') for x in parsed])
+    parsed = header.split(" ", 1)[1].split(",") or []
+    return dict([x.strip().split("=") for x in parsed])
 
 
 class SentryAPIFactory(object):
     """
     Check permission based on Sentry payload
     """
 
     def __init__(self, request):
         self.__acl__ = []
         self.possibly_public = False
-        if request.headers.get('X-Sentry-Auth', '').startswith('Sentry'):
-            header_string = request.headers['X-Sentry-Auth']
+        if request.headers.get("X-Sentry-Auth", "").startswith("Sentry"):
+            header_string = request.headers["X-Sentry-Auth"]
             result = parse_sentry_header(header_string)
-        elif request.headers.get('Authorization', '').startswith('Sentry'):
-            header_string = request.headers['Authorization']
+        elif request.headers.get("Authorization", "").startswith("Sentry"):
+            header_string = request.headers["Authorization"]
             result = parse_sentry_header(header_string)
         else:
-            result = dict((k, v) for k, v in list(request.GET.items())
-                          if k.startswith('sentry_'))
-        key = result.get('sentry_key')
-        log.info('sentry request {}'.format(result))
+            result = dict(
+                (k, v) for k, v in list(request.GET.items()) if k.startswith("sentry_")
+            )
+        key = result.get("sentry_key")
+        log.info("sentry request {}".format(result))
 
         self.resource = ApplicationService.by_api_key_cached()(key)
-        if not self.resource or \
-            result.get('sentry_client', '').startswith('raven-js'):
+        if not self.resource or result.get("sentry_client", "").startswith("raven-js"):
             self.resource = ApplicationService.by_public_api_key(
-                key, from_cache=True, request=request)
+                key, from_cache=True, request=request
+            )
         if self.resource:
-            self.__acl__.append((Allow, Everyone, 'create',))
+            self.__acl__.append((Allow, Everyone, "create"))
 
 
 class ResourcePluginConfigFactory(object):
-
     def __init__(self, request):
         Resource = appenlight.models.resource.Resource
         self.__acl__ = []
         self.resource = None
-        plugin_id = to_integer_safe(request.matchdict.get('id'))
+        plugin_id = to_integer_safe(request.matchdict.get("id"))
         self.plugin = PluginConfigService.by_id(plugin_id)
         if not self.plugin:
             raise HTTPNotFound()
         if self.plugin.resource_id:
             self.resource = ResourceService.by_resource_id(self.plugin.resource_id)
         if self.resource:
             self.__acl__ = self.resource.__acl__
         if request.user and self.resource:
             permissions = ResourceService.perms_for_user(self.resource, request.user)
             for perm_user, perm_name in permission_to_04_acls(permissions):
                 self.__acl__.append(rewrite_root_perm(perm_user, perm_name))
 
         add_root_superperm(request, self)
 
 
 class ResourceJSONBodyFactory(object):
     """
     Checks permissions to specific resource based on user permissions or
     API key headers from json body
     """
 
     def __init__(self, request):
         Resource = appenlight.models.resource.Resource
 
         self.__acl__ = []
-        resource_id = request.unsafe_json_body().get('resource_id')
+        resource_id = request.unsafe_json_body().get("resource_id")
         resource_id = to_integer_safe(resource_id)
         self.resource = ResourceService.by_resource_id(resource_id)
         if self.resource and request.user:
             self.__acl__ = self.resource.__acl__
             permissions = ResourceService.perms_for_user(self.resource, request.user)
             for perm_user, perm_name in permission_to_04_acls(permissions):
                 self.__acl__.append(rewrite_root_perm(perm_user, perm_name))
         add_root_superperm(request, self)
 
 
 class ResourcePluginMixedFactory(object):
     def __init__(self, request):
         Resource = appenlight.models.resource.Resource
         self.__acl__ = []
         json_body = request.safe_json_body
         self.resource = None
         if json_body:
-            resource_id = json_body.get('resource_id')
+            resource_id = json_body.get("resource_id")
         else:
-            resource_id = request.GET.get('resource_id')
+            resource_id = request.GET.get("resource_id")
         if resource_id:
             resource_id = to_integer_safe(resource_id)
             self.resource = ResourceService.by_resource_id(resource_id)
         if self.resource and request.user:
             self.__acl__ = self.resource.__acl__
             permissions = ResourceService.perms_for_user(self.resource, request.user)
             for perm_user, perm_name in permission_to_04_acls(permissions):
                 self.__acl__.append(rewrite_root_perm(perm_user, perm_name))
         add_root_superperm(request, self)

backend/src/appenlight/subscribers.py

@@ -1,169 +1,236 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import hashlib
 import os
 
 from pyramid.i18n import TranslationStringFactory
 from pyramid import threadlocal
 
-_ = TranslationStringFactory('pyramid')
+_ = TranslationStringFactory("pyramid")
 
 from appenlight import security
 from appenlight.lib import helpers, generate_random_string
 from appenlight.models.services.config import ConfigService
 
 
 def gen_urls(request):
     urls = {
-        'baseUrl': request.route_url('/'),
-        'applicationsNoId': request.route_url('applications_no_id'),
-        'applications': request.route_url('applications', resource_id='REPLACE_ID').replace('REPLACE_ID',':resourceId'),
-        'applicationsProperty': request.route_url('applications_property',key='REPLACE_KEY', resource_id='REPLACE_ID').replace('REPLACE_ID',':resourceId').replace('REPLACE_KEY',':key'),
-        'configsNoId': request.route_url('admin_configs'),
-        'configs': request.route_url('admin_config', key='REPLACE_KEY', section='REPLACE_SECTION').replace('REPLACE_SECTION',':section').replace('REPLACE_KEY',':key'),
-        'docs': 'http://getappenlight.com/page/api/main.html',
-        'eventsNoId': request.route_url('events_no_id'),
-        'events': request.route_url('events', event_id='REPLACE_ID').replace('REPLACE_ID',':eventId'),
-        'eventsProperty': request.route_url('events_property',key='REPLACE_KEY', event_id='REPLACE_ID').replace('REPLACE_ID',':eventId').replace('REPLACE_KEY',':key'),
-        'groupsNoId': request.route_url('groups_no_id'),
-        'groups': request.route_url('groups', group_id='REPLACE_ID').replace('REPLACE_ID',':groupId'),
-        'groupsProperty': request.route_url('groups_property',key='REPLACE_KEY', group_id='REPLACE_ID').replace('REPLACE_ID',':groupId').replace('REPLACE_KEY',':key'),
-        'logsNoId': request.route_url('logs_no_id'),
-        'integrationAction': request.route_url('integrations_id',action='REPLACE_ACT', resource_id='REPLACE_RID', integration='REPLACE_IID').replace('REPLACE_RID',':resourceId').replace('REPLACE_ACT',':action').replace('REPLACE_IID',':integration'),
-        'usersNoId': request.route_url('users_no_id'),
-        'users': request.route_url('users', user_id='REPLACE_ID').replace('REPLACE_ID',':userId'),
-        'usersProperty': request.route_url('users_property',key='REPLACE_KEY', user_id='REPLACE_ID').replace('REPLACE_ID',':userId').replace('REPLACE_KEY',':key'),
-        'userSelf': request.route_url('users_self'),
-        'userSelfProperty': request.route_url('users_self_property',key='REPLACE_KEY').replace('REPLACE_KEY',':key'),
-        'reports': request.route_url('reports'),
-        'reportGroup': request.route_url('report_groups', group_id='REPLACE_RID').replace('REPLACE_RID',':groupId'),
-        'reportGroupProperty': request.route_url('report_groups_property', key='REPLACE_KEY', group_id='REPLACE_GID').replace('REPLACE_KEY',':key').replace('REPLACE_GID',':groupId'),
-        'pluginConfigsNoId': request.route_url('plugin_configs', plugin_name='REPLACE_TYPE').replace('REPLACE_TYPE',':plugin_name'),
-        'pluginConfigs': request.route_url('plugin_config', id='REPLACE_ID', plugin_name='REPLACE_TYPE').replace('REPLACE_ID',':id').replace('REPLACE_TYPE',':plugin_name'),
-        'resourceProperty': request.route_url('resources_property',key='REPLACE_KEY', resource_id='REPLACE_ID').replace('REPLACE_ID',':resourceId').replace('REPLACE_KEY',':key'),
-        'slowReports': request.route_url('slow_reports'),
-        'sectionView': request.route_url('section_view', section='REPLACE_S', view='REPLACE_V').replace('REPLACE_S',':section').replace('REPLACE_V',':view'),
-        'otherRoutes': {
-            'register': request.route_url('register'),
-            'lostPassword': request.route_url('lost_password'),
-            'lostPasswordGenerate': request.route_url('lost_password_generate'),
-            'signOut': request.route_url('ziggurat.routes.sign_out')
+        "baseUrl": request.route_url("/"),
+        "applicationsNoId": request.route_url("applications_no_id"),
+        "applications": request.route_url(
+            "applications", resource_id="REPLACE_ID"
+        ).replace("REPLACE_ID", ":resourceId"),
+        "applicationsProperty": request.route_url(
+            "applications_property", key="REPLACE_KEY", resource_id="REPLACE_ID"
+        )
+        .replace("REPLACE_ID", ":resourceId")
+        .replace("REPLACE_KEY", ":key"),
+        "configsNoId": request.route_url("admin_configs"),
+        "configs": request.route_url(
+            "admin_config", key="REPLACE_KEY", section="REPLACE_SECTION"
+        )
+        .replace("REPLACE_SECTION", ":section")
+        .replace("REPLACE_KEY", ":key"),
+        "docs": "http://getappenlight.com/page/api/main.html",
+        "eventsNoId": request.route_url("events_no_id"),
+        "events": request.route_url("events", event_id="REPLACE_ID").replace(
+            "REPLACE_ID", ":eventId"
+        ),
+        "eventsProperty": request.route_url(
+            "events_property", key="REPLACE_KEY", event_id="REPLACE_ID"
+        )
+        .replace("REPLACE_ID", ":eventId")
+        .replace("REPLACE_KEY", ":key"),
+        "groupsNoId": request.route_url("groups_no_id"),
+        "groups": request.route_url("groups", group_id="REPLACE_ID").replace(
+            "REPLACE_ID", ":groupId"
+        ),
+        "groupsProperty": request.route_url(
+            "groups_property", key="REPLACE_KEY", group_id="REPLACE_ID"
+        )
+        .replace("REPLACE_ID", ":groupId")
+        .replace("REPLACE_KEY", ":key"),
+        "logsNoId": request.route_url("logs_no_id"),
+        "integrationAction": request.route_url(
+            "integrations_id",
+            action="REPLACE_ACT",
+            resource_id="REPLACE_RID",
+            integration="REPLACE_IID",
+        )
+        .replace("REPLACE_RID", ":resourceId")
+        .replace("REPLACE_ACT", ":action")
+        .replace("REPLACE_IID", ":integration"),
+        "usersNoId": request.route_url("users_no_id"),
+        "users": request.route_url("users", user_id="REPLACE_ID").replace(
+            "REPLACE_ID", ":userId"
+        ),
+        "usersProperty": request.route_url(
+            "users_property", key="REPLACE_KEY", user_id="REPLACE_ID"
+        )
+        .replace("REPLACE_ID", ":userId")
+        .replace("REPLACE_KEY", ":key"),
+        "userSelf": request.route_url("users_self"),
+        "userSelfProperty": request.route_url(
+            "users_self_property", key="REPLACE_KEY"
+        ).replace("REPLACE_KEY", ":key"),
+        "reports": request.route_url("reports"),
+        "reportGroup": request.route_url(
+            "report_groups", group_id="REPLACE_RID"
+        ).replace("REPLACE_RID", ":groupId"),
+        "reportGroupProperty": request.route_url(
+            "report_groups_property", key="REPLACE_KEY", group_id="REPLACE_GID"
+        )
+        .replace("REPLACE_KEY", ":key")
+        .replace("REPLACE_GID", ":groupId"),
+        "pluginConfigsNoId": request.route_url(
+            "plugin_configs", plugin_name="REPLACE_TYPE"
+        ).replace("REPLACE_TYPE", ":plugin_name"),
+        "pluginConfigs": request.route_url(
+            "plugin_config", id="REPLACE_ID", plugin_name="REPLACE_TYPE"
+        )
+        .replace("REPLACE_ID", ":id")
+        .replace("REPLACE_TYPE", ":plugin_name"),
+        "resourceProperty": request.route_url(
+            "resources_property", key="REPLACE_KEY", resource_id="REPLACE_ID"
+        )
+        .replace("REPLACE_ID", ":resourceId")
+        .replace("REPLACE_KEY", ":key"),
+        "slowReports": request.route_url("slow_reports"),
+        "sectionView": request.route_url(
+            "section_view", section="REPLACE_S", view="REPLACE_V"
+        )
+        .replace("REPLACE_S", ":section")
+        .replace("REPLACE_V", ":view"),
+        "otherRoutes": {
+            "register": request.route_url("register"),
+            "lostPassword": request.route_url("lost_password"),
+            "lostPasswordGenerate": request.route_url("lost_password_generate"),
+            "signOut": request.route_url("ziggurat.routes.sign_out"),
         },
-        'social_auth': {
-            'google': request.route_url('social_auth', provider='google'),
-            'twitter': request.route_url('social_auth', provider='twitter'),
-            'bitbucket': request.route_url('social_auth', provider='bitbucket'),
-            'github': request.route_url('social_auth', provider='github'),
+        "social_auth": {
+            "google": request.route_url("social_auth", provider="google"),
+            "twitter": request.route_url("social_auth", provider="twitter"),
+            "bitbucket": request.route_url("social_auth", provider="bitbucket"),
+            "github": request.route_url("social_auth", provider="github"),
         },
         "plugins": {},
-        "adminAction": request.route_url('admin', action="REPLACE_ACT").replace('REPLACE_ACT',':action')
+        "adminAction": request.route_url("admin", action="REPLACE_ACT").replace(
+            "REPLACE_ACT", ":action"
+        ),
     }
     return urls
 
+
 def new_request(event):
     environ = event.request.environ
-    event.request.response.headers['X-Frame-Options'] = 'SAMEORIGIN'
-    event.request.response.headers['X-XSS-Protection'] = '1; mode=block'
+    event.request.response.headers["X-Frame-Options"] = "SAMEORIGIN"
+    event.request.response.headers["X-XSS-Protection"] = "1; mode=block"
     # can this be enabled on non https deployments?
     # event.request.response.headers['Strict-Transport-Security'] = 'max-age=31536000; includeSubdomains;'
 
     # do not send XSRF token with /api calls
-    if not event.request.path.startswith('/api'):
-        if environ['wsgi.url_scheme'] == 'https':
+    if not event.request.path.startswith("/api"):
+        if environ["wsgi.url_scheme"] == "https":
             event.request.response.set_cookie(
-                'XSRF-TOKEN', event.request.session.get_csrf_token(),
-                secure=True)
+                "XSRF-TOKEN", event.request.session.get_csrf_token(), secure=True
+            )
         else:
             event.request.response.set_cookie(
-                'XSRF-TOKEN', event.request.session.get_csrf_token())
+                "XSRF-TOKEN", event.request.session.get_csrf_token()
+            )
     if event.request.user:
-        event.request.response.headers[
-            'x-appenlight-uid'] = '%s' % event.request.user.id
+        event.request.response.headers["x-appenlight-uid"] = (
+            "%s" % event.request.user.id
+        )
 
 
 def add_renderer_globals(event):
     request = event.get("request") or threadlocal.get_current_request()
 
     renderer_globals = event
     renderer_globals["h"] = helpers
     renderer_globals["js_hash"] = request.registry.js_hash
     renderer_globals["css_hash"] = request.registry.css_hash
-    renderer_globals['_'] = _
-    renderer_globals['security'] = security
-    renderer_globals['flash_msgs'] = []
-    renderer_globals['appenlight_plugins'] = []
+    renderer_globals["_"] = _
+    renderer_globals["security"] = security
+    renderer_globals["flash_msgs"] = []
+    renderer_globals["appenlight_plugins"] = []
 
-    if 'jinja' in event['renderer_info'].type:
-        renderer_globals['url_list'] = gen_urls(request)
+    if "jinja" in event["renderer_info"].type:
+        renderer_globals["url_list"] = gen_urls(request)
         # add footer html and some other global vars to renderer
         for module, config in request.registry.appenlight_plugins.items():
-            if config['url_gen']:
-                urls = config['url_gen'](request)
-                renderer_globals['url_list']['plugins'][module] = urls
-
-            renderer_globals['appenlight_plugins'].append(
-                {'name': module,
-                 'config': {
-                     'javascript':config['javascript'],
-                     'header_html':config['header_html']
-                 }})
+            if config["url_gen"]:
+                urls = config["url_gen"](request)
+                renderer_globals["url_list"]["plugins"][module] = urls
+
+            renderer_globals["appenlight_plugins"].append(
+                {
+                    "name": module,
+                    "config": {
+                        "javascript": config["javascript"],
+                        "header_html": config["header_html"],
+                    },
+                }
+            )
 
         footer_config = ConfigService.by_key_and_section(
-            'template_footer_html', 'global', default_value='')
+            "template_footer_html", "global", default_value=""
+        )
 
-        renderer_globals['template_footer_html'] = footer_config.value
+        renderer_globals["template_footer_html"] = footer_config.value
         try:
-            renderer_globals['root_administrator'] = request.has_permission(
-                'root_administration', security.RootFactory(request))
+            renderer_globals["root_administrator"] = request.has_permission(
+                "root_administration", security.RootFactory(request)
+            )
         except AttributeError:
-            renderer_globals['root_administrator'] = False
+            renderer_globals["root_administrator"] = False
 
-    renderer_globals['_mail_url'] = request.registry.settings['_mail_url']
+    renderer_globals["_mail_url"] = request.registry.settings["_mail_url"]
 
     if not request:
         return
 
     # do not sens flash headers with /api calls
-    if not request.path.startswith('/api'):
+    if not request.path.startswith("/api"):
         flash_msgs = helpers.get_type_formatted_flash(request)
-        renderer_globals['flash_msgs'] = flash_msgs
+        renderer_globals["flash_msgs"] = flash_msgs
         request.add_flash_to_headers()
 
+
 def application_created(app):
-    webassets_dir = app.app.registry.settings.get('webassets.dir')
+    webassets_dir = app.app.registry.settings.get("webassets.dir")
     js_hash = generate_random_string()
     css_hash = generate_random_string()
     if webassets_dir:
         js_hasher = hashlib.md5()
         css_hasher = hashlib.md5()
         for root, dirs, files in os.walk(webassets_dir):
             for name in files:
                 filename = os.path.join(root, name)
-                if name.endswith('css'):
-                    with open(filename, 'r', encoding='utf8',
-                              errors='replace') as f:
+                if name.endswith("css"):
+                    with open(filename, "r", encoding="utf8", errors="replace") as f:
                         for line in f:
-                            css_hasher.update(line.encode('utf8'))
-                elif name.endswith('js'):
-                    with open(filename, 'r', encoding='utf8',
-                              errors='replace') as f:
+                            css_hasher.update(line.encode("utf8"))
+                elif name.endswith("js"):
+                    with open(filename, "r", encoding="utf8", errors="replace") as f:
                         for line in f:
-                            js_hasher.update(line.encode('utf8'))
+                            js_hasher.update(line.encode("utf8"))
         js_hash = js_hasher.hexdigest()
         css_hash = css_hasher.hexdigest()
     app.app.registry.js_hash = js_hash
     app.app.registry.css_hash = css_hash

backend/src/appenlight/tests/__init__.py

@@ -1,16 +1,15 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-

backend/src/appenlight/tests/conftest.py

@@ -1,190 +1,196 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import mock
 import os
 import pytest
 import transaction
 from datetime import datetime
 
 from alembic.config import Config
 from alembic import command
 from collections import OrderedDict
 from zope.sqlalchemy import mark_changed
 from pyramid.paster import get_appsettings
 from pyramid import testing
 
 
 @pytest.fixture
 def base_app(request, mocker):
     # disable email sending
-    mocker.patch('pyramid_mailer.mailer_factory_from_settings', mocker.Mock())
+    mocker.patch("pyramid_mailer.mailer_factory_from_settings", mocker.Mock())
 
     from appenlight import main
     import transaction
+
     current_dir = os.path.dirname(os.path.abspath(__file__))
-    path = os.path.join(current_dir, '../../../../',
-                        os.environ.get("APPENLIGHT_INI", 'testing.ini'))
+    path = os.path.join(
+        current_dir, "../../../../", os.environ.get("APPENLIGHT_INI", "testing.ini")
+    )
     # appsettings from ini
     app_settings = get_appsettings(path, name="appenlight")
     app = main({}, **app_settings)
-    app_request = testing.DummyRequest(base_url='https://appenlight.com')
+    app_request = testing.DummyRequest(base_url="https://appenlight.com")
     app_request.tm = transaction.manager
     app_request.add_flash_to_headers = mock.Mock()
     testing.setUp(registry=app.registry, request=app_request)
 
     def teardown():
         testing.tearDown()
 
     request.addfinalizer(teardown)
 
     return app
 
 
 @pytest.fixture
 def with_migrations(request, base_app):
     settings = base_app.registry.settings
     alembic_cfg = Config()
-    alembic_cfg.set_main_option("script_location",
-                                "ziggurat_foundations:migrations")
+    alembic_cfg.set_main_option("script_location", "ziggurat_foundations:migrations")
     alembic_cfg.set_main_option("sqlalchemy.url", settings["sqlalchemy.url"])
     command.upgrade(alembic_cfg, "head")
     alembic_cfg = Config()
     alembic_cfg.set_main_option("script_location", "appenlight:migrations")
     alembic_cfg.set_main_option("sqlalchemy.url", settings["sqlalchemy.url"])
     command.upgrade(alembic_cfg, "head")
 
     for plugin_name, config in base_app.registry.appenlight_plugins.items():
-        if config['sqlalchemy_migrations']:
+        if config["sqlalchemy_migrations"]:
             alembic_cfg = Config()
-            alembic_cfg.set_main_option("script_location",
-                                        config['sqlalchemy_migrations'])
             alembic_cfg.set_main_option(
-                "sqlalchemy.url",
-                base_app.registry.settings["sqlalchemy.url"])
+                "script_location", config["sqlalchemy_migrations"]
+            )
+            alembic_cfg.set_main_option(
+                "sqlalchemy.url", base_app.registry.settings["sqlalchemy.url"]
+            )
             command.upgrade(alembic_cfg, "head")
 
 
 @pytest.fixture
 def default_data(base_app):
     from appenlight.models.services.config import ConfigService
     from appenlight.lib import get_callable
+
     transaction.begin()
     ConfigService.setup_default_values()
     for plugin_name, config in base_app.registry.appenlight_plugins.items():
-        if config['default_values_setter']:
-            get_callable(config['default_values_setter'])()
+        if config["default_values_setter"]:
+            get_callable(config["default_values_setter"])()
     transaction.commit()
 
 
 @pytest.fixture
 def clean_tables(request):
     from appenlight.models import Base, DBSession
 
     def fin():
         tables = Base.metadata.tables.keys()
         transaction.begin()
         for t in tables:
-            if not t.startswith('alembic_'):
-                DBSession.execute('truncate %s cascade' % t)
+            if not t.startswith("alembic_"):
+                DBSession.execute("truncate %s cascade" % t)
         session = DBSession()
         mark_changed(session)
         transaction.commit()
 
     request.addfinalizer(fin)
 
 
 @pytest.fixture
 def default_user():
     from appenlight.models import DBSession
     from appenlight.models.user import User
     from appenlight.models.auth_token import AuthToken
+
     transaction.begin()
     session = DBSession()
-    user = User(id=1,
-                user_name='testuser',
-                status=1,
-                email='[e-mail]')
+    user = User(id=1, user_name="testuser", status=1, email="[e-mail]")
     session.add(user)
-    token = AuthToken(token='1234')
+    token = AuthToken(token="1234")
     user.auth_tokens.append(token)
     session.execute("SELECT nextval('users_id_seq')")
     transaction.commit()
     return user
 
 
 @pytest.fixture
 def default_application(default_user):
     from appenlight.models import DBSession
     from appenlight.models.application import Application
 
     transaction.begin()
     session = DBSession()
-    application = Application(
-        resource_id=1, resource_name='testapp', api_key='xxxx')
+    application = Application(resource_id=1, resource_name="testapp", api_key="xxxx")
     session.add(application)
     default_user.resources.append(application)
     session.execute("SELECT nextval('resources_resource_id_seq')")
     transaction.commit()
     return application
 
 
 @pytest.fixture
 def report_type_matrix():
     from appenlight.models.report import REPORT_TYPE_MATRIX
+
     return REPORT_TYPE_MATRIX
 
 
 @pytest.fixture
 def chart_series():
     series = []
 
     for x in range(1, 7):
-        tmp_list = [('key', 'X'), ('0_1', x)]
+        tmp_list = [("key", "X"), ("0_1", x)]
         if x % 2 == 0:
-            tmp_list.append(('0_2', x))
+            tmp_list.append(("0_2", x))
         if x % 3 == 0:
-            tmp_list.append(('0_3', x))
+            tmp_list.append(("0_3", x))
 
-        series.append(
-            OrderedDict(tmp_list)
-        )
+        series.append(OrderedDict(tmp_list))
     return series
 
 
 @pytest.fixture
 def log_schema():
     from appenlight.validators import LogListSchema
+
     schema = LogListSchema().bind(utcnow=datetime.utcnow())
     return schema
 
+
 @pytest.fixture
 def general_metrics_schema():
     from appenlight.validators import GeneralMetricsListSchema
+
     schema = GeneralMetricsListSchema().bind(utcnow=datetime.utcnow())
     return schema
 
+
 @pytest.fixture
 def request_metrics_schema():
     from appenlight.validators import MetricsListSchema
+
     schema = MetricsListSchema().bind(utcnow=datetime.utcnow())
     return schema
 
+
 @pytest.fixture
 def report_05_schema():
     from appenlight.validators import ReportListSchema_0_5
+
     schema = ReportListSchema_0_5().bind(utcnow=datetime.utcnow())
     return schema

backend/src/appenlight/tests/payload_examples.py

@@ -1,855 +1,977 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 # -*- coding: utf-8 -*-
 
 from datetime import datetime
 
 now = datetime.utcnow().date()
 
-REQUEST_METRICS_EXAMPLES = [{
-    "server": "some.server.hostname",
-    "timestamp": now.strftime('%Y-%m-%dT%H:%M:%S.0'),
-    "metrics": [
-        ["dir/module:func",
-         {"custom": 0.0,
-          "custom_calls": 0,
-          "main": 0.01664,
-          "nosql": 0.00061,
-          "nosql_calls": 23,
-          "remote": 0.0,
-          "remote_calls": 0,
-          "requests": 1,
-          "sql": 0.00105,
-          "sql_calls": 2,
-          "tmpl": 0.0,
-          "tmpl_calls": 0}],
-        ["SomeView.function",
-         {"custom": 0.0,
-          "custom_calls": 0,
-          "main": 0.647261,
-          "nosql": 0.306554,
-          "nosql_calls": 140,
-          "remote": 0.0,
-          "remote_calls": 0,
-          "requests": 28,
-          "sql": 0.0,
-          "sql_calls": 0,
-          "tmpl": 0.0,
-          "tmpl_calls": 0}]]
-}]
+REQUEST_METRICS_EXAMPLES = [
+    {
+        "server": "some.server.hostname",
+        "timestamp": now.strftime("%Y-%m-%dT%H:%M:%S.0"),
+        "metrics": [
+            [
+                "dir/module:func",
+                {
+                    "custom": 0.0,
+                    "custom_calls": 0,
+                    "main": 0.01664,
+                    "nosql": 0.00061,
+                    "nosql_calls": 23,
+                    "remote": 0.0,
+                    "remote_calls": 0,
+                    "requests": 1,
+                    "sql": 0.00105,
+                    "sql_calls": 2,
+                    "tmpl": 0.0,
+                    "tmpl_calls": 0,
+                },
+            ],
+            [
+                "SomeView.function",
+                {
+                    "custom": 0.0,
+                    "custom_calls": 0,
+                    "main": 0.647261,
+                    "nosql": 0.306554,
+                    "nosql_calls": 140,
+                    "remote": 0.0,
+                    "remote_calls": 0,
+                    "requests": 28,
+                    "sql": 0.0,
+                    "sql_calls": 0,
+                    "tmpl": 0.0,
+                    "tmpl_calls": 0,
+                },
+            ],
+        ],
+    }
+]
 
 LOG_EXAMPLES = [
     {
         "log_level": "WARNING",
         "message": "OMG ValueError happened",
         "namespace": "some.namespace.indicator",
         "request_id": "SOME_UUID",
         "server": "some server",
-        "tags": [["tag_name", "tag_value"],
-                 ["tag_name2", 2]
-
-                 ],
-        "date": now.strftime('%Y-%m-%dT%H:%M:%S.%f')
+        "tags": [["tag_name", "tag_value"], ["tag_name2", 2]],
+        "date": now.strftime("%Y-%m-%dT%H:%M:%S.%f"),
     },
     {
         "log_level": "ERROR",
         "message": "OMG ValueError happened2",
         "namespace": "some.namespace.indicator",
         "request_id": "SOME_UUID",
         "server": "some server",
-        "date": now.strftime('%Y-%m-%dT%H:%M:%S.%f')
-    }
+        "date": now.strftime("%Y-%m-%dT%H:%M:%S.%f"),
+    },
 ]
 
 PARSED_REPORT_404 = {
-    'report_details': [{
-        'username': 'foo',
-        'url': 'http://localhost:6543/test/error?aaa=1&bbb=2',
-        'ip': '127.0.0.1',
-        'start_time': now.strftime(
-            '%Y-%m-%dT%H:%M:%S.0'),
-        'slow_calls': [],
-        'request': {'COOKIES': {
-            'country': 'US',
-            'sessionId': '***',
-            'test_group_id': '5',
-            'http_referer': 'http://localhost:5000/'},
-            'POST': {},
-            'GET': {'aaa': ['1'], 'bbb': ['2']},
-            'HTTP_METHOD': 'GET',
-        },
-        'user_agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0.1) Gecko/20100101 Firefox/10.0.1',
-        'message': '',
-        'end_time': now.strftime(
-            '%Y-%m-%dT%H:%M:%S.0'),
-        'request_stats': {}
-    }],
-    'error': '404 Not Found',
-    'server': "servername/instancename",
-    'priority': 5,
-    'client': 'appenlight-python',
-    'language': 'python',
-    'http_status': 404}
+    "report_details": [
+        {
+            "username": "foo",
+            "url": "http://localhost:6543/test/error?aaa=1&bbb=2",
+            "ip": "127.0.0.1",
+            "start_time": now.strftime("%Y-%m-%dT%H:%M:%S.0"),
+            "slow_calls": [],
+            "request": {
+                "COOKIES": {
+                    "country": "US",
+                    "sessionId": "***",
+                    "test_group_id": "5",
+                    "http_referer": "http://localhost:5000/",
+                },
+                "POST": {},
+                "GET": {"aaa": ["1"], "bbb": ["2"]},
+                "HTTP_METHOD": "GET",
+            },
+            "user_agent": "Mozilla/5.0 (X11; Linux x86_64; rv:10.0.1) Gecko/20100101 Firefox/10.0.1",
+            "message": "",
+            "end_time": now.strftime("%Y-%m-%dT%H:%M:%S.0"),
+            "request_stats": {},
+        }
+    ],
+    "error": "404 Not Found",
+    "server": "servername/instancename",
+    "priority": 5,
+    "client": "appenlight-python",
+    "language": "python",
+    "http_status": 404,
+}
 
 PYTHON_PAYLOAD_0_4 = {
     "client": "your-client-name-python",
     "language": "python",
-    'view_name': 'views/foo:bar',
-    'server': "servername/instancename",
+    "view_name": "views/foo:bar",
+    "server": "servername/instancename",
     "priority": 5,
     "error": "OMG ValueError happened test",
     "occurences": 1,
     "http_status": 500,
     "report_details": [
-        {"username": "USER",
-         "url": "HTTP://SOMEURL",
-         "ip": "127.0.0.1",
-         "start_time": now.strftime(
-             '%Y-%m-%dT%H:%M:%S.0'),
-         "end_time": now.strftime(
-             '%Y-%m-%dT%H:%M:%S.0'),
-         "user_agent": "BROWSER_AGENT",
-         "message": "arbitrary text that will get attached to your report",
-         "request_id": "SOME_UUID",
-         "request": {"REQUEST_METHOD": "GET",
-                     "PATH_INFO": "/FOO/BAR",
-                     "POST": {"FOO": "BAZ", "XXX": "YYY"}
-                     },
-         "slow_calls": [
-             {
-                 "type": "sql",
-                 "start": now.strftime(
-                     '%Y-%m-%dT%H:%M:%S.0'),
-                 "end": now.strftime(
-                     '%Y-%m-%dT%H:%M:%S.0'),
-                 "subtype": "postgresql",
-                 "parameters": ["QPARAM1", "QPARAM2",
-                                "QPARAMX"],
-                 "statement": "QUERY"}],
-         "request_stats": {"main": 0.50779,
-                           "nosql": 0.01008,
-                           "nosql_calls": 17.0,
-                           "remote": 0.0,
-                           "remote_calls": 0.0,
-                           "custom": 0.0,
-                           "custom_calls": 0.0,
-                           "sql": 0.42423,
-                           "sql_calls": 1.0,
-                           "tmpl": 0.0,
-                           "tmpl_calls": 0.0},
-         "traceback": [
-             {"cline": "return foo_bar_baz(1,2,3)",
-              "file": "somedir/somefile.py",
-              "fn": "somefunction",
-              "line": 454,
-              "vars": [["a_list",
-                        ["1",
-                         "2",
-                         "4",
-                         "5",
-                         "6"]],
-                       ["b",
-                        {1: "2", "ccc": "ddd", "1": "a"}],
-                       ["obj",
-                        "<object object at 0x7f0030853dc0>"]]},
-             {"cline": "OMG ValueError happened",
-              "file": "",
-              "fn": "",
-              "line": "",
-              "vars": []}]
-         }]
+        {
+            "username": "USER",
+            "url": "HTTP://SOMEURL",
+            "ip": "127.0.0.1",
+            "start_time": now.strftime("%Y-%m-%dT%H:%M:%S.0"),
+            "end_time": now.strftime("%Y-%m-%dT%H:%M:%S.0"),
+            "user_agent": "BROWSER_AGENT",
+            "message": "arbitrary text that will get attached to your report",
+            "request_id": "SOME_UUID",
+            "request": {
+                "REQUEST_METHOD": "GET",
+                "PATH_INFO": "/FOO/BAR",
+                "POST": {"FOO": "BAZ", "XXX": "YYY"},
+            },
+            "slow_calls": [
+                {
+                    "type": "sql",
+                    "start": now.strftime("%Y-%m-%dT%H:%M:%S.0"),
+                    "end": now.strftime("%Y-%m-%dT%H:%M:%S.0"),
+                    "subtype": "postgresql",
+                    "parameters": ["QPARAM1", "QPARAM2", "QPARAMX"],
+                    "statement": "QUERY",
+                }
+            ],
+            "request_stats": {
+                "main": 0.50779,
+                "nosql": 0.01008,
+                "nosql_calls": 17.0,
+                "remote": 0.0,
+                "remote_calls": 0.0,
+                "custom": 0.0,
+                "custom_calls": 0.0,
+                "sql": 0.42423,
+                "sql_calls": 1.0,
+                "tmpl": 0.0,
+                "tmpl_calls": 0.0,
+            },
+            "traceback": [
+                {
+                    "cline": "return foo_bar_baz(1,2,3)",
+                    "file": "somedir/somefile.py",
+                    "fn": "somefunction",
+                    "line": 454,
+                    "vars": [
+                        ["a_list", ["1", "2", "4", "5", "6"]],
+                        ["b", {1: "2", "ccc": "ddd", "1": "a"}],
+                        ["obj", "<object object at 0x7f0030853dc0>"],
+                    ],
+                },
+                {
+                    "cline": "OMG ValueError happened",
+                    "file": "",
+                    "fn": "",
+                    "line": "",
+                    "vars": [],
+                },
+            ],
+        }
+    ],
 }
 
 PYTHON_PAYLOAD_0_5 = {
     "client": "your-client-name-python",
     "language": "python",
-    'view_name': 'views/foo:bar',
-    'server': "servername/instancename",
+    "view_name": "views/foo:bar",
+    "server": "servername/instancename",
     "priority": 5,
     "error": "OMG ValueError happened test",
     "occurences": 1,
     "http_status": 500,
     "username": "USER",
     "url": "HTTP://SOMEURL",
     "ip": "127.0.0.1",
-    "start_time": now.strftime(
-        '%Y-%m-%dT%H:%M:%S.0'),
-    "end_time": now.strftime(
-        '%Y-%m-%dT%H:%M:%S.0'),
+    "start_time": now.strftime("%Y-%m-%dT%H:%M:%S.0"),
+    "end_time": now.strftime("%Y-%m-%dT%H:%M:%S.0"),
     "user_agent": "BROWSER_AGENT",
     "message": "arbitrary text that will get attached to your report",
     "request_id": "SOME_UUID",
-    "request": {"REQUEST_METHOD": "GET",
-                "PATH_INFO": "/FOO/BAR",
-                "POST": {"FOO": "BAZ", "XXX": "YYY"}},
+    "request": {
+        "REQUEST_METHOD": "GET",
+        "PATH_INFO": "/FOO/BAR",
+        "POST": {"FOO": "BAZ", "XXX": "YYY"},
+    },
     "slow_calls": [
         {
             "type": "sql",
-            "start": now.strftime(
-                '%Y-%m-%dT%H:%M:%S.0'),
-            "end": now.strftime(
-                '%Y-%m-%dT%H:%M:%S.0'),
+            "start": now.strftime("%Y-%m-%dT%H:%M:%S.0"),
+            "end": now.strftime("%Y-%m-%dT%H:%M:%S.0"),
             "subtype": "postgresql",
             "parameters": ["QPARAM1", "QPARAM2", "QPARAMX"],
-            "statement": "QUERY"}],
-    "request_stats": {"main": 0.50779,
-                      "nosql": 0.01008,
-                      "nosql_calls": 17.0,
-                      "remote": 0.0,
-                      "remote_calls": 0.0,
-                      "custom": 0.0,
-                      "custom_calls": 0.0,
-                      "sql": 0.42423,
-                      "sql_calls": 1.0,
-                      "tmpl": 0.0,
-                      "tmpl_calls": 0.0},
+            "statement": "QUERY",
+        }
+    ],
+    "request_stats": {
+        "main": 0.50779,
+        "nosql": 0.01008,
+        "nosql_calls": 17.0,
+        "remote": 0.0,
+        "remote_calls": 0.0,
+        "custom": 0.0,
+        "custom_calls": 0.0,
+        "sql": 0.42423,
+        "sql_calls": 1.0,
+        "tmpl": 0.0,
+        "tmpl_calls": 0.0,
+    },
     "traceback": [
-        {"cline": "return foo_bar_baz(1,2,3)",
-         "file": "somedir/somefile.py",
-         "fn": "somefunction",
-         "line": 454,
-         "vars": [["a_list",
-                   ["1",
-                    "2",
-                    "4",
-                    "5",
-                    "6"]],
-                  ["b", {1: "2", "ccc": "ddd", "1": "a"}],
-                  ["obj",
-                   "<object object at 0x7f0030853dc0>"]]},
-        {"cline": "OMG ValueError happened",
-         "file": "",
-         "fn": "",
-         "line": "",
-         "vars": []}]
+        {
+            "cline": "return foo_bar_baz(1,2,3)",
+            "file": "somedir/somefile.py",
+            "fn": "somefunction",
+            "line": 454,
+            "vars": [
+                ["a_list", ["1", "2", "4", "5", "6"]],
+                ["b", {1: "2", "ccc": "ddd", "1": "a"}],
+                ["obj", "<object object at 0x7f0030853dc0>"],
+            ],
+        },
+        {
+            "cline": "OMG ValueError happened",
+            "file": "",
+            "fn": "",
+            "line": "",
+            "vars": [],
+        },
+    ],
 }
 
 PHP_PAYLOAD = {
-    'client': 'php',
-    'error': 'Nie mo\u017cna ustali\u0107 \u017c\u0105dania "feed.xml".',
-    'error_type': '',
-    'http_status': 404,
-    'language': 'unknown',
-    'priority': 1,
-    'report_details': [{'end_time': None,
-                        'group_string': None,
-                        'ip': None,
-                        'message': 'exception \'CHttpException\' with message \'Nie mo\u017cna ustali\u0107 \u017c\u0105dania "feed.xml".\' in /home/dobryslownik/www/sites/dobryslownik/vendor/yiisoft/yii/framework/web/CWebApplication.php:286\nStack trace:\n#0 /home/dobryslownik/www/sites/dobryslownik/common/components/WebApplication.php(34): CWebApplication->runController(\'feed.xml\')\n#1 /home/dobryslownik/www/sites/dobryslownik/vendor/yiisoft/yii/framework/web/CWebApplication.php(141): WebApplication->runController(\'feed.xml\')\n#2 /home/dobryslownik/www/sites/dobryslownik/vendor/yiisoft/yii/framework/base/CApplication.php(180): CWebApplication->processRequest()\n#3 /home/dobryslownik/www/sites/dobryslownik/frontend/www/index.php(23): CApplication->run()\n#4 {main}',
-                        'occurences': 1,
-                        'request': {
-                            'COOKIES': [],
-                            'FILES': [],
-                            'GET': [],
-                            'POST': [],
-                            'REQUEST_METHOD': None,
-                            'SERVER': {
-                                'DOCUMENT_ROOT': '/home/dobryslownik/www/sites/dobryslownik/frontend/www/',
-                                'GATEWAY_INTERFACE': 'CGI/1.1',
-                                'HTTPS': 'on',
-                                'HTTP_ACCEPT': '*/*',
-                                'HTTP_ACCEPT_ENCODING': 'gzip, deflate',
-                                'HTTP_ACCEPT_LANGUAGE': 'pl-PL',
-                                'HTTP_CONNECTION': 'close',
-                                'HTTP_HOST': 'dobryslownik.pl',
-                                'HTTP_IF_MODIFIED_SINCE': 'Wed, 30 Jul 2014 18:26:32 GMT',
-                                'HTTP_IF_NONE_MATCH': '"45de3-2a3-4ff6d4b9fbe7f"',
-                                'HTTP_USER_AGENT': 'Apple-PubSub/28',
-                                'HTTP_X_FORWARDED_FOR': '195.150.190.186',
-                                'HTTP_X_FORWARDED_PROTO': 'https',
-                                'PATH': '/bin:/usr/bin:/usr/ucb:/usr/bsd:/usr/local/bin',
-                                'PHP_SELF': '/index.php',
-                                'QUERY_STRING': '',
-                                'REDIRECT_HTTPS': 'on',
-                                'REDIRECT_STATUS': '200',
-                                'REDIRECT_UNIQUE_ID': 'VFAhZQoCaXIAAAkd414AAAAC',
-                                'REDIRECT_URL': '/feed.xml',
-                                'REMOTE_ADDR': '195.150.190.186',
-                                'REMOTE_PORT': '41728',
-                                'REQUEST_METHOD': 'GET',
-                                'REQUEST_TIME': 1414537573,
-                                'REQUEST_TIME_FLOAT': 1414537573.32,
-                                'REQUEST_URI': '/feed.xml',
-                                'SCRIPT_FILENAME': '/home/dobryslownik/www/sites/dobryslownik/frontend/www/index.php',
-                                'SCRIPT_NAME': '/index.php',
-                                'SERVER_ADDR': '10.2.105.114',
-                                'SERVER_ADMIN': '[no address given]',
-                                'SERVER_NAME': 'dobryslownik.pl',
-                                'SERVER_SIGNATURE': '',
-                                'SERVER_SOFTWARE': 'Apache/2.2.22 (Ubuntu) PHP/5.4.17',
-                                'UNIQUE_ID': 'VFAg4AoCaXIAAAkd40UAAAAC'},
-                            'SESSION': []},
-                        'request_id': 'VFAg4AoCaXIAAAkd40UAAAAC',
-                        'request_stats': {'custom': 0,
-                                          'custom_calls': 0,
-                                          'main': 0,
-                                          'nosql': 0.0,
-                                          'nosql_calls': 0.0,
-                                          'remote': 0.0,
-                                          'remote_calls': 0.0,
-                                          'sql': 0.0,
-                                          'sql_calls': 0.0,
-                                          'tmpl': 0.0,
-                                          'tmpl_calls': 0.0,
-                                          'unknown': 0.0},
-                        'slow_calls': [],
-                        'start_time': None,
-                        'frameinfo': [
-                            {'cline': None,
-                             'file': '/home/dobryslownik/www/sites/dobryslownik/common/components/WebApplication.php',
-                             'fn': 'CWebApplication->runController',
-                             'line': 34,
-                             'vars': ['feed.xml']},
-                            {'cline': None,
-                             'file': '/home/dobryslownik/www/sites/dobryslownik/vendor/yiisoft/yii/framework/web/CWebApplication.php',
-                             'fn': 'WebApplication->runController',
-                             'line': 141,
-                             'vars': ['feed.xml']},
-                            {'cline': None,
-                             'file': '/home/dobryslownik/www/sites/dobryslownik/vendor/yiisoft/yii/framework/base/CApplication.php',
-                             'fn': 'CWebApplication->processRequest',
-                             'line': 180,
-                             'vars': []},
-                            {'cline': None,
-                             'file': '/home/dobryslownik/www/sites/dobryslownik/frontend/www/index.php',
-                             'fn': 'CApplication->run',
-                             'line': 23,
-                             'vars': []}],
-                        'url': 'https://dobryslownik.pl/feed.xml',
-                        'user_agent': 'magpie-crawler/1.1 (U; Linux amd64; en-GB; +http://www.brandwatch.net)',
-                        'username': 'guest'}],
-    'server': 'unknown',
-    'traceback': '',
-    'view_name': ''}
+    "client": "php",
+    "error": 'Nie mo\u017cna ustali\u0107 \u017c\u0105dania "feed.xml".',
+    "error_type": "",
+    "http_status": 404,
+    "language": "unknown",
+    "priority": 1,
+    "report_details": [
+        {
+            "end_time": None,
+            "group_string": None,
+            "ip": None,
+            "message": "exception 'CHttpException' with message 'Nie mo\u017cna ustali\u0107 \u017c\u0105dania \"feed.xml\".' in /home/dobryslownik/www/sites/dobryslownik/vendor/yiisoft/yii/framework/web/CWebApplication.php:286\nStack trace:\n#0 /home/dobryslownik/www/sites/dobryslownik/common/components/WebApplication.php(34): CWebApplication->runController('feed.xml')\n#1 /home/dobryslownik/www/sites/dobryslownik/vendor/yiisoft/yii/framework/web/CWebApplication.php(141): WebApplication->runController('feed.xml')\n#2 /home/dobryslownik/www/sites/dobryslownik/vendor/yiisoft/yii/framework/base/CApplication.php(180): CWebApplication->processRequest()\n#3 /home/dobryslownik/www/sites/dobryslownik/frontend/www/index.php(23): CApplication->run()\n#4 {main}",
+            "occurences": 1,
+            "request": {
+                "COOKIES": [],
+                "FILES": [],
+                "GET": [],
+                "POST": [],
+                "REQUEST_METHOD": None,
+                "SERVER": {
+                    "DOCUMENT_ROOT": "/home/dobryslownik/www/sites/dobryslownik/frontend/www/",
+                    "GATEWAY_INTERFACE": "CGI/1.1",
+                    "HTTPS": "on",
+                    "HTTP_ACCEPT": "*/*",
+                    "HTTP_ACCEPT_ENCODING": "gzip, deflate",
+                    "HTTP_ACCEPT_LANGUAGE": "pl-PL",
+                    "HTTP_CONNECTION": "close",
+                    "HTTP_HOST": "dobryslownik.pl",
+                    "HTTP_IF_MODIFIED_SINCE": "Wed, 30 Jul 2014 18:26:32 GMT",
+                    "HTTP_IF_NONE_MATCH": '"45de3-2a3-4ff6d4b9fbe7f"',
+                    "HTTP_USER_AGENT": "Apple-PubSub/28",
+                    "HTTP_X_FORWARDED_FOR": "195.150.190.186",
+                    "HTTP_X_FORWARDED_PROTO": "https",
+                    "PATH": "/bin:/usr/bin:/usr/ucb:/usr/bsd:/usr/local/bin",
+                    "PHP_SELF": "/index.php",
+                    "QUERY_STRING": "",
+                    "REDIRECT_HTTPS": "on",
+                    "REDIRECT_STATUS": "200",
+                    "REDIRECT_UNIQUE_ID": "VFAhZQoCaXIAAAkd414AAAAC",
+                    "REDIRECT_URL": "/feed.xml",
+                    "REMOTE_ADDR": "195.150.190.186",
+                    "REMOTE_PORT": "41728",
+                    "REQUEST_METHOD": "GET",
+                    "REQUEST_TIME": 1414537573,
+                    "REQUEST_TIME_FLOAT": 1414537573.32,
+                    "REQUEST_URI": "/feed.xml",
+                    "SCRIPT_FILENAME": "/home/dobryslownik/www/sites/dobryslownik/frontend/www/index.php",
+                    "SCRIPT_NAME": "/index.php",
+                    "SERVER_ADDR": "10.2.105.114",
+                    "SERVER_ADMIN": "[no address given]",
+                    "SERVER_NAME": "dobryslownik.pl",
+                    "SERVER_SIGNATURE": "",
+                    "SERVER_SOFTWARE": "Apache/2.2.22 (Ubuntu) PHP/5.4.17",
+                    "UNIQUE_ID": "VFAg4AoCaXIAAAkd40UAAAAC",
+                },
+                "SESSION": [],
+            },
+            "request_id": "VFAg4AoCaXIAAAkd40UAAAAC",
+            "request_stats": {
+                "custom": 0,
+                "custom_calls": 0,
+                "main": 0,
+                "nosql": 0.0,
+                "nosql_calls": 0.0,
+                "remote": 0.0,
+                "remote_calls": 0.0,
+                "sql": 0.0,
+                "sql_calls": 0.0,
+                "tmpl": 0.0,
+                "tmpl_calls": 0.0,
+                "unknown": 0.0,
+            },
+            "slow_calls": [],
+            "start_time": None,
+            "frameinfo": [
+                {
+                    "cline": None,
+                    "file": "/home/dobryslownik/www/sites/dobryslownik/common/components/WebApplication.php",
+                    "fn": "CWebApplication->runController",
+                    "line": 34,
+                    "vars": ["feed.xml"],
+                },
+                {
+                    "cline": None,
+                    "file": "/home/dobryslownik/www/sites/dobryslownik/vendor/yiisoft/yii/framework/web/CWebApplication.php",
+                    "fn": "WebApplication->runController",
+                    "line": 141,
+                    "vars": ["feed.xml"],
+                },
+                {
+                    "cline": None,
+                    "file": "/home/dobryslownik/www/sites/dobryslownik/vendor/yiisoft/yii/framework/base/CApplication.php",
+                    "fn": "CWebApplication->processRequest",
+                    "line": 180,
+                    "vars": [],
+                },
+                {
+                    "cline": None,
+                    "file": "/home/dobryslownik/www/sites/dobryslownik/frontend/www/index.php",
+                    "fn": "CApplication->run",
+                    "line": 23,
+                    "vars": [],
+                },
+            ],
+            "url": "https://dobryslownik.pl/feed.xml",
+            "user_agent": "magpie-crawler/1.1 (U; Linux amd64; en-GB; +http://www.brandwatch.net)",
+            "username": "guest",
+        }
+    ],
+    "server": "unknown",
+    "traceback": "",
+    "view_name": "",
+}
 
 JS_PAYLOAD = {
-    "client": "javascript", "language": "javascript",
+    "client": "javascript",
+    "language": "javascript",
     "error_type": "ReferenceError: non_existant_var is not defined",
-    "occurences": 1, "priority": 5, "server": "jstest.appenlight",
-    "http_status": 500, "report_details": [{
-        "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/537.36",
-        "start_time": "2014-10-29T19:59:30.589Z",
-        "frameinfo": [
-            {
-                "cline": "",
-                "file": "http://demo.appenlight.com/#",
-                "fn": "HTMLAnchorElement.onclick",
-                "line": 79,
-                "vars": []},
-            {
-                "cline": "",
-                "file": "http://demo.appenlight.com/static/js/demo.js",
-                "fn": "test_error",
-                "line": 7,
-                "vars": []},
-            {
-                "cline": "ReferenceError: non_existant_var is not defined",
-                "file": "http://demo.appenlight.com/static/js/demo.js",
-                "fn": "something",
-                "line": 2,
-                "vars": []}],
-        "url": "http://demo.appenlight.com/#",
-        "server": "jstest.appenlight",
-        "username": "i_am_mario",
-        "ip": "127.0.0.1",
-        "request_id": "0.01984176435507834"}]}
+    "occurences": 1,
+    "priority": 5,
+    "server": "jstest.appenlight",
+    "http_status": 500,
+    "report_details": [
+        {
+            "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/537.36",
+            "start_time": "2014-10-29T19:59:30.589Z",
+            "frameinfo": [
+                {
+                    "cline": "",
+                    "file": "http://demo.appenlight.com/#",
+                    "fn": "HTMLAnchorElement.onclick",
+                    "line": 79,
+                    "vars": [],
+                },
+                {
+                    "cline": "",
+                    "file": "http://demo.appenlight.com/static/js/demo.js",
+                    "fn": "test_error",
+                    "line": 7,
+                    "vars": [],
+                },
+                {
+                    "cline": "ReferenceError: non_existant_var is not defined",
+                    "file": "http://demo.appenlight.com/static/js/demo.js",
+                    "fn": "something",
+                    "line": 2,
+                    "vars": [],
+                },
+            ],
+            "url": "http://demo.appenlight.com/#",
+            "server": "jstest.appenlight",
+            "username": "i_am_mario",
+            "ip": "127.0.0.1",
+            "request_id": "0.01984176435507834",
+        }
+    ],
+}
 
 AIRBRAKE_RUBY_EXAMPLE = """
 <?xml version="1.0" ?>
 <notice version="2.4">
     <api-key>APPENLIGHT_API_KEY</api-key>
     <notifier>
         <name>Airbrake Notifier</name>
         <version>3.1.7</version>
         <url>https://github.com/airbrake/airbrake</url>
     </notifier>
     <error>
         <class>NameError</class>
         <message>NameError: undefined local variable or method `sdfdfdf' for #&lt;#&lt;Class:0x000000039a8b90&gt;:0x00000002c53df0&gt;</message>
         <backtrace>
             <line file="[PROJECT_ROOT]/app/views/welcome/index.html.erb" method="_app_views_welcome_index_html_erb___2570061166873166679_31748940" number="3"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_view/template.rb" method="block in render" number="145"/>
             <line file="[GEM_ROOT]/gems/activesupport-3.2.11/lib/active_support/notifications.rb" method="instrument" number="125"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_view/template.rb" method="render" number="143"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_view/renderer/template_renderer.rb" method="block (2 levels) in render_template" number="47"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_view/renderer/abstract_renderer.rb" method="block in instrument" number="38"/>
             <line file="[GEM_ROOT]/gems/activesupport-3.2.11/lib/active_support/notifications.rb" method="block in instrument" number="123"/>
             <line file="[GEM_ROOT]/gems/activesupport-3.2.11/lib/active_support/notifications/instrumenter.rb" method="instrument" number="20"/>
             <line file="[GEM_ROOT]/gems/activesupport-3.2.11/lib/active_support/notifications.rb" method="instrument" number="123"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_view/renderer/abstract_renderer.rb" method="instrument" number="38"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_view/renderer/template_renderer.rb" method="block in render_template" number="46"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_view/renderer/template_renderer.rb" method="render_with_layout" number="54"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_view/renderer/template_renderer.rb" method="render_template" number="45"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_view/renderer/template_renderer.rb" method="render" number="18"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_view/renderer/renderer.rb" method="render_template" number="36"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_view/renderer/renderer.rb" method="render" number="17"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/abstract_controller/rendering.rb" method="_render_template" number="110"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_controller/metal/streaming.rb" method="_render_template" number="225"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/abstract_controller/rendering.rb" method="render_to_body" number="103"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_controller/metal/renderers.rb" method="render_to_body" number="28"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_controller/metal/compatibility.rb" method="render_to_body" number="50"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/abstract_controller/rendering.rb" method="render" number="88"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_controller/metal/rendering.rb" method="render" number="16"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_controller/metal/instrumentation.rb" method="block (2 levels) in render" number="40"/>
             <line file="[GEM_ROOT]/gems/activesupport-3.2.11/lib/active_support/core_ext/benchmark.rb" method="block in ms" number="5"/>
             <line file="/home/ergo/.rbenv/versions/1.9.3-p327/lib/ruby/1.9.1/benchmark.rb" method="realtime" number="295"/>
             <line file="[GEM_ROOT]/gems/activesupport-3.2.11/lib/active_support/core_ext/benchmark.rb" method="ms" number="5"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_controller/metal/instrumentation.rb" method="block in render" number="40"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_controller/metal/instrumentation.rb" method="cleanup_view_runtime" number="83"/>
             <line file="[GEM_ROOT]/gems/activerecord-3.2.11/lib/active_record/railties/controller_runtime.rb" method="cleanup_view_runtime" number="24"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_controller/metal/instrumentation.rb" method="render" number="39"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_controller/metal/implicit_render.rb" method="default_render" number="10"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_controller/metal/implicit_render.rb" method="send_action" number="5"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/abstract_controller/base.rb" method="process_action" number="167"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_controller/metal/rendering.rb" method="process_action" number="10"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/abstract_controller/callbacks.rb" method="block in process_action" number="18"/>
             <line file="[GEM_ROOT]/gems/activesupport-3.2.11/lib/active_support/callbacks.rb" method="_run__2035827918079922105__process_action__2488436794202946797__callbacks" number="414"/>
             <line file="[GEM_ROOT]/gems/activesupport-3.2.11/lib/active_support/callbacks.rb" method="__run_callback" number="405"/>
             <line file="[GEM_ROOT]/gems/activesupport-3.2.11/lib/active_support/callbacks.rb" method="_run_process_action_callbacks" number="385"/>
             <line file="[GEM_ROOT]/gems/activesupport-3.2.11/lib/active_support/callbacks.rb" method="run_callbacks" number="81"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/abstract_controller/callbacks.rb" method="process_action" number="17"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_controller/metal/rescue.rb" method="process_action" number="29"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_controller/metal/instrumentation.rb" method="block in process_action" number="30"/>
             <line file="[GEM_ROOT]/gems/activesupport-3.2.11/lib/active_support/notifications.rb" method="block in instrument" number="123"/>
             <line file="[GEM_ROOT]/gems/activesupport-3.2.11/lib/active_support/notifications/instrumenter.rb" method="instrument" number="20"/>
             <line file="[GEM_ROOT]/gems/activesupport-3.2.11/lib/active_support/notifications.rb" method="instrument" number="123"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_controller/metal/instrumentation.rb" method="process_action" number="29"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_controller/metal/params_wrapper.rb" method="process_action" number="207"/>
             <line file="[GEM_ROOT]/gems/activerecord-3.2.11/lib/active_record/railties/controller_runtime.rb" method="process_action" number="18"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/abstract_controller/base.rb" method="process" number="121"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/abstract_controller/rendering.rb" method="process" number="45"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_controller/metal.rb" method="dispatch" number="203"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_controller/metal/rack_delegation.rb" method="dispatch" number="14"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_controller/metal.rb" method="block in action" number="246"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_dispatch/routing/route_set.rb" method="call" number="73"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_dispatch/routing/route_set.rb" method="dispatch" number="73"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_dispatch/routing/route_set.rb" method="call" number="36"/>
             <line file="[GEM_ROOT]/gems/journey-1.0.4/lib/journey/router.rb" method="block in call" number="68"/>
             <line file="[GEM_ROOT]/gems/journey-1.0.4/lib/journey/router.rb" method="each" number="56"/>
             <line file="[GEM_ROOT]/gems/journey-1.0.4/lib/journey/router.rb" method="call" number="56"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_dispatch/routing/route_set.rb" method="call" number="601"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_dispatch/middleware/best_standards_support.rb" method="call" number="17"/>
             <line file="[GEM_ROOT]/gems/rack-1.4.4/lib/rack/etag.rb" method="call" number="23"/>
             <line file="[GEM_ROOT]/gems/rack-1.4.4/lib/rack/conditionalget.rb" method="call" number="25"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_dispatch/middleware/head.rb" method="call" number="14"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_dispatch/middleware/params_parser.rb" method="call" number="21"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_dispatch/middleware/flash.rb" method="call" number="242"/>
             <line file="[GEM_ROOT]/gems/rack-1.4.4/lib/rack/session/abstract/id.rb" method="context" number="210"/>
             <line file="[GEM_ROOT]/gems/rack-1.4.4/lib/rack/session/abstract/id.rb" method="call" number="205"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_dispatch/middleware/cookies.rb" method="call" number="341"/>
             <line file="[GEM_ROOT]/gems/activerecord-3.2.11/lib/active_record/query_cache.rb" method="call" number="64"/>
             <line file="[GEM_ROOT]/gems/activerecord-3.2.11/lib/active_record/connection_adapters/abstract/connection_pool.rb" method="call" number="479"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_dispatch/middleware/callbacks.rb" method="block in call" number="28"/>
             <line file="[GEM_ROOT]/gems/activesupport-3.2.11/lib/active_support/callbacks.rb" method="_run__3791281566140176909__call__1380026543027572563__callbacks" number="405"/>
             <line file="[GEM_ROOT]/gems/activesupport-3.2.11/lib/active_support/callbacks.rb" method="__run_callback" number="405"/>
             <line file="[GEM_ROOT]/gems/activesupport-3.2.11/lib/active_support/callbacks.rb" method="_run_call_callbacks" number="385"/>
             <line file="[GEM_ROOT]/gems/activesupport-3.2.11/lib/active_support/callbacks.rb" method="run_callbacks" number="81"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_dispatch/middleware/callbacks.rb" method="call" number="27"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_dispatch/middleware/reloader.rb" method="call" number="65"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_dispatch/middleware/remote_ip.rb" method="call" number="31"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_dispatch/middleware/debug_exceptions.rb" method="call" number="16"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_dispatch/middleware/show_exceptions.rb" method="call" number="56"/>
             <line file="[GEM_ROOT]/gems/railties-3.2.11/lib/rails/rack/logger.rb" method="call_app" number="32"/>
             <line file="[GEM_ROOT]/gems/railties-3.2.11/lib/rails/rack/logger.rb" method="block in call" number="16"/>
             <line file="[GEM_ROOT]/gems/activesupport-3.2.11/lib/active_support/tagged_logging.rb" method="tagged" number="22"/>
             <line file="[GEM_ROOT]/gems/railties-3.2.11/lib/rails/rack/logger.rb" method="call" number="16"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_dispatch/middleware/request_id.rb" method="call" number="22"/>
             <line file="[GEM_ROOT]/gems/rack-1.4.4/lib/rack/methodoverride.rb" method="call" number="21"/>
             <line file="[GEM_ROOT]/gems/rack-1.4.4/lib/rack/runtime.rb" method="call" number="17"/>
             <line file="[GEM_ROOT]/gems/activesupport-3.2.11/lib/active_support/cache/strategy/local_cache.rb" method="call" number="72"/>
             <line file="[GEM_ROOT]/gems/rack-1.4.4/lib/rack/lock.rb" method="call" number="15"/>
             <line file="[GEM_ROOT]/gems/actionpack-3.2.11/lib/action_dispatch/middleware/static.rb" method="call" number="62"/>
             <line file="[GEM_ROOT]/gems/railties-3.2.11/lib/rails/engine.rb" method="call" number="479"/>
             <line file="[GEM_ROOT]/gems/railties-3.2.11/lib/rails/application.rb" method="call" number="223"/>
             <line file="[GEM_ROOT]/gems/rack-1.4.4/lib/rack/content_length.rb" method="call" number="14"/>
             <line file="[GEM_ROOT]/gems/railties-3.2.11/lib/rails/rack/log_tailer.rb" method="call" number="17"/>
             <line file="[GEM_ROOT]/gems/rack-1.4.4/lib/rack/handler/webrick.rb" method="service" number="59"/>
             <line file="/home/ergo/.rbenv/versions/1.9.3-p327/lib/ruby/1.9.1/webrick/httpserver.rb" method="service" number="138"/>
             <line file="/home/ergo/.rbenv/versions/1.9.3-p327/lib/ruby/1.9.1/webrick/httpserver.rb" method="run" number="94"/>
             <line file="/home/ergo/.rbenv/versions/1.9.3-p327/lib/ruby/1.9.1/webrick/server.rb" method="block in start_thread" number="191"/>
         </backtrace>
     </error>
     <request>
         <url>http://0.0.0.0:3000/welcome/index?test=1234</url>
         <component>welcome</component>
         <action>index</action>
         <params>
             <var key="test">1234</var>
             <var key="controller">welcome</var>
             <var key="action">index</var>
         </params>
         <session>
             <var key="session_id">4706af9678e4b94f2bb66e1d85ced382</var>
             <var key="_csrf_token">h9R7MuRtnNX6ZDK6vI1pIJV3dYYtlEx1mPw/nzyIVTA=</var>
         </session>
         <cgi-data>
             <var key="GATEWAY_INTERFACE">CGI/1.1</var>
             <var key="PATH_INFO">/welcome/index</var>
             <var key="QUERY_STRING">test=1234</var>
             <var key="REMOTE_ADDR">127.0.0.1</var>
             <var key="REMOTE_HOST">localhost</var>
             <var key="REQUEST_METHOD">GET</var>
             <var key="REQUEST_URI">http://0.0.0.0:3000/welcome/index?test=1234</var>
             <var key="SCRIPT_NAME"/>
             <var key="SERVER_NAME">0.0.0.0</var>
             <var key="SERVER_PORT">3000</var>
             <var key="SERVER_PROTOCOL">HTTP/1.1</var>
             <var key="SERVER_SOFTWARE">WEBrick/1.3.1 (Ruby/1.9.3/2012-11-10)</var>
             <var key="HTTP_HOST">0.0.0.0:3000</var>
             <var key="HTTP_CONNECTION">keep-alive</var>
             <var key="HTTP_CACHE_CONTROL">max-age=0</var>
             <var key="HTTP_ACCEPT">text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8</var>
             <var key="HTTP_USER_AGENT">Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.56 Safari/537.17</var>
             <var key="HTTP_DNT">1</var>
             <var key="HTTP_ACCEPT_ENCODING">gzip,deflate,sdch</var>
             <var key="HTTP_ACCEPT_LANGUAGE">pl,en-US;q=0.8,en;q=0.6</var>
             <var key="HTTP_ACCEPT_CHARSET">ISO-8859-1,utf-8;q=0.7,*;q=0.3</var>
             <var key="HTTP_COOKIE">_rails_app_session=BAh7B0kiD3Nlc3Npb25faWQGOgZFRkkiJTQ3MDZhZjk2NzhlNGI5NGYyYmI2NmUxZDg1Y2VkMzgyBjsAVEkiEF9jc3JmX3Rva2VuBjsARkkiMWg5UjdNdVJ0bk5YNlpESzZ2STFwSUpWM2RZWXRsRXgxbVB3L256eUlWVEE9BjsARg%3D%3D--08a5940133e1c7f7ca58ed5154e3a8e7acae337a</var>
             <var key="rack.version">[&quot;1&quot;, &quot;1&quot;]</var>
             <var key="rack.input">#&lt;StringIO:0x00000003b6ca08&gt;</var>
             <var key="rack.errors">#&lt;IO:0x00000001e76228&gt;</var>
             <var key="rack.multithread">false</var>
             <var key="rack.multiprocess">false</var>
             <var key="rack.run_once">false</var>
             <var key="rack.url_scheme">http</var>
             <var key="HTTP_VERSION">HTTP/1.1</var>
             <var key="REQUEST_PATH">/welcome/index</var>
             <var key="ORIGINAL_FULLPATH">/welcome/index?test=1234</var>
             <var key="action_dispatch.routes">#&lt;ActionDispatch::Routing::RouteSet:0x00000002b31648&gt;</var>
             <var key="action_dispatch.parameter_filter">[&quot;password&quot;]</var>
             <var key="action_dispatch.show_exceptions">true</var>
             <var key="action_dispatch.show_detailed_exceptions">true</var>
             <var key="action_dispatch.logger">#&lt;ActiveSupport::TaggedLogging:0x00000002ae79d0&gt;</var>
             <var key="action_dispatch.backtrace_cleaner">#&lt;Rails::BacktraceCleaner:0x000000029a7250&gt;</var>
             <var key="action_dispatch.request_id">c11b2267f3ad8b00a1768cae35559fa1</var>
             <var key="action_dispatch.remote_ip">127.0.0.1</var>
             <var key="rack.session">
                 <var key="session_id">4706af9678e4b94f2bb66e1d85ced382</var>
                 <var key="_csrf_token">h9R7MuRtnNX6ZDK6vI1pIJV3dYYtlEx1mPw/nzyIVTA=</var>
             </var>
             <var key="rack.session.options">
                 <var key="path">/</var>
                 <var key="domain"/>
                 <var key="expire_after"/>
                 <var key="secure">false</var>
                 <var key="httponly">true</var>
                 <var key="defer">false</var>
                 <var key="renew">false</var>
                 <var key="secret">e96386a74a0c95f5c8eeaf95e47d14962973a0ff9e9d32841703559423b1</var>
                 <var key="coder">#&lt;Rack::Session::Cookie::Base64::Marshal:0x00000002daf528&gt;</var>
                 <var key="id">4706af9678e4b94f2bb66e1d85ced382</var>
             </var>
             <var key="rack.request.cookie_hash">
                 <var key="_rails_app_session">BAh7B0kiD3Nlc3Npb25faWQGOgZFRkkiJTQ3MDZhZjk2NzhlNGI5NGYyYmI2NmUxZDg1Y2VkMzgyBjsAVEkiEF9jc3JmX3Rva2VuBjsARkkiMWg5UjdNdVJ0bk5YNlpESzZ2STFwSUpWM2RZWXRsRXgxbVB3L256eUlWVEE9BjsARg==--08a5940133e1c7f7ca58ed5154e3a8e7acae337a</var>
             </var>
             <var key="rack.request.cookie_string">_rails_app_session=BAh7B0kiD3Nlc3Npb25faWQGOgZFRkkiJTQ3MDZhZjk2NzhlNGI5NGYyYmI2NmUxZDg1Y2VkMzgyBjsAVEkiEF9jc3JmX3Rva2VuBjsARkkiMWg5UjdNdVJ0bk5YNlpESzZ2STFwSUpWM2RZWXRsRXgxbVB3L256eUlWVEE9BjsARg%3D%3D--08a5940133e1c7f7ca58ed5154e3a8e7acae337a</var>
             <var key="action_dispatch.cookies">#&lt;ActionDispatch::Cookies::CookieJar:0x00000002c51e38&gt;</var>
             <var key="action_dispatch.request.unsigned_session_cookie">
                 <var key="session_id">4706af9678e4b94f2bb66e1d85ced382</var>
                 <var key="_csrf_token">h9R7MuRtnNX6ZDK6vI1pIJV3dYYtlEx1mPw/nzyIVTA=</var>
             </var>
             <var key="action_dispatch.request.path_parameters">
                 <var key="controller">welcome</var>
                 <var key="action">index</var>
             </var>
             <var key="action_controller.instance">#&lt;WelcomeController:0x00000002c4f020&gt;</var>
             <var key="action_dispatch.request.content_type"/>
             <var key="action_dispatch.request.request_parameters"/>
             <var key="rack.request.query_string">test=1234</var>
             <var key="rack.request.query_hash">
                 <var key="test">1234</var>
             </var>
             <var key="action_dispatch.request.query_parameters">
                 <var key="test">1234</var>
             </var>
             <var key="action_dispatch.request.parameters">
                 <var key="test">1234</var>
                 <var key="controller">welcome</var>
                 <var key="action">index</var>
             </var>
             <var key="action_dispatch.request.formats">[&quot;text/html&quot;]</var>
         </cgi-data>
     </request>
     <server-environment>
         <project-root>/home/ergo/workspace/rails_app</project-root>
         <environment-name>development</environment-name>
         <hostname>ergo-desktop</hostname>
     </server-environment>
     <framework>Rails: 3.2.11</framework>
 </notice>
-""".replace('\n', '').replace('  ', '')
+""".replace(
+    "\n", ""
+).replace(
+    "  ", ""
+)
 
-AIRBRAKE_EXAMPLE_SHORT = '''
+AIRBRAKE_EXAMPLE_SHORT = """
 <?xml version="1.0" encoding="UTF-8"?>
 <notice version="2.3">
   <api-key>76fdb93ab2cf276ec080671a8b3d3866</api-key>
   <notifier>
     <name>Airbrake Notifier</name>
     <version>3.1.6</version>
     <url>http://api.airbrake.io</url>
   </notifier>
   <error>
     <class>RuntimeError</class>
     <message>RuntimeError: I've made a huge mistake</message>
     <backtrace>
       <line method="public" file="/testapp/app/models/user.rb" number="53"/>
       <line method="index" file="/testapp/app/controllers/users_controller.rb" number="14"/>
     </backtrace>
   </error>
   <request>
     <url>http://example.com</url>
     <component/>
     <action/>
     <cgi-data>
       <var key="SERVER_NAME">example.org</var>
       <var key="HTTP_USER_AGENT">Mozilla</var>
     </cgi-data>
   </request>
   <server-environment>
     <project-root>/testapp</project-root>
     <environment-name>production</environment-name>
     <app-version>1.0.0</app-version>
   </server-environment>
 </notice>
-'''.replace('\n', '').replace('  ', '')
+""".replace(
+    "\n", ""
+).replace(
+    "  ", ""
+)
 
 SENTRY_PYTHON_PAYLOAD_7 = {
-    'culprit': 'djangoapp.views in error',
-    'event_id': '9fae652c8c1c4d6a8eee09260f613a98',
-    'exception': {
-        'values': [
-            {'module': 'exceptions',
-             'stacktrace': {'frames': [{
-                 'abs_path': '/home/ergo/venvs/appenlight/local/lib/python2.7/site-packages/django/core/handlers/base.py',
-                 'context_line': 'response = wrapped_callback(request, *callback_args, **callback_kwargs)',
-                 'filename': 'django/core/handlers/base.py',
-                 'function': 'get_response',
-                 'in_app': False,
-                 'lineno': 111,
-                 'module': 'django.core.handlers.base',
-                 'post_context': [
-                     '                except Exception as e:',
-                     '                    # If the view raised an exception, run it through exception',
-                     '                    # middleware, and if the exception middleware returns a',
-                     '                    # response, use that. Otherwise, reraise the exception.',
-                     '                    for middleware_method in self._exception_middleware:'],
-                 'pre_context': [
-                     '                        break',
-                     '',
-                     '            if response is None:',
-                     '                wrapped_callback = self.make_view_atomic(callback)',
-                     '                try:'],
-                 'vars': {
-                     'callback': '<function error from djangoapp.views at 0x7fe7c9f2cb90>',
-                     'callback_args': [],
-                     'callback_kwargs': {},
-                     'e': "Exception(u'test 500 \\u0142\\xf3\\u201c\\u0107\\u201c\\u0107\\u017c\\u0105',)",
-                     'middleware_method': '<bound method MessageMiddleware.process_request of <django.contrib.messages.middleware.MessageMiddleware object at 0x7fe7c8b0c950>>',
-                     'request': '<WSGIRequest at 0x140633490316304>',
-                     'resolver': "<RegexURLResolver 'djangoapp.urls' (None:None) ^/>",
-                     'resolver_match': "ResolverMatch(func=<function error at 0x7fe7c9f2cb90>, args=(), kwargs={}, url_name='error', app_name='None', namespace='')",
-                     'response': None,
-                     'self': '<django.core.handlers.wsgi.WSGIHandler object at 0x7fe7cf75a790>',
-                     'urlconf': "'djangoapp.urls'",
-                     'wrapped_callback': '<function error from djangoapp.views at 0x7fe7c9f2cb90>'}},
-                 {
-                     'abs_path': '/home/ergo/IdeaProjects/django_raven/djangoapp/views.py',
-                     'context_line': "raise Exception(u'test 500 \u0142\xf3\u201c\u0107\u201c\u0107\u017c\u0105')",
-                     'filename': 'djangoapp/views.py',
-                     'function': 'error',
-                     'in_app': False,
-                     'lineno': 84,
-                     'module': 'djangoapp.views',
-                     'post_context': [
-                         '',
-                         '',
-                         'def notfound(request):',
-                         "    raise Http404('404 appenlight exception test')",
-                         ''],
-                     'pre_context': [
-                         '    c.execute("INSERT INTO stocks VALUES (\'2006-01-05\',\'BUY\',\'RHAT\',100,35.14)")',
-                         '    c.execute("INSERT INTO stocks VALUES (\'2006-01-05\',\'BUY\',\'RHAT\',100,35.14)")',
-                         '    conn.commit()',
-                         '    c.close()',
-                         "    request.POST.get('DUPA')"],
-                     'vars': {
-                         'c': '<sqlite3.Cursor object at 0x7fe7c82af8f0>',
-                         'conn': '<sqlite3.Connection object at 0x7fe7c8b23bf8>',
-                         'request': '<WSGIRequest at 0x140633490316304>'}}]},
-             'type': 'Exception',
-             'value': 'test 500 \u0142\xf3\u201c\u0107\u201c\u0107\u017c\u0105'}]},
-    'extra': {
-        'sys.argv': ["'manage.py'", "'runserver'"]},
-    'level': 40,
-    'message': 'Exception: test 500 \u0142\xf3\u201c\u0107\u201c\u0107\u017c\u0105',
-    'modules': {'django': '1.7.1',
-                 'python': '2.7.6',
-                 'raven': '5.9.2'},
-    'platform': 'python',
-    'project': 'sentry',
-    'release': 'test',
-    'request': {'cookies': {
-        'appenlight': 'X'},
-        'data': None,
-        'env': {'REMOTE_ADDR': '127.0.0.1',
-                 'SERVER_NAME': 'localhost',
-                 'SERVER_PORT': '8000'},
-        'headers': {
-            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
-            'Accept-Encoding': 'gzip, deflate, sdch',
-            'Accept-Language': 'en-US,en;q=0.8,pl;q=0.6',
-            'Connection': 'keep-alive',
-            'Content-Length': '',
-            'Content-Type': 'text/plain',
-            'Cookie': 'appenlight=X',
-            'Dnt': '1',
-            'Host': '127.0.0.1:8000',
-            'Upgrade-Insecure-Requests': '1',
-            'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36'},
-        'method': 'GET',
-        'query_string': '',
-        'url': 'http://127.0.0.1:8000/error'},
-    'server_name': 'ergo-virtual-machine',
-    'tags': {'site': 'example.com'},
-    'time_spent': None,
-    'timestamp': now.strftime('%Y-%m-%dT%H:%M:%SZ')}
+    "culprit": "djangoapp.views in error",
+    "event_id": "9fae652c8c1c4d6a8eee09260f613a98",
+    "exception": {
+        "values": [
+            {
+                "module": "exceptions",
+                "stacktrace": {
+                    "frames": [
+                        {
+                            "abs_path": "/home/ergo/venvs/appenlight/local/lib/python2.7/site-packages/django/core/handlers/base.py",
+                            "context_line": "response = wrapped_callback(request, *callback_args, **callback_kwargs)",
+                            "filename": "django/core/handlers/base.py",
+                            "function": "get_response",
+                            "in_app": False,
+                            "lineno": 111,
+                            "module": "django.core.handlers.base",
+                            "post_context": [
+                                "                except Exception as e:",
+                                "                    # If the view raised an exception, run it through exception",
+                                "                    # middleware, and if the exception middleware returns a",
+                                "                    # response, use that. Otherwise, reraise the exception.",
+                                "                    for middleware_method in self._exception_middleware:",
+                            ],
+                            "pre_context": [
+                                "                        break",
+                                "",
+                                "            if response is None:",
+                                "                wrapped_callback = self.make_view_atomic(callback)",
+                                "                try:",
+                            ],
+                            "vars": {
+                                "callback": "<function error from djangoapp.views at 0x7fe7c9f2cb90>",
+                                "callback_args": [],
+                                "callback_kwargs": {},
+                                "e": "Exception(u'test 500 \\u0142\\xf3\\u201c\\u0107\\u201c\\u0107\\u017c\\u0105',)",
+                                "middleware_method": "<bound method MessageMiddleware.process_request of <django.contrib.messages.middleware.MessageMiddleware object at 0x7fe7c8b0c950>>",
+                                "request": "<WSGIRequest at 0x140633490316304>",
+                                "resolver": "<RegexURLResolver 'djangoapp.urls' (None:None) ^/>",
+                                "resolver_match": "ResolverMatch(func=<function error at 0x7fe7c9f2cb90>, args=(), kwargs={}, url_name='error', app_name='None', namespace='')",
+                                "response": None,
+                                "self": "<django.core.handlers.wsgi.WSGIHandler object at 0x7fe7cf75a790>",
+                                "urlconf": "'djangoapp.urls'",
+                                "wrapped_callback": "<function error from djangoapp.views at 0x7fe7c9f2cb90>",
+                            },
+                        },
+                        {
+                            "abs_path": "/home/ergo/IdeaProjects/django_raven/djangoapp/views.py",
+                            "context_line": "raise Exception(u'test 500 \u0142\xf3\u201c\u0107\u201c\u0107\u017c\u0105')",
+                            "filename": "djangoapp/views.py",
+                            "function": "error",
+                            "in_app": False,
+                            "lineno": 84,
+                            "module": "djangoapp.views",
+                            "post_context": [
+                                "",
+                                "",
+                                "def notfound(request):",
+                                "    raise Http404('404 appenlight exception test')",
+                                "",
+                            ],
+                            "pre_context": [
+                                "    c.execute(\"INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14)\")",
+                                "    c.execute(\"INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14)\")",
+                                "    conn.commit()",
+                                "    c.close()",
+                                "    request.POST.get('DUPA')",
+                            ],
+                            "vars": {
+                                "c": "<sqlite3.Cursor object at 0x7fe7c82af8f0>",
+                                "conn": "<sqlite3.Connection object at 0x7fe7c8b23bf8>",
+                                "request": "<WSGIRequest at 0x140633490316304>",
+                            },
+                        },
+                    ]
+                },
+                "type": "Exception",
+                "value": "test 500 \u0142\xf3\u201c\u0107\u201c\u0107\u017c\u0105",
+            }
+        ]
+    },
+    "extra": {"sys.argv": ["'manage.py'", "'runserver'"]},
+    "level": 40,
+    "message": "Exception: test 500 \u0142\xf3\u201c\u0107\u201c\u0107\u017c\u0105",
+    "modules": {"django": "1.7.1", "python": "2.7.6", "raven": "5.9.2"},
+    "platform": "python",
+    "project": "sentry",
+    "release": "test",
+    "request": {
+        "cookies": {"appenlight": "X"},
+        "data": None,
+        "env": {
+            "REMOTE_ADDR": "127.0.0.1",
+            "SERVER_NAME": "localhost",
+            "SERVER_PORT": "8000",
+        },
+        "headers": {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+            "Accept-Encoding": "gzip, deflate, sdch",
+            "Accept-Language": "en-US,en;q=0.8,pl;q=0.6",
+            "Connection": "keep-alive",
+            "Content-Length": "",
+            "Content-Type": "text/plain",
+            "Cookie": "appenlight=X",
+            "Dnt": "1",
+            "Host": "127.0.0.1:8000",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36",
+        },
+        "method": "GET",
+        "query_string": "",
+        "url": "http://127.0.0.1:8000/error",
+    },
+    "server_name": "ergo-virtual-machine",
+    "tags": {"site": "example.com"},
+    "time_spent": None,
+    "timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
+}
 
 
 SENTRY_JS_PAYLOAD_7 = {
-    "project": "sentry", "logger": "javascript",
-    "platform": "javascript", "request": {"headers": {
-        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.111 Safari/537.36"},
-        "url": "http://localhost:6543/test/js_error#/"},
-    "exception": {"values": [{"type": "ReferenceError",
-                              "value": "fateqtwetew is not defined",
-                              "stacktrace": {"frames": [{
-                                  "filename": "https://cdn.ravenjs.com/2.0.0/angular/raven.min.js",
-                                  "lineno": 1,
-                                  "colno": 4466,
-                                  "function": "c",
-                                  "in_app": False},
-                                  {
-                                      "filename": "http://localhost:6543/test/js_error",
-                                      "lineno": 47,
-                                      "colno": 19,
-                                      "function": "?",
-                                      "in_app": True}]}}]},
+    "project": "sentry",
+    "logger": "javascript",
+    "platform": "javascript",
+    "request": {
+        "headers": {
+            "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.111 Safari/537.36"
+        },
+        "url": "http://localhost:6543/test/js_error#/",
+    },
+    "exception": {
+        "values": [
+            {
+                "type": "ReferenceError",
+                "value": "fateqtwetew is not defined",
+                "stacktrace": {
+                    "frames": [
+                        {
+                            "filename": "https://cdn.ravenjs.com/2.0.0/angular/raven.min.js",
+                            "lineno": 1,
+                            "colno": 4466,
+                            "function": "c",
+                            "in_app": False,
+                        },
+                        {
+                            "filename": "http://localhost:6543/test/js_error",
+                            "lineno": 47,
+                            "colno": 19,
+                            "function": "?",
+                            "in_app": True,
+                        },
+                    ]
+                },
+            }
+        ]
+    },
     "culprit": "http://localhost:6543/test/js_error",
     "message": "ReferenceError: fateqtwetew is not defined",
     "extra": {"session:duration": 5009},
-    "event_id": "2bf514aaf0e94f35a8f435a0d29a888b"}
+    "event_id": "2bf514aaf0e94f35a8f435a0d29a888b",
+}
 
 SENTRY_JS_PAYLOAD_7_2 = {
-    "project": "sentry", "logger": "javascript",
-    "platform": "javascript", "request": {"headers": {
-        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.111 Safari/537.36"},
-        "url": "http://localhost:6543/#/report/927/9558"},
-    "exception": {"values": [{"type": "Error",
-                              "value": "[$injector:modulerr] http://errors.angularjs.org/1.5.0-rc.0/$injector/modulerr?p0=appenlight&p1=Erro…",
-                              "stacktrace": {"frames": [{
-                                  "filename": "http://localhost:6543/static/js/appenlight.js?rev=752",
-                                  "lineno": 1647,
-                                  "colno": 112,
-                                  "function": "?",
-                                  "in_app": True},
-                                  {
-                                      "filename": "http://localhost:6543/static/js/appenlight.js?rev=752",
-                                      "lineno": 1363,
-                                      "colno": 41,
-                                      "function": "be",
-                                      "in_app": True},
-                                  {
-                                      "filename": "http://localhost:6543/static/js/appenlight.js?rev=752",
-                                      "lineno": 1364,
-                                      "colno": 225,
-                                      "function": "zc",
-                                      "in_app": True},
-                                  {
-                                      "filename": "http://localhost:6543/static/js/appenlight.js?rev=752",
-                                      "lineno": 1363,
-                                      "colno": 421,
-                                      "function": "c",
-                                      "in_app": True},
-                                  {
-                                      "filename": "http://localhost:6543/static/js/appenlight.js?rev=752",
-                                      "lineno": 1386,
-                                      "colno": 360,
-                                      "function": "fb",
-                                      "in_app": True},
-                                  {
-                                      "filename": "http://localhost:6543/static/js/appenlight.js?rev=752",
-                                      "lineno": 1383,
-                                      "colno": 49,
-                                      "function": "g",
-                                      "in_app": True},
-                                  {
-                                      "filename": "http://localhost:6543/static/js/appenlight.js?rev=752",
-                                      "lineno": 1351,
-                                      "colno": 344,
-                                      "function": "n",
-                                      "in_app": True},
-                                  {
-                                      "filename": "http://localhost:6543/static/js/appenlight.js?rev=752",
-                                      "lineno": 1383,
-                                      "colno": 475,
-                                      "function": "?",
-                                      "in_app": True},
-                                  {
-                                      "filename": "http://localhost:6543/static/js/appenlight.js?rev=752",
-                                      "lineno": 1350,
-                                      "colno": 421,
-                                      "function": "?",
-                                      "in_app": True}]}}]},
+    "project": "sentry",
+    "logger": "javascript",
+    "platform": "javascript",
+    "request": {
+        "headers": {
+            "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.111 Safari/537.36"
+        },
+        "url": "http://localhost:6543/#/report/927/9558",
+    },
+    "exception": {
+        "values": [
+            {
+                "type": "Error",
+                "value": "[$injector:modulerr] http://errors.angularjs.org/1.5.0-rc.0/$injector/modulerr?p0=appenlight&p1=Erro…",
+                "stacktrace": {
+                    "frames": [
+                        {
+                            "filename": "http://localhost:6543/static/js/appenlight.js?rev=752",
+                            "lineno": 1647,
+                            "colno": 112,
+                            "function": "?",
+                            "in_app": True,
+                        },
+                        {
+                            "filename": "http://localhost:6543/static/js/appenlight.js?rev=752",
+                            "lineno": 1363,
+                            "colno": 41,
+                            "function": "be",
+                            "in_app": True,
+                        },
+                        {
+                            "filename": "http://localhost:6543/static/js/appenlight.js?rev=752",
+                            "lineno": 1364,
+                            "colno": 225,
+                            "function": "zc",
+                            "in_app": True,
+                        },
+                        {
+                            "filename": "http://localhost:6543/static/js/appenlight.js?rev=752",
+                            "lineno": 1363,
+                            "colno": 421,
+                            "function": "c",
+                            "in_app": True,
+                        },
+                        {
+                            "filename": "http://localhost:6543/static/js/appenlight.js?rev=752",
+                            "lineno": 1386,
+                            "colno": 360,
+                            "function": "fb",
+                            "in_app": True,
+                        },
+                        {
+                            "filename": "http://localhost:6543/static/js/appenlight.js?rev=752",
+                            "lineno": 1383,
+                            "colno": 49,
+                            "function": "g",
+                            "in_app": True,
+                        },
+                        {
+                            "filename": "http://localhost:6543/static/js/appenlight.js?rev=752",
+                            "lineno": 1351,
+                            "colno": 344,
+                            "function": "n",
+                            "in_app": True,
+                        },
+                        {
+                            "filename": "http://localhost:6543/static/js/appenlight.js?rev=752",
+                            "lineno": 1383,
+                            "colno": 475,
+                            "function": "?",
+                            "in_app": True,
+                        },
+                        {
+                            "filename": "http://localhost:6543/static/js/appenlight.js?rev=752",
+                            "lineno": 1350,
+                            "colno": 421,
+                            "function": "?",
+                            "in_app": True,
+                        },
+                    ]
+                },
+            }
+        ]
+    },
     "culprit": "http://localhost:6543/static/js/appenlight.js?rev=752",
     "message": "Error: [$injector:modulerr] http://errors.angularjs.org/1.5.0-rc.0/$injector/modulerr?p0=appenlight&…",
     "extra": {"session:duration": 330},
-    "event_id": "c50b5b6a13994f54b1d8da0c2e0e767a"}
+    "event_id": "c50b5b6a13994f54b1d8da0c2e0e767a",
+}
 
 SENTRY_LOG_PAYLOAD_7 = {
-    "project": "sentry", "sentry.interfaces.Message": {
-        "message": "TEST from django logging", "params": []},
+    "project": "sentry",
+    "sentry.interfaces.Message": {"message": "TEST from django logging", "params": []},
     "server_name": "ergo-virtual-machine",
     "culprit": "testlogger in index",
-    "extra": {"thread": 139723601139456, "process": 24645,
-              "sys.argv": ["'manage.py'", "'runserver'"],
-              "price": 6, "threadName": "'Thread-1'",
-              "filename": "'views.py'",
-              "processName": "'MainProcess'",
-              "tag": "'extra'", "dupa": True, "lineno": 22,
-              "asctime": "'2016-01-18 05:24:29,001'",
-              "pathname": "'/home/ergo/IdeaProjects/django_raven/djangoapp/views.py'"},
+    "extra": {
+        "thread": 139723601139456,
+        "process": 24645,
+        "sys.argv": ["'manage.py'", "'runserver'"],
+        "price": 6,
+        "threadName": "'Thread-1'",
+        "filename": "'views.py'",
+        "processName": "'MainProcess'",
+        "tag": "'extra'",
+        "dupa": True,
+        "lineno": 22,
+        "asctime": "'2016-01-18 05:24:29,001'",
+        "pathname": "'/home/ergo/IdeaProjects/django_raven/djangoapp/views.py'",
+    },
     "event_id": "9a6172f2e6d2444582f83a6c333d9cfb",
-    "timestamp": now.strftime('%Y-%m-%dT%H:%M:%SZ'),
+    "timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
     "tags": {"site": "example.com"},
-    "modules": {"python": "2.7.6", "raven": "5.9.2",
-                "django": "1.7.1"}, "time_spent": None,
-    "platform": "python", "release": "test",
-    "logger": "testlogger", "level": 50,
-    "message": "TEST from django logging"}
+    "modules": {"python": "2.7.6", "raven": "5.9.2", "django": "1.7.1"},
+    "time_spent": None,
+    "platform": "python",
+    "release": "test",
+    "logger": "testlogger",
+    "level": 50,
+    "message": "TEST from django logging",
+}
 
 METRICS_PAYLOAD = {
     "namespace": "some.monitor",
-    "timestamp": now.strftime('%Y-%m-%dT%H:%M:%S.0'),
+    "timestamp": now.strftime("%Y-%m-%dT%H:%M:%S.0"),
     "server_name": "server.name",
-    "tags": [["usage_foo", 15.5],
-             ["usage_bar", 63],
-             ]
+    "tags": [["usage_foo", 15.5], ["usage_bar", 63]],
 }
 
 
 SENTRY_PYTHON_ENCODED = b'x\x9c\xedXmo\xdbF\x12\xfe+\x0b\xdd\x07I9\x89/z\x97[\x05p\x1d5\x0ej\'\x81,\xa7\xed\xd59bE\x8d$V|\xcb\xeeR\x96k\xf8\xbf\xdf\xcc.II\xb6\xdc\xf4\xd2\x16\xb8\x03\xa2 \x06\xb9\xdc\x9d\x9dy\xe6\x99y\x96\xbc\xaf\xa4"\xf9\x15|U9a\x15\t\xb1\x12w\x95\x06\xab\xc0\xd6\x87T\x05I\x8c\xc3\xf7\x95\r\x0f3\x90x\xf9\xcb}E*\xee\xaf\x95\xe0>\xe8G\x0b\xc1\xa3\xe2\xd1"\x8b\xfd|Me\t\xca\x13 \xd3$\x96@\x06\xf9Lz)W+zf\xaf\x92\x08l\x10\xcb\xc4\xde@\xbc\x91vz\xa7VI\xdc\xb2\xc3\xc4\xe7\xa1\x1d\x06\xb3b\xc4\xea\xdb2P\xd0LqO\xbe\x04i\xcf\x7f\xe51.\xf3\x13\x01\xf6\x8a\xc7\xf3\x10\x84\xb4g\\\x82\x95j\xbfS\x01\x9e\x9f\xc4\n\xb6\x14\xd0/\x15\xb6\xf7\x0b\x16\xac\xf0\x88\x05\x92\xbdMb8\xa15\xec\xd1\xefV\xf04\x85\xb9\x87\xbe\x843\xdc\x98\x8d\x98\x84paE|\r\xde&\x80[\x8f\xab$\n\xfcZ1\xa1~\xcc\n\x02y\xd4:\xfdJ7FO6\xab\t\xf8\x84X\xab\x06{Q\x0cy\\,%\xde\xef\x06\xd6\xb74tt[\x9386.\xf2\xc7\xb8d\x18\xe6G\xc2&\x91\xea\x00\x9c\xc7\xeb\xff\xc1\xce\x92(\ry\x10\x13Vj\x05\x8c\xa2EoU&b\x98k\xc4X\x8d3?\x89"\xb4\x0cB$\xa2n=\xb6\xf2Ga\xc6y\x81\x0cb\xe4S\xecC\x89e\x83\xa9\xbb\x14\xa4\xf5}\xce\xa5)\xde\xd5O\x8cw\xdf\x7f\xf7\xe19DuZb\xa4"BZ\x98\xb2<=\xe2y:\xfa\r\x17R3\x96x[)\xf1\xa9eU\x85p\xb3\xae\xe3\xb0\x9b\x9b\xccq;\xad\x9b\x9b\xed\xa2\x8d\xd7-\xc7\xf5\xf5\x90\xd3\x7fr\xe7\xb8\xfd\xfc\xae[m\xe8D\x1cd\x8b\xe0\xa5M\x11\x88$\xdc\x80\xf0"\xae|\xcd\xfdI>rI\x035\xaa\x98\x91\xe14\xd2\xc0\xa2(\xa4\xa5qm0\xb23\xaa\xd5\x1b\xccd{t\xff\xd0`\x99\x08uL\xa3bN\x9a\xea{9\xa2\xed\xf4\x15\x96\x8a\xbe\xd5N\x15\x89\xf0\x02\x89\xd5\x18\xcfA\xc0\x1c\xbdX\xf0P\x02>\x8e\x829V\x10\x9a\x07/\x02,8zV\xf9v\x96d\xf1\x9c\x99\x01v\tRb\xe5]\x963-\xec\x17\xb8\x03\xd9\xd3De\xc9\x82}kB\xb0\x88\\"\x98Y\x91Y$\xad\xdd\x06\xd6\x13C,\x99Q\xdfa\\1g\xdb_\xb4{\xdd\xb6\x03}\x7f\xe8\xbc|I\xaeS\xc9iwJ\xdbh\xa4(y\xebV.\x03\xeb\xc7\xab\xd7o\xce\xcd\xc8ScC\xd7\x1f\xf6\xba\xceK\x03\x83vU\x9b\xa3E\x93\xdcu=\xdbm\x0f\x07}\xb75p;\xbdA\xabc\x16\x14\xc9\xd4+\x8a\xb6f\x08\xcf\x16"\x89\xd8\xa3\x9c\xed\x07\xe1\xf7\x06\x83\xce@\x9by\\\xdc\x7f\xd2\\\xc1&mf\x02K\xd8^O.\nB\xb1\xea\xce\x08\xd2DVYM\x97\x1e\xfd\xa9\xb3\x7f\xdb\x07q\xe5\x1d\x84\xea\xe1a\x8f&x\x1fga\x88#h\x01\x93\xa9\x13\xf0\xd8n\x85VD\xc9<\x0bu%\x1dM\x0fud\xdao\x11\x84@\xac\xdcM|\xbeu\x87A\x0cq\x823\xdd\xce\x10o\x83\xd8\xc3-\xf7\xc8\x9aw.\x8f\xe6\x91\xbd\xcf4V\xdd\xb2\x0b\xae\x96r\xe6\xcd\xee\xbc\x1d)kh7\xe7F\x9d\xc2\xfa\x9f\x97\xb0\xfd\xdfL\x00_\xd3\x82/m\xc0\x7f\xa1\xce\x1d\x95\x97\xc73\x9f\x91\xa6\xcfk\xe4\x7f\x9d\xca#\xa0\xfc\x8d\xda\xf6U]\xbe\xaa\xcbWu\xf9\xffQ\x97\xfe_\xa0.\x7f\xea\xd8\xfeDit\xae~Gbn\x13\xb1\xd6\xa5\x97\x8b\x87\'8\xaa\x8e]Bg\x9b\xd2~^?|\x0b\xb6\xe0g\nj7\x957o\xaf\xc6\x93){\xf3v\xfa\x8eI\x95\xf8k\xc9>\x9c^\\\x8f\xafX\xad\xdar\x9c^\xd3q\x9b\xd4w\xaa\xdf]\xff\x8c\x7f\'\xe7\xa7\xd3j\xc3u\x9cF\xbbk\xb9\x9d\xfaM\xa5\x94\x81\xbf\xc9j\x12\xc7\x16\xb5\xe1@\xd5\xf6\xb6\xf2\xc3D\xc2n \xc7\xdbz\xff\xeejj\xa1R\xd7\xaa\xaf\xae\xdf\x9fV\xeb\xcf\xbf\xe9\xd0\xff9,X\x9c\xa8\x05\xf5\xa0"e\xf5R\x82\x04\x0f0\xb9\xe7J\xa5\x1d\xa7S\xab\xe2\x1fSE\xd8^\xb1\x94J\xe1a\xd4\xd2\xabFe\x0ez\xbf\xafKG~\nQ\xef\xdb\xd6Y&dr\xa4u\xb4\x1c\xde\x99\x7fq\xeb@p\x0ew\xc1\x010\x15\x7f\xa4\xe3\xcd\x17\xfd>.<VSe;8^I\x8fYU\xd6\xcf\xa0\xfbG\xcb\xc7\xc0y\\\x0b\x8d\x14f\x8e\x83Zh\xc4\xcfh\xdf\xc1\xb5\x96A\xa3\x82X4\x1f)\nz9<P\xd8\xcaAhe\x8etT\xfa\xb3\x05\r\x7f\xf1\xbe\xf9\xae~\x16\xa6"PG\xc0cA\xdei\x8d\xa8\x08R\xe3\x02H*\xdd\xe6&\x10*\xe3a3\xe2\xfe\x8a\xb0\xd1\xdfV\x94\xe0\x9a0\xf2NZ\xd8\x126\x9a\xa3\xd5\x88\xc7(a\x88w\x95fUE\x16\x1b\x83\xd5\x8av\x02\xb0\xe4\x95\x17h\x15u\x17\xbd\xe1\x00\x86>\xcc\x16\xb3N\xdb\x1f\xce\x16=\x07\xda\xb3\xd6\x00|\x7f\x01\xd0=\xe4\x1623I\xd6\x01\x18\x96R\x9f\xcf\x84\xfe|Sq55\xb0\xf1\xd2\xcd\n\x89\x7fb\xdbn\xabo9\xf8\xcf=\x198\x8ec\x97\xd1\xad\x80\xa3\xc2\x1b\x1bg\x94\xeeX5/ ^\x9anE3N}B\xbfy\x81\x08e\x18\x89\xc6 n^_5 \xfe\xe6\xd3\xc8\xb1\x06\x8d4\xd4\x17\xbd\xbd\xd9\xe3\xd8O\xe6A\xbc\xd4\'\xee\xdf\x82\xb4\xc1\xb0HC\xae\x90Ur\x8e\xa7\x1a\x9c\xb9\xe38MZ\x03\xa4M\x1e\x06\x1b\xd8Y1I\xde*{\xa5\xa2\xb0\x81\xd9\t\x03\x9f\xd3\x02{K#\xff\xdc>\x1e\x8d\x8c#\xc3F\x10\xa1\xa7\xf6-\xcc\xd2\xc6\x0b\xfb\x85q\x93\xec^\xa7K\x81\xf16\xdf`\x12\xfcL@3/Mi`\xc3\x19\xafbU^\x9f\'\xa6\x88\x0f\xb13\xbe\x13\xf2\xf4\xac\xc0}\xa4W\x9c!\x1f\xa0I8\x8aD\xa3\x1f\xf1m\x13]\x19\xe9U\xd7\x98\xf9\xe6\xe9\x12\xcc\x16\x97\xc9oA\x18r\xbbk9\xac\xf6\x93\xeb~\xc3.\x828\xdb\xb2\xed\xa0\xe7\xf5:uv\x8a\x91\xc1\x8f0\xfb!Pv\xb7\xdd\xb7\xda=V\xfb\xe1|zy\xd1`a\xb0\x06\xf6\x1a\xfcuRgg+A\x8a\xd2%\x07[\xbd\x9ek\r;\xec\x8a/\xb8\x08\xf2U9\xd6:\xb3\xd3\xbc\xd04\xaa\xfa\xdc\xac\xa9\x82\xef:\x9a\x00\xd8\xec?\x8c\'\xde\xdb\xd3\xcb1\xcd\xd2o=+\x02\x01\xe7\xe4\xcf\xde\xbf\x9bL\xe9Y\x81\xc4d|\xf9n:\xf6N_\xbd\x9a\x1c@\xa5\xed"\xb6\xe2\xce\x93x\xbe3L\xd0\xcd\x9a+\xbe;3\xec\x8e\x90\xaf\xc7S\xbd&\xc4\x8a \xe8:N\xd9\x03\x0c;\xcd\x9b\x17M\xc5\xb7/C6-\x984Bjci\x7fL%kW\xac!\xce\xd2\xed%\x88\xc0\x93\xa9\xc1=\xdf\x18\x83G\xc1\x10\x11\xcd\xcc-\xe73\xa5\xe2Q\xaa\xb7q\\\x14\xb8n\xd3\xe9L\xdd\xdeI\xc79i\r\xffE\x93\xe8\x15m\xee\x8b,\x9a\xc9\'\xdfQ\x91\x89\xb0L\xc4]\xd1\x9f\xc2d\xb9\x04]hE\\\xbbc\xc1\xfef\xa8\x06\xad6b\xda\x1aZ\x9dn\xbf7\xdc\x01u\xbf\xd7Y\xb0]\xe9\n\xef\xd1j\xbe4\xbd\x91\x1e6\n\xb3\'\xf8\xe6\x96\xc1\x03E=\xcf\x04\xcf\xab\xab\x04[\x1f\xa7i\xd9t|5\xdd?F2r\x94\xb2\xb4\xd7\x8d\xb1by\x16*s\xb0\xd9\x0f\xcc\xc0\xbe\x1f\xd3\x1cf\xd9\xf2wc\x1at\x9d\xfe^P\x9fu\xf0\n\xdf<\xd0\x1f\x96\x0f\xd1\x1bC\xa8\xdb\x12\xeb\xf6\xfbL%\xecH_\x1b86O\x03\xdb|\xef\xb6\xf1\xbc\x82\xa7\xc6\xa3\x01}4\x07\xd8\x10\xb8,\x95\xa4r\xb8\x7f)E\'\xec\xcbu\xc6\xa4\xc9\xb0\x84>\x17\x98\x84!:!\xd0YH\x93S\xce\xd7\x86E\xd8\x85\xf3^\xb8cs!:\x1a\xf1f\xce\xd3\x87\x87\xff\x00`\xb1k\xbd'
 
-SENTRY_RUBY_ENCODED = b'eJzVVttu4zYQ/RVCfUgLRNfYsi002z5sf6DoU7cLgyJHEmOKFEjKW6+Rf++QkmM7ziJF0T7UMBzNhZzLOTPKMYI9KLcVPKoioDndLAvIH5YlrykHVpd8sSlhXfNstWTRfdSDtbQFdP4djP4o9sIKrX4xRpuKcBQ5cFIfSIa+TqC3o/2A3kWWl3G2jLPFb3lZLcuqyGePrR0wgahSo5T3kcR0ZFQtsvtoMPoJ2ItlkNQ12vR4mRnrA56Wum3BoIzPbJSDEegcOYyZmJoIRVJCHZFCAdmgiwWzB7NVtPfpg2l1vBfGjVTGPWUduvn6NB8l2Kg6RpQ5LK2nQoYgi6RISvSY1ANluxvlXsCXV8o9POn6RodRfJWvtAaYNvxGbcdh0MZd6k04XSZZ8oBiLVqESvTUK/PZpx6F5CHxB9QUQaP4VEqe5EWSn1XxqKSmPFiy4Mu0YqMxCEwcmn22AMrCekSTVeJRhj+BjY7WEuJO650Nvg/Bt6GGcupPZ8kmaFro4y+GDgMYOye78mqpayoDB7GkkL/I1yqIUxShY8zJaglBuQiFP1mtwi3rUI3UuqFdSG1qjMcuiGWy8CKyLXaHwcOLXcmuVDGnjk7dQqomWREI2gsltr77SIJivjmb9Z5oqFpi9HD7KJ0YqHHxoIPh5Kv0TrfCiJBpifX4Rgz2wE6prlE2E5/yOVUvxnOADHUPQSekvWBBkGMOA/KGOuBbSzEp8XWGODsfirnuw21CtbNt9WLrXC/jbx11Aq5D7nz/cw+Ar8JwzWazr9RTBRG28SXVFlNB+34inufGNI3KmUNsKK6fOai/wuLUsx24CaLyWhefWoDghVtdp03oUL4JDHCdAeob0cBMpaXXfhWq0TPdiujZc9YZBPuIj462dnoarc/4GMwMBj/Qfg3sqRx9Ez4dI0+UtzYfxgheaHu1Aqd1Mq0oXIVsh3EZ+Gsbg3touhYB3CK5HWaFckQICo1oE24VeSR3nXNDlablpqCroixZTetFttqs101RNIyXWVY2Bd1U7zn8nBcr3+Ukr9bZOksnBO7+UH6IFQ9/8eczkhMJfJ3Rd4TRwY0GCFUH8tIfS750gnWk8xOtCB8NMoxMGwHNRDfEdcKSWiKAIQAhOa7l7CIoxqO13Q7Udeft7ZfHqNiEQfQjDrL64Cccl7RCJFe4ENQWg0aVMyOEfeWT3XoHPPCrZ1VySpnrEG5+n2yN1i8vlQbnen4hnCI/37+BCCEcmlMPvtdz8Y/k+PzDXJb/iGaqdNvi2lY/XVgIqaEV6lvtnT4GLBuBBEpdnUUTFRYQhY9a3bkXLEKZBLy/vTpwuumEE3n8QOCm12mne0j9izBNcD5TP7qpn+EYxyRZTPLlnMZhSlMp6jTIaU0t3KA1Z3cB16Y849VQaW8BO1d6ECD538HrOoM3UAuXvMmEf43Pb4B5MUn4fj2D/h7Hw5X+n5Ybsm/eIfvlSP1zjv+/upX+x/35/Oy/fwFCRniE'
+SENTRY_RUBY_ENCODED = b"eJzVVttu4zYQ/RVCfUgLRNfYsi002z5sf6DoU7cLgyJHEmOKFEjKW6+Rf++QkmM7ziJF0T7UMBzNhZzLOTPKMYI9KLcVPKoioDndLAvIH5YlrykHVpd8sSlhXfNstWTRfdSDtbQFdP4djP4o9sIKrX4xRpuKcBQ5cFIfSIa+TqC3o/2A3kWWl3G2jLPFb3lZLcuqyGePrR0wgahSo5T3kcR0ZFQtsvtoMPoJ2ItlkNQ12vR4mRnrA56Wum3BoIzPbJSDEegcOYyZmJoIRVJCHZFCAdmgiwWzB7NVtPfpg2l1vBfGjVTGPWUduvn6NB8l2Kg6RpQ5LK2nQoYgi6RISvSY1ANluxvlXsCXV8o9POn6RodRfJWvtAaYNvxGbcdh0MZd6k04XSZZ8oBiLVqESvTUK/PZpx6F5CHxB9QUQaP4VEqe5EWSn1XxqKSmPFiy4Mu0YqMxCEwcmn22AMrCekSTVeJRhj+BjY7WEuJO650Nvg/Bt6GGcupPZ8kmaFro4y+GDgMYOye78mqpayoDB7GkkL/I1yqIUxShY8zJaglBuQiFP1mtwi3rUI3UuqFdSG1qjMcuiGWy8CKyLXaHwcOLXcmuVDGnjk7dQqomWREI2gsltr77SIJivjmb9Z5oqFpi9HD7KJ0YqHHxoIPh5Kv0TrfCiJBpifX4Rgz2wE6prlE2E5/yOVUvxnOADHUPQSekvWBBkGMOA/KGOuBbSzEp8XWGODsfirnuw21CtbNt9WLrXC/jbx11Aq5D7nz/cw+Ar8JwzWazr9RTBRG28SXVFlNB+34inufGNI3KmUNsKK6fOai/wuLUsx24CaLyWhefWoDghVtdp03oUL4JDHCdAeob0cBMpaXXfhWq0TPdiujZc9YZBPuIj462dnoarc/4GMwMBj/Qfg3sqRx9Ez4dI0+UtzYfxgheaHu1Aqd1Mq0oXIVsh3EZ+Gsbg3touhYB3CK5HWaFckQICo1oE24VeSR3nXNDlablpqCroixZTetFttqs101RNIyXWVY2Bd1U7zn8nBcr3+Ukr9bZOksnBO7+UH6IFQ9/8eczkhMJfJ3Rd4TRwY0GCFUH8tIfS750gnWk8xOtCB8NMoxMGwHNRDfEdcKSWiKAIQAhOa7l7CIoxqO13Q7Udeft7ZfHqNiEQfQjDrL64Cccl7RCJFe4ENQWg0aVMyOEfeWT3XoHPPCrZ1VySpnrEG5+n2yN1i8vlQbnen4hnCI/37+BCCEcmlMPvtdz8Y/k+PzDXJb/iGaqdNvi2lY/XVgIqaEV6lvtnT4GLBuBBEpdnUUTFRYQhY9a3bkXLEKZBLy/vTpwuumEE3n8QOCm12mne0j9izBNcD5TP7qpn+EYxyRZTPLlnMZhSlMp6jTIaU0t3KA1Z3cB16Y849VQaW8BO1d6ECD538HrOoM3UAuXvMmEf43Pb4B5MUn4fj2D/h7Hw5X+n5Ybsm/eIfvlSP1zjv+/upX+x/35/Oy/fwFCRniE"

backend/src/appenlight/tests/test_integration.py

@@ -1,110 +1,118 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import pytest
 import json
 from webtest import TestApp
 
 
[e-mail]('base_app', 'with_migrations', 'clean_tables',
-                         'default_application')
[e-mail](
+    "base_app", "with_migrations", "clean_tables", "default_application"
+)
 class TestAPIReportsView(object):
     def test_no_json_payload(self, base_app):
         app = TestApp(base_app)
-        url_path = '/api/reports'
-        headers = {'x-appenlight-api-key': 'xxxx'}
-        res = app.post(url_path, {}, status=400,
-                       headers=headers)
+        url_path = "/api/reports"
+        headers = {"x-appenlight-api-key": "xxxx"}
+        res = app.post(url_path, {}, status=400, headers=headers)
 
     def test_wrong_json_payload(self, base_app):
         app = TestApp(base_app)
-        url_path = '/api/reports'
-        headers = {'x-appenlight-api-key': 'xxxx'}
+        url_path = "/api/reports"
+        headers = {"x-appenlight-api-key": "xxxx"}
         res = app.post(url_path, {}, status=400, headers=headers)
 
     def test_correct_json_payload(self, base_app):
         import appenlight.tests.payload_examples as payload_examples
+
         app = TestApp(base_app)
-        url_path = '/api/reports'
-        headers = {'x-appenlight-api-key': 'xxxx'}
-        res = app.post_json(url_path, [payload_examples.PYTHON_PAYLOAD_0_5],
-                            headers=headers)
+        url_path = "/api/reports"
+        headers = {"x-appenlight-api-key": "xxxx"}
+        res = app.post_json(
+            url_path, [payload_examples.PYTHON_PAYLOAD_0_5], headers=headers
+        )
 
     def test_json_payload_wrong_key(self, base_app):
         import appenlight.tests.payload_examples as payload_examples
+
         app = TestApp(base_app)
-        url_path = '/api/reports'
-        res = app.post(url_path,
-                       json.dumps([payload_examples.PYTHON_PAYLOAD_0_5]),
-                       status=403)
+        url_path = "/api/reports"
+        res = app.post(
+            url_path, json.dumps([payload_examples.PYTHON_PAYLOAD_0_5]), status=403
+        )
 
 
[e-mail]('base_app', 'with_migrations', 'clean_tables',
-                         'default_data', 'default_application')
[e-mail](
+    "base_app", "with_migrations", "clean_tables", "default_data", "default_application"
+)
 class TestRegistrationView(object):
     def test_register_empty(self, base_app):
-        url_path = '/register'
+        url_path = "/register"
         app = TestApp(base_app)
-        resp = app.get('/')
-        cookies = resp.headers.getall('Set-Cookie')
+        resp = app.get("/")
+        cookies = resp.headers.getall("Set-Cookie")
         cookie = None
-        for name, value in [c.split('=', 1) for c in cookies]:
-            if name == 'XSRF-TOKEN':
-                cookie = value.split(';')[0]
-        headers = {'X-XSRF-TOKEN': cookie}
-        res = app.post(url_path,
-                       params={'user_name': '',
-                               'user_password': '',
-                               'email': ''},
-                       headers=headers)
-        assert 'This field is required.' in res
+        for name, value in [c.split("=", 1) for c in cookies]:
+            if name == "XSRF-TOKEN":
+                cookie = value.split(";")[0]
+        headers = {"X-XSRF-TOKEN": cookie}
+        res = app.post(
+            url_path,
+            params={"user_name": "", "user_password": "", "email": ""},
+            headers=headers,
+        )
+        assert "This field is required." in res
 
     def test_register_proper(self, base_app):
-        url_path = '/register'
+        url_path = "/register"
         app = TestApp(base_app)
-        resp = app.get('/')
-        cookies = resp.headers.getall('Set-Cookie')
+        resp = app.get("/")
+        cookies = resp.headers.getall("Set-Cookie")
         cookie = None
-        for name, value in [c.split('=', 1) for c in cookies]:
-            if name == 'XSRF-TOKEN':
-                cookie = value.split(';')[0]
-        headers = {'X-XSRF-TOKEN': cookie}
-        res = app.post(url_path,
-                       params={'user_name': 'user_foo',
-                               'user_password': 'passbar',
-                               'email': '[e-mail]'},
-                       headers=headers,
-                       status=302)
+        for name, value in [c.split("=", 1) for c in cookies]:
+            if name == "XSRF-TOKEN":
+                cookie = value.split(";")[0]
+        headers = {"X-XSRF-TOKEN": cookie}
+        res = app.post(
+            url_path,
+            params={
+                "user_name": "user_foo",
+                "user_password": "passbar",
+                "email": "[e-mail]",
+            },
+            headers=headers,
+            status=302,
+        )
 
 
[e-mail]('base_app', 'with_migrations', 'clean_tables',
-                         'default_data', 'default_application')
[e-mail](
+    "base_app", "with_migrations", "clean_tables", "default_data", "default_application"
+)
 class TestRegistrationAuthTokenView(object):
-
     def test_create_application_bad(self, base_app):
-        url_path = '/applications'
+        url_path = "/applications"
         app = TestApp(base_app)
-        headers = {'x-appenlight-auth-token': ''}
-        app.post_json(url_path,
-                      params={'resource_name': 'user_foo'},
-                      headers=headers, status=403)
+        headers = {"x-appenlight-auth-token": ""}
+        app.post_json(
+            url_path, params={"resource_name": "user_foo"}, headers=headers, status=403
+        )
 
     def test_create_application_proper(self, base_app):
-        url_path = '/applications'
+        url_path = "/applications"
         app = TestApp(base_app)
-        headers = {'x-appenlight-auth-token': '1234'}
-        app.post_json(url_path,
-                      params={'resource_name': 'user_foo'},
-                      headers=headers, status=200)
+        headers = {"x-appenlight-auth-token": "1234"}
+        app.post_json(
+            url_path, params={"resource_name": "user_foo"}, headers=headers, status=200
+        )

backend/src/appenlight/tests/test_unit.py

@@ -1,1592 +1,1610 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import copy
 import logging
 import mock
 import pyramid
 import pytest
 import sqlalchemy as sa
 import webob
 
 from datetime import datetime
 from pyramid import testing
 
 
 from appenlight.models import DBSession
 from appenlight.lib.ext_json import json
 
 
 log = logging.getLogger(__name__)
 
 
 class DummyContext(object):
     pass
 
 
[e-mail]('base_app')
[e-mail]("base_app")
 class BasicTest(object):
     pass
 
 
[e-mail]('base_app')
[e-mail]("base_app")
 class TestMigration(object):
     def test_migration(self):
         assert 1 == 1
 
 
 class TestSentryProto_7(object):
     def test_log_payload(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.lib.enums import ParsedSentryEventType
         from appenlight.lib.utils.sentry import parse_sentry_event
+
         event_dict, event_type = parse_sentry_event(
-            payload_examples.SENTRY_LOG_PAYLOAD_7)
+            payload_examples.SENTRY_LOG_PAYLOAD_7
+        )
         assert ParsedSentryEventType.LOG == event_type
-        assert event_dict['log_level'] == 'CRITICAL'
-        assert event_dict['message'] == 'TEST from django logging'
-        assert event_dict['namespace'] == 'testlogger'
-        assert event_dict['request_id'] == '9a6172f2e6d2444582f83a6c333d9cfb'
-        assert event_dict['server'] == 'ergo-virtual-machine'
-        assert event_dict['date'] == datetime.utcnow().date().strftime(
-            '%Y-%m-%dT%H:%M:%SZ')
-        tags = [('site', 'example.com'),
-                ('sys.argv', ["'manage.py'", "'runserver'"]),
-                ('price', 6),
-                ('tag', "'extra'"),
-                ('dupa', True),
-                ('project', 'sentry'),
-                ('sentry_culprit', 'testlogger in index'),
-                ('sentry_language', 'python'),
-                ('sentry_release', 'test')]
-        assert sorted(event_dict['tags']) == sorted(tags)
+        assert event_dict["log_level"] == "CRITICAL"
+        assert event_dict["message"] == "TEST from django logging"
+        assert event_dict["namespace"] == "testlogger"
+        assert event_dict["request_id"] == "9a6172f2e6d2444582f83a6c333d9cfb"
+        assert event_dict["server"] == "ergo-virtual-machine"
+        assert event_dict["date"] == datetime.utcnow().date().strftime(
+            "%Y-%m-%dT%H:%M:%SZ"
+        )
+        tags = [
+            ("site", "example.com"),
+            ("sys.argv", ["'manage.py'", "'runserver'"]),
+            ("price", 6),
+            ("tag", "'extra'"),
+            ("dupa", True),
+            ("project", "sentry"),
+            ("sentry_culprit", "testlogger in index"),
+            ("sentry_language", "python"),
+            ("sentry_release", "test"),
+        ]
+        assert sorted(event_dict["tags"]) == sorted(tags)
 
     def test_report_payload(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.lib.enums import ParsedSentryEventType
         from appenlight.lib.utils.sentry import parse_sentry_event
-        utcnow = datetime.utcnow().date().strftime('%Y-%m-%dT%H:%M:%SZ')
+
+        utcnow = datetime.utcnow().date().strftime("%Y-%m-%dT%H:%M:%SZ")
         event_dict, event_type = parse_sentry_event(
-            payload_examples.SENTRY_PYTHON_PAYLOAD_7)
+            payload_examples.SENTRY_PYTHON_PAYLOAD_7
+        )
         assert ParsedSentryEventType.ERROR_REPORT == event_type
-        assert event_dict['client'] == 'sentry'
-        assert event_dict[
-                   'error'] == 'Exception: test 500 ' \
-                               '\u0142\xf3\u201c\u0107\u201c\u0107\u017c\u0105'
-        assert event_dict['language'] == 'python'
-        assert event_dict['ip'] == '127.0.0.1'
-        assert event_dict['request_id'] == '9fae652c8c1c4d6a8eee09260f613a98'
-        assert event_dict['server'] == 'ergo-virtual-machine'
-        assert event_dict['start_time'] == utcnow
-        assert event_dict['url'] == 'http://127.0.0.1:8000/error'
-        assert event_dict['user_agent'] == 'Mozilla/5.0 (X11; Linux x86_64) ' \
-                                           'AppleWebKit/537.36 (KHTML, ' \
-                                           'like Gecko) Chrome/47.0.2526.106 ' \
-                                           'Safari/537.36'
-        assert event_dict['view_name'] == 'djangoapp.views in error'
-        tags = [('site', 'example.com'), ('sentry_release', 'test')]
-        assert sorted(event_dict['tags']) == sorted(tags)
-        extra = [('sys.argv', ["'manage.py'", "'runserver'"]),
-                 ('project', 'sentry')]
-        assert sorted(event_dict['extra']) == sorted(extra)
-        request = event_dict['request']
-        assert request['url'] == 'http://127.0.0.1:8000/error'
-        assert request['cookies'] == {'appenlight': 'X'}
-        assert request['data'] is None
-        assert request['method'] == 'GET'
-        assert request['query_string'] == ''
-        assert request['env'] == {'REMOTE_ADDR': '127.0.0.1',
-                                  'SERVER_NAME': 'localhost',
-                                  'SERVER_PORT': '8000'}
-        assert request['headers'] == {
-            'Accept': 'text/html,application/xhtml+xml,'
-                       'application/xml;q=0.9,image/webp,*/*;q=0.8',
-            'Accept-Encoding': 'gzip, deflate, sdch',
-            'Accept-Language': 'en-US,en;q=0.8,pl;q=0.6',
-            'Connection': 'keep-alive',
-            'Content-Length': '',
-            'Content-Type': 'text/plain',
-            'Cookie': 'appenlight=X',
-            'Dnt': '1',
-            'Host': '127.0.0.1:8000',
-            'Upgrade-Insecure-Requests': '1',
-            'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) '
-                           'AppleWebKit/537.36 (KHTML, like Gecko) '
-                           'Chrome/47.0.2526.106 Safari/537.36'}
-        traceback = event_dict['traceback']
-        assert traceback[0]['cline'] == 'response = wrapped_callback(request, ' \
-                                        '*callback_args, **callback_kwargs)'
-        assert traceback[0]['file'] == 'django/core/handlers/base.py'
-        assert traceback[0]['fn'] == 'get_response'
-        assert traceback[0]['line'] == 111
-        assert traceback[0]['module'] == 'django.core.handlers.base'
-
-        assert traceback[1]['cline'] == "raise Exception(u'test 500 " \
-                                        "\u0142\xf3\u201c\u0107\u201c\u0107" \
-                                        "\u017c\u0105')"
-        assert traceback[1]['file'] == 'djangoapp/views.py'
-        assert traceback[1]['fn'] == 'error'
-        assert traceback[1]['line'] == 84
-        assert traceback[1]['module'] == 'djangoapp.views'
-        assert sorted(traceback[1]['vars']) == sorted([
-            ('c',
-             '<sqlite3.Cursor object at 0x7fe7c82af8f0>'),
-            ('request',
-             '<WSGIRequest at 0x140633490316304>'),
-            ('conn',
-             '<sqlite3.Connection object at 0x7fe7c8b23bf8>')])
+        assert event_dict["client"] == "sentry"
+        assert (
+            event_dict["error"] == "Exception: test 500 "
+            "\u0142\xf3\u201c\u0107\u201c\u0107\u017c\u0105"
+        )
+        assert event_dict["language"] == "python"
+        assert event_dict["ip"] == "127.0.0.1"
+        assert event_dict["request_id"] == "9fae652c8c1c4d6a8eee09260f613a98"
+        assert event_dict["server"] == "ergo-virtual-machine"
+        assert event_dict["start_time"] == utcnow
+        assert event_dict["url"] == "http://127.0.0.1:8000/error"
+        assert (
+            event_dict["user_agent"] == "Mozilla/5.0 (X11; Linux x86_64) "
+            "AppleWebKit/537.36 (KHTML, "
+            "like Gecko) Chrome/47.0.2526.106 "
+            "Safari/537.36"
+        )
+        assert event_dict["view_name"] == "djangoapp.views in error"
+        tags = [("site", "example.com"), ("sentry_release", "test")]
+        assert sorted(event_dict["tags"]) == sorted(tags)
+        extra = [("sys.argv", ["'manage.py'", "'runserver'"]), ("project", "sentry")]
+        assert sorted(event_dict["extra"]) == sorted(extra)
+        request = event_dict["request"]
+        assert request["url"] == "http://127.0.0.1:8000/error"
+        assert request["cookies"] == {"appenlight": "X"}
+        assert request["data"] is None
+        assert request["method"] == "GET"
+        assert request["query_string"] == ""
+        assert request["env"] == {
+            "REMOTE_ADDR": "127.0.0.1",
+            "SERVER_NAME": "localhost",
+            "SERVER_PORT": "8000",
+        }
+        assert request["headers"] == {
+            "Accept": "text/html,application/xhtml+xml,"
+            "application/xml;q=0.9,image/webp,*/*;q=0.8",
+            "Accept-Encoding": "gzip, deflate, sdch",
+            "Accept-Language": "en-US,en;q=0.8,pl;q=0.6",
+            "Connection": "keep-alive",
+            "Content-Length": "",
+            "Content-Type": "text/plain",
+            "Cookie": "appenlight=X",
+            "Dnt": "1",
+            "Host": "127.0.0.1:8000",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) "
+            "AppleWebKit/537.36 (KHTML, like Gecko) "
+            "Chrome/47.0.2526.106 Safari/537.36",
+        }
+        traceback = event_dict["traceback"]
+        assert (
+            traceback[0]["cline"] == "response = wrapped_callback(request, "
+            "*callback_args, **callback_kwargs)"
+        )
+        assert traceback[0]["file"] == "django/core/handlers/base.py"
+        assert traceback[0]["fn"] == "get_response"
+        assert traceback[0]["line"] == 111
+        assert traceback[0]["module"] == "django.core.handlers.base"
+
+        assert (
+            traceback[1]["cline"] == "raise Exception(u'test 500 "
+            "\u0142\xf3\u201c\u0107\u201c\u0107"
+            "\u017c\u0105')"
+        )
+        assert traceback[1]["file"] == "djangoapp/views.py"
+        assert traceback[1]["fn"] == "error"
+        assert traceback[1]["line"] == 84
+        assert traceback[1]["module"] == "djangoapp.views"
+        assert sorted(traceback[1]["vars"]) == sorted(
+            [
+                ("c", "<sqlite3.Cursor object at 0x7fe7c82af8f0>"),
+                ("request", "<WSGIRequest at 0x140633490316304>"),
+                ("conn", "<sqlite3.Connection object at 0x7fe7c8b23bf8>"),
+            ]
+        )
 
 
 class TestAPIReports_0_5_Validation(object):
-    @pytest.mark.parametrize('dummy_json', ['', {}, [], None])
+    @pytest.mark.parametrize("dummy_json", ["", {}, [], None])
     def test_no_payload(self, dummy_json):
         import colander
         from appenlight.validators import ReportListSchema_0_5
+
         utcnow = datetime.utcnow()
         schema = ReportListSchema_0_5().bind(utcnow=utcnow)
         with pytest.raises(colander.Invalid):
             schema.deserialize(dummy_json)
 
     def test_minimal_payload(self):
         dummy_json = [{}]
         import colander
         from appenlight.validators import ReportListSchema_0_5
+
         utcnow = datetime.utcnow()
         schema = ReportListSchema_0_5().bind(utcnow=utcnow)
         with pytest.raises(colander.Invalid):
             schema.deserialize(dummy_json)
 
     def test_minimal_payload(self):
-        dummy_json = [{'report_details': [{}]}]
+        dummy_json = [{"report_details": [{}]}]
         from appenlight.validators import ReportListSchema_0_5
+
         utcnow = datetime.utcnow()
         schema = ReportListSchema_0_5().bind(utcnow=utcnow)
 
         deserialized = schema.deserialize(dummy_json)
 
         expected_deserialization = [
-            {'language': 'unknown',
-             'server': 'unknown',
-             'occurences': 1,
-             'priority': 5,
-             'view_name': '',
-             'client': 'unknown',
-             'http_status': 200,
-             'error': '',
-             'tags': None,
-             'username': '',
-             'traceback': None,
-             'extra': None,
-             'url': '',
-             'ip': None,
-             'start_time': utcnow,
-             'group_string': None,
-             'request': {},
-             'request_stats': None,
-             'end_time': None,
-             'request_id': '',
-             'message': '',
-             'slow_calls': [],
-             'user_agent': ''
-             }
+            {
+                "language": "unknown",
+                "server": "unknown",
+                "occurences": 1,
+                "priority": 5,
+                "view_name": "",
+                "client": "unknown",
+                "http_status": 200,
+                "error": "",
+                "tags": None,
+                "username": "",
+                "traceback": None,
+                "extra": None,
+                "url": "",
+                "ip": None,
+                "start_time": utcnow,
+                "group_string": None,
+                "request": {},
+                "request_stats": None,
+                "end_time": None,
+                "request_id": "",
+                "message": "",
+                "slow_calls": [],
+                "user_agent": "",
+            }
         ]
         assert deserialized == expected_deserialization
 
     def test_full_payload(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.validators import ReportListSchema_0_5
+
         PYTHON_PAYLOAD = copy.deepcopy(payload_examples.PYTHON_PAYLOAD_0_5)
         utcnow = datetime.utcnow()
         schema = ReportListSchema_0_5().bind(utcnow=utcnow)
-        PYTHON_PAYLOAD["tags"] = [("foo", 1), ("action", "test"), ("baz", 1.1),
-                                  ("date",
-                                   utcnow.strftime('%Y-%m-%dT%H:%M:%S.0'))]
+        PYTHON_PAYLOAD["tags"] = [
+            ("foo", 1),
+            ("action", "test"),
+            ("baz", 1.1),
+            ("date", utcnow.strftime("%Y-%m-%dT%H:%M:%S.0")),
+        ]
         dummy_json = [PYTHON_PAYLOAD]
         deserialized = schema.deserialize(dummy_json)[0]
-        assert deserialized['error'] == PYTHON_PAYLOAD['error']
-        assert deserialized['language'] == PYTHON_PAYLOAD['language']
-        assert deserialized['server'] == PYTHON_PAYLOAD['server']
-        assert deserialized['priority'] == PYTHON_PAYLOAD['priority']
-        assert deserialized['view_name'] == PYTHON_PAYLOAD['view_name']
-        assert deserialized['client'] == PYTHON_PAYLOAD['client']
-        assert deserialized['http_status'] == PYTHON_PAYLOAD['http_status']
-        assert deserialized['error'] == PYTHON_PAYLOAD['error']
-        assert deserialized['occurences'] == PYTHON_PAYLOAD['occurences']
-        assert deserialized['username'] == PYTHON_PAYLOAD['username']
-        assert deserialized['traceback'] == PYTHON_PAYLOAD['traceback']
-        assert deserialized['url'] == PYTHON_PAYLOAD['url']
-        assert deserialized['ip'] == PYTHON_PAYLOAD['ip']
-        assert deserialized['start_time'].strftime('%Y-%m-%dT%H:%M:%S.0') == \
-               PYTHON_PAYLOAD['start_time']
-        assert deserialized['ip'] == PYTHON_PAYLOAD['ip']
-        assert deserialized['group_string'] is None
-        assert deserialized['request_stats'] == PYTHON_PAYLOAD['request_stats']
-        assert deserialized['end_time'].strftime('%Y-%m-%dT%H:%M:%S.0') == \
-               PYTHON_PAYLOAD['end_time']
-        assert deserialized['request_id'] == PYTHON_PAYLOAD['request_id']
-        assert deserialized['message'] == PYTHON_PAYLOAD['message']
-        assert deserialized['user_agent'] == PYTHON_PAYLOAD['user_agent']
-        assert deserialized['slow_calls'][0]['start'].strftime(
-            '%Y-%m-%dT%H:%M:%S.0') == PYTHON_PAYLOAD['slow_calls'][0][
-                   'start']
-        assert deserialized['slow_calls'][0]['end'].strftime(
-            '%Y-%m-%dT%H:%M:%S.0') == PYTHON_PAYLOAD['slow_calls'][0][
-                   'end']
-        assert deserialized['slow_calls'][0]['statement'] == \
-               PYTHON_PAYLOAD['slow_calls'][0]['statement']
-        assert deserialized['slow_calls'][0]['parameters'] == \
-               PYTHON_PAYLOAD['slow_calls'][0]['parameters']
-        assert deserialized['slow_calls'][0]['type'] == \
-               PYTHON_PAYLOAD['slow_calls'][0]['type']
-        assert deserialized['slow_calls'][0]['subtype'] == \
-               PYTHON_PAYLOAD['slow_calls'][0]['subtype']
-        assert deserialized['slow_calls'][0]['location'] == ''
-        assert deserialized['tags'] == [
-            ('foo', 1), ('action', 'test'),
-            ('baz', 1.1), ('date', utcnow.strftime('%Y-%m-%dT%H:%M:%S.0'))]
-
-
[e-mail]('log_schema')
+        assert deserialized["error"] == PYTHON_PAYLOAD["error"]
+        assert deserialized["language"] == PYTHON_PAYLOAD["language"]
+        assert deserialized["server"] == PYTHON_PAYLOAD["server"]
+        assert deserialized["priority"] == PYTHON_PAYLOAD["priority"]
+        assert deserialized["view_name"] == PYTHON_PAYLOAD["view_name"]
+        assert deserialized["client"] == PYTHON_PAYLOAD["client"]
+        assert deserialized["http_status"] == PYTHON_PAYLOAD["http_status"]
+        assert deserialized["error"] == PYTHON_PAYLOAD["error"]
+        assert deserialized["occurences"] == PYTHON_PAYLOAD["occurences"]
+        assert deserialized["username"] == PYTHON_PAYLOAD["username"]
+        assert deserialized["traceback"] == PYTHON_PAYLOAD["traceback"]
+        assert deserialized["url"] == PYTHON_PAYLOAD["url"]
+        assert deserialized["ip"] == PYTHON_PAYLOAD["ip"]
+        assert (
+            deserialized["start_time"].strftime("%Y-%m-%dT%H:%M:%S.0")
+            == PYTHON_PAYLOAD["start_time"]
+        )
+        assert deserialized["ip"] == PYTHON_PAYLOAD["ip"]
+        assert deserialized["group_string"] is None
+        assert deserialized["request_stats"] == PYTHON_PAYLOAD["request_stats"]
+        assert (
+            deserialized["end_time"].strftime("%Y-%m-%dT%H:%M:%S.0")
+            == PYTHON_PAYLOAD["end_time"]
+        )
+        assert deserialized["request_id"] == PYTHON_PAYLOAD["request_id"]
+        assert deserialized["message"] == PYTHON_PAYLOAD["message"]
+        assert deserialized["user_agent"] == PYTHON_PAYLOAD["user_agent"]
+        assert (
+            deserialized["slow_calls"][0]["start"].strftime("%Y-%m-%dT%H:%M:%S.0")
+            == PYTHON_PAYLOAD["slow_calls"][0]["start"]
+        )
+        assert (
+            deserialized["slow_calls"][0]["end"].strftime("%Y-%m-%dT%H:%M:%S.0")
+            == PYTHON_PAYLOAD["slow_calls"][0]["end"]
+        )
+        assert (
+            deserialized["slow_calls"][0]["statement"]
+            == PYTHON_PAYLOAD["slow_calls"][0]["statement"]
+        )
+        assert (
+            deserialized["slow_calls"][0]["parameters"]
+            == PYTHON_PAYLOAD["slow_calls"][0]["parameters"]
+        )
+        assert (
+            deserialized["slow_calls"][0]["type"]
+            == PYTHON_PAYLOAD["slow_calls"][0]["type"]
+        )
+        assert (
+            deserialized["slow_calls"][0]["subtype"]
+            == PYTHON_PAYLOAD["slow_calls"][0]["subtype"]
+        )
+        assert deserialized["slow_calls"][0]["location"] == ""
+        assert deserialized["tags"] == [
+            ("foo", 1),
+            ("action", "test"),
+            ("baz", 1.1),
+            ("date", utcnow.strftime("%Y-%m-%dT%H:%M:%S.0")),
+        ]
+
+
[e-mail]("log_schema")
 class TestAPILogsValidation(object):
-    @pytest.mark.parametrize('dummy_json', ['', {}, [], None])
+    @pytest.mark.parametrize("dummy_json", ["", {}, [], None])
     def test_no_payload(self, dummy_json, log_schema):
         import colander
 
         with pytest.raises(colander.Invalid):
             log_schema.deserialize(dummy_json)
 
     def test_minimal_payload(self, log_schema):
         dummy_json = [{}]
         deserialized = log_schema.deserialize(dummy_json)[0]
-        expected = {'log_level': 'UNKNOWN',
-                    'namespace': '',
-                    'server': 'unknown',
-                    'request_id': '',
-                    'primary_key': None,
-                    'date': datetime.utcnow(),
-                    'message': '',
-                    'tags': None}
-        assert deserialized['log_level'] == expected['log_level']
-        assert deserialized['message'] == expected['message']
-        assert deserialized['namespace'] == expected['namespace']
-        assert deserialized['request_id'] == expected['request_id']
-        assert deserialized['server'] == expected['server']
-        assert deserialized['tags'] == expected['tags']
-        assert deserialized['primary_key'] == expected['primary_key']
+        expected = {
+            "log_level": "UNKNOWN",
+            "namespace": "",
+            "server": "unknown",
+            "request_id": "",
+            "primary_key": None,
+            "date": datetime.utcnow(),
+            "message": "",
+            "tags": None,
+        }
+        assert deserialized["log_level"] == expected["log_level"]
+        assert deserialized["message"] == expected["message"]
+        assert deserialized["namespace"] == expected["namespace"]
+        assert deserialized["request_id"] == expected["request_id"]
+        assert deserialized["server"] == expected["server"]
+        assert deserialized["tags"] == expected["tags"]
+        assert deserialized["primary_key"] == expected["primary_key"]
 
     def test_normal_payload(self, log_schema):
         import appenlight.tests.payload_examples as payload_examples
+
         deserialized = log_schema.deserialize(payload_examples.LOG_EXAMPLES)[0]
         expected = payload_examples.LOG_EXAMPLES[0]
-        assert deserialized['log_level'] == expected['log_level']
-        assert deserialized['message'] == expected['message']
-        assert deserialized['namespace'] == expected['namespace']
-        assert deserialized['request_id'] == expected['request_id']
-        assert deserialized['server'] == expected['server']
-        assert deserialized['date'].strftime('%Y-%m-%dT%H:%M:%S.%f') == \
-               expected['date']
-        assert deserialized['tags'][0][0] == "tag_name"
-        assert deserialized['tags'][0][1] == "tag_value"
-        assert deserialized['tags'][1][0] == "tag_name2"
-        assert deserialized['tags'][1][1] == 2
+        assert deserialized["log_level"] == expected["log_level"]
+        assert deserialized["message"] == expected["message"]
+        assert deserialized["namespace"] == expected["namespace"]
+        assert deserialized["request_id"] == expected["request_id"]
+        assert deserialized["server"] == expected["server"]
+        assert deserialized["date"].strftime("%Y-%m-%dT%H:%M:%S.%f") == expected["date"]
+        assert deserialized["tags"][0][0] == "tag_name"
+        assert deserialized["tags"][0][1] == "tag_value"
+        assert deserialized["tags"][1][0] == "tag_name2"
+        assert deserialized["tags"][1][1] == 2
 
     def test_normal_payload_date_without_microseconds(self, log_schema):
         import appenlight.tests.payload_examples as payload_examples
+
         LOG_EXAMPLE = copy.deepcopy(payload_examples.LOG_EXAMPLES)
-        LOG_EXAMPLE[0]['date'] = datetime.utcnow().strftime(
-            '%Y-%m-%dT%H:%M:%S')
+        LOG_EXAMPLE[0]["date"] = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S")
         deserialized = log_schema.deserialize(LOG_EXAMPLE)
-        assert deserialized[0]['date'].strftime('%Y-%m-%dT%H:%M:%S') == \
-               LOG_EXAMPLE[0]['date']
+        assert (
+            deserialized[0]["date"].strftime("%Y-%m-%dT%H:%M:%S")
+            == LOG_EXAMPLE[0]["date"]
+        )
 
     def test_normal_payload_date_without_seconds(self, log_schema):
         import appenlight.tests.payload_examples as payload_examples
+
         LOG_EXAMPLE = copy.deepcopy(payload_examples.LOG_EXAMPLES)
-        LOG_EXAMPLE[0]['date'] = datetime.utcnow().date().strftime(
-            '%Y-%m-%dT%H:%M')
+        LOG_EXAMPLE[0]["date"] = datetime.utcnow().date().strftime("%Y-%m-%dT%H:%M")
         deserialized = log_schema.deserialize(LOG_EXAMPLE)
-        assert deserialized[0]['date'].strftime('%Y-%m-%dT%H:%M') == \
-               LOG_EXAMPLE[0]['date']
+        assert (
+            deserialized[0]["date"].strftime("%Y-%m-%dT%H:%M") == LOG_EXAMPLE[0]["date"]
+        )
 
     def test_payload_empty_date(self, log_schema):
         import appenlight.tests.payload_examples as payload_examples
+
         LOG_EXAMPLE = copy.deepcopy(payload_examples.LOG_EXAMPLES)
-        LOG_EXAMPLE[0]['date'] = None
+        LOG_EXAMPLE[0]["date"] = None
         deserialized = log_schema.deserialize(LOG_EXAMPLE)
-        assert deserialized[0]['date'].strftime('%Y-%m-%dT%H:%M') is not None
+        assert deserialized[0]["date"].strftime("%Y-%m-%dT%H:%M") is not None
 
     def test_payload_no_date(self, log_schema):
         import appenlight.tests.payload_examples as payload_examples
+
         LOG_EXAMPLE = copy.deepcopy(payload_examples.LOG_EXAMPLES)
-        LOG_EXAMPLE[0].pop('date', None)
+        LOG_EXAMPLE[0].pop("date", None)
         deserialized = log_schema.deserialize(LOG_EXAMPLE)
-        assert deserialized[0]['date'].strftime('%Y-%m-%dT%H:%M') is not None
+        assert deserialized[0]["date"].strftime("%Y-%m-%dT%H:%M") is not None
 
 
[e-mail]('general_metrics_schema')
[e-mail]("general_metrics_schema")
 class TestAPIGeneralMetricsValidation(object):
-    @pytest.mark.parametrize('dummy_json', ['', {}, [], None])
+    @pytest.mark.parametrize("dummy_json", ["", {}, [], None])
     def test_no_payload(self, dummy_json, general_metrics_schema):
         import colander
 
         with pytest.raises(colander.Invalid):
             general_metrics_schema.deserialize(dummy_json)
 
     def test_minimal_payload(self, general_metrics_schema):
-        dummy_json = [{'tags': [['counter_a', 15.5], ['counter_b', 63]]}]
+        dummy_json = [{"tags": [["counter_a", 15.5], ["counter_b", 63]]}]
         deserialized = general_metrics_schema.deserialize(dummy_json)[0]
-        expected = {'namespace': '',
-                    'server_name': 'unknown',
-                    'tags': [('counter_a', 15.5), ('counter_b', 63)],
-                    'timestamp': datetime.utcnow()}
-        assert deserialized['namespace'] == expected['namespace']
-        assert deserialized['server_name'] == expected['server_name']
-        assert deserialized['tags'] == expected['tags']
+        expected = {
+            "namespace": "",
+            "server_name": "unknown",
+            "tags": [("counter_a", 15.5), ("counter_b", 63)],
+            "timestamp": datetime.utcnow(),
+        }
+        assert deserialized["namespace"] == expected["namespace"]
+        assert deserialized["server_name"] == expected["server_name"]
+        assert deserialized["tags"] == expected["tags"]
 
     def test_normal_payload(self, general_metrics_schema):
         import appenlight.tests.payload_examples as payload_examples
+
         dummy_json = [payload_examples.METRICS_PAYLOAD]
         deserialized = general_metrics_schema.deserialize(dummy_json)[0]
-        expected = {'namespace': 'some.monitor',
-                    'server_name': 'server.name',
-                    'tags': [('usage_foo', 15.5), ('usage_bar', 63)],
-                    'timestamp': datetime.utcnow()}
-        assert deserialized['namespace'] == expected['namespace']
-        assert deserialized['server_name'] == expected['server_name']
-        assert deserialized['tags'] == expected['tags']
+        expected = {
+            "namespace": "some.monitor",
+            "server_name": "server.name",
+            "tags": [("usage_foo", 15.5), ("usage_bar", 63)],
+            "timestamp": datetime.utcnow(),
+        }
+        assert deserialized["namespace"] == expected["namespace"]
+        assert deserialized["server_name"] == expected["server_name"]
+        assert deserialized["tags"] == expected["tags"]
 
 
[e-mail]('request_metrics_schema')
[e-mail]("request_metrics_schema")
 class TestAPIRequestMetricsValidation(object):
-    @pytest.mark.parametrize('dummy_json', ['', {}, [], None])
+    @pytest.mark.parametrize("dummy_json", ["", {}, [], None])
     def test_no_payload(self, dummy_json, request_metrics_schema):
         import colander
 
         with pytest.raises(colander.Invalid):
             print(request_metrics_schema.deserialize(dummy_json))
 
     def test_normal_payload(self, request_metrics_schema):
         import appenlight.tests.payload_examples as payload_examples
+
         dummy_json = payload_examples.REQUEST_METRICS_EXAMPLES
         deserialized = request_metrics_schema.deserialize(dummy_json)[0]
-        expected = {'metrics': [('dir/module:func',
-                                 {'custom': 0.0,
-                                  'custom_calls': 0.0,
-                                  'main': 0.01664,
-                                  'nosql': 0.00061,
-                                  'nosql_calls': 23.0,
-                                  'remote': 0.0,
-                                  'remote_calls': 0.0,
-                                  'requests': 1,
-                                  'sql': 0.00105,
-                                  'sql_calls': 2.0,
-                                  'tmpl': 0.0,
-                                  'tmpl_calls': 0.0}),
-                                ('SomeView.function',
-                                 {'custom': 0.0,
-                                  'custom_calls': 0.0,
-                                  'main': 0.647261,
-                                  'nosql': 0.306554,
-                                  'nosql_calls': 140.0,
-                                  'remote': 0.0,
-                                  'remote_calls': 0.0,
-                                  'requests': 28,
-                                  'sql': 0.0,
-                                  'sql_calls': 0.0,
-                                  'tmpl': 0.0,
-                                  'tmpl_calls': 0.0})],
-                    'server': 'some.server.hostname',
-                    'timestamp': datetime.utcnow()}
-        assert deserialized['server'] == expected['server']
-        metric = deserialized['metrics'][0]
-        expected_metric = expected['metrics'][0]
+        expected = {
+            "metrics": [
+                (
+                    "dir/module:func",
+                    {
+                        "custom": 0.0,
+                        "custom_calls": 0.0,
+                        "main": 0.01664,
+                        "nosql": 0.00061,
+                        "nosql_calls": 23.0,
+                        "remote": 0.0,
+                        "remote_calls": 0.0,
+                        "requests": 1,
+                        "sql": 0.00105,
+                        "sql_calls": 2.0,
+                        "tmpl": 0.0,
+                        "tmpl_calls": 0.0,
+                    },
+                ),
+                (
+                    "SomeView.function",
+                    {
+                        "custom": 0.0,
+                        "custom_calls": 0.0,
+                        "main": 0.647261,
+                        "nosql": 0.306554,
+                        "nosql_calls": 140.0,
+                        "remote": 0.0,
+                        "remote_calls": 0.0,
+                        "requests": 28,
+                        "sql": 0.0,
+                        "sql_calls": 0.0,
+                        "tmpl": 0.0,
+                        "tmpl_calls": 0.0,
+                    },
+                ),
+            ],
+            "server": "some.server.hostname",
+            "timestamp": datetime.utcnow(),
+        }
+        assert deserialized["server"] == expected["server"]
+        metric = deserialized["metrics"][0]
+        expected_metric = expected["metrics"][0]
         assert metric[0] == expected_metric[0]
         assert sorted(metric[1].items()) == sorted(expected_metric[1].items())
 
 
[e-mail]('default_application')
[e-mail]('base_app', 'with_migrations', 'clean_tables')
[e-mail]("default_application")
[e-mail]("base_app", "with_migrations", "clean_tables")
 class TestAPIReportsView(object):
     def test_no_json_payload(self, default_application):
         import colander
         from appenlight.models.services.application import ApplicationService
         from appenlight.views.api import reports_create
 
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
-        request = testing.DummyRequest(
-            headers={'Content-Type': 'application/json'})
-        request.unsafe_json_body = ''
+        request = testing.DummyRequest(headers={"Content-Type": "application/json"})
+        request.unsafe_json_body = ""
         request.context = context
         route = mock.Mock()
-        route.name = 'api_reports'
+        route.name = "api_reports"
         request.matched_route = route
         with pytest.raises(colander.Invalid):
             response = reports_create(request)
 
     def test_single_proper_json_0_5_payload(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.views.api import reports_create
         from appenlight.models.services.application import ApplicationService
         from appenlight.models.report_group import ReportGroup
+
         route = mock.Mock()
-        route.name = 'api_reports'
+        route.name = "api_reports"
         request = pyramid.threadlocal.get_current_request()
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request.context = context
         request.matched_route = route
         PYTHON_PAYLOAD = payload_examples.PYTHON_PAYLOAD_0_5
         request.unsafe_json_body = [copy.deepcopy(PYTHON_PAYLOAD)]
         reports_create(request)
         query = DBSession.query(ReportGroup)
         report = query.first()
         assert query.count() == 1
         assert report.total_reports == 1
 
     def test_grouping_0_5(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.views.api import reports_create
         from appenlight.models.services.application import ApplicationService
         from appenlight.models.report_group import ReportGroup
+
         route = mock.Mock()
-        route.name = 'api_reports'
+        route.name = "api_reports"
         request = pyramid.threadlocal.get_current_request()
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request.context = context
         request.matched_route = route
         PYTHON_PAYLOAD = payload_examples.PYTHON_PAYLOAD_0_5
-        request.unsafe_json_body = [copy.deepcopy(PYTHON_PAYLOAD),
-                                    copy.deepcopy(PYTHON_PAYLOAD)]
+        request.unsafe_json_body = [
+            copy.deepcopy(PYTHON_PAYLOAD),
+            copy.deepcopy(PYTHON_PAYLOAD),
+        ]
         reports_create(request)
         query = DBSession.query(ReportGroup)
         report = query.first()
         assert query.count() == 1
         assert report.total_reports == 2
 
     def test_grouping_different_reports_0_5(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.views.api import reports_create
         from appenlight.models.services.application import ApplicationService
         from appenlight.models.report_group import ReportGroup
+
         route = mock.Mock()
-        route.name = 'api_reports'
+        route.name = "api_reports"
         request = pyramid.threadlocal.get_current_request()
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request.context = context
         request.matched_route = route
         PYTHON_PAYLOAD = payload_examples.PYTHON_PAYLOAD_0_5
         PARSED_REPORT_404 = payload_examples.PARSED_REPORT_404
-        request.unsafe_json_body = [copy.deepcopy(PYTHON_PAYLOAD),
-                                    copy.deepcopy(PARSED_REPORT_404)]
+        request.unsafe_json_body = [
+            copy.deepcopy(PYTHON_PAYLOAD),
+            copy.deepcopy(PARSED_REPORT_404),
+        ]
         reports_create(request)
         query = DBSession.query(ReportGroup)
         report = query.first()
         assert query.count() == 2
         assert report.total_reports == 1
 
 
[e-mail]('default_application')
[e-mail]('base_app', 'with_migrations', 'clean_tables')
[e-mail]("default_application")
[e-mail]("base_app", "with_migrations", "clean_tables")
 class TestAirbrakeXMLView(object):
-
     def test_normal_payload_parsing(self):
         import datetime
         import defusedxml.ElementTree as ElementTree
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.lib.utils.airbrake import parse_airbrake_xml
         from appenlight.validators import ReportListSchema_0_5
 
         context = DummyContext()
-        request = testing.DummyRequest(
-            headers={'Content-Type': 'application/xml'})
+        request = testing.DummyRequest(headers={"Content-Type": "application/xml"})
         request.context = context
         request.context.possibly_public = False
         root = ElementTree.fromstring(payload_examples.AIRBRAKE_RUBY_EXAMPLE)
         request.context.airbrake_xml_etree = root
         error_dict = parse_airbrake_xml(request)
         schema = ReportListSchema_0_5().bind(utcnow=datetime.datetime.utcnow())
         deserialized_report = schema.deserialize([error_dict])[0]
-        assert deserialized_report['client'] == 'Airbrake Notifier'
-        assert deserialized_report['error'] == 'NameError: undefined local variable or method `sdfdfdf\' for #<#<Class:0x000000039a8b90>:0x00000002c53df0>'
-        assert deserialized_report['http_status'] == 500
-        assert deserialized_report['language'] == 'unknown'
-        assert deserialized_report['message'] == ''
-        assert deserialized_report['occurences'] == 1
-        assert deserialized_report['priority'] == 5
-        d_request = deserialized_report['request']
-        assert d_request['GET'] == {'test': '1234'}
-        assert d_request['action_dispatch.request.parameters'] == {
-            'action': 'index',
-            'controller': 'welcome',
-            'test': '1234'}
-        assert deserialized_report['request_id'] == 'c11b2267f3ad8b00a1768cae35559fa1'
-        assert deserialized_report['server'] == 'ergo-desktop'
-        assert deserialized_report['traceback'][0] == {
-            'cline': 'block in start_thread',
-            'file': '/home/ergo/.rbenv/versions/1.9.3-p327/lib/ruby/1.9.1/webrick/server.rb',
-            'fn': 'block in start_thread',
-            'line': '191',
-            'module': '',
-            'vars': {}}
-        assert deserialized_report['traceback'][-1] == {
-            'cline': '_app_views_welcome_index_html_erb___2570061166873166679_31748940',
-            'file': '[PROJECT_ROOT]/app/views/welcome/index.html.erb',
-            'fn': '_app_views_welcome_index_html_erb___2570061166873166679_31748940',
-            'line': '3',
-            'module': '',
-            'vars': {}}
-        assert deserialized_report['url'] == 'http://0.0.0.0:3000/welcome/index?test=1234'
-        assert deserialized_report['view_name'] == 'welcome:index'
+        assert deserialized_report["client"] == "Airbrake Notifier"
+        assert (
+            deserialized_report["error"]
+            == "NameError: undefined local variable or method `sdfdfdf' for #<#<Class:0x000000039a8b90>:0x00000002c53df0>"
+        )
+        assert deserialized_report["http_status"] == 500
+        assert deserialized_report["language"] == "unknown"
+        assert deserialized_report["message"] == ""
+        assert deserialized_report["occurences"] == 1
+        assert deserialized_report["priority"] == 5
+        d_request = deserialized_report["request"]
+        assert d_request["GET"] == {"test": "1234"}
+        assert d_request["action_dispatch.request.parameters"] == {
+            "action": "index",
+            "controller": "welcome",
+            "test": "1234",
+        }
+        assert deserialized_report["request_id"] == "c11b2267f3ad8b00a1768cae35559fa1"
+        assert deserialized_report["server"] == "ergo-desktop"
+        assert deserialized_report["traceback"][0] == {
+            "cline": "block in start_thread",
+            "file": "/home/ergo/.rbenv/versions/1.9.3-p327/lib/ruby/1.9.1/webrick/server.rb",
+            "fn": "block in start_thread",
+            "line": "191",
+            "module": "",
+            "vars": {},
+        }
+        assert deserialized_report["traceback"][-1] == {
+            "cline": "_app_views_welcome_index_html_erb___2570061166873166679_31748940",
+            "file": "[PROJECT_ROOT]/app/views/welcome/index.html.erb",
+            "fn": "_app_views_welcome_index_html_erb___2570061166873166679_31748940",
+            "line": "3",
+            "module": "",
+            "vars": {},
+        }
+        assert (
+            deserialized_report["url"] == "http://0.0.0.0:3000/welcome/index?test=1234"
+        )
+        assert deserialized_report["view_name"] == "welcome:index"
 
     def test_normal_payload_view(self):
         import defusedxml.ElementTree as ElementTree
         import appenlight.tests.payload_examples as payload_examples
 
         from appenlight.models.services.application import ApplicationService
         from appenlight.views.api import airbrake_xml_compat
 
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
-        request = testing.DummyRequest(
-            headers={'Content-Type': 'application/xml'})
+        request = testing.DummyRequest(headers={"Content-Type": "application/xml"})
         request.context = context
         request.context.possibly_public = False
         root = ElementTree.fromstring(payload_examples.AIRBRAKE_RUBY_EXAMPLE)
         request.context.airbrake_xml_etree = root
         route = mock.Mock()
-        route.name = 'api_airbrake'
+        route.name = "api_airbrake"
         request.matched_route = route
         result = airbrake_xml_compat(request)
-        assert '<notice><id>' in result
+        assert "<notice><id>" in result
 
 
[e-mail]('default_application')
[e-mail]('base_app', 'with_migrations', 'clean_tables')
[e-mail]("default_application")
[e-mail]("base_app", "with_migrations", "clean_tables")
 class TestAPILogView(object):
     def test_no_json_payload(self, base_app):
         import colander
         from appenlight.models.services.application import ApplicationService
         from appenlight.views.api import logs_create
 
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
-        request = testing.DummyRequest(
-            headers={'Content-Type': 'application/json'})
+        request = testing.DummyRequest(headers={"Content-Type": "application/json"})
         request.context = context
         request.registry = base_app.registry
-        request.unsafe_json_body = ''
+        request.unsafe_json_body = ""
         route = mock.Mock()
-        route.name = 'api_logs'
+        route.name = "api_logs"
         request.matched_route = route
         with pytest.raises(colander.Invalid):
             response = logs_create(request)
 
     def test_single_json_payload(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.models.log import Log
         from appenlight.views.api import logs_create
         from appenlight.models.services.application import ApplicationService
+
         route = mock.Mock()
-        route.name = 'api_logs'
+        route.name = "api_logs"
         request = pyramid.threadlocal.get_current_request()
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request.context = context
         request.matched_route = route
-        request.unsafe_json_body = [copy.deepcopy(
-            payload_examples.LOG_EXAMPLES[0])]
+        request.unsafe_json_body = [copy.deepcopy(payload_examples.LOG_EXAMPLES[0])]
         logs_create(request)
         query = DBSession.query(Log)
         log = query.first()
         assert query.count() == 1
         assert log.message == "OMG ValueError happened"
 
     def test_multiple_json_payload(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.models.log import Log
         from appenlight.views.api import logs_create
         from appenlight.models.services.application import ApplicationService
+
         route = mock.Mock()
-        route.name = 'api_logs'
+        route.name = "api_logs"
         request = pyramid.threadlocal.get_current_request()
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request.context = context
         request.matched_route = route
         LOG_PAYLOAD = payload_examples.LOG_EXAMPLES[0]
         LOG_PAYLOAD2 = payload_examples.LOG_EXAMPLES[1]
         request.unsafe_json_body = copy.deepcopy([LOG_PAYLOAD, LOG_PAYLOAD2])
         logs_create(request)
         query = DBSession.query(Log).order_by(sa.asc(Log.log_id))
         assert query.count() == 2
         assert query[0].message == "OMG ValueError happened"
         assert query[1].message == "OMG ValueError happened2"
 
     def test_public_key_rewriting(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.models.log import Log
         from appenlight.views.api import logs_create
         from appenlight.models.services.application import ApplicationService
+
         route = mock.Mock()
-        route.name = 'api_logs'
+        route.name = "api_logs"
         request = pyramid.threadlocal.get_current_request()
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request.context = context
         request.matched_route = route
 
         LOG_PAYLOAD = copy.deepcopy(payload_examples.LOG_EXAMPLES[0])
         LOG_PAYLOAD2 = copy.deepcopy(payload_examples.LOG_EXAMPLES[1])
-        LOG_PAYLOAD['primary_key'] = 'X2'
-        LOG_PAYLOAD2['primary_key'] = 'X2'
+        LOG_PAYLOAD["primary_key"] = "X2"
+        LOG_PAYLOAD2["primary_key"] = "X2"
         request.unsafe_json_body = [LOG_PAYLOAD, LOG_PAYLOAD2]
         logs_create(request)
 
         query = DBSession.query(Log).order_by(sa.asc(Log.log_id))
         assert query.count() == 1
         assert query[0].message == "OMG ValueError happened2"
 
[e-mail]('default_application')
[e-mail]('base_app', 'with_migrations', 'clean_tables')
+
[e-mail]("default_application")
[e-mail]("base_app", "with_migrations", "clean_tables")
 class TestAPIGeneralMetricsView(object):
     def test_no_json_payload(self, base_app):
         import colander
         from appenlight.models.services.application import ApplicationService
         from appenlight.views.api import general_metrics_create
+
         route = mock.Mock()
-        route.name = 'api_general_metrics'
+        route.name = "api_general_metrics"
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
-        request = testing.DummyRequest(
-            headers={'Content-Type': 'application/json'})
+        request = testing.DummyRequest(headers={"Content-Type": "application/json"})
         request.context = context
         request.registry = base_app.registry
-        request.unsafe_json_body = ''
+        request.unsafe_json_body = ""
         request.matched_route = route
         with pytest.raises(colander.Invalid):
             general_metrics_create(request)
 
     def test_single_json_payload(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.models.metric import Metric
         from appenlight.views.api import general_metrics_create
         from appenlight.models.services.application import ApplicationService
+
         route = mock.Mock()
-        route.name = 'api_general_metric'
+        route.name = "api_general_metric"
         request = pyramid.threadlocal.get_current_request()
         request.matched_route = route
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request.context = context
         request.unsafe_json_body = payload_examples.METRICS_PAYLOAD
         general_metrics_create(request)
         query = DBSession.query(Metric)
         metric = query.first()
         assert query.count() == 1
-        assert metric.namespace == 'some.monitor'
+        assert metric.namespace == "some.monitor"
 
     def test_multiple_json_payload(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.models.metric import Metric
         from appenlight.views.api import general_metrics_create
         from appenlight.models.services.application import ApplicationService
+
         route = mock.Mock()
-        route.name = 'api_general_metrics'
+        route.name = "api_general_metrics"
         request = pyramid.threadlocal.get_current_request()
         request.matched_route = route
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request.context = context
         request.unsafe_json_body = [
             copy.deepcopy(payload_examples.METRICS_PAYLOAD),
             copy.deepcopy(payload_examples.METRICS_PAYLOAD),
         ]
         general_metrics_create(request)
         query = DBSession.query(Metric)
         metric = query.first()
         assert query.count() == 2
-        assert metric.namespace == 'some.monitor'
+        assert metric.namespace == "some.monitor"
 
 
 class TestGroupingMessageReplacements(object):
     def replace_default_repr_python(self):
-        test_str = '''
+        test_str = """
         ConnectionError: ConnectionError((<urllib3.connection.HTTPConnection object at 0x7f87a0ba9fd0>, 'Connection to domain.gr timed out. (connect timeout=10)')) caused by: ConnectTimeoutError((<urllib3.connection.HTTPConnection object at 0x7f87a0ba9fd0>, 'Connection to domain.gr timed out. (connect timeout=10)'))
-        '''
-        regex = r'<(.*?) object at (.*?)>'
+        """
+        regex = r"<(.*?) object at (.*?)>"
 
 
 class TestRulesKeyGetter(object):
     def test_default_dict_getter_top_key(self):
         from appenlight.lib.rule import Rule
-        struct = {
-            "a": {
-                "b": 'b',
-                "c": {
-                    "d": 'd',
-                    "g": {
-                        "h": 'h'
-                    }
-                },
-                "e": 'e'
-            },
-            "f": 'f'
-        }
+
+        struct = {"a": {"b": "b", "c": {"d": "d", "g": {"h": "h"}}, "e": "e"}, "f": "f"}
         result = Rule.default_dict_struct_getter(struct, "a")
-        assert result == struct['a']
+        assert result == struct["a"]
 
     def test_default_dict_getter_sub_key(self):
         from appenlight.lib.rule import Rule
-        struct = {
-            "a": {
-                "b": 'b',
-                "c": {
-                    "d": 'd',
-                    "g": {
-                        "h": 'h'
-                    }
-                },
-                "e": 'e'
-            },
-            "f": 'f'
-        }
-        result = Rule.default_dict_struct_getter(struct, 'a:b')
-        assert result == struct['a']['b']
-        result = Rule.default_dict_struct_getter(struct, 'a:c:d')
-        assert result == struct['a']['c']['d']
+
+        struct = {"a": {"b": "b", "c": {"d": "d", "g": {"h": "h"}}, "e": "e"}, "f": "f"}
+        result = Rule.default_dict_struct_getter(struct, "a:b")
+        assert result == struct["a"]["b"]
+        result = Rule.default_dict_struct_getter(struct, "a:c:d")
+        assert result == struct["a"]["c"]["d"]
 
     def test_default_obj_getter_top_key(self):
         from appenlight.lib.rule import Rule
+
         class TestStruct(object):
             def __init__(self, a, b):
                 self.a = a
                 self.b = b
 
-        struct = TestStruct(a='a',
-                            b=TestStruct(a='x', b='y'))
+        struct = TestStruct(a="a", b=TestStruct(a="x", b="y"))
         result = Rule.default_obj_struct_getter(struct, "a")
         assert result == struct.a
 
     def test_default_obj_getter_sub_key(self):
         from appenlight.lib.rule import Rule
+
         class TestStruct(object):
             def __init__(self, name, a, b):
                 self.name = name
                 self.a = a
                 self.b = b
 
             def __repr__(self):
-                return '<obj {}>'.format(self.name)
+                return "<obj {}>".format(self.name)
 
-        c = TestStruct('c', a=5, b='z')
-        b = TestStruct('b', a=c, b='y')
-        struct = TestStruct('a', a='a', b=b)
-        result = Rule.default_obj_struct_getter(struct, 'b:b')
+        c = TestStruct("c", a=5, b="z")
+        b = TestStruct("b", a=c, b="y")
+        struct = TestStruct("a", a="a", b=b)
+        result = Rule.default_obj_struct_getter(struct, "b:b")
         assert result == struct.b.b
-        result = Rule.default_obj_struct_getter(struct, 'b:a:b')
+        result = Rule.default_obj_struct_getter(struct, "b:a:b")
         assert result == struct.b.a.b
 
 
[e-mail]('report_type_matrix')
-class TestRulesParsing():
-    @pytest.mark.parametrize("op, struct_value, test_value, match_result", [
-        ('eq', 500, 500, True),
-        ('eq', 600, 500, False),
-        ('eq', 300, 500, False),
-        ('eq', "300", 500, False),
-        ('eq', "600", 500, False),
-        ('eq', "500", 500, True),
-        ('ne', 500, 500, False),
-        ('ne', 600, 500, True),
-        ('ne', 300, 500, True),
-        ('ne', "300", 500, True),
-        ('ne', "600", 500, True),
-        ('ne', "500", 500, False),
-        ('ge', 500, 500, True),
-        ('ge', 600, 500, True),
-        ('ge', 499, 500, False),
-        ('gt', 499, 500, False),
-        ('gt', 500, 500, False),
-        ('gt', 501, 500, True),
-        ('le', 499, 500, True),
-        ('le', 500, 500, True),
-        ('le', 501, 500, False),
-        ('lt', 499, 500, True),
-        ('lt', 500, 500, False),
-        ('lt', 501, 500, False),
-    ])
-    def test_single_op_int(self, op, struct_value, test_value, match_result,
-                           report_type_matrix):
[e-mail]("report_type_matrix")
+class TestRulesParsing:
+    @pytest.mark.parametrize(
+        "op, struct_value, test_value, match_result",
+        [
+            ("eq", 500, 500, True),
+            ("eq", 600, 500, False),
+            ("eq", 300, 500, False),
+            ("eq", "300", 500, False),
+            ("eq", "600", 500, False),
+            ("eq", "500", 500, True),
+            ("ne", 500, 500, False),
+            ("ne", 600, 500, True),
+            ("ne", 300, 500, True),
+            ("ne", "300", 500, True),
+            ("ne", "600", 500, True),
+            ("ne", "500", 500, False),
+            ("ge", 500, 500, True),
+            ("ge", 600, 500, True),
+            ("ge", 499, 500, False),
+            ("gt", 499, 500, False),
+            ("gt", 500, 500, False),
+            ("gt", 501, 500, True),
+            ("le", 499, 500, True),
+            ("le", 500, 500, True),
+            ("le", 501, 500, False),
+            ("lt", 499, 500, True),
+            ("lt", 500, 500, False),
+            ("lt", 501, 500, False),
+        ],
+    )
+    def test_single_op_int(
+        self, op, struct_value, test_value, match_result, report_type_matrix
+    ):
         from appenlight.lib.rule import Rule
-        rule_config = {
-            "op": op,
-            "field": "http_status",
-            "value": test_value
-        }
+
+        rule_config = {"op": op, "field": "http_status", "value": test_value}
         rule = Rule(rule_config, report_type_matrix)
 
-        data = {
-            "http_status": struct_value
-        }
+        data = {"http_status": struct_value}
         assert rule.match(data) is match_result
 
-    @pytest.mark.parametrize("op, struct_value, test_value, match_result", [
-        ('ge', "500.01", 500, True),
-        ('ge', "500.01", 500.02, False),
-        ('le', "500.01", 500.02, True)
-    ])
-    def test_single_op_float(self, op, struct_value, test_value, match_result,
-                             report_type_matrix):
+    @pytest.mark.parametrize(
+        "op, struct_value, test_value, match_result",
+        [
+            ("ge", "500.01", 500, True),
+            ("ge", "500.01", 500.02, False),
+            ("le", "500.01", 500.02, True),
+        ],
+    )
+    def test_single_op_float(
+        self, op, struct_value, test_value, match_result, report_type_matrix
+    ):
         from appenlight.lib.rule import Rule
-        rule_config = {
-            "op": op,
-            "field": "duration",
-            "value": test_value
-        }
+
+        rule_config = {"op": op, "field": "duration", "value": test_value}
         rule = Rule(rule_config, report_type_matrix)
 
-        data = {
-            "duration": struct_value
-        }
+        data = {"duration": struct_value}
         assert rule.match(data) is match_result
 
-    @pytest.mark.parametrize("op, struct_value, test_value, match_result", [
-        ('contains', 'foo bar baz', 'foo', True),
-        ('contains', 'foo bar baz', 'bar', True),
-        ('contains', 'foo bar baz', 'dupa', False),
-        ('startswith', 'foo bar baz', 'foo', True),
-        ('startswith', 'foo bar baz', 'bar', False),
-        ('endswith', 'foo bar baz', 'baz', True),
-        ('endswith', 'foo bar baz', 'bar', False),
-    ])
-    def test_single_op_string(self, op, struct_value, test_value,
-                              match_result, report_type_matrix):
+    @pytest.mark.parametrize(
+        "op, struct_value, test_value, match_result",
+        [
+            ("contains", "foo bar baz", "foo", True),
+            ("contains", "foo bar baz", "bar", True),
+            ("contains", "foo bar baz", "dupa", False),
+            ("startswith", "foo bar baz", "foo", True),
+            ("startswith", "foo bar baz", "bar", False),
+            ("endswith", "foo bar baz", "baz", True),
+            ("endswith", "foo bar baz", "bar", False),
+        ],
+    )
+    def test_single_op_string(
+        self, op, struct_value, test_value, match_result, report_type_matrix
+    ):
         from appenlight.lib.rule import Rule
-        rule_config = {
-            "op": op,
-            "field": "error",
-            "value": test_value
-        }
+
+        rule_config = {"op": op, "field": "error", "value": test_value}
         rule = Rule(rule_config, report_type_matrix)
 
-        data = {
-            "error": struct_value
-        }
+        data = {"error": struct_value}
         assert rule.match(data) is match_result
 
-    @pytest.mark.parametrize("field, value, s_type", [
-        ('field_unicode', 500, str),
-        ('field_unicode', 500.0, str),
-        ('field_unicode', "500", str),
-        ('field_int', "500", int),
-        ('field_int', 500, int),
-        ('field_int', 500.0, int),
-        ('field_float', "500", float),
-        ('field_float', 500, float),
-        ('field_float', 500.0, float),
-    ])
+    @pytest.mark.parametrize(
+        "field, value, s_type",
+        [
+            ("field_unicode", 500, str),
+            ("field_unicode", 500.0, str),
+            ("field_unicode", "500", str),
+            ("field_int", "500", int),
+            ("field_int", 500, int),
+            ("field_int", 500.0, int),
+            ("field_float", "500", float),
+            ("field_float", 500, float),
+            ("field_float", 500.0, float),
+        ],
+    )
     def test_type_normalization(self, field, value, s_type):
         from appenlight.lib.rule import Rule
+
         type_matrix = {
-            'field_unicode': {"type": 'unicode'},
-            'field_float': {"type": 'float'},
-            'field_int': {"type": 'int'},
+            "field_unicode": {"type": "unicode"},
+            "field_float": {"type": "float"},
+            "field_int": {"type": "int"},
         }
 
         rule = Rule({}, type_matrix)
         n_value = rule.normalized_type(field, value)
         assert isinstance(n_value, s_type) is True
 
 
[e-mail]('report_type_matrix')
-class TestNestedRuleParsing():
-
-    @pytest.mark.parametrize("data, result", [
-        ({"http_status": 501, "group": {"priority": 7, "occurences": 11}},
-         False),
-        ({"http_status": 101, "group": {"priority": 7, "occurences": 11}},
-         False),
-        ({"http_status": 500, "group": {"priority": 1, "occurences": 11}},
-         False),
-        ({"http_status": 101, "group": {"priority": 3, "occurences": 5}},
-         True),
-    ])
[e-mail]("report_type_matrix")
+class TestNestedRuleParsing:
+    @pytest.mark.parametrize(
+        "data, result",
+        [
+            ({"http_status": 501, "group": {"priority": 7, "occurences": 11}}, False),
+            ({"http_status": 101, "group": {"priority": 7, "occurences": 11}}, False),
+            ({"http_status": 500, "group": {"priority": 1, "occurences": 11}}, False),
+            ({"http_status": 101, "group": {"priority": 3, "occurences": 5}}, True),
+        ],
+    )
     def test_NOT_rule(self, data, result, report_type_matrix):
         from appenlight.lib.rule import Rule
+
         rule_config = {
             "field": "__NOT__",
             "rules": [
-                {
-                    "op": "ge",
-                    "field": "group:occurences",
-                    "value": "10"
-                },
-                {
-                    "op": "ge",
-                    "field": "group:priority",
-                    "value": "4"
-                }
-            ]
+                {"op": "ge", "field": "group:occurences", "value": "10"},
+                {"op": "ge", "field": "group:priority", "value": "4"},
+            ],
         }
 
         rule = Rule(rule_config, report_type_matrix)
         assert rule.match(data) is result
 
-    @pytest.mark.parametrize("data, result", [
-        ({"http_status": 501, "group": {"priority": 7, "occurences": 11}},
-         True),
-        ({"http_status": 101, "group": {"priority": 7, "occurences": 11}},
-         True),
-        ({"http_status": 500, "group": {"priority": 1, "occurences": 1}},
-         True),
-        ({"http_status": 101, "group": {"priority": 3, "occurences": 11}},
-         False),
-    ])
+    @pytest.mark.parametrize(
+        "data, result",
+        [
+            ({"http_status": 501, "group": {"priority": 7, "occurences": 11}}, True),
+            ({"http_status": 101, "group": {"priority": 7, "occurences": 11}}, True),
+            ({"http_status": 500, "group": {"priority": 1, "occurences": 1}}, True),
+            ({"http_status": 101, "group": {"priority": 3, "occurences": 11}}, False),
+        ],
+    )
     def test_nested_OR_AND_rule(self, data, result, report_type_matrix):
         from appenlight.lib.rule import Rule
+
         rule_config = {
             "field": "__OR__",
             "rules": [
                 {
                     "field": "__AND__",
                     "rules": [
-                        {
-                            "op": "ge",
-                            "field": "group:occurences",
-                            "value": "10"
-                        },
-                        {
-                            "op": "ge",
-                            "field": "group:priority",
-                            "value": "4"
-                        }
-                    ]
+                        {"op": "ge", "field": "group:occurences", "value": "10"},
+                        {"op": "ge", "field": "group:priority", "value": "4"},
+                    ],
                 },
-                {
-                    "op": "eq",
-                    "field": "http_status",
-                    "value": "500"
-                }
-            ]
+                {"op": "eq", "field": "http_status", "value": "500"},
+            ],
         }
 
         rule = Rule(rule_config, report_type_matrix)
         assert rule.match(data) is result
 
-    @pytest.mark.parametrize("data, result", [
-        ({"http_status": 501, "group": {"priority": 7, "occurences": 11}},
-         True),
-        ({"http_status": 101, "group": {"priority": 7, "occurences": 11}},
-         True),
-        ({"http_status": 500, "group": {"priority": 1, "occurences": 1}},
-         True),
-        ({"http_status": 101, "group": {"priority": 3, "occurences": 1}},
-         False),
-    ])
+    @pytest.mark.parametrize(
+        "data, result",
+        [
+            ({"http_status": 501, "group": {"priority": 7, "occurences": 11}}, True),
+            ({"http_status": 101, "group": {"priority": 7, "occurences": 11}}, True),
+            ({"http_status": 500, "group": {"priority": 1, "occurences": 1}}, True),
+            ({"http_status": 101, "group": {"priority": 3, "occurences": 1}}, False),
+        ],
+    )
     def test_nested_OR_OR_rule(self, data, result, report_type_matrix):
         from appenlight.lib.rule import Rule
+
         rule_config = {
             "field": "__OR__",
             "rules": [
-                {"field": "__OR__",
-                 "rules": [
-                     {"op": "ge",
-                      "field": "group:occurences",
-                      "value": "10"
-                      },
-                     {"op": "ge",
-                      "field": "group:priority",
-                      "value": "4"
-                      }
-                 ]
-                 },
-                {"op": "eq",
-                 "field": "http_status",
-                 "value": "500"
-                 }
-            ]
+                {
+                    "field": "__OR__",
+                    "rules": [
+                        {"op": "ge", "field": "group:occurences", "value": "10"},
+                        {"op": "ge", "field": "group:priority", "value": "4"},
+                    ],
+                },
+                {"op": "eq", "field": "http_status", "value": "500"},
+            ],
         }
 
         rule = Rule(rule_config, report_type_matrix)
         assert rule.match(data) is result
 
-    @pytest.mark.parametrize("data, result", [
-        ({"http_status": 500, "group": {"priority": 7, "occurences": 11}},
-         True),
-        ({"http_status": 101, "group": {"priority": 7, "occurences": 11}},
-         False),
-        ({"http_status": 500, "group": {"priority": 1, "occurences": 1}},
-         False),
-        ({"http_status": 101, "group": {"priority": 3, "occurences": 1}},
-         False),
-    ])
+    @pytest.mark.parametrize(
+        "data, result",
+        [
+            ({"http_status": 500, "group": {"priority": 7, "occurences": 11}}, True),
+            ({"http_status": 101, "group": {"priority": 7, "occurences": 11}}, False),
+            ({"http_status": 500, "group": {"priority": 1, "occurences": 1}}, False),
+            ({"http_status": 101, "group": {"priority": 3, "occurences": 1}}, False),
+        ],
+    )
     def test_nested_AND_AND_rule(self, data, result, report_type_matrix):
         from appenlight.lib.rule import Rule
+
         rule_config = {
             "field": "__AND__",
             "rules": [
-                {"field": "__AND__",
-                 "rules": [
-                     {"op": "ge",
-                      "field": "group:occurences",
-                      "value": "10"
-                      },
-                     {"op": "ge",
-                      "field": "group:priority",
-                      "value": "4"
-                      }]
-                 },
-                {"op": "eq",
-                 "field": "http_status",
-                 "value": "500"
-                 }
-            ]
+                {
+                    "field": "__AND__",
+                    "rules": [
+                        {"op": "ge", "field": "group:occurences", "value": "10"},
+                        {"op": "ge", "field": "group:priority", "value": "4"},
+                    ],
+                },
+                {"op": "eq", "field": "http_status", "value": "500"},
+            ],
         }
 
         rule = Rule(rule_config, report_type_matrix)
         assert rule.match(data) is result
 
-    @pytest.mark.parametrize("data, result", [
-        ({"http_status": 500, "group": {"priority": 7, "occurences": 11},
-          "url_path": '/test/register', "error": "foo test bar"}, True),
-        ({"http_status": 500, "group": {"priority": 7, "occurences": 11},
-          "url_path": '/test/register', "error": "foo INVALID bar"}, False),
-    ])
+    @pytest.mark.parametrize(
+        "data, result",
+        [
+            (
+                {
+                    "http_status": 500,
+                    "group": {"priority": 7, "occurences": 11},
+                    "url_path": "/test/register",
+                    "error": "foo test bar",
+                },
+                True,
+            ),
+            (
+                {
+                    "http_status": 500,
+                    "group": {"priority": 7, "occurences": 11},
+                    "url_path": "/test/register",
+                    "error": "foo INVALID bar",
+                },
+                False,
+            ),
+        ],
+    )
     def test_nested_AND_AND_AND_rule(self, data, result, report_type_matrix):
         from appenlight.lib.rule import Rule
+
         rule_config = {
             "field": "__AND__",
             "rules": [
-                {"field": "__AND__",
-                 "rules": [
-                     {"op": "ge",
-                      "field": "group:occurences",
-                      "value": "10"
-                      },
-                     {"field": "__AND__",
-                      "rules": [
-                          {"op": "endswith",
-                           "field": "url_path",
-                           "value": "register"},
-                          {"op": "contains",
-                           "field": "error",
-                           "value": "test"}]}]
-                 },
-                {"op": "eq",
-                 "field": "http_status",
-                 "value": "500"
-                 }
-            ]
+                {
+                    "field": "__AND__",
+                    "rules": [
+                        {"op": "ge", "field": "group:occurences", "value": "10"},
+                        {
+                            "field": "__AND__",
+                            "rules": [
+                                {
+                                    "op": "endswith",
+                                    "field": "url_path",
+                                    "value": "register",
+                                },
+                                {"op": "contains", "field": "error", "value": "test"},
+                            ],
+                        },
+                    ],
+                },
+                {"op": "eq", "field": "http_status", "value": "500"},
+            ],
         }
 
         rule = Rule(rule_config, report_type_matrix)
         assert rule.match(data) is result
 
-    @pytest.mark.parametrize("data, result", [
-        ({"http_status": 500, "group": {"priority": 7, "occurences": 11},
-          "url_path": 6, "error": 3}, False),
-        ({"http_status": 500, "group": {"priority": 7, "occurences": 11},
-          "url_path": '/test/register', "error": "foo INVALID bar"}, True),
-    ])
+    @pytest.mark.parametrize(
+        "data, result",
+        [
+            (
+                {
+                    "http_status": 500,
+                    "group": {"priority": 7, "occurences": 11},
+                    "url_path": 6,
+                    "error": 3,
+                },
+                False,
+            ),
+            (
+                {
+                    "http_status": 500,
+                    "group": {"priority": 7, "occurences": 11},
+                    "url_path": "/test/register",
+                    "error": "foo INVALID bar",
+                },
+                True,
+            ),
+        ],
+    )
     def test_nested_AND_AND_OR_rule(self, data, result, report_type_matrix):
         from appenlight.lib.rule import Rule
+
         rule_config = {
             "field": "__AND__",
             "rules": [
-                {"field": "__AND__",
-                 "rules": [
-                     {"op": "ge",
-                      "field": "group:occurences",
-                      "value": "10"
-                      },
-                     {"field": "__OR__",
-                      "rules": [
-                          {"op": "endswith",
-                           "field": "url_path",
-                           "value": "register"
-                           },
-                          {"op": "contains",
-                           "field": "error",
-                           "value": "test"
-                           }]}]
-                 },
-                {"op": "eq",
-                 "field": "http_status",
-                 "value": "500"
-                 }
-            ]
+                {
+                    "field": "__AND__",
+                    "rules": [
+                        {"op": "ge", "field": "group:occurences", "value": "10"},
+                        {
+                            "field": "__OR__",
+                            "rules": [
+                                {
+                                    "op": "endswith",
+                                    "field": "url_path",
+                                    "value": "register",
+                                },
+                                {"op": "contains", "field": "error", "value": "test"},
+                            ],
+                        },
+                    ],
+                },
+                {"op": "eq", "field": "http_status", "value": "500"},
+            ],
         }
 
         rule = Rule(rule_config, report_type_matrix)
         assert rule.match(data) is result
 
-    @pytest.mark.parametrize("op, field, value, should_fail", [
-        ('eq', 'http_status', "1", False),
-        ('ne', 'http_status', "1", False),
-        ('ne', 'http_status', "foo", True),
-        ('startswith', 'http_status', "1", True),
-        ('eq', 'group:priority', "1", False),
-        ('ne', 'group:priority', "1", False),
-        ('ge', 'group:priority', "1", False),
-        ('le', 'group:priority', "1", False),
-        ('startswith', 'group:priority', "1", True),
-        ('eq', 'url_domain', "1", False),
-        ('ne', 'url_domain', "1", False),
-        ('startswith', 'url_domain', "1", False),
-        ('endswith', 'url_domain', "1", False),
-        ('contains', 'url_domain', "1", False),
-        ('ge', 'url_domain', "1", True),
-        ('eq', 'url_path', "1", False),
-        ('ne', 'url_path', "1", False),
-        ('startswith', 'url_path', "1", False),
-        ('endswith', 'url_path', "1", False),
-        ('contains', 'url_path', "1", False),
-        ('ge', 'url_path', "1", True),
-        ('eq', 'error', "1", False),
-        ('ne', 'error', "1", False),
-        ('startswith', 'error', "1", False),
-        ('endswith', 'error', "1", False),
-        ('contains', 'error', "1", False),
-        ('ge', 'error', "1", True),
-        ('ge', 'url_path', "1", True),
-        ('eq', 'tags:server_name', "1", False),
-        ('ne', 'tags:server_name', "1", False),
-        ('startswith', 'tags:server_name', "1", False),
-        ('endswith', 'tags:server_name', "1", False),
-        ('contains', 'tags:server_name', "1", False),
-        ('ge', 'tags:server_name', "1", True),
-        ('contains', 'traceback', "1", False),
-        ('ge', 'traceback', "1", True),
-        ('eq', 'group:occurences', "1", False),
-        ('ne', 'group:occurences', "1", False),
-        ('ge', 'group:occurences', "1", False),
-        ('le', 'group:occurences', "1", False),
-        ('contains', 'group:occurences', "1", True),
-    ])
-    def test_rule_validation(self, op, field, value, should_fail,
-                             report_type_matrix):
+    @pytest.mark.parametrize(
+        "op, field, value, should_fail",
+        [
+            ("eq", "http_status", "1", False),
+            ("ne", "http_status", "1", False),
+            ("ne", "http_status", "foo", True),
+            ("startswith", "http_status", "1", True),
+            ("eq", "group:priority", "1", False),
+            ("ne", "group:priority", "1", False),
+            ("ge", "group:priority", "1", False),
+            ("le", "group:priority", "1", False),
+            ("startswith", "group:priority", "1", True),
+            ("eq", "url_domain", "1", False),
+            ("ne", "url_domain", "1", False),
+            ("startswith", "url_domain", "1", False),
+            ("endswith", "url_domain", "1", False),
+            ("contains", "url_domain", "1", False),
+            ("ge", "url_domain", "1", True),
+            ("eq", "url_path", "1", False),
+            ("ne", "url_path", "1", False),
+            ("startswith", "url_path", "1", False),
+            ("endswith", "url_path", "1", False),
+            ("contains", "url_path", "1", False),
+            ("ge", "url_path", "1", True),
+            ("eq", "error", "1", False),
+            ("ne", "error", "1", False),
+            ("startswith", "error", "1", False),
+            ("endswith", "error", "1", False),
+            ("contains", "error", "1", False),
+            ("ge", "error", "1", True),
+            ("ge", "url_path", "1", True),
+            ("eq", "tags:server_name", "1", False),
+            ("ne", "tags:server_name", "1", False),
+            ("startswith", "tags:server_name", "1", False),
+            ("endswith", "tags:server_name", "1", False),
+            ("contains", "tags:server_name", "1", False),
+            ("ge", "tags:server_name", "1", True),
+            ("contains", "traceback", "1", False),
+            ("ge", "traceback", "1", True),
+            ("eq", "group:occurences", "1", False),
+            ("ne", "group:occurences", "1", False),
+            ("ge", "group:occurences", "1", False),
+            ("le", "group:occurences", "1", False),
+            ("contains", "group:occurences", "1", True),
+        ],
+    )
+    def test_rule_validation(self, op, field, value, should_fail, report_type_matrix):
         import colander
         from appenlight.validators import build_rule_schema
-        rule_config = {
-            "op": op,
-            "field": field,
-            "value": value
-        }
+
+        rule_config = {"op": op, "field": field, "value": value}
 
         schema = build_rule_schema(rule_config, report_type_matrix)
         if should_fail:
             with pytest.raises(colander.Invalid):
                 schema.deserialize(rule_config)
         else:
             schema.deserialize(rule_config)
 
     def test_nested_proper_rule_validation(self, report_type_matrix):
         from appenlight.validators import build_rule_schema
+
         rule_config = {
             "field": "__AND__",
             "rules": [
                 {
                     "field": "__AND__",
                     "rules": [
-                        {
-                            "op": "ge",
-                            "field": "group:occurences",
-                            "value": "10"
-                        },
+                        {"op": "ge", "field": "group:occurences", "value": "10"},
                         {
                             "field": "__OR__",
                             "rules": [
                                 {
                                     "op": "endswith",
                                     "field": "url_path",
-                                    "value": "register"
+                                    "value": "register",
                                 },
-                                {
-                                    "op": "contains",
-                                    "field": "error",
-                                    "value": "test"
-                                }
-                            ]
-                        }
-                    ]
+                                {"op": "contains", "field": "error", "value": "test"},
+                            ],
+                        },
+                    ],
                 },
-                {
-                    "op": "eq",
-                    "field": "http_status",
-                    "value": "500"
-                }
-            ]
+                {"op": "eq", "field": "http_status", "value": "500"},
+            ],
         }
 
         schema = build_rule_schema(rule_config, report_type_matrix)
         deserialized = schema.deserialize(rule_config)
 
     def test_nested_bad_rule_validation(self, report_type_matrix):
         import colander
         from appenlight.validators import build_rule_schema
+
         rule_config = {
             "field": "__AND__",
             "rules": [
                 {
                     "field": "__AND__",
                     "rules": [
-                        {
-                            "op": "ge",
-                            "field": "group:occurences",
-                            "value": "10"
-                        },
+                        {"op": "ge", "field": "group:occurences", "value": "10"},
                         {
                             "field": "__OR__",
                             "rules": [
-                                {
-                                    "op": "gt",
-                                    "field": "url_path",
-                                    "value": "register"
-                                },
-                                {
-                                    "op": "contains",
-                                    "field": "error",
-                                    "value": "test"
-                                }
-                            ]
-                        }
-                    ]
+                                {"op": "gt", "field": "url_path", "value": "register"},
+                                {"op": "contains", "field": "error", "value": "test"},
+                            ],
+                        },
+                    ],
                 },
-                {
-                    "op": "eq",
-                    "field": "http_status",
-                    "value": "500"
-                }
-            ]
+                {"op": "eq", "field": "http_status", "value": "500"},
+            ],
         }
 
         schema = build_rule_schema(rule_config, report_type_matrix)
         with pytest.raises(colander.Invalid):
             deserialized = schema.deserialize(rule_config)
 
     def test_config_manipulator(self):
         from appenlight.lib.rule import Rule
+
         type_matrix = {
-            'a': {"type": 'int',
-                  "ops": ('eq', 'ne', 'ge', 'le',)},
-            'b': {"type": 'int',
-                  "ops": ('eq', 'ne', 'ge', 'le',)},
+            "a": {"type": "int", "ops": ("eq", "ne", "ge", "le")},
+            "b": {"type": "int", "ops": ("eq", "ne", "ge", "le")},
         }
         rule_config = {
             "field": "__OR__",
             "rules": [
                 {
                     "field": "__OR__",
-                    "rules": [
-                        {
-                            "op": "ge",
-                            "field": "a",
-                            "value": "10"
-                        }
-                    ]
+                    "rules": [{"op": "ge", "field": "a", "value": "10"}],
                 },
-                {
-                    "op": "eq",
-                    "field": "b",
-                    "value": "500"
-                }
-            ]
+                {"op": "eq", "field": "b", "value": "500"},
+            ],
         }
 
         def rule_manipulator(rule):
-            if 'value' in rule.config:
-                rule.config['value'] = "1"
+            if "value" in rule.config:
+                rule.config["value"] = "1"
 
-        rule = Rule(rule_config, type_matrix,
-                    config_manipulator=rule_manipulator)
-        rule.match({"a": 1,
-                    "b": "2"})
-        assert rule.config['rules'][0]['rules'][0]['value'] == "1"
-        assert rule.config['rules'][1]['value'] == "1"
-        assert rule.type_matrix["b"]['type'] == "int"
+        rule = Rule(rule_config, type_matrix, config_manipulator=rule_manipulator)
+        rule.match({"a": 1, "b": "2"})
+        assert rule.config["rules"][0]["rules"][0]["value"] == "1"
+        assert rule.config["rules"][1]["value"] == "1"
+        assert rule.type_matrix["b"]["type"] == "int"
 
     def test_dynamic_config_manipulator(self):
         from appenlight.lib.rule import Rule
+
         rule_config = {
             "field": "__OR__",
             "rules": [
                 {
                     "field": "__OR__",
-                    "rules": [
-                        {
-                            "op": "ge",
-                            "field": "a",
-                            "value": "10"
-                        }
-                    ]
+                    "rules": [{"op": "ge", "field": "a", "value": "10"}],
                 },
-                {
-                    "op": "eq",
-                    "field": "b",
-                    "value": "500"
-                }
-            ]
+                {"op": "eq", "field": "b", "value": "500"},
+            ],
         }
 
         def rule_manipulator(rule):
             rule.type_matrix = {
-                'a': {"type": 'int',
-                      "ops": ('eq', 'ne', 'ge', 'le',)},
-                'b': {"type": 'unicode',
-                      "ops": ('eq', 'ne', 'ge', 'le',)},
+                "a": {"type": "int", "ops": ("eq", "ne", "ge", "le")},
+                "b": {"type": "unicode", "ops": ("eq", "ne", "ge", "le")},
             }
 
-            if 'value' in rule.config:
-                if rule.config['field'] == 'a':
-                    rule.config['value'] = "1"
-                elif rule.config['field'] == 'b':
-                    rule.config['value'] = "2"
+            if "value" in rule.config:
+                if rule.config["field"] == "a":
+                    rule.config["value"] = "1"
+                elif rule.config["field"] == "b":
+                    rule.config["value"] = "2"
 
-        rule = Rule(rule_config, {},
-                    config_manipulator=rule_manipulator)
-        rule.match({"a": 11,
-                    "b": "55"})
-        assert rule.config['rules'][0]['rules'][0]['value'] == "1"
-        assert rule.config['rules'][1]['value'] == "2"
-        assert rule.type_matrix["b"]['type'] == "unicode"
+        rule = Rule(rule_config, {}, config_manipulator=rule_manipulator)
+        rule.match({"a": 11, "b": "55"})
+        assert rule.config["rules"][0]["rules"][0]["value"] == "1"
+        assert rule.config["rules"][1]["value"] == "2"
+        assert rule.type_matrix["b"]["type"] == "unicode"
 
 
[e-mail]('base_app', 'with_migrations')
[e-mail]("base_app", "with_migrations")
 class TestViewsWithForms(object):
     def test_bad_csrf(self):
         from appenlight.forms import CSRFException
         from appenlight.views.index import register
-        post_data = {'dupa': 'dupa'}
+
+        post_data = {"dupa": "dupa"}
         request = testing.DummyRequest(post=post_data)
         request.POST = webob.multidict.MultiDict(request.POST)
         with pytest.raises(CSRFException):
             register(request)
 
     def test_proper_csrf(self):
         from appenlight.views.index import register
+
         request = pyramid.threadlocal.get_current_request()
-        post_data = {'dupa': 'dupa',
-                     'csrf_token': request.session.get_csrf_token()}
+        post_data = {"dupa": "dupa", "csrf_token": request.session.get_csrf_token()}
         request = testing.DummyRequest(post=post_data)
         request.POST = webob.multidict.MultiDict(request.POST)
         result = register(request)
-        assert result['form'].errors['email'][0] == 'This field is required.'
+        assert result["form"].errors["email"][0] == "This field is required."
 
 
[e-mail]('base_app', 'with_migrations', 'default_data')
[e-mail]("base_app", "with_migrations", "default_data")
 class TestRegistration(object):
     def test_invalid_form(self):
         from appenlight.views.index import register
+
         request = pyramid.threadlocal.get_current_request()
-        post_data = {'user_name': '',
-                     'user_password': '',
-                     'email': '',
-                     'csrf_token': request.session.get_csrf_token()}
+        post_data = {
+            "user_name": "",
+            "user_password": "",
+            "email": "",
+            "csrf_token": request.session.get_csrf_token(),
+        }
         request = testing.DummyRequest(post=post_data)
         request.POST = webob.multidict.MultiDict(request.POST)
         result = register(request)
-        assert result['form'].errors['user_name'][0] == \
-               'This field is required.'
+        assert result["form"].errors["user_name"][0] == "This field is required."
 
     def test_valid_form(self):
         from appenlight.views.index import register
         from ziggurat_foundations.models.services.user import UserService
+
         request = pyramid.threadlocal.get_current_request()
-        post_data = {'user_name': 'foo',
-                     'user_password': 'barr',
-                     'email': '[e-mail]',
-                     'csrf_token': request.session.get_csrf_token()}
+        post_data = {
+            "user_name": "foo",
+            "user_password": "barr",
+            "email": "[e-mail]",
+            "csrf_token": request.session.get_csrf_token(),
+        }
         request = testing.DummyRequest(post=post_data)
         request.add_flash_to_headers = mock.Mock()
         request.POST = webob.multidict.MultiDict(request.POST)
-        assert UserService.by_user_name('foo') is None
+        assert UserService.by_user_name("foo") is None
         register(request)
-        user = UserService.by_user_name('foo')
-        assert user.user_name == 'foo'
+        user = UserService.by_user_name("foo")
+        assert user.user_name == "foo"
         assert len(user.user_password) >= 60
 
 
[e-mail]('base_app', 'with_migrations', 'clean_tables',
-                         'default_user')
[e-mail]("base_app", "with_migrations", "clean_tables", "default_user")
 class TestApplicationCreation(object):
     def test_wrong_data(self):
         import appenlight.views.applications as applications
         from ziggurat_foundations.models.services.user import UserService
+
         request = pyramid.threadlocal.get_current_request()
-        request.user = UserService.by_user_name('testuser')
+        request.user = UserService.by_user_name("testuser")
         request.unsafe_json_body = {}
-        request.headers['X-XSRF-TOKEN'] = request.session.get_csrf_token()
+        request.headers["X-XSRF-TOKEN"] = request.session.get_csrf_token()
         response = applications.application_create(request)
         assert response.code == 422
 
     def test_proper_data(self):
         import appenlight.views.applications as applications
         from ziggurat_foundations.models.services.user import UserService
 
         request = pyramid.threadlocal.get_current_request()
-        request.user = UserService.by_user_name('testuser')
-        request.unsafe_json_body = {"resource_name": "app name",
-                                    "domains": "foo"}
-        request.headers['X-XSRF-TOKEN'] = request.session.get_csrf_token()
+        request.user = UserService.by_user_name("testuser")
+        request.unsafe_json_body = {"resource_name": "app name", "domains": "foo"}
+        request.headers["X-XSRF-TOKEN"] = request.session.get_csrf_token()
         app_dict = applications.application_create(request)
-        assert app_dict['public_key'] is not None
-        assert app_dict['api_key'] is not None
-        assert app_dict['resource_name'] == 'app name'
-        assert app_dict['owner_group_id'] is None
-        assert app_dict['resource_id'] is not None
-        assert app_dict['default_grouping'] == 'url_traceback'
-        assert app_dict['possible_permissions'] == ('view', 'update_reports')
-        assert app_dict['slow_report_threshold'] == 10
-        assert app_dict['owner_user_name'] == 'testuser'
-        assert app_dict['owner_user_id'] == request.user.id
-        assert app_dict['domains'] is 'foo'
-        assert app_dict['postprocessing_rules'] == []
-        assert app_dict['error_report_threshold'] == 10
-        assert app_dict['allow_permanent_storage'] is False
-        assert app_dict['resource_type'] == 'application'
-        assert app_dict['current_permissions'] == []
-
-
[e-mail]('default_application')
[e-mail]('base_app', 'with_migrations', 'clean_tables')
+        assert app_dict["public_key"] is not None
+        assert app_dict["api_key"] is not None
+        assert app_dict["resource_name"] == "app name"
+        assert app_dict["owner_group_id"] is None
+        assert app_dict["resource_id"] is not None
+        assert app_dict["default_grouping"] == "url_traceback"
+        assert app_dict["possible_permissions"] == ("view", "update_reports")
+        assert app_dict["slow_report_threshold"] == 10
+        assert app_dict["owner_user_name"] == "testuser"
+        assert app_dict["owner_user_id"] == request.user.id
+        assert app_dict["domains"] is "foo"
+        assert app_dict["postprocessing_rules"] == []
+        assert app_dict["error_report_threshold"] == 10
+        assert app_dict["allow_permanent_storage"] is False
+        assert app_dict["resource_type"] == "application"
+        assert app_dict["current_permissions"] == []
+
+
[e-mail]("default_application")
[e-mail]("base_app", "with_migrations", "clean_tables")
 class TestAPISentryView(object):
     def test_no_payload(self, default_application):
         import colander
         from appenlight.models.services.application import ApplicationService
         from appenlight.views.api import sentry_compat
         from appenlight.lib.request import JSONException
 
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
-        request = testing.DummyRequest(
-            headers={'Content-Type': 'application/json'})
-        request.unsafe_json_body = ''
+        request = testing.DummyRequest(headers={"Content-Type": "application/json"})
+        request.unsafe_json_body = ""
         request.context = context
         route = mock.Mock()
-        route.name = 'api_sentry'
+        route.name = "api_sentry"
         request.matched_route = route
         with pytest.raises(JSONException):
             sentry_compat(request)
 
     def test_java_client_payload(self):
         from appenlight.views.api import sentry_compat
         from appenlight.models.services.application import ApplicationService
         from appenlight.models.report_group import ReportGroup
+
         route = mock.Mock()
-        route.name = 'api_sentry'
+        route.name = "api_sentry"
         request = pyramid.threadlocal.get_current_request()
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         context.resource.allow_permanent_storage = True
         request.context = context
         request.matched_route = route
-        request.body = b'eJy1UmFr2zAQ/S0T+7BCLOzYThp/C6xjG6SDLd/GCBf57Ki' \
-                       b'RJSHJJiXkv+/UlC7p2kAZA33Ru6f33t1pz3BAHVayZhWr87' \
-                       b'JMs+I6q3MsrifFep2vc1iXM1HMpgBTNmIdeg8tEvlmJ9AGa' \
-                       b'fQ7goOkQoDOUmGcZpMkLZO0WGZFRadMiaHIR1EVnTMu3k3b' \
-                       b'oiMgqJrXpgOpOVjLLTiPkWAVhMa4jih3MAAholfWyUDAksz' \
-                       b'm1iopICbg8fWH52B8VWXZVYwHrWfV/jBipD2gW2no8CFMa5' \
-                       b'JButCDSjoQG6mR6LgLDojPPn/7sbydL25ep34HGl+y3DiE+' \
-                       b'lH0xXBXjMzFBsXW99SS7pWKYXRw91zqgK4BgZ4/DZVVP/cs' \
-                       b'3NuzSZPfAKqP2Cdj4tw7U/cKH0fEFeiWQFqE2FIHAmMPjaN' \
-                       b'Y/kHvbzY/JqdHUq9o/KxqQHkcsabX4piDuT4aK+pXG1ZNi/' \
-                       b'IwOpEyruXC1LiB3vPO3BmOOxTUCIqv5LIg5H12oh9cf0l+P' \
-                       b'MvP5P8kddgoFIEvMGzM5cRSD2aLJ6qTdHKm6nv9pPcRFba0' \
-                       b'Kd0eleeCFuGN+9JZ9TaXIn/V5JYMBvxXg3L6PwzSE4dkfOb' \
-                       b'w7CtfWmP85SdCs8OvA53fUV19cg=='
+        request.body = (
+            b"eJy1UmFr2zAQ/S0T+7BCLOzYThp/C6xjG6SDLd/GCBf57Ki"
+            b"RJSHJJiXkv+/UlC7p2kAZA33Ru6f33t1pz3BAHVayZhWr87"
+            b"JMs+I6q3MsrifFep2vc1iXM1HMpgBTNmIdeg8tEvlmJ9AGa"
+            b"fQ7goOkQoDOUmGcZpMkLZO0WGZFRadMiaHIR1EVnTMu3k3b"
+            b"oiMgqJrXpgOpOVjLLTiPkWAVhMa4jih3MAAholfWyUDAksz"
+            b"m1iopICbg8fWH52B8VWXZVYwHrWfV/jBipD2gW2no8CFMa5"
+            b"JButCDSjoQG6mR6LgLDojPPn/7sbydL25ep34HGl+y3DiE+"
+            b"lH0xXBXjMzFBsXW99SS7pWKYXRw91zqgK4BgZ4/DZVVP/cs"
+            b"3NuzSZPfAKqP2Cdj4tw7U/cKH0fEFeiWQFqE2FIHAmMPjaN"
+            b"Y/kHvbzY/JqdHUq9o/KxqQHkcsabX4piDuT4aK+pXG1ZNi/"
+            b"IwOpEyruXC1LiB3vPO3BmOOxTUCIqv5LIg5H12oh9cf0l+P"
+            b"MvP5P8kddgoFIEvMGzM5cRSD2aLJ6qTdHKm6nv9pPcRFba0"
+            b"Kd0eleeCFuGN+9JZ9TaXIn/V5JYMBvxXg3L6PwzSE4dkfOb"
+            b"w7CtfWmP85SdCs8OvA53fUV19cg=="
+        )
         sentry_compat(request)
         query = DBSession.query(ReportGroup)
         report = query.first()
         assert query.count() == 1
         assert report.total_reports == 1
 
     def test_ruby_client_payload(self):
         from appenlight.views.api import sentry_compat
         from appenlight.models.services.application import ApplicationService
         from appenlight.models.report_group import ReportGroup
         from appenlight.tests.payload_examples import SENTRY_RUBY_ENCODED
+
         route = mock.Mock()
-        route.name = 'api_sentry'
+        route.name = "api_sentry"
         request = testing.DummyRequest(
-            headers={'Content-Type': 'application/octet-stream',
-                     'User-Agent': 'sentry-ruby/1.0.0',
-                     'X-Sentry-Auth': 'Sentry sentry_version=5, '
-                                      'sentry_client=raven-ruby/1.0.0, '
-                                      'sentry_timestamp=1462378483, '
-                                      'sentry_key=xxx, sentry_secret=xxx'
-                     })
+            headers={
+                "Content-Type": "application/octet-stream",
+                "User-Agent": "sentry-ruby/1.0.0",
+                "X-Sentry-Auth": "Sentry sentry_version=5, "
+                "sentry_client=raven-ruby/1.0.0, "
+                "sentry_timestamp=1462378483, "
+                "sentry_key=xxx, sentry_secret=xxx",
+            }
+        )
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         context.resource.allow_permanent_storage = True
         request.context = context
         request.matched_route = route
         request.body = SENTRY_RUBY_ENCODED
         sentry_compat(request)
         query = DBSession.query(ReportGroup)
         report = query.first()
         assert query.count() == 1
         assert report.total_reports == 1
 
     def test_python_client_decoded_payload(self):
         from appenlight.views.api import sentry_compat
         from appenlight.models.services.application import ApplicationService
         from appenlight.models.report_group import ReportGroup
         from appenlight.tests.payload_examples import SENTRY_PYTHON_PAYLOAD_7
+
         route = mock.Mock()
-        route.name = 'api_sentry'
+        route.name = "api_sentry"
         request = pyramid.threadlocal.get_current_request()
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         context.resource.allow_permanent_storage = True
         request.context = context
         request.matched_route = route
-        request.body = json.dumps(SENTRY_PYTHON_PAYLOAD_7).encode('utf8')
+        request.body = json.dumps(SENTRY_PYTHON_PAYLOAD_7).encode("utf8")
         sentry_compat(request)
         query = DBSession.query(ReportGroup)
         report = query.first()
         assert query.count() == 1
         assert report.total_reports == 1
 
     def test_python_client_encoded_payload(self):
         from appenlight.views.api import sentry_compat
         from appenlight.models.services.application import ApplicationService
         from appenlight.models.report_group import ReportGroup
         from appenlight.tests.payload_examples import SENTRY_PYTHON_ENCODED
+
         route = mock.Mock()
-        route.name = 'api_sentry'
+        route.name = "api_sentry"
         request = testing.DummyRequest(
-            headers={'Content-Type': 'application/octet-stream',
-                     'Content-Encoding': 'deflate',
-                     'User-Agent': 'sentry-ruby/1.0.0',
-                     'X-Sentry-Auth': 'Sentry sentry_version=5, '
-                                      'sentry_client=raven-ruby/1.0.0, '
-                                      'sentry_timestamp=1462378483, '
-                                      'sentry_key=xxx, sentry_secret=xxx'
-                     })
+            headers={
+                "Content-Type": "application/octet-stream",
+                "Content-Encoding": "deflate",
+                "User-Agent": "sentry-ruby/1.0.0",
+                "X-Sentry-Auth": "Sentry sentry_version=5, "
+                "sentry_client=raven-ruby/1.0.0, "
+                "sentry_timestamp=1462378483, "
+                "sentry_key=xxx, sentry_secret=xxx",
+            }
+        )
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         context.resource.allow_permanent_storage = True
         request.context = context
         request.matched_route = route
         request.body = SENTRY_PYTHON_ENCODED
         sentry_compat(request)
         query = DBSession.query(ReportGroup)
         report = query.first()
         assert query.count() == 1
         assert report.total_reports == 1

backend/src/appenlight/validators.py

@@ -1,768 +1,831 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import datetime
 
 import colander
 from colander import null
 
 # those keywords are here so we can distingush between searching for tags and
 # normal properties of reports/logs
-accepted_search_params = ['resource',
-                          'request_id',
-                          'start_date',
-                          'end_date',
-                          'page',
-                          'min_occurences',
-                          'http_status',
-                          'priority',
-                          'error',
-                          'url_path',
-                          'url_domain',
-                          'report_status',
-                          'min_duration',
-                          'max_duration',
-                          'message',
-                          'level',
-                          'namespace']
+accepted_search_params = [
+    "resource",
+    "request_id",
+    "start_date",
+    "end_date",
+    "page",
+    "min_occurences",
+    "http_status",
+    "priority",
+    "error",
+    "url_path",
+    "url_domain",
+    "report_status",
+    "min_duration",
+    "max_duration",
+    "message",
+    "level",
+    "namespace",
+]
 
 
 @colander.deferred
 def deferred_utcnow(node, kw):
-    return kw['utcnow']
+    return kw["utcnow"]
 
 
 @colander.deferred
 def optional_limited_date(node, kw):
-    if not kw.get('allow_permanent_storage'):
+    if not kw.get("allow_permanent_storage"):
         return limited_date
 
 
 def lowercase_preparer(input_data):
     """
     Transforms a list of string entries to lowercase
     Used in search query validation
     """
     if not input_data:
         return input_data
     return [x.lower() for x in input_data]
 
 
 def shortener_factory(cutoff_size=32):
     """
     Limits the input data to specific character count
     :arg cutoff_cutoff_size How much characters to store
 
     """
 
     def shortener(input_data):
         if not input_data:
             return input_data
         else:
             if isinstance(input_data, str):
                 return input_data[:cutoff_size]
             else:
                 return input_data
 
     return shortener
 
 
 def cast_to_unicode_or_null(value):
     if value is not colander.null:
         return str(value)
     return None
 
 
 class NonTZDate(colander.DateTime):
     """ Returns null for incorrect date format - also removes tz info"""
 
     def deserialize(self, node, cstruct):
         # disabled for now
         # if cstruct and isinstance(cstruct, str):
         #     if ':' not in cstruct:
         #         cstruct += ':0.0'
         #     if '.' not in cstruct:
         #         cstruct += '.0'
         value = super(NonTZDate, self).deserialize(node, cstruct)
         if value:
             return value.replace(tzinfo=None)
         return value
 
 
 class UnknownType(object):
     """
     Universal type that will accept a deserialized JSON object and store it unaltered
     """
 
     def serialize(self, node, appstruct):
         if appstruct is null:
             return null
         return appstruct
 
     def deserialize(self, node, cstruct):
         if cstruct is null:
             return null
         return cstruct
 
     def cstruct_children(self):
         return []
 
 
 # SLOW REPORT SCHEMA
 
+
 def rewrite_type(input_data):
     """
     Fix for legacy appenlight clients
     """
-    if input_data == 'remote_call':
-        return 'remote'
+    if input_data == "remote_call":
+        return "remote"
     return input_data
 
 
 class ExtraTupleSchema(colander.TupleSchema):
-    name = colander.SchemaNode(colander.String(),
-                               validator=colander.Length(1, 64))
-    value = colander.SchemaNode(UnknownType(),
-                                preparer=shortener_factory(512),
-                                missing=None)
+    name = colander.SchemaNode(colander.String(), validator=colander.Length(1, 64))
+    value = colander.SchemaNode(
+        UnknownType(), preparer=shortener_factory(512), missing=None
+    )
 
 
 class ExtraSchemaList(colander.SequenceSchema):
     tag = ExtraTupleSchema()
     missing = None
 
 
 class TagsTupleSchema(colander.TupleSchema):
-    name = colander.SchemaNode(colander.String(),
-                               validator=colander.Length(1, 128))
-    value = colander.SchemaNode(UnknownType(),
-                                preparer=shortener_factory(128),
-                                missing=None)
+    name = colander.SchemaNode(colander.String(), validator=colander.Length(1, 128))
+    value = colander.SchemaNode(
+        UnknownType(), preparer=shortener_factory(128), missing=None
+    )
 
 
 class TagSchemaList(colander.SequenceSchema):
     tag = TagsTupleSchema()
     missing = None
 
 
 class NumericTagsTupleSchema(colander.TupleSchema):
-    name = colander.SchemaNode(colander.String(),
-                               validator=colander.Length(1, 128))
+    name = colander.SchemaNode(colander.String(), validator=colander.Length(1, 128))
     value = colander.SchemaNode(colander.Float(), missing=0)
 
 
 class NumericTagSchemaList(colander.SequenceSchema):
     tag = NumericTagsTupleSchema()
     missing = None
 
 
 class SlowCallSchema(colander.MappingSchema):
     """
     Validates slow call format in slow call list
     """
+
     start = colander.SchemaNode(NonTZDate())
     end = colander.SchemaNode(NonTZDate())
-    statement = colander.SchemaNode(colander.String(), missing='')
+    statement = colander.SchemaNode(colander.String(), missing="")
     parameters = colander.SchemaNode(UnknownType(), missing=None)
     type = colander.SchemaNode(
         colander.String(),
         preparer=rewrite_type,
         validator=colander.OneOf(
-            ['tmpl', 'sql', 'nosql', 'remote', 'unknown', 'custom']),
-        missing='unknown')
-    subtype = colander.SchemaNode(colander.String(),
-                                  validator=colander.Length(1, 16),
-                                  missing='unknown')
-    location = colander.SchemaNode(colander.String(),
-                                   validator=colander.Length(1, 255),
-                                   missing='')
+            ["tmpl", "sql", "nosql", "remote", "unknown", "custom"]
+        ),
+        missing="unknown",
+    )
+    subtype = colander.SchemaNode(
+        colander.String(), validator=colander.Length(1, 16), missing="unknown"
+    )
+    location = colander.SchemaNode(
+        colander.String(), validator=colander.Length(1, 255), missing=""
+    )
 
 
 def limited_date(node, value):
     """ checks to make sure that the value is not older/newer than 2h """
     past_hours = 72
     future_hours = 2
-    min_time = datetime.datetime.utcnow() - datetime.timedelta(
-        hours=past_hours)
-    max_time = datetime.datetime.utcnow() + datetime.timedelta(
-        hours=future_hours)
+    min_time = datetime.datetime.utcnow() - datetime.timedelta(hours=past_hours)
+    max_time = datetime.datetime.utcnow() + datetime.timedelta(hours=future_hours)
     if min_time > value:
-        msg = '%r is older from current UTC time by ' + str(past_hours)
-        msg += ' hours. Ask administrator to enable permanent logging for ' \
-               'your application to store logs with dates in past.'
+        msg = "%r is older from current UTC time by " + str(past_hours)
+        msg += (
+            " hours. Ask administrator to enable permanent logging for "
+            "your application to store logs with dates in past."
+        )
         raise colander.Invalid(node, msg % value)
     if max_time < value:
-        msg = '%r is newer from current UTC time by ' + str(future_hours)
-        msg += ' hours. Ask administrator to enable permanent logging for ' \
-               'your application to store logs with dates in future.'
+        msg = "%r is newer from current UTC time by " + str(future_hours)
+        msg += (
+            " hours. Ask administrator to enable permanent logging for "
+            "your application to store logs with dates in future."
+        )
         raise colander.Invalid(node, msg % value)
 
 
 class SlowCallListSchema(colander.SequenceSchema):
     """
     Validates list of individual slow calls
     """
+
     slow_call = SlowCallSchema()
 
 
 class RequestStatsSchema(colander.MappingSchema):
     """
     Validates format of requests statistics dictionary
     """
-    main = colander.SchemaNode(colander.Float(), validator=colander.Range(0),
-                               missing=0)
-    sql = colander.SchemaNode(colander.Float(), validator=colander.Range(0),
-                              missing=0)
-    nosql = colander.SchemaNode(colander.Float(), validator=colander.Range(0),
-                                missing=0)
-    remote = colander.SchemaNode(colander.Float(), validator=colander.Range(0),
-                                 missing=0)
-    tmpl = colander.SchemaNode(colander.Float(), validator=colander.Range(0),
-                               missing=0)
-    custom = colander.SchemaNode(colander.Float(), validator=colander.Range(0),
-                                 missing=0)
-    sql_calls = colander.SchemaNode(colander.Float(),
-                                    validator=colander.Range(0),
-                                    missing=0)
-    nosql_calls = colander.SchemaNode(colander.Float(),
-                                      validator=colander.Range(0),
-                                      missing=0)
-    remote_calls = colander.SchemaNode(colander.Float(),
-                                       validator=colander.Range(0),
-                                       missing=0)
-    tmpl_calls = colander.SchemaNode(colander.Float(),
-                                     validator=colander.Range(0),
-                                     missing=0)
-    custom_calls = colander.SchemaNode(colander.Float(),
-                                       validator=colander.Range(0),
-                                       missing=0)
+
+    main = colander.SchemaNode(colander.Float(), validator=colander.Range(0), missing=0)
+    sql = colander.SchemaNode(colander.Float(), validator=colander.Range(0), missing=0)
+    nosql = colander.SchemaNode(
+        colander.Float(), validator=colander.Range(0), missing=0
+    )
+    remote = colander.SchemaNode(
+        colander.Float(), validator=colander.Range(0), missing=0
+    )
+    tmpl = colander.SchemaNode(colander.Float(), validator=colander.Range(0), missing=0)
+    custom = colander.SchemaNode(
+        colander.Float(), validator=colander.Range(0), missing=0
+    )
+    sql_calls = colander.SchemaNode(
+        colander.Float(), validator=colander.Range(0), missing=0
+    )
+    nosql_calls = colander.SchemaNode(
+        colander.Float(), validator=colander.Range(0), missing=0
+    )
+    remote_calls = colander.SchemaNode(
+        colander.Float(), validator=colander.Range(0), missing=0
+    )
+    tmpl_calls = colander.SchemaNode(
+        colander.Float(), validator=colander.Range(0), missing=0
+    )
+    custom_calls = colander.SchemaNode(
+        colander.Float(), validator=colander.Range(0), missing=0
+    )
 
 
 class FrameInfoVarSchema(colander.SequenceSchema):
     """
     Validates format of frame variables of a traceback
     """
-    vars = colander.SchemaNode(UnknownType(),
-                               validator=colander.Length(2, 2))
+
+    vars = colander.SchemaNode(UnknownType(), validator=colander.Length(2, 2))
 
 
 class FrameInfoSchema(colander.MappingSchema):
     """
     Validates format of a traceback line
     """
-    cline = colander.SchemaNode(colander.String(), missing='')
-    module = colander.SchemaNode(colander.String(), missing='')
-    line = colander.SchemaNode(colander.String(), missing='')
-    file = colander.SchemaNode(colander.String(), missing='')
-    fn = colander.SchemaNode(colander.String(), missing='')
+
+    cline = colander.SchemaNode(colander.String(), missing="")
+    module = colander.SchemaNode(colander.String(), missing="")
+    line = colander.SchemaNode(colander.String(), missing="")
+    file = colander.SchemaNode(colander.String(), missing="")
+    fn = colander.SchemaNode(colander.String(), missing="")
     vars = FrameInfoVarSchema()
 
 
 class FrameInfoListSchema(colander.SequenceSchema):
     """
     Validates format of list of traceback lines
     """
+
     frame = colander.SchemaNode(UnknownType())
 
 
 class ReportDetailBaseSchema(colander.MappingSchema):
     """
     Validates format of report - ie. request parameters and stats for a request in report group
     """
-    username = colander.SchemaNode(colander.String(),
-                                   preparer=[shortener_factory(255),
-                                             lambda x: x or ''],
-                                   missing='')
-    request_id = colander.SchemaNode(colander.String(),
-                                     preparer=shortener_factory(40),
-                                     missing='')
-    url = colander.SchemaNode(colander.String(),
-                              preparer=shortener_factory(1024), missing='')
-    ip = colander.SchemaNode(colander.String(), preparer=shortener_factory(39),
-                             missing=None)
-    start_time = colander.SchemaNode(NonTZDate(),
-                                     validator=optional_limited_date,
-                                     missing=deferred_utcnow)
-    end_time = colander.SchemaNode(NonTZDate(),
-                                   validator=optional_limited_date,
-                                   missing=None)
-    user_agent = colander.SchemaNode(colander.String(),
-                                     preparer=[shortener_factory(512),
-                                               lambda x: x or ''],
-                                     missing='')
-    message = colander.SchemaNode(colander.String(),
-                                  preparer=shortener_factory(2048),
-                                  missing='')
-    group_string = colander.SchemaNode(colander.String(),
-                                       preparer=shortener_factory(512),
-                                       missing=None)
+
+    username = colander.SchemaNode(
+        colander.String(),
+        preparer=[shortener_factory(255), lambda x: x or ""],
+        missing="",
+    )
+    request_id = colander.SchemaNode(
+        colander.String(), preparer=shortener_factory(40), missing=""
+    )
+    url = colander.SchemaNode(
+        colander.String(), preparer=shortener_factory(1024), missing=""
+    )
+    ip = colander.SchemaNode(
+        colander.String(), preparer=shortener_factory(39), missing=None
+    )
+    start_time = colander.SchemaNode(
+        NonTZDate(), validator=optional_limited_date, missing=deferred_utcnow
+    )
+    end_time = colander.SchemaNode(
+        NonTZDate(), validator=optional_limited_date, missing=None
+    )
+    user_agent = colander.SchemaNode(
+        colander.String(),
+        preparer=[shortener_factory(512), lambda x: x or ""],
+        missing="",
+    )
+    message = colander.SchemaNode(
+        colander.String(), preparer=shortener_factory(2048), missing=""
+    )
+    group_string = colander.SchemaNode(
+        colander.String(), preparer=shortener_factory(512), missing=None
+    )
     request_stats = RequestStatsSchema(missing=None)
-    request = colander.SchemaNode(colander.Mapping(unknown='preserve'),
-                                  missing={})
+    request = colander.SchemaNode(colander.Mapping(unknown="preserve"), missing={})
     traceback = FrameInfoListSchema(missing=None)
     slow_calls = SlowCallListSchema(missing=[])
     extra = ExtraSchemaList()
 
 
 class ReportDetailSchema_0_5(ReportDetailBaseSchema):
     pass
 
 
 class ReportDetailSchemaPermissiveDate_0_5(ReportDetailSchema_0_5):
     start_time = colander.SchemaNode(NonTZDate(), missing=deferred_utcnow)
     end_time = colander.SchemaNode(NonTZDate(), missing=None)
 
 
 class ReportSchemaBase(colander.MappingSchema):
     """
     Validates format of report group
     """
-    client = colander.SchemaNode(colander.String(),
-                                 preparer=lambda x: x or 'unknown')
+
+    client = colander.SchemaNode(colander.String(), preparer=lambda x: x or "unknown")
     server = colander.SchemaNode(
         colander.String(),
-        preparer=[
-            lambda x: x.lower() if x else 'unknown', shortener_factory(128)],
-        missing='unknown')
-    priority = colander.SchemaNode(colander.Int(),
-                                   preparer=[lambda x: x or 5],
-                                   validator=colander.Range(1, 10),
-                                   missing=5)
-    language = colander.SchemaNode(colander.String(), missing='unknown')
-    error = colander.SchemaNode(colander.String(),
-                                preparer=shortener_factory(512),
-                                missing='')
-    view_name = colander.SchemaNode(colander.String(),
-                                    preparer=[shortener_factory(128),
-                                              lambda x: x or ''],
-                                    missing='')
-    http_status = colander.SchemaNode(colander.Int(),
-                                      preparer=[lambda x: x or 200],
-                                      validator=colander.Range(1))
-
-    occurences = colander.SchemaNode(colander.Int(),
-                                     validator=colander.Range(1, 99999999999),
-                                     missing=1)
+        preparer=[lambda x: x.lower() if x else "unknown", shortener_factory(128)],
+        missing="unknown",
+    )
+    priority = colander.SchemaNode(
+        colander.Int(),
+        preparer=[lambda x: x or 5],
+        validator=colander.Range(1, 10),
+        missing=5,
+    )
+    language = colander.SchemaNode(colander.String(), missing="unknown")
+    error = colander.SchemaNode(
+        colander.String(), preparer=shortener_factory(512), missing=""
+    )
+    view_name = colander.SchemaNode(
+        colander.String(),
+        preparer=[shortener_factory(128), lambda x: x or ""],
+        missing="",
+    )
+    http_status = colander.SchemaNode(
+        colander.Int(), preparer=[lambda x: x or 200], validator=colander.Range(1)
+    )
+
+    occurences = colander.SchemaNode(
+        colander.Int(), validator=colander.Range(1, 99999999999), missing=1
+    )
     tags = TagSchemaList()
 
 
 class ReportSchema_0_5(ReportSchemaBase, ReportDetailSchema_0_5):
     pass
 
 
-class ReportSchemaPermissiveDate_0_5(ReportSchemaBase,
-                                 ReportDetailSchemaPermissiveDate_0_5):
+class ReportSchemaPermissiveDate_0_5(
+    ReportSchemaBase, ReportDetailSchemaPermissiveDate_0_5
+):
     pass
 
 
 class ReportListSchema_0_5(colander.SequenceSchema):
     """
     Validates format of list of report groups
     """
+
     report = ReportSchema_0_5()
     validator = colander.Length(1)
 
 
 class ReportListPermissiveDateSchema_0_5(colander.SequenceSchema):
     """
     Validates format of list of report groups
     """
+
     report = ReportSchemaPermissiveDate_0_5()
     validator = colander.Length(1)
 
 
 class LogSchema(colander.MappingSchema):
     """
     Validates format if individual log entry
     """
-    primary_key = colander.SchemaNode(UnknownType(),
-                                      preparer=[cast_to_unicode_or_null,
-                                                shortener_factory(128)],
-                                      missing=None)
-    log_level = colander.SchemaNode(colander.String(),
-                                    preparer=shortener_factory(10),
-                                    missing='UNKNOWN')
-    message = colander.SchemaNode(colander.String(),
-                                  preparer=shortener_factory(4096),
-                                  missing='')
-    namespace = colander.SchemaNode(colander.String(),
-                                    preparer=shortener_factory(128),
-                                    missing='')
-    request_id = colander.SchemaNode(colander.String(),
-                                     preparer=shortener_factory(40),
-                                     missing='')
-    server = colander.SchemaNode(colander.String(),
-                                 preparer=shortener_factory(128),
-                                 missing='unknown')
-    date = colander.SchemaNode(NonTZDate(),
-                               validator=limited_date,
-                               missing=deferred_utcnow)
+
+    primary_key = colander.SchemaNode(
+        UnknownType(),
+        preparer=[cast_to_unicode_or_null, shortener_factory(128)],
+        missing=None,
+    )
+    log_level = colander.SchemaNode(
+        colander.String(), preparer=shortener_factory(10), missing="UNKNOWN"
+    )
+    message = colander.SchemaNode(
+        colander.String(), preparer=shortener_factory(4096), missing=""
+    )
+    namespace = colander.SchemaNode(
+        colander.String(), preparer=shortener_factory(128), missing=""
+    )
+    request_id = colander.SchemaNode(
+        colander.String(), preparer=shortener_factory(40), missing=""
+    )
+    server = colander.SchemaNode(
+        colander.String(), preparer=shortener_factory(128), missing="unknown"
+    )
+    date = colander.SchemaNode(
+        NonTZDate(), validator=limited_date, missing=deferred_utcnow
+    )
     tags = TagSchemaList()
 
 
 class LogSchemaPermanent(LogSchema):
-    date = colander.SchemaNode(NonTZDate(),
-                               missing=deferred_utcnow)
+    date = colander.SchemaNode(NonTZDate(), missing=deferred_utcnow)
     permanent = colander.SchemaNode(colander.Boolean(), missing=False)
 
 
 class LogListSchema(colander.SequenceSchema):
     """
     Validates format of list of log entries
     """
+
     log = LogSchema()
     validator = colander.Length(1)
 
 
 class LogListPermanentSchema(colander.SequenceSchema):
     """
     Validates format of list of log entries
     """
+
     log = LogSchemaPermanent()
     validator = colander.Length(1)
 
 
 class ViewRequestStatsSchema(RequestStatsSchema):
-    requests = colander.SchemaNode(colander.Integer(),
-                                   validator=colander.Range(0),
-                                   missing=0)
+    requests = colander.SchemaNode(
+        colander.Integer(), validator=colander.Range(0), missing=0
+    )
 
 
 class ViewMetricTupleSchema(colander.TupleSchema):
     """
     Validates list of views and their corresponding request stats object ie:
     ["dir/module:func",{"custom": 0.0..}]
     """
-    view_name = colander.SchemaNode(colander.String(),
-                                    preparer=[shortener_factory(128),
-                                              lambda x: x or 'unknown'],
-                                    missing='unknown')
+
+    view_name = colander.SchemaNode(
+        colander.String(),
+        preparer=[shortener_factory(128), lambda x: x or "unknown"],
+        missing="unknown",
+    )
     metrics = ViewRequestStatsSchema()
 
 
 class ViewMetricListSchema(colander.SequenceSchema):
     """
     Validates view breakdown stats objects list
     {metrics key of server/time object}
     """
+
     view_tuple = ViewMetricTupleSchema()
     validator = colander.Length(1)
 
 
 class ViewMetricSchema(colander.MappingSchema):
     """
     Validates server/timeinterval object, ie:
     {server/time object}
 
     """
-    timestamp = colander.SchemaNode(NonTZDate(),
-                                    validator=limited_date,
-                                    missing=None)
-    server = colander.SchemaNode(colander.String(),
-                                 preparer=[shortener_factory(128),
-                                           lambda x: x or 'unknown'],
-                                 missing='unknown')
+
+    timestamp = colander.SchemaNode(NonTZDate(), validator=limited_date, missing=None)
+    server = colander.SchemaNode(
+        colander.String(),
+        preparer=[shortener_factory(128), lambda x: x or "unknown"],
+        missing="unknown",
+    )
     metrics = ViewMetricListSchema()
 
 
 class GeneralMetricSchema(colander.MappingSchema):
     """
     Validates universal metric schema
 
     """
-    namespace = colander.SchemaNode(colander.String(), missing='',
-                                    preparer=shortener_factory(128))
 
-    server_name = colander.SchemaNode(colander.String(),
-                                      preparer=[shortener_factory(128),
-                                                lambda x: x or 'unknown'],
-                                      missing='unknown')
-    timestamp = colander.SchemaNode(NonTZDate(), validator=limited_date,
-                                    missing=deferred_utcnow)
+    namespace = colander.SchemaNode(
+        colander.String(), missing="", preparer=shortener_factory(128)
+    )
+
+    server_name = colander.SchemaNode(
+        colander.String(),
+        preparer=[shortener_factory(128), lambda x: x or "unknown"],
+        missing="unknown",
+    )
+    timestamp = colander.SchemaNode(
+        NonTZDate(), validator=limited_date, missing=deferred_utcnow
+    )
     tags = TagSchemaList(missing=colander.required)
 
 
 class GeneralMetricPermanentSchema(GeneralMetricSchema):
     """
     Validates universal metric schema
 
     """
+
     timestamp = colander.SchemaNode(NonTZDate(), missing=deferred_utcnow)
 
 
 class GeneralMetricsListSchema(colander.SequenceSchema):
     metric = GeneralMetricSchema()
     validator = colander.Length(1)
 
 
 class GeneralMetricsPermanentListSchema(colander.SequenceSchema):
     metric = GeneralMetricPermanentSchema()
     validator = colander.Length(1)
 
 
 class MetricsListSchema(colander.SequenceSchema):
     """
     Validates list of metrics objects ie:
     [{server/time object}, ] part
 
 
     """
+
     metric = ViewMetricSchema()
     validator = colander.Length(1)
 
 
 class StringToAppList(object):
     """
     Returns validated list of application ids from user query and
     set of applications user is allowed to look at
     transform string to list containing single integer
     """
 
     def serialize(self, node, appstruct):
         if appstruct is null:
             return null
         return appstruct
 
     def deserialize(self, node, cstruct):
         if cstruct is null:
             return null
 
-        apps = set([int(a) for a in node.bindings['resources']])
+        apps = set([int(a) for a in node.bindings["resources"]])
 
         if isinstance(cstruct, str):
             cstruct = [cstruct]
 
         cstruct = [int(a) for a in cstruct]
 
         valid_apps = list(apps.intersection(set(cstruct)))
         if valid_apps:
             return valid_apps
         return null
 
     def cstruct_children(self):
         return []
 
 
 @colander.deferred
 def possible_applications_validator(node, kw):
-    possible_apps = [int(a) for a in kw['resources']]
-    return colander.All(colander.ContainsOnly(possible_apps),
-                        colander.Length(1))
+    possible_apps = [int(a) for a in kw["resources"]]
+    return colander.All(colander.ContainsOnly(possible_apps), colander.Length(1))
 
 
 @colander.deferred
 def possible_applications(node, kw):
-    return [int(a) for a in kw['resources']]
+    return [int(a) for a in kw["resources"]]
 
 
 @colander.deferred
 def today_start(node, kw):
-    return datetime.datetime.utcnow().replace(second=0, microsecond=0,
-                                              minute=0,
-                                              hour=0)
+    return datetime.datetime.utcnow().replace(second=0, microsecond=0, minute=0, hour=0)
 
 
 @colander.deferred
 def today_end(node, kw):
-    return datetime.datetime.utcnow().replace(second=0, microsecond=0,
-                                              minute=59, hour=23)
+    return datetime.datetime.utcnow().replace(
+        second=0, microsecond=0, minute=59, hour=23
+    )
 
 
 @colander.deferred
 def old_start(node, kw):
     t_delta = datetime.timedelta(days=90)
-    return datetime.datetime.utcnow().replace(second=0, microsecond=0,
-                                              minute=0,
-                                              hour=0) - t_delta
+    return (
+        datetime.datetime.utcnow().replace(second=0, microsecond=0, minute=0, hour=0)
+        - t_delta
+    )
 
 
 @colander.deferred
 def today_end(node, kw):
-    return datetime.datetime.utcnow().replace(second=0, microsecond=0,
-                                              minute=59, hour=23)
+    return datetime.datetime.utcnow().replace(
+        second=0, microsecond=0, minute=59, hour=23
+    )
 
 
 class PermissiveDate(colander.DateTime):
     """ Returns null for incorrect date format - also removes tz info"""
 
     def deserialize(self, node, cstruct):
         if not cstruct:
             return null
 
         try:
             result = colander.iso8601.parse_date(
-                cstruct, default_timezone=self.default_tzinfo)
+                cstruct, default_timezone=self.default_tzinfo
+            )
         except colander.iso8601.ParseError:
             return null
         return result.replace(tzinfo=None)
 
 
 class LogSearchSchema(colander.MappingSchema):
     def schema_type(self, **kw):
-        return colander.Mapping(unknown='preserve')
-
-    resource = colander.SchemaNode(StringToAppList(),
-                                   validator=possible_applications_validator,
-                                   missing=possible_applications)
-
-    message = colander.SchemaNode(colander.Sequence(accept_scalar=True),
-                                  colander.SchemaNode(colander.String()),
-                                  missing=None)
-    level = colander.SchemaNode(colander.Sequence(accept_scalar=True),
-                                colander.SchemaNode(colander.String()),
-                                preparer=lowercase_preparer,
-                                missing=None)
-    namespace = colander.SchemaNode(colander.Sequence(accept_scalar=True),
-                                    colander.SchemaNode(colander.String()),
-                                    preparer=lowercase_preparer,
-                                    missing=None)
-    request_id = colander.SchemaNode(colander.Sequence(accept_scalar=True),
-                                     colander.SchemaNode(colander.String()),
-                                     preparer=lowercase_preparer,
-                                     missing=None)
-    start_date = colander.SchemaNode(PermissiveDate(),
-                                     missing=None)
-    end_date = colander.SchemaNode(PermissiveDate(),
-                                   missing=None)
-    page = colander.SchemaNode(colander.Integer(),
-                               validator=colander.Range(min=1),
-                               missing=1)
+        return colander.Mapping(unknown="preserve")
+
+    resource = colander.SchemaNode(
+        StringToAppList(),
+        validator=possible_applications_validator,
+        missing=possible_applications,
+    )
+
+    message = colander.SchemaNode(
+        colander.Sequence(accept_scalar=True),
+        colander.SchemaNode(colander.String()),
+        missing=None,
+    )
+    level = colander.SchemaNode(
+        colander.Sequence(accept_scalar=True),
+        colander.SchemaNode(colander.String()),
+        preparer=lowercase_preparer,
+        missing=None,
+    )
+    namespace = colander.SchemaNode(
+        colander.Sequence(accept_scalar=True),
+        colander.SchemaNode(colander.String()),
+        preparer=lowercase_preparer,
+        missing=None,
+    )
+    request_id = colander.SchemaNode(
+        colander.Sequence(accept_scalar=True),
+        colander.SchemaNode(colander.String()),
+        preparer=lowercase_preparer,
+        missing=None,
+    )
+    start_date = colander.SchemaNode(PermissiveDate(), missing=None)
+    end_date = colander.SchemaNode(PermissiveDate(), missing=None)
+    page = colander.SchemaNode(
+        colander.Integer(), validator=colander.Range(min=1), missing=1
+    )
 
 
 class ReportSearchSchema(colander.MappingSchema):
     def schema_type(self, **kw):
-        return colander.Mapping(unknown='preserve')
-
-    resource = colander.SchemaNode(StringToAppList(),
-                                   validator=possible_applications_validator,
-                                   missing=possible_applications)
-    request_id = colander.SchemaNode(colander.Sequence(accept_scalar=True),
-                                     colander.SchemaNode(colander.String()),
-                                     missing=None)
-    start_date = colander.SchemaNode(PermissiveDate(),
-                                     missing=None)
-    end_date = colander.SchemaNode(PermissiveDate(),
-                                   missing=None)
-    page = colander.SchemaNode(colander.Integer(),
-                               validator=colander.Range(min=1),
-                               missing=1)
+        return colander.Mapping(unknown="preserve")
+
+    resource = colander.SchemaNode(
+        StringToAppList(),
+        validator=possible_applications_validator,
+        missing=possible_applications,
+    )
+    request_id = colander.SchemaNode(
+        colander.Sequence(accept_scalar=True),
+        colander.SchemaNode(colander.String()),
+        missing=None,
+    )
+    start_date = colander.SchemaNode(PermissiveDate(), missing=None)
+    end_date = colander.SchemaNode(PermissiveDate(), missing=None)
+    page = colander.SchemaNode(
+        colander.Integer(), validator=colander.Range(min=1), missing=1
+    )
 
     min_occurences = colander.SchemaNode(
         colander.Sequence(accept_scalar=True),
         colander.SchemaNode(colander.Integer()),
-        missing=None)
-
-    http_status = colander.SchemaNode(colander.Sequence(accept_scalar=True),
-                                      colander.SchemaNode(colander.Integer()),
-                                      missing=None)
-    priority = colander.SchemaNode(colander.Sequence(accept_scalar=True),
-                                   colander.SchemaNode(colander.Integer()),
-                                   missing=None)
-    error = colander.SchemaNode(colander.Sequence(accept_scalar=True),
-                                colander.SchemaNode(colander.String()),
-                                missing=None)
-    url_path = colander.SchemaNode(colander.Sequence(accept_scalar=True),
-                                   colander.SchemaNode(colander.String()),
-                                   missing=None)
-    url_domain = colander.SchemaNode(colander.Sequence(accept_scalar=True),
-                                     colander.SchemaNode(colander.String()),
-                                     missing=None)
-    report_status = colander.SchemaNode(colander.Sequence(accept_scalar=True),
-                                        colander.SchemaNode(colander.String()),
-                                        missing=None)
-    min_duration = colander.SchemaNode(colander.Sequence(accept_scalar=True),
-                                       colander.SchemaNode(colander.Float()),
-                                       missing=None)
-    max_duration = colander.SchemaNode(colander.Sequence(accept_scalar=True),
-                                       colander.SchemaNode(colander.Float()),
-                                       missing=None)
+        missing=None,
+    )
+
+    http_status = colander.SchemaNode(
+        colander.Sequence(accept_scalar=True),
+        colander.SchemaNode(colander.Integer()),
+        missing=None,
+    )
+    priority = colander.SchemaNode(
+        colander.Sequence(accept_scalar=True),
+        colander.SchemaNode(colander.Integer()),
+        missing=None,
+    )
+    error = colander.SchemaNode(
+        colander.Sequence(accept_scalar=True),
+        colander.SchemaNode(colander.String()),
+        missing=None,
+    )
+    url_path = colander.SchemaNode(
+        colander.Sequence(accept_scalar=True),
+        colander.SchemaNode(colander.String()),
+        missing=None,
+    )
+    url_domain = colander.SchemaNode(
+        colander.Sequence(accept_scalar=True),
+        colander.SchemaNode(colander.String()),
+        missing=None,
+    )
+    report_status = colander.SchemaNode(
+        colander.Sequence(accept_scalar=True),
+        colander.SchemaNode(colander.String()),
+        missing=None,
+    )
+    min_duration = colander.SchemaNode(
+        colander.Sequence(accept_scalar=True),
+        colander.SchemaNode(colander.Float()),
+        missing=None,
+    )
+    max_duration = colander.SchemaNode(
+        colander.Sequence(accept_scalar=True),
+        colander.SchemaNode(colander.Float()),
+        missing=None,
+    )
 
 
 class TagSchema(colander.MappingSchema):
     """
     Used in log search
     """
-    name = colander.SchemaNode(colander.String(),
-                               validator=colander.Length(1, 32))
-    value = colander.SchemaNode(colander.Sequence(accept_scalar=True),
-                                colander.SchemaNode(colander.String(),
-                                                    validator=colander.Length(
-                                                        1, 128)),
-                                missing=None)
-    op = colander.SchemaNode(colander.String(),
-                             validator=colander.Length(1, 128),
-                             missing=None)
+
+    name = colander.SchemaNode(colander.String(), validator=colander.Length(1, 32))
+    value = colander.SchemaNode(
+        colander.Sequence(accept_scalar=True),
+        colander.SchemaNode(colander.String(), validator=colander.Length(1, 128)),
+        missing=None,
+    )
+    op = colander.SchemaNode(
+        colander.String(), validator=colander.Length(1, 128), missing=None
+    )
 
 
 class TagListSchema(colander.SequenceSchema):
     tag = TagSchema()
 
 
 class RuleFieldType(object):
     """ Validator which succeeds if the value passed to it is one of
     a fixed set of values """
 
     def __init__(self, cast_to):
         self.cast_to = cast_to
 
     def __call__(self, node, value):
         try:
-            if self.cast_to == 'int':
+            if self.cast_to == "int":
                 int(value)
-            elif self.cast_to == 'float':
+            elif self.cast_to == "float":
                 float(value)
-            elif self.cast_to == 'unicode':
+            elif self.cast_to == "unicode":
                 str(value)
         except:
-            raise colander.Invalid(node,
-                                   "Can't cast {} to {}".format(
-                                       value, self.cast_to))
+            raise colander.Invalid(
+                node, "Can't cast {} to {}".format(value, self.cast_to)
+            )
 
 
 def build_rule_schema(ruleset, check_matrix):
     """
     Accepts ruleset and a map of fields/possible operations and builds
     validation class
     """
 
     schema = colander.SchemaNode(colander.Mapping())
-    schema.add(colander.SchemaNode(colander.String(), name='field'))
+    schema.add(colander.SchemaNode(colander.String(), name="field"))
 
-    if ruleset['field'] in ['__AND__', '__OR__', '__NOT__']:
-        subrules = colander.SchemaNode(colander.Tuple(), name='rules')
-        for rule in ruleset['rules']:
+    if ruleset["field"] in ["__AND__", "__OR__", "__NOT__"]:
+        subrules = colander.SchemaNode(colander.Tuple(), name="rules")
+        for rule in ruleset["rules"]:
             subrules.add(build_rule_schema(rule, check_matrix))
         schema.add(subrules)
     else:
-        op_choices = check_matrix[ruleset['field']]['ops']
-        cast_to = check_matrix[ruleset['field']]['type']
-        schema.add(colander.SchemaNode(colander.String(),
-                                       validator=colander.OneOf(op_choices),
-                                       name='op'))
-
-        schema.add(colander.SchemaNode(colander.String(),
-                                       name='value',
-                                       validator=RuleFieldType(cast_to)))
+        op_choices = check_matrix[ruleset["field"]]["ops"]
+        cast_to = check_matrix[ruleset["field"]]["type"]
+        schema.add(
+            colander.SchemaNode(
+                colander.String(), validator=colander.OneOf(op_choices), name="op"
+            )
+        )
+
+        schema.add(
+            colander.SchemaNode(
+                colander.String(), name="value", validator=RuleFieldType(cast_to)
+            )
+        )
     return schema
 
 
 class ConfigTypeSchema(colander.MappingSchema):
     type = colander.SchemaNode(colander.String(), missing=None)
     config = colander.SchemaNode(UnknownType(), missing=None)
 
 
 class MappingListSchema(colander.SequenceSchema):
     config = colander.SchemaNode(UnknownType())

backend/src/appenlight/views/__init__.py

@@ -1,241 +1,301 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 """View handlers package.
 """
 from pyramid.response import Response
 import logging
 import simplejson as json
 from appenlight.lib import helpers
 
 log = logging.getLogger(__name__)
 
 
 def includeme(config):
     """Add the application's view handlers.
     """
 
-    config.add_route('/', '/')
-    config.add_route('angular_app_ui_ix', '/ui')
-    config.add_route('angular_app_ui', '/ui/*remainder')
+    config.add_route("/", "/")
+    config.add_route("angular_app_ui_ix", "/ui")
+    config.add_route("angular_app_ui", "/ui/*remainder")
 
     # applications API
-    config.add_route('applications_no_id', '/applications')
-    config.add_route('applications', '/applications/{resource_id}',
-                     factory='appenlight.security.ResourceFactory')
-    config.add_route('applications_property',
-                     '/applications/{resource_id}/{key}',
-                     factory='appenlight.security.ResourceFactory')
+    config.add_route("applications_no_id", "/applications")
     config.add_route(
-        'integrations_id',
-        '/applications/{resource_id}/integrations/{integration}/{action}',
-        factory='appenlight.security.ResourceFactory')
+        "applications",
+        "/applications/{resource_id}",
+        factory="appenlight.security.ResourceFactory",
+    )
+    config.add_route(
+        "applications_property",
+        "/applications/{resource_id}/{key}",
+        factory="appenlight.security.ResourceFactory",
+    )
+    config.add_route(
+        "integrations_id",
+        "/applications/{resource_id}/integrations/{integration}/{action}",
+        factory="appenlight.security.ResourceFactory",
+    )
 
     # users API
-    config.add_route('users_self', '/users/self')
-    config.add_route('users_self_property', '/users/self/{key}')
-    config.add_route('users_no_id', '/users')
-    config.add_route('users', '/users/{user_id}')
-    config.add_route('users_property', '/users/{user_id}/{key}')
+    config.add_route("users_self", "/users/self")
+    config.add_route("users_self_property", "/users/self/{key}")
+    config.add_route("users_no_id", "/users")
+    config.add_route("users", "/users/{user_id}")
+    config.add_route("users_property", "/users/{user_id}/{key}")
 
     # events
-    config.add_route('events_no_id', '/events')
-    config.add_route('events', '/events/{event_id}')
-    config.add_route('events_property', '/events/{event_id}/{key}')
+    config.add_route("events_no_id", "/events")
+    config.add_route("events", "/events/{event_id}")
+    config.add_route("events_property", "/events/{event_id}/{key}")
 
     # groups
-    config.add_route('groups_no_id', '/groups')
-    config.add_route('groups', '/groups/{group_id}')
-    config.add_route('groups_property', '/groups/{group_id}/{key}')
+    config.add_route("groups_no_id", "/groups")
+    config.add_route("groups", "/groups/{group_id}")
+    config.add_route("groups_property", "/groups/{group_id}/{key}")
 
     # reports API
-    config.add_route('reports', '/reports')
-    config.add_route('slow_reports', '/slow_reports')
-    config.add_route('report_groups', '/report_groups/{group_id}',
-                     factory='appenlight.security.ResourceReportFactory')
-    config.add_route('report_groups_property',
-                     '/report_groups/{group_id}/{key}',
-                     factory='appenlight.security.ResourceReportFactory')
-
-    #generic resource API
-    config.add_route('resources_property',
-                     '/resources/{resource_id}/{key}',
-                     factory='appenlight.security.ResourceFactory')
+    config.add_route("reports", "/reports")
+    config.add_route("slow_reports", "/slow_reports")
+    config.add_route(
+        "report_groups",
+        "/report_groups/{group_id}",
+        factory="appenlight.security.ResourceReportFactory",
+    )
+    config.add_route(
+        "report_groups_property",
+        "/report_groups/{group_id}/{key}",
+        factory="appenlight.security.ResourceReportFactory",
+    )
+
+    # generic resource API
+    config.add_route(
+        "resources_property",
+        "/resources/{resource_id}/{key}",
+        factory="appenlight.security.ResourceFactory",
+    )
 
     # plugin configs API
-    config.add_route('plugin_configs', '/plugin_configs/{plugin_name}',
-                     factory='appenlight.security.ResourcePluginMixedFactory')
-    config.add_route('plugin_config', '/plugin_configs/{plugin_name}/{id}',
-                     factory='appenlight.security.ResourcePluginConfigFactory')
+    config.add_route(
+        "plugin_configs",
+        "/plugin_configs/{plugin_name}",
+        factory="appenlight.security.ResourcePluginMixedFactory",
+    )
+    config.add_route(
+        "plugin_config",
+        "/plugin_configs/{plugin_name}/{id}",
+        factory="appenlight.security.ResourcePluginConfigFactory",
+    )
 
     # client endpoints API
-    config.add_route('api_reports', '/api/reports',
-                     factory='appenlight.security.APIFactory')
-    config.add_route('api_report', '/api/report',
-                     factory='appenlight.security.APIFactory')
-    config.add_route('api_logs', '/api/logs',
-                     factory='appenlight.security.APIFactory')
-    config.add_route('api_log', '/api/log',
-                     factory='appenlight.security.APIFactory')
-    config.add_route('api_slow_reports', '/api/slow_reports',
-                     factory='appenlight.security.APIFactory')
-    config.add_route('api_request_stats', '/api/request_stats',
-                     factory='appenlight.security.APIFactory')
-    config.add_route('api_metrics', '/api/metrics',
-                     factory='appenlight.security.APIFactory')
-    config.add_route('api_general_metrics', '/api/general_metrics',
-                     factory='appenlight.security.APIFactory')
-    config.add_route('api_general_metric', '/api/general_metric',
-                     factory='appenlight.security.APIFactory')
-    config.add_route('api_airbrake', '/notifier_api/v2/{action}',
-                     factory='appenlight.security.AirbrakeV2APIFactory')
-    config.add_route('api_sentry', '/api/{project}/store',
-                     factory='appenlight.security.SentryAPIFactory')
-    config.add_route('api_sentry_slash', '/api/{project}/store/',
-                     factory='appenlight.security.SentryAPIFactory')
+    config.add_route(
+        "api_reports", "/api/reports", factory="appenlight.security.APIFactory"
+    )
+    config.add_route(
+        "api_report", "/api/report", factory="appenlight.security.APIFactory"
+    )
+    config.add_route("api_logs", "/api/logs", factory="appenlight.security.APIFactory")
+    config.add_route("api_log", "/api/log", factory="appenlight.security.APIFactory")
+    config.add_route(
+        "api_slow_reports",
+        "/api/slow_reports",
+        factory="appenlight.security.APIFactory",
+    )
+    config.add_route(
+        "api_request_stats",
+        "/api/request_stats",
+        factory="appenlight.security.APIFactory",
+    )
+    config.add_route(
+        "api_metrics", "/api/metrics", factory="appenlight.security.APIFactory"
+    )
+    config.add_route(
+        "api_general_metrics",
+        "/api/general_metrics",
+        factory="appenlight.security.APIFactory",
+    )
+    config.add_route(
+        "api_general_metric",
+        "/api/general_metric",
+        factory="appenlight.security.APIFactory",
+    )
+    config.add_route(
+        "api_airbrake",
+        "/notifier_api/v2/{action}",
+        factory="appenlight.security.AirbrakeV2APIFactory",
+    )
+    config.add_route(
+        "api_sentry",
+        "/api/{project}/store",
+        factory="appenlight.security.SentryAPIFactory",
+    )
+    config.add_route(
+        "api_sentry_slash",
+        "/api/{project}/store/",
+        factory="appenlight.security.SentryAPIFactory",
+    )
 
     # other
-    config.add_route('register', '/register')
-    config.add_route('register_ajax', '/register_ajax')
-    config.add_route('lost_password', '/lost_password')
-    config.add_route('lost_password_generate', '/lost_password_generate')
-    config.add_route('logs_no_id', '/logs')
-    config.add_route('forbidden', '/forbidden')
-    config.add_route('test', '/test/{action}')
-    config.add_route('section_view', '/sections/{section}/{view}')
-
-    config.add_view('appenlight.views.forbidden_view',
-                    context='pyramid.exceptions.Forbidden',
-                    renderer='appenlight:templates/forbidden.jinja2',
-                    permission='__no_permission_required__')
-    config.add_view('appenlight.views.not_found_view',
-                    context='pyramid.exceptions.NotFound',
-                    renderer='appenlight:templates/not_found.jinja2',
-                    permission='__no_permission_required__')
-    config.add_view('appenlight.views.csrf_view',
-                    context='appenlight.lib.request.CSRFException',
-                    renderer='appenlight:templates/forbidden.jinja2',
-                    permission='__no_permission_required__')
-    config.add_view('appenlight.views.csrf_view',
-                    context='appenlight.forms.CSRFException',
-                    renderer='appenlight:templates/forbidden.jinja2',
-                    permission='__no_permission_required__')
-    config.add_view('appenlight.views.colander_invalid_view',
-                    context='colander.Invalid',
-                    renderer='json',
-                    permission='__no_permission_required__')
-    config.add_view('appenlight.views.bad_json_view',
-                    context='appenlight.lib.request.JSONException',
-                    renderer='json',
-                    permission='__no_permission_required__')
+    config.add_route("register", "/register")
+    config.add_route("register_ajax", "/register_ajax")
+    config.add_route("lost_password", "/lost_password")
+    config.add_route("lost_password_generate", "/lost_password_generate")
+    config.add_route("logs_no_id", "/logs")
+    config.add_route("forbidden", "/forbidden")
+    config.add_route("test", "/test/{action}")
+    config.add_route("section_view", "/sections/{section}/{view}")
+
+    config.add_view(
+        "appenlight.views.forbidden_view",
+        context="pyramid.exceptions.Forbidden",
+        renderer="appenlight:templates/forbidden.jinja2",
+        permission="__no_permission_required__",
+    )
+    config.add_view(
+        "appenlight.views.not_found_view",
+        context="pyramid.exceptions.NotFound",
+        renderer="appenlight:templates/not_found.jinja2",
+        permission="__no_permission_required__",
+    )
+    config.add_view(
+        "appenlight.views.csrf_view",
+        context="appenlight.lib.request.CSRFException",
+        renderer="appenlight:templates/forbidden.jinja2",
+        permission="__no_permission_required__",
+    )
+    config.add_view(
+        "appenlight.views.csrf_view",
+        context="appenlight.forms.CSRFException",
+        renderer="appenlight:templates/forbidden.jinja2",
+        permission="__no_permission_required__",
+    )
+    config.add_view(
+        "appenlight.views.colander_invalid_view",
+        context="colander.Invalid",
+        renderer="json",
+        permission="__no_permission_required__",
+    )
+    config.add_view(
+        "appenlight.views.bad_json_view",
+        context="appenlight.lib.request.JSONException",
+        renderer="json",
+        permission="__no_permission_required__",
+    )
 
     # handle authomatic
-    config.add_route('social_auth', '/social_auth/{provider}')
-    config.add_route('social_auth_abort', '/social_auth/{provider}/abort')
+    config.add_route("social_auth", "/social_auth/{provider}")
+    config.add_route("social_auth_abort", "/social_auth/{provider}/abort")
 
     # only use in production
-    if (config.registry.settings.get('pyramid.reload_templates') is False
-        and config.registry.settings.get('pyramid.debug_templates') is False):
-        config.add_view('appenlight.views.error_view',
-                        context=Exception,
-                        renderer='appenlight:templates/error.jinja2',
-                        permission='__no_permission_required__')
+    if (
+        config.registry.settings.get("pyramid.reload_templates") is False
+        and config.registry.settings.get("pyramid.debug_templates") is False
+    ):
+        config.add_view(
+            "appenlight.views.error_view",
+            context=Exception,
+            renderer="appenlight:templates/error.jinja2",
+            permission="__no_permission_required__",
+        )
 
 
 def bad_json_view(exc, request):
-    request.environ['appenlight.ignore_error'] = 1
-    request.response.headers.add('X-AppEnlight-Error', 'Incorrect JSON')
+    request.environ["appenlight.ignore_error"] = 1
+    request.response.headers.add("X-AppEnlight-Error", "Incorrect JSON")
     request.response.status_int = 400
     return "Incorrect JSON"
 
 
 def colander_invalid_view(exc, request):
-    request.environ['appenlight.ignore_error'] = 1
-    log.warning('API version %s, %s' % (
-        request.params.get('protocol_version'),
-        request.context.resource))
-    log.warning('Invalid payload sent')
+    request.environ["appenlight.ignore_error"] = 1
+    log.warning(
+        "API version %s, %s"
+        % (request.params.get("protocol_version"), request.context.resource)
+    )
+    log.warning("Invalid payload sent")
     errors = exc.asdict()
-    request.response.headers.add('X-AppEnlight-Error', 'Invalid payload sent')
+    request.response.headers.add("X-AppEnlight-Error", "Invalid payload sent")
     request.response.status_int = 422
     return errors
 
 
 def csrf_view(exc, request):
     request.response.status = 403
     from ..models import DBSession
+
     request.environ["appenlight.ignore_error"] = 1
-    request.response.headers.add('X-AppEnlight-Error', str(exc))
+    request.response.headers.add("X-AppEnlight-Error", str(exc))
     if request.user:
         request.user = DBSession.merge(request.user)
-    return {'forbidden_view': True, 'csrf': True}
+    return {"forbidden_view": True, "csrf": True}
 
 
 def not_found_view(exc, request):
     request.response.status = 404
     from ..models import DBSession
 
     if request.user:
         request.user = DBSession.merge(request.user)
 
     if request.user:
-        request.response.headers['x-appenlight-uid'] = '%s' % request.user.id
-    request.response.headers['x-appenlight-flash'] = json.dumps(
-        helpers.get_flash(request))
+        request.response.headers["x-appenlight-uid"] = "%s" % request.user.id
+    request.response.headers["x-appenlight-flash"] = json.dumps(
+        helpers.get_flash(request)
+    )
 
     return {}
 
 
 def forbidden_view(exc, request):
     # dont serve html for api requests
     from ..models import DBSession
 
     if request.user:
         request.user = DBSession.merge(request.user)
-    if request.path.startswith('/api'):
-        logging.warning('Wrong API Key sent')
+    if request.path.startswith("/api"):
+        logging.warning("Wrong API Key sent")
         logging.info(request.url)
-        logging.info(
-            '\n'.join(
-                ['%s:%s' % (k, v) for k, v in request.headers.items()]))
+        logging.info("\n".join(["%s:%s" % (k, v) for k, v in request.headers.items()]))
         resp = Response(
-            "Wrong api key",
-            headers=(('X-AppEnlight-Error', 'Incorrect API key',),))
+            "Wrong api key", headers=(("X-AppEnlight-Error", "Incorrect API key"),)
+        )
         resp.status_int = 403
         return resp
 
     if request.user:
-        request.response.headers['x-appenlight-uid'] = '%s' % request.user.id
-    request.response.headers['x-appenlight-flash'] = json.dumps(
-        helpers.get_flash(request))
+        request.response.headers["x-appenlight-uid"] = "%s" % request.user.id
+    request.response.headers["x-appenlight-flash"] = json.dumps(
+        helpers.get_flash(request)
+    )
     request.response.status = 403
-    return {'forbidden_view': True}
+    return {"forbidden_view": True}
 
 
 def error_view(exc, request):
     from ..models import DBSession
+
     if request.user:
         request.user = DBSession.merge(request.user)
-    if request.path.startswith('/api'):
+    if request.path.startswith("/api"):
         resp = Response(
             "There was a problem handling your request please try again",
-            headers=(('X-AppEnlight-Error', 'Problem handling request',),)
+            headers=(("X-AppEnlight-Error", "Problem handling request"),),
         )
         resp.status_int = 500
         return resp
     log.error(exc)
     request.response.status = 500
     return {}

backend/src/appenlight/views/admin/__init__.py

@@ -1,31 +1,31 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 """View handlers package.
 """
 import logging
 
 log = logging.getLogger(__name__)
 
 
 def includeme(config):
     """Add the application's view handlers.
     """
-    config.add_route('admin_users_no_id', '/admin/users/')
-    config.add_route('admin_users', '/admin/users/{user_id}')
-    config.add_route('admin', '/admin/{action}')
-    config.add_route('admin_configs', '/configs')
-    config.add_route('admin_config', '/configs/{key}/{section}')
+    config.add_route("admin_users_no_id", "/admin/users/")
+    config.add_route("admin_users", "/admin/users/{user_id}")
+    config.add_route("admin", "/admin/{action}")
+    config.add_route("admin_configs", "/configs")
+    config.add_route("admin_config", "/configs/{key}/{section}")

backend/src/appenlight/views/admin/admin.py

@@ -1,192 +1,203 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 import os
 import pkg_resources
 
 from datetime import datetime, timedelta
 
 import psutil
 import redis
 
 from pyramid.view import view_config
 from appenlight.models import DBSession
 from appenlight.models import Datastores
 from appenlight.lib.redis_keys import REDIS_KEYS
 
 
 def bytes2human(total):
     giga = 1024.0 ** 3
     mega = 1024.0 ** 2
     kilo = 1024.0
     if giga <= total:
-        return '{:0.1f}G'.format(total / giga)
+        return "{:0.1f}G".format(total / giga)
     elif mega <= total:
-        return '{:0.1f}M'.format(total / mega)
+        return "{:0.1f}M".format(total / mega)
     else:
-        return '{:0.1f}K'.format(total / kilo)
+        return "{:0.1f}K".format(total / kilo)
 
 
 log = logging.getLogger(__name__)
 
 
-@view_config(route_name='section_view',
-             match_param=['section=admin_section', 'view=system'],
-             renderer='json', permission='root_administration')
+@view_config(
+    route_name="section_view",
+    match_param=["section=admin_section", "view=system"],
+    renderer="json",
+    permission="root_administration",
+)
 def system(request):
-    current_time = datetime.utcnow(). \
-                       replace(second=0, microsecond=0) - timedelta(minutes=1)
+    current_time = datetime.utcnow().replace(second=0, microsecond=0) - timedelta(
+        minutes=1
+    )
     # global app counter
     processed_reports = request.registry.redis_conn.get(
-        REDIS_KEYS['counters']['reports_per_minute'].format(current_time))
+        REDIS_KEYS["counters"]["reports_per_minute"].format(current_time)
+    )
     processed_reports = int(processed_reports) if processed_reports else 0
     processed_logs = request.registry.redis_conn.get(
-        REDIS_KEYS['counters']['logs_per_minute'].format(current_time))
+        REDIS_KEYS["counters"]["logs_per_minute"].format(current_time)
+    )
     processed_logs = int(processed_logs) if processed_logs else 0
     processed_metrics = request.registry.redis_conn.get(
-        REDIS_KEYS['counters']['metrics_per_minute'].format(current_time))
+        REDIS_KEYS["counters"]["metrics_per_minute"].format(current_time)
+    )
     processed_metrics = int(processed_metrics) if processed_metrics else 0
 
     waiting_reports = 0
     waiting_logs = 0
     waiting_metrics = 0
     waiting_other = 0
 
-    if 'redis' in request.registry.settings['celery.broker_type']:
+    if "redis" in request.registry.settings["celery.broker_type"]:
         redis_client = redis.StrictRedis.from_url(
-            request.registry.settings['celery.broker_url'])
-        waiting_reports = redis_client.llen('reports')
-        waiting_logs = redis_client.llen('logs')
-        waiting_metrics = redis_client.llen('metrics')
-        waiting_other = redis_client.llen('default')
+            request.registry.settings["celery.broker_url"]
+        )
+        waiting_reports = redis_client.llen("reports")
+        waiting_logs = redis_client.llen("logs")
+        waiting_metrics = redis_client.llen("metrics")
+        waiting_other = redis_client.llen("default")
 
     # process
     def replace_inf(val):
-        return val if val != psutil.RLIM_INFINITY else 'unlimited'
+        return val if val != psutil.RLIM_INFINITY else "unlimited"
 
     p = psutil.Process()
     fd = p.rlimit(psutil.RLIMIT_NOFILE)
     memlock = p.rlimit(psutil.RLIMIT_MEMLOCK)
     self_info = {
-        'fds': {'soft': replace_inf(fd[0]),
-                'hard': replace_inf(fd[1])},
-        'memlock': {'soft': replace_inf(memlock[0]),
-                    'hard': replace_inf(memlock[1])},
+        "fds": {"soft": replace_inf(fd[0]), "hard": replace_inf(fd[1])},
+        "memlock": {"soft": replace_inf(memlock[0]), "hard": replace_inf(memlock[1])},
     }
 
     # disks
     disks = []
     for part in psutil.disk_partitions(all=False):
-        if os.name == 'nt':
-            if 'cdrom' in part.opts or part.fstype == '':
+        if os.name == "nt":
+            if "cdrom" in part.opts or part.fstype == "":
                 continue
         usage = psutil.disk_usage(part.mountpoint)
-        disks.append({
-            'device': part.device,
-            'total': bytes2human(usage.total),
-            'used': bytes2human(usage.used),
-            'free': bytes2human(usage.free),
-            'percentage': int(usage.percent),
-            'mountpoint': part.mountpoint,
-            'fstype': part.fstype
-        })
+        disks.append(
+            {
+                "device": part.device,
+                "total": bytes2human(usage.total),
+                "used": bytes2human(usage.used),
+                "free": bytes2human(usage.free),
+                "percentage": int(usage.percent),
+                "mountpoint": part.mountpoint,
+                "fstype": part.fstype,
+            }
+        )
 
     # memory
     memory_v = psutil.virtual_memory()
     memory_s = psutil.swap_memory()
 
     memory = {
-        'total': bytes2human(memory_v.total),
-        'available': bytes2human(memory_v.available),
-        'percentage': memory_v.percent,
-        'used': bytes2human(memory_v.used),
-        'free': bytes2human(memory_v.free),
-        'active': bytes2human(memory_v.active),
-        'inactive': bytes2human(memory_v.inactive),
-        'buffers': bytes2human(memory_v.buffers),
-        'cached': bytes2human(memory_v.cached),
-        'swap_total': bytes2human(memory_s.total),
-        'swap_used': bytes2human(memory_s.used)
+        "total": bytes2human(memory_v.total),
+        "available": bytes2human(memory_v.available),
+        "percentage": memory_v.percent,
+        "used": bytes2human(memory_v.used),
+        "free": bytes2human(memory_v.free),
+        "active": bytes2human(memory_v.active),
+        "inactive": bytes2human(memory_v.inactive),
+        "buffers": bytes2human(memory_v.buffers),
+        "cached": bytes2human(memory_v.cached),
+        "swap_total": bytes2human(memory_s.total),
+        "swap_used": bytes2human(memory_s.used),
     }
 
     # load
     system_load = os.getloadavg()
 
     # processes
     min_mem = 1024 * 1024 * 40  # 40MB
     process_info = []
     for p in psutil.process_iter():
         mem_used = p.memory_info().rss
         if mem_used < min_mem:
             continue
-        process_info.append({'owner': p.username(),
-                             'pid': p.pid,
-                             'cpu': round(p.cpu_percent(interval=0), 1),
-                             'mem_percentage': round(p.memory_percent(),1),
-                             'mem_usage': bytes2human(mem_used),
-                             'name': p.name(),
-                             'command': ' '.join(p.cmdline())
-                             })
-    process_info = sorted(process_info, key=lambda x: x['mem_percentage'],
-                          reverse=True)
+        process_info.append(
+            {
+                "owner": p.username(),
+                "pid": p.pid,
+                "cpu": round(p.cpu_percent(interval=0), 1),
+                "mem_percentage": round(p.memory_percent(), 1),
+                "mem_usage": bytes2human(mem_used),
+                "name": p.name(),
+                "command": " ".join(p.cmdline()),
+            }
+        )
+    process_info = sorted(process_info, key=lambda x: x["mem_percentage"], reverse=True)
 
     # pg tables
 
-    db_size_query = '''
+    db_size_query = """
     SELECT tablename, pg_total_relation_size(tablename::text) size
     FROM pg_tables WHERE tablename NOT LIKE 'pg_%' AND 
-    tablename NOT LIKE 'sql_%' ORDER BY size DESC;'''
+    tablename NOT LIKE 'sql_%' ORDER BY size DESC;"""
 
     db_tables = []
     for row in DBSession.execute(db_size_query):
-        db_tables.append({"size_human": bytes2human(row.size),
-                          "table_name": row.tablename})
+        db_tables.append(
+            {"size_human": bytes2human(row.size), "table_name": row.tablename}
+        )
 
     # es indices
     es_indices = []
-    result = Datastores.es.indices.stats(metric=['store, docs'])
-    for ix, stats in result['indices'].items():
-        size = stats['primaries']['store']['size_in_bytes']
-        es_indices.append({'name': ix,
-                           'size': size,
-                           'size_human': bytes2human(size)})
+    result = Datastores.es.indices.stats(metric=["store, docs"])
+    for ix, stats in result["indices"].items():
+        size = stats["primaries"]["store"]["size_in_bytes"]
+        es_indices.append({"name": ix, "size": size, "size_human": bytes2human(size)})
 
     # packages
 
-    packages = ({'name': p.project_name, 'version': p.version}
-                for p in pkg_resources.working_set)
-
-    return {'db_tables': db_tables,
-            'es_indices': sorted(es_indices,
-                                 key=lambda x: x['size'], reverse=True),
-            'process_info': process_info,
-            'system_load': system_load,
-            'disks': disks,
-            'memory': memory,
-            'packages': sorted(packages, key=lambda x: x['name'].lower()),
-            'current_time': current_time,
-            'queue_stats': {
-                'processed_reports': processed_reports,
-                'processed_logs': processed_logs,
-                'processed_metrics': processed_metrics,
-                'waiting_reports': waiting_reports,
-                'waiting_logs': waiting_logs,
-                'waiting_metrics': waiting_metrics,
-                'waiting_other': waiting_other
-            },
-            'self_info': self_info
-            }
+    packages = (
+        {"name": p.project_name, "version": p.version}
+        for p in pkg_resources.working_set
+    )
+
+    return {
+        "db_tables": db_tables,
+        "es_indices": sorted(es_indices, key=lambda x: x["size"], reverse=True),
+        "process_info": process_info,
+        "system_load": system_load,
+        "disks": disks,
+        "memory": memory,
+        "packages": sorted(packages, key=lambda x: x["name"].lower()),
+        "current_time": current_time,
+        "queue_stats": {
+            "processed_reports": processed_reports,
+            "processed_logs": processed_logs,
+            "processed_metrics": processed_metrics,
+            "waiting_reports": waiting_reports,
+            "waiting_logs": waiting_logs,
+            "waiting_metrics": waiting_metrics,
+            "waiting_other": waiting_other,
+        },
+        "self_info": self_info,
+    }

backend/src/appenlight/views/admin/config.py

@@ -1,47 +1,55 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from pyramid.view import view_config
 from pyramid.httpexceptions import HTTPNotFound
 from appenlight.models.services.config import ConfigService
 
 import logging
 
 log = logging.getLogger(__name__)
 
 
-@view_config(route_name='admin_configs', renderer='json',
-             permission='root_administration', request_method='GET')
+@view_config(
+    route_name="admin_configs",
+    renderer="json",
+    permission="root_administration",
+    request_method="GET",
+)
 def query(request):
     ConfigService.setup_default_values()
     pairs = []
-    for value in request.GET.getall('filter'):
-        split = value.split(':', 1)
-        pairs.append({'key': split[0], 'section': split[1]})
+    for value in request.GET.getall("filter"):
+        split = value.split(":", 1)
+        pairs.append({"key": split[0], "section": split[1]})
     return [c for c in ConfigService.filtered_key_and_section(pairs)]
 
 
-@view_config(route_name='admin_config', renderer='json',
-             permission='root_administration', request_method='POST')
+@view_config(
+    route_name="admin_config",
+    renderer="json",
+    permission="root_administration",
+    request_method="POST",
+)
 def post(request):
     row = ConfigService.by_key_and_section(
-        key=request.matchdict.get('key'),
-        section=request.matchdict.get('section'))
+        key=request.matchdict.get("key"), section=request.matchdict.get("section")
+    )
     if not row:
         raise HTTPNotFound()
     row.value = None
-    row.value = request.unsafe_json_body['value']
+    row.value = request.unsafe_json_body["value"]
     return row

backend/src/appenlight/views/admin/partitions.py

@@ -1,148 +1,166 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from pyramid.view import view_config
 from appenlight.models import DBSession, Datastores
 from appenlight.forms import get_partition_deletion_form
 
 import logging
 
 from zope.sqlalchemy import mark_changed
 from datetime import datetime
 import sqlalchemy as sa
 
 log = logging.getLogger(__name__)
 
 
 def get_partition_stats():
     table_query = """
         SELECT table_name
         FROM information_schema.tables
         GROUP BY table_name
         ORDER BY table_name
     """
 
     permanent_partitions = {}
     daily_partitions = {}
 
     def is_int(data):
         try:
             int(data)
             return True
         except Exception:
             pass
         return False
 
     def add_key(key, holder):
         if not ix_time in holder:
-            holder[ix_time] = {'pg': [], 'elasticsearch': []}
+            holder[ix_time] = {"pg": [], "elasticsearch": []}
 
-    for partition in list(Datastores.es.indices.get_alias('rcae*')):
-        if not partition.startswith('rcae'):
+    for partition in list(Datastores.es.indices.get_alias("rcae*")):
+        if not partition.startswith("rcae"):
             continue
-        split_data = partition.split('_')
+        split_data = partition.split("_")
         permanent = False
         # if we dont have a day then treat it as permanent partion
         if False in list(map(is_int, split_data[-3:])):
-            ix_time = datetime(year=int(split_data[-2]),
-                               month=int(split_data[-1]),
-                               day=1).date()
+            ix_time = datetime(
+                year=int(split_data[-2]), month=int(split_data[-1]), day=1
+            ).date()
             permanent = True
         else:
-            ix_time = datetime(year=int(split_data[-3]),
-                               month=int(split_data[-2]),
-                               day=int(split_data[-1])).date()
+            ix_time = datetime(
+                year=int(split_data[-3]),
+                month=int(split_data[-2]),
+                day=int(split_data[-1]),
+            ).date()
 
         ix_time = str(ix_time)
         if permanent:
             add_key(ix_time, permanent_partitions)
             if ix_time not in permanent_partitions:
-                permanent_partitions[ix_time]['elasticsearch'] = []
-            permanent_partitions[ix_time]['elasticsearch'].append(partition)
+                permanent_partitions[ix_time]["elasticsearch"] = []
+            permanent_partitions[ix_time]["elasticsearch"].append(partition)
         else:
             add_key(ix_time, daily_partitions)
             if ix_time not in daily_partitions:
-                daily_partitions[ix_time]['elasticsearch'] = []
-            daily_partitions[ix_time]['elasticsearch'].append(partition)
+                daily_partitions[ix_time]["elasticsearch"] = []
+            daily_partitions[ix_time]["elasticsearch"].append(partition)
 
     for row in DBSession.execute(table_query):
-        splitted = row['table_name'].split('_')
-        if 'p' in splitted:
+        splitted = row["table_name"].split("_")
+        if "p" in splitted:
             # dealing with partition
-            split_data = [int(x) for x in splitted[splitted.index('p') + 1:]]
+            split_data = [int(x) for x in splitted[splitted.index("p") + 1 :]]
             if len(split_data) == 3:
-                ix_time = datetime(split_data[0], split_data[1],
-                                   split_data[2]).date()
+                ix_time = datetime(split_data[0], split_data[1], split_data[2]).date()
                 ix_time = str(ix_time)
                 add_key(ix_time, daily_partitions)
-                daily_partitions[ix_time]['pg'].append(row['table_name'])
+                daily_partitions[ix_time]["pg"].append(row["table_name"])
             else:
                 ix_time = datetime(split_data[0], split_data[1], 1).date()
                 ix_time = str(ix_time)
                 add_key(ix_time, permanent_partitions)
-                permanent_partitions[ix_time]['pg'].append(row['table_name'])
+                permanent_partitions[ix_time]["pg"].append(row["table_name"])
 
     return permanent_partitions, daily_partitions
 
 
-@view_config(route_name='section_view', permission='root_administration',
-             match_param=['section=admin_section', 'view=partitions'],
-             renderer='json', request_method='GET')
+@view_config(
+    route_name="section_view",
+    permission="root_administration",
+    match_param=["section=admin_section", "view=partitions"],
+    renderer="json",
+    request_method="GET",
+)
 def index(request):
     permanent_partitions, daily_partitions = get_partition_stats()
 
-    return {"permanent_partitions": sorted(list(permanent_partitions.items()),
-                                           key=lambda x: x[0], reverse=True),
-            "daily_partitions": sorted(list(daily_partitions.items()),
-                                       key=lambda x: x[0], reverse=True)}
-
-
-@view_config(route_name='section_view', request_method='POST',
-             match_param=['section=admin_section', 'view=partitions_remove'],
-             renderer='json', permission='root_administration')
+    return {
+        "permanent_partitions": sorted(
+            list(permanent_partitions.items()), key=lambda x: x[0], reverse=True
+        ),
+        "daily_partitions": sorted(
+            list(daily_partitions.items()), key=lambda x: x[0], reverse=True
+        ),
+    }
+
+
+@view_config(
+    route_name="section_view",
+    request_method="POST",
+    match_param=["section=admin_section", "view=partitions_remove"],
+    renderer="json",
+    permission="root_administration",
+)
 def partitions_remove(request):
     permanent_partitions, daily_partitions = get_partition_stats()
     pg_partitions = []
     es_partitions = []
     for item in list(permanent_partitions.values()) + list(daily_partitions.values()):
-        es_partitions.extend(item['elasticsearch'])
-        pg_partitions.extend(item['pg'])
+        es_partitions.extend(item["elasticsearch"])
+        pg_partitions.extend(item["pg"])
     FormCls = get_partition_deletion_form(es_partitions, pg_partitions)
-    form = FormCls(es_index=request.unsafe_json_body['es_indices'],
-                   pg_index=request.unsafe_json_body['pg_indices'],
-                   confirm=request.unsafe_json_body['confirm'],
-                   csrf_context=request)
+    form = FormCls(
+        es_index=request.unsafe_json_body["es_indices"],
+        pg_index=request.unsafe_json_body["pg_indices"],
+        confirm=request.unsafe_json_body["confirm"],
+        csrf_context=request,
+    )
     if form.validate():
-        for ix in form.data['es_index']:
-            log.warning('deleting ES partition: {}'.format(ix))
+        for ix in form.data["es_index"]:
+            log.warning("deleting ES partition: {}".format(ix))
             Datastores.es.indices.delete(ix)
-        for ix in form.data['pg_index']:
-            log.warning('deleting PG partition: {}'.format(ix))
-            stmt = sa.text('DROP TABLE %s CASCADE' % sa.text(ix))
+        for ix in form.data["pg_index"]:
+            log.warning("deleting PG partition: {}".format(ix))
+            stmt = sa.text("DROP TABLE %s CASCADE" % sa.text(ix))
             session = DBSession()
             session.connection().execute(stmt)
             mark_changed(session)
 
     for field, error in form.errors.items():
-        msg = '%s: %s' % (field, error[0])
-        request.session.flash(msg, 'error')
+        msg = "%s: %s" % (field, error[0])
+        request.session.flash(msg, "error")
 
     permanent_partitions, daily_partitions = get_partition_stats()
     return {
         "permanent_partitions": sorted(
-            list(permanent_partitions.items()), key=lambda x: x[0], reverse=True),
+            list(permanent_partitions.items()), key=lambda x: x[0], reverse=True
+        ),
         "daily_partitions": sorted(
-            list(daily_partitions.items()), key=lambda x: x[0], reverse=True)}
+            list(daily_partitions.items()), key=lambda x: x[0], reverse=True
+        ),
+    }

backend/src/appenlight/views/admin/users.py

@@ -1,36 +1,39 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from pyramid.view import view_config
 from pyramid.httpexceptions import HTTPFound, HTTPNotFound
 from pyramid import security
 from ziggurat_foundations.models.services.user import UserService
 
 import logging
 
 log = logging.getLogger(__name__)
 
 
-@view_config(route_name='section_view', permission='root_administration',
-             match_param=['section=admin_section', 'view=relogin_user'],
-             renderer='json', request_method='GET')
+@view_config(
+    route_name="section_view",
+    permission="root_administration",
+    match_param=["section=admin_section", "view=relogin_user"],
+    renderer="json",
+    request_method="GET",
+)
 def relogin_to_user(request):
-    user = UserService.by_id(request.GET.get('user_id'))
+    user = UserService.by_id(request.GET.get("user_id"))
     if not user:
         return HTTPNotFound()
     headers = security.remember(request, user.id)
-    return HTTPFound(location=request.route_url('/'),
-                     headers=headers)
+    return HTTPFound(location=request.route_url("/"), headers=headers)

backend/src/appenlight/views/api.py

@@ -1,435 +1,485 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import base64
 import io
 import datetime
 import json
 import logging
 import urllib.request, urllib.parse, urllib.error
 import zlib
 
 from gzip import GzipFile
 from pyramid.view import view_config
 from pyramid.httpexceptions import HTTPBadRequest
 
 import appenlight.celery.tasks as tasks
 from appenlight.lib.api import rate_limiting, check_cors
 from appenlight.lib.enums import ParsedSentryEventType
 from appenlight.lib.utils import parse_proto
 from appenlight.lib.utils.airbrake import parse_airbrake_xml
 from appenlight.lib.utils.date_utils import convert_date
 from appenlight.lib.utils.sentry import parse_sentry_event
 from appenlight.lib.request import JSONException
-from appenlight.validators import (LogListSchema,
-                                   MetricsListSchema,
-                                   GeneralMetricsListSchema,
-                                   GeneralMetricsPermanentListSchema,
-                                   GeneralMetricSchema,
-                                   GeneralMetricPermanentSchema,
-                                   LogListPermanentSchema,
-                                   ReportListSchema_0_5,
-                                   LogSchema,
-                                   LogSchemaPermanent,
-                                   ReportSchema_0_5)
+from appenlight.validators import (
+    LogListSchema,
+    MetricsListSchema,
+    GeneralMetricsListSchema,
+    GeneralMetricsPermanentListSchema,
+    GeneralMetricSchema,
+    GeneralMetricPermanentSchema,
+    LogListPermanentSchema,
+    ReportListSchema_0_5,
+    LogSchema,
+    LogSchemaPermanent,
+    ReportSchema_0_5,
+)
 
 log = logging.getLogger(__name__)
 
 
-@view_config(route_name='api_logs', renderer='string', permission='create',
-             require_csrf=False)
-@view_config(route_name='api_log', renderer='string', permission='create',
-             require_csrf=False)
+@view_config(
+    route_name="api_logs", renderer="string", permission="create", require_csrf=False
+)
+@view_config(
+    route_name="api_log", renderer="string", permission="create", require_csrf=False
+)
 def logs_create(request):
     """
     Endpoint for log aggregation
     """
     application = request.context.resource
-    if request.method.upper() == 'OPTIONS':
+    if request.method.upper() == "OPTIONS":
         return check_cors(request, application)
     else:
         check_cors(request, application, should_return=False)
 
     params = dict(request.params.copy())
-    proto_version = parse_proto(params.get('protocol_version', ''))
+    proto_version = parse_proto(params.get("protocol_version", ""))
     payload = request.unsafe_json_body
-    sequence_accepted = request.matched_route.name == 'api_logs'
+    sequence_accepted = request.matched_route.name == "api_logs"
 
     if sequence_accepted:
         if application.allow_permanent_storage:
-            schema = LogListPermanentSchema().bind(
-                utcnow=datetime.datetime.utcnow())
+            schema = LogListPermanentSchema().bind(utcnow=datetime.datetime.utcnow())
         else:
-            schema = LogListSchema().bind(
-                utcnow=datetime.datetime.utcnow())
+            schema = LogListSchema().bind(utcnow=datetime.datetime.utcnow())
     else:
         if application.allow_permanent_storage:
-            schema = LogSchemaPermanent().bind(
-                utcnow=datetime.datetime.utcnow())
+            schema = LogSchemaPermanent().bind(utcnow=datetime.datetime.utcnow())
         else:
-            schema = LogSchema().bind(
-                utcnow=datetime.datetime.utcnow())
+            schema = LogSchema().bind(utcnow=datetime.datetime.utcnow())
 
     deserialized_logs = schema.deserialize(payload)
     if sequence_accepted is False:
         deserialized_logs = [deserialized_logs]
 
-    rate_limiting(request, application, 'per_application_logs_rate_limit',
-                  len(deserialized_logs))
+    rate_limiting(
+        request, application, "per_application_logs_rate_limit", len(deserialized_logs)
+    )
 
     # pprint.pprint(deserialized_logs)
 
     # we need to split those out so we can process the pkey ones one by one
-    non_pkey_logs = [log_dict for log_dict in deserialized_logs
-                     if not log_dict['primary_key']]
+    non_pkey_logs = [
+        log_dict for log_dict in deserialized_logs if not log_dict["primary_key"]
+    ]
     pkey_dict = {}
     # try to process the logs as best as we can and group together to reduce
     # the amount of
     for log_dict in deserialized_logs:
-        if log_dict['primary_key']:
-            key = (log_dict['primary_key'], log_dict['namespace'],)
+        if log_dict["primary_key"]:
+            key = (log_dict["primary_key"], log_dict["namespace"])
             if not key in pkey_dict:
                 pkey_dict[key] = []
             pkey_dict[key].append(log_dict)
 
     if non_pkey_logs:
-        log.debug('%s non-pkey logs received: %s' % (application,
-                                                     len(non_pkey_logs)))
+        log.debug("%s non-pkey logs received: %s" % (application, len(non_pkey_logs)))
         tasks.add_logs.delay(application.resource_id, params, non_pkey_logs)
     if pkey_dict:
         logs_to_insert = []
         for primary_key_tuple, payload in pkey_dict.items():
-            sorted_logs = sorted(payload, key=lambda x: x['date'])
+            sorted_logs = sorted(payload, key=lambda x: x["date"])
             logs_to_insert.append(sorted_logs[-1])
-        log.debug('%s pkey logs received: %s' % (application,
-                                                 len(logs_to_insert)))
+        log.debug("%s pkey logs received: %s" % (application, len(logs_to_insert)))
         tasks.add_logs.delay(application.resource_id, params, logs_to_insert)
 
-    log.info('LOG call %s %s client:%s' % (
-        application, proto_version, request.headers.get('user_agent')))
-    return 'OK: Logs accepted'
-
-
-@view_config(route_name='api_request_stats', renderer='string',
-             permission='create', require_csrf=False)
-@view_config(route_name='api_metrics', renderer='string',
-             permission='create', require_csrf=False)
+    log.info(
+        "LOG call %s %s client:%s"
+        % (application, proto_version, request.headers.get("user_agent"))
+    )
+    return "OK: Logs accepted"
+
+
+@view_config(
+    route_name="api_request_stats",
+    renderer="string",
+    permission="create",
+    require_csrf=False,
+)
+@view_config(
+    route_name="api_metrics", renderer="string", permission="create", require_csrf=False
+)
 def request_metrics_create(request):
     """
     Endpoint for performance metrics, aggregates view performance stats
     and converts them to general metric row
     """
     application = request.context.resource
-    if request.method.upper() == 'OPTIONS':
+    if request.method.upper() == "OPTIONS":
         return check_cors(request, application)
     else:
         check_cors(request, application, should_return=False)
 
     params = dict(request.params.copy())
-    proto_version = parse_proto(params.get('protocol_version', ''))
+    proto_version = parse_proto(params.get("protocol_version", ""))
 
     payload = request.unsafe_json_body
     schema = MetricsListSchema()
     dataset = schema.deserialize(payload)
 
-    rate_limiting(request, application, 'per_application_metrics_rate_limit',
-                  len(dataset))
+    rate_limiting(
+        request, application, "per_application_metrics_rate_limit", len(dataset)
+    )
 
     # looping report data
     metrics = {}
     for metric in dataset:
-        server_name = metric.get('server', '').lower() or 'unknown'
-        start_interval = convert_date(metric['timestamp'])
+        server_name = metric.get("server", "").lower() or "unknown"
+        start_interval = convert_date(metric["timestamp"])
         start_interval = start_interval.replace(second=0, microsecond=0)
 
-        for view_name, view_metrics in metric['metrics']:
-            key = '%s%s%s' % (metric['server'], start_interval, view_name)
+        for view_name, view_metrics in metric["metrics"]:
+            key = "%s%s%s" % (metric["server"], start_interval, view_name)
             if start_interval not in metrics:
-                metrics[key] = {"requests": 0, "main": 0, "sql": 0,
-                                "nosql": 0, "remote": 0, "tmpl": 0,
-                                "custom": 0, 'sql_calls': 0,
-                                'nosql_calls': 0,
-                                'remote_calls': 0, 'tmpl_calls': 0,
-                                'custom_calls': 0,
-                                "start_interval": start_interval,
-                                "server_name": server_name,
-                                "view_name": view_name
-                                }
-            metrics[key]["requests"] += int(view_metrics['requests'])
-            metrics[key]["main"] += round(view_metrics['main'], 5)
-            metrics[key]["sql"] += round(view_metrics['sql'], 5)
-            metrics[key]["nosql"] += round(view_metrics['nosql'], 5)
-            metrics[key]["remote"] += round(view_metrics['remote'], 5)
-            metrics[key]["tmpl"] += round(view_metrics['tmpl'], 5)
-            metrics[key]["custom"] += round(view_metrics.get('custom', 0.0),
-                                            5)
-            metrics[key]["sql_calls"] += int(
-                view_metrics.get('sql_calls', 0))
-            metrics[key]["nosql_calls"] += int(
-                view_metrics.get('nosql_calls', 0))
-            metrics[key]["remote_calls"] += int(
-                view_metrics.get('remote_calls', 0))
-            metrics[key]["tmpl_calls"] += int(
-                view_metrics.get('tmpl_calls', 0))
-            metrics[key]["custom_calls"] += int(
-                view_metrics.get('custom_calls', 0))
+                metrics[key] = {
+                    "requests": 0,
+                    "main": 0,
+                    "sql": 0,
+                    "nosql": 0,
+                    "remote": 0,
+                    "tmpl": 0,
+                    "custom": 0,
+                    "sql_calls": 0,
+                    "nosql_calls": 0,
+                    "remote_calls": 0,
+                    "tmpl_calls": 0,
+                    "custom_calls": 0,
+                    "start_interval": start_interval,
+                    "server_name": server_name,
+                    "view_name": view_name,
+                }
+            metrics[key]["requests"] += int(view_metrics["requests"])
+            metrics[key]["main"] += round(view_metrics["main"], 5)
+            metrics[key]["sql"] += round(view_metrics["sql"], 5)
+            metrics[key]["nosql"] += round(view_metrics["nosql"], 5)
+            metrics[key]["remote"] += round(view_metrics["remote"], 5)
+            metrics[key]["tmpl"] += round(view_metrics["tmpl"], 5)
+            metrics[key]["custom"] += round(view_metrics.get("custom", 0.0), 5)
+            metrics[key]["sql_calls"] += int(view_metrics.get("sql_calls", 0))
+            metrics[key]["nosql_calls"] += int(view_metrics.get("nosql_calls", 0))
+            metrics[key]["remote_calls"] += int(view_metrics.get("remote_calls", 0))
+            metrics[key]["tmpl_calls"] += int(view_metrics.get("tmpl_calls", 0))
+            metrics[key]["custom_calls"] += int(view_metrics.get("custom_calls", 0))
 
             if not metrics[key]["requests"]:
                 # fix this here because validator can't
                 metrics[key]["requests"] = 1
                 # metrics dict is being built to minimize
                 # the amount of queries used
                 # in case we get multiple rows from same minute
 
     normalized_metrics = []
     for metric in metrics.values():
         new_metric = {
-            'namespace': 'appenlight.request_metric',
-            'timestamp': metric.pop('start_interval'),
-            'server_name': metric['server_name'],
-            'tags': list(metric.items())
+            "namespace": "appenlight.request_metric",
+            "timestamp": metric.pop("start_interval"),
+            "server_name": metric["server_name"],
+            "tags": list(metric.items()),
         }
         normalized_metrics.append(new_metric)
 
-    tasks.add_metrics.delay(application.resource_id, params,
-                            normalized_metrics, proto_version)
-
-    log.info('REQUEST METRICS call {} {} client:{}'.format(
-        application.resource_name, proto_version,
-        request.headers.get('user_agent')))
-    return 'OK: request metrics accepted'
-
-
-@view_config(route_name='api_general_metrics', renderer='string',
-             permission='create', require_csrf=False)
-@view_config(route_name='api_general_metric', renderer='string',
-             permission='create', require_csrf=False)
+    tasks.add_metrics.delay(
+        application.resource_id, params, normalized_metrics, proto_version
+    )
+
+    log.info(
+        "REQUEST METRICS call {} {} client:{}".format(
+            application.resource_name, proto_version, request.headers.get("user_agent")
+        )
+    )
+    return "OK: request metrics accepted"
+
+
+@view_config(
+    route_name="api_general_metrics",
+    renderer="string",
+    permission="create",
+    require_csrf=False,
+)
+@view_config(
+    route_name="api_general_metric",
+    renderer="string",
+    permission="create",
+    require_csrf=False,
+)
 def general_metrics_create(request):
     """
     Endpoint for general metrics aggregation
     """
     application = request.context.resource
-    if request.method.upper() == 'OPTIONS':
+    if request.method.upper() == "OPTIONS":
         return check_cors(request, application)
     else:
         check_cors(request, application, should_return=False)
 
     params = dict(request.params.copy())
-    proto_version = parse_proto(params.get('protocol_version', ''))
+    proto_version = parse_proto(params.get("protocol_version", ""))
     payload = request.unsafe_json_body
-    sequence_accepted = request.matched_route.name == 'api_general_metrics'
+    sequence_accepted = request.matched_route.name == "api_general_metrics"
     if sequence_accepted:
         if application.allow_permanent_storage:
             schema = GeneralMetricsPermanentListSchema().bind(
-                utcnow=datetime.datetime.utcnow())
+                utcnow=datetime.datetime.utcnow()
+            )
         else:
-            schema = GeneralMetricsListSchema().bind(
-                utcnow=datetime.datetime.utcnow())
+            schema = GeneralMetricsListSchema().bind(utcnow=datetime.datetime.utcnow())
     else:
         if application.allow_permanent_storage:
             schema = GeneralMetricPermanentSchema().bind(
-                utcnow=datetime.datetime.utcnow())
+                utcnow=datetime.datetime.utcnow()
+            )
         else:
-            schema = GeneralMetricSchema().bind(
-                utcnow=datetime.datetime.utcnow())
+            schema = GeneralMetricSchema().bind(utcnow=datetime.datetime.utcnow())
 
     deserialized_metrics = schema.deserialize(payload)
     if sequence_accepted is False:
         deserialized_metrics = [deserialized_metrics]
 
-    rate_limiting(request, application, 'per_application_metrics_rate_limit',
-                  len(deserialized_metrics))
-
-    tasks.add_metrics.delay(application.resource_id, params,
-                            deserialized_metrics, proto_version)
-
-    log.info('METRICS call {} {} client:{}'.format(
-        application.resource_name, proto_version,
-        request.headers.get('user_agent')))
-    return 'OK: Metrics accepted'
-
-
-@view_config(route_name='api_reports', renderer='string', permission='create',
-             require_csrf=False)
-@view_config(route_name='api_slow_reports', renderer='string',
-             permission='create', require_csrf=False)
-@view_config(route_name='api_report', renderer='string', permission='create',
-             require_csrf=False)
+    rate_limiting(
+        request,
+        application,
+        "per_application_metrics_rate_limit",
+        len(deserialized_metrics),
+    )
+
+    tasks.add_metrics.delay(
+        application.resource_id, params, deserialized_metrics, proto_version
+    )
+
+    log.info(
+        "METRICS call {} {} client:{}".format(
+            application.resource_name, proto_version, request.headers.get("user_agent")
+        )
+    )
+    return "OK: Metrics accepted"
+
+
+@view_config(
+    route_name="api_reports", renderer="string", permission="create", require_csrf=False
+)
+@view_config(
+    route_name="api_slow_reports",
+    renderer="string",
+    permission="create",
+    require_csrf=False,
+)
+@view_config(
+    route_name="api_report", renderer="string", permission="create", require_csrf=False
+)
 def reports_create(request):
     """
     Endpoint for exception and slowness reports
     """
     # route_url('reports')
     application = request.context.resource
-    if request.method.upper() == 'OPTIONS':
+    if request.method.upper() == "OPTIONS":
         return check_cors(request, application)
     else:
         check_cors(request, application, should_return=False)
     params = dict(request.params.copy())
-    proto_version = parse_proto(params.get('protocol_version', ''))
+    proto_version = parse_proto(params.get("protocol_version", ""))
     payload = request.unsafe_json_body
-    sequence_accepted = request.matched_route.name == 'api_reports'
+    sequence_accepted = request.matched_route.name == "api_reports"
 
     if sequence_accepted:
-        schema = ReportListSchema_0_5().bind(
-            utcnow=datetime.datetime.utcnow())
+        schema = ReportListSchema_0_5().bind(utcnow=datetime.datetime.utcnow())
     else:
-        schema = ReportSchema_0_5().bind(
-            utcnow=datetime.datetime.utcnow())
+        schema = ReportSchema_0_5().bind(utcnow=datetime.datetime.utcnow())
 
     deserialized_reports = schema.deserialize(payload)
     if sequence_accepted is False:
         deserialized_reports = [deserialized_reports]
     if deserialized_reports:
-        rate_limiting(request, application,
-                      'per_application_reports_rate_limit',
-                      len(deserialized_reports))
+        rate_limiting(
+            request,
+            application,
+            "per_application_reports_rate_limit",
+            len(deserialized_reports),
+        )
 
         # pprint.pprint(deserialized_reports)
-        tasks.add_reports.delay(application.resource_id, params,
-                                deserialized_reports)
-    log.info('REPORT call  %s, %s client:%s' % (
-        application,
-        proto_version,
-        request.headers.get('user_agent'))
-             )
-    return 'OK: Reports accepted'
-
-
-@view_config(route_name='api_airbrake', renderer='string', permission='create',
-             require_csrf=False)
+        tasks.add_reports.delay(application.resource_id, params, deserialized_reports)
+    log.info(
+        "REPORT call  %s, %s client:%s"
+        % (application, proto_version, request.headers.get("user_agent"))
+    )
+    return "OK: Reports accepted"
+
+
+@view_config(
+    route_name="api_airbrake",
+    renderer="string",
+    permission="create",
+    require_csrf=False,
+)
 def airbrake_xml_compat(request):
     """
     Airbrake compatible endpoint for XML reports
     """
     application = request.context.resource
-    if request.method.upper() == 'OPTIONS':
+    if request.method.upper() == "OPTIONS":
         return check_cors(request, application)
     else:
         check_cors(request, application, should_return=False)
 
     params = dict(request.params.copy())
 
     error_dict = parse_airbrake_xml(request)
     schema = ReportListSchema_0_5().bind(utcnow=datetime.datetime.utcnow())
     deserialized_reports = schema.deserialize([error_dict])
-    rate_limiting(request, application, 'per_application_reports_rate_limit',
-                  len(deserialized_reports))
-
-    tasks.add_reports.delay(application.resource_id, params,
-                            deserialized_reports)
-    log.info('%s AIRBRAKE call for application %s, api_ver:%s client:%s' % (
-        500, application.resource_name,
-        request.params.get('protocol_version', 'unknown'),
-        request.headers.get('user_agent'))
-             )
-    return '<notice><id>no-id</id><url>%s</url></notice>' % \
-           request.registry.settings['mailing.app_url']
+    rate_limiting(
+        request,
+        application,
+        "per_application_reports_rate_limit",
+        len(deserialized_reports),
+    )
+
+    tasks.add_reports.delay(application.resource_id, params, deserialized_reports)
+    log.info(
+        "%s AIRBRAKE call for application %s, api_ver:%s client:%s"
+        % (
+            500,
+            application.resource_name,
+            request.params.get("protocol_version", "unknown"),
+            request.headers.get("user_agent"),
+        )
+    )
+    return (
+        "<notice><id>no-id</id><url>%s</url></notice>"
+        % request.registry.settings["mailing.app_url"]
+    )
 
 
 def decompress_gzip(data):
     try:
         fp = io.StringIO(data)
         with GzipFile(fileobj=fp) as f:
             return f.read()
     except Exception as exc:
         raise
         log.error(exc)
         raise HTTPBadRequest()
 
 
 def decompress_zlib(data):
     try:
         return zlib.decompress(data)
     except Exception as exc:
         raise
         log.error(exc)
         raise HTTPBadRequest()
 
 
 def decode_b64(data):
     try:
         return base64.b64decode(data)
     except Exception as exc:
         raise
         log.error(exc)
         raise HTTPBadRequest()
 
 
-@view_config(route_name='api_sentry', renderer='string', permission='create',
-             require_csrf=False)
-@view_config(route_name='api_sentry_slash', renderer='string',
-             permission='create', require_csrf=False)
+@view_config(
+    route_name="api_sentry", renderer="string", permission="create", require_csrf=False
+)
+@view_config(
+    route_name="api_sentry_slash",
+    renderer="string",
+    permission="create",
+    require_csrf=False,
+)
 def sentry_compat(request):
     """
     Sentry compatible endpoint
     """
     application = request.context.resource
-    if request.method.upper() == 'OPTIONS':
+    if request.method.upper() == "OPTIONS":
         return check_cors(request, application)
     else:
         check_cors(request, application, should_return=False)
 
     # handle various report encoding
-    content_encoding = request.headers.get('Content-Encoding')
-    content_type = request.headers.get('Content-Type')
-    if content_encoding == 'gzip':
+    content_encoding = request.headers.get("Content-Encoding")
+    content_type = request.headers.get("Content-Type")
+    if content_encoding == "gzip":
         body = decompress_gzip(request.body)
-    elif content_encoding == 'deflate':
+    elif content_encoding == "deflate":
         body = decompress_zlib(request.body)
     else:
         body = request.body
         # attempt to fix string before decoding for stupid clients
-        if content_type == 'application/x-www-form-urlencoded':
-            body = urllib.parse.unquote(body.decode('utf8'))
-        check_char = '{' if isinstance(body, str) else b'{'
+        if content_type == "application/x-www-form-urlencoded":
+            body = urllib.parse.unquote(body.decode("utf8"))
+        check_char = "{" if isinstance(body, str) else b"{"
         if not body.startswith(check_char):
             try:
                 body = decode_b64(body)
                 body = decompress_zlib(body)
             except Exception as exc:
                 log.info(exc)
 
     try:
-        json_body = json.loads(body.decode('utf8'))
+        json_body = json.loads(body.decode("utf8"))
     except ValueError:
         raise JSONException("Incorrect JSON")
 
     event, event_type = parse_sentry_event(json_body)
 
     if event_type == ParsedSentryEventType.LOG:
         if application.allow_permanent_storage:
-            schema = LogSchemaPermanent().bind(
-                utcnow=datetime.datetime.utcnow())
+            schema = LogSchemaPermanent().bind(utcnow=datetime.datetime.utcnow())
         else:
-            schema = LogSchema().bind(
-                utcnow=datetime.datetime.utcnow())
+            schema = LogSchema().bind(utcnow=datetime.datetime.utcnow())
         deserialized_logs = schema.deserialize(event)
         non_pkey_logs = [deserialized_logs]
-        log.debug('%s non-pkey logs received: %s' % (application,
-                                                     len(non_pkey_logs)))
+        log.debug("%s non-pkey logs received: %s" % (application, len(non_pkey_logs)))
         tasks.add_logs.delay(application.resource_id, {}, non_pkey_logs)
     if event_type == ParsedSentryEventType.ERROR_REPORT:
         schema = ReportSchema_0_5().bind(
             utcnow=datetime.datetime.utcnow(),
-            allow_permanent_storage=application.allow_permanent_storage)
+            allow_permanent_storage=application.allow_permanent_storage,
+        )
         deserialized_reports = [schema.deserialize(event)]
-        rate_limiting(request, application,
-                      'per_application_reports_rate_limit',
-                      len(deserialized_reports))
-        tasks.add_reports.delay(application.resource_id, {},
-                                deserialized_reports)
-    return 'OK: Events accepted'
+        rate_limiting(
+            request,
+            application,
+            "per_application_reports_rate_limit",
+            len(deserialized_reports),
+        )
+        tasks.add_reports.delay(application.resource_id, {}, deserialized_reports)
+    return "OK: Events accepted"

backend/src/appenlight/views/applications.py

@@ -1,759 +1,846 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import copy
 import json
 import logging
 
 from datetime import datetime, timedelta
 
 import colander
 from pyramid.httpexceptions import HTTPFound, HTTPUnprocessableEntity
 from pyramid.view import view_config
 from webob.multidict import MultiDict
 from zope.sqlalchemy import mark_changed
 from ziggurat_foundations.permissions import ANY_PERMISSION
 
 import appenlight.forms as forms
 from appenlight.models import DBSession
 from appenlight.models.resource import Resource
 from appenlight.models.application import Application
-from appenlight.models.application_postprocess_conf import \
-    ApplicationPostprocessConf
+from appenlight.models.application_postprocess_conf import ApplicationPostprocessConf
 from ziggurat_foundations.models.services.user import UserService
 from ziggurat_foundations.models.services.resource import ResourceService
-from ziggurat_foundations.models.services.user_resource_permission import UserResourcePermissionService
+from ziggurat_foundations.models.services.user_resource_permission import (
+    UserResourcePermissionService,
+)
 from appenlight.models.user_resource_permission import UserResourcePermission
 from appenlight.models.group_resource_permission import GroupResourcePermission
 from appenlight.models.services.application import ApplicationService
-from appenlight.models.services.application_postprocess_conf import \
-    ApplicationPostprocessConfService
+from appenlight.models.services.application_postprocess_conf import (
+    ApplicationPostprocessConfService,
+)
 from appenlight.models.services.group import GroupService
-from appenlight.models.services.group_resource_permission import \
-    GroupResourcePermissionService
+from appenlight.models.services.group_resource_permission import (
+    GroupResourcePermissionService,
+)
 from appenlight.models.services.request_metric import RequestMetricService
 from appenlight.models.services.report_group import ReportGroupService
 from appenlight.models.services.slow_call import SlowCallService
 from appenlight.lib import helpers as h
 from appenlight.lib.utils import build_filter_settings_from_query_dict
 from appenlight.security import RootFactory
 from appenlight.models.report import REPORT_TYPE_MATRIX
 from appenlight.validators import build_rule_schema
 
 _ = str
 
 log = logging.getLogger(__name__)
 
 
 def app_not_found(request, id):
     """
     Redirects on non found and sets a flash message
     """
-    request.session.flash(_('Application not found'), 'warning')
-    return HTTPFound(
-        location=request.route_url('applications', action='index'))
+    request.session.flash(_("Application not found"), "warning")
+    return HTTPFound(location=request.route_url("applications", action="index"))
 
 
-@view_config(route_name='applications_no_id',
-             renderer='json', request_method="GET", permission='authenticated')
+@view_config(
+    route_name="applications_no_id",
+    renderer="json",
+    request_method="GET",
+    permission="authenticated",
+)
 def applications_list(request):
     """
     Applications list
 
     if query params contain ?type=foo, it will list applications
     with one of those permissions for user,
     otherwise only list of owned applications will
     be returned
 
     appending ?root_list while being administration will allow to list all
     applications in the system
 
     """
-    is_root = request.has_permission('root_administration',
-                                     RootFactory(request))
-    if is_root and request.GET.get('root_list'):
+    is_root = request.has_permission("root_administration", RootFactory(request))
+    if is_root and request.GET.get("root_list"):
         resources = Resource.all().order_by(Resource.resource_name)
-        resource_type = request.GET.get('resource_type', 'application')
+        resource_type = request.GET.get("resource_type", "application")
         if resource_type:
-            resources = resources.filter(
-                Resource.resource_type == resource_type)
+            resources = resources.filter(Resource.resource_type == resource_type)
     else:
-        permissions = request.params.getall('permission')
+        permissions = request.params.getall("permission")
         if permissions:
             resources = UserService.resources_with_perms(
                 request.user,
                 permissions,
-                resource_types=[request.GET.get('resource_type',
-                                                'application')])
+                resource_types=[request.GET.get("resource_type", "application")],
+            )
         else:
             resources = request.user.resources.filter(
-                Application.resource_type == request.GET.get(
-                    'resource_type',
-                    'application'))
-    return [r.get_dict(include_keys=['resource_id', 'resource_name', 'domains',
-                                     'owner_user_name', 'owner_group_name'])
-            for
-            r in resources]
-
-
-@view_config(route_name='applications', renderer='json',
-             request_method="GET", permission='view')
+                Application.resource_type
+                == request.GET.get("resource_type", "application")
+            )
+    return [
+        r.get_dict(
+            include_keys=[
+                "resource_id",
+                "resource_name",
+                "domains",
+                "owner_user_name",
+                "owner_group_name",
+            ]
+        )
+        for r in resources
+    ]
+
+
+@view_config(
+    route_name="applications", renderer="json", request_method="GET", permission="view"
+)
 def application_GET(request):
     resource = request.context.resource
     include_sensitive_info = False
-    if request.has_permission('edit'):
+    if request.has_permission("edit"):
         include_sensitive_info = True
     resource_dict = resource.get_dict(
         include_perms=include_sensitive_info,
-        include_processing_rules=include_sensitive_info)
+        include_processing_rules=include_sensitive_info,
+    )
     return resource_dict
 
 
-@view_config(route_name='applications_no_id', request_method="POST",
-             renderer='json', permission='create_resources')
+@view_config(
+    route_name="applications_no_id",
+    request_method="POST",
+    renderer="json",
+    permission="create_resources",
+)
 def application_create(request):
     """
     Creates new application instances
     """
     user = request.user
-    form = forms.ApplicationCreateForm(MultiDict(request.unsafe_json_body),
-                                       csrf_context=request)
+    form = forms.ApplicationCreateForm(
+        MultiDict(request.unsafe_json_body), csrf_context=request
+    )
     if form.validate():
         session = DBSession()
         resource = Application()
         DBSession.add(resource)
         form.populate_obj(resource)
         resource.api_key = resource.generate_api_key()
         user.resources.append(resource)
-        request.session.flash(_('Application created'))
+        request.session.flash(_("Application created"))
         DBSession.flush()
         mark_changed(session)
     else:
         return HTTPUnprocessableEntity(body=form.errors_json)
 
     return resource.get_dict()
 
 
-@view_config(route_name='applications', request_method="PATCH",
-             renderer='json', permission='edit')
+@view_config(
+    route_name="applications",
+    request_method="PATCH",
+    renderer="json",
+    permission="edit",
+)
 def application_update(request):
     """
     Updates main application configuration
     """
     resource = request.context.resource
     if not resource:
         return app_not_found()
 
     # disallow setting permanent storage by non-admins
     # use default/non-resource based context for this check
     req_dict = copy.copy(request.unsafe_json_body)
-    if not request.has_permission('root_administration', RootFactory(request)):
-        req_dict['allow_permanent_storage'] = ''
-    if not req_dict.get('uptime_url'):
+    if not request.has_permission("root_administration", RootFactory(request)):
+        req_dict["allow_permanent_storage"] = ""
+    if not req_dict.get("uptime_url"):
         # needed cause validator is still triggered by default
-        req_dict.pop('uptime_url', '')
-    application_form = forms.ApplicationUpdateForm(MultiDict(req_dict),
-                                                   csrf_context=request)
+        req_dict.pop("uptime_url", "")
+    application_form = forms.ApplicationUpdateForm(
+        MultiDict(req_dict), csrf_context=request
+    )
     if application_form.validate():
         application_form.populate_obj(resource)
-        request.session.flash(_('Application updated'))
+        request.session.flash(_("Application updated"))
     else:
         return HTTPUnprocessableEntity(body=application_form.errors_json)
 
     include_sensitive_info = False
-    if request.has_permission('edit'):
+    if request.has_permission("edit"):
         include_sensitive_info = True
     resource_dict = resource.get_dict(
         include_perms=include_sensitive_info,
-        include_processing_rules=include_sensitive_info)
+        include_processing_rules=include_sensitive_info,
+    )
     return resource_dict
 
 
-@view_config(route_name='applications_property', match_param='key=api_key',
-             request_method="POST", renderer='json',
-             permission='delete')
+@view_config(
+    route_name="applications_property",
+    match_param="key=api_key",
+    request_method="POST",
+    renderer="json",
+    permission="delete",
+)
 def application_regenerate_key(request):
     """
     Regenerates API keys for application
     """
     resource = request.context.resource
 
-    form = forms.CheckPasswordForm(MultiDict(request.unsafe_json_body),
-                                   csrf_context=request)
+    form = forms.CheckPasswordForm(
+        MultiDict(request.unsafe_json_body), csrf_context=request
+    )
     form.password.user = request.user
 
     if form.validate():
         resource.api_key = resource.generate_api_key()
         resource.public_key = resource.generate_api_key()
-        msg = 'API keys regenerated - please update your application config.'
+        msg = "API keys regenerated - please update your application config."
         request.session.flash(_(msg))
     else:
         return HTTPUnprocessableEntity(body=form.errors_json)
 
-    if request.has_permission('edit'):
+    if request.has_permission("edit"):
         include_sensitive_info = True
     resource_dict = resource.get_dict(
         include_perms=include_sensitive_info,
-        include_processing_rules=include_sensitive_info)
+        include_processing_rules=include_sensitive_info,
+    )
     return resource_dict
 
 
-@view_config(route_name='applications_property',
-             match_param='key=delete_resource',
-             request_method="PATCH", renderer='json', permission='delete')
+@view_config(
+    route_name="applications_property",
+    match_param="key=delete_resource",
+    request_method="PATCH",
+    renderer="json",
+    permission="delete",
+)
 def application_remove(request):
     """
     Removes application resources
     """
     resource = request.context.resource
     # we need polymorphic object here, to properly launch sqlalchemy events
     resource = ApplicationService.by_id(resource.resource_id)
-    form = forms.CheckPasswordForm(MultiDict(request.safe_json_body or {}),
-                                   csrf_context=request)
+    form = forms.CheckPasswordForm(
+        MultiDict(request.safe_json_body or {}), csrf_context=request
+    )
     form.password.user = request.user
     if form.validate():
         DBSession.delete(resource)
-        request.session.flash(_('Application removed'))
+        request.session.flash(_("Application removed"))
     else:
         return HTTPUnprocessableEntity(body=form.errors_json)
 
     return True
 
 
-@view_config(route_name='applications_property', match_param='key=owner',
-             request_method="PATCH", renderer='json', permission='delete')
+@view_config(
+    route_name="applications_property",
+    match_param="key=owner",
+    request_method="PATCH",
+    renderer="json",
+    permission="delete",
+)
 def application_ownership_transfer(request):
     """
     Allows application owner to transfer application ownership to other user
     """
     resource = request.context.resource
     form = forms.ChangeApplicationOwnerForm(
-        MultiDict(request.safe_json_body or {}), csrf_context=request)
+        MultiDict(request.safe_json_body or {}), csrf_context=request
+    )
     form.password.user = request.user
     if form.validate():
         user = UserService.by_user_name(form.user_name.data)
         user.resources.append(resource)
         # remove integrations to not leak security data of external applications
         for integration in resource.integrations[:]:
             resource.integrations.remove(integration)
-        request.session.flash(_('Application transfered'))
+        request.session.flash(_("Application transfered"))
     else:
         return HTTPUnprocessableEntity(body=form.errors_json)
     return True
 
 
-@view_config(route_name='applications_property',
-             match_param='key=postprocessing_rules', renderer='json',
-             request_method='POST', permission='edit')
+@view_config(
+    route_name="applications_property",
+    match_param="key=postprocessing_rules",
+    renderer="json",
+    request_method="POST",
+    permission="edit",
+)
 def applications_postprocess_POST(request):
     """
     Creates new postprocessing rules for applications
     """
     resource = request.context.resource
     conf = ApplicationPostprocessConf()
-    conf.do = 'postprocess'
-    conf.new_value = '1'
+    conf.do = "postprocess"
+    conf.new_value = "1"
     resource.postprocess_conf.append(conf)
     DBSession.flush()
     return conf.get_dict()
 
 
-@view_config(route_name='applications_property',
-             match_param='key=postprocessing_rules', renderer='json',
-             request_method='PATCH', permission='edit')
+@view_config(
+    route_name="applications_property",
+    match_param="key=postprocessing_rules",
+    renderer="json",
+    request_method="PATCH",
+    permission="edit",
+)
 def applications_postprocess_PATCH(request):
     """
     Creates new postprocessing rules for applications
     """
     json_body = request.unsafe_json_body
 
-    schema = build_rule_schema(json_body['rule'], REPORT_TYPE_MATRIX)
+    schema = build_rule_schema(json_body["rule"], REPORT_TYPE_MATRIX)
     try:
-        schema.deserialize(json_body['rule'])
+        schema.deserialize(json_body["rule"])
     except colander.Invalid as exc:
         return HTTPUnprocessableEntity(body=json.dumps(exc.asdict()))
 
     resource = request.context.resource
     conf = ApplicationPostprocessConfService.by_pkey_and_resource_id(
-        json_body['pkey'], resource.resource_id)
-    conf.rule = request.unsafe_json_body['rule']
+        json_body["pkey"], resource.resource_id
+    )
+    conf.rule = request.unsafe_json_body["rule"]
     # for now hardcode int since we dont support anything else so far
-    conf.new_value = int(request.unsafe_json_body['new_value'])
+    conf.new_value = int(request.unsafe_json_body["new_value"])
     return conf.get_dict()
 
 
-@view_config(route_name='applications_property',
-             match_param='key=postprocessing_rules', renderer='json',
-             request_method='DELETE', permission='edit')
+@view_config(
+    route_name="applications_property",
+    match_param="key=postprocessing_rules",
+    renderer="json",
+    request_method="DELETE",
+    permission="edit",
+)
 def applications_postprocess_DELETE(request):
     """
     Removes application postprocessing rules
     """
     form = forms.ReactorForm(request.POST, csrf_context=request)
     resource = request.context.resource
     if form.validate():
         for postprocess_conf in resource.postprocess_conf:
-            if postprocess_conf.pkey == int(request.GET['pkey']):
+            if postprocess_conf.pkey == int(request.GET["pkey"]):
                 # remove rule
                 DBSession.delete(postprocess_conf)
     return True
 
 
-@view_config(route_name='applications_property',
-             match_param='key=report_graphs', renderer='json',
-             permission='view')
-@view_config(route_name='applications_property',
-             match_param='key=slow_report_graphs', renderer='json',
-             permission='view')
+@view_config(
+    route_name="applications_property",
+    match_param="key=report_graphs",
+    renderer="json",
+    permission="view",
+)
+@view_config(
+    route_name="applications_property",
+    match_param="key=slow_report_graphs",
+    renderer="json",
+    permission="view",
+)
 def get_application_report_stats(request):
     query_params = request.GET.mixed()
-    query_params['resource'] = (request.context.resource.resource_id,)
+    query_params["resource"] = (request.context.resource.resource_id,)
 
-    filter_settings = build_filter_settings_from_query_dict(request,
-                                                            query_params)
-    if not filter_settings.get('end_date'):
+    filter_settings = build_filter_settings_from_query_dict(request, query_params)
+    if not filter_settings.get("end_date"):
         end_date = datetime.utcnow().replace(microsecond=0, second=0)
-        filter_settings['end_date'] = end_date
+        filter_settings["end_date"] = end_date
 
-    if not filter_settings.get('start_date'):
+    if not filter_settings.get("start_date"):
         delta = timedelta(hours=1)
-        filter_settings['start_date'] = filter_settings['end_date'] - delta
+        filter_settings["start_date"] = filter_settings["end_date"] - delta
 
     result = ReportGroupService.get_report_stats(request, filter_settings)
     return result
 
 
-@view_config(route_name='applications_property',
-             match_param='key=metrics_graphs', renderer='json',
-             permission='view')
+@view_config(
+    route_name="applications_property",
+    match_param="key=metrics_graphs",
+    renderer="json",
+    permission="view",
+)
 def metrics_graphs(request):
     """
     Handles metric dashboard graphs
     Returns information for time/tier breakdown
     """
     query_params = request.GET.mixed()
-    query_params['resource'] = (request.context.resource.resource_id,)
+    query_params["resource"] = (request.context.resource.resource_id,)
 
-    filter_settings = build_filter_settings_from_query_dict(request,
-                                                            query_params)
+    filter_settings = build_filter_settings_from_query_dict(request, query_params)
 
-    if not filter_settings.get('end_date'):
+    if not filter_settings.get("end_date"):
         end_date = datetime.utcnow().replace(microsecond=0, second=0)
-        filter_settings['end_date'] = end_date
+        filter_settings["end_date"] = end_date
 
     delta = timedelta(hours=1)
-    if not filter_settings.get('start_date'):
-        filter_settings['start_date'] = filter_settings['end_date'] - delta
-    if filter_settings['end_date'] <= filter_settings['start_date']:
-        filter_settings['end_date'] = filter_settings['start_date']
+    if not filter_settings.get("start_date"):
+        filter_settings["start_date"] = filter_settings["end_date"] - delta
+    if filter_settings["end_date"] <= filter_settings["start_date"]:
+        filter_settings["end_date"] = filter_settings["start_date"]
 
-    delta = filter_settings['end_date'] - filter_settings['start_date']
-    if delta < h.time_deltas.get('12h')['delta']:
+    delta = filter_settings["end_date"] - filter_settings["start_date"]
+    if delta < h.time_deltas.get("12h")["delta"]:
         divide_by_min = 1
-    elif delta <= h.time_deltas.get('3d')['delta']:
+    elif delta <= h.time_deltas.get("3d")["delta"]:
         divide_by_min = 5.0
-    elif delta >= h.time_deltas.get('2w')['delta']:
+    elif delta >= h.time_deltas.get("2w")["delta"]:
         divide_by_min = 60.0 * 24
     else:
         divide_by_min = 60.0
 
-    results = RequestMetricService.get_metrics_stats(
-        request, filter_settings)
+    results = RequestMetricService.get_metrics_stats(request, filter_settings)
     # because requests are PER SECOND / we divide 1 min stats by 60
     # requests are normalized to 1 min average
     # results are average seconds time spent per request in specific area
     for point in results:
-        if point['requests']:
-            point['main'] = (point['main'] - point['sql'] -
-                             point['nosql'] - point['remote'] -
-                             point['tmpl'] -
-                             point['custom']) / point['requests']
-            point['sql'] = point['sql'] / point['requests']
-            point['nosql'] = point['nosql'] / point['requests']
-            point['remote'] = point['remote'] / point['requests']
-            point['tmpl'] = point['tmpl'] / point['requests']
-            point['custom'] = point['custom'] / point['requests']
-            point['requests_2'] = point['requests'] / 60.0 / divide_by_min
-
-    selected_types = ['main', 'sql', 'nosql', 'remote', 'tmpl', 'custom']
+        if point["requests"]:
+            point["main"] = (
+                point["main"]
+                - point["sql"]
+                - point["nosql"]
+                - point["remote"]
+                - point["tmpl"]
+                - point["custom"]
+            ) / point["requests"]
+            point["sql"] = point["sql"] / point["requests"]
+            point["nosql"] = point["nosql"] / point["requests"]
+            point["remote"] = point["remote"] / point["requests"]
+            point["tmpl"] = point["tmpl"] / point["requests"]
+            point["custom"] = point["custom"] / point["requests"]
+            point["requests_2"] = point["requests"] / 60.0 / divide_by_min
+
+    selected_types = ["main", "sql", "nosql", "remote", "tmpl", "custom"]
 
     for point in results:
         for stat_type in selected_types:
             point[stat_type] = round(point.get(stat_type, 0), 3)
 
     return results
 
 
-@view_config(route_name='applications_property',
-             match_param='key=response_graphs', renderer='json',
-             permission='view')
+@view_config(
+    route_name="applications_property",
+    match_param="key=response_graphs",
+    renderer="json",
+    permission="view",
+)
 def response_graphs(request):
     """
     Handles dashboard infomation for avg. response time split by today,
     2 days ago and week ago
     """
     query_params = request.GET.mixed()
-    query_params['resource'] = (request.context.resource.resource_id,)
+    query_params["resource"] = (request.context.resource.resource_id,)
 
-    filter_settings = build_filter_settings_from_query_dict(request,
-                                                            query_params)
+    filter_settings = build_filter_settings_from_query_dict(request, query_params)
 
-    if not filter_settings.get('end_date'):
+    if not filter_settings.get("end_date"):
         end_date = datetime.utcnow().replace(microsecond=0, second=0)
-        filter_settings['end_date'] = end_date
+        filter_settings["end_date"] = end_date
 
     delta = timedelta(hours=1)
-    if not filter_settings.get('start_date'):
-        filter_settings['start_date'] = filter_settings['end_date'] - delta
+    if not filter_settings.get("start_date"):
+        filter_settings["start_date"] = filter_settings["end_date"] - delta
 
-    result_now = RequestMetricService.get_metrics_stats(
-        request, filter_settings)
+    result_now = RequestMetricService.get_metrics_stats(request, filter_settings)
 
     filter_settings_2d = filter_settings.copy()
-    filter_settings_2d['start_date'] = filter_settings['start_date'] - \
-                                       timedelta(days=2)
-    filter_settings_2d['end_date'] = filter_settings['end_date'] - \
-                                     timedelta(days=2)
-    result_2d = RequestMetricService.get_metrics_stats(
-        request, filter_settings_2d)
+    filter_settings_2d["start_date"] = filter_settings["start_date"] - timedelta(days=2)
+    filter_settings_2d["end_date"] = filter_settings["end_date"] - timedelta(days=2)
+    result_2d = RequestMetricService.get_metrics_stats(request, filter_settings_2d)
 
     filter_settings_7d = filter_settings.copy()
-    filter_settings_7d['start_date'] = filter_settings['start_date'] - \
-                                       timedelta(days=7)
-    filter_settings_7d['end_date'] = filter_settings['end_date'] - \
-                                     timedelta(days=7)
-    result_7d = RequestMetricService.get_metrics_stats(
-        request, filter_settings_7d)
+    filter_settings_7d["start_date"] = filter_settings["start_date"] - timedelta(days=7)
+    filter_settings_7d["end_date"] = filter_settings["end_date"] - timedelta(days=7)
+    result_7d = RequestMetricService.get_metrics_stats(request, filter_settings_7d)
 
     plot_data = []
 
     for item in result_now:
-        point = {'x': item['x'], 'today': 0, 'days_ago_2': 0,
-                 'days_ago_7': 0}
-        if item['requests']:
-            point['today'] = round(item['main'] / item['requests'], 3)
+        point = {"x": item["x"], "today": 0, "days_ago_2": 0, "days_ago_7": 0}
+        if item["requests"]:
+            point["today"] = round(item["main"] / item["requests"], 3)
         plot_data.append(point)
 
-    for i, item in enumerate(result_2d[:len(plot_data)]):
-        plot_data[i]['days_ago_2'] = 0
+    for i, item in enumerate(result_2d[: len(plot_data)]):
+        plot_data[i]["days_ago_2"] = 0
         point = result_2d[i]
-        if point['requests']:
-            plot_data[i]['days_ago_2'] = round(point['main'] /
-                                               point['requests'], 3)
+        if point["requests"]:
+            plot_data[i]["days_ago_2"] = round(point["main"] / point["requests"], 3)
 
-    for i, item in enumerate(result_7d[:len(plot_data)]):
-        plot_data[i]['days_ago_7'] = 0
+    for i, item in enumerate(result_7d[: len(plot_data)]):
+        plot_data[i]["days_ago_7"] = 0
         point = result_7d[i]
-        if point['requests']:
-            plot_data[i]['days_ago_7'] = round(point['main'] /
-                                               point['requests'], 3)
+        if point["requests"]:
+            plot_data[i]["days_ago_7"] = round(point["main"] / point["requests"], 3)
 
     return plot_data
 
 
-@view_config(route_name='applications_property',
-             match_param='key=requests_graphs', renderer='json',
-             permission='view')
+@view_config(
+    route_name="applications_property",
+    match_param="key=requests_graphs",
+    renderer="json",
+    permission="view",
+)
 def requests_graphs(request):
     """
     Handles dashboard infomation for avg. response time split by today,
     2 days ago and week ago
     """
     query_params = request.GET.mixed()
-    query_params['resource'] = (request.context.resource.resource_id,)
+    query_params["resource"] = (request.context.resource.resource_id,)
 
-    filter_settings = build_filter_settings_from_query_dict(request,
-                                                            query_params)
+    filter_settings = build_filter_settings_from_query_dict(request, query_params)
 
-    if not filter_settings.get('end_date'):
+    if not filter_settings.get("end_date"):
         end_date = datetime.utcnow().replace(microsecond=0, second=0)
-        filter_settings['end_date'] = end_date
+        filter_settings["end_date"] = end_date
 
     delta = timedelta(hours=1)
-    if not filter_settings.get('start_date'):
-        filter_settings['start_date'] = filter_settings['end_date'] - delta
-
-    result_now = RequestMetricService.get_metrics_stats(
-        request, filter_settings)
-
-    delta = filter_settings['end_date'] - filter_settings['start_date']
-    if delta < h.time_deltas.get('12h')['delta']:
-        seconds = h.time_deltas['1m']['minutes'] * 60.0
-    elif delta <= h.time_deltas.get('3d')['delta']:
-        seconds = h.time_deltas['5m']['minutes'] * 60.0
-    elif delta >= h.time_deltas.get('2w')['delta']:
-        seconds = h.time_deltas['24h']['minutes'] * 60.0
+    if not filter_settings.get("start_date"):
+        filter_settings["start_date"] = filter_settings["end_date"] - delta
+
+    result_now = RequestMetricService.get_metrics_stats(request, filter_settings)
+
+    delta = filter_settings["end_date"] - filter_settings["start_date"]
+    if delta < h.time_deltas.get("12h")["delta"]:
+        seconds = h.time_deltas["1m"]["minutes"] * 60.0
+    elif delta <= h.time_deltas.get("3d")["delta"]:
+        seconds = h.time_deltas["5m"]["minutes"] * 60.0
+    elif delta >= h.time_deltas.get("2w")["delta"]:
+        seconds = h.time_deltas["24h"]["minutes"] * 60.0
     else:
-        seconds = h.time_deltas['1h']['minutes'] * 60.0
+        seconds = h.time_deltas["1h"]["minutes"] * 60.0
 
     for item in result_now:
-        if item['requests']:
-            item['requests'] = round(item['requests'] / seconds, 3)
+        if item["requests"]:
+            item["requests"] = round(item["requests"] / seconds, 3)
     return result_now
 
 
-@view_config(route_name='applications_property',
-             match_param='key=apdex_stats', renderer='json',
-             permission='view')
+@view_config(
+    route_name="applications_property",
+    match_param="key=apdex_stats",
+    renderer="json",
+    permission="view",
+)
 def get_apdex_stats(request):
     """
     Returns information and calculates APDEX score per server for dashboard
     server information (upper right stats boxes)
     """
     query_params = request.GET.mixed()
-    query_params['resource'] = (request.context.resource.resource_id,)
+    query_params["resource"] = (request.context.resource.resource_id,)
 
-    filter_settings = build_filter_settings_from_query_dict(request,
-                                                            query_params)
+    filter_settings = build_filter_settings_from_query_dict(request, query_params)
     # make sure we have only one resource here to don't produce
     # weird results when we have wrong app in app selector
-    filter_settings['resource'] = [filter_settings['resource'][0]]
+    filter_settings["resource"] = [filter_settings["resource"][0]]
 
-    if not filter_settings.get('end_date'):
+    if not filter_settings.get("end_date"):
         end_date = datetime.utcnow().replace(microsecond=0, second=0)
-        filter_settings['end_date'] = end_date
+        filter_settings["end_date"] = end_date
 
     delta = timedelta(hours=1)
-    if not filter_settings.get('start_date'):
-        filter_settings['start_date'] = filter_settings['end_date'] - delta
+    if not filter_settings.get("start_date"):
+        filter_settings["start_date"] = filter_settings["end_date"] - delta
 
     return RequestMetricService.get_apdex_stats(request, filter_settings)
 
 
-@view_config(route_name='applications_property', match_param='key=slow_calls',
-             renderer='json', permission='view')
+@view_config(
+    route_name="applications_property",
+    match_param="key=slow_calls",
+    renderer="json",
+    permission="view",
+)
 def get_slow_calls(request):
     """
     Returns information for time consuming calls in specific time interval
     """
     query_params = request.GET.mixed()
-    query_params['resource'] = (request.context.resource.resource_id,)
+    query_params["resource"] = (request.context.resource.resource_id,)
 
-    filter_settings = build_filter_settings_from_query_dict(request,
-                                                            query_params)
+    filter_settings = build_filter_settings_from_query_dict(request, query_params)
 
-    if not filter_settings.get('end_date'):
+    if not filter_settings.get("end_date"):
         end_date = datetime.utcnow().replace(microsecond=0, second=0)
-        filter_settings['end_date'] = end_date
+        filter_settings["end_date"] = end_date
 
     delta = timedelta(hours=1)
-    if not filter_settings.get('start_date'):
-        filter_settings['start_date'] = filter_settings['end_date'] - delta
+    if not filter_settings.get("start_date"):
+        filter_settings["start_date"] = filter_settings["end_date"] - delta
 
     return SlowCallService.get_time_consuming_calls(request, filter_settings)
 
 
-@view_config(route_name='applications_property',
-             match_param='key=requests_breakdown',
-             renderer='json', permission='view')
+@view_config(
+    route_name="applications_property",
+    match_param="key=requests_breakdown",
+    renderer="json",
+    permission="view",
+)
 def get_requests_breakdown(request):
     """
     Used on dashboard to get information which views are most used in
     a time interval
     """
     query_params = request.GET.mixed()
-    query_params['resource'] = (request.context.resource.resource_id,)
+    query_params["resource"] = (request.context.resource.resource_id,)
 
-    filter_settings = build_filter_settings_from_query_dict(request,
-                                                            query_params)
-    if not filter_settings.get('end_date'):
+    filter_settings = build_filter_settings_from_query_dict(request, query_params)
+    if not filter_settings.get("end_date"):
         end_date = datetime.utcnow().replace(microsecond=0, second=0)
-        filter_settings['end_date'] = end_date
+        filter_settings["end_date"] = end_date
 
-    if not filter_settings.get('start_date'):
+    if not filter_settings.get("start_date"):
         delta = timedelta(hours=1)
-        filter_settings['start_date'] = filter_settings['end_date'] - delta
+        filter_settings["start_date"] = filter_settings["end_date"] - delta
 
-    series = RequestMetricService.get_requests_breakdown(
-        request, filter_settings)
+    series = RequestMetricService.get_requests_breakdown(request, filter_settings)
 
     results = []
     for row in series:
-        d_row = {'avg_response': round(row['main'] / row['requests'], 3),
-                 'requests': row['requests'],
-                 'main': row['main'],
-                 'view_name': row['key'],
-                 'latest_details': row['latest_details'],
-                 'percentage': round(row['percentage'] * 100, 1)}
+        d_row = {
+            "avg_response": round(row["main"] / row["requests"], 3),
+            "requests": row["requests"],
+            "main": row["main"],
+            "view_name": row["key"],
+            "latest_details": row["latest_details"],
+            "percentage": round(row["percentage"] * 100, 1),
+        }
 
         results.append(d_row)
 
     return results
 
 
-@view_config(route_name='applications_property',
-             match_param='key=trending_reports', renderer='json',
-             permission='view')
+@view_config(
+    route_name="applications_property",
+    match_param="key=trending_reports",
+    renderer="json",
+    permission="view",
+)
 def trending_reports(request):
     """
     Returns exception/slow reports trending for specific time interval
     """
     query_params = request.GET.mixed().copy()
     # pop report type to rewrite it to tag later
-    report_type = query_params.pop('report_type', None)
+    report_type = query_params.pop("report_type", None)
     if report_type:
-        query_params['type'] = report_type
+        query_params["type"] = report_type
 
-    query_params['resource'] = (request.context.resource.resource_id,)
+    query_params["resource"] = (request.context.resource.resource_id,)
 
-    filter_settings = build_filter_settings_from_query_dict(request,
-                                                            query_params)
+    filter_settings = build_filter_settings_from_query_dict(request, query_params)
 
-    if not filter_settings.get('end_date'):
+    if not filter_settings.get("end_date"):
         end_date = datetime.utcnow().replace(microsecond=0, second=0)
-        filter_settings['end_date'] = end_date
+        filter_settings["end_date"] = end_date
 
-    if not filter_settings.get('start_date'):
+    if not filter_settings.get("start_date"):
         delta = timedelta(hours=1)
-        filter_settings['start_date'] = filter_settings['end_date'] - delta
+        filter_settings["start_date"] = filter_settings["end_date"] - delta
 
     results = ReportGroupService.get_trending(request, filter_settings)
 
     trending = []
     for occurences, group in results:
         report_group = group.get_dict(request)
         # show the occurences in time range instead of global ones
-        report_group['occurences'] = occurences
+        report_group["occurences"] = occurences
         trending.append(report_group)
 
     return trending
 
 
-@view_config(route_name='applications_property',
-             match_param='key=integrations',
-             renderer='json', permission='view')
+@view_config(
+    route_name="applications_property",
+    match_param="key=integrations",
+    renderer="json",
+    permission="view",
+)
 def integrations(request):
     """
     Integration list for given application
     """
     application = request.context.resource
-    return {'resource': application}
+    return {"resource": application}
 
 
-@view_config(route_name='applications_property',
-             match_param='key=user_permissions', renderer='json',
-             permission='owner', request_method='POST')
+@view_config(
+    route_name="applications_property",
+    match_param="key=user_permissions",
+    renderer="json",
+    permission="owner",
+    request_method="POST",
+)
 def user_resource_permission_create(request):
     """
     Set new permissions for user for a resource
     """
     resource = request.context.resource
-    user_name = request.unsafe_json_body.get('user_name')
+    user_name = request.unsafe_json_body.get("user_name")
     user = UserService.by_user_name(user_name)
     if not user:
         user = UserService.by_email(user_name)
     if not user:
         return False
 
-    for perm_name in request.unsafe_json_body.get('permissions', []):
+    for perm_name in request.unsafe_json_body.get("permissions", []):
         permission = UserResourcePermissionService.by_resource_user_and_perm(
-            user.id, perm_name, resource.resource_id)
+            user.id, perm_name, resource.resource_id
+        )
         if not permission:
-            permission = UserResourcePermission(perm_name=perm_name,
-                                                user_id=user.id)
+            permission = UserResourcePermission(perm_name=perm_name, user_id=user.id)
             resource.user_permissions.append(permission)
     DBSession.flush()
-    perms = [p.perm_name for p in ResourceService.perms_for_user(resource, user)
-             if p.type == 'user']
-    result = {'user_name': user.user_name,
-              'permissions': list(set(perms))}
+    perms = [
+        p.perm_name
+        for p in ResourceService.perms_for_user(resource, user)
+        if p.type == "user"
+    ]
+    result = {"user_name": user.user_name, "permissions": list(set(perms))}
     return result
 
 
-@view_config(route_name='applications_property',
-             match_param='key=user_permissions', renderer='json',
-             permission='owner', request_method='DELETE')
+@view_config(
+    route_name="applications_property",
+    match_param="key=user_permissions",
+    renderer="json",
+    permission="owner",
+    request_method="DELETE",
+)
 def user_resource_permission_delete(request):
     """
     Removes user permission from specific resource
     """
     resource = request.context.resource
 
-    user = UserService.by_user_name(request.GET.get('user_name'))
+    user = UserService.by_user_name(request.GET.get("user_name"))
     if not user:
         return False
 
-    for perm_name in request.GET.getall('permissions'):
+    for perm_name in request.GET.getall("permissions"):
         permission = UserResourcePermissionService.by_resource_user_and_perm(
-            user.id, perm_name, resource.resource_id)
+            user.id, perm_name, resource.resource_id
+        )
         resource.user_permissions.remove(permission)
     DBSession.flush()
-    perms = [p.perm_name for p in ResourceService.perms_for_user(resource, user)
-             if p.type == 'user']
-    result = {'user_name': user.user_name,
-              'permissions': list(set(perms))}
+    perms = [
+        p.perm_name
+        for p in ResourceService.perms_for_user(resource, user)
+        if p.type == "user"
+    ]
+    result = {"user_name": user.user_name, "permissions": list(set(perms))}
     return result
 
 
-@view_config(route_name='applications_property',
-             match_param='key=group_permissions', renderer='json',
-             permission='owner', request_method='POST')
+@view_config(
+    route_name="applications_property",
+    match_param="key=group_permissions",
+    renderer="json",
+    permission="owner",
+    request_method="POST",
+)
 def group_resource_permission_create(request):
     """
     Set new permissions for group for a resource
     """
     resource = request.context.resource
-    group = GroupService.by_id(request.unsafe_json_body.get('group_id'))
+    group = GroupService.by_id(request.unsafe_json_body.get("group_id"))
     if not group:
         return False
 
-    for perm_name in request.unsafe_json_body.get('permissions', []):
+    for perm_name in request.unsafe_json_body.get("permissions", []):
         permission = GroupResourcePermissionService.by_resource_group_and_perm(
-            group.id, perm_name, resource.resource_id)
+            group.id, perm_name, resource.resource_id
+        )
         if not permission:
-            permission = GroupResourcePermission(perm_name=perm_name,
-                                                 group_id=group.id)
+            permission = GroupResourcePermission(perm_name=perm_name, group_id=group.id)
             resource.group_permissions.append(permission)
     DBSession.flush()
     perm_tuples = ResourceService.groups_for_perm(
-        resource,
-        ANY_PERMISSION,
-        limit_group_permissions=True,
-        group_ids=[group.id])
-    perms = [p.perm_name for p in perm_tuples if p.type == 'group']
-    result = {'group': group.get_dict(),
-              'permissions': list(set(perms))}
+        resource, ANY_PERMISSION, limit_group_permissions=True, group_ids=[group.id]
+    )
+    perms = [p.perm_name for p in perm_tuples if p.type == "group"]
+    result = {"group": group.get_dict(), "permissions": list(set(perms))}
     return result
 
 
-@view_config(route_name='applications_property',
-             match_param='key=group_permissions', renderer='json',
-             permission='owner', request_method='DELETE')
+@view_config(
+    route_name="applications_property",
+    match_param="key=group_permissions",
+    renderer="json",
+    permission="owner",
+    request_method="DELETE",
+)
 def group_resource_permission_delete(request):
     """
     Removes group permission from specific resource
     """
     form = forms.ReactorForm(request.POST, csrf_context=request)
     form.validate()
     resource = request.context.resource
-    group = GroupService.by_id(request.GET.get('group_id'))
+    group = GroupService.by_id(request.GET.get("group_id"))
     if not group:
         return False
 
-    for perm_name in request.GET.getall('permissions'):
+    for perm_name in request.GET.getall("permissions"):
         permission = GroupResourcePermissionService.by_resource_group_and_perm(
-            group.id, perm_name, resource.resource_id)
+            group.id, perm_name, resource.resource_id
+        )
         resource.group_permissions.remove(permission)
     DBSession.flush()
     perm_tuples = ResourceService.groups_for_perm(
-        resource,
-        ANY_PERMISSION,
-        limit_group_permissions=True,
-        group_ids=[group.id])
-    perms = [p.perm_name for p in perm_tuples if p.type == 'group']
-    result = {'group': group.get_dict(),
-              'permissions': list(set(perms))}
+        resource, ANY_PERMISSION, limit_group_permissions=True, group_ids=[group.id]
+    )
+    perms = [p.perm_name for p in perm_tuples if p.type == "group"]
+    result = {"group": group.get_dict(), "permissions": list(set(perms))}
     return result

backend/src/appenlight/views/events.py

@@ -1,60 +1,62 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from appenlight.lib.helpers import gen_pagination_headers
 from appenlight.models.services.event import EventService
 from pyramid.view import view_config
 from pyramid.httpexceptions import HTTPBadRequest, HTTPNotFound
 from ziggurat_foundations.models.services.user import UserService
 
 
-@view_config(route_name='events_no_id',
-             renderer='json', permission='authenticated')
+@view_config(route_name="events_no_id", renderer="json", permission="authenticated")
 def fetch_events(request):
     """
     Returns list of log entries from Elasticsearch
     """
     event_paginator = EventService.get_paginator(
-        user=request.user,
-        page=1,
-        items_per_page=100
+        user=request.user, page=1, items_per_page=100
     )
     headers = gen_pagination_headers(request, event_paginator)
     request.response.headers.update(headers)
 
     return [ev.get_dict() for ev in event_paginator.items]
 
 
-@view_config(route_name='events', renderer='json', request_method='PATCH',
-             permission='authenticated')
+@view_config(
+    route_name="events",
+    renderer="json",
+    request_method="PATCH",
+    permission="authenticated",
+)
 def event_PATCH(request):
     resources = UserService.resources_with_perms(
-        request.user, ['view'], resource_types=request.registry.resource_types)
+        request.user, ["view"], resource_types=request.registry.resource_types
+    )
     event = EventService.for_resource(
-        [r.resource_id for r in resources],
-        event_id=request.matchdict['event_id']).first()
+        [r.resource_id for r in resources], event_id=request.matchdict["event_id"]
+    ).first()
     if not event:
         return HTTPNotFound()
-    allowed_keys = ['status']
+    allowed_keys = ["status"]
     for k, v in request.unsafe_json_body.items():
         if k in allowed_keys:
-            if k == 'status':
+            if k == "status":
                 event.close()
             else:
                 setattr(event, k, v)
         else:
             return HTTPBadRequest()
     return event.get_dict()

backend/src/appenlight/views/groups.py

@@ -1,192 +1,246 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 
 from pyramid.view import view_config
 from pyramid.httpexceptions import HTTPUnprocessableEntity, HTTPNotFound
 
 from ziggurat_foundations.models.services.user import UserService
 from appenlight.lib.utils import permission_tuple_to_dict
 from appenlight.models.services.config import ConfigService
 from appenlight.models.group import Group
 from appenlight.models.services.group import GroupService
 from appenlight.models.user import User
 from appenlight.models import DBSession
 from appenlight import forms
 from webob.multidict import MultiDict
 
 log = logging.getLogger(__name__)
 
 _ = str
 
 
-@view_config(route_name='groups_no_id', renderer='json',
-             request_method="GET", permission='authenticated')
+@view_config(
+    route_name="groups_no_id",
+    renderer="json",
+    request_method="GET",
+    permission="authenticated",
+)
 def groups_list(request):
     """
     Returns groups list
     """
     groups = Group.all().order_by(Group.group_name)
     list_groups = ConfigService.by_key_and_section(
-        'list_groups_to_non_admins', 'global')
-    if list_groups.value or request.has_permission('root_administration'):
+        "list_groups_to_non_admins", "global"
+    )
+    if list_groups.value or request.has_permission("root_administration"):
         return [g.get_dict() for g in groups]
     else:
         return []
 
 
-@view_config(route_name='groups_no_id', renderer='json',
-             request_method="POST", permission='root_administration')
+@view_config(
+    route_name="groups_no_id",
+    renderer="json",
+    request_method="POST",
+    permission="root_administration",
+)
 def groups_create(request):
     """
     Returns groups list
     """
     form = forms.GroupCreateForm(
-        MultiDict(request.safe_json_body or {}), csrf_context=request)
+        MultiDict(request.safe_json_body or {}), csrf_context=request
+    )
     if form.validate():
-        log.info('registering group')
+        log.info("registering group")
         group = Group()
         # insert new group here
         DBSession.add(group)
         form.populate_obj(group)
-        request.session.flash(_('Group created'))
+        request.session.flash(_("Group created"))
         DBSession.flush()
         return group.get_dict(include_perms=True)
     else:
         return HTTPUnprocessableEntity(body=form.errors_json)
 
 
-@view_config(route_name='groups', renderer='json',
-             request_method="DELETE", permission='root_administration')
+@view_config(
+    route_name="groups",
+    renderer="json",
+    request_method="DELETE",
+    permission="root_administration",
+)
 def groups_DELETE(request):
     """
     Removes a groups permanently from db
     """
-    msg = _('You cannot remove administrator group from the system')
-    group = GroupService.by_id(request.matchdict.get('group_id'))
+    msg = _("You cannot remove administrator group from the system")
+    group = GroupService.by_id(request.matchdict.get("group_id"))
     if group:
         if group.id == 1:
-            request.session.flash(msg, 'warning')
+            request.session.flash(msg, "warning")
         else:
             DBSession.delete(group)
-            request.session.flash(_('Group removed'))
+            request.session.flash(_("Group removed"))
             return True
     request.response.status = 422
     return False
 
 
-@view_config(route_name='groups', renderer='json',
-             request_method="GET", permission='root_administration')
-@view_config(route_name='groups', renderer='json',
-             request_method="PATCH", permission='root_administration')
+@view_config(
+    route_name="groups",
+    renderer="json",
+    request_method="GET",
+    permission="root_administration",
+)
+@view_config(
+    route_name="groups",
+    renderer="json",
+    request_method="PATCH",
+    permission="root_administration",
+)
 def group_update(request):
     """
     Updates group object
     """
-    group = GroupService.by_id(request.matchdict.get('group_id'))
+    group = GroupService.by_id(request.matchdict.get("group_id"))
     if not group:
         return HTTPNotFound()
 
-    if request.method == 'PATCH':
+    if request.method == "PATCH":
         form = forms.GroupCreateForm(
-            MultiDict(request.unsafe_json_body), csrf_context=request)
+            MultiDict(request.unsafe_json_body), csrf_context=request
+        )
         form._modified_group = group
         if form.validate():
             form.populate_obj(group)
         else:
             return HTTPUnprocessableEntity(body=form.errors_json)
     return group.get_dict(include_perms=True)
 
 
-@view_config(route_name='groups_property',
-             match_param='key=resource_permissions',
-             renderer='json', permission='root_administration')
+@view_config(
+    route_name="groups_property",
+    match_param="key=resource_permissions",
+    renderer="json",
+    permission="root_administration",
+)
 def groups_resource_permissions_list(request):
     """
     Get list of permissions assigned to specific resources
     """
-    group = GroupService.by_id(request.matchdict.get('group_id'))
+    group = GroupService.by_id(request.matchdict.get("group_id"))
     if not group:
         return HTTPNotFound()
-    return [permission_tuple_to_dict(perm) for perm in
-            GroupService.resources_with_possible_perms(group)]
-
-
-@view_config(route_name='groups_property',
-             match_param='key=users', request_method="GET",
-             renderer='json', permission='root_administration')
+    return [
+        permission_tuple_to_dict(perm)
+        for perm in GroupService.resources_with_possible_perms(group)
+    ]
+
+
+@view_config(
+    route_name="groups_property",
+    match_param="key=users",
+    request_method="GET",
+    renderer="json",
+    permission="root_administration",
+)
 def groups_users_list(request):
     """
     Get list of permissions assigned to specific resources
     """
-    group = GroupService.by_id(request.matchdict.get('group_id'))
+    group = GroupService.by_id(request.matchdict.get("group_id"))
     if not group:
         return HTTPNotFound()
-    props = ['user_name', 'id', 'first_name', 'last_name', 'email',
-             'last_login_date', 'status']
+    props = [
+        "user_name",
+        "id",
+        "first_name",
+        "last_name",
+        "email",
+        "last_login_date",
+        "status",
+    ]
     users_dicts = []
     for user in group.users:
         u_dict = user.get_dict(include_keys=props)
-        u_dict['gravatar_url'] = UserService.gravatar_url(user, s=20)
+        u_dict["gravatar_url"] = UserService.gravatar_url(user, s=20)
         users_dicts.append(u_dict)
     return users_dicts
 
 
-@view_config(route_name='groups_property',
-             match_param='key=users', request_method="DELETE",
-             renderer='json', permission='root_administration')
+@view_config(
+    route_name="groups_property",
+    match_param="key=users",
+    request_method="DELETE",
+    renderer="json",
+    permission="root_administration",
+)
 def groups_users_remove(request):
     """
     Get list of permissions assigned to specific resources
     """
-    group = GroupService.by_id(request.matchdict.get('group_id'))
-    user = UserService.by_user_name(request.GET.get('user_name'))
+    group = GroupService.by_id(request.matchdict.get("group_id"))
+    user = UserService.by_user_name(request.GET.get("user_name"))
     if not group or not user:
         return HTTPNotFound()
     if len(group.users) > 1:
         group.users.remove(user)
         msg = "User removed from group"
         request.session.flash(msg)
         group.member_count = group.users_dynamic.count()
         return True
     msg = "Administrator group needs to contain at least one user"
-    request.session.flash(msg, 'warning')
+    request.session.flash(msg, "warning")
     return False
 
 
-@view_config(route_name='groups_property',
-             match_param='key=users', request_method="POST",
-             renderer='json', permission='root_administration')
+@view_config(
+    route_name="groups_property",
+    match_param="key=users",
+    request_method="POST",
+    renderer="json",
+    permission="root_administration",
+)
 def groups_users_add(request):
     """
     Get list of permissions assigned to specific resources
     """
-    group = GroupService.by_id(request.matchdict.get('group_id'))
-    user = UserService.by_user_name(request.unsafe_json_body.get('user_name'))
+    group = GroupService.by_id(request.matchdict.get("group_id"))
+    user = UserService.by_user_name(request.unsafe_json_body.get("user_name"))
     if not user:
-        user = UserService.by_email(request.unsafe_json_body.get('user_name'))
+        user = UserService.by_email(request.unsafe_json_body.get("user_name"))
 
     if not group or not user:
         return HTTPNotFound()
     if user not in group.users:
         group.users.append(user)
         group.member_count = group.users_dynamic.count()
-    props = ['user_name', 'id', 'first_name', 'last_name', 'email',
-             'last_login_date', 'status']
+    props = [
+        "user_name",
+        "id",
+        "first_name",
+        "last_name",
+        "email",
+        "last_login_date",
+        "status",
+    ]
     u_dict = user.get_dict(include_keys=props)
-    u_dict['gravatar_url'] = UserService.gravatar_url(user, s=20)
+    u_dict["gravatar_url"] = UserService.gravatar_url(user, s=20)
     return u_dict

backend/src/appenlight/views/index.py

@@ -1,242 +1,263 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import datetime
 import logging
 import uuid
 
 import pyramid.security as security
 
 from pyramid.view import view_config
 from pyramid.httpexceptions import HTTPFound
 from pyramid.response import Response
 from pyramid.security import NO_PERMISSION_REQUIRED
 from ziggurat_foundations.ext.pyramid.sign_in import ZigguratSignInSuccess
 from ziggurat_foundations.ext.pyramid.sign_in import ZigguratSignInBadAuth
 from ziggurat_foundations.ext.pyramid.sign_in import ZigguratSignOut
 from ziggurat_foundations.models.services.user import UserService
 
 from appenlight.lib.social import handle_social_data
 from appenlight.models import DBSession
 from appenlight.models.user import User
 from appenlight.models.services.user import UserService
 from appenlight.subscribers import _
 from appenlight import forms
 from webob.multidict import MultiDict
 
 log = logging.getLogger(__name__)
 
 
 @view_config(context=ZigguratSignInSuccess, permission=NO_PERMISSION_REQUIRED)
 def sign_in(request):
     """
     Performs sign in by sending proper user identification headers
     Regenerates CSRF token
     """
     user = request.context.user
     if user.status == 1:
         request.session.new_csrf_token()
         user.last_login_date = datetime.datetime.utcnow()
-        social_data = request.session.get('zigg.social_auth')
+        social_data = request.session.get("zigg.social_auth")
         if social_data:
             handle_social_data(request, user, social_data)
     else:
-        request.session.flash(_('Account got disabled'))
+        request.session.flash(_("Account got disabled"))
 
-    if request.context.came_from != '/':
-        return HTTPFound(location=request.context.came_from,
-                         headers=request.context.headers)
+    if request.context.came_from != "/":
+        return HTTPFound(
+            location=request.context.came_from, headers=request.context.headers
+        )
     else:
-        return HTTPFound(location=request.route_url('/'),
-                         headers=request.context.headers)
+        return HTTPFound(
+            location=request.route_url("/"), headers=request.context.headers
+        )
 
 
 @view_config(context=ZigguratSignInBadAuth, permission=NO_PERMISSION_REQUIRED)
 def bad_auth(request):
     """
     Handles incorrect login flow
     """
-    request.session.flash(_('Incorrect username or password'), 'warning')
-    return HTTPFound(location=request.route_url('register'),
-                     headers=request.context.headers)
+    request.session.flash(_("Incorrect username or password"), "warning")
+    return HTTPFound(
+        location=request.route_url("register"), headers=request.context.headers
+    )
 
 
 @view_config(context=ZigguratSignOut, permission=NO_PERMISSION_REQUIRED)
 def sign_out(request):
     """
     Removes user identification cookie
     """
-    return HTTPFound(location=request.route_url('register'),
-                     headers=request.context.headers)
+    return HTTPFound(
+        location=request.route_url("register"), headers=request.context.headers
+    )
 
 
-@view_config(route_name='lost_password',
-             renderer='appenlight:templates/user/lost_password.jinja2',
-             permission=NO_PERMISSION_REQUIRED)
+@view_config(
+    route_name="lost_password",
+    renderer="appenlight:templates/user/lost_password.jinja2",
+    permission=NO_PERMISSION_REQUIRED,
+)
 def lost_password(request):
     """
     Presents lost password page - sends password reset link to
     specified email address.
     This link is valid only for 10 minutes
     """
     form = forms.LostPasswordForm(request.POST, csrf_context=request)
-    if request.method == 'POST' and form.validate():
+    if request.method == "POST" and form.validate():
         user = UserService.by_email(form.email.data)
         if user:
             UserService.regenerate_security_code(user)
             user.security_code_date = datetime.datetime.utcnow()
             email_vars = {
-                'user': user,
-                'request': request,
-                'email_title': "AppEnlight :: New password request"
+                "user": user,
+                "request": request,
+                "email_title": "AppEnlight :: New password request",
             }
             UserService.send_email(
-                request, recipients=[user.email],
+                request,
+                recipients=[user.email],
                 variables=email_vars,
-                template='/email_templates/lost_password.jinja2')
-            msg = 'Password reset email had been sent. ' \
-                  'Please check your mailbox for further instructions.'
+                template="/email_templates/lost_password.jinja2",
+            )
+            msg = (
+                "Password reset email had been sent. "
+                "Please check your mailbox for further instructions."
+            )
             request.session.flash(_(msg))
-            return HTTPFound(location=request.route_url('lost_password'))
+            return HTTPFound(location=request.route_url("lost_password"))
     return {"form": form}
 
 
-@view_config(route_name='lost_password_generate',
-             permission=NO_PERMISSION_REQUIRED,
-             renderer='appenlight:templates/user/lost_password_generate.jinja2')
+@view_config(
+    route_name="lost_password_generate",
+    permission=NO_PERMISSION_REQUIRED,
+    renderer="appenlight:templates/user/lost_password_generate.jinja2",
+)
 def lost_password_generate(request):
     """
     Shows new password form - perform time check and set new password for user
     """
     user = UserService.by_user_name_and_security_code(
-        request.GET.get('user_name'), request.GET.get('security_code'))
+        request.GET.get("user_name"), request.GET.get("security_code")
+    )
     if user:
         delta = datetime.datetime.utcnow() - user.security_code_date
 
     if user and delta.total_seconds() < 600:
         form = forms.NewPasswordForm(request.POST, csrf_context=request)
         if request.method == "POST" and form.validate():
             UserService.set_password(user, form.new_password.data)
-            request.session.flash(_('You can sign in with your new password.'))
-            return HTTPFound(location=request.route_url('register'))
+            request.session.flash(_("You can sign in with your new password."))
+            return HTTPFound(location=request.route_url("register"))
         else:
             return {"form": form}
     else:
-        return Response('Security code expired')
+        return Response("Security code expired")
 
 
-@view_config(route_name='register',
-             renderer='appenlight:templates/user/register.jinja2',
-             permission=NO_PERMISSION_REQUIRED)
+@view_config(
+    route_name="register",
+    renderer="appenlight:templates/user/register.jinja2",
+    permission=NO_PERMISSION_REQUIRED,
+)
 def register(request):
     """
     Render register page with form
     Also handles oAuth flow for registration
     """
-    login_url = request.route_url('ziggurat.routes.sign_in')
+    login_url = request.route_url("ziggurat.routes.sign_in")
     if request.query_string:
-        query_string = '?%s' % request.query_string
+        query_string = "?%s" % request.query_string
     else:
-        query_string = ''
-    referrer = '%s%s' % (request.path, query_string)
+        query_string = ""
+    referrer = "%s%s" % (request.path, query_string)
 
-    if referrer in [login_url, '/register', '/register?sign_in=1']:
-        referrer = '/'  # never use the login form itself as came_from
+    if referrer in [login_url, "/register", "/register?sign_in=1"]:
+        referrer = "/"  # never use the login form itself as came_from
     sign_in_form = forms.SignInForm(
-        came_from=request.params.get('came_from', referrer),
-        csrf_context=request)
+        came_from=request.params.get("came_from", referrer), csrf_context=request
+    )
 
     # populate form from oAuth session data returned by authomatic
-    social_data = request.session.get('zigg.social_auth')
-    if request.method != 'POST' and social_data:
+    social_data = request.session.get("zigg.social_auth")
+    if request.method != "POST" and social_data:
         log.debug(social_data)
-        user_name = social_data['user'].get('user_name', '').split('@')[0]
-        form_data = {
-            'user_name': user_name,
-            'email': social_data['user'].get('email')
-        }
-        form_data['user_password'] = str(uuid.uuid4())
-        form = forms.UserRegisterForm(MultiDict(form_data),
-                                      csrf_context=request)
+        user_name = social_data["user"].get("user_name", "").split("@")[0]
+        form_data = {"user_name": user_name, "email": social_data["user"].get("email")}
+        form_data["user_password"] = str(uuid.uuid4())
+        form = forms.UserRegisterForm(MultiDict(form_data), csrf_context=request)
         form.user_password.widget.hide_value = False
     else:
         form = forms.UserRegisterForm(request.POST, csrf_context=request)
-    if request.method == 'POST' and form.validate():
-        log.info('registering user')
+    if request.method == "POST" and form.validate():
+        log.info("registering user")
         # insert new user here
-        if request.registry.settings['appenlight.disable_registration']:
-            request.session.flash(_('Registration is currently disabled.'))
-            return HTTPFound(location=request.route_url('/'))
+        if request.registry.settings["appenlight.disable_registration"]:
+            request.session.flash(_("Registration is currently disabled."))
+            return HTTPFound(location=request.route_url("/"))
 
         new_user = User()
         DBSession.add(new_user)
         form.populate_obj(new_user)
         UserService.regenerate_security_code(new_user)
         new_user.status = 1
         UserService.set_password(new_user, new_user.user_password)
-        new_user.registration_ip = request.environ.get('REMOTE_ADDR')
+        new_user.registration_ip = request.environ.get("REMOTE_ADDR")
 
         if social_data:
             handle_social_data(request, new_user, social_data)
 
-        email_vars = {'user': new_user,
-                      'request': request,
-                      'email_title': "AppEnlight :: Start information"}
+        email_vars = {
+            "user": new_user,
+            "request": request,
+            "email_title": "AppEnlight :: Start information",
+        }
         UserService.send_email(
-            request, recipients=[new_user.email], variables=email_vars,
-            template='/email_templates/registered.jinja2')
-        request.session.flash(_('You have successfully registered.'))
+            request,
+            recipients=[new_user.email],
+            variables=email_vars,
+            template="/email_templates/registered.jinja2",
+        )
+        request.session.flash(_("You have successfully registered."))
         DBSession.flush()
         headers = security.remember(request, new_user.id)
-        return HTTPFound(location=request.route_url('/'),
-                         headers=headers)
+        return HTTPFound(location=request.route_url("/"), headers=headers)
     settings = request.registry.settings
     social_plugins = {}
-    if settings.get('authomatic.pr.twitter.key', ''):
-        social_plugins['twitter'] = True
-    if settings.get('authomatic.pr.google.key', ''):
-        social_plugins['google'] = True
-    if settings.get('authomatic.pr.github.key', ''):
-        social_plugins['github'] = True
-    if settings.get('authomatic.pr.bitbucket.key', ''):
-        social_plugins['bitbucket'] = True
+    if settings.get("authomatic.pr.twitter.key", ""):
+        social_plugins["twitter"] = True
+    if settings.get("authomatic.pr.google.key", ""):
+        social_plugins["google"] = True
+    if settings.get("authomatic.pr.github.key", ""):
+        social_plugins["github"] = True
+    if settings.get("authomatic.pr.bitbucket.key", ""):
+        social_plugins["bitbucket"] = True
 
     return {
         "form": form,
         "sign_in_form": sign_in_form,
-        "social_plugins": social_plugins
+        "social_plugins": social_plugins,
     }
 
 
-@view_config(route_name='/',
-             renderer='appenlight:templates/app.jinja2',
-             permission=NO_PERMISSION_REQUIRED)
-@view_config(route_name='angular_app_ui',
-             renderer='appenlight:templates/app.jinja2',
-             permission=NO_PERMISSION_REQUIRED)
-@view_config(route_name='angular_app_ui_ix',
-             renderer='appenlight:templates/app.jinja2',
-             permission=NO_PERMISSION_REQUIRED)
+@view_config(
+    route_name="/",
+    renderer="appenlight:templates/app.jinja2",
+    permission=NO_PERMISSION_REQUIRED,
+)
+@view_config(
+    route_name="angular_app_ui",
+    renderer="appenlight:templates/app.jinja2",
+    permission=NO_PERMISSION_REQUIRED,
+)
+@view_config(
+    route_name="angular_app_ui_ix",
+    renderer="appenlight:templates/app.jinja2",
+    permission=NO_PERMISSION_REQUIRED,
+)
 def app_main_index(request):
     """
     Render dashoard/report browser page page along with:
      - flash messages
      - application list
      - assigned reports
      - latest events
      (those last two come from subscribers.py that sets global renderer variables)
     """
     return {}

backend/src/appenlight/views/integrations/__init__.py

@@ -1,244 +1,266 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 import random
 from datetime import datetime
 
 from pyramid.httpexceptions import HTTPFound
 from pyramid.view import view_config
 
 from appenlight.models import DBSession
 from appenlight.models.event import Event
 from appenlight.models.integrations import IntegrationBase
 from appenlight.models.alert_channel import AlertChannel
 from appenlight.models.report_group import ReportGroup
 from appenlight.models.services.alert_channel import AlertChannelService
 from appenlight.lib import generate_random_string
 
 log = logging.getLogger(__name__)
 
 dummy_report = ReportGroup()
-dummy_report.error = "ProtocolError: ('Connection aborted.', " \
-                     "error(111, 'Connection refused'))"
+dummy_report.error = (
+    "ProtocolError: ('Connection aborted.', " "error(111, 'Connection refused'))"
+)
 dummy_report.total_reports = 4
 dummy_report.occurences = 4
 
 dummy_report2 = ReportGroup()
-dummy_report2.error = "UnboundLocalError: local variable " \
-                      "'hits' referenced before assignment"
+dummy_report2.error = (
+    "UnboundLocalError: local variable " "'hits' referenced before assignment"
+)
 dummy_report2.total_reports = 8
 dummy_report2.occurences = 8
 
 dummy_reports = [(4, dummy_report), (8, dummy_report2)]
 
 
 class IntegrationView(object):
     """
     Base class for 3rd party integrations setup views
     """
 
     def __init__(self, request):
         self.request = request
         resource = self.request.context.resource
-        integration_name = request.matchdict['integration']
+        integration_name = request.matchdict["integration"]
         integration = IntegrationBase.by_app_id_and_integration_name(
-            resource.resource_id, integration_name)
+            resource.resource_id, integration_name
+        )
         if integration:
             dict_config = integration.config
         else:
             dict_config = {}
         self.integration = integration
         self.integration_config = dict_config
 
-    @view_config(route_name='integrations_id',
-                 request_method="DELETE",
-                 renderer='json',
-                 permission='edit')
+    @view_config(
+        route_name="integrations_id",
+        request_method="DELETE",
+        renderer="json",
+        permission="edit",
+    )
     def remove_integration(self):
         if self.integration:
             DBSession.delete(self.integration)
-            self.request.session.flash('Integration removed')
-        return ''
-
-    @view_config(route_name='integrations_id',
-                 request_method="POST",
-                 match_param=['action=test_report_notification'],
-                 renderer='json',
-                 permission='edit')
+            self.request.session.flash("Integration removed")
+        return ""
+
+    @view_config(
+        route_name="integrations_id",
+        request_method="POST",
+        match_param=["action=test_report_notification"],
+        renderer="json",
+        permission="edit",
+    )
     def test_report_notification(self):
         if not self.integration:
-            self.request.session.flash('Integration needs to be configured',
-                                       'warning')
+            self.request.session.flash("Integration needs to be configured", "warning")
             return False
 
         resource = self.integration.resource
 
         channel = AlertChannelService.by_integration_id(self.integration.id)
 
         if random.random() < 0.5:
             confirmed_reports = dummy_reports
         else:
             confirmed_reports = [random.choice(dummy_reports)]
 
-        channel.notify_reports(resource=resource,
-                               user=self.request.user,
-                               request=self.request,
-                               since_when=datetime.utcnow(),
-                               reports=confirmed_reports)
-        self.request.session.flash('Report notification sent')
+        channel.notify_reports(
+            resource=resource,
+            user=self.request.user,
+            request=self.request,
+            since_when=datetime.utcnow(),
+            reports=confirmed_reports,
+        )
+        self.request.session.flash("Report notification sent")
         return True
 
-    @view_config(route_name='integrations_id',
-                 request_method="POST",
-                 match_param=['action=test_error_alert'],
-                 renderer='json',
-                 permission='edit')
+    @view_config(
+        route_name="integrations_id",
+        request_method="POST",
+        match_param=["action=test_error_alert"],
+        renderer="json",
+        permission="edit",
+    )
     def test_error_alert(self):
         if not self.integration:
-            self.request.session.flash('Integration needs to be configured',
-                                       'warning')
+            self.request.session.flash("Integration needs to be configured", "warning")
             return False
 
         resource = self.integration.resource
 
-        event_name = random.choice(('error_report_alert',
-                                    'slow_report_alert',))
-        new_event = Event(resource_id=resource.resource_id,
-                          event_type=Event.types[event_name],
-                          start_date=datetime.utcnow(),
-                          status=Event.statuses['active'],
-                          values={'reports': random.randint(11, 99),
-                                  'threshold': 10}
-                          )
+        event_name = random.choice(("error_report_alert", "slow_report_alert"))
+        new_event = Event(
+            resource_id=resource.resource_id,
+            event_type=Event.types[event_name],
+            start_date=datetime.utcnow(),
+            status=Event.statuses["active"],
+            values={"reports": random.randint(11, 99), "threshold": 10},
+        )
 
         channel = AlertChannelService.by_integration_id(self.integration.id)
 
-        channel.notify_alert(resource=resource,
-                             event=new_event,
-                             user=self.request.user,
-                             request=self.request)
-        self.request.session.flash('Notification sent')
+        channel.notify_alert(
+            resource=resource,
+            event=new_event,
+            user=self.request.user,
+            request=self.request,
+        )
+        self.request.session.flash("Notification sent")
         return True
 
-    @view_config(route_name='integrations_id',
-                 request_method="POST",
-                 match_param=['action=test_daily_digest'],
-                 renderer='json',
-                 permission='edit')
+    @view_config(
+        route_name="integrations_id",
+        request_method="POST",
+        match_param=["action=test_daily_digest"],
+        renderer="json",
+        permission="edit",
+    )
     def test_daily_digest(self):
         if not self.integration:
-            self.request.session.flash('Integration needs to be configured',
-                                       'warning')
+            self.request.session.flash("Integration needs to be configured", "warning")
             return False
 
         resource = self.integration.resource
         channel = AlertChannelService.by_integration_id(self.integration.id)
 
-        channel.send_digest(resource=resource,
-                            user=self.request.user,
-                            request=self.request,
-                            since_when=datetime.utcnow(),
-                            reports=dummy_reports)
-        self.request.session.flash('Notification sent')
+        channel.send_digest(
+            resource=resource,
+            user=self.request.user,
+            request=self.request,
+            since_when=datetime.utcnow(),
+            reports=dummy_reports,
+        )
+        self.request.session.flash("Notification sent")
         return True
 
-    @view_config(route_name='integrations_id',
-                 request_method="POST",
-                 match_param=['action=test_uptime_alert'],
-                 renderer='json',
-                 permission='edit')
+    @view_config(
+        route_name="integrations_id",
+        request_method="POST",
+        match_param=["action=test_uptime_alert"],
+        renderer="json",
+        permission="edit",
+    )
     def test_uptime_alert(self):
         if not self.integration:
-            self.request.session.flash('Integration needs to be configured',
-                                       'warning')
+            self.request.session.flash("Integration needs to be configured", "warning")
             return False
 
         resource = self.integration.resource
 
-        new_event = Event(resource_id=resource.resource_id,
-                          event_type=Event.types['uptime_alert'],
-                          start_date=datetime.utcnow(),
-                          status=Event.statuses['active'],
-                          values={"status_code": 500,
-                                  "tries": 2,
-                                  "response_time": 0})
+        new_event = Event(
+            resource_id=resource.resource_id,
+            event_type=Event.types["uptime_alert"],
+            start_date=datetime.utcnow(),
+            status=Event.statuses["active"],
+            values={"status_code": 500, "tries": 2, "response_time": 0},
+        )
 
         channel = AlertChannelService.by_integration_id(self.integration.id)
-        channel.notify_uptime_alert(resource=resource,
-                                    event=new_event,
-                                    user=self.request.user,
-                                    request=self.request)
-
-        self.request.session.flash('Notification sent')
+        channel.notify_uptime_alert(
+            resource=resource,
+            event=new_event,
+            user=self.request.user,
+            request=self.request,
+        )
+
+        self.request.session.flash("Notification sent")
         return True
 
-    @view_config(route_name='integrations_id',
-                 request_method="POST",
-                 match_param=['action=test_chart_alert'],
-                 renderer='json',
-                 permission='edit')
+    @view_config(
+        route_name="integrations_id",
+        request_method="POST",
+        match_param=["action=test_chart_alert"],
+        renderer="json",
+        permission="edit",
+    )
     def test_chart_alert(self):
         if not self.integration:
-            self.request.session.flash('Integration needs to be configured',
-                                       'warning')
+            self.request.session.flash("Integration needs to be configured", "warning")
             return False
 
         resource = self.integration.resource
 
         chart_values = {
-            "matched_rule": {'name': 'Fraud attempt limit'},
-            "matched_step_values": {"labels": {
-                "0_1": {"human_label": "Attempts sum"}},
-                "values": {"0_1": random.randint(11, 55),
-                           "key": "2015-12-16T15:49:00"}},
+            "matched_rule": {"name": "Fraud attempt limit"},
+            "matched_step_values": {
+                "labels": {"0_1": {"human_label": "Attempts sum"}},
+                "values": {"0_1": random.randint(11, 55), "key": "2015-12-16T15:49:00"},
+            },
             "start_interval": datetime.utcnow(),
             "resource": 1,
             "chart_name": "Fraud attempts per day",
             "chart_uuid": "some_uuid",
             "step_size": 3600,
-            "action_name": "Notify excessive fraud attempts"}
-
-        new_event = Event(resource_id=resource.resource_id,
-                          event_type=Event.types['chart_alert'],
-                          status=Event.statuses['active'],
-                          values=chart_values,
-                          target_uuid="some_uuid",
-                          start_date=datetime.utcnow())
+            "action_name": "Notify excessive fraud attempts",
+        }
+
+        new_event = Event(
+            resource_id=resource.resource_id,
+            event_type=Event.types["chart_alert"],
+            status=Event.statuses["active"],
+            values=chart_values,
+            target_uuid="some_uuid",
+            start_date=datetime.utcnow(),
+        )
 
         channel = AlertChannelService.by_integration_id(self.integration.id)
-        channel.notify_chart_alert(resource=resource,
-                                   event=new_event,
-                                   user=self.request.user,
-                                   request=self.request)
-
-        self.request.session.flash('Notification sent')
+        channel.notify_chart_alert(
+            resource=resource,
+            event=new_event,
+            user=self.request.user,
+            request=self.request,
+        )
+
+        self.request.session.flash("Notification sent")
         return True
 
     def create_missing_channel(self, resource, channel_name):
         """
         Recreates alert channel for the owner of integration
         """
         if self.integration:
             if not self.integration.channel:
                 channel = AlertChannel()
                 channel.channel_name = channel_name
                 channel.channel_validated = True
                 channel.channel_value = resource.resource_id
                 channel.integration_id = self.integration.id
                 security_code = generate_random_string(10)
-                channel.channel_json_conf = {'security_code': security_code}
+                channel.channel_json_conf = {"security_code": security_code}
                 resource.owner.alert_channels.append(channel)

backend/src/appenlight/views/integrations/bitbucket.py

@@ -1,121 +1,129 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from appenlight.models.integrations.bitbucket import BitbucketIntegration, \
-    IntegrationException
+from appenlight.models.integrations.bitbucket import (
+    BitbucketIntegration,
+    IntegrationException,
+)
 from appenlight.models.report_comment import ReportComment
 from appenlight.models.services.report_group import ReportGroupService
 from pyramid.view import view_config
 from appenlight import forms
 import logging
 from datetime import datetime
 from pyramid.httpexceptions import HTTPUnprocessableEntity
 from webob.multidict import MultiDict
 
 log = logging.getLogger(__name__)
 
 from . import IntegrationView
 
 
 class BitbucketView(IntegrationView):
-    @view_config(route_name='integrations_id',
-                 match_param=['action=info', 'integration=bitbucket'],
-                 renderer='json')
+    @view_config(
+        route_name="integrations_id",
+        match_param=["action=info", "integration=bitbucket"],
+        renderer="json",
+    )
     def get_bitbucket_info(self):
         """
         Grab information about possible priority levels and assignable users
         """
         try:
             client = BitbucketIntegration.create_client(
                 self.request,
-                self.integration.config['user_name'],
-                self.integration.config['repo_name'])
+                self.integration.config["user_name"],
+                self.integration.config["repo_name"],
+            )
         except IntegrationException as e:
             self.request.response.status_code = 503
-            return {'error_messages': [str(e)]}
+            return {"error_messages": [str(e)]}
         assignees = client.get_assignees()
         priorities = client.get_priorities()
-        return {'assignees': assignees,
-                'priorities': priorities}
+        return {"assignees": assignees, "priorities": priorities}
 
-    @view_config(route_name='integrations_id',
-                 match_param=['action=create-issue',
-                              'integration=bitbucket'],
-                 renderer='json')
+    @view_config(
+        route_name="integrations_id",
+        match_param=["action=create-issue", "integration=bitbucket"],
+        renderer="json",
+    )
     def create_issue(self):
         """
         Creates a new issue in bitbucket issue tracker from report group
         """
-        report = ReportGroupService.by_id(
-            self.request.unsafe_json_body['group_id'])
+        report = ReportGroupService.by_id(self.request.unsafe_json_body["group_id"])
         form_data = {
-            'title': self.request.unsafe_json_body.get('title',
-                                                       'Unknown Title'),
-            'content': self.request.unsafe_json_body.get('content', ''),
-            'kind': 'bug',
-            'priority': self.request.unsafe_json_body['priority'],
-            'responsible': self.request.unsafe_json_body['responsible']['user']
+            "title": self.request.unsafe_json_body.get("title", "Unknown Title"),
+            "content": self.request.unsafe_json_body.get("content", ""),
+            "kind": "bug",
+            "priority": self.request.unsafe_json_body["priority"],
+            "responsible": self.request.unsafe_json_body["responsible"]["user"],
         }
 
         try:
             client = BitbucketIntegration.create_client(
                 self.request,
-                self.integration.config['user_name'],
-                self.integration.config['repo_name'])
+                self.integration.config["user_name"],
+                self.integration.config["repo_name"],
+            )
             issue = client.create_issue(form_data)
         except IntegrationException as e:
             self.request.response.status_code = 503
-            return {'error_messages': [str(e)]}
+            return {"error_messages": [str(e)]}
 
-        comment_body = 'Bitbucket issue created: %s ' % issue['web_url']
-        comment = ReportComment(owner_id=self.request.user.id,
-                                report_time=report.first_timestamp,
-                                body=comment_body)
+        comment_body = "Bitbucket issue created: %s " % issue["web_url"]
+        comment = ReportComment(
+            owner_id=self.request.user.id,
+            report_time=report.first_timestamp,
+            body=comment_body,
+        )
         report.comments.append(comment)
         return True
 
-    @view_config(route_name='integrations_id',
-                 match_param=['action=setup', 'integration=bitbucket'],
-                 renderer='json',
-                 permission='edit')
+    @view_config(
+        route_name="integrations_id",
+        match_param=["action=setup", "integration=bitbucket"],
+        renderer="json",
+        permission="edit",
+    )
     def setup(self):
         """
         Validates and creates integration between application and bitbucket
         """
         resource = self.request.context.resource
         form = forms.IntegrationBitbucketForm(
             MultiDict(self.request.safe_json_body or {}),
-            csrf_context=self.request, **self.integration_config)
-        if self.request.method == 'POST' and form.validate():
+            csrf_context=self.request,
+            **self.integration_config
+        )
+        if self.request.method == "POST" and form.validate():
             integration_config = {
-                'repo_name': form.repo_name.data,
-                'user_name': form.user_name.data,
-                'host_name': 'https://bitbucket.org'
+                "repo_name": form.repo_name.data,
+                "user_name": form.user_name.data,
+                "host_name": "https://bitbucket.org",
             }
             if not self.integration:
                 # add new integration
-                self.integration = BitbucketIntegration(
-                    modified_date=datetime.utcnow(),
-                )
-                self.request.session.flash('Integration added')
+                self.integration = BitbucketIntegration(modified_date=datetime.utcnow())
+                self.request.session.flash("Integration added")
                 resource.integrations.append(self.integration)
             else:
-                self.request.session.flash('Integration updated')
+                self.request.session.flash("Integration updated")
             self.integration.config = integration_config
             return integration_config
-        elif self.request.method == 'POST':
+        elif self.request.method == "POST":
             return HTTPUnprocessableEntity(body=form.errors_json)
         return self.integration_config

backend/src/appenlight/views/integrations/campfire.py

@@ -1,78 +1,80 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from ...models import DBSession
-from ...models.integrations.campfire import CampfireIntegration, \
-    IntegrationException
+from ...models.integrations.campfire import CampfireIntegration, IntegrationException
 from ...models.alert_channel import AlertChannel
 from ...lib import generate_random_string
 from pyramid.httpexceptions import HTTPFound, HTTPUnprocessableEntity
 from pyramid.view import view_config
 from ... import forms
 import logging
 from datetime import datetime
 from webob.multidict import MultiDict
 
 log = logging.getLogger(__name__)
 
 from . import IntegrationView
 
 
 class CampfireView(IntegrationView):
-    @view_config(route_name='integrations_id',
-                 match_param=['action=info', 'integration=campfire'],
-                 renderer='json')
+    @view_config(
+        route_name="integrations_id",
+        match_param=["action=info", "integration=campfire"],
+        renderer="json",
+    )
     def get_info(self):
         pass
 
-    @view_config(route_name='integrations_id',
-                 match_param=['action=setup', 'integration=campfire'],
-                 renderer='json',
-                 permission='edit')
+    @view_config(
+        route_name="integrations_id",
+        match_param=["action=setup", "integration=campfire"],
+        renderer="json",
+        permission="edit",
+    )
     def setup(self):
         """
         Validates and creates integration between application and campfire
         """
         resource = self.request.context.resource
-        self.create_missing_channel(resource, 'campfire')
+        self.create_missing_channel(resource, "campfire")
 
         form = forms.IntegrationCampfireForm(
             MultiDict(self.request.safe_json_body or {}),
             csrf_context=self.request,
-            **self.integration_config)
+            **self.integration_config
+        )
 
-        if self.request.method == 'POST' and form.validate():
+        if self.request.method == "POST" and form.validate():
             integration_config = {
-                'account': form.account.data,
-                'api_token': form.api_token.data,
-                'rooms': form.rooms.data,
+                "account": form.account.data,
+                "api_token": form.api_token.data,
+                "rooms": form.rooms.data,
             }
             if not self.integration:
                 # add new integration
-                self.integration = CampfireIntegration(
-                    modified_date=datetime.utcnow(),
-                )
-                self.request.session.flash('Integration added')
+                self.integration = CampfireIntegration(modified_date=datetime.utcnow())
+                self.request.session.flash("Integration added")
                 resource.integrations.append(self.integration)
             else:
-                self.request.session.flash('Integration updated')
+                self.request.session.flash("Integration updated")
             self.integration.config = integration_config
             DBSession.flush()
-            self.create_missing_channel(resource, 'campfire')
+            self.create_missing_channel(resource, "campfire")
             return integration_config
-        elif self.request.method == 'POST':
+        elif self.request.method == "POST":
             return HTTPUnprocessableEntity(body=form.errors_json)
         return self.integration_config

backend/src/appenlight/views/integrations/flowdock.py

@@ -1,71 +1,73 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from appenlight.models import DBSession
 from appenlight.models.integrations.flowdock import FlowdockIntegration
 from pyramid.httpexceptions import HTTPUnprocessableEntity
 from pyramid.view import view_config
 from appenlight import forms
 import logging
 from datetime import datetime
 from webob.multidict import MultiDict
 
 log = logging.getLogger(__name__)
 
 from . import IntegrationView
 
 
 class FlowdockView(IntegrationView):
-    @view_config(route_name='integrations_id',
-                 match_param=['action=info', 'integration=flowdock'],
-                 renderer='json')
+    @view_config(
+        route_name="integrations_id",
+        match_param=["action=info", "integration=flowdock"],
+        renderer="json",
+    )
     def get_info(self):
         pass
 
-    @view_config(route_name='integrations_id',
-                 match_param=['action=setup', 'integration=flowdock'],
-                 renderer='json',
-                 permission='edit')
+    @view_config(
+        route_name="integrations_id",
+        match_param=["action=setup", "integration=flowdock"],
+        renderer="json",
+        permission="edit",
+    )
     def setup(self):
         """
         Validates and creates integration between application and flowdock
         """
         resource = self.request.context.resource
-        self.create_missing_channel(resource, 'flowdock')
+        self.create_missing_channel(resource, "flowdock")
 
         form = forms.IntegrationFlowdockForm(
             MultiDict(self.request.safe_json_body or {}),
-            csrf_context=self.request, **self.integration_config)
-        if self.request.method == 'POST' and form.validate():
-            integration_config = {
-                'api_token': form.api_token.data,
-            }
+            csrf_context=self.request,
+            **self.integration_config
+        )
+        if self.request.method == "POST" and form.validate():
+            integration_config = {"api_token": form.api_token.data}
             if not self.integration:
                 # add new integration
-                self.integration = FlowdockIntegration(
-                    modified_date=datetime.utcnow(),
-                )
-                self.request.session.flash('Integration added')
+                self.integration = FlowdockIntegration(modified_date=datetime.utcnow())
+                self.request.session.flash("Integration added")
                 resource.integrations.append(self.integration)
             else:
-                self.request.session.flash('Integration updated')
+                self.request.session.flash("Integration updated")
             self.integration.config = integration_config
             DBSession.flush()
-            self.create_missing_channel(resource, 'flowdock')
+            self.create_missing_channel(resource, "flowdock")
             return integration_config
-        elif self.request.method == 'POST':
+        elif self.request.method == "POST":
             return HTTPUnprocessableEntity(body=form.errors_json)
         return self.integration_config

backend/src/appenlight/views/integrations/github.py

@@ -1,122 +1,129 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from appenlight.models import DBSession
-from appenlight.models.integrations.github import GithubIntegration, \
-    IntegrationException
+from appenlight.models.integrations.github import (
+    GithubIntegration,
+    IntegrationException,
+)
 from appenlight.models.report_comment import ReportComment
 from appenlight.models.services.report_group import ReportGroupService
 from pyramid.view import view_config
 from appenlight import forms
 import logging
 from datetime import datetime
 from pyramid.httpexceptions import HTTPFound, HTTPUnprocessableEntity
 from webob.multidict import MultiDict
 
 log = logging.getLogger(__name__)
 
 from . import IntegrationView
 
 
 class GithubView(IntegrationView):
-    @view_config(route_name='integrations_id',
-                 match_param=['action=info', 'integration=github'],
-                 renderer='json')
+    @view_config(
+        route_name="integrations_id",
+        match_param=["action=info", "integration=github"],
+        renderer="json",
+    )
     def get_github_info(self):
         """
         Grab information about possible priority statuses and assignable users
         """
         try:
             client = GithubIntegration.create_client(
                 self.request,
-                self.integration.config['user_name'],
-                self.integration.config['repo_name'])
+                self.integration.config["user_name"],
+                self.integration.config["repo_name"],
+            )
         except IntegrationException as e:
             self.request.response.status_code = 503
-            return {'error_messages': [str(e)]}
+            return {"error_messages": [str(e)]}
         try:
             assignees = client.get_assignees()
             statuses = client.get_statuses()
         except IntegrationException as e:
-            return {'error_messages': [str(e)]}
-        return {'assignees': assignees,
-                'statuses': statuses}
+            return {"error_messages": [str(e)]}
+        return {"assignees": assignees, "statuses": statuses}
 
-    @view_config(route_name='integrations_id',
-                 match_param=['action=create-issue', 'integration=github'],
-                 renderer='json')
+    @view_config(
+        route_name="integrations_id",
+        match_param=["action=create-issue", "integration=github"],
+        renderer="json",
+    )
     def create_issue(self):
         """
         Creates a new issue in github issue tracker from report group
         """
-        report = ReportGroupService.by_id(
-            self.request.unsafe_json_body['group_id'])
+        report = ReportGroupService.by_id(self.request.unsafe_json_body["group_id"])
         form_data = {
-            'title': self.request.unsafe_json_body.get('title',
-                                                       'Unknown Title'),
-            'content': self.request.unsafe_json_body.get('content'),
-            'kind': [self.request.unsafe_json_body['status']],
-            'responsible': self.request.unsafe_json_body['responsible']['user']
+            "title": self.request.unsafe_json_body.get("title", "Unknown Title"),
+            "content": self.request.unsafe_json_body.get("content"),
+            "kind": [self.request.unsafe_json_body["status"]],
+            "responsible": self.request.unsafe_json_body["responsible"]["user"],
         }
 
         try:
             client = GithubIntegration.create_client(
                 self.request,
-                self.integration.config['user_name'],
-                self.integration.config['repo_name'])
+                self.integration.config["user_name"],
+                self.integration.config["repo_name"],
+            )
             issue = client.create_issue(form_data)
         except IntegrationException as e:
             self.request.response.status_code = 503
-            return {'error_messages': [str(e)]}
+            return {"error_messages": [str(e)]}
 
-        comment_body = 'Github issue created: %s ' % issue['web_url']
-        comment = ReportComment(owner_id=self.request.user.id,
-                                report_time=report.first_timestamp,
-                                body=comment_body)
+        comment_body = "Github issue created: %s " % issue["web_url"]
+        comment = ReportComment(
+            owner_id=self.request.user.id,
+            report_time=report.first_timestamp,
+            body=comment_body,
+        )
         report.comments.append(comment)
         return True
 
-    @view_config(route_name='integrations_id',
-                 match_param=['action=setup', 'integration=github'],
-                 renderer='json',
-                 permission='edit')
+    @view_config(
+        route_name="integrations_id",
+        match_param=["action=setup", "integration=github"],
+        renderer="json",
+        permission="edit",
+    )
     def setup(self):
         """
         Validates and creates integration between application and github
         """
         resource = self.request.context.resource
         form = forms.IntegrationGithubForm(
             MultiDict(self.request.safe_json_body or {}),
             csrf_context=self.request,
-            **self.integration_config)
-        if self.request.method == 'POST' and form.validate():
+            **self.integration_config
+        )
+        if self.request.method == "POST" and form.validate():
             integration_config = {
-                'repo_name': form.repo_name.data,
-                'user_name': form.user_name.data,
-                'host_name': 'https://api.github.com'
+                "repo_name": form.repo_name.data,
+                "user_name": form.user_name.data,
+                "host_name": "https://api.github.com",
             }
             if not self.integration:
-                self.integration = GithubIntegration(
-                    modified_date=datetime.utcnow(),
-
-                )
+                self.integration = GithubIntegration(modified_date=datetime.utcnow())
             self.integration.config = integration_config
             resource.integrations.append(self.integration)
-            self.request.session.flash('Integration updated')
+            self.request.session.flash("Integration updated")
             return integration_config
-        elif self.request.method == 'POST':
+        elif self.request.method == "POST":
             return HTTPUnprocessableEntity(body=form.errors_json)
         return self.integration_config

backend/src/appenlight/views/integrations/hipchat.py

@@ -1,73 +1,77 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 from datetime import datetime
 
 from appenlight.models import DBSession
 from appenlight.models.integrations.hipchat import HipchatIntegration
 from pyramid.httpexceptions import HTTPUnprocessableEntity
 from pyramid.view import view_config
 from appenlight import forms
 
 from webob.multidict import MultiDict
 
 log = logging.getLogger(__name__)
 
 from . import IntegrationView
 
 
 class HipchatView(IntegrationView):
-    @view_config(route_name='integrations_id',
-                 match_param=['action=info', 'integration=hipchat'],
-                 renderer='json')
+    @view_config(
+        route_name="integrations_id",
+        match_param=["action=info", "integration=hipchat"],
+        renderer="json",
+    )
     def get_info(self):
         pass
 
-    @view_config(route_name='integrations_id',
-                 match_param=['action=setup', 'integration=hipchat'],
-                 renderer='json',
-                 permission='edit')
+    @view_config(
+        route_name="integrations_id",
+        match_param=["action=setup", "integration=hipchat"],
+        renderer="json",
+        permission="edit",
+    )
     def setup(self):
         """
         Validates and creates integration between application and hipchat
         """
         resource = self.request.context.resource
-        self.create_missing_channel(resource, 'hipchat')
+        self.create_missing_channel(resource, "hipchat")
         form = forms.IntegrationHipchatForm(
             MultiDict(self.request.safe_json_body or {}),
-            csrf_context=self.request, **self.integration_config)
-        if self.request.method == 'POST' and form.validate():
+            csrf_context=self.request,
+            **self.integration_config
+        )
+        if self.request.method == "POST" and form.validate():
             integration_config = {
-                'api_token': form.api_token.data,
-                'rooms': form.rooms.data,
+                "api_token": form.api_token.data,
+                "rooms": form.rooms.data,
             }
             if not self.integration:
                 # add new integration
-                self.integration = HipchatIntegration(
-                    modified_date=datetime.utcnow(),
-                )
-                self.request.session.flash('Integration added')
+                self.integration = HipchatIntegration(modified_date=datetime.utcnow())
+                self.request.session.flash("Integration added")
                 resource.integrations.append(self.integration)
             else:
-                self.request.session.flash('Integration updated')
+                self.request.session.flash("Integration updated")
             self.integration.config = integration_config
             DBSession.flush()
-            self.create_missing_channel(resource, 'hipchat')
+            self.create_missing_channel(resource, "hipchat")
             return integration_config
-        elif self.request.method == 'POST':
+        elif self.request.method == "POST":
             return HTTPUnprocessableEntity(body=form.errors_json)
         return self.integration_config

backend/src/appenlight/views/integrations/jira.py

@@ -1,148 +1,158 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import copy
 import logging
-from appenlight.models.integrations.jira import JiraIntegration, \
-    JiraClient, IntegrationException
+from appenlight.models.integrations.jira import (
+    JiraIntegration,
+    JiraClient,
+    IntegrationException,
+)
 from appenlight.models.report_comment import ReportComment
 from appenlight.models.services.report_group import ReportGroupService
 from pyramid.view import view_config
 from appenlight import forms
 from datetime import datetime
 from pyramid.httpexceptions import HTTPFound, HTTPUnprocessableEntity
 from webob.multidict import MultiDict
 
 log = logging.getLogger(__name__)
 
 from . import IntegrationView
 
 
 class JiraView(IntegrationView):
     def create_client(self, form=None):
         """
         Creates a client that can make authenticated requests to jira
         """
         if self.integration and not form:
-            user_name = self.integration.config['user_name']
-            password = self.integration.config['password']
-            host_name = self.integration.config['host_name']
-            project = self.integration.config['project']
+            user_name = self.integration.config["user_name"]
+            password = self.integration.config["password"]
+            host_name = self.integration.config["host_name"]
+            project = self.integration.config["project"]
         else:
             user_name, password = form.user_name.data, form.password.data
             host_name = form.host_name.data
             project = form.host_name.data
-        client = JiraClient(user_name, password, host_name, project,
-                            request=self.request)
+        client = JiraClient(
+            user_name, password, host_name, project, request=self.request
+        )
         return client
 
-    @view_config(route_name='integrations_id',
-                 match_param=['action=info', 'integration=jira'],
-                 renderer='json')
+    @view_config(
+        route_name="integrations_id",
+        match_param=["action=info", "integration=jira"],
+        renderer="json",
+    )
     def get_jira_info(self):
         """
         Get basic metadata - assignees and priority levels from tracker
         """
         try:
             client = self.create_client()
         except IntegrationException as e:
             self.request.response.status_code = 503
-            return {'error_messages': [str(e)]}
+            return {"error_messages": [str(e)]}
         assignees = []
         priorities = []
         issue_types = []
         possible_issue_types = client.get_issue_types(self.request)
         for issue_type in possible_issue_types:
-            for field in issue_type['fields']:
-                if field['id'] == 'assignee':
-                    assignees = field['values']
-                if field['id'] == 'priority':
-                    priorities = field['values']
-            issue_types.append({'name':issue_type['name'],
-                                'id':issue_type['id']})
-        return {'assignees': assignees,
-                'priorities': priorities,
-                'issue_types': issue_types}
+            for field in issue_type["fields"]:
+                if field["id"] == "assignee":
+                    assignees = field["values"]
+                if field["id"] == "priority":
+                    priorities = field["values"]
+            issue_types.append({"name": issue_type["name"], "id": issue_type["id"]})
+        return {
+            "assignees": assignees,
+            "priorities": priorities,
+            "issue_types": issue_types,
+        }
 
-    @view_config(route_name='integrations_id',
-                 match_param=['action=create-issue',
-                              'integration=jira'],
-                 renderer='json')
+    @view_config(
+        route_name="integrations_id",
+        match_param=["action=create-issue", "integration=jira"],
+        renderer="json",
+    )
     def create_issue(self):
         """
         Creates a new issue in jira from report group
         """
-        report = ReportGroupService.by_id(
-            self.request.unsafe_json_body['group_id'])
+        report = ReportGroupService.by_id(self.request.unsafe_json_body["group_id"])
         form_data = {
-            'title': self.request.unsafe_json_body.get('title',
-                                                       'Unknown Title'),
-            'content': self.request.unsafe_json_body.get('content', ''),
-            'issue_type': self.request.unsafe_json_body['issue_type']['id'],
-            'priority': self.request.unsafe_json_body['priority']['id'],
-            'responsible': self.request.unsafe_json_body['responsible']['id'],
-            'project': self.integration.config['project']
+            "title": self.request.unsafe_json_body.get("title", "Unknown Title"),
+            "content": self.request.unsafe_json_body.get("content", ""),
+            "issue_type": self.request.unsafe_json_body["issue_type"]["id"],
+            "priority": self.request.unsafe_json_body["priority"]["id"],
+            "responsible": self.request.unsafe_json_body["responsible"]["id"],
+            "project": self.integration.config["project"],
         }
         try:
             client = self.create_client()
             issue = client.create_issue(form_data, request=self.request)
         except IntegrationException as e:
             self.request.response.status_code = 503
-            return {'error_messages': [str(e)]}
+            return {"error_messages": [str(e)]}
 
-        comment_body = 'Jira issue created: %s ' % issue['web_url']
-        comment = ReportComment(owner_id=self.request.user.id,
-                                report_time=report.first_timestamp,
-                                body=comment_body)
+        comment_body = "Jira issue created: %s " % issue["web_url"]
+        comment = ReportComment(
+            owner_id=self.request.user.id,
+            report_time=report.first_timestamp,
+            body=comment_body,
+        )
         report.comments.append(comment)
         return True
 
-    @view_config(route_name='integrations_id',
-                 match_param=['action=setup', 'integration=jira'],
-                 renderer='json',
-                 permission='edit')
+    @view_config(
+        route_name="integrations_id",
+        match_param=["action=setup", "integration=jira"],
+        renderer="json",
+        permission="edit",
+    )
     def setup(self):
         """
         Validates and creates integration between application and jira
         """
         resource = self.request.context.resource
         form = forms.IntegrationJiraForm(
             MultiDict(self.request.safe_json_body or {}),
-            csrf_context=self.request, **self.integration_config)
-        if self.request.method == 'POST' and form.validate():
+            csrf_context=self.request,
+            **self.integration_config
+        )
+        if self.request.method == "POST" and form.validate():
             integration_config = {
-                'user_name': form.user_name.data,
-                'password': form.password.data,
-                'host_name': form.host_name.data,
-                'project': form.project.data
+                "user_name": form.user_name.data,
+                "password": form.password.data,
+                "host_name": form.host_name.data,
+                "project": form.project.data,
             }
             if not self.integration:
                 # add new integration
-                self.integration = JiraIntegration(
-                    modified_date=datetime.utcnow(),
-                )
-                self.request.session.flash('Integration added')
+                self.integration = JiraIntegration(modified_date=datetime.utcnow())
+                self.request.session.flash("Integration added")
                 resource.integrations.append(self.integration)
             else:
-                self.request.session.flash('Integration updated')
+                self.request.session.flash("Integration updated")
             self.integration.config = integration_config
             return integration_config
-        elif self.request.method == 'POST':
+        elif self.request.method == "POST":
             return HTTPUnprocessableEntity(body=form.errors_json)
 
         to_return = copy.deepcopy(self.integration_config)
-        to_return.pop('password', None)
+        to_return.pop("password", None)
         return to_return

backend/src/appenlight/views/integrations/slack.py

@@ -1,73 +1,74 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 
 from appenlight.models import DBSession
-from appenlight.models.integrations.slack import SlackIntegration, \
-    IntegrationException
+from appenlight.models.integrations.slack import SlackIntegration, IntegrationException
 from pyramid.httpexceptions import HTTPUnprocessableEntity
 from pyramid.view import view_config
 from appenlight import forms
 from datetime import datetime
 from webob.multidict import MultiDict
 
 log = logging.getLogger(__name__)
 
 from . import IntegrationView
 
 
 class SlackView(IntegrationView):
-    @view_config(route_name='integrations_id',
-                 match_param=['action=info', 'integration=slack'],
-                 renderer='json')
+    @view_config(
+        route_name="integrations_id",
+        match_param=["action=info", "integration=slack"],
+        renderer="json",
+    )
     def get_info(self):
         pass
 
-    @view_config(route_name='integrations_id',
-                 match_param=['action=setup', 'integration=slack'],
-                 renderer='json',
-                 permission='edit')
+    @view_config(
+        route_name="integrations_id",
+        match_param=["action=setup", "integration=slack"],
+        renderer="json",
+        permission="edit",
+    )
     def setup(self):
         """
         Validates and creates integration between application and slack
         """
         resource = self.request.context.resource
-        self.create_missing_channel(resource, 'slack')
+        self.create_missing_channel(resource, "slack")
         form = forms.IntegrationSlackForm(
             MultiDict(self.request.safe_json_body or {}),
-            csrf_context=self.request, **self.integration_config)
+            csrf_context=self.request,
+            **self.integration_config
+        )
 
-        if self.request.method == 'POST' and form.validate():
-            integration_config = {
-                'webhook_url': form.webhook_url.data
-            }
+        if self.request.method == "POST" and form.validate():
+            integration_config = {"webhook_url": form.webhook_url.data}
             if not self.integration:
                 # add new integration
-                self.integration = SlackIntegration(
-                    modified_date=datetime.utcnow(),
-                )
-                self.request.session.flash('Integration added')
+                self.integration = SlackIntegration(modified_date=datetime.utcnow())
+                self.request.session.flash("Integration added")
                 resource.integrations.append(self.integration)
             else:
-                self.request.session.flash('Integration updated')
+                self.request.session.flash("Integration updated")
             self.integration.config = integration_config
             DBSession.flush()
-            self.create_missing_channel(resource, 'slack')
+            self.create_missing_channel(resource, "slack")
             return integration_config
-        elif self.request.method == 'POST':
+        elif self.request.method == "POST":
             return HTTPUnprocessableEntity(body=form.errors_json)
         return self.integration_config

backend/src/appenlight/views/integrations/webhooks.py

@@ -1,72 +1,79 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from appenlight.models import DBSession
-from appenlight.models.integrations.webhooks import WebhooksIntegration, \
-    IntegrationException
+from appenlight.models.integrations.webhooks import (
+    WebhooksIntegration,
+    IntegrationException,
+)
 from pyramid.httpexceptions import HTTPUnprocessableEntity
 from pyramid.view import view_config
 from appenlight import forms
 import logging
 from datetime import datetime
 from webob.multidict import MultiDict
 
 log = logging.getLogger(__name__)
 
 from . import IntegrationView
 
 
 class WebhooksView(IntegrationView):
-    @view_config(route_name='integrations_id',
-                 match_param=['action=info', 'integration=webhooks'],
-                 renderer='json')
+    @view_config(
+        route_name="integrations_id",
+        match_param=["action=info", "integration=webhooks"],
+        renderer="json",
+    )
     def get_info(self):
         pass
 
-    @view_config(route_name='integrations_id',
-                 match_param=['action=setup', 'integration=webhooks'],
-                 renderer='json', permission='edit')
+    @view_config(
+        route_name="integrations_id",
+        match_param=["action=setup", "integration=webhooks"],
+        renderer="json",
+        permission="edit",
+    )
     def setup(self):
         """
         Creates webhook integration
         """
         resource = self.request.context.resource
-        self.create_missing_channel(resource, 'webhooks')
+        self.create_missing_channel(resource, "webhooks")
 
         form = forms.IntegrationWebhooksForm(
             MultiDict(self.request.safe_json_body or {}),
-            csrf_context=self.request, **self.integration_config)
-        if self.request.method == 'POST' and form.validate():
+            csrf_context=self.request,
+            **self.integration_config
+        )
+        if self.request.method == "POST" and form.validate():
             integration_config = {
-                'reports_webhook': form.reports_webhook.data,
-                'alerts_webhook': form.alerts_webhook.data,
+                "reports_webhook": form.reports_webhook.data,
+                "alerts_webhook": form.alerts_webhook.data,
             }
             if not self.integration:
                 # add new integration
-                self.integration = WebhooksIntegration(
-                    modified_date=datetime.utcnow(),
-                )
-                self.request.session.flash('Integration added')
+                self.integration = WebhooksIntegration(modified_date=datetime.utcnow())
+                self.request.session.flash("Integration added")
                 resource.integrations.append(self.integration)
             else:
-                self.request.session.flash('Integration updated')
+                self.request.session.flash("Integration updated")
             self.integration.config = integration_config
             DBSession.flush()
-            self.create_missing_channel(resource, 'webhooks')
+            self.create_missing_channel(resource, "webhooks")
             return integration_config
-        elif self.request.method == 'POST':
+        elif self.request.method == "POST":
             return HTTPUnprocessableEntity(body=form.errors_json)
         return self.integration_config

backend/src/appenlight/views/logs.py

@@ -1,224 +1,219 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 from datetime import datetime, timedelta
 
 from pyramid.view import view_config
 from pyramid.httpexceptions import HTTPUnprocessableEntity
 from appenlight.models import Datastores, Log
 from appenlight.models.services.log import LogService
-from appenlight.lib.utils import (build_filter_settings_from_query_dict,
-                                  es_index_name_limiter)
+from appenlight.lib.utils import (
+    build_filter_settings_from_query_dict,
+    es_index_name_limiter,
+)
 from appenlight.lib.helpers import gen_pagination_headers
 from appenlight.celery.tasks import logs_cleanup
 
 log = logging.getLogger(__name__)
 
-section_filters_key = 'appenlight:logs:filter:%s'
+section_filters_key = "appenlight:logs:filter:%s"
 
 
-@view_config(route_name='logs_no_id', renderer='json',
-             permission='authenticated')
+@view_config(route_name="logs_no_id", renderer="json", permission="authenticated")
 def fetch_logs(request):
     """
     Returns list of log entries from Elasticsearch
     """
 
-    filter_settings = build_filter_settings_from_query_dict(request,
-                                                            request.GET.mixed())
+    filter_settings = build_filter_settings_from_query_dict(
+        request, request.GET.mixed()
+    )
     logs_paginator = LogService.get_paginator_by_app_ids(
-        app_ids=filter_settings['resource'],
-        page=filter_settings['page'],
-        filter_settings=filter_settings
+        app_ids=filter_settings["resource"],
+        page=filter_settings["page"],
+        filter_settings=filter_settings,
     )
     headers = gen_pagination_headers(request, logs_paginator)
     request.response.headers.update(headers)
 
     return [l.get_dict() for l in logs_paginator.sa_items]
 
 
-@view_config(route_name='section_view',
-             match_param=['section=logs_section', 'view=fetch_series'],
-             renderer='json', permission='authenticated')
+@view_config(
+    route_name="section_view",
+    match_param=["section=logs_section", "view=fetch_series"],
+    renderer="json",
+    permission="authenticated",
+)
 def logs_fetch_series(request):
     """
     Handles metric dashboard graphs
     Returns information for time/tier breakdown
     """
-    filter_settings = build_filter_settings_from_query_dict(request,
-                                                            request.GET.mixed())
+    filter_settings = build_filter_settings_from_query_dict(
+        request, request.GET.mixed()
+    )
     paginator = LogService.get_paginator_by_app_ids(
-        app_ids=filter_settings['resource'],
-        page=1, filter_settings=filter_settings, items_per_page=1)
+        app_ids=filter_settings["resource"],
+        page=1,
+        filter_settings=filter_settings,
+        items_per_page=1,
+    )
     now = datetime.utcnow().replace(microsecond=0, second=0)
     delta = timedelta(days=7)
     if paginator.sa_items:
-        start_date = paginator.sa_items[-1].timestamp.replace(microsecond=0,
-                                                              second=0)
-        filter_settings['start_date'] = start_date - delta
+        start_date = paginator.sa_items[-1].timestamp.replace(microsecond=0, second=0)
+        filter_settings["start_date"] = start_date - delta
     else:
-        filter_settings['start_date'] = now - delta
-    filter_settings['end_date'] = filter_settings['start_date'] \
-                                  + timedelta(days=7)
+        filter_settings["start_date"] = now - delta
+    filter_settings["end_date"] = filter_settings["start_date"] + timedelta(days=7)
 
-    @request.registry.cache_regions.redis_sec_30.cache_on_arguments(
-        'logs_graphs')
+    @request.registry.cache_regions.redis_sec_30.cache_on_arguments("logs_graphs")
     def cached(apps, search_params, delta, now):
         data = LogService.get_time_series_aggregate(
-            filter_settings['resource'], filter_settings)
+            filter_settings["resource"], filter_settings
+        )
         if not data:
             return []
-        buckets = data['aggregations']['events_over_time']['buckets']
-        return [{"x": datetime.utcfromtimestamp(item["key"] / 1000),
-                 "logs": item["doc_count"]} for item in buckets]
+        buckets = data["aggregations"]["events_over_time"]["buckets"]
+        return [
+            {
+                "x": datetime.utcfromtimestamp(item["key"] / 1000),
+                "logs": item["doc_count"],
+            }
+            for item in buckets
+        ]
 
     return cached(filter_settings, request.GET.mixed(), delta, now)
 
 
-@view_config(route_name='logs_no_id', renderer='json', request_method="DELETE",
-             permission='authenticated')
+@view_config(
+    route_name="logs_no_id",
+    renderer="json",
+    request_method="DELETE",
+    permission="authenticated",
+)
 def logs_mass_delete(request):
     params = request.GET.mixed()
-    if 'resource' not in params:
+    if "resource" not in params:
         raise HTTPUnprocessableEntity()
     # this might be '' and then colander will not validate the schema
-    if not params.get('namespace'):
-        params.pop('namespace', None)
+    if not params.get("namespace"):
+        params.pop("namespace", None)
     filter_settings = build_filter_settings_from_query_dict(
-        request, params, resource_permissions=['update_reports'])
+        request, params, resource_permissions=["update_reports"]
+    )
 
-    resource_id = list(filter_settings['resource'])[0]
+    resource_id = list(filter_settings["resource"])[0]
     # filter settings returns list of all of users applications
     # if app is not matching - normally we would not care as its used for search
     # but here user playing with params would possibly wipe out their whole data
-    if int(resource_id) != int(params['resource']):
+    if int(resource_id) != int(params["resource"]):
         raise HTTPUnprocessableEntity()
 
     logs_cleanup.delay(resource_id, filter_settings)
-    msg = 'Log cleanup process started - it may take a while for ' \
-          'everything to get removed'
+    msg = (
+        "Log cleanup process started - it may take a while for "
+        "everything to get removed"
+    )
     request.session.flash(msg)
     return {}
 
 
-@view_config(route_name='section_view',
-             match_param=("view=common_tags", "section=logs_section"),
-             renderer='json', permission='authenticated')
+@view_config(
+    route_name="section_view",
+    match_param=("view=common_tags", "section=logs_section"),
+    renderer="json",
+    permission="authenticated",
+)
 def common_tags(request):
     config = request.GET.mixed()
-    filter_settings = build_filter_settings_from_query_dict(request,
-                                                            config)
+    filter_settings = build_filter_settings_from_query_dict(request, config)
 
     resources = list(filter_settings["resource"])
     query = {
         "query": {
             "filtered": {
-                "filter": {
-                    "and": [{"terms": {"resource_id": list(resources)}}]
-                }
+                "filter": {"and": [{"terms": {"resource_id": list(resources)}}]}
             }
         }
     }
-    start_date = filter_settings.get('start_date')
-    end_date = filter_settings.get('end_date')
-    filter_part = query['query']['filtered']['filter']['and']
+    start_date = filter_settings.get("start_date")
+    end_date = filter_settings.get("end_date")
+    filter_part = query["query"]["filtered"]["filter"]["and"]
 
     date_range = {"range": {"timestamp": {}}}
     if start_date:
         date_range["range"]["timestamp"]["gte"] = start_date
     if end_date:
         date_range["range"]["timestamp"]["lte"] = end_date
     if start_date or end_date:
         filter_part.append(date_range)
 
-    levels = filter_settings.get('level')
+    levels = filter_settings.get("level")
     if levels:
-        filter_part.append({"terms": {'log_level': levels}})
-    namespaces = filter_settings.get('namespace')
+        filter_part.append({"terms": {"log_level": levels}})
+    namespaces = filter_settings.get("namespace")
     if namespaces:
-        filter_part.append({"terms": {'namespace': namespaces}})
+        filter_part.append({"terms": {"namespace": namespaces}})
 
-    query["aggs"] = {
-        "sub_agg": {
-            "terms": {
-                "field": "tag_list",
-                "size": 50
-            }
-        }
-    }
+    query["aggs"] = {"sub_agg": {"terms": {"field": "tag_list", "size": 50}}}
     # tags
-    index_names = es_index_name_limiter(
-        ixtypes=[config.get('datasource', 'logs')])
-    result = Datastores.es.search(body=query, index=index_names, doc_type='log',
-                                  size=0)
-    tag_buckets = result['aggregations']['sub_agg'].get('buckets', [])
+    index_names = es_index_name_limiter(ixtypes=[config.get("datasource", "logs")])
+    result = Datastores.es.search(body=query, index=index_names, doc_type="log", size=0)
+    tag_buckets = result["aggregations"]["sub_agg"].get("buckets", [])
     # namespaces
-    query["aggs"] = {
-        "sub_agg": {
-            "terms": {
-                "field": "namespace",
-                "size": 50
-            }
-        }
-    }
-    result = Datastores.es.search(body=query, index=index_names, doc_type='log',
-                                  size=0)
-    namespaces_buckets = result['aggregations']['sub_agg'].get('buckets', [])
+    query["aggs"] = {"sub_agg": {"terms": {"field": "namespace", "size": 50}}}
+    result = Datastores.es.search(body=query, index=index_names, doc_type="log", size=0)
+    namespaces_buckets = result["aggregations"]["sub_agg"].get("buckets", [])
     return {
-        "tags": [item['key'] for item in tag_buckets],
-        "namespaces": [item['key'] for item in namespaces_buckets]
+        "tags": [item["key"] for item in tag_buckets],
+        "namespaces": [item["key"] for item in namespaces_buckets],
     }
 
 
-@view_config(route_name='section_view',
-             match_param=("view=common_values", "section=logs_section"),
-             renderer='json', permission='authenticated')
+@view_config(
+    route_name="section_view",
+    match_param=("view=common_values", "section=logs_section"),
+    renderer="json",
+    permission="authenticated",
+)
 def common_values(request):
     config = request.GET.mixed()
-    datasource = config.pop('datasource', 'logs')
-    filter_settings = build_filter_settings_from_query_dict(request,
-                                                            config)
+    datasource = config.pop("datasource", "logs")
+    filter_settings = build_filter_settings_from_query_dict(request, config)
     resources = list(filter_settings["resource"])
-    tag_name = filter_settings['tags'][0]['value'][0]
+    tag_name = filter_settings["tags"][0]["value"][0]
     query = {
-        'query': {
-            'filtered': {
-                'filter': {
-                    'and': [
-                        {'terms': {'resource_id': list(resources)}},
-                        {'terms': {
-                            'namespace': filter_settings['namespace']}}
+        "query": {
+            "filtered": {
+                "filter": {
+                    "and": [
+                        {"terms": {"resource_id": list(resources)}},
+                        {"terms": {"namespace": filter_settings["namespace"]}},
                     ]
                 }
             }
         }
     }
-    query['aggs'] = {
-        'sub_agg': {
-            'terms': {
-                'field': 'tags.{}.values'.format(tag_name),
-                'size': 50
-            }
-        }
+    query["aggs"] = {
+        "sub_agg": {"terms": {"field": "tags.{}.values".format(tag_name), "size": 50}}
     }
     index_names = es_index_name_limiter(ixtypes=[datasource])
-    result = Datastores.es.search(body=query, index=index_names, doc_type='log',
-                                  size=0)
-    values_buckets = result['aggregations']['sub_agg'].get('buckets', [])
-    return {
-        "values": [item['key'] for item in values_buckets]
-    }
+    result = Datastores.es.search(body=query, index=index_names, doc_type="log", size=0)
+    values_buckets = result["aggregations"]["sub_agg"].get("buckets", [])
+    return {"values": [item["key"] for item in values_buckets]}

backend/src/appenlight/views/plugin_configs.py

@@ -1,32 +1,37 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from pyramid.view import view_config
 from appenlight.models.services.plugin_config import PluginConfigService
 
 import logging
 
 log = logging.getLogger(__name__)
 
 
-@view_config(route_name='plugin_configs', renderer='json',
-             permission='edit', request_method='GET')
+@view_config(
+    route_name="plugin_configs",
+    renderer="json",
+    permission="edit",
+    request_method="GET",
+)
 def query(request):
     configs = PluginConfigService.by_query(
-        request.params.get('resource_id'),
-        plugin_name=request.matchdict.get('plugin_name'),
-        section=request.params.get('section'))
+        request.params.get("resource_id"),
+        plugin_name=request.matchdict.get("plugin_name"),
+        section=request.params.get("section"),
+    )
     return [c for c in configs]

backend/src/appenlight/views/reports.py

@@ -1,297 +1,353 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 
 from datetime import datetime, timedelta
 from pyramid.view import view_config
 from pyramid.httpexceptions import HTTPUnprocessableEntity
 
 from ziggurat_foundations.models.services.resource import ResourceService
 from ziggurat_foundations.models.services.user import UserService
 
 from appenlight.models import DBSession
 from appenlight.models.user import User
 from appenlight.models.report_comment import ReportComment
 from appenlight.models.report_assignment import ReportAssignment
 from appenlight.models.services.user import UserService
 from appenlight.models.services.report_group import ReportGroupService
 from appenlight import forms
 from appenlight.lib.enums import ReportType
 from appenlight.lib.helpers import gen_pagination_headers
 from appenlight.lib.utils import build_filter_settings_from_query_dict
-from appenlight.validators import ReportSearchSchema, TagListSchema, \
-    accepted_search_params
+from appenlight.validators import (
+    ReportSearchSchema,
+    TagListSchema,
+    accepted_search_params,
+)
 from webob.multidict import MultiDict
 
 _ = str
 
 log = logging.getLogger(__name__)
 
-section_filters_key = 'appenlight:reports:filter:%s'
+section_filters_key = "appenlight:reports:filter:%s"
 
 
-@view_config(route_name='reports', renderer='json', permission='authenticated')
-@view_config(route_name='slow_reports', renderer='json',
-             permission='authenticated')
+@view_config(route_name="reports", renderer="json", permission="authenticated")
+@view_config(route_name="slow_reports", renderer="json", permission="authenticated")
 def index(request):
     """
     Returns list of report groups based on user search query
     """
     if request.user:
         request.user.last_login_date = datetime.utcnow()
 
     applications = UserService.resources_with_perms(
-        request.user, ['view'], resource_types=['application'])
+        request.user, ["view"], resource_types=["application"]
+    )
 
     search_params = request.GET.mixed()
 
     all_possible_app_ids = set([app.resource_id for app in applications])
     schema = ReportSearchSchema().bind(resources=all_possible_app_ids)
     tag_schema = TagListSchema()
     filter_settings = schema.deserialize(search_params)
-    tag_list = [{"name": k, "value": v} for k, v in filter_settings.items()
-                if k not in accepted_search_params]
+    tag_list = [
+        {"name": k, "value": v}
+        for k, v in filter_settings.items()
+        if k not in accepted_search_params
+    ]
     tags = tag_schema.deserialize(tag_list)
-    filter_settings['tags'] = tags
-    if request.matched_route.name == 'slow_reports':
-        filter_settings['report_type'] = [ReportType.slow]
+    filter_settings["tags"] = tags
+    if request.matched_route.name == "slow_reports":
+        filter_settings["report_type"] = [ReportType.slow]
     else:
-        filter_settings['report_type'] = [ReportType.error]
+        filter_settings["report_type"] = [ReportType.error]
 
     reports_paginator = ReportGroupService.get_paginator_by_app_ids(
-        app_ids=filter_settings['resource'],
-        page=filter_settings['page'],
-        filter_settings=filter_settings
+        app_ids=filter_settings["resource"],
+        page=filter_settings["page"],
+        filter_settings=filter_settings,
     )
     reports = []
-    include_keys = ('id', 'http_status', 'report_type', 'resource_name',
-                    'front_url', 'resource_id', 'error', 'url_path', 'tags',
-                    'duration')
+    include_keys = (
+        "id",
+        "http_status",
+        "report_type",
+        "resource_name",
+        "front_url",
+        "resource_id",
+        "error",
+        "url_path",
+        "tags",
+        "duration",
+    )
     for report in reports_paginator.sa_items:
         reports.append(report.get_dict(request, include_keys=include_keys))
     headers = gen_pagination_headers(request, reports_paginator)
     request.response.headers.update(headers)
     return reports
 
 
-@view_config(route_name='report_groups', renderer='json', permission='view',
-             request_method="GET")
+@view_config(
+    route_name="report_groups", renderer="json", permission="view", request_method="GET"
+)
 def view_report(request):
     """
     Show individual detailed report group along with latest report
     """
     report_group = request.context.report_group
     if not report_group.read:
         report_group.read = True
 
-    report_id = request.params.get('reportId', request.params.get('report_id'))
-    report_dict = report_group.get_report(report_id).get_dict(request,
-                                                              details=True)
+    report_id = request.params.get("reportId", request.params.get("report_id"))
+    report_dict = report_group.get_report(report_id).get_dict(request, details=True)
     # disallow browsing other occurences by anonymous
     if not request.user:
-        report_dict.pop('group_next_report', None)
-        report_dict.pop('group_previous_report', None)
+        report_dict.pop("group_next_report", None)
+        report_dict.pop("group_previous_report", None)
     return report_dict
 
 
-@view_config(route_name='report_groups', renderer='json',
-             permission='update_reports', request_method='DELETE')
+@view_config(
+    route_name="report_groups",
+    renderer="json",
+    permission="update_reports",
+    request_method="DELETE",
+)
 def remove(request):
     """
     Used to remove reourt groups from database
     """
     report = request.context.report_group
     form = forms.ReactorForm(request.POST, csrf_context=request)
     form.validate()
     DBSession.delete(report)
     return True
 
 
-@view_config(route_name='report_groups_property', match_param='key=comments',
-             renderer='json', permission='view', request_method="POST")
+@view_config(
+    route_name="report_groups_property",
+    match_param="key=comments",
+    renderer="json",
+    permission="view",
+    request_method="POST",
+)
 def comment_create(request):
     """
     Creates user comments for report group, sends email notifications
     of said comments
     """
     report_group = request.context.report_group
     application = request.context.resource
-    form = forms.CommentForm(MultiDict(request.unsafe_json_body),
-                             csrf_context=request)
-    if request.method == 'POST' and form.validate():
-        comment = ReportComment(owner_id=request.user.id,
-                                report_time=report_group.first_timestamp)
+    form = forms.CommentForm(MultiDict(request.unsafe_json_body), csrf_context=request)
+    if request.method == "POST" and form.validate():
+        comment = ReportComment(
+            owner_id=request.user.id, report_time=report_group.first_timestamp
+        )
         form.populate_obj(comment)
         report_group.comments.append(comment)
-        perm_list = ResourceService.users_for_perm(application, 'view')
+        perm_list = ResourceService.users_for_perm(application, "view")
         uids_to_notify = []
         users_to_notify = []
         for perm in perm_list:
             user = perm.user
-            if ('@{}'.format(user.user_name) in comment.body and
-                        user.id not in uids_to_notify):
+            if (
+                "@{}".format(user.user_name) in comment.body
+                and user.id not in uids_to_notify
+            ):
                 uids_to_notify.append(user.id)
                 users_to_notify.append(user)
 
         commenters = ReportGroupService.users_commenting(
-            report_group, exclude_user_id=request.user.id)
+            report_group, exclude_user_id=request.user.id
+        )
         for user in commenters:
             if user.id not in uids_to_notify:
                 uids_to_notify.append(user.id)
                 users_to_notify.append(user)
 
         for user in users_to_notify:
-            email_vars = {'user': user,
-                          'commenting_user': request.user,
-                          'request': request,
-                          'application': application,
-                          'report_group': report_group,
-                          'comment': comment,
-                          'email_title': "AppEnlight :: New comment"}
+            email_vars = {
+                "user": user,
+                "commenting_user": request.user,
+                "request": request,
+                "application": application,
+                "report_group": report_group,
+                "comment": comment,
+                "email_title": "AppEnlight :: New comment",
+            }
             UserService.send_email(
                 request,
                 recipients=[user.email],
                 variables=email_vars,
-                template='/email_templates/new_comment_report.jinja2')
-        request.session.flash(_('Your comment was created'))
+                template="/email_templates/new_comment_report.jinja2",
+            )
+        request.session.flash(_("Your comment was created"))
         return comment.get_dict()
     else:
         return form.errors
 
 
-@view_config(route_name='report_groups_property',
-             match_param='key=assigned_users', renderer='json',
-             permission='update_reports', request_method="GET")
+@view_config(
+    route_name="report_groups_property",
+    match_param="key=assigned_users",
+    renderer="json",
+    permission="update_reports",
+    request_method="GET",
+)
 def assigned_users(request):
     """
     Returns list of users a specific report group is assigned for review
     """
     report_group = request.context.report_group
     application = request.context.resource
-    users = set([p.user for p in ResourceService.users_for_perm(application, 'view')])
+    users = set([p.user for p in ResourceService.users_for_perm(application, "view")])
     currently_assigned = [u.user_name for u in report_group.assigned_users]
-    user_status = {'assigned': [], 'unassigned': []}
+    user_status = {"assigned": [], "unassigned": []}
     # handle users
     for user in users:
-        user_dict = {'user_name': user.user_name,
-                     'gravatar_url': UserService.gravatar_url(user),
-                     'name': '%s %s' % (user.first_name, user.last_name,)}
+        user_dict = {
+            "user_name": user.user_name,
+            "gravatar_url": UserService.gravatar_url(user),
+            "name": "%s %s" % (user.first_name, user.last_name),
+        }
         if user.user_name in currently_assigned:
-            user_status['assigned'].append(user_dict)
-        elif user_dict not in user_status['unassigned']:
-            user_status['unassigned'].append(user_dict)
+            user_status["assigned"].append(user_dict)
+        elif user_dict not in user_status["unassigned"]:
+            user_status["unassigned"].append(user_dict)
     return user_status
 
 
-@view_config(route_name='report_groups_property',
-             match_param='key=assigned_users', renderer='json',
-             permission='update_reports', request_method="PATCH")
+@view_config(
+    route_name="report_groups_property",
+    match_param="key=assigned_users",
+    renderer="json",
+    permission="update_reports",
+    request_method="PATCH",
+)
 def assign_users(request):
     """
     Assigns specific report group to user for review - send email notification
     """
     report_group = request.context.report_group
     application = request.context.resource
     currently_assigned = [u.user_name for u in report_group.assigned_users]
     new_assigns = request.unsafe_json_body
 
     # first unassign old users
-    for user_name in new_assigns['unassigned']:
+    for user_name in new_assigns["unassigned"]:
         if user_name in currently_assigned:
             user = UserService.by_user_name(user_name)
             report_group.assigned_users.remove(user)
-            comment = ReportComment(owner_id=request.user.id,
-                                    report_time=report_group.first_timestamp)
-            comment.body = 'Unassigned group from @%s' % user_name
+            comment = ReportComment(
+                owner_id=request.user.id, report_time=report_group.first_timestamp
+            )
+            comment.body = "Unassigned group from @%s" % user_name
             report_group.comments.append(comment)
 
     # assign new users
-    for user_name in new_assigns['assigned']:
+    for user_name in new_assigns["assigned"]:
         if user_name not in currently_assigned:
             user = UserService.by_user_name(user_name)
             if user in report_group.assigned_users:
                 report_group.assigned_users.remove(user)
             DBSession.flush()
             assignment = ReportAssignment(
                 owner_id=user.id,
                 report_time=report_group.first_timestamp,
-                group_id=report_group.id)
+                group_id=report_group.id,
+            )
             DBSession.add(assignment)
 
-            comment = ReportComment(owner_id=request.user.id,
-                                    report_time=report_group.first_timestamp)
-            comment.body = 'Assigned report_group to @%s' % user_name
+            comment = ReportComment(
+                owner_id=request.user.id, report_time=report_group.first_timestamp
+            )
+            comment.body = "Assigned report_group to @%s" % user_name
             report_group.comments.append(comment)
 
-            email_vars = {'user': user,
-                          'request': request,
-                          'application': application,
-                          'report_group': report_group,
-                          'email_title': "AppEnlight :: Assigned Report"}
-            UserService.send_email(request, recipients=[user.email],
-                                   variables=email_vars,
-                                   template='/email_templates/assigned_report.jinja2')
+            email_vars = {
+                "user": user,
+                "request": request,
+                "application": application,
+                "report_group": report_group,
+                "email_title": "AppEnlight :: Assigned Report",
+            }
+            UserService.send_email(
+                request,
+                recipients=[user.email],
+                variables=email_vars,
+                template="/email_templates/assigned_report.jinja2",
+            )
 
     return True
 
 
-@view_config(route_name='report_groups_property', match_param='key=history',
-             renderer='json', permission='view')
+@view_config(
+    route_name="report_groups_property",
+    match_param="key=history",
+    renderer="json",
+    permission="view",
+)
 def history(request):
     """ Separate error graph or similar graph"""
     report_group = request.context.report_group
     query_params = request.GET.mixed()
-    query_params['resource'] = (report_group.resource_id,)
+    query_params["resource"] = (report_group.resource_id,)
 
-    filter_settings = build_filter_settings_from_query_dict(request,
-                                                            query_params)
-    if not filter_settings.get('end_date'):
+    filter_settings = build_filter_settings_from_query_dict(request, query_params)
+    if not filter_settings.get("end_date"):
         end_date = datetime.utcnow().replace(microsecond=0, second=0)
-        filter_settings['end_date'] = end_date
+        filter_settings["end_date"] = end_date
 
-    if not filter_settings.get('start_date'):
+    if not filter_settings.get("start_date"):
         delta = timedelta(days=30)
-        filter_settings['start_date'] = filter_settings['end_date'] - delta
+        filter_settings["start_date"] = filter_settings["end_date"] - delta
 
-    filter_settings['group_id'] = report_group.id
+    filter_settings["group_id"] = report_group.id
 
     result = ReportGroupService.get_report_stats(request, filter_settings)
 
     plot_data = []
     for row in result:
         point = {
-            'x': row['x'],
-            'reports': row['report'] + row['slow_report'] + row['not_found']}
+            "x": row["x"],
+            "reports": row["report"] + row["slow_report"] + row["not_found"],
+        }
         plot_data.append(point)
 
     return plot_data
 
 
-@view_config(route_name='report_groups', renderer='json',
-             permission='update_reports', request_method="PATCH")
+@view_config(
+    route_name="report_groups",
+    renderer="json",
+    permission="update_reports",
+    request_method="PATCH",
+)
 def report_groups_PATCH(request):
     """
     Used to update the report group fixed status
     """
     report_group = request.context.report_group
-    allowed_keys = ['public', 'fixed']
+    allowed_keys = ["public", "fixed"]
     for k, v in request.unsafe_json_body.items():
         if k in allowed_keys:
             setattr(report_group, k, v)
         else:
             return HTTPUnprocessableEntity()
     return report_group.get_dict(request)

backend/src/appenlight/views/tests.py

@@ -1,445 +1,576 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import copy
 import logging
 import datetime
 import time
 import random
 import redis
 import pyramid.renderers
 import requests
 
 from ziggurat_foundations.models.services.user import UserService
 
 import appenlight.celery.tasks
 from pyramid.view import view_config
 from pyramid_mailer.message import Message
 from appenlight_client.timing import time_trace
 from appenlight.models import DBSession, Datastores
 from appenlight.models.user import User
 from appenlight.models.report_group import ReportGroup
 from appenlight.models.event import Event
 from appenlight.models.services.report_group import ReportGroupService
 from appenlight.models.services.event import EventService
 from appenlight.lib.enums import ReportType
 
 log = logging.getLogger(__name__)
 
 GLOBAL_REQ = None
 
 
-@view_config(route_name='test', match_param='action=mail',
-             renderer='string', permission='root_administration')
+@view_config(
+    route_name="test",
+    match_param="action=mail",
+    renderer="string",
+    permission="root_administration",
+)
 def mail(request):
     """
     Test email communication
     """
-    request.environ['HTTP_HOST'] = 'appenlight.com'
-    request.environ['wsgi.url_scheme'] = 'https'
-    renderer_vars = {"title": "You have just registered on AppEnlight",
-                     "username": "test",
-                     "email": "grzegżółka",
-                     'firstname': 'dupa'}
+    request.environ["HTTP_HOST"] = "appenlight.com"
+    request.environ["wsgi.url_scheme"] = "https"
+    renderer_vars = {
+        "title": "You have just registered on AppEnlight",
+        "username": "test",
+        "email": "grzegżółka",
+        "firstname": "dupa",
+    }
     # return vars
-    html = pyramid.renderers.render('/email_templates/registered.jinja2',
-                                    renderer_vars,
-                                    request=request)
-    message = Message(subject="hello world %s" % random.randint(1, 9999),
-                      sender="[e-mail]",
-                      recipients=["[e-mail]"],
-                      html=html)
+    html = pyramid.renderers.render(
+        "/email_templates/registered.jinja2", renderer_vars, request=request
+    )
+    message = Message(
+        subject="hello world %s" % random.randint(1, 9999),
+        sender="[e-mail]",
+        recipients=["[e-mail]"],
+        html=html,
+    )
     request.registry.mailer.send(message)
     return html
     return vars
 
 
-@view_config(route_name='test', match_param='action=alerting',
-             renderer='appenlight:templates/tests/alerting.jinja2',
-             permission='root_administration')
+@view_config(
+    route_name="test",
+    match_param="action=alerting",
+    renderer="appenlight:templates/tests/alerting.jinja2",
+    permission="root_administration",
+)
 def alerting_test(request):
     """
     Allows to test send data on various registered alerting channels
     """
-    applications = UserService.resources_with_perms(request.user, ['view'], resource_types=['application'])
+    applications = UserService.resources_with_perms(
+        request.user, ["view"], resource_types=["application"]
+    )
     # what we can select in total
     all_possible_app_ids = [app.resource_id for app in applications]
     resource = applications[0]
 
     alert_channels = []
     for channel in request.user.alert_channels:
         alert_channels.append(channel.get_dict())
 
-    cname = request.params.get('channel_name')
-    cvalue = request.params.get('channel_value')
-    event_name = request.params.get('event_name')
+    cname = request.params.get("channel_name")
+    cvalue = request.params.get("channel_value")
+    event_name = request.params.get("event_name")
     if cname and cvalue:
         for channel in request.user.alert_channels:
-            if (channel.channel_value == cvalue and
-                        channel.channel_name == cname):
+            if channel.channel_value == cvalue and channel.channel_name == cname:
                 break
-        if event_name in ['error_report_alert', 'slow_report_alert']:
+        if event_name in ["error_report_alert", "slow_report_alert"]:
             # opened
-            new_event = Event(resource_id=resource.resource_id,
-                              event_type=Event.types[event_name],
-                              start_date=datetime.datetime.utcnow(),
-                              status=Event.statuses['active'],
-                              values={'reports': 5,
-                                      'threshold': 10}
-                              )
-            channel.notify_alert(resource=resource,
-                                 event=new_event,
-                                 user=request.user,
-                                 request=request)
+            new_event = Event(
+                resource_id=resource.resource_id,
+                event_type=Event.types[event_name],
+                start_date=datetime.datetime.utcnow(),
+                status=Event.statuses["active"],
+                values={"reports": 5, "threshold": 10},
+            )
+            channel.notify_alert(
+                resource=resource, event=new_event, user=request.user, request=request
+            )
 
             # closed
-            ev_type = Event.types[event_name.replace('open', 'close')]
-            new_event = Event(resource_id=resource.resource_id,
-                              event_type=ev_type,
-                              start_date=datetime.datetime.utcnow(),
-                              status=Event.statuses['closed'],
-                              values={'reports': 5,
-                                      'threshold': 10})
-            channel.notify_alert(resource=resource,
-                                 event=new_event,
-                                 user=request.user,
-                                 request=request)
-        elif event_name == 'notify_reports':
-            report = ReportGroupService.by_app_ids(all_possible_app_ids) \
-                .filter(ReportGroup.report_type == ReportType.error).first()
+            ev_type = Event.types[event_name.replace("open", "close")]
+            new_event = Event(
+                resource_id=resource.resource_id,
+                event_type=ev_type,
+                start_date=datetime.datetime.utcnow(),
+                status=Event.statuses["closed"],
+                values={"reports": 5, "threshold": 10},
+            )
+            channel.notify_alert(
+                resource=resource, event=new_event, user=request.user, request=request
+            )
+        elif event_name == "notify_reports":
+            report = (
+                ReportGroupService.by_app_ids(all_possible_app_ids)
+                .filter(ReportGroup.report_type == ReportType.error)
+                .first()
+            )
             confirmed_reports = [(5, report), (1, report)]
-            channel.notify_reports(resource=resource,
-                                   user=request.user,
-                                   request=request,
-                                   since_when=datetime.datetime.utcnow(),
-                                   reports=confirmed_reports)
+            channel.notify_reports(
+                resource=resource,
+                user=request.user,
+                request=request,
+                since_when=datetime.datetime.utcnow(),
+                reports=confirmed_reports,
+            )
             confirmed_reports = [(5, report)]
-            channel.notify_reports(resource=resource,
-                                   user=request.user,
-                                   request=request,
-                                   since_when=datetime.datetime.utcnow(),
-                                   reports=confirmed_reports)
-        elif event_name == 'notify_uptime':
-            new_event = Event(resource_id=resource.resource_id,
-                              event_type=Event.types['uptime_alert'],
-                              start_date=datetime.datetime.utcnow(),
-                              status=Event.statuses['active'],
-                              values={"status_code": 500,
-                                      "tries": 2,
-                                      "response_time": 0})
-            channel.notify_uptime_alert(resource=resource,
-                                        event=new_event,
-                                        user=request.user,
-                                        request=request)
-        elif event_name == 'chart_alert':
+            channel.notify_reports(
+                resource=resource,
+                user=request.user,
+                request=request,
+                since_when=datetime.datetime.utcnow(),
+                reports=confirmed_reports,
+            )
+        elif event_name == "notify_uptime":
+            new_event = Event(
+                resource_id=resource.resource_id,
+                event_type=Event.types["uptime_alert"],
+                start_date=datetime.datetime.utcnow(),
+                status=Event.statuses["active"],
+                values={"status_code": 500, "tries": 2, "response_time": 0},
+            )
+            channel.notify_uptime_alert(
+                resource=resource, event=new_event, user=request.user, request=request
+            )
+        elif event_name == "chart_alert":
             event = EventService.by_type_and_status(
-                event_types=(Event.types['chart_alert'],),
-                status_types=(Event.statuses['active'],)).first()
-            channel.notify_chart_alert(resource=event.resource,
-                                       event=event,
-                                       user=request.user,
-                                       request=request)
-        elif event_name == 'daily_digest':
-            since_when = datetime.datetime.utcnow() - datetime.timedelta(
-                hours=8)
-            filter_settings = {'resource': [resource.resource_id],
-                               'tags': [{'name': 'type',
-                                         'value': ['error'], 'op': None}],
-                               'type': 'error', 'start_date': since_when}
+                event_types=(Event.types["chart_alert"],),
+                status_types=(Event.statuses["active"],),
+            ).first()
+            channel.notify_chart_alert(
+                resource=event.resource, event=event, user=request.user, request=request
+            )
+        elif event_name == "daily_digest":
+            since_when = datetime.datetime.utcnow() - datetime.timedelta(hours=8)
+            filter_settings = {
+                "resource": [resource.resource_id],
+                "tags": [{"name": "type", "value": ["error"], "op": None}],
+                "type": "error",
+                "start_date": since_when,
+            }
 
             reports = ReportGroupService.get_trending(
-                request, filter_settings=filter_settings, limit=50)
-            channel.send_digest(resource=resource,
-                                user=request.user,
-                                request=request,
-                                since_when=datetime.datetime.utcnow(),
-                                reports=reports)
-
-    return {'alert_channels': alert_channels,
-            'applications': dict([(app.resource_id, app.resource_name)
-                                  for app in applications.all()])}
-
-
-@view_config(route_name='test', match_param='action=error',
-             renderer='string', permission='root_administration')
+                request, filter_settings=filter_settings, limit=50
+            )
+            channel.send_digest(
+                resource=resource,
+                user=request.user,
+                request=request,
+                since_when=datetime.datetime.utcnow(),
+                reports=reports,
+            )
+
+    return {
+        "alert_channels": alert_channels,
+        "applications": dict(
+            [(app.resource_id, app.resource_name) for app in applications.all()]
+        ),
+    }
+
+
+@view_config(
+    route_name="test",
+    match_param="action=error",
+    renderer="string",
+    permission="root_administration",
+)
 def error(request):
     """
     Raises an internal error with some test data for testing purposes
     """
-    request.environ['appenlight.message'] = 'test message'
-    request.environ['appenlight.extra']['dupa'] = 'dupa'
-    request.environ['appenlight.extra']['message'] = 'message'
-    request.environ['appenlight.tags']['action'] = 'test_error'
-    request.environ['appenlight.tags']['count'] = 5
+    request.environ["appenlight.message"] = "test message"
+    request.environ["appenlight.extra"]["dupa"] = "dupa"
+    request.environ["appenlight.extra"]["message"] = "message"
+    request.environ["appenlight.tags"]["action"] = "test_error"
+    request.environ["appenlight.tags"]["count"] = 5
     log.debug(chr(960))
-    log.debug('debug')
+    log.debug("debug")
     log.info(chr(960))
-    log.info('INFO')
-    log.warning('warning')
+    log.info("INFO")
+    log.warning("warning")
 
-    @time_trace(name='error.foobar', min_duration=0.1)
+    @time_trace(name="error.foobar", min_duration=0.1)
     def fooobar():
         time.sleep(0.12)
         return 1
 
     fooobar()
 
     def foobar(somearg):
-        raise Exception('test')
+        raise Exception("test")
 
     client = redis.StrictRedis()
-    client.setex('testval', 10, 'foo')
-    request.environ['appenlight.force_send'] = 1
+    client.setex("testval", 10, "foo")
+    request.environ["appenlight.force_send"] = 1
 
     # stats, result = get_local_storage(local_timing).get_thread_stats()
     # import pprint
     # pprint.pprint(stats)
     # pprint.pprint(result)
     # print 'entries', len(result)
-    request.environ['appenlight.username'] = 'ErgO'
-    raise Exception(chr(960) + '%s' % random.randint(1, 5))
+    request.environ["appenlight.username"] = "ErgO"
+    raise Exception(chr(960) + "%s" % random.randint(1, 5))
     return {}
 
 
-@view_config(route_name='test', match_param='action=task',
-             renderer='string', permission='root_administration')
+@view_config(
+    route_name="test",
+    match_param="action=task",
+    renderer="string",
+    permission="root_administration",
+)
 def test_task(request):
     """
     Test erroneous celery task
     """
     import appenlight.celery.tasks
 
     appenlight.celery.tasks.test_exception_task.delay()
-    return 'task sent'
+    return "task sent"
 
 
-@view_config(route_name='test', match_param='action=task_retry',
-             renderer='string', permission='root_administration')
+@view_config(
+    route_name="test",
+    match_param="action=task_retry",
+    renderer="string",
+    permission="root_administration",
+)
 def test_task_retry(request):
     """
     Test erroneous celery task
     """
     import appenlight.celery.tasks
 
     appenlight.celery.tasks.test_retry_exception_task.delay()
-    return 'task sent'
+    return "task sent"
 
 
-@view_config(route_name='test', match_param='action=celery_emails',
-             renderer='string', permission='root_administration')
+@view_config(
+    route_name="test",
+    match_param="action=celery_emails",
+    renderer="string",
+    permission="root_administration",
+)
 def test_celery_emails(request):
     import appenlight.celery.tasks
+
     appenlight.celery.tasks.alerting.delay()
-    return 'task sent'
+    return "task sent"
 
 
-@view_config(route_name='test', match_param='action=daily_digest',
-             renderer='string', permission='root_administration')
+@view_config(
+    route_name="test",
+    match_param="action=daily_digest",
+    renderer="string",
+    permission="root_administration",
+)
 def test_celery_daily_digest(request):
     import appenlight.celery.tasks
+
     appenlight.celery.tasks.daily_digest.delay()
-    return 'task sent'
+    return "task sent"
 
 
-@view_config(route_name='test', match_param='action=celery_alerting',
-             renderer='string', permission='root_administration')
+@view_config(
+    route_name="test",
+    match_param="action=celery_alerting",
+    renderer="string",
+    permission="root_administration",
+)
 def test_celery_alerting(request):
     import appenlight.celery.tasks
+
     appenlight.celery.tasks.alerting()
-    return 'task sent'
+    return "task sent"
 
 
-@view_config(route_name='test', match_param='action=logging',
-             renderer='string', permission='root_administration')
+@view_config(
+    route_name="test",
+    match_param="action=logging",
+    renderer="string",
+    permission="root_administration",
+)
 def logs(request):
     """
     Test some in-app logging
     """
     log.debug(chr(960))
-    log.debug('debug')
+    log.debug("debug")
     log.info(chr(960))
-    log.info('INFO')
-    log.warning('Matched GET /\xc4\x85\xc5\xbc\xc4\x87'
-                '\xc4\x99\xc4\x99\xc4\x85/summary')
-    log.warning('XXXXMatched GET /\xc4\x85\xc5\xbc\xc4'
-                '\x87\xc4\x99\xc4\x99\xc4\x85/summary')
-    log.warning('DUPA /ążćęęą')
+    log.info("INFO")
+    log.warning(
+        "Matched GET /\xc4\x85\xc5\xbc\xc4\x87" "\xc4\x99\xc4\x99\xc4\x85/summary"
+    )
+    log.warning(
+        "XXXXMatched GET /\xc4\x85\xc5\xbc\xc4" "\x87\xc4\x99\xc4\x99\xc4\x85/summary"
+    )
+    log.warning("DUPA /ążćęęą")
     log.warning("g\u017ceg\u017c\u00f3\u0142ka")
-    log.error('TEST Lorem ipsum2',
-              extra={'user': 'ergo', 'commit': 'sog8ds0g7sdih12hh1j512h5k'})
-    log.fatal('TEST Lorem ipsum3')
-    log.warning('TEST Lorem ipsum',
-                extra={"action": 'purchase',
-                       "price": random.random() * 100,
-                       "quantity": random.randint(1, 99)})
-    log.warning('test_pkey',
-                extra={"action": 'test_pkey', "price": random.random() * 100,
-                       'ae_primary_key': 1,
-                       "quantity": random.randint(1, 99)})
-    log.warning('test_pkey2',
-                extra={"action": 'test_pkey', "price": random.random() * 100,
-                       'ae_primary_key': 'b',
-                       'ae_permanent': 't',
-                       "quantity": random.randint(1, 99)})
-    log.warning('test_pkey3',
-                extra={"action": 'test_pkey', "price": random.random() * 100,
-                       'ae_primary_key': 1,
-                       "quantity": random.randint(1, 99)})
-    log.warning('test_pkey4',
-                extra={"action": 'test_pkey', "price": random.random() * 100,
-                       'ae_primary_key': 'b',
-                       'ae_permanent': True,
-                       "quantity": random.randint(1, 99)})
-    request.environ['appenlight.force_send'] = 1
+    log.error(
+        "TEST Lorem ipsum2",
+        extra={"user": "ergo", "commit": "sog8ds0g7sdih12hh1j512h5k"},
+    )
+    log.fatal("TEST Lorem ipsum3")
+    log.warning(
+        "TEST Lorem ipsum",
+        extra={
+            "action": "purchase",
+            "price": random.random() * 100,
+            "quantity": random.randint(1, 99),
+        },
+    )
+    log.warning(
+        "test_pkey",
+        extra={
+            "action": "test_pkey",
+            "price": random.random() * 100,
+            "ae_primary_key": 1,
+            "quantity": random.randint(1, 99),
+        },
+    )
+    log.warning(
+        "test_pkey2",
+        extra={
+            "action": "test_pkey",
+            "price": random.random() * 100,
+            "ae_primary_key": "b",
+            "ae_permanent": "t",
+            "quantity": random.randint(1, 99),
+        },
+    )
+    log.warning(
+        "test_pkey3",
+        extra={
+            "action": "test_pkey",
+            "price": random.random() * 100,
+            "ae_primary_key": 1,
+            "quantity": random.randint(1, 99),
+        },
+    )
+    log.warning(
+        "test_pkey4",
+        extra={
+            "action": "test_pkey",
+            "price": random.random() * 100,
+            "ae_primary_key": "b",
+            "ae_permanent": True,
+            "quantity": random.randint(1, 99),
+        },
+    )
+    request.environ["appenlight.force_send"] = 1
     return {}
 
 
-@view_config(route_name='test', match_param='action=transaction',
-             renderer='string', permission='root_administration')
+@view_config(
+    route_name="test",
+    match_param="action=transaction",
+    renderer="string",
+    permission="root_administration",
+)
 def transaction_test(request):
     """
     Test transactions
     """
     try:
         result = DBSession.execute("SELECT 1/0")
     except:
         request.tm.abort()
     result = DBSession.execute("SELECT 1")
-    return 'OK'
+    return "OK"
 
 
-@view_config(route_name='test', match_param='action=slow_request',
-             renderer='string', permission='root_administration')
+@view_config(
+    route_name="test",
+    match_param="action=slow_request",
+    renderer="string",
+    permission="root_administration",
+)
 def slow_request(request):
     """
     Test a request that has some slow entries - including nested calls
     """
     users = DBSession.query(User).all()
     import random
+
     some_val = random.random()
     import threading
+
     t_id = id(threading.currentThread())
-    log.warning('slow_log %s %s ' % (some_val, t_id))
-    log.critical('tid %s' % t_id)
+    log.warning("slow_log %s %s " % (some_val, t_id))
+    log.critical("tid %s" % t_id)
 
-    @time_trace(name='baz_func %s' % some_val, min_duration=0.1)
+    @time_trace(name="baz_func %s" % some_val, min_duration=0.1)
     def baz(arg):
         time.sleep(0.32)
         return arg
 
-    requests.get('http://ubuntu.com')
+    requests.get("http://ubuntu.com")
 
-    @time_trace(name='foo_func %s %s' % (some_val, t_id), min_duration=0.1)
+    @time_trace(name="foo_func %s %s" % (some_val, t_id), min_duration=0.1)
     def foo(arg):
         time.sleep(0.52)
-        log.warning('foo_func %s %s' % (some_val, t_id))
-        requests.get('http://ubuntu.com?test=%s' % some_val)
+        log.warning("foo_func %s %s" % (some_val, t_id))
+        requests.get("http://ubuntu.com?test=%s" % some_val)
         return bar(arg)
 
-    @time_trace(name='bar_func %s %s' % (some_val, t_id), min_duration=0.1)
+    @time_trace(name="bar_func %s %s" % (some_val, t_id), min_duration=0.1)
     def bar(arg):
-        log.warning('bar_func %s %s' % (some_val, t_id))
+        log.warning("bar_func %s %s" % (some_val, t_id))
         time.sleep(1.52)
         baz(arg)
         baz(arg)
         return baz(arg)
 
-    foo('a')
+    foo("a")
     return {}
 
 
-@view_config(route_name='test', match_param='action=styling',
-             renderer='appenlight:templates/tests/styling.jinja2',
-             permission='__no_permission_required__')
+@view_config(
+    route_name="test",
+    match_param="action=styling",
+    renderer="appenlight:templates/tests/styling.jinja2",
+    permission="__no_permission_required__",
+)
 def styling(request):
     """
     Some styling test page
     """
     _ = str
-    request.session.flash(_(
-        'Your password got updated. '
-        'Next time log in with your new credentials.'))
-    request.session.flash(_(
-        'Something went wrong when we '
-        'tried to authorize you via external provider'),
-        'warning')
-    request.session.flash(_(
-        'Unfortunately there was a problem '
-        'processing your payment, please try again later.'),
-        'error')
+    request.session.flash(
+        _("Your password got updated. " "Next time log in with your new credentials.")
+    )
+    request.session.flash(
+        _(
+            "Something went wrong when we "
+            "tried to authorize you via external provider"
+        ),
+        "warning",
+    )
+    request.session.flash(
+        _(
+            "Unfortunately there was a problem "
+            "processing your payment, please try again later."
+        ),
+        "error",
+    )
     return {}
 
 
-@view_config(route_name='test', match_param='action=js_error',
-             renderer='appenlight:templates/tests/js_error.jinja2',
-             permission='__no_permission_required__')
+@view_config(
+    route_name="test",
+    match_param="action=js_error",
+    renderer="appenlight:templates/tests/js_error.jinja2",
+    permission="__no_permission_required__",
+)
 def js(request):
     """
     Used for testing javasctipt client for error catching
     """
     return {}
 
 
-@view_config(route_name='test', match_param='action=js_log',
-             renderer='appenlight:templates/tests/js_log.jinja2',
-             permission='__no_permission_required__')
+@view_config(
+    route_name="test",
+    match_param="action=js_log",
+    renderer="appenlight:templates/tests/js_log.jinja2",
+    permission="__no_permission_required__",
+)
 def js_log(request):
     """
     Used for testing javasctipt client for logging
     """
     return {}
 
 
-@view_config(route_name='test', match_param='action=log_requests',
-             renderer='string',
-             permission='__no_permission_required__')
+@view_config(
+    route_name="test",
+    match_param="action=log_requests",
+    renderer="string",
+    permission="__no_permission_required__",
+)
 def log_requests(request):
     """
     Util view for printing json requests
     """
     return {}
 
 
-@view_config(route_name='test', match_param='action=url', renderer='string',
-             permission='__no_permission_required__')
+@view_config(
+    route_name="test",
+    match_param="action=url",
+    renderer="string",
+    permission="__no_permission_required__",
+)
 def log_requests(request):
     """
     I have no fucking clue why I needed that ;-)
     """
-    return request.route_url('reports', _app_url='https://appenlight.com')
+    return request.route_url("reports", _app_url="https://appenlight.com")
 
 
 class TestClass(object):
     """
     Used to test if class-based view name resolution works correctly
     """
 
     def __init__(self, request):
         self.request = request
 
-    @view_config(route_name='test', match_param='action=test_a',
-                 renderer='string', permission='root_administration')
-    @view_config(route_name='test', match_param='action=test_c',
-                 renderer='string', permission='root_administration')
-    @view_config(route_name='test', match_param='action=test_d',
-                 renderer='string', permission='root_administration')
+    @view_config(
+        route_name="test",
+        match_param="action=test_a",
+        renderer="string",
+        permission="root_administration",
+    )
+    @view_config(
+        route_name="test",
+        match_param="action=test_c",
+        renderer="string",
+        permission="root_administration",
+    )
+    @view_config(
+        route_name="test",
+        match_param="action=test_d",
+        renderer="string",
+        permission="root_administration",
+    )
     def test_a(self):
-        return 'ok'
-
-    @view_config(route_name='test', match_param='action=test_b',
-                 renderer='string', permission='root_administration')
+        return "ok"
+
+    @view_config(
+        route_name="test",
+        match_param="action=test_b",
+        renderer="string",
+        permission="root_administration",
+    )
     def test_b(self):
-        return 'ok'
+        return "ok"

backend/src/appenlight/views/user.py

@@ -1,678 +1,841 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import colander
 import datetime
 import json
 import logging
 import uuid
 import pyramid.security as security
 import appenlight.lib.helpers as h
 
 from authomatic.adapters import WebObAdapter
 from pyramid.view import view_config
 from pyramid.httpexceptions import HTTPFound, HTTPUnprocessableEntity
 from pyramid.httpexceptions import HTTPNotFound, HTTPBadRequest
 from pyramid.security import NO_PERMISSION_REQUIRED
-from ziggurat_foundations.models.services.external_identity import \
-    ExternalIdentityService
+from ziggurat_foundations.models.services.external_identity import (
+    ExternalIdentityService,
+)
 from ziggurat_foundations.models.services.user import UserService
 
 from appenlight.lib import generate_random_string
 from appenlight.lib.social import handle_social_data
-from appenlight.lib.utils import channelstream_request, add_cors_headers, \
-    permission_tuple_to_dict
+from appenlight.lib.utils import (
+    channelstream_request,
+    add_cors_headers,
+    permission_tuple_to_dict,
+)
 from appenlight.models import DBSession
 from appenlight.models.alert_channels.email import EmailAlertChannel
 from appenlight.models.alert_channel_action import AlertChannelAction
 from appenlight.models.services.alert_channel import AlertChannelService
-from appenlight.models.services.alert_channel_action import \
-    AlertChannelActionService
+from appenlight.models.services.alert_channel_action import AlertChannelActionService
 from appenlight.models.auth_token import AuthToken
 from appenlight.models.report import REPORT_TYPE_MATRIX
 from appenlight.models.user import User
 from appenlight.models.services.user import UserService
 from appenlight.subscribers import _
 from appenlight.validators import build_rule_schema
 from appenlight import forms
 from webob.multidict import MultiDict
 
 log = logging.getLogger(__name__)
 
 
-@view_config(route_name='users_no_id', renderer='json',
-             request_method="GET", permission='root_administration')
+@view_config(
+    route_name="users_no_id",
+    renderer="json",
+    request_method="GET",
+    permission="root_administration",
+)
 def users_list(request):
     """
     Returns users list
     """
-    props = ['user_name', 'id', 'first_name', 'last_name', 'email',
-             'last_login_date', 'status']
+    props = [
+        "user_name",
+        "id",
+        "first_name",
+        "last_name",
+        "email",
+        "last_login_date",
+        "status",
+    ]
     users = UserService.all()
     users_dicts = []
     for user in users:
         u_dict = user.get_dict(include_keys=props)
-        u_dict['gravatar_url'] = UserService.gravatar_url(user, s=20)
+        u_dict["gravatar_url"] = UserService.gravatar_url(user, s=20)
         users_dicts.append(u_dict)
     return users_dicts
 
 
-@view_config(route_name='users_no_id', renderer='json',
-             request_method="POST", permission='root_administration')
+@view_config(
+    route_name="users_no_id",
+    renderer="json",
+    request_method="POST",
+    permission="root_administration",
+)
 def users_create(request):
     """
     Returns users list
     """
-    form = forms.UserCreateForm(MultiDict(request.safe_json_body or {}),
-                                csrf_context=request)
+    form = forms.UserCreateForm(
+        MultiDict(request.safe_json_body or {}), csrf_context=request
+    )
     if form.validate():
-        log.info('registering user')
+        log.info("registering user")
         # probably not needed in the future since this requires root anyways
         # lets keep this here in case we lower view permission in the future
         # if request.registry.settings['appenlight.disable_registration']:
         #     return HTTPUnprocessableEntity(body={'error': 'Registration is currently disabled.'})
         user = User()
         # insert new user here
         DBSession.add(user)
         form.populate_obj(user)
         UserService.regenerate_security_code(user)
         UserService.set_password(user, user.user_password)
         user.status = 1 if form.status.data else 0
-        request.session.flash(_('User created'))
+        request.session.flash(_("User created"))
         DBSession.flush()
-        return user.get_dict(exclude_keys=['security_code_date', 'notes',
-                                           'security_code', 'user_password'])
+        return user.get_dict(
+            exclude_keys=[
+                "security_code_date",
+                "notes",
+                "security_code",
+                "user_password",
+            ]
+        )
     else:
         return HTTPUnprocessableEntity(body=form.errors_json)
 
 
-@view_config(route_name='users', renderer='json',
-             request_method="GET", permission='root_administration')
-@view_config(route_name='users', renderer='json',
-             request_method="PATCH", permission='root_administration')
+@view_config(
+    route_name="users",
+    renderer="json",
+    request_method="GET",
+    permission="root_administration",
+)
+@view_config(
+    route_name="users",
+    renderer="json",
+    request_method="PATCH",
+    permission="root_administration",
+)
 def users_update(request):
     """
     Updates user object
     """
-    user = UserService.by_id(request.matchdict.get('user_id'))
+    user = UserService.by_id(request.matchdict.get("user_id"))
     if not user:
         return HTTPNotFound()
     post_data = request.safe_json_body or {}
-    if request.method == 'PATCH':
-        form = forms.UserUpdateForm(MultiDict(post_data),
-                                    csrf_context=request)
+    if request.method == "PATCH":
+        form = forms.UserUpdateForm(MultiDict(post_data), csrf_context=request)
         if form.validate():
             form.populate_obj(user, ignore_none=True)
             if form.user_password.data:
                 UserService.set_password(user, user.user_password)
             if form.status.data:
                 user.status = 1
             else:
                 user.status = 0
         else:
             return HTTPUnprocessableEntity(body=form.errors_json)
-    return user.get_dict(exclude_keys=['security_code_date', 'notes',
-                                       'security_code', 'user_password'])
+    return user.get_dict(
+        exclude_keys=["security_code_date", "notes", "security_code", "user_password"]
+    )
 
 
-@view_config(route_name='users_property',
-             match_param='key=resource_permissions',
-             renderer='json', permission='authenticated')
+@view_config(
+    route_name="users_property",
+    match_param="key=resource_permissions",
+    renderer="json",
+    permission="authenticated",
+)
 def users_resource_permissions_list(request):
     """
     Get list of permissions assigned to specific resources
     """
-    user = UserService.by_id(request.matchdict.get('user_id'))
+    user = UserService.by_id(request.matchdict.get("user_id"))
     if not user:
         return HTTPNotFound()
-    return [permission_tuple_to_dict(perm) for perm in
-            UserService.resources_with_possible_perms(user)]
-
-
-@view_config(route_name='users', renderer='json',
-             request_method="DELETE", permission='root_administration')
+    return [
+        permission_tuple_to_dict(perm)
+        for perm in UserService.resources_with_possible_perms(user)
+    ]
+
+
+@view_config(
+    route_name="users",
+    renderer="json",
+    request_method="DELETE",
+    permission="root_administration",
+)
 def users_DELETE(request):
     """
     Removes a user permanently from db - makes a check to see if after the
     operation there will be at least one admin left
     """
-    msg = _('There needs to be at least one administrator in the system')
-    user = UserService.by_id(request.matchdict.get('user_id'))
+    msg = _("There needs to be at least one administrator in the system")
+    user = UserService.by_id(request.matchdict.get("user_id"))
     if user:
-        users = UserService.users_for_perms(['root_administration']).all()
+        users = UserService.users_for_perms(["root_administration"]).all()
         if len(users) < 2 and user.id == users[0].id:
-            request.session.flash(msg, 'warning')
+            request.session.flash(msg, "warning")
         else:
             DBSession.delete(user)
-            request.session.flash(_('User removed'))
+            request.session.flash(_("User removed"))
             return True
     request.response.status = 422
     return False
 
 
-@view_config(route_name='users_self', renderer='json',
-             request_method="GET", permission='authenticated')
-@view_config(route_name='users_self', renderer='json',
-             request_method="PATCH", permission='authenticated')
+@view_config(
+    route_name="users_self",
+    renderer="json",
+    request_method="GET",
+    permission="authenticated",
+)
+@view_config(
+    route_name="users_self",
+    renderer="json",
+    request_method="PATCH",
+    permission="authenticated",
+)
 def users_self(request):
     """
     Updates user personal information
     """
 
-    if request.method == 'PATCH':
+    if request.method == "PATCH":
         form = forms.gen_user_profile_form()(
-            MultiDict(request.unsafe_json_body),
-            csrf_context=request)
+            MultiDict(request.unsafe_json_body), csrf_context=request
+        )
         if form.validate():
             form.populate_obj(request.user)
-            request.session.flash(_('Your profile got updated.'))
+            request.session.flash(_("Your profile got updated."))
         else:
             return HTTPUnprocessableEntity(body=form.errors_json)
     return request.user.get_dict(
-        exclude_keys=['security_code_date', 'notes', 'security_code',
-                      'user_password'],
-        extended_info=True)
-
-
-@view_config(route_name='users_self_property',
-             match_param='key=external_identities', renderer='json',
-             request_method='GET', permission='authenticated')
+        exclude_keys=["security_code_date", "notes", "security_code", "user_password"],
+        extended_info=True,
+    )
+
+
+@view_config(
+    route_name="users_self_property",
+    match_param="key=external_identities",
+    renderer="json",
+    request_method="GET",
+    permission="authenticated",
+)
 def users_external_identies(request):
     user = request.user
-    identities = [{'provider': ident.provider_name,
-                   'id': ident.external_user_name} for ident
-                  in user.external_identities.all()]
+    identities = [
+        {"provider": ident.provider_name, "id": ident.external_user_name}
+        for ident in user.external_identities.all()
+    ]
     return identities
 
 
-@view_config(route_name='users_self_property',
-             match_param='key=external_identities', renderer='json',
-             request_method='DELETE', permission='authenticated')
+@view_config(
+    route_name="users_self_property",
+    match_param="key=external_identities",
+    renderer="json",
+    request_method="DELETE",
+    permission="authenticated",
+)
 def users_external_identies_DELETE(request):
     """
     Unbinds external identities(google,twitter etc.) from user account
     """
     user = request.user
     for identity in user.external_identities.all():
-        log.info('found identity %s' % identity)
-        if (identity.provider_name == request.params.get('provider') and
-                    identity.external_user_name == request.params.get('id')):
-            log.info('remove identity %s' % identity)
+        log.info("found identity %s" % identity)
+        if identity.provider_name == request.params.get(
+            "provider"
+        ) and identity.external_user_name == request.params.get("id"):
+            log.info("remove identity %s" % identity)
             DBSession.delete(identity)
             return True
     return False
 
 
-@view_config(route_name='users_self_property',
-             match_param='key=password', renderer='json',
-             request_method='PATCH', permission='authenticated')
+@view_config(
+    route_name="users_self_property",
+    match_param="key=password",
+    renderer="json",
+    request_method="PATCH",
+    permission="authenticated",
+)
 def users_password(request):
     """
     Sets new password for user account
     """
     user = request.user
-    form = forms.ChangePasswordForm(MultiDict(request.unsafe_json_body),
-                                    csrf_context=request)
+    form = forms.ChangePasswordForm(
+        MultiDict(request.unsafe_json_body), csrf_context=request
+    )
     form.old_password.user = user
     if form.validate():
         UserService.regenerate_security_code(user)
         UserService.set_password(user, form.new_password.data)
-        msg = 'Your password got updated. ' \
-              'Next time log in with your new credentials.'
+        msg = (
+            "Your password got updated. " "Next time log in with your new credentials."
+        )
         request.session.flash(_(msg))
         return True
     else:
         return HTTPUnprocessableEntity(body=form.errors_json)
     return False
 
 
-@view_config(route_name='users_self_property', match_param='key=websocket',
-             renderer='json', permission='authenticated')
+@view_config(
+    route_name="users_self_property",
+    match_param="key=websocket",
+    renderer="json",
+    permission="authenticated",
+)
 def users_websocket(request):
     """
     Handle authorization of users trying to connect
     """
     # handle preflight request
     user = request.user
-    if request.method == 'OPTIONS':
-        res = request.response.body('OK')
+    if request.method == "OPTIONS":
+        res = request.response.body("OK")
         add_cors_headers(res)
         return res
-    applications = UserService.resources_with_perms(user, ['view'], resource_types=['application'])
-    channels = ['app_%s' % app.resource_id for app in applications]
-    payload = {"username": user.user_name,
-               "conn_id": str(uuid.uuid4()),
-               "channels": channels
-               }
+    applications = UserService.resources_with_perms(
+        user, ["view"], resource_types=["application"]
+    )
+    channels = ["app_%s" % app.resource_id for app in applications]
+    payload = {
+        "username": user.user_name,
+        "conn_id": str(uuid.uuid4()),
+        "channels": channels,
+    }
     settings = request.registry.settings
     response = channelstream_request(
-        settings['cometd.secret'], '/connect', payload,
-        servers=[request.registry.settings['cometd_servers']],
-        throw_exceptions=True)
+        settings["cometd.secret"],
+        "/connect",
+        payload,
+        servers=[request.registry.settings["cometd_servers"]],
+        throw_exceptions=True,
+    )
     return payload
 
 
-@view_config(route_name='users_self_property', request_method="GET",
-             match_param='key=alert_channels', renderer='json',
-             permission='authenticated')
+@view_config(
+    route_name="users_self_property",
+    request_method="GET",
+    match_param="key=alert_channels",
+    renderer="json",
+    permission="authenticated",
+)
 def alert_channels(request):
     """
     Lists all available alert channels
     """
     user = request.user
     return [c.get_dict(extended_info=True) for c in user.alert_channels]
 
 
-@view_config(route_name='users_self_property', match_param='key=alert_actions',
-             request_method="GET", renderer='json', permission='authenticated')
+@view_config(
+    route_name="users_self_property",
+    match_param="key=alert_actions",
+    request_method="GET",
+    renderer="json",
+    permission="authenticated",
+)
 def alert_actions(request):
     """
     Lists all available alert channels
     """
     user = request.user
     return [r.get_dict(extended_info=True) for r in user.alert_actions]
 
 
-@view_config(route_name='users_self_property', renderer='json',
-             match_param='key=alert_channels_rules', request_method='POST',
-             permission='authenticated')
+@view_config(
+    route_name="users_self_property",
+    renderer="json",
+    match_param="key=alert_channels_rules",
+    request_method="POST",
+    permission="authenticated",
+)
 def alert_channels_rule_POST(request):
     """
     Creates new notification rule for specific alert channel
     """
     user = request.user
-    alert_action = AlertChannelAction(owner_id=request.user.id,
-                                      type='report')
+    alert_action = AlertChannelAction(owner_id=request.user.id, type="report")
     DBSession.add(alert_action)
     DBSession.flush()
     return alert_action.get_dict()
 
 
-@view_config(route_name='users_self_property', permission='authenticated',
-             match_param='key=alert_channels_rules',
-             renderer='json', request_method='DELETE')
+@view_config(
+    route_name="users_self_property",
+    permission="authenticated",
+    match_param="key=alert_channels_rules",
+    renderer="json",
+    request_method="DELETE",
+)
 def alert_channels_rule_DELETE(request):
     """
     Removes specific alert channel rule
     """
     user = request.user
     rule_action = AlertChannelActionService.by_owner_id_and_pkey(
-        user.id,
-        request.GET.get('pkey'))
+        user.id, request.GET.get("pkey")
+    )
     if rule_action:
         DBSession.delete(rule_action)
         return True
     return HTTPNotFound()
 
 
-@view_config(route_name='users_self_property', permission='authenticated',
-             match_param='key=alert_channels_rules',
-             renderer='json', request_method='PATCH')
+@view_config(
+    route_name="users_self_property",
+    permission="authenticated",
+    match_param="key=alert_channels_rules",
+    renderer="json",
+    request_method="PATCH",
+)
 def alert_channels_rule_PATCH(request):
     """
     Removes specific alert channel rule
     """
     user = request.user
     json_body = request.unsafe_json_body
 
-    schema = build_rule_schema(json_body['rule'], REPORT_TYPE_MATRIX)
+    schema = build_rule_schema(json_body["rule"], REPORT_TYPE_MATRIX)
     try:
-        schema.deserialize(json_body['rule'])
+        schema.deserialize(json_body["rule"])
     except colander.Invalid as exc:
         return HTTPUnprocessableEntity(body=json.dumps(exc.asdict()))
 
     rule_action = AlertChannelActionService.by_owner_id_and_pkey(
-        user.id,
-        request.GET.get('pkey'))
+        user.id, request.GET.get("pkey")
+    )
 
     if rule_action:
-        rule_action.rule = json_body['rule']
-        rule_action.resource_id = json_body['resource_id']
-        rule_action.action = json_body['action']
+        rule_action.rule = json_body["rule"]
+        rule_action.resource_id = json_body["resource_id"]
+        rule_action.action = json_body["action"]
         return rule_action.get_dict()
     return HTTPNotFound()
 
 
-@view_config(route_name='users_self_property', permission='authenticated',
-             match_param='key=alert_channels',
-             renderer='json', request_method='PATCH')
+@view_config(
+    route_name="users_self_property",
+    permission="authenticated",
+    match_param="key=alert_channels",
+    renderer="json",
+    request_method="PATCH",
+)
 def alert_channels_PATCH(request):
     user = request.user
-    channel_name = request.GET.get('channel_name')
-    channel_value = request.GET.get('channel_value')
+    channel_name = request.GET.get("channel_name")
+    channel_value = request.GET.get("channel_value")
     # iterate over channels
     channel = None
     for channel in user.alert_channels:
-        if (channel.channel_name == channel_name and
-                    channel.channel_value == channel_value):
+        if (
+            channel.channel_name == channel_name
+            and channel.channel_value == channel_value
+        ):
             break
     if not channel:
         return HTTPNotFound()
 
-    allowed_keys = ['daily_digest', 'send_alerts']
+    allowed_keys = ["daily_digest", "send_alerts"]
     for k, v in request.unsafe_json_body.items():
         if k in allowed_keys:
             setattr(channel, k, v)
         else:
             return HTTPBadRequest()
     return channel.get_dict()
 
 
-@view_config(route_name='users_self_property', permission='authenticated',
-             match_param='key=alert_channels',
-             request_method="POST", renderer='json')
+@view_config(
+    route_name="users_self_property",
+    permission="authenticated",
+    match_param="key=alert_channels",
+    request_method="POST",
+    renderer="json",
+)
 def alert_channels_POST(request):
     """
     Creates a new email alert channel for user, sends a validation email
     """
     user = request.user
-    form = forms.EmailChannelCreateForm(MultiDict(request.unsafe_json_body),
-                                        csrf_context=request)
+    form = forms.EmailChannelCreateForm(
+        MultiDict(request.unsafe_json_body), csrf_context=request
+    )
     if not form.validate():
         return HTTPUnprocessableEntity(body=form.errors_json)
 
     email = form.email.data.strip()
     channel = EmailAlertChannel()
-    channel.channel_name = 'email'
+    channel.channel_name = "email"
     channel.channel_value = email
     security_code = generate_random_string(10)
-    channel.channel_json_conf = {'security_code': security_code}
+    channel.channel_json_conf = {"security_code": security_code}
     user.alert_channels.append(channel)
 
-    email_vars = {'user': user,
-                  'email': email,
-                  'request': request,
-                  'security_code': security_code,
-                  'email_title': "AppEnlight :: "
-                                 "Please authorize your email"}
-
-    UserService.send_email(request, recipients=[email],
-                           variables=email_vars,
-                           template='/email_templates/authorize_email.jinja2')
-    request.session.flash(_('Your alert channel was '
-                            'added to the system.'))
+    email_vars = {
+        "user": user,
+        "email": email,
+        "request": request,
+        "security_code": security_code,
+        "email_title": "AppEnlight :: " "Please authorize your email",
+    }
+
+    UserService.send_email(
+        request,
+        recipients=[email],
+        variables=email_vars,
+        template="/email_templates/authorize_email.jinja2",
+    )
+    request.session.flash(_("Your alert channel was " "added to the system."))
     request.session.flash(
-        _('You need to authorize your email channel, a message was '
-          'sent containing necessary information.'),
-        'warning')
+        _(
+            "You need to authorize your email channel, a message was "
+            "sent containing necessary information."
+        ),
+        "warning",
+    )
     DBSession.flush()
     channel.get_dict()
 
 
-@view_config(route_name='section_view',
-             match_param=['section=user_section',
-                          'view=alert_channels_authorize'],
-             renderer='string', permission='authenticated')
+@view_config(
+    route_name="section_view",
+    match_param=["section=user_section", "view=alert_channels_authorize"],
+    renderer="string",
+    permission="authenticated",
+)
 def alert_channels_authorize(request):
     """
     Performs alert channel authorization based on auth code sent in email
     """
     user = request.user
     for channel in user.alert_channels:
-        security_code = request.params.get('security_code', '')
-        if channel.channel_json_conf['security_code'] == security_code:
+        security_code = request.params.get("security_code", "")
+        if channel.channel_json_conf["security_code"] == security_code:
             channel.channel_validated = True
-            request.session.flash(_('Your email was authorized.'))
-    return HTTPFound(location=request.route_url('/'))
+            request.session.flash(_("Your email was authorized."))
+    return HTTPFound(location=request.route_url("/"))
 
 
-@view_config(route_name='users_self_property', request_method="DELETE",
-             match_param='key=alert_channels', renderer='json',
-             permission='authenticated')
+@view_config(
+    route_name="users_self_property",
+    request_method="DELETE",
+    match_param="key=alert_channels",
+    renderer="json",
+    permission="authenticated",
+)
 def alert_channel_DELETE(request):
     """
     Removes alert channel from users channel
     """
     user = request.user
     channel = None
     for chan in user.alert_channels:
-        if (chan.channel_name == request.params.get('channel_name') and
-                    chan.channel_value == request.params.get('channel_value')):
+        if chan.channel_name == request.params.get(
+            "channel_name"
+        ) and chan.channel_value == request.params.get("channel_value"):
             channel = chan
             break
     if channel:
         user.alert_channels.remove(channel)
-        request.session.flash(_('Your channel was removed.'))
+        request.session.flash(_("Your channel was removed."))
         return True
     return False
 
 
-@view_config(route_name='users_self_property', permission='authenticated',
-             match_param='key=alert_channels_actions_binds',
-             renderer='json', request_method="POST")
+@view_config(
+    route_name="users_self_property",
+    permission="authenticated",
+    match_param="key=alert_channels_actions_binds",
+    renderer="json",
+    request_method="POST",
+)
 def alert_channels_actions_binds_POST(request):
     """
     Adds alert action to users channels
     """
     user = request.user
     json_body = request.unsafe_json_body
     channel = AlertChannelService.by_owner_id_and_pkey(
-        user.id,
-        json_body.get('channel_pkey'))
+        user.id, json_body.get("channel_pkey")
+    )
 
     rule_action = AlertChannelActionService.by_owner_id_and_pkey(
-        user.id,
-        json_body.get('action_pkey'))
+        user.id, json_body.get("action_pkey")
+    )
 
     if channel and rule_action:
         if channel.pkey not in [c.pkey for c in rule_action.channels]:
             rule_action.channels.append(channel)
             return rule_action.get_dict(extended_info=True)
     return HTTPUnprocessableEntity()
 
 
-@view_config(route_name='users_self_property', request_method="DELETE",
-             match_param='key=alert_channels_actions_binds',
-             renderer='json', permission='authenticated')
+@view_config(
+    route_name="users_self_property",
+    request_method="DELETE",
+    match_param="key=alert_channels_actions_binds",
+    renderer="json",
+    permission="authenticated",
+)
 def alert_channels_actions_binds_DELETE(request):
     """
     Removes alert action from users channels
     """
     user = request.user
     channel = AlertChannelService.by_owner_id_and_pkey(
-        user.id,
-        request.GET.get('channel_pkey'))
+        user.id, request.GET.get("channel_pkey")
+    )
 
     rule_action = AlertChannelActionService.by_owner_id_and_pkey(
-        user.id,
-        request.GET.get('action_pkey'))
+        user.id, request.GET.get("action_pkey")
+    )
 
     if channel and rule_action:
         if channel.pkey in [c.pkey for c in rule_action.channels]:
             rule_action.channels.remove(channel)
             return rule_action.get_dict(extended_info=True)
     return HTTPUnprocessableEntity()
 
 
-@view_config(route_name='social_auth_abort',
-             renderer='string', permission=NO_PERMISSION_REQUIRED)
+@view_config(
+    route_name="social_auth_abort", renderer="string", permission=NO_PERMISSION_REQUIRED
+)
 def oauth_abort(request):
     """
     Handles problems with authorization via velruse
     """
 
 
-@view_config(route_name='social_auth', permission=NO_PERMISSION_REQUIRED)
+@view_config(route_name="social_auth", permission=NO_PERMISSION_REQUIRED)
 def social_auth(request):
     # Get the internal provider name URL variable.
-    provider_name = request.matchdict.get('provider')
+    provider_name = request.matchdict.get("provider")
 
     # Start the login procedure.
     adapter = WebObAdapter(request, request.response)
     result = request.authomatic.login(adapter, provider_name)
     if result:
         if result.error:
             return handle_auth_error(request, result)
         elif result.user:
             return handle_auth_success(request, result)
     return request.response
 
 
 def handle_auth_error(request, result):
     # Login procedure finished with an error.
-    request.session.pop('zigg.social_auth', None)
-    request.session.flash(_('Something went wrong when we tried to '
-                            'authorize you via external provider. '
-                            'Please try again.'), 'warning')
+    request.session.pop("zigg.social_auth", None)
+    request.session.flash(
+        _(
+            "Something went wrong when we tried to "
+            "authorize you via external provider. "
+            "Please try again."
+        ),
+        "warning",
+    )
 
-    return HTTPFound(location=request.route_url('/'))
+    return HTTPFound(location=request.route_url("/"))
 
 
 def handle_auth_success(request, result):
     # Hooray, we have the user!
     # OAuth 2.0 and OAuth 1.0a provide only limited user data on login,
     # We need to update the user to get more info.
     if result.user:
         result.user.update()
 
     social_data = {
-        'user': {'data': result.user.data},
-        'credentials': result.user.credentials
+        "user": {"data": result.user.data},
+        "credentials": result.user.credentials,
     }
     # normalize data
-    social_data['user']['id'] = result.user.id
-    user_name = result.user.username or ''
+    social_data["user"]["id"] = result.user.id
+    user_name = result.user.username or ""
     # use email name as username for google
-    if (social_data['credentials'].provider_name == 'google' and
-            result.user.email):
+    if social_data["credentials"].provider_name == "google" and result.user.email:
         user_name = result.user.email
-    social_data['user']['user_name'] = user_name
-    social_data['user']['email'] = result.user.email or ''
+    social_data["user"]["user_name"] = user_name
+    social_data["user"]["email"] = result.user.email or ""
 
-    request.session['zigg.social_auth'] = social_data
+    request.session["zigg.social_auth"] = social_data
     # user is logged so bind his external identity with account
     if request.user:
         handle_social_data(request, request.user, social_data)
-        request.session.pop('zigg.social_auth', None)
-        return HTTPFound(location=request.route_url('/'))
+        request.session.pop("zigg.social_auth", None)
+        return HTTPFound(location=request.route_url("/"))
     else:
         user = ExternalIdentityService.user_by_external_id_and_provider(
-            social_data['user']['id'],
-            social_data['credentials'].provider_name
+            social_data["user"]["id"], social_data["credentials"].provider_name
         )
         # fix legacy accounts with wrong google ID
-        if not user and social_data['credentials'].provider_name == 'google':
+        if not user and social_data["credentials"].provider_name == "google":
             user = ExternalIdentityService.user_by_external_id_and_provider(
-                social_data['user']['email'],
-                social_data['credentials'].provider_name)
+                social_data["user"]["email"], social_data["credentials"].provider_name
+            )
 
         # user tokens are already found in our db
         if user:
             handle_social_data(request, user, social_data)
             headers = security.remember(request, user.id)
-            request.session.pop('zigg.social_auth', None)
-            return HTTPFound(location=request.route_url('/'), headers=headers)
+            request.session.pop("zigg.social_auth", None)
+            return HTTPFound(location=request.route_url("/"), headers=headers)
         else:
-            msg = 'You need to finish registration ' \
-                  'process to bind your external identity to your account ' \
-                  'or sign in to existing account'
+            msg = (
+                "You need to finish registration "
+                "process to bind your external identity to your account "
+                "or sign in to existing account"
+            )
             request.session.flash(msg)
-            return HTTPFound(location=request.route_url('register'))
+            return HTTPFound(location=request.route_url("register"))
 
 
-@view_config(route_name='section_view', permission='authenticated',
-             match_param=['section=users_section', 'view=search_users'],
-             renderer='json')
+@view_config(
+    route_name="section_view",
+    permission="authenticated",
+    match_param=["section=users_section", "view=search_users"],
+    renderer="json",
+)
 def search_users(request):
     """
     Returns a list of users for autocomplete
     """
     user = request.user
     items_returned = []
-    like_condition = request.params.get('user_name', '') + '%'
+    like_condition = request.params.get("user_name", "") + "%"
     # first append used if email is passed
-    found_user = UserService.by_email(request.params.get('user_name', ''))
+    found_user = UserService.by_email(request.params.get("user_name", ""))
     if found_user:
-        name = '{} {}'.format(found_user.first_name, found_user.last_name)
-        items_returned.append({'user': found_user.user_name, 'name': name})
+        name = "{} {}".format(found_user.first_name, found_user.last_name)
+        items_returned.append({"user": found_user.user_name, "name": name})
     for found_user in UserService.user_names_like(like_condition).limit(20):
-        name = '{} {}'.format(found_user.first_name, found_user.last_name)
-        items_returned.append({'user': found_user.user_name, 'name': name})
+        name = "{} {}".format(found_user.first_name, found_user.last_name)
+        items_returned.append({"user": found_user.user_name, "name": name})
     return items_returned
 
 
-@view_config(route_name='users_self_property', match_param='key=auth_tokens',
-             request_method="GET", renderer='json', permission='authenticated')
-@view_config(route_name='users_property', match_param='key=auth_tokens',
-             request_method="GET", renderer='json', permission='authenticated')
+@view_config(
+    route_name="users_self_property",
+    match_param="key=auth_tokens",
+    request_method="GET",
+    renderer="json",
+    permission="authenticated",
+)
+@view_config(
+    route_name="users_property",
+    match_param="key=auth_tokens",
+    request_method="GET",
+    renderer="json",
+    permission="authenticated",
+)
 def auth_tokens_list(request):
     """
     Lists all available alert channels
     """
-    if request.matched_route.name == 'users_self_property':
+    if request.matched_route.name == "users_self_property":
         user = request.user
     else:
-        user = UserService.by_id(request.matchdict.get('user_id'))
+        user = UserService.by_id(request.matchdict.get("user_id"))
         if not user:
             return HTTPNotFound()
     return [c.get_dict() for c in user.auth_tokens]
 
 
-@view_config(route_name='users_self_property', match_param='key=auth_tokens',
-             request_method="POST", renderer='json',
-             permission='authenticated')
-@view_config(route_name='users_property', match_param='key=auth_tokens',
-             request_method="POST", renderer='json',
-             permission='authenticated')
+@view_config(
+    route_name="users_self_property",
+    match_param="key=auth_tokens",
+    request_method="POST",
+    renderer="json",
+    permission="authenticated",
+)
+@view_config(
+    route_name="users_property",
+    match_param="key=auth_tokens",
+    request_method="POST",
+    renderer="json",
+    permission="authenticated",
+)
 def auth_tokens_POST(request):
     """
     Lists all available alert channels
     """
-    if request.matched_route.name == 'users_self_property':
+    if request.matched_route.name == "users_self_property":
         user = request.user
     else:
-        user = UserService.by_id(request.matchdict.get('user_id'))
+        user = UserService.by_id(request.matchdict.get("user_id"))
         if not user:
             return HTTPNotFound()
 
     req_data = request.safe_json_body or {}
-    if not req_data.get('expires'):
-        req_data.pop('expires', None)
+    if not req_data.get("expires"):
+        req_data.pop("expires", None)
     form = forms.AuthTokenCreateForm(MultiDict(req_data), csrf_context=request)
     if not form.validate():
         return HTTPUnprocessableEntity(body=form.errors_json)
     token = AuthToken()
     form.populate_obj(token)
     if token.expires:
-        interval = h.time_deltas.get(token.expires)['delta']
+        interval = h.time_deltas.get(token.expires)["delta"]
         token.expires = datetime.datetime.utcnow() + interval
     user.auth_tokens.append(token)
     DBSession.flush()
     return token.get_dict()
 
 
-@view_config(route_name='users_self_property', match_param='key=auth_tokens',
-             request_method="DELETE", renderer='json',
-             permission='authenticated')
-@view_config(route_name='users_property', match_param='key=auth_tokens',
-             request_method="DELETE", renderer='json',
-             permission='authenticated')
+@view_config(
+    route_name="users_self_property",
+    match_param="key=auth_tokens",
+    request_method="DELETE",
+    renderer="json",
+    permission="authenticated",
+)
+@view_config(
+    route_name="users_property",
+    match_param="key=auth_tokens",
+    request_method="DELETE",
+    renderer="json",
+    permission="authenticated",
+)
 def auth_tokens_DELETE(request):
     """
     Lists all available alert channels
     """
-    if request.matched_route.name == 'users_self_property':
+    if request.matched_route.name == "users_self_property":
         user = request.user
     else:
-        user = UserService.by_id(request.matchdict.get('user_id'))
+        user = UserService.by_id(request.matchdict.get("user_id"))
         if not user:
             return HTTPNotFound()
 
     for token in user.auth_tokens:
-        if token.token == request.params.get('token'):
+        if token.token == request.params.get("token"):
             user.auth_tokens.remove(token)
             return True
     return False