metrics: rename RequestMetric module to Metric

backend/README.md

@@ -1,47 +1,47 b''
 # appenlight README
 
 
 To run the app you need to have meet prerequsites:
 
 - running elasticsearch (2.3+ tested)
 - running postgresql (9.5+ required)
 - running redis
 
 # Setup basics
 
 Set up the basic application database schema:
 
     appenlight_initialize_db config.ini
 
 Set up basic elasticsearch schema:
 
     appenlight-reindex-elasticsearch -c config.ini -t all
 
 Installed the appenlight uptime plugin
 
 # Running
 
 To run the application itself:
 
     pserve --reload development.ini
 
 To run celery queue processing:
 
     celery worker -A appenlight.celery -Q "reports,logs,metrics,default" --ini=development.ini
 
 To run celery beats scheduling:
 
     celery beat -A appenlight.celery --ini=development.ini
 
-You should also run the channelstream websocket server for real-time notifications
+You should also run the `channelstream websocket server for real-time notifications
 
     channelstream -i filename.ini
 
 # Testing
 
 To run test suite:
 
     py.test appenlight/tests/tests.py --cov appenlight (this looks for testing.ini in repo root)
 
 WARNING!!!
 Some tests will insert data into elasticsearch or redis based on testing.ini

backend/src/appenlight/celery/tasks.py

@@ -1,634 +1,634 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # AppEnlight Enterprise Edition, including its added features, Support
 # services, and proprietary license terms, please see
 # https://rhodecode.com/licenses/
 
 import bisect
 import collections
 import math
 from datetime import datetime, timedelta
 
 import sqlalchemy as sa
 import pyelasticsearch
 
 from celery.utils.log import get_task_logger
 from zope.sqlalchemy import mark_changed
 from pyramid.threadlocal import get_current_request, get_current_registry
 from appenlight.celery import celery
 from appenlight.models.report_group import ReportGroup
 from appenlight.models import DBSession, Datastores
 from appenlight.models.report import Report
 from appenlight.models.log import Log
-from appenlight.models.request_metric import Metric
+from appenlight.models.metric import Metric
 from appenlight.models.event import Event
 
 from appenlight.models.services.application import ApplicationService
 from appenlight.models.services.event import EventService
 from appenlight.models.services.log import LogService
 from appenlight.models.services.report import ReportService
 from appenlight.models.services.report_group import ReportGroupService
 from appenlight.models.services.user import UserService
 from appenlight.models.tag import Tag
 from appenlight.lib import print_traceback
 from appenlight.lib.utils import parse_proto, in_batches
 from appenlight.lib.ext_json import json
 from appenlight.lib.redis_keys import REDIS_KEYS
 from appenlight.lib.enums import ReportType
 
 log = get_task_logger(__name__)
 
 sample_boundries = list(range(100, 1000, 100)) + \
                    list(range(1000, 10000, 1000)) + \
                    list(range(10000, 100000, 5000))
 
 
 def pick_sample(total_occurences, report_type=None):
     every = 1.0
     position = bisect.bisect_left(sample_boundries, total_occurences)
     if position > 0:
         if report_type == ReportType.not_found:
             divide = 10.0
         else:
             divide = 100.0
         every = sample_boundries[position - 1] / divide
     return total_occurences % every == 0
 
 
 @celery.task(queue="default", default_retry_delay=1, max_retries=2)
 def test_exception_task():
     log.error('test celery log', extra={'location': 'celery'})
     log.warning('test celery log', extra={'location': 'celery'})
     raise Exception('Celery exception test')
 
 
 @celery.task(queue="default", default_retry_delay=1, max_retries=2)
 def test_retry_exception_task():
     try:
         import time
 
         time.sleep(1.3)
         log.error('test retry celery log', extra={'location': 'celery'})
         log.warning('test retry celery log', extra={'location': 'celery'})
         raise Exception('Celery exception test')
     except Exception as exc:
         test_retry_exception_task.retry(exc=exc)
 
 
 @celery.task(queue="reports", default_retry_delay=600, max_retries=144)
 def add_reports(resource_id, params, dataset, environ=None, **kwargs):
     proto_version = parse_proto(params.get('protocol_version', ''))
     current_time = datetime.utcnow().replace(second=0, microsecond=0)
     try:
         # we will store solr docs here for single insert
         es_report_docs = {}
         es_report_group_docs = {}
         resource = ApplicationService.by_id(resource_id)
 
         tags = []
         es_slow_calls_docs = {}
         es_reports_stats_rows = {}
         for report_data in dataset:
             # build report details for later
             added_details = 0
             report = Report()
             report.set_data(report_data, resource, proto_version)
             report._skip_ft_index = True
 
             report_group = ReportGroupService.by_hash_and_resource(
                 report.resource_id,
                 report.grouping_hash
             )
             occurences = report_data.get('occurences', 1)
             if not report_group:
                 # total reports will be +1 moment later
                 report_group = ReportGroup(grouping_hash=report.grouping_hash,
                                            occurences=0, total_reports=0,
                                            last_report=0,
                                            priority=report.priority,
                                            error=report.error,
                                            first_timestamp=report.start_time)
                 report_group._skip_ft_index = True
                 report_group.report_type = report.report_type
             report.report_group_time = report_group.first_timestamp
             add_sample = pick_sample(report_group.occurences,
                                      report_type=report_group.report_type)
             if add_sample:
                 resource.report_groups.append(report_group)
                 report_group.reports.append(report)
                 added_details += 1
                 DBSession.flush()
                 if report.partition_id not in es_report_docs:
                     es_report_docs[report.partition_id] = []
                 es_report_docs[report.partition_id].append(report.es_doc())
                 tags.extend(list(report.tags.items()))
                 slow_calls = report.add_slow_calls(report_data, report_group)
                 DBSession.flush()
                 for s_call in slow_calls:
                     if s_call.partition_id not in es_slow_calls_docs:
                         es_slow_calls_docs[s_call.partition_id] = []
                     es_slow_calls_docs[s_call.partition_id].append(
                         s_call.es_doc())
                     # try generating new stat rows if needed
             else:
                 # required for postprocessing to not fail later
                 report.report_group = report_group
 
             stat_row = ReportService.generate_stat_rows(
                 report, resource, report_group)
             if stat_row.partition_id not in es_reports_stats_rows:
                 es_reports_stats_rows[stat_row.partition_id] = []
             es_reports_stats_rows[stat_row.partition_id].append(
                 stat_row.es_doc())
 
             # see if we should mark 10th occurence of report
             last_occurences_10 = int(math.floor(report_group.occurences / 10))
             curr_occurences_10 = int(math.floor(
                 (report_group.occurences + report.occurences) / 10))
             last_occurences_100 = int(
                 math.floor(report_group.occurences / 100))
             curr_occurences_100 = int(math.floor(
                 (report_group.occurences + report.occurences) / 100))
             notify_occurences_10 = last_occurences_10 != curr_occurences_10
             notify_occurences_100 = last_occurences_100 != curr_occurences_100
             report_group.occurences = ReportGroup.occurences + occurences
             report_group.last_timestamp = report.start_time
             report_group.summed_duration = ReportGroup.summed_duration + report.duration
             summed_duration = ReportGroup.summed_duration + report.duration
             summed_occurences = ReportGroup.occurences + occurences
             report_group.average_duration = summed_duration / summed_occurences
             report_group.run_postprocessing(report)
             if added_details:
                 report_group.total_reports = ReportGroup.total_reports + 1
                 report_group.last_report = report.id
             report_group.set_notification_info(notify_10=notify_occurences_10,
                                                notify_100=notify_occurences_100)
             DBSession.flush()
             report_group.get_report().notify_channel(report_group)
             if report_group.partition_id not in es_report_group_docs:
                 es_report_group_docs[report_group.partition_id] = []
             es_report_group_docs[report_group.partition_id].append(
                 report_group.es_doc())
 
             action = 'REPORT'
             log_msg = '%s: %s %s, client: %s, proto: %s' % (
                 action,
                 report_data.get('http_status', 'unknown'),
                 str(resource),
                 report_data.get('client'),
                 proto_version)
             log.info(log_msg)
         total_reports = len(dataset)
         key = REDIS_KEYS['counters']['reports_per_minute'].format(current_time)
         Datastores.redis.incr(key, total_reports)
         Datastores.redis.expire(key, 3600 * 24)
         key = REDIS_KEYS['counters']['reports_per_minute_per_app'].format(
             resource_id, current_time)
         Datastores.redis.incr(key, total_reports)
         Datastores.redis.expire(key, 3600 * 24)
 
         add_reports_es(es_report_group_docs, es_report_docs)
         add_reports_slow_calls_es(es_slow_calls_docs)
         add_reports_stats_rows_es(es_reports_stats_rows)
         return True
     except Exception as exc:
         print_traceback(log)
         add_reports.retry(exc=exc)
 
 
 @celery.task(queue="es", default_retry_delay=600, max_retries=144)
 def add_reports_es(report_group_docs, report_docs):
     for k, v in report_group_docs.items():
         Datastores.es.bulk_index(k, 'report_group', v, id_field="_id")
     for k, v in report_docs.items():
         Datastores.es.bulk_index(k, 'report', v, id_field="_id",
                                  parent_field='_parent')
 
 
 @celery.task(queue="es", default_retry_delay=600, max_retries=144)
 def add_reports_slow_calls_es(es_docs):
     for k, v in es_docs.items():
         Datastores.es.bulk_index(k, 'log', v)
 
 
 @celery.task(queue="es", default_retry_delay=600, max_retries=144)
 def add_reports_stats_rows_es(es_docs):
     for k, v in es_docs.items():
         Datastores.es.bulk_index(k, 'log', v)
 
 
 @celery.task(queue="logs", default_retry_delay=600, max_retries=144)
 def add_logs(resource_id, request, dataset, environ=None, **kwargs):
     proto_version = request.get('protocol_version')
     current_time = datetime.utcnow().replace(second=0, microsecond=0)
 
     try:
         es_docs = collections.defaultdict(list)
         application = ApplicationService.by_id(resource_id)
         ns_pairs = []
         for entry in dataset:
             # gather pk and ns so we can remove older versions of row later
             if entry['primary_key'] is not None:
                 ns_pairs.append({"pk": entry['primary_key'],
                                  "ns": entry['namespace']})
             log_entry = Log()
             log_entry.set_data(entry, resource=application)
             log_entry._skip_ft_index = True
             application.logs.append(log_entry)
             DBSession.flush()
             # insert non pk rows first
             if entry['primary_key'] is None:
                 es_docs[log_entry.partition_id].append(log_entry.es_doc())
 
         # 2nd pass to delete all log entries from db foe same pk/ns pair
         if ns_pairs:
             ids_to_delete = []
             es_docs = collections.defaultdict(list)
             es_docs_to_delete = collections.defaultdict(list)
             found_pkey_logs = LogService.query_by_primary_key_and_namespace(
                 list_of_pairs=ns_pairs)
             log_dict = {}
             for log_entry in found_pkey_logs:
                 log_key = (log_entry.primary_key, log_entry.namespace)
                 if log_key not in log_dict:
                     log_dict[log_key] = []
                 log_dict[log_key].append(log_entry)
 
             for ns, entry_list in log_dict.items():
                 entry_list = sorted(entry_list, key=lambda x: x.timestamp)
                 # newest row needs to be indexed in es
                 log_entry = entry_list[-1]
                 # delete everything from pg and ES, leave the last row in pg
                 for e in entry_list[:-1]:
                     ids_to_delete.append(e.log_id)
                     es_docs_to_delete[e.partition_id].append(e.delete_hash)
 
                 es_docs_to_delete[log_entry.partition_id].append(
                     log_entry.delete_hash)
 
                 es_docs[log_entry.partition_id].append(log_entry.es_doc())
 
             if ids_to_delete:
                 query = DBSession.query(Log).filter(
                     Log.log_id.in_(ids_to_delete))
                 query.delete(synchronize_session=False)
             if es_docs_to_delete:
                 # batch this to avoid problems with default ES bulk limits
                 for es_index in es_docs_to_delete.keys():
                     for batch in in_batches(es_docs_to_delete[es_index], 20):
                         query = {'terms': {'delete_hash': batch}}
 
                         try:
                             Datastores.es.delete_by_query(
                                 es_index, 'log', query)
                         except pyelasticsearch.ElasticHttpNotFoundError as exc:
                             log.error(exc)
 
         total_logs = len(dataset)
 
         log_msg = 'LOG_NEW: %s, entries: %s, proto:%s' % (
             str(application),
             total_logs,
             proto_version)
         log.info(log_msg)
         # mark_changed(session)
         key = REDIS_KEYS['counters']['logs_per_minute'].format(current_time)
         Datastores.redis.incr(key, total_logs)
         Datastores.redis.expire(key, 3600 * 24)
         key = REDIS_KEYS['counters']['logs_per_minute_per_app'].format(
             resource_id, current_time)
         Datastores.redis.incr(key, total_logs)
         Datastores.redis.expire(key, 3600 * 24)
         add_logs_es(es_docs)
         return True
     except Exception as exc:
         print_traceback(log)
         add_logs.retry(exc=exc)
 
 
 @celery.task(queue="es", default_retry_delay=600, max_retries=144)
 def add_logs_es(es_docs):
     for k, v in es_docs.items():
         Datastores.es.bulk_index(k, 'log', v)
 
 
 @celery.task(queue="metrics", default_retry_delay=600, max_retries=144)
 def add_metrics(resource_id, request, dataset, proto_version):
     current_time = datetime.utcnow().replace(second=0, microsecond=0)
     try:
         application = ApplicationService.by_id_cached()(resource_id)
         application = DBSession.merge(application, load=False)
         es_docs = []
         rows = []
         for metric in dataset:
             tags = dict(metric['tags'])
             server_n = tags.get('server_name', metric['server_name']).lower()
             tags['server_name'] = server_n or 'unknown'
             new_metric = Metric(
                 timestamp=metric['timestamp'],
                 resource_id=application.resource_id,
                 namespace=metric['namespace'],
                 tags=tags)
             rows.append(new_metric)
             es_docs.append(new_metric.es_doc())
         session = DBSession()
         session.bulk_save_objects(rows)
         session.flush()
 
         action = 'METRICS'
         metrics_msg = '%s: %s, metrics: %s, proto:%s' % (
             action,
             str(application),
             len(dataset),
             proto_version
         )
         log.info(metrics_msg)
 
         mark_changed(session)
         key = REDIS_KEYS['counters']['metrics_per_minute'].format(current_time)
         Datastores.redis.incr(key, len(rows))
         Datastores.redis.expire(key, 3600 * 24)
         key = REDIS_KEYS['counters']['metrics_per_minute_per_app'].format(
             resource_id, current_time)
         Datastores.redis.incr(key, len(rows))
         Datastores.redis.expire(key, 3600 * 24)
         add_metrics_es(es_docs)
         return True
     except Exception as exc:
         print_traceback(log)
         add_metrics.retry(exc=exc)
 
 
 @celery.task(queue="es", default_retry_delay=600, max_retries=144)
 def add_metrics_es(es_docs):
     for doc in es_docs:
         partition = 'rcae_m_%s' % doc['timestamp'].strftime('%Y_%m_%d')
         Datastores.es.index(partition, 'log', doc)
 
 
 @celery.task(queue="default", default_retry_delay=5, max_retries=2)
 def check_user_report_notifications(resource_id):
     since_when = datetime.utcnow()
     try:
         request = get_current_request()
         application = ApplicationService.by_id(resource_id)
         if not application:
             return
         error_key = REDIS_KEYS['reports_to_notify_per_type_per_app'].format(
             ReportType.error, resource_id)
         slow_key = REDIS_KEYS['reports_to_notify_per_type_per_app'].format(
             ReportType.slow, resource_id)
         error_group_ids = Datastores.redis.smembers(error_key)
         slow_group_ids = Datastores.redis.smembers(slow_key)
         Datastores.redis.delete(error_key)
         Datastores.redis.delete(slow_key)
         err_gids = [int(g_id) for g_id in error_group_ids]
         slow_gids = [int(g_id) for g_id in list(slow_group_ids)]
         group_ids = err_gids + slow_gids
         occurence_dict = {}
         for g_id in group_ids:
             key = REDIS_KEYS['counters']['report_group_occurences'].format(
                 g_id)
             val = Datastores.redis.get(key)
             Datastores.redis.delete(key)
             if val:
                 occurence_dict[g_id] = int(val)
             else:
                 occurence_dict[g_id] = 1
         report_groups = ReportGroupService.by_ids(group_ids)
         report_groups.options(sa.orm.joinedload(ReportGroup.last_report_ref))
 
         ApplicationService.check_for_groups_alert(
             application, 'alert', report_groups=report_groups,
             occurence_dict=occurence_dict)
         users = set([p.user for p in application.users_for_perm('view')])
         report_groups = report_groups.all()
         for user in users:
             UserService.report_notify(user, request, application,
                                       report_groups=report_groups,
                                       occurence_dict=occurence_dict)
         for group in report_groups:
             # marks report_groups as notified
             if not group.notified:
                 group.notified = True
     except Exception as exc:
         print_traceback(log)
         raise
 
 
 @celery.task(queue="default", default_retry_delay=5, max_retries=2)
 def check_alerts(resource_id):
     since_when = datetime.utcnow()
     try:
         request = get_current_request()
         application = ApplicationService.by_id(resource_id)
         if not application:
             return
         error_key = REDIS_KEYS[
             'reports_to_notify_per_type_per_app_alerting'].format(
             ReportType.error, resource_id)
         slow_key = REDIS_KEYS[
             'reports_to_notify_per_type_per_app_alerting'].format(
             ReportType.slow, resource_id)
         error_group_ids = Datastores.redis.smembers(error_key)
         slow_group_ids = Datastores.redis.smembers(slow_key)
         Datastores.redis.delete(error_key)
         Datastores.redis.delete(slow_key)
         err_gids = [int(g_id) for g_id in error_group_ids]
         slow_gids = [int(g_id) for g_id in list(slow_group_ids)]
         group_ids = err_gids + slow_gids
         occurence_dict = {}
         for g_id in group_ids:
             key = REDIS_KEYS['counters'][
                 'report_group_occurences_alerting'].format(
                 g_id)
             val = Datastores.redis.get(key)
             Datastores.redis.delete(key)
             if val:
                 occurence_dict[g_id] = int(val)
             else:
                 occurence_dict[g_id] = 1
         report_groups = ReportGroupService.by_ids(group_ids)
         report_groups.options(sa.orm.joinedload(ReportGroup.last_report_ref))
 
         ApplicationService.check_for_groups_alert(
             application, 'alert', report_groups=report_groups,
             occurence_dict=occurence_dict, since_when=since_when)
     except Exception as exc:
         print_traceback(log)
         raise
 
 
 @celery.task(queue="default", default_retry_delay=1, max_retries=2)
 def close_alerts():
     log.warning('Checking alerts')
     since_when = datetime.utcnow()
     try:
         event_types = [Event.types['error_report_alert'],
                        Event.types['slow_report_alert'], ]
         statuses = [Event.statuses['active']]
         # get events older than 5 min
         events = EventService.by_type_and_status(
             event_types,
             statuses,
             older_than=(since_when - timedelta(minutes=5)))
         for event in events:
             # see if we can close them
             event.validate_or_close(
                 since_when=(since_when - timedelta(minutes=1)))
     except Exception as exc:
         print_traceback(log)
         raise
 
 
 @celery.task(queue="default", default_retry_delay=600, max_retries=144)
 def update_tag_counter(tag_name, tag_value, count):
     try:
         query = DBSession.query(Tag).filter(Tag.name == tag_name).filter(
             sa.cast(Tag.value, sa.types.TEXT) == sa.cast(json.dumps(tag_value),
                                                          sa.types.TEXT))
         query.update({'times_seen': Tag.times_seen + count,
                       'last_timestamp': datetime.utcnow()},
                      synchronize_session=False)
         session = DBSession()
         mark_changed(session)
         return True
     except Exception as exc:
         print_traceback(log)
         update_tag_counter.retry(exc=exc)
 
 
 @celery.task(queue="default")
 def update_tag_counters():
     """
     Sets task to update counters for application tags
     """
     tags = Datastores.redis.lrange(REDIS_KEYS['seen_tag_list'], 0, -1)
     Datastores.redis.delete(REDIS_KEYS['seen_tag_list'])
     c = collections.Counter(tags)
     for t_json, count in c.items():
         tag_info = json.loads(t_json)
         update_tag_counter.delay(tag_info[0], tag_info[1], count)
 
 
 @celery.task(queue="default")
 def daily_digest():
     """
     Sends daily digest with top 50 error reports
     """
     request = get_current_request()
     apps = Datastores.redis.smembers(REDIS_KEYS['apps_that_had_reports'])
     Datastores.redis.delete(REDIS_KEYS['apps_that_had_reports'])
     since_when = datetime.utcnow() - timedelta(hours=8)
     log.warning('Generating daily digests')
     for resource_id in apps:
         resource_id = resource_id.decode('utf8')
         end_date = datetime.utcnow().replace(microsecond=0, second=0)
         filter_settings = {'resource': [resource_id],
                            'tags': [{'name': 'type',
                                      'value': ['error'], 'op': None}],
                            'type': 'error', 'start_date': since_when,
                            'end_date': end_date}
 
         reports = ReportGroupService.get_trending(
             request, filter_settings=filter_settings, limit=50)
 
         application = ApplicationService.by_id(resource_id)
         if application:
             users = set([p.user for p in application.users_for_perm('view')])
             for user in users:
                 user.send_digest(request, application, reports=reports,
                                  since_when=since_when)
 
 
 @celery.task(queue="default")
 def notifications_reports():
     """
     Loop that checks redis for info and then issues new tasks to celery to
     issue notifications
     """
     apps = Datastores.redis.smembers(REDIS_KEYS['apps_that_had_reports'])
     Datastores.redis.delete(REDIS_KEYS['apps_that_had_reports'])
     for app in apps:
         log.warning('Notify for app: %s' % app)
         check_user_report_notifications.delay(app.decode('utf8'))
 
 @celery.task(queue="default")
 def alerting_reports():
     """
     Loop that checks redis for info and then issues new tasks to celery to
     perform the following:
     - which applications should have new alerts opened
     """
 
     apps = Datastores.redis.smembers(REDIS_KEYS['apps_that_had_reports_alerting'])
     Datastores.redis.delete(REDIS_KEYS['apps_that_had_reports_alerting'])
     for app in apps:
         log.warning('Notify for app: %s' % app)
         check_alerts.delay(app.decode('utf8'))
 
 
 @celery.task(queue="default", soft_time_limit=3600 * 4,
              hard_time_limit=3600 * 4, max_retries=144)
 def logs_cleanup(resource_id, filter_settings):
     request = get_current_request()
     request.tm.begin()
     es_query = {
         "_source": False,
         "size": 5000,
         "query": {
             "filtered": {
                 "filter": {
                     "and": [{"term": {"resource_id": resource_id}}]
                 }
             }
         }
     }
 
     query = DBSession.query(Log).filter(Log.resource_id == resource_id)
     if filter_settings['namespace']:
         query = query.filter(Log.namespace == filter_settings['namespace'][0])
         es_query['query']['filtered']['filter']['and'].append(
             {"term": {"namespace": filter_settings['namespace'][0]}}
         )
     query.delete(synchronize_session=False)
     request.tm.commit()
     result = request.es_conn.search(es_query, index='rcae_l_*',
                                     doc_type='log', es_scroll='1m',
                                     es_search_type='scan')
     scroll_id = result['_scroll_id']
     while True:
         log.warning('log_cleanup, app:{} ns:{} batch'.format(
             resource_id,
             filter_settings['namespace']
         ))
         es_docs_to_delete = []
         result = request.es_conn.send_request(
             'POST', ['_search', 'scroll'],
             body=scroll_id, query_params={"scroll": '1m'})
         scroll_id = result['_scroll_id']
         if not result['hits']['hits']:
             break
         for doc in result['hits']['hits']:
             es_docs_to_delete.append({"id": doc['_id'],
                                       "index": doc['_index']})
 
         for batch in in_batches(es_docs_to_delete, 10):
             Datastores.es.bulk([Datastores.es.delete_op(doc_type='log',
                                                         **to_del)
                                 for to_del in batch])

backend/src/appenlight/models/__init__.py

@@ -1,135 +1,135 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # AppEnlight Enterprise Edition, including its added features, Support
 # services, and proprietary license terms, please see
 # https://rhodecode.com/licenses/
 
 import logging
 
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy import MetaData
 from sqlalchemy.orm import scoped_session
 from sqlalchemy.orm import sessionmaker
 from zope.sqlalchemy import ZopeTransactionExtension
 import ziggurat_foundations
 from ziggurat_foundations.models.base import get_db_session
 
 log = logging.getLogger(__name__)
 
 DBSession = scoped_session(sessionmaker(extension=ZopeTransactionExtension()))
 
 NAMING_CONVENTION = {
     "ix": 'ix_%(column_0_label)s',
     "uq": "uq_%(table_name)s_%(column_0_name)s",
     "ck": "ck_%(table_name)s_%(constraint_name)s",
     "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
     "pk": "pk_%(table_name)s"
 }
 
 metadata = MetaData(naming_convention=NAMING_CONVENTION)
 Base = declarative_base(metadata=metadata)
 
 # optional for request.db approach
 ziggurat_foundations.models.DBSession = DBSession
 
 
 class Datastores(object):
     redis = None
     es = None
 
 
 def register_datastores(es_conn, redis_conn, redis_lockmgr):
     Datastores.es = es_conn
     Datastores.redis = redis_conn
     Datastores.lockmgr = redis_lockmgr
 
 
 class SliceableESQuery(object):
     def __init__(self, query, sort_query=None, aggregations=False, **kwconfig):
         self.query = query
         self.sort_query = sort_query
         self.aggregations = aggregations
         self.items_per_page = kwconfig.pop('items_per_page', 10)
         self.page = kwconfig.pop('page', 1)
         self.kwconfig = kwconfig
         self.result = None
 
     def __getitem__(self, index):
         config = self.kwconfig.copy()
         config['es_from'] = index.start
         query = self.query.copy()
         if self.sort_query:
             query.update(self.sort_query)
         self.result = Datastores.es.search(query, size=self.items_per_page,
                                            **config)
         if self.aggregations:
             self.items = self.result.get('aggregations')
         else:
             self.items = self.result['hits']['hits']
 
         return self.items
 
     def __iter__(self):
         return self.result
 
     def __len__(self):
         config = self.kwconfig.copy()
         query = self.query.copy()
         self.result = Datastores.es.search(query, size=self.items_per_page,
                                            **config)
         if self.aggregations:
             self.items = self.result.get('aggregations')
         else:
             self.items = self.result['hits']['hits']
 
         count = int(self.result['hits']['total'])
         return count if count < 5000 else 5000
 
 
 from appenlight.models.resource import Resource
 from appenlight.models.application import Application
 from appenlight.models.user import User
 from appenlight.models.alert_channel import AlertChannel
 from appenlight.models.alert_channel_action import AlertChannelAction
-from appenlight.models.request_metric import Metric
+from appenlight.models.metric import Metric
 from appenlight.models.application_postprocess_conf import \
     ApplicationPostprocessConf
 from appenlight.models.auth_token import AuthToken
 from appenlight.models.event import Event
 from appenlight.models.external_identity import ExternalIdentity
 from appenlight.models.group import Group
 from appenlight.models.group_permission import GroupPermission
 from appenlight.models.group_resource_permission import GroupResourcePermission
 from appenlight.models.log import Log
 from appenlight.models.plugin_config import PluginConfig
 from appenlight.models.report import Report
 from appenlight.models.report_group import ReportGroup
 from appenlight.models.report_comment import ReportComment
 from appenlight.models.report_assignment import ReportAssignment
 from appenlight.models.report_stat import ReportStat
 from appenlight.models.slow_call import SlowCall
 from appenlight.models.tag import Tag
 from appenlight.models.user_group import UserGroup
 from appenlight.models.user_permission import UserPermission
 from appenlight.models.user_resource_permission import UserResourcePermission
 from ziggurat_foundations import ziggurat_model_init
 
 ziggurat_model_init(User, Group, UserGroup, GroupPermission, UserPermission,
                     UserResourcePermission, GroupResourcePermission,
                     Resource,
                     ExternalIdentity, passwordmanager=None)

backend/src/appenlight/scripts/reindex_elasticsearch.py

@@ -1,435 +1,435 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # AppEnlight Enterprise Edition, including its added features, Support
 # services, and proprietary license terms, please see
 # https://rhodecode.com/licenses/
 
 import argparse
 import datetime
 import logging
 
 import sqlalchemy as sa
 from collections import defaultdict
 from pyramid.paster import setup_logging
 from pyramid.paster import bootstrap
 from appenlight.models import (
     DBSession,
     Datastores,
     metadata
 )
 from appenlight.lib import get_callable
 from appenlight.models.report_group import ReportGroup
 from appenlight.models.report import Report
 from appenlight.models.report_stat import ReportStat
 from appenlight.models.log import Log
 from appenlight.models.slow_call import SlowCall
-from appenlight.models.request_metric import Metric
+from appenlight.models.metric import Metric
 
 
 log = logging.getLogger(__name__)
 
 tables = {
     'slow_calls_p_': [],
     'reports_stats_p_': [],
     'reports_p_': [],
     'reports_groups_p_': [],
     'logs_p_': [],
     'metrics_p_': [],
 }
 
 def detect_tables(table_prefix):
     found_tables = []
     db_tables_query = '''
     SELECT tablename FROM pg_tables WHERE tablename NOT LIKE 'pg_%' AND
     tablename NOT LIKE 'sql_%' ORDER BY tablename ASC;'''
 
     for table in DBSession.execute(db_tables_query).fetchall():
         tablename = table.tablename
         if tablename.startswith(table_prefix):
             t = sa.Table(tablename, metadata, autoload=True,
                          autoload_with=DBSession.bind.engine)
             found_tables.append(t)
     return found_tables
 
 
 def main():
     """
     Recreates Elasticsearch indexes
     Performs reindex of whole db to Elasticsearch
 
     """
 
     # need parser twice because we first need to load ini file
     # bootstrap pyramid and then load plugins
     pre_parser = argparse.ArgumentParser(
         description='Reindex AppEnlight data',
         add_help=False)
     pre_parser.add_argument('-c', '--config', required=True,
                             help='Configuration ini file of application')
     pre_parser.add_argument('-h', '--help', help='Show help', nargs='?')
     pre_parser.add_argument('-t', '--types', nargs='+',
                             help='Which parts of database should get reindexed')
     args = pre_parser.parse_args()
 
     config_uri = args.config
     setup_logging(config_uri)
     log.setLevel(logging.INFO)
     env = bootstrap(config_uri)
     parser = argparse.ArgumentParser(description='Reindex AppEnlight data')
     choices = {
         'reports': 'appenlight.scripts.reindex_elasticsearch:reindex_reports',
         'logs': 'appenlight.scripts.reindex_elasticsearch:reindex_logs',
         'metrics': 'appenlight.scripts.reindex_elasticsearch:reindex_metrics',
         'slow_calls': 'appenlight.scripts.reindex_elasticsearch:reindex_slow_calls',
         'template': 'appenlight.scripts.reindex_elasticsearch:update_template'
     }
     for k, v in env['registry'].appenlight_plugins.items():
         if v.get('fulltext_indexer'):
             choices[k] = v['fulltext_indexer']
     parser.add_argument('-t', '--types', nargs='*',
                         choices=['all'] + list(choices.keys()), default=['all'],
                         help='Which parts of database should get reindexed')
     parser.add_argument('-c', '--config', required=True,
                         help='Configuration ini file of application')
     args = parser.parse_args()
 
 
     if 'all' in args.types:
         args.types = list(choices.keys())
 
     log.info('settings {}'.format(args.types))
 
     if 'template' in args.types:
         get_callable(choices['template'])()
         args.types.remove('template')
     for selected in args.types:
         get_callable(choices[selected])()
 
 
 def update_template():
     try:
         Datastores.es.send_request("delete", ['_template', 'rcae'],
                                    query_params={})
     except Exception as e:
         print(e)
     log.info('updating elasticsearch template')
     tag_templates = [
         {"values": {
             "path_match": "tags.*",
             "mapping": {
                 "type": "object",
                 "properties": {
                     "values": {"type": "string", "analyzer": "tag_value"},
                     "numeric_values": {"type": "float"}
                 }
             }
         }}
     ]
 
     template_schema = {
         "template": "rcae_*",
         "settings": {
             "index": {
                 "refresh_interval": "5s",
                 "translog": {"interval": "5s",
                              "durability": "async"}
             },
             "number_of_shards": 5,
             "analysis": {
                 "analyzer": {
                     "url_path": {
                         "type": "custom",
                         "char_filter": [],
                         "tokenizer": "path_hierarchy",
                         "filter": []
                     },
                     "tag_value": {
                         "type": "custom",
                         "char_filter": [],
                         "tokenizer": "keyword",
                         "filter": ["lowercase"]
                     },
                 }
             },
         },
         "mappings": {
             "report_group": {
                 "_all": {"enabled": False},
                 "dynamic_templates": tag_templates,
                 "properties": {
                     "pg_id": {"type": "string", "index": "not_analyzed"},
                     "resource_id": {"type": "integer"},
                     "priority": {"type": "integer"},
                     "error": {"type": "string", "analyzer": "simple"},
                     "read": {"type": "boolean"},
                     "occurences": {"type": "integer"},
                     "fixed": {"type": "boolean"},
                     "first_timestamp": {"type": "date"},
                     "last_timestamp": {"type": "date"},
                     "average_duration": {"type": "float"},
                     "summed_duration": {"type": "float"},
                     "public": {"type": "boolean"}
                 }
             },
             "report": {
                 "_all": {"enabled": False},
                 "dynamic_templates": tag_templates,
                 "properties": {
                     "pg_id": {"type": "string", "index": "not_analyzed"},
                     "resource_id": {"type": "integer"},
                     "group_id": {"type": "string"},
                     "http_status": {"type": "integer"},
                     "ip": {"type": "string", "index": "not_analyzed"},
                     "url_domain": {"type": "string", "analyzer": "simple"},
                     "url_path": {"type": "string", "analyzer": "url_path"},
                     "error": {"type": "string", "analyzer": "simple"},
                     "report_type": {"type": "integer"},
                     "start_time": {"type": "date"},
                     "request_id": {"type": "string", "index": "not_analyzed"},
                     "end_time": {"type": "date"},
                     "duration": {"type": "float"},
                     "tags": {
                         "type": "object"
                     },
                     "tag_list": {"type": "string", "analyzer": "tag_value"},
                     "extra": {
                         "type": "object"
                     },
                 },
                 "_parent": {"type": "report_group"}
             },
             "log": {
                 "_all": {"enabled": False},
                 "dynamic_templates": tag_templates,
                 "properties": {
                     "pg_id": {"type": "string", "index": "not_analyzed"},
                     "delete_hash": {"type": "string", "index": "not_analyzed"},
                     "resource_id": {"type": "integer"},
                     "timestamp": {"type": "date"},
                     "permanent": {"type": "boolean"},
                     "request_id": {"type": "string", "index": "not_analyzed"},
                     "log_level": {"type": "string", "analyzer": "simple"},
                     "message": {"type": "string", "analyzer": "simple"},
                     "namespace": {"type": "string", "index": "not_analyzed"},
                     "tags": {
                         "type": "object"
                     },
                     "tag_list": {"type": "string", "analyzer": "tag_value"}
                 }
             }
         }
     }
 
     Datastores.es.send_request('PUT', ['_template', 'rcae'],
                                body=template_schema, query_params={})
 
 
 def reindex_reports():
     reports_groups_tables = detect_tables('reports_groups_p_')
     try:
         Datastores.es.delete_index('rcae_r*')
     except Exception as e:
         log.error(e)
 
     log.info('reindexing report groups')
     i = 0
     task_start = datetime.datetime.now()
     for partition_table in reports_groups_tables:
         conn = DBSession.connection().execution_options(stream_results=True)
         result = conn.execute(partition_table.select())
         while True:
             chunk = result.fetchmany(2000)
             if not chunk:
                 break
             es_docs = defaultdict(list)
             for row in chunk:
                 i += 1
                 item = ReportGroup(**dict(list(row.items())))
                 d_range = item.partition_id
                 es_docs[d_range].append(item.es_doc())
             if es_docs:
                 name = partition_table.name
                 log.info('round {}, {}'.format(i, name))
                 for k, v in es_docs.items():
                     Datastores.es.bulk_index(k, 'report_group', v,
                                              id_field="_id")
 
     log.info(
         'total docs {} {}'.format(i, datetime.datetime.now() - task_start))
 
     i = 0
     log.info('reindexing reports')
     task_start = datetime.datetime.now()
     reports_tables = detect_tables('reports_p_')
     for partition_table in reports_tables:
         conn = DBSession.connection().execution_options(stream_results=True)
         result = conn.execute(partition_table.select())
         while True:
             chunk = result.fetchmany(2000)
             if not chunk:
                 break
             es_docs = defaultdict(list)
             for row in chunk:
                 i += 1
                 item = Report(**dict(list(row.items())))
                 d_range = item.partition_id
                 es_docs[d_range].append(item.es_doc())
             if es_docs:
                 name = partition_table.name
                 log.info('round {}, {}'.format(i, name))
                 for k, v in es_docs.items():
                     Datastores.es.bulk_index(k, 'report', v, id_field="_id",
                                              parent_field='_parent')
 
     log.info(
         'total docs {} {}'.format(i, datetime.datetime.now() - task_start))
 
     log.info('reindexing reports stats')
     i = 0
     task_start = datetime.datetime.now()
     reports_stats_tables = detect_tables('reports_stats_p_')
     for partition_table in reports_stats_tables:
         conn = DBSession.connection().execution_options(stream_results=True)
         result = conn.execute(partition_table.select())
         while True:
             chunk = result.fetchmany(2000)
             if not chunk:
                 break
             es_docs = defaultdict(list)
             for row in chunk:
                 rd = dict(list(row.items()))
                 # remove legacy columns
                 # TODO: remove the column later
                 rd.pop('size', None)
                 item = ReportStat(**rd)
                 i += 1
                 d_range = item.partition_id
                 es_docs[d_range].append(item.es_doc())
             if es_docs:
                 name = partition_table.name
                 log.info('round  {}, {}'.format(i, name))
                 for k, v in es_docs.items():
                     Datastores.es.bulk_index(k, 'log', v)
 
     log.info(
         'total docs {} {}'.format(i, datetime.datetime.now() - task_start))
 
 
 def reindex_logs():
     try:
         Datastores.es.delete_index('rcae_l*')
     except Exception as e:
         log.error(e)
 
     # logs
     log.info('reindexing logs')
     i = 0
     task_start = datetime.datetime.now()
     log_tables = detect_tables('logs_p_')
     for partition_table in log_tables:
         conn = DBSession.connection().execution_options(stream_results=True)
         result = conn.execute(partition_table.select())
         while True:
             chunk = result.fetchmany(2000)
             if not chunk:
                 break
             es_docs = defaultdict(list)
 
             for row in chunk:
                 i += 1
                 item = Log(**dict(list(row.items())))
                 d_range = item.partition_id
                 es_docs[d_range].append(item.es_doc())
             if es_docs:
                 name = partition_table.name
                 log.info('round  {}, {}'.format(i, name))
                 for k, v in es_docs.items():
                     Datastores.es.bulk_index(k, 'log', v)
 
     log.info(
         'total docs {} {}'.format(i, datetime.datetime.now() - task_start))
 
 
 def reindex_metrics():
     try:
         Datastores.es.delete_index('rcae_m*')
     except Exception as e:
         print(e)
 
     log.info('reindexing applications metrics')
     i = 0
     task_start = datetime.datetime.now()
     metric_tables = detect_tables('metrics_p_')
     for partition_table in metric_tables:
         conn = DBSession.connection().execution_options(stream_results=True)
         result = conn.execute(partition_table.select())
         while True:
             chunk = result.fetchmany(2000)
             if not chunk:
                 break
             es_docs = defaultdict(list)
             for row in chunk:
                 i += 1
                 item = Metric(**dict(list(row.items())))
                 d_range = item.partition_id
                 es_docs[d_range].append(item.es_doc())
             if es_docs:
                 name = partition_table.name
                 log.info('round  {}, {}'.format(i, name))
                 for k, v in es_docs.items():
                     Datastores.es.bulk_index(k, 'log', v)
 
     log.info(
         'total docs {} {}'.format(i, datetime.datetime.now() - task_start))
 
 
 def reindex_slow_calls():
     try:
         Datastores.es.delete_index('rcae_sc*')
     except Exception as e:
         print(e)
 
     log.info('reindexing slow calls')
     i = 0
     task_start = datetime.datetime.now()
     slow_calls_tables = detect_tables('slow_calls_p_')
     for partition_table in slow_calls_tables:
         conn = DBSession.connection().execution_options(stream_results=True)
         result = conn.execute(partition_table.select())
         while True:
             chunk = result.fetchmany(2000)
             if not chunk:
                 break
             es_docs = defaultdict(list)
             for row in chunk:
                 i += 1
                 item = SlowCall(**dict(list(row.items())))
                 d_range = item.partition_id
                 es_docs[d_range].append(item.es_doc())
             if es_docs:
                 name = partition_table.name
                 log.info('round  {}, {}'.format(i, name))
                 for k, v in es_docs.items():
                     Datastores.es.bulk_index(k, 'log', v)
 
     log.info(
         'total docs {} {}'.format(i, datetime.datetime.now() - task_start))
 
 
 if __name__ == '__main__':
     main()

backend/src/appenlight/tests/test_unit.py

@@ -1,1703 +1,1703 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # AppEnlight Enterprise Edition, including its added features, Support
 # services, and proprietary license terms, please see
 # https://rhodecode.com/licenses/
 
 import copy
 import logging
 import mock
 import pyramid
 import pytest
 import sqlalchemy as sa
 import webob
 
 from datetime import datetime
 from pyramid import testing
 
 
 from appenlight.models import DBSession
 from appenlight.lib.ext_json import json
 
 
 log = logging.getLogger(__name__)
 
 
 class DummyContext(object):
     pass
 
 
 @pytest.mark.usefixtures('base_app')
 class BasicTest(object):
     pass
 
 
 @pytest.mark.usefixtures('base_app')
 class TestMigration(object):
     def test_migration(self):
         assert 1 == 1
 
 
 class TestAPIReports_0_4_Validation(object):
     @pytest.mark.parametrize('dummy_json', ['', {}, [], None])
     def test_no_payload(self, dummy_json):
         import colander
         from appenlight.validators import ReportListSchema_0_4
         utcnow = datetime.utcnow()
         schema = ReportListSchema_0_4().bind(utcnow=utcnow)
         with pytest.raises(colander.Invalid):
             schema.deserialize(dummy_json)
 
     def test_minimal_payload(self, report_04_schema):
         dummy_json = [{}]
         import colander
         from appenlight.validators import ReportListSchema_0_4
         utcnow = datetime.utcnow()
         schema = ReportListSchema_0_4().bind(utcnow=utcnow)
         with pytest.raises(colander.Invalid):
             schema.deserialize(dummy_json)
 
     def test_minimal_payload(self):
         from appenlight.validators import ReportListSchema_0_4
         dummy_json = [{'report_details': [{}]}]
         utcnow = datetime.utcnow()
         schema = ReportListSchema_0_4().bind(utcnow=utcnow)
         deserialized = schema.deserialize(dummy_json)
 
         expected_deserialization = [
             {'error_type': '',
              'language': 'unknown',
              'report_details': [
                  {'username': '',
                   'traceback': None,
                   'extra': None,
                   'frameinfo': None,
                   'url': '',
                   'ip': None,
                   'start_time': utcnow,
                   'group_string': None,
                   'request': {},
                   'request_stats': None,
                   'end_time': None,
                   'request_id': '',
                   'message': '',
                   'slow_calls': [],
                   'user_agent': ''}],
              'server': 'unknown',
              'occurences': 1,
              'priority': 5,
              'view_name': '',
              'client': 'unknown',
              'http_status': 200,
              'error': '',
              'tags': None}
         ]
         assert deserialized == expected_deserialization
 
     def test_full_payload(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.validators import ReportListSchema_0_4
         utcnow = datetime.utcnow()
         schema = ReportListSchema_0_4().bind(utcnow=utcnow)
         PYTHON_PAYLOAD = copy.deepcopy(payload_examples.PYTHON_PAYLOAD_0_4)
         utcnow = datetime.utcnow()
         PYTHON_PAYLOAD["tags"] = [("foo", 1), ("action", "test"), ("baz", 1.1),
                                   ("date",
                                    utcnow.strftime('%Y-%m-%dT%H:%M:%S.0'))]
         dummy_json = [PYTHON_PAYLOAD]
 
         deserialized = schema.deserialize(dummy_json)
         assert deserialized[0]['error'] == PYTHON_PAYLOAD['error']
         assert deserialized[0]['language'] == PYTHON_PAYLOAD['language']
         assert deserialized[0]['server'] == PYTHON_PAYLOAD['server']
         assert deserialized[0]['priority'] == PYTHON_PAYLOAD['priority']
         assert deserialized[0]['view_name'] == PYTHON_PAYLOAD['view_name']
         assert deserialized[0]['client'] == PYTHON_PAYLOAD['client']
         assert deserialized[0]['http_status'] == PYTHON_PAYLOAD['http_status']
         assert deserialized[0]['error'] == PYTHON_PAYLOAD['error']
         assert deserialized[0]['occurences'] == PYTHON_PAYLOAD['occurences']
         first_detail = deserialized[0]['report_details'][0]
         payload_detail = PYTHON_PAYLOAD['report_details'][0]
         assert first_detail['username'] == payload_detail['username']
         assert first_detail['traceback'] == payload_detail['traceback']
         assert first_detail['url'] == payload_detail['url']
         assert first_detail['ip'] == payload_detail['ip']
         assert first_detail['start_time'].strftime('%Y-%m-%dT%H:%M:%S.0') == \
                payload_detail['start_time']
         assert first_detail['ip'] == payload_detail['ip']
         assert first_detail['group_string'] is None
         assert first_detail['request_stats'] == payload_detail['request_stats']
         assert first_detail['end_time'].strftime('%Y-%m-%dT%H:%M:%S.0') == \
                payload_detail['end_time']
         assert first_detail['request_id'] == payload_detail['request_id']
         assert first_detail['message'] == payload_detail['message']
         assert first_detail['user_agent'] == payload_detail['user_agent']
         slow_call = first_detail['slow_calls'][0]
         expected_slow_call = payload_detail['slow_calls'][0]
         assert slow_call['start'].strftime('%Y-%m-%dT%H:%M:%S.0') == \
                expected_slow_call['start']
         assert slow_call['end'].strftime('%Y-%m-%dT%H:%M:%S.0') == \
                expected_slow_call['end']
         assert slow_call['statement'] == expected_slow_call['statement']
         assert slow_call['parameters'] == expected_slow_call['parameters']
         assert slow_call['type'] == expected_slow_call['type']
         assert slow_call['subtype'] == expected_slow_call['subtype']
         assert slow_call['location'] == ''
         assert deserialized[0]['tags'] == [
             ('foo', 1), ('action', 'test'),
             ('baz', 1.1), ('date', utcnow.strftime('%Y-%m-%dT%H:%M:%S.0'))]
 
 
 class TestSentryProto_7(object):
     def test_log_payload(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.lib.enums import ParsedSentryEventType
         from appenlight.lib.utils.sentry import parse_sentry_event
         event_dict, event_type = parse_sentry_event(
             payload_examples.SENTRY_LOG_PAYLOAD_7)
         assert ParsedSentryEventType.LOG == event_type
         assert event_dict['log_level'] == 'CRITICAL'
         assert event_dict['message'] == 'TEST from django logging'
         assert event_dict['namespace'] == 'testlogger'
         assert event_dict['request_id'] == '9a6172f2e6d2444582f83a6c333d9cfb'
         assert event_dict['server'] == 'ergo-virtual-machine'
         assert event_dict['date'] == datetime.utcnow().date().strftime(
             '%Y-%m-%dT%H:%M:%SZ')
         tags = [('site', 'example.com'),
                 ('sys.argv', ["'manage.py'", "'runserver'"]),
                 ('price', 6),
                 ('tag', "'extra'"),
                 ('dupa', True),
                 ('project', 'sentry'),
                 ('sentry_culprit', 'testlogger in index'),
                 ('sentry_language', 'python'),
                 ('sentry_release', 'test')]
         assert sorted(event_dict['tags']) == sorted(tags)
 
     def test_report_payload(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.lib.enums import ParsedSentryEventType
         from appenlight.lib.utils.sentry import parse_sentry_event
         utcnow = datetime.utcnow().date().strftime('%Y-%m-%dT%H:%M:%SZ')
         event_dict, event_type = parse_sentry_event(
             payload_examples.SENTRY_PYTHON_PAYLOAD_7)
         assert ParsedSentryEventType.ERROR_REPORT == event_type
         assert event_dict['client'] == 'sentry'
         assert event_dict[
                    'error'] == 'Exception: test 500 ' \
                                '\u0142\xf3\u201c\u0107\u201c\u0107\u017c\u0105'
         assert event_dict['language'] == 'python'
         assert event_dict['ip'] == '127.0.0.1'
         assert event_dict['request_id'] == '9fae652c8c1c4d6a8eee09260f613a98'
         assert event_dict['server'] == 'ergo-virtual-machine'
         assert event_dict['start_time'] == utcnow
         assert event_dict['url'] == 'http://127.0.0.1:8000/error'
         assert event_dict['user_agent'] == 'Mozilla/5.0 (X11; Linux x86_64) ' \
                                            'AppleWebKit/537.36 (KHTML, ' \
                                            'like Gecko) Chrome/47.0.2526.106 ' \
                                            'Safari/537.36'
         assert event_dict['view_name'] == 'djangoapp.views in error'
         tags = [('site', 'example.com'), ('sentry_release', 'test')]
         assert sorted(event_dict['tags']) == sorted(tags)
         extra = [('sys.argv', ["'manage.py'", "'runserver'"]),
                  ('project', 'sentry')]
         assert sorted(event_dict['extra']) == sorted(extra)
         request = event_dict['request']
         assert request['url'] == 'http://127.0.0.1:8000/error'
         assert request['cookies'] == {'appenlight': 'X'}
         assert request['data'] is None
         assert request['method'] == 'GET'
         assert request['query_string'] == ''
         assert request['env'] == {'REMOTE_ADDR': '127.0.0.1',
                                   'SERVER_NAME': 'localhost',
                                   'SERVER_PORT': '8000'}
         assert request['headers'] == {
             'Accept': 'text/html,application/xhtml+xml,'
                        'application/xml;q=0.9,image/webp,*/*;q=0.8',
             'Accept-Encoding': 'gzip, deflate, sdch',
             'Accept-Language': 'en-US,en;q=0.8,pl;q=0.6',
             'Connection': 'keep-alive',
             'Content-Length': '',
             'Content-Type': 'text/plain',
             'Cookie': 'appenlight=X',
             'Dnt': '1',
             'Host': '127.0.0.1:8000',
             'Upgrade-Insecure-Requests': '1',
             'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) '
                            'AppleWebKit/537.36 (KHTML, like Gecko) '
                            'Chrome/47.0.2526.106 Safari/537.36'}
         traceback = event_dict['traceback']
         assert traceback[0]['cline'] == 'response = wrapped_callback(request, ' \
                                         '*callback_args, **callback_kwargs)'
         assert traceback[0]['file'] == 'django/core/handlers/base.py'
         assert traceback[0]['fn'] == 'get_response'
         assert traceback[0]['line'] == 111
         assert traceback[0]['module'] == 'django.core.handlers.base'
 
         assert traceback[1]['cline'] == "raise Exception(u'test 500 " \
                                         "\u0142\xf3\u201c\u0107\u201c\u0107" \
                                         "\u017c\u0105')"
         assert traceback[1]['file'] == 'djangoapp/views.py'
         assert traceback[1]['fn'] == 'error'
         assert traceback[1]['line'] == 84
         assert traceback[1]['module'] == 'djangoapp.views'
         assert sorted(traceback[1]['vars']) == sorted([
             ('c',
              '<sqlite3.Cursor object at 0x7fe7c82af8f0>'),
             ('request',
              '<WSGIRequest at 0x140633490316304>'),
             ('conn',
              '<sqlite3.Connection object at 0x7fe7c8b23bf8>')])
 
 
 class TestAPIReports_0_5_Validation(object):
     @pytest.mark.parametrize('dummy_json', ['', {}, [], None])
     def test_no_payload(self, dummy_json):
         import colander
         from appenlight.validators import ReportListSchema_0_5
         utcnow = datetime.utcnow()
         schema = ReportListSchema_0_5().bind(utcnow=utcnow)
         with pytest.raises(colander.Invalid):
             schema.deserialize(dummy_json)
 
     def test_minimal_payload(self):
         dummy_json = [{}]
         import colander
         from appenlight.validators import ReportListSchema_0_5
         utcnow = datetime.utcnow()
         schema = ReportListSchema_0_5().bind(utcnow=utcnow)
         with pytest.raises(colander.Invalid):
             schema.deserialize(dummy_json)
 
     def test_minimal_payload(self):
         dummy_json = [{'report_details': [{}]}]
         from appenlight.validators import ReportListSchema_0_5
         utcnow = datetime.utcnow()
         schema = ReportListSchema_0_5().bind(utcnow=utcnow)
 
         deserialized = schema.deserialize(dummy_json)
 
         expected_deserialization = [
             {'language': 'unknown',
              'server': 'unknown',
              'occurences': 1,
              'priority': 5,
              'view_name': '',
              'client': 'unknown',
              'http_status': 200,
              'error': '',
              'tags': None,
              'username': '',
              'traceback': None,
              'extra': None,
              'url': '',
              'ip': None,
              'start_time': utcnow,
              'group_string': None,
              'request': {},
              'request_stats': None,
              'end_time': None,
              'request_id': '',
              'message': '',
              'slow_calls': [],
              'user_agent': ''
              }
         ]
         assert deserialized == expected_deserialization
 
     def test_full_payload(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.validators import ReportListSchema_0_5
         PYTHON_PAYLOAD = copy.deepcopy(payload_examples.PYTHON_PAYLOAD_0_5)
         utcnow = datetime.utcnow()
         schema = ReportListSchema_0_5().bind(utcnow=utcnow)
         PYTHON_PAYLOAD["tags"] = [("foo", 1), ("action", "test"), ("baz", 1.1),
                                   ("date",
                                    utcnow.strftime('%Y-%m-%dT%H:%M:%S.0'))]
         dummy_json = [PYTHON_PAYLOAD]
         deserialized = schema.deserialize(dummy_json)[0]
         assert deserialized['error'] == PYTHON_PAYLOAD['error']
         assert deserialized['language'] == PYTHON_PAYLOAD['language']
         assert deserialized['server'] == PYTHON_PAYLOAD['server']
         assert deserialized['priority'] == PYTHON_PAYLOAD['priority']
         assert deserialized['view_name'] == PYTHON_PAYLOAD['view_name']
         assert deserialized['client'] == PYTHON_PAYLOAD['client']
         assert deserialized['http_status'] == PYTHON_PAYLOAD['http_status']
         assert deserialized['error'] == PYTHON_PAYLOAD['error']
         assert deserialized['occurences'] == PYTHON_PAYLOAD['occurences']
         assert deserialized['username'] == PYTHON_PAYLOAD['username']
         assert deserialized['traceback'] == PYTHON_PAYLOAD['traceback']
         assert deserialized['url'] == PYTHON_PAYLOAD['url']
         assert deserialized['ip'] == PYTHON_PAYLOAD['ip']
         assert deserialized['start_time'].strftime('%Y-%m-%dT%H:%M:%S.0') == \
                PYTHON_PAYLOAD['start_time']
         assert deserialized['ip'] == PYTHON_PAYLOAD['ip']
         assert deserialized['group_string'] is None
         assert deserialized['request_stats'] == PYTHON_PAYLOAD['request_stats']
         assert deserialized['end_time'].strftime('%Y-%m-%dT%H:%M:%S.0') == \
                PYTHON_PAYLOAD['end_time']
         assert deserialized['request_id'] == PYTHON_PAYLOAD['request_id']
         assert deserialized['message'] == PYTHON_PAYLOAD['message']
         assert deserialized['user_agent'] == PYTHON_PAYLOAD['user_agent']
         assert deserialized['slow_calls'][0]['start'].strftime(
             '%Y-%m-%dT%H:%M:%S.0') == PYTHON_PAYLOAD['slow_calls'][0][
                    'start']
         assert deserialized['slow_calls'][0]['end'].strftime(
             '%Y-%m-%dT%H:%M:%S.0') == PYTHON_PAYLOAD['slow_calls'][0][
                    'end']
         assert deserialized['slow_calls'][0]['statement'] == \
                PYTHON_PAYLOAD['slow_calls'][0]['statement']
         assert deserialized['slow_calls'][0]['parameters'] == \
                PYTHON_PAYLOAD['slow_calls'][0]['parameters']
         assert deserialized['slow_calls'][0]['type'] == \
                PYTHON_PAYLOAD['slow_calls'][0]['type']
         assert deserialized['slow_calls'][0]['subtype'] == \
                PYTHON_PAYLOAD['slow_calls'][0]['subtype']
         assert deserialized['slow_calls'][0]['location'] == ''
         assert deserialized['tags'] == [
             ('foo', 1), ('action', 'test'),
             ('baz', 1.1), ('date', utcnow.strftime('%Y-%m-%dT%H:%M:%S.0'))]
 
 
 @pytest.mark.usefixtures('log_schema')
 class TestAPILogsValidation(object):
     @pytest.mark.parametrize('dummy_json', ['', {}, [], None])
     def test_no_payload(self, dummy_json, log_schema):
         import colander
 
         with pytest.raises(colander.Invalid):
             log_schema.deserialize(dummy_json)
 
     def test_minimal_payload(self, log_schema):
         dummy_json = [{}]
         deserialized = log_schema.deserialize(dummy_json)[0]
         expected = {'log_level': 'UNKNOWN',
                     'namespace': '',
                     'server': 'unknown',
                     'request_id': '',
                     'primary_key': None,
                     'date': datetime.utcnow(),
                     'message': '',
                     'tags': None}
         assert deserialized['log_level'] == expected['log_level']
         assert deserialized['message'] == expected['message']
         assert deserialized['namespace'] == expected['namespace']
         assert deserialized['request_id'] == expected['request_id']
         assert deserialized['server'] == expected['server']
         assert deserialized['tags'] == expected['tags']
         assert deserialized['primary_key'] == expected['primary_key']
 
     def test_normal_payload(self, log_schema):
         import appenlight.tests.payload_examples as payload_examples
         deserialized = log_schema.deserialize(payload_examples.LOG_EXAMPLES)[0]
         expected = payload_examples.LOG_EXAMPLES[0]
         assert deserialized['log_level'] == expected['log_level']
         assert deserialized['message'] == expected['message']
         assert deserialized['namespace'] == expected['namespace']
         assert deserialized['request_id'] == expected['request_id']
         assert deserialized['server'] == expected['server']
         assert deserialized['date'].strftime('%Y-%m-%dT%H:%M:%S.%f') == \
                expected['date']
         assert deserialized['tags'][0][0] == "tag_name"
         assert deserialized['tags'][0][1] == "tag_value"
         assert deserialized['tags'][1][0] == "tag_name2"
         assert deserialized['tags'][1][1] == 2
 
     def test_normal_payload_date_without_microseconds(self, log_schema):
         import appenlight.tests.payload_examples as payload_examples
         LOG_EXAMPLE = copy.deepcopy(payload_examples.LOG_EXAMPLES)
         LOG_EXAMPLE[0]['date'] = datetime.utcnow().strftime(
             '%Y-%m-%dT%H:%M:%S')
         deserialized = log_schema.deserialize(LOG_EXAMPLE)
         assert deserialized[0]['date'].strftime('%Y-%m-%dT%H:%M:%S') == \
                LOG_EXAMPLE[0]['date']
 
     def test_normal_payload_date_without_seconds(self, log_schema):
         import appenlight.tests.payload_examples as payload_examples
         LOG_EXAMPLE = copy.deepcopy(payload_examples.LOG_EXAMPLES)
         LOG_EXAMPLE[0]['date'] = datetime.utcnow().date().strftime(
             '%Y-%m-%dT%H:%M')
         deserialized = log_schema.deserialize(LOG_EXAMPLE)
         assert deserialized[0]['date'].strftime('%Y-%m-%dT%H:%M') == \
                LOG_EXAMPLE[0]['date']
 
     def test_payload_empty_date(self, log_schema):
         import appenlight.tests.payload_examples as payload_examples
         LOG_EXAMPLE = copy.deepcopy(payload_examples.LOG_EXAMPLES)
         LOG_EXAMPLE[0]['date'] = None
         deserialized = log_schema.deserialize(LOG_EXAMPLE)
         assert deserialized[0]['date'].strftime('%Y-%m-%dT%H:%M') is not None
 
     def test_payload_no_date(self, log_schema):
         import appenlight.tests.payload_examples as payload_examples
         LOG_EXAMPLE = copy.deepcopy(payload_examples.LOG_EXAMPLES)
         LOG_EXAMPLE[0].pop('date', None)
         deserialized = log_schema.deserialize(LOG_EXAMPLE)
         assert deserialized[0]['date'].strftime('%Y-%m-%dT%H:%M') is not None
 
 
 @pytest.mark.usefixtures('general_metrics_schema')
 class TestAPIGeneralMetricsValidation(object):
     @pytest.mark.parametrize('dummy_json', ['', {}, [], None])
     def test_no_payload(self, dummy_json, general_metrics_schema):
         import colander
 
         with pytest.raises(colander.Invalid):
             general_metrics_schema.deserialize(dummy_json)
 
     def test_minimal_payload(self, general_metrics_schema):
         dummy_json = [{}]
         deserialized = general_metrics_schema.deserialize(dummy_json)[0]
         expected = {'namespace': '',
                     'server_name': 'unknown',
                     'tags': None,
                     'timestamp': datetime.utcnow()}
         assert deserialized['namespace'] == expected['namespace']
         assert deserialized['server_name'] == expected['server_name']
         assert deserialized['tags'] == expected['tags']
 
     def test_normal_payload(self, general_metrics_schema):
         import appenlight.tests.payload_examples as payload_examples
         dummy_json = [payload_examples.METRICS_PAYLOAD]
         deserialized = general_metrics_schema.deserialize(dummy_json)[0]
         expected = {'namespace': 'some.monitor',
                     'server_name': 'server.name',
                     'tags': [('usage_foo', 15.5), ('usage_bar', 63)],
                     'timestamp': datetime.utcnow()}
         assert deserialized['namespace'] == expected['namespace']
         assert deserialized['server_name'] == expected['server_name']
         assert deserialized['tags'] == expected['tags']
 
 
 @pytest.mark.usefixtures('request_metrics_schema')
 class TestAPIRequestMetricsValidation(object):
     @pytest.mark.parametrize('dummy_json', ['', {}, [], None])
     def test_no_payload(self, dummy_json, request_metrics_schema):
         import colander
 
         with pytest.raises(colander.Invalid):
             print(request_metrics_schema.deserialize(dummy_json))
 
     def test_normal_payload(self, request_metrics_schema):
         import appenlight.tests.payload_examples as payload_examples
         dummy_json = payload_examples.REQUEST_METRICS_EXAMPLES
         deserialized = request_metrics_schema.deserialize(dummy_json)[0]
         expected = {'metrics': [('dir/module:func',
                                  {'custom': 0.0,
                                   'custom_calls': 0.0,
                                   'main': 0.01664,
                                   'nosql': 0.00061,
                                   'nosql_calls': 23.0,
                                   'remote': 0.0,
                                   'remote_calls': 0.0,
                                   'requests': 1,
                                   'sql': 0.00105,
                                   'sql_calls': 2.0,
                                   'tmpl': 0.0,
                                   'tmpl_calls': 0.0}),
                                 ('SomeView.function',
                                  {'custom': 0.0,
                                   'custom_calls': 0.0,
                                   'main': 0.647261,
                                   'nosql': 0.306554,
                                   'nosql_calls': 140.0,
                                   'remote': 0.0,
                                   'remote_calls': 0.0,
                                   'requests': 28,
                                   'sql': 0.0,
                                   'sql_calls': 0.0,
                                   'tmpl': 0.0,
                                   'tmpl_calls': 0.0})],
                     'server': 'some.server.hostname',
                     'timestamp': datetime.utcnow()}
         assert deserialized['server'] == expected['server']
         metric = deserialized['metrics'][0]
         expected_metric = expected['metrics'][0]
         assert metric[0] == expected_metric[0]
         assert sorted(metric[1].items()) == sorted(expected_metric[1].items())
 
 
 @pytest.mark.usefixtures('default_application')
 @pytest.mark.usefixtures('base_app', 'with_migrations', 'clean_tables')
 class TestAPIReportsView(object):
     def test_no_json_payload(self, default_application):
         import colander
         from appenlight.models.services.application import ApplicationService
         from appenlight.views.api import reports_create
 
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request = testing.DummyRequest(
             headers={'Content-Type': 'application/json'})
         request.unsafe_json_body = ''
         request.context = context
         route = mock.Mock()
         route.name = 'api_reports'
         request.matched_route = route
         with pytest.raises(colander.Invalid):
             response = reports_create(request)
 
     def test_single_proper_json_0_5_payload(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.views.api import reports_create
         from appenlight.models.services.application import ApplicationService
         from appenlight.models.report_group import ReportGroup
         route = mock.Mock()
         route.name = 'api_reports'
         request = pyramid.threadlocal.get_current_request()
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request.context = context
         request.matched_route = route
         PYTHON_PAYLOAD = payload_examples.PYTHON_PAYLOAD_0_5
         request.unsafe_json_body = [copy.deepcopy(PYTHON_PAYLOAD)]
         reports_create(request)
         query = DBSession.query(ReportGroup)
         report = query.first()
         assert query.count() == 1
         assert report.total_reports == 1
 
     def test_grouping_0_5(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.views.api import reports_create
         from appenlight.models.services.application import ApplicationService
         from appenlight.models.report_group import ReportGroup
         route = mock.Mock()
         route.name = 'api_reports'
         request = pyramid.threadlocal.get_current_request()
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request.context = context
         request.matched_route = route
         PYTHON_PAYLOAD = payload_examples.PYTHON_PAYLOAD_0_5
         request.unsafe_json_body = [copy.deepcopy(PYTHON_PAYLOAD),
                                     copy.deepcopy(PYTHON_PAYLOAD)]
         reports_create(request)
         query = DBSession.query(ReportGroup)
         report = query.first()
         assert query.count() == 1
         assert report.total_reports == 2
 
     def test_grouping_different_reports_0_5(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.views.api import reports_create
         from appenlight.models.services.application import ApplicationService
         from appenlight.models.report_group import ReportGroup
         route = mock.Mock()
         route.name = 'api_reports'
         request = pyramid.threadlocal.get_current_request()
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request.context = context
         request.matched_route = route
         PYTHON_PAYLOAD = payload_examples.PYTHON_PAYLOAD_0_5
         PARSED_REPORT_404 = payload_examples.PARSED_REPORT_404
         request.unsafe_json_body = [copy.deepcopy(PYTHON_PAYLOAD),
                                     copy.deepcopy(PARSED_REPORT_404)]
         reports_create(request)
         query = DBSession.query(ReportGroup)
         report = query.first()
         assert query.count() == 2
         assert report.total_reports == 1
 
 
 @pytest.mark.usefixtures('default_application')
 @pytest.mark.usefixtures('base_app', 'with_migrations', 'clean_tables')
 class TestAirbrakeXMLView(object):
 
     def test_normal_payload_parsing(self):
         import datetime
         import defusedxml.ElementTree as ElementTree
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.lib.utils.airbrake import parse_airbrake_xml
         from appenlight.validators import ReportListSchema_0_5
 
         context = DummyContext()
         request = testing.DummyRequest(
             headers={'Content-Type': 'application/xml'})
         request.context = context
         request.context.possibly_public = False
         root = ElementTree.fromstring(payload_examples.AIRBRAKE_RUBY_EXAMPLE)
         request.context.airbrake_xml_etree = root
         error_dict = parse_airbrake_xml(request)
         schema = ReportListSchema_0_5().bind(utcnow=datetime.datetime.utcnow())
         deserialized_report = schema.deserialize([error_dict])[0]
         assert deserialized_report['client'] == 'Airbrake Notifier'
         assert deserialized_report['error'] == 'NameError: undefined local variable or method `sdfdfdf\' for #<#<Class:0x000000039a8b90>:0x00000002c53df0>'
         assert deserialized_report['http_status'] == 500
         assert deserialized_report['language'] == 'unknown'
         assert deserialized_report['message'] == ''
         assert deserialized_report['occurences'] == 1
         assert deserialized_report['priority'] == 5
         d_request = deserialized_report['request']
         assert d_request['GET'] == {'test': '1234'}
         assert d_request['action_dispatch.request.parameters'] == {
             'action': 'index',
             'controller': 'welcome',
             'test': '1234'}
         assert deserialized_report['request_id'] == 'c11b2267f3ad8b00a1768cae35559fa1'
         assert deserialized_report['server'] == 'ergo-desktop'
         assert deserialized_report['traceback'][0] == {
             'cline': 'block in start_thread',
             'file': '/home/ergo/.rbenv/versions/1.9.3-p327/lib/ruby/1.9.1/webrick/server.rb',
             'fn': 'block in start_thread',
             'line': '191',
             'module': '',
             'vars': {}}
         assert deserialized_report['traceback'][-1] == {
             'cline': '_app_views_welcome_index_html_erb___2570061166873166679_31748940',
             'file': '[PROJECT_ROOT]/app/views/welcome/index.html.erb',
             'fn': '_app_views_welcome_index_html_erb___2570061166873166679_31748940',
             'line': '3',
             'module': '',
             'vars': {}}
         assert deserialized_report['url'] == 'http://0.0.0.0:3000/welcome/index?test=1234'
         assert deserialized_report['view_name'] == 'welcome:index'
 
     def test_normal_payload_view(self):
         import defusedxml.ElementTree as ElementTree
         import appenlight.tests.payload_examples as payload_examples
 
         from appenlight.models.services.application import ApplicationService
         from appenlight.views.api import airbrake_xml_compat
 
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request = testing.DummyRequest(
             headers={'Content-Type': 'application/xml'})
         request.context = context
         request.context.possibly_public = False
         root = ElementTree.fromstring(payload_examples.AIRBRAKE_RUBY_EXAMPLE)
         request.context.airbrake_xml_etree = root
         route = mock.Mock()
         route.name = 'api_airbrake'
         request.matched_route = route
         result = airbrake_xml_compat(request)
         assert '<notice><id>' in result
 
 
 @pytest.mark.usefixtures('default_application')
 @pytest.mark.usefixtures('base_app', 'with_migrations', 'clean_tables')
 class TestAPILogView(object):
     def test_no_json_payload(self, base_app):
         import colander
         from appenlight.models.services.application import ApplicationService
         from appenlight.views.api import logs_create
 
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request = testing.DummyRequest(
             headers={'Content-Type': 'application/json'})
         request.context = context
         request.registry = base_app.registry
         request.unsafe_json_body = ''
         route = mock.Mock()
         route.name = 'api_logs'
         request.matched_route = route
         with pytest.raises(colander.Invalid):
             response = logs_create(request)
 
     def test_single_json_payload(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.models.log import Log
         from appenlight.views.api import logs_create
         from appenlight.models.services.application import ApplicationService
         route = mock.Mock()
         route.name = 'api_logs'
         request = pyramid.threadlocal.get_current_request()
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request.context = context
         request.matched_route = route
         request.unsafe_json_body = [copy.deepcopy(
             payload_examples.LOG_EXAMPLES[0])]
         logs_create(request)
         query = DBSession.query(Log)
         log = query.first()
         assert query.count() == 1
         assert log.message == "OMG ValueError happened"
 
     def test_multiple_json_payload(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.models.log import Log
         from appenlight.views.api import logs_create
         from appenlight.models.services.application import ApplicationService
         route = mock.Mock()
         route.name = 'api_logs'
         request = pyramid.threadlocal.get_current_request()
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request.context = context
         request.matched_route = route
         LOG_PAYLOAD = payload_examples.LOG_EXAMPLES[0]
         LOG_PAYLOAD2 = payload_examples.LOG_EXAMPLES[1]
         request.unsafe_json_body = copy.deepcopy([LOG_PAYLOAD, LOG_PAYLOAD2])
         logs_create(request)
         query = DBSession.query(Log).order_by(sa.asc(Log.log_id))
         assert query.count() == 2
         assert query[0].message == "OMG ValueError happened"
         assert query[1].message == "OMG ValueError happened2"
 
     def test_public_key_rewriting(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.models.log import Log
         from appenlight.views.api import logs_create
         from appenlight.models.services.application import ApplicationService
         route = mock.Mock()
         route.name = 'api_logs'
         request = pyramid.threadlocal.get_current_request()
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request.context = context
         request.matched_route = route
 
         LOG_PAYLOAD = copy.deepcopy(payload_examples.LOG_EXAMPLES[0])
         LOG_PAYLOAD2 = copy.deepcopy(payload_examples.LOG_EXAMPLES[1])
         LOG_PAYLOAD['primary_key'] = 'X2'
         LOG_PAYLOAD2['primary_key'] = 'X2'
         request.unsafe_json_body = [LOG_PAYLOAD, LOG_PAYLOAD2]
         logs_create(request)
 
         query = DBSession.query(Log).order_by(sa.asc(Log.log_id))
         assert query.count() == 1
         assert query[0].message == "OMG ValueError happened2"
 
 @pytest.mark.usefixtures('default_application')
 @pytest.mark.usefixtures('base_app', 'with_migrations', 'clean_tables')
 class TestAPIGeneralMetricsView(object):
     def test_no_json_payload(self, base_app):
         import colander
         from appenlight.models.services.application import ApplicationService
         from appenlight.views.api import general_metrics_create
         route = mock.Mock()
         route.name = 'api_general_metrics'
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request = testing.DummyRequest(
             headers={'Content-Type': 'application/json'})
         request.context = context
         request.registry = base_app.registry
         request.unsafe_json_body = ''
         request.matched_route = route
         with pytest.raises(colander.Invalid):
             general_metrics_create(request)
 
     def test_single_json_payload(self):
         import appenlight.tests.payload_examples as payload_examples
-        from appenlight.models.request_metric import Metric
+        from appenlight.models.metric import Metric
         from appenlight.views.api import general_metrics_create
         from appenlight.models.services.application import ApplicationService
         route = mock.Mock()
         route.name = 'api_general_metric'
         request = pyramid.threadlocal.get_current_request()
         request.matched_route = route
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request.context = context
         request.unsafe_json_body = payload_examples.METRICS_PAYLOAD
         general_metrics_create(request)
         query = DBSession.query(Metric)
         metric = query.first()
         assert query.count() == 1
         assert metric.namespace == 'some.monitor'
 
     def test_multiple_json_payload(self):
         import appenlight.tests.payload_examples as payload_examples
-        from appenlight.models.request_metric import Metric
+        from appenlight.models.metric import Metric
         from appenlight.views.api import general_metrics_create
         from appenlight.models.services.application import ApplicationService
         route = mock.Mock()
         route.name = 'api_general_metrics'
         request = pyramid.threadlocal.get_current_request()
         request.matched_route = route
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request.context = context
         request.unsafe_json_body = [
             copy.deepcopy(payload_examples.METRICS_PAYLOAD),
             copy.deepcopy(payload_examples.METRICS_PAYLOAD),
         ]
         general_metrics_create(request)
         query = DBSession.query(Metric)
         metric = query.first()
         assert query.count() == 2
         assert metric.namespace == 'some.monitor'
 
 
 class TestGroupingMessageReplacements(object):
     def replace_default_repr_python(self):
         test_str = '''
         ConnectionError: ConnectionError((<urllib3.connection.HTTPConnection object at 0x7f87a0ba9fd0>, 'Connection to domain.gr timed out. (connect timeout=10)')) caused by: ConnectTimeoutError((<urllib3.connection.HTTPConnection object at 0x7f87a0ba9fd0>, 'Connection to domain.gr timed out. (connect timeout=10)'))
         '''
         regex = r'<(.*?) object at (.*?)>'
 
 
 class TestRulesKeyGetter(object):
     def test_default_dict_getter_top_key(self):
         from appenlight.lib.rule import Rule
         struct = {
             "a": {
                 "b": 'b',
                 "c": {
                     "d": 'd',
                     "g": {
                         "h": 'h'
                     }
                 },
                 "e": 'e'
             },
             "f": 'f'
         }
         result = Rule.default_dict_struct_getter(struct, "a")
         assert result == struct['a']
 
     def test_default_dict_getter_sub_key(self):
         from appenlight.lib.rule import Rule
         struct = {
             "a": {
                 "b": 'b',
                 "c": {
                     "d": 'd',
                     "g": {
                         "h": 'h'
                     }
                 },
                 "e": 'e'
             },
             "f": 'f'
         }
         result = Rule.default_dict_struct_getter(struct, 'a:b')
         assert result == struct['a']['b']
         result = Rule.default_dict_struct_getter(struct, 'a:c:d')
         assert result == struct['a']['c']['d']
 
     def test_default_obj_getter_top_key(self):
         from appenlight.lib.rule import Rule
         class TestStruct(object):
             def __init__(self, a, b):
                 self.a = a
                 self.b = b
 
         struct = TestStruct(a='a',
                             b=TestStruct(a='x', b='y'))
         result = Rule.default_obj_struct_getter(struct, "a")
         assert result == struct.a
 
     def test_default_obj_getter_sub_key(self):
         from appenlight.lib.rule import Rule
         class TestStruct(object):
             def __init__(self, name, a, b):
                 self.name = name
                 self.a = a
                 self.b = b
 
             def __repr__(self):
                 return '<obj {}>'.format(self.name)
 
         c = TestStruct('c', a=5, b='z')
         b = TestStruct('b', a=c, b='y')
         struct = TestStruct('a', a='a', b=b)
         result = Rule.default_obj_struct_getter(struct, 'b:b')
         assert result == struct.b.b
         result = Rule.default_obj_struct_getter(struct, 'b:a:b')
         assert result == struct.b.a.b
 
 
 @pytest.mark.usefixtures('report_type_matrix')
 class TestRulesParsing():
     @pytest.mark.parametrize("op, struct_value, test_value, match_result", [
         ('eq', 500, 500, True),
         ('eq', 600, 500, False),
         ('eq', 300, 500, False),
         ('eq', "300", 500, False),
         ('eq', "600", 500, False),
         ('eq', "500", 500, True),
         ('ne', 500, 500, False),
         ('ne', 600, 500, True),
         ('ne', 300, 500, True),
         ('ne', "300", 500, True),
         ('ne', "600", 500, True),
         ('ne', "500", 500, False),
         ('ge', 500, 500, True),
         ('ge', 600, 500, True),
         ('ge', 499, 500, False),
         ('gt', 499, 500, False),
         ('gt', 500, 500, False),
         ('gt', 501, 500, True),
         ('le', 499, 500, True),
         ('le', 500, 500, True),
         ('le', 501, 500, False),
         ('lt', 499, 500, True),
         ('lt', 500, 500, False),
         ('lt', 501, 500, False),
     ])
     def test_single_op_int(self, op, struct_value, test_value, match_result,
                            report_type_matrix):
         from appenlight.lib.rule import Rule
         rule_config = {
             "op": op,
             "field": "http_status",
             "value": test_value
         }
         rule = Rule(rule_config, report_type_matrix)
 
         data = {
             "http_status": struct_value
         }
         assert rule.match(data) is match_result
 
     @pytest.mark.parametrize("op, struct_value, test_value, match_result", [
         ('ge', "500.01", 500, True),
         ('ge', "500.01", 500.02, False),
         ('le', "500.01", 500.02, True)
     ])
     def test_single_op_float(self, op, struct_value, test_value, match_result,
                              report_type_matrix):
         from appenlight.lib.rule import Rule
         rule_config = {
             "op": op,
             "field": "duration",
             "value": test_value
         }
         rule = Rule(rule_config, report_type_matrix)
 
         data = {
             "duration": struct_value
         }
         assert rule.match(data) is match_result
 
     @pytest.mark.parametrize("op, struct_value, test_value, match_result", [
         ('contains', 'foo bar baz', 'foo', True),
         ('contains', 'foo bar baz', 'bar', True),
         ('contains', 'foo bar baz', 'dupa', False),
         ('startswith', 'foo bar baz', 'foo', True),
         ('startswith', 'foo bar baz', 'bar', False),
         ('endswith', 'foo bar baz', 'baz', True),
         ('endswith', 'foo bar baz', 'bar', False),
     ])
     def test_single_op_string(self, op, struct_value, test_value,
                               match_result, report_type_matrix):
         from appenlight.lib.rule import Rule
         rule_config = {
             "op": op,
             "field": "error",
             "value": test_value
         }
         rule = Rule(rule_config, report_type_matrix)
 
         data = {
             "error": struct_value
         }
         assert rule.match(data) is match_result
 
     @pytest.mark.parametrize("field, value, s_type", [
         ('field_unicode', 500, str),
         ('field_unicode', 500.0, str),
         ('field_unicode', "500", str),
         ('field_int', "500", int),
         ('field_int', 500, int),
         ('field_int', 500.0, int),
         ('field_float', "500", float),
         ('field_float', 500, float),
         ('field_float', 500.0, float),
     ])
     def test_type_normalization(self, field, value, s_type):
         from appenlight.lib.rule import Rule
         type_matrix = {
             'field_unicode': {"type": 'unicode'},
             'field_float': {"type": 'float'},
             'field_int': {"type": 'int'},
         }
 
         rule = Rule({}, type_matrix)
         n_value = rule.normalized_type(field, value)
         assert isinstance(n_value, s_type) is True
 
 
 @pytest.mark.usefixtures('report_type_matrix')
 class TestNestedRuleParsing():
 
     @pytest.mark.parametrize("data, result", [
         ({"http_status": 501, "group": {"priority": 7, "occurences": 11}},
          False),
         ({"http_status": 101, "group": {"priority": 7, "occurences": 11}},
          False),
         ({"http_status": 500, "group": {"priority": 1, "occurences": 11}},
          False),
         ({"http_status": 101, "group": {"priority": 3, "occurences": 5}},
          True),
     ])
     def test_NOT_rule(self, data, result, report_type_matrix):
         from appenlight.lib.rule import Rule
         rule_config = {
             "field": "__NOT__",
             "rules": [
                 {
                     "op": "ge",
                     "field": "group:occurences",
                     "value": "10"
                 },
                 {
                     "op": "ge",
                     "field": "group:priority",
                     "value": "4"
                 }
             ]
         }
 
         rule = Rule(rule_config, report_type_matrix)
         assert rule.match(data) is result
 
     @pytest.mark.parametrize("data, result", [
         ({"http_status": 501, "group": {"priority": 7, "occurences": 11}},
          True),
         ({"http_status": 101, "group": {"priority": 7, "occurences": 11}},
          True),
         ({"http_status": 500, "group": {"priority": 1, "occurences": 1}},
          True),
         ({"http_status": 101, "group": {"priority": 3, "occurences": 11}},
          False),
     ])
     def test_nested_OR_AND_rule(self, data, result, report_type_matrix):
         from appenlight.lib.rule import Rule
         rule_config = {
             "field": "__OR__",
             "rules": [
                 {
                     "field": "__AND__",
                     "rules": [
                         {
                             "op": "ge",
                             "field": "group:occurences",
                             "value": "10"
                         },
                         {
                             "op": "ge",
                             "field": "group:priority",
                             "value": "4"
                         }
                     ]
                 },
                 {
                     "op": "eq",
                     "field": "http_status",
                     "value": "500"
                 }
             ]
         }
 
         rule = Rule(rule_config, report_type_matrix)
         assert rule.match(data) is result
 
     @pytest.mark.parametrize("data, result", [
         ({"http_status": 501, "group": {"priority": 7, "occurences": 11}},
          True),
         ({"http_status": 101, "group": {"priority": 7, "occurences": 11}},
          True),
         ({"http_status": 500, "group": {"priority": 1, "occurences": 1}},
          True),
         ({"http_status": 101, "group": {"priority": 3, "occurences": 1}},
          False),
     ])
     def test_nested_OR_OR_rule(self, data, result, report_type_matrix):
         from appenlight.lib.rule import Rule
         rule_config = {
             "field": "__OR__",
             "rules": [
                 {"field": "__OR__",
                  "rules": [
                      {"op": "ge",
                       "field": "group:occurences",
                       "value": "10"
                       },
                      {"op": "ge",
                       "field": "group:priority",
                       "value": "4"
                       }
                  ]
                  },
                 {"op": "eq",
                  "field": "http_status",
                  "value": "500"
                  }
             ]
         }
 
         rule = Rule(rule_config, report_type_matrix)
         assert rule.match(data) is result
 
     @pytest.mark.parametrize("data, result", [
         ({"http_status": 500, "group": {"priority": 7, "occurences": 11}},
          True),
         ({"http_status": 101, "group": {"priority": 7, "occurences": 11}},
          False),
         ({"http_status": 500, "group": {"priority": 1, "occurences": 1}},
          False),
         ({"http_status": 101, "group": {"priority": 3, "occurences": 1}},
          False),
     ])
     def test_nested_AND_AND_rule(self, data, result, report_type_matrix):
         from appenlight.lib.rule import Rule
         rule_config = {
             "field": "__AND__",
             "rules": [
                 {"field": "__AND__",
                  "rules": [
                      {"op": "ge",
                       "field": "group:occurences",
                       "value": "10"
                       },
                      {"op": "ge",
                       "field": "group:priority",
                       "value": "4"
                       }]
                  },
                 {"op": "eq",
                  "field": "http_status",
                  "value": "500"
                  }
             ]
         }
 
         rule = Rule(rule_config, report_type_matrix)
         assert rule.match(data) is result
 
     @pytest.mark.parametrize("data, result", [
         ({"http_status": 500, "group": {"priority": 7, "occurences": 11},
           "url_path": '/test/register', "error": "foo test bar"}, True),
         ({"http_status": 500, "group": {"priority": 7, "occurences": 11},
           "url_path": '/test/register', "error": "foo INVALID bar"}, False),
     ])
     def test_nested_AND_AND_AND_rule(self, data, result, report_type_matrix):
         from appenlight.lib.rule import Rule
         rule_config = {
             "field": "__AND__",
             "rules": [
                 {"field": "__AND__",
                  "rules": [
                      {"op": "ge",
                       "field": "group:occurences",
                       "value": "10"
                       },
                      {"field": "__AND__",
                       "rules": [
                           {"op": "endswith",
                            "field": "url_path",
                            "value": "register"},
                           {"op": "contains",
                            "field": "error",
                            "value": "test"}]}]
                  },
                 {"op": "eq",
                  "field": "http_status",
                  "value": "500"
                  }
             ]
         }
 
         rule = Rule(rule_config, report_type_matrix)
         assert rule.match(data) is result
 
     @pytest.mark.parametrize("data, result", [
         ({"http_status": 500, "group": {"priority": 7, "occurences": 11},
           "url_path": 6, "error": 3}, False),
         ({"http_status": 500, "group": {"priority": 7, "occurences": 11},
           "url_path": '/test/register', "error": "foo INVALID bar"}, True),
     ])
     def test_nested_AND_AND_OR_rule(self, data, result, report_type_matrix):
         from appenlight.lib.rule import Rule
         rule_config = {
             "field": "__AND__",
             "rules": [
                 {"field": "__AND__",
                  "rules": [
                      {"op": "ge",
                       "field": "group:occurences",
                       "value": "10"
                       },
                      {"field": "__OR__",
                       "rules": [
                           {"op": "endswith",
                            "field": "url_path",
                            "value": "register"
                            },
                           {"op": "contains",
                            "field": "error",
                            "value": "test"
                            }]}]
                  },
                 {"op": "eq",
                  "field": "http_status",
                  "value": "500"
                  }
             ]
         }
 
         rule = Rule(rule_config, report_type_matrix)
         assert rule.match(data) is result
 
     @pytest.mark.parametrize("op, field, value, should_fail", [
         ('eq', 'http_status', "1", False),
         ('ne', 'http_status', "1", False),
         ('ne', 'http_status', "foo", True),
         ('startswith', 'http_status', "1", True),
         ('eq', 'group:priority', "1", False),
         ('ne', 'group:priority', "1", False),
         ('ge', 'group:priority', "1", False),
         ('le', 'group:priority', "1", False),
         ('startswith', 'group:priority', "1", True),
         ('eq', 'url_domain', "1", False),
         ('ne', 'url_domain', "1", False),
         ('startswith', 'url_domain', "1", False),
         ('endswith', 'url_domain', "1", False),
         ('contains', 'url_domain', "1", False),
         ('ge', 'url_domain', "1", True),
         ('eq', 'url_path', "1", False),
         ('ne', 'url_path', "1", False),
         ('startswith', 'url_path', "1", False),
         ('endswith', 'url_path', "1", False),
         ('contains', 'url_path', "1", False),
         ('ge', 'url_path', "1", True),
         ('eq', 'error', "1", False),
         ('ne', 'error', "1", False),
         ('startswith', 'error', "1", False),
         ('endswith', 'error', "1", False),
         ('contains', 'error', "1", False),
         ('ge', 'error', "1", True),
         ('ge', 'url_path', "1", True),
         ('eq', 'tags:server_name', "1", False),
         ('ne', 'tags:server_name', "1", False),
         ('startswith', 'tags:server_name', "1", False),
         ('endswith', 'tags:server_name', "1", False),
         ('contains', 'tags:server_name', "1", False),
         ('ge', 'tags:server_name', "1", True),
         ('contains', 'traceback', "1", False),
         ('ge', 'traceback', "1", True),
         ('eq', 'group:occurences', "1", False),
         ('ne', 'group:occurences', "1", False),
         ('ge', 'group:occurences', "1", False),
         ('le', 'group:occurences', "1", False),
         ('contains', 'group:occurences', "1", True),
     ])
     def test_rule_validation(self, op, field, value, should_fail,
                              report_type_matrix):
         import colander
         from appenlight.validators import build_rule_schema
         rule_config = {
             "op": op,
             "field": field,
             "value": value
         }
 
         schema = build_rule_schema(rule_config, report_type_matrix)
         if should_fail:
             with pytest.raises(colander.Invalid):
                 schema.deserialize(rule_config)
         else:
             schema.deserialize(rule_config)
 
     def test_nested_proper_rule_validation(self, report_type_matrix):
         from appenlight.validators import build_rule_schema
         rule_config = {
             "field": "__AND__",
             "rules": [
                 {
                     "field": "__AND__",
                     "rules": [
                         {
                             "op": "ge",
                             "field": "group:occurences",
                             "value": "10"
                         },
                         {
                             "field": "__OR__",
                             "rules": [
                                 {
                                     "op": "endswith",
                                     "field": "url_path",
                                     "value": "register"
                                 },
                                 {
                                     "op": "contains",
                                     "field": "error",
                                     "value": "test"
                                 }
                             ]
                         }
                     ]
                 },
                 {
                     "op": "eq",
                     "field": "http_status",
                     "value": "500"
                 }
             ]
         }
 
         schema = build_rule_schema(rule_config, report_type_matrix)
         deserialized = schema.deserialize(rule_config)
 
     def test_nested_bad_rule_validation(self, report_type_matrix):
         import colander
         from appenlight.validators import build_rule_schema
         rule_config = {
             "field": "__AND__",
             "rules": [
                 {
                     "field": "__AND__",
                     "rules": [
                         {
                             "op": "ge",
                             "field": "group:occurences",
                             "value": "10"
                         },
                         {
                             "field": "__OR__",
                             "rules": [
                                 {
                                     "op": "gt",
                                     "field": "url_path",
                                     "value": "register"
                                 },
                                 {
                                     "op": "contains",
                                     "field": "error",
                                     "value": "test"
                                 }
                             ]
                         }
                     ]
                 },
                 {
                     "op": "eq",
                     "field": "http_status",
                     "value": "500"
                 }
             ]
         }
 
         schema = build_rule_schema(rule_config, report_type_matrix)
         with pytest.raises(colander.Invalid):
             deserialized = schema.deserialize(rule_config)
 
     def test_config_manipulator(self):
         from appenlight.lib.rule import Rule
         type_matrix = {
             'a': {"type": 'int',
                   "ops": ('eq', 'ne', 'ge', 'le',)},
             'b': {"type": 'int',
                   "ops": ('eq', 'ne', 'ge', 'le',)},
         }
         rule_config = {
             "field": "__OR__",
             "rules": [
                 {
                     "field": "__OR__",
                     "rules": [
                         {
                             "op": "ge",
                             "field": "a",
                             "value": "10"
                         }
                     ]
                 },
                 {
                     "op": "eq",
                     "field": "b",
                     "value": "500"
                 }
             ]
         }
 
         def rule_manipulator(rule):
             if 'value' in rule.config:
                 rule.config['value'] = "1"
 
         rule = Rule(rule_config, type_matrix,
                     config_manipulator=rule_manipulator)
         rule.match({"a": 1,
                     "b": "2"})
         assert rule.config['rules'][0]['rules'][0]['value'] == "1"
         assert rule.config['rules'][1]['value'] == "1"
         assert rule.type_matrix["b"]['type'] == "int"
 
     def test_dynamic_config_manipulator(self):
         from appenlight.lib.rule import Rule
         rule_config = {
             "field": "__OR__",
             "rules": [
                 {
                     "field": "__OR__",
                     "rules": [
                         {
                             "op": "ge",
                             "field": "a",
                             "value": "10"
                         }
                     ]
                 },
                 {
                     "op": "eq",
                     "field": "b",
                     "value": "500"
                 }
             ]
         }
 
         def rule_manipulator(rule):
             rule.type_matrix = {
                 'a': {"type": 'int',
                       "ops": ('eq', 'ne', 'ge', 'le',)},
                 'b': {"type": 'unicode',
                       "ops": ('eq', 'ne', 'ge', 'le',)},
             }
 
             if 'value' in rule.config:
                 if rule.config['field'] == 'a':
                     rule.config['value'] = "1"
                 elif rule.config['field'] == 'b':
                     rule.config['value'] = "2"
 
         rule = Rule(rule_config, {},
                     config_manipulator=rule_manipulator)
         rule.match({"a": 11,
                     "b": "55"})
         assert rule.config['rules'][0]['rules'][0]['value'] == "1"
         assert rule.config['rules'][1]['value'] == "2"
         assert rule.type_matrix["b"]['type'] == "unicode"
 
 
 @pytest.mark.usefixtures('base_app', 'with_migrations')
 class TestViewsWithForms(object):
     def test_bad_csrf(self):
         from appenlight.forms import CSRFException
         from appenlight.views.index import register
         post_data = {'dupa': 'dupa'}
         request = testing.DummyRequest(post=post_data)
         request.POST = webob.multidict.MultiDict(request.POST)
         with pytest.raises(CSRFException):
             register(request)
 
     def test_proper_csrf(self):
         from appenlight.views.index import register
         request = pyramid.threadlocal.get_current_request()
         post_data = {'dupa': 'dupa',
                      'csrf_token': request.session.get_csrf_token()}
         request = testing.DummyRequest(post=post_data)
         request.POST = webob.multidict.MultiDict(request.POST)
         result = register(request)
         assert result['form'].errors['email'][0] == 'This field is required.'
 
 
 @pytest.mark.usefixtures('base_app', 'with_migrations', 'default_data')
 class TestRegistration(object):
     def test_invalid_form(self):
         from appenlight.views.index import register
         request = pyramid.threadlocal.get_current_request()
         post_data = {'user_name': '',
                      'user_password': '',
                      'email': '',
                      'csrf_token': request.session.get_csrf_token()}
         request = testing.DummyRequest(post=post_data)
         request.POST = webob.multidict.MultiDict(request.POST)
         result = register(request)
         assert result['form'].errors['user_name'][0] == \
                'This field is required.'
 
     def test_valid_form(self):
         from appenlight.views.index import register
         from ziggurat_foundations.models.services.user import UserService
         request = pyramid.threadlocal.get_current_request()
         post_data = {'user_name': 'foo',
                      'user_password': 'barr',
                      'email': '[e-mail]',
                      'csrf_token': request.session.get_csrf_token()}
         request = testing.DummyRequest(post=post_data)
         request.add_flash_to_headers = mock.Mock()
         request.POST = webob.multidict.MultiDict(request.POST)
         assert UserService.by_user_name('foo') is None
         register(request)
         user = UserService.by_user_name('foo')
         assert user.user_name == 'foo'
         assert len(user.user_password) == 60
 
 
 @pytest.mark.usefixtures('base_app', 'with_migrations', 'clean_tables',
                          'default_user')
 class TestApplicationCreation(object):
     def test_wrong_data(self):
         import appenlight.views.applications as applications
         from ziggurat_foundations.models.services.user import UserService
         request = pyramid.threadlocal.get_current_request()
         request.user = UserService.by_user_name('testuser')
         request.unsafe_json_body = {}
         request.headers['X-XSRF-TOKEN'] = request.session.get_csrf_token()
         response = applications.application_create(request)
         assert response.code == 422
 
     def test_proper_data(self):
         import appenlight.views.applications as applications
         from ziggurat_foundations.models.services.user import UserService
 
         request = pyramid.threadlocal.get_current_request()
         request.user = UserService.by_user_name('testuser')
         request.unsafe_json_body = {"resource_name": "app name",
                                     "domains": "foo"}
         request.headers['X-XSRF-TOKEN'] = request.session.get_csrf_token()
         app_dict = applications.application_create(request)
         assert app_dict['public_key'] is not None
         assert app_dict['api_key'] is not None
         assert app_dict['resource_name'] == 'app name'
         assert app_dict['owner_group_id'] is None
         assert app_dict['resource_id'] is not None
         assert app_dict['default_grouping'] == 'url_traceback'
         assert app_dict['possible_permissions'] == ('view', 'update_reports')
         assert app_dict['slow_report_threshold'] == 10
         assert app_dict['owner_user_name'] == 'testuser'
         assert app_dict['owner_user_id'] == request.user.id
         assert app_dict['domains'] is 'foo'
         assert app_dict['postprocessing_rules'] == []
         assert app_dict['error_report_threshold'] == 10
         assert app_dict['allow_permanent_storage'] is False
         assert app_dict['resource_type'] == 'application'
         assert app_dict['current_permissions'] == []
 
 
 @pytest.mark.usefixtures('default_application')
 @pytest.mark.usefixtures('base_app', 'with_migrations', 'clean_tables')
 class TestAPISentryView(object):
     def test_no_payload(self, default_application):
         import colander
         from appenlight.models.services.application import ApplicationService
         from appenlight.views.api import sentry_compat
         from appenlight.lib.request import JSONException
 
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request = testing.DummyRequest(
             headers={'Content-Type': 'application/json'})
         request.unsafe_json_body = ''
         request.context = context
         route = mock.Mock()
         route.name = 'api_sentry'
         request.matched_route = route
         with pytest.raises(JSONException):
             sentry_compat(request)
 
     def test_java_client_payload(self):
         from appenlight.views.api import sentry_compat
         from appenlight.models.services.application import ApplicationService
         from appenlight.models.report_group import ReportGroup
         route = mock.Mock()
         route.name = 'api_sentry'
         request = pyramid.threadlocal.get_current_request()
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request.context = context
         request.matched_route = route
         request.body = b'eJy1UmFr2zAQ/S0T+7BCLOzYThp/C6xjG6SDLd/GCBf57Ki' \
                        b'RJSHJJiXkv+/UlC7p2kAZA33Ru6f33t1pz3BAHVayZhWr87' \
                        b'JMs+I6q3MsrifFep2vc1iXM1HMpgBTNmIdeg8tEvlmJ9AGa' \
                        b'fQ7goOkQoDOUmGcZpMkLZO0WGZFRadMiaHIR1EVnTMu3k3b' \
                        b'oiMgqJrXpgOpOVjLLTiPkWAVhMa4jih3MAAholfWyUDAksz' \
                        b'm1iopICbg8fWH52B8VWXZVYwHrWfV/jBipD2gW2no8CFMa5' \
                        b'JButCDSjoQG6mR6LgLDojPPn/7sbydL25ep34HGl+y3DiE+' \
                        b'lH0xXBXjMzFBsXW99SS7pWKYXRw91zqgK4BgZ4/DZVVP/cs' \
                        b'3NuzSZPfAKqP2Cdj4tw7U/cKH0fEFeiWQFqE2FIHAmMPjaN' \
                        b'Y/kHvbzY/JqdHUq9o/KxqQHkcsabX4piDuT4aK+pXG1ZNi/' \
                        b'IwOpEyruXC1LiB3vPO3BmOOxTUCIqv5LIg5H12oh9cf0l+P' \
                        b'MvP5P8kddgoFIEvMGzM5cRSD2aLJ6qTdHKm6nv9pPcRFba0' \
                        b'Kd0eleeCFuGN+9JZ9TaXIn/V5JYMBvxXg3L6PwzSE4dkfOb' \
                        b'w7CtfWmP85SdCs8OvA53fUV19cg=='
         sentry_compat(request)
         query = DBSession.query(ReportGroup)
         report = query.first()
         assert query.count() == 1
         assert report.total_reports == 1
 
     def test_ruby_client_payload(self):
         from appenlight.views.api import sentry_compat
         from appenlight.models.services.application import ApplicationService
         from appenlight.models.report_group import ReportGroup
         from appenlight.tests.payload_examples import SENTRY_RUBY_ENCODED
         route = mock.Mock()
         route.name = 'api_sentry'
         request = testing.DummyRequest(
             headers={'Content-Type': 'application/octet-stream',
                      'User-Agent': 'sentry-ruby/1.0.0',
                      'X-Sentry-Auth': 'Sentry sentry_version=5, '
                                       'sentry_client=raven-ruby/1.0.0, '
                                       'sentry_timestamp=1462378483, '
                                       'sentry_key=xxx, sentry_secret=xxx'
                      })
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request.context = context
         request.matched_route = route
         request.body = SENTRY_RUBY_ENCODED
         sentry_compat(request)
         query = DBSession.query(ReportGroup)
         report = query.first()
         assert query.count() == 1
         assert report.total_reports == 1
 
     def test_python_client_decoded_payload(self):
         from appenlight.views.api import sentry_compat
         from appenlight.models.services.application import ApplicationService
         from appenlight.models.report_group import ReportGroup
         from appenlight.tests.payload_examples import SENTRY_PYTHON_PAYLOAD_7
         route = mock.Mock()
         route.name = 'api_sentry'
         request = pyramid.threadlocal.get_current_request()
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request.context = context
         request.matched_route = route
         request.body = json.dumps(SENTRY_PYTHON_PAYLOAD_7).encode('utf8')
         sentry_compat(request)
         query = DBSession.query(ReportGroup)
         report = query.first()
         assert query.count() == 1
         assert report.total_reports == 1
 
     def test_python_client_encoded_payload(self):
         from appenlight.views.api import sentry_compat
         from appenlight.models.services.application import ApplicationService
         from appenlight.models.report_group import ReportGroup
         from appenlight.tests.payload_examples import SENTRY_PYTHON_ENCODED
         route = mock.Mock()
         route.name = 'api_sentry'
         request = testing.DummyRequest(
             headers={'Content-Type': 'application/octet-stream',
                      'Content-Encoding': 'deflate',
                      'User-Agent': 'sentry-ruby/1.0.0',
                      'X-Sentry-Auth': 'Sentry sentry_version=5, '
                                       'sentry_client=raven-ruby/1.0.0, '
                                       'sentry_timestamp=1462378483, '
                                       'sentry_key=xxx, sentry_secret=xxx'
                      })
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request.context = context
         request.matched_route = route
         request.body = SENTRY_PYTHON_ENCODED
         sentry_compat(request)
         query = DBSession.query(ReportGroup)
         report = query.first()
         assert query.count() == 1
         assert report.total_reports == 1