|
|
# -*- coding: utf-8 -*-
|
|
|
|
|
|
# Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
|
|
|
#
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
# you may not use this file except in compliance with the License.
|
|
|
# You may obtain a copy of the License at
|
|
|
#
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
#
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
# See the License for the specific language governing permissions and
|
|
|
# limitations under the License.
|
|
|
|
|
|
import logging
|
|
|
import paginate
|
|
|
import sqlalchemy as sa
|
|
|
import appenlight.lib.helpers as h
|
|
|
|
|
|
from datetime import datetime
|
|
|
|
|
|
from appenlight.models import get_db_session, Datastores
|
|
|
from appenlight.models.report import Report
|
|
|
from appenlight.models.report_group import ReportGroup
|
|
|
from appenlight.models.report_comment import ReportComment
|
|
|
from appenlight.models.user import User
|
|
|
from appenlight.models.services.base import BaseService
|
|
|
from appenlight.lib.enums import ReportType
|
|
|
from appenlight.lib.utils import es_index_name_limiter
|
|
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
|
class ReportGroupService(BaseService):
|
|
|
@classmethod
|
|
|
def get_trending(cls, request, filter_settings, limit=15,
|
|
|
db_session=None):
|
|
|
"""
|
|
|
Returns report groups trending for specific time interval
|
|
|
"""
|
|
|
db_session = get_db_session(db_session)
|
|
|
|
|
|
tags = []
|
|
|
if filter_settings.get('tags'):
|
|
|
for tag in filter_settings['tags']:
|
|
|
tags.append(
|
|
|
{'terms': {
|
|
|
'tags.{}.values'.format(tag['name']): tag['value']}})
|
|
|
|
|
|
index_names = es_index_name_limiter(
|
|
|
start_date=filter_settings['start_date'],
|
|
|
end_date=filter_settings['end_date'],
|
|
|
ixtypes=['reports'])
|
|
|
|
|
|
if not index_names or not filter_settings['resource']:
|
|
|
return []
|
|
|
|
|
|
es_query = {
|
|
|
'aggs': {'parent_agg': {'aggs': {'groups': {'aggs': {
|
|
|
'sub_agg': {
|
|
|
'value_count': {'field': 'tags.group_id.values'}}},
|
|
|
'filter': {'exists': {'field': 'tags.group_id.values'}}}},
|
|
|
'terms': {'field': 'tags.group_id.values', 'size': limit}}},
|
|
|
'query': {'filtered': {
|
|
|
'filter': {'and': [
|
|
|
{'terms': {
|
|
|
'resource_id': [filter_settings['resource'][0]]}
|
|
|
},
|
|
|
{'range': {'timestamp': {
|
|
|
'gte': filter_settings['start_date'],
|
|
|
'lte': filter_settings['end_date']}}}]
|
|
|
}
|
|
|
}}
|
|
|
}
|
|
|
if tags:
|
|
|
es_query['query']['filtered']['filter']['and'].extend(tags)
|
|
|
|
|
|
result = Datastores.es.search(
|
|
|
es_query, index=index_names, doc_type='log', size=0)
|
|
|
series = []
|
|
|
for bucket in result['aggregations']['parent_agg']['buckets']:
|
|
|
series.append({
|
|
|
'key': bucket['key'],
|
|
|
'groups': bucket['groups']['sub_agg']['value']
|
|
|
})
|
|
|
|
|
|
report_groups_d = {}
|
|
|
for g in series:
|
|
|
report_groups_d[int(g['key'])] = g['groups'] or 0
|
|
|
|
|
|
query = db_session.query(ReportGroup)
|
|
|
query = query.filter(ReportGroup.id.in_(list(report_groups_d.keys())))
|
|
|
query = query.options(
|
|
|
sa.orm.joinedload(ReportGroup.last_report_ref))
|
|
|
results = [(report_groups_d[group.id], group,) for group in query]
|
|
|
return sorted(results, reverse=True, key=lambda x:x[0])
|
|
|
|
|
|
@classmethod
|
|
|
def get_search_iterator(cls, app_ids=None, page=1, items_per_page=50,
|
|
|
order_by=None, filter_settings=None, limit=None):
|
|
|
if not app_ids:
|
|
|
return {}
|
|
|
if not filter_settings:
|
|
|
filter_settings = {}
|
|
|
|
|
|
query = {
|
|
|
"size": 0,
|
|
|
"query": {
|
|
|
"filtered": {
|
|
|
"filter": {
|
|
|
"and": [{"terms": {"resource_id": list(app_ids)}}]
|
|
|
}
|
|
|
}
|
|
|
},
|
|
|
|
|
|
"aggs": {
|
|
|
"top_groups": {
|
|
|
"terms": {
|
|
|
"size": 5000,
|
|
|
"field": "_parent",
|
|
|
"order": {
|
|
|
"newest": "desc"
|
|
|
}
|
|
|
},
|
|
|
"aggs": {
|
|
|
"top_reports_hits": {
|
|
|
"top_hits": {"size": 1,
|
|
|
"sort": {"start_time": "desc"}
|
|
|
}
|
|
|
},
|
|
|
"newest": {
|
|
|
"max": {"field": "start_time"}
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
|
|
|
start_date = filter_settings.get('start_date')
|
|
|
end_date = filter_settings.get('end_date')
|
|
|
filter_part = query['query']['filtered']['filter']['and']
|
|
|
date_range = {"range": {"start_time": {}}}
|
|
|
if start_date:
|
|
|
date_range["range"]["start_time"]["gte"] = start_date
|
|
|
if end_date:
|
|
|
date_range["range"]["start_time"]["lte"] = end_date
|
|
|
if start_date or end_date:
|
|
|
filter_part.append(date_range)
|
|
|
|
|
|
priorities = filter_settings.get('priority')
|
|
|
|
|
|
for tag in filter_settings.get('tags', []):
|
|
|
tag_values = [v.lower() for v in tag['value']]
|
|
|
key = "tags.%s.values" % tag['name'].replace('.', '_')
|
|
|
filter_part.append({"terms": {key: tag_values}})
|
|
|
|
|
|
if priorities:
|
|
|
filter_part.append({"has_parent": {
|
|
|
"parent_type": "report_group",
|
|
|
"query": {
|
|
|
"terms": {'priority': priorities}
|
|
|
}}})
|
|
|
|
|
|
min_occurences = filter_settings.get('min_occurences')
|
|
|
if min_occurences:
|
|
|
filter_part.append({"has_parent": {
|
|
|
"parent_type": "report_group",
|
|
|
"query": {
|
|
|
"range": {'occurences': {"gte": min_occurences[0]}}
|
|
|
}}})
|
|
|
|
|
|
min_duration = filter_settings.get('min_duration')
|
|
|
max_duration = filter_settings.get('max_duration')
|
|
|
|
|
|
request_ids = filter_settings.get('request_id')
|
|
|
if request_ids:
|
|
|
filter_part.append({"terms": {'request_id': request_ids}})
|
|
|
|
|
|
duration_range = {"range": {"average_duration": {}}}
|
|
|
if min_duration:
|
|
|
duration_range["range"]["average_duration"]["gte"] = \
|
|
|
min_duration[0]
|
|
|
if max_duration:
|
|
|
duration_range["range"]["average_duration"]["lte"] = \
|
|
|
max_duration[0]
|
|
|
if min_duration or max_duration:
|
|
|
filter_part.append({"has_parent": {
|
|
|
"parent_type": "report_group",
|
|
|
"query": duration_range}})
|
|
|
|
|
|
http_status = filter_settings.get('http_status')
|
|
|
report_type = filter_settings.get('report_type', [ReportType.error])
|
|
|
# set error report type if http status is not found
|
|
|
# and we are dealing with slow reports
|
|
|
if not http_status or ReportType.slow in report_type:
|
|
|
filter_part.append({"terms": {'report_type': report_type}})
|
|
|
if http_status:
|
|
|
filter_part.append({"terms": {'http_status': http_status}})
|
|
|
|
|
|
messages = filter_settings.get('message')
|
|
|
if messages:
|
|
|
condition = {'match': {"message": ' '.join(messages)}}
|
|
|
query['query']['filtered']['query'] = condition
|
|
|
errors = filter_settings.get('error')
|
|
|
if errors:
|
|
|
condition = {'match': {"error": ' '.join(errors)}}
|
|
|
query['query']['filtered']['query'] = condition
|
|
|
url_domains = filter_settings.get('url_domain')
|
|
|
if url_domains:
|
|
|
condition = {'terms': {"url_domain": url_domains}}
|
|
|
query['query']['filtered']['query'] = condition
|
|
|
url_paths = filter_settings.get('url_path')
|
|
|
if url_paths:
|
|
|
condition = {'terms': {"url_path": url_paths}}
|
|
|
query['query']['filtered']['query'] = condition
|
|
|
|
|
|
if filter_settings.get('report_status'):
|
|
|
for status in filter_settings.get('report_status'):
|
|
|
if status == 'never_reviewed':
|
|
|
filter_part.append({"has_parent": {
|
|
|
"parent_type": "report_group",
|
|
|
"query": {
|
|
|
"term": {"read": False}
|
|
|
}}})
|
|
|
elif status == 'reviewed':
|
|
|
filter_part.append({"has_parent": {
|
|
|
"parent_type": "report_group",
|
|
|
"query": {
|
|
|
"term": {"read": True}
|
|
|
}}})
|
|
|
elif status == 'public':
|
|
|
filter_part.append({"has_parent": {
|
|
|
"parent_type": "report_group",
|
|
|
"query": {
|
|
|
"term": {"public": True}
|
|
|
}}})
|
|
|
elif status == 'fixed':
|
|
|
filter_part.append({"has_parent": {
|
|
|
"parent_type": "report_group",
|
|
|
"query": {
|
|
|
"term": {"fixed": True}
|
|
|
}}})
|
|
|
|
|
|
# logging.getLogger('pyelasticsearch').setLevel(logging.DEBUG)
|
|
|
index_names = es_index_name_limiter(filter_settings.get('start_date'),
|
|
|
filter_settings.get('end_date'),
|
|
|
ixtypes=['reports'])
|
|
|
if index_names:
|
|
|
results = Datastores.es.search(
|
|
|
query, index=index_names, doc_type=["report", "report_group"],
|
|
|
size=0)
|
|
|
else:
|
|
|
return []
|
|
|
return results['aggregations']
|
|
|
|
|
|
@classmethod
|
|
|
def get_paginator_by_app_ids(cls, app_ids=None, page=1, item_count=None,
|
|
|
items_per_page=50, order_by=None,
|
|
|
filter_settings=None,
|
|
|
exclude_columns=None, db_session=None):
|
|
|
if not filter_settings:
|
|
|
filter_settings = {}
|
|
|
results = cls.get_search_iterator(app_ids, page, items_per_page,
|
|
|
order_by, filter_settings)
|
|
|
|
|
|
ordered_ids = []
|
|
|
if results:
|
|
|
for item in results['top_groups']['buckets']:
|
|
|
pg_id = item['top_reports_hits']['hits']['hits'][0]['_source'][
|
|
|
'pg_id']
|
|
|
ordered_ids.append(pg_id)
|
|
|
log.info(filter_settings)
|
|
|
paginator = paginate.Page(ordered_ids, items_per_page=items_per_page,
|
|
|
**filter_settings)
|
|
|
sa_items = ()
|
|
|
if paginator.items:
|
|
|
db_session = get_db_session(db_session)
|
|
|
# latest report detail
|
|
|
query = db_session.query(Report)
|
|
|
query = query.options(sa.orm.joinedload(Report.report_group))
|
|
|
query = query.filter(Report.id.in_(paginator.items))
|
|
|
if filter_settings.get('order_col'):
|
|
|
order_col = filter_settings.get('order_col')
|
|
|
if filter_settings.get('order_dir') == 'dsc':
|
|
|
sort_on = 'desc'
|
|
|
else:
|
|
|
sort_on = 'asc'
|
|
|
if order_col == 'when':
|
|
|
order_col = 'last_timestamp'
|
|
|
query = query.order_by(getattr(sa, sort_on)(
|
|
|
getattr(ReportGroup, order_col)))
|
|
|
sa_items = query.all()
|
|
|
sorted_instance_list = []
|
|
|
for i_id in ordered_ids:
|
|
|
for report in sa_items:
|
|
|
if (str(report.id) == i_id and
|
|
|
report not in sorted_instance_list):
|
|
|
sorted_instance_list.append(report)
|
|
|
paginator.sa_items = sorted_instance_list
|
|
|
return paginator
|
|
|
|
|
|
@classmethod
|
|
|
def by_app_ids(cls, app_ids=None, order_by=True, db_session=None):
|
|
|
db_session = get_db_session(db_session)
|
|
|
q = db_session.query(ReportGroup)
|
|
|
if app_ids:
|
|
|
q = q.filter(ReportGroup.resource_id.in_(app_ids))
|
|
|
if order_by:
|
|
|
q = q.order_by(sa.desc(ReportGroup.id))
|
|
|
return q
|
|
|
|
|
|
@classmethod
|
|
|
def by_id(cls, group_id, app_ids=None, db_session=None):
|
|
|
db_session = get_db_session(db_session)
|
|
|
q = db_session.query(ReportGroup).filter(
|
|
|
ReportGroup.id == int(group_id))
|
|
|
if app_ids:
|
|
|
q = q.filter(ReportGroup.resource_id.in_(app_ids))
|
|
|
return q.first()
|
|
|
|
|
|
@classmethod
|
|
|
def by_ids(cls, group_ids=None, db_session=None):
|
|
|
db_session = get_db_session(db_session)
|
|
|
query = db_session.query(ReportGroup)
|
|
|
query = query.filter(ReportGroup.id.in_(group_ids))
|
|
|
return query
|
|
|
|
|
|
@classmethod
|
|
|
def by_hash_and_resource(cls, resource_id, grouping_hash, since_when=None,
|
|
|
db_session=None):
|
|
|
db_session = get_db_session(db_session)
|
|
|
q = db_session.query(ReportGroup)
|
|
|
q = q.filter(ReportGroup.resource_id == resource_id)
|
|
|
q = q.filter(ReportGroup.grouping_hash == grouping_hash)
|
|
|
q = q.filter(ReportGroup.fixed == False)
|
|
|
if since_when:
|
|
|
q = q.filter(ReportGroup.first_timestamp >= since_when)
|
|
|
return q.first()
|
|
|
|
|
|
@classmethod
|
|
|
def users_commenting(cls, report_group, exclude_user_id=None,
|
|
|
db_session=None):
|
|
|
db_session = get_db_session(None, report_group)
|
|
|
query = db_session.query(User).distinct()
|
|
|
query = query.filter(User.id == ReportComment.owner_id)
|
|
|
query = query.filter(ReportComment.group_id == report_group.id)
|
|
|
if exclude_user_id:
|
|
|
query = query.filter(ReportComment.owner_id != exclude_user_id)
|
|
|
return query
|
|
|
|
|
|
@classmethod
|
|
|
def affected_users_count(cls, report_group, db_session=None):
|
|
|
db_session = get_db_session(db_session)
|
|
|
query = db_session.query(sa.func.count(Report.username))
|
|
|
query = query.filter(Report.group_id == report_group.id)
|
|
|
query = query.filter(Report.username != '')
|
|
|
query = query.filter(Report.username != None)
|
|
|
query = query.group_by(Report.username)
|
|
|
return query.count()
|
|
|
|
|
|
@classmethod
|
|
|
def top_affected_users(cls, report_group, db_session=None):
|
|
|
db_session = get_db_session(db_session)
|
|
|
count_label = sa.func.count(Report.username).label('count')
|
|
|
query = db_session.query(Report.username, count_label)
|
|
|
query = query.filter(Report.group_id == report_group.id)
|
|
|
query = query.filter(Report.username != None)
|
|
|
query = query.filter(Report.username != '')
|
|
|
query = query.group_by(Report.username)
|
|
|
query = query.order_by(sa.desc(count_label))
|
|
|
query = query.limit(50)
|
|
|
return query
|
|
|
|
|
|
@classmethod
|
|
|
def get_report_stats(cls, request, filter_settings):
|
|
|
"""
|
|
|
Gets report dashboard graphs
|
|
|
Returns information for BAR charts with occurences/interval information
|
|
|
detailed means version that returns time intervals - non detailed
|
|
|
returns total sum
|
|
|
"""
|
|
|
delta = filter_settings['end_date'] - filter_settings['start_date']
|
|
|
if delta < h.time_deltas.get('12h')['delta']:
|
|
|
interval = '1m'
|
|
|
elif delta <= h.time_deltas.get('3d')['delta']:
|
|
|
interval = '5m'
|
|
|
elif delta >= h.time_deltas.get('2w')['delta']:
|
|
|
interval = '24h'
|
|
|
else:
|
|
|
interval = '1h'
|
|
|
|
|
|
group_id = filter_settings.get('group_id')
|
|
|
|
|
|
es_query = {
|
|
|
'aggs': {'parent_agg': {'aggs': {'types': {
|
|
|
'aggs': {'sub_agg': {'terms': {'field': 'tags.type.values'}}},
|
|
|
'filter': {
|
|
|
'and': [{'exists': {'field': 'tags.type.values'}}]}
|
|
|
}},
|
|
|
'date_histogram': {'extended_bounds': {
|
|
|
'max': filter_settings['end_date'],
|
|
|
'min': filter_settings['start_date']},
|
|
|
'field': 'timestamp',
|
|
|
'interval': interval,
|
|
|
'min_doc_count': 0}}},
|
|
|
'query': {'filtered': {
|
|
|
'filter': {'and': [
|
|
|
{'terms': {
|
|
|
'resource_id': [filter_settings['resource'][0]]}},
|
|
|
{'range': {'timestamp': {
|
|
|
'gte': filter_settings['start_date'],
|
|
|
'lte': filter_settings['end_date']}}}]
|
|
|
}
|
|
|
}}
|
|
|
}
|
|
|
if group_id:
|
|
|
parent_agg = es_query['aggs']['parent_agg']
|
|
|
filters = parent_agg['aggs']['types']['filter']['and']
|
|
|
filters.append({'terms': {'tags.group_id.values': [group_id]}})
|
|
|
|
|
|
index_names = es_index_name_limiter(
|
|
|
start_date=filter_settings['start_date'],
|
|
|
end_date=filter_settings['end_date'],
|
|
|
ixtypes=['reports'])
|
|
|
|
|
|
if not index_names:
|
|
|
return []
|
|
|
|
|
|
result = Datastores.es.search(es_query,
|
|
|
index=index_names,
|
|
|
doc_type='log',
|
|
|
size=0)
|
|
|
series = []
|
|
|
for bucket in result['aggregations']['parent_agg']['buckets']:
|
|
|
point = {
|
|
|
'x': datetime.utcfromtimestamp(int(bucket['key']) / 1000),
|
|
|
'report': 0,
|
|
|
'not_found': 0,
|
|
|
'slow_report': 0
|
|
|
}
|
|
|
for subbucket in bucket['types']['sub_agg']['buckets']:
|
|
|
if subbucket['key'] == 'slow':
|
|
|
point['slow_report'] = subbucket['doc_count']
|
|
|
elif subbucket['key'] == 'error':
|
|
|
point['report'] = subbucket['doc_count']
|
|
|
elif subbucket['key'] == 'not_found':
|
|
|
point['not_found'] = subbucket['doc_count']
|
|
|
series.append(point)
|
|
|
return series
|
|
|
|
