# -*- coding: utf-8 -*- # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging import paginate import sqlalchemy as sa import appenlight.lib.helpers as h from datetime import datetime from appenlight.models import get_db_session, Datastores from appenlight.models.report import Report from appenlight.models.report_group import ReportGroup from appenlight.models.report_comment import ReportComment from appenlight.models.user import User from appenlight.models.services.base import BaseService from appenlight.lib.enums import ReportType from appenlight.lib.utils import es_index_name_limiter log = logging.getLogger(__name__) class ReportGroupService(BaseService): @classmethod def get_trending(cls, request, filter_settings, limit=15, db_session=None): """ Returns report groups trending for specific time interval """ db_session = get_db_session(db_session) tags = [] if filter_settings.get("tags"): for tag in filter_settings["tags"]: tags.append( {"terms": {"tags.{}.values".format(tag["name"]): tag["value"]}} ) index_names = es_index_name_limiter( start_date=filter_settings["start_date"], end_date=filter_settings["end_date"], ixtypes=["reports"], ) if not index_names or not filter_settings["resource"]: return [] es_query = { "aggs": { "parent_agg": { "aggs": { "groups": { "aggs": { "sub_agg": { "value_count": { "field": "tags.group_id.values.keyword" } } }, "filter": {"exists": {"field": "tags.group_id.values"}}, } }, "terms": {"field": "tags.group_id.values.keyword", "size": limit}, } }, "query": { "bool": { "filter": [ {"terms": {"resource_id": [filter_settings["resource"][0]]}}, { "range": { "timestamp": { "gte": filter_settings["start_date"], "lte": filter_settings["end_date"], } } }, ] } }, } if tags: es_query["query"]["bool"]["filter"].extend(tags) result = Datastores.es.search( body=es_query, index=index_names, doc_type="report", size=0 ) series = [] for bucket in result["aggregations"]["parent_agg"]["buckets"]: series.append( {"key": bucket["key"], "groups": bucket["groups"]["sub_agg"]["value"]} ) report_groups_d = {} for g in series: report_groups_d[int(g["key"])] = g["groups"] or 0 query = db_session.query(ReportGroup) query = query.filter(ReportGroup.id.in_(list(report_groups_d.keys()))) query = query.options(sa.orm.joinedload(ReportGroup.last_report_ref)) results = [(report_groups_d[group.id], group) for group in query] return sorted(results, reverse=True, key=lambda x: x[0]) @classmethod def get_search_iterator( cls, app_ids=None, page=1, items_per_page=50, order_by=None, filter_settings=None, limit=None, ): if not app_ids: return {} if not filter_settings: filter_settings = {} query = { "size": 0, "query": { "bool": { "must": [], "should": [], "filter": [{"terms": {"resource_id": list(app_ids)}}], } }, "aggs": { "top_groups": { "terms": { "size": 5000, "field": "join_field#report_group", "order": {"newest": "desc"}, }, "aggs": { "top_reports_hits": { "top_hits": {"size": 1, "sort": {"start_time": "desc"}} }, "newest": {"max": {"field": "start_time"}}, }, } }, } start_date = filter_settings.get("start_date") end_date = filter_settings.get("end_date") filter_part = query["query"]["bool"]["filter"] date_range = {"range": {"start_time": {}}} if start_date: date_range["range"]["start_time"]["gte"] = start_date if end_date: date_range["range"]["start_time"]["lte"] = end_date if start_date or end_date: filter_part.append(date_range) priorities = filter_settings.get("priority") for tag in filter_settings.get("tags", []): tag_values = [v.lower() for v in tag["value"]] key = "tags.%s.values" % tag["name"].replace(".", "_") filter_part.append({"terms": {key: tag_values}}) if priorities: filter_part.append( { "has_parent": { "parent_type": "report_group", "query": {"terms": {"priority": priorities}}, } } ) min_occurences = filter_settings.get("min_occurences") if min_occurences: filter_part.append( { "has_parent": { "parent_type": "report_group", "query": {"range": {"occurences": {"gte": min_occurences[0]}}}, } } ) min_duration = filter_settings.get("min_duration") max_duration = filter_settings.get("max_duration") request_ids = filter_settings.get("request_id") if request_ids: filter_part.append({"terms": {"request_id": request_ids}}) duration_range = {"range": {"average_duration": {}}} if min_duration: duration_range["range"]["average_duration"]["gte"] = min_duration[0] if max_duration: duration_range["range"]["average_duration"]["lte"] = max_duration[0] if min_duration or max_duration: filter_part.append( {"has_parent": {"parent_type": "report_group", "query": duration_range}} ) http_status = filter_settings.get("http_status") report_type = filter_settings.get("report_type", [ReportType.error]) # set error report type if http status is not found # and we are dealing with slow reports if not http_status or ReportType.slow in report_type: filter_part.append({"terms": {"report_type": report_type}}) if http_status: filter_part.append({"terms": {"http_status": http_status}}) messages = filter_settings.get("message") if messages: condition = {"match": {"message": " ".join(messages)}} query["query"]["bool"]["must"].append(condition) errors = filter_settings.get("error") if errors: condition = {"match": {"error": " ".join(errors)}} query["query"]["bool"]["must"].append(condition) url_domains = filter_settings.get("url_domain") if url_domains: condition = {"terms": {"url_domain": url_domains}} query["query"]["bool"]["must"].append(condition) url_paths = filter_settings.get("url_path") if url_paths: condition = {"terms": {"url_path": url_paths}} query["query"]["bool"]["must"].append(condition) if filter_settings.get("report_status"): for status in filter_settings.get("report_status"): if status == "never_reviewed": filter_part.append( { "has_parent": { "parent_type": "report_group", "query": {"term": {"read": False}}, } } ) elif status == "reviewed": filter_part.append( { "has_parent": { "parent_type": "report_group", "query": {"term": {"read": True}}, } } ) elif status == "public": filter_part.append( { "has_parent": { "parent_type": "report_group", "query": {"term": {"public": True}}, } } ) elif status == "fixed": filter_part.append( { "has_parent": { "parent_type": "report_group", "query": {"term": {"fixed": True}}, } } ) # logging.getLogger('pyelasticsearch').setLevel(logging.DEBUG) index_names = es_index_name_limiter( filter_settings.get("start_date"), filter_settings.get("end_date"), ixtypes=["reports"], ) if index_names: results = Datastores.es.search( body=query, index=index_names, doc_type=["report", "report_group"], size=0, ) else: return [] return results["aggregations"] @classmethod def get_paginator_by_app_ids( cls, app_ids=None, page=1, item_count=None, items_per_page=50, order_by=None, filter_settings=None, exclude_columns=None, db_session=None, ): if not filter_settings: filter_settings = {} results = cls.get_search_iterator( app_ids, page, items_per_page, order_by, filter_settings ) ordered_ids = [] if results: for item in results["top_groups"]["buckets"]: pg_id = item["top_reports_hits"]["hits"]["hits"][0]["_source"][ "report_id" ] ordered_ids.append(pg_id) log.info(filter_settings) paginator = paginate.Page( ordered_ids, items_per_page=items_per_page, **filter_settings ) sa_items = () if paginator.items: db_session = get_db_session(db_session) # latest report detail query = db_session.query(Report) query = query.options(sa.orm.joinedload(Report.report_group)) query = query.filter(Report.id.in_(paginator.items)) if filter_settings.get("order_col"): order_col = filter_settings.get("order_col") if filter_settings.get("order_dir") == "dsc": sort_on = "desc" else: sort_on = "asc" if order_col == "when": order_col = "last_timestamp" query = query.order_by( getattr(sa, sort_on)(getattr(ReportGroup, order_col)) ) sa_items = query.all() sorted_instance_list = [] for i_id in ordered_ids: for report in sa_items: if str(report.id) == i_id and report not in sorted_instance_list: sorted_instance_list.append(report) paginator.sa_items = sorted_instance_list return paginator @classmethod def by_app_ids(cls, app_ids=None, order_by=True, db_session=None): db_session = get_db_session(db_session) q = db_session.query(ReportGroup) if app_ids: q = q.filter(ReportGroup.resource_id.in_(app_ids)) if order_by: q = q.order_by(sa.desc(ReportGroup.id)) return q @classmethod def by_id(cls, group_id, app_ids=None, db_session=None): db_session = get_db_session(db_session) q = db_session.query(ReportGroup).filter(ReportGroup.id == int(group_id)) if app_ids: q = q.filter(ReportGroup.resource_id.in_(app_ids)) return q.first() @classmethod def by_ids(cls, group_ids=None, db_session=None): db_session = get_db_session(db_session) query = db_session.query(ReportGroup) query = query.filter(ReportGroup.id.in_(group_ids)) return query @classmethod def by_hash_and_resource( cls, resource_id, grouping_hash, since_when=None, db_session=None ): db_session = get_db_session(db_session) q = db_session.query(ReportGroup) q = q.filter(ReportGroup.resource_id == resource_id) q = q.filter(ReportGroup.grouping_hash == grouping_hash) q = q.filter(ReportGroup.fixed == False) if since_when: q = q.filter(ReportGroup.first_timestamp >= since_when) return q.first() @classmethod def users_commenting(cls, report_group, exclude_user_id=None, db_session=None): db_session = get_db_session(None, report_group) query = db_session.query(User).distinct() query = query.filter(User.id == ReportComment.owner_id) query = query.filter(ReportComment.group_id == report_group.id) if exclude_user_id: query = query.filter(ReportComment.owner_id != exclude_user_id) return query @classmethod def affected_users_count(cls, report_group, db_session=None): db_session = get_db_session(db_session) query = db_session.query(sa.func.count(Report.username)) query = query.filter(Report.group_id == report_group.id) query = query.filter(Report.username != "") query = query.filter(Report.username != None) query = query.group_by(Report.username) return query.count() @classmethod def top_affected_users(cls, report_group, db_session=None): db_session = get_db_session(db_session) count_label = sa.func.count(Report.username).label("count") query = db_session.query(Report.username, count_label) query = query.filter(Report.group_id == report_group.id) query = query.filter(Report.username != None) query = query.filter(Report.username != "") query = query.group_by(Report.username) query = query.order_by(sa.desc(count_label)) query = query.limit(50) return query @classmethod def get_report_stats(cls, request, filter_settings): """ Gets report dashboard graphs Returns information for BAR charts with occurences/interval information detailed means version that returns time intervals - non detailed returns total sum """ delta = filter_settings["end_date"] - filter_settings["start_date"] if delta < h.time_deltas.get("12h")["delta"]: interval = "1m" elif delta <= h.time_deltas.get("3d")["delta"]: interval = "5m" elif delta >= h.time_deltas.get("2w")["delta"]: interval = "24h" else: interval = "1h" group_id = filter_settings.get("group_id") es_query = { "aggs": { "parent_agg": { "aggs": { "types": { "aggs": { "sub_agg": { "terms": {"field": "tags.type.values.keyword"} } }, "filter": { "bool": { "filter": [ {"exists": {"field": "tags.type.values"}} ] } }, } }, "date_histogram": { "extended_bounds": { "max": filter_settings["end_date"], "min": filter_settings["start_date"], }, "field": "timestamp", "interval": interval, "min_doc_count": 0, }, } }, "query": { "bool": { "filter": [ {"terms": {"resource_id": [filter_settings["resource"][0]]}}, { "range": { "timestamp": { "gte": filter_settings["start_date"], "lte": filter_settings["end_date"], } } }, ] } }, } if group_id: parent_agg = es_query["aggs"]["parent_agg"] filters = parent_agg["aggs"]["types"]["filter"]["bool"]["filter"] filters.append({"terms": {"tags.group_id.values": [group_id]}}) index_names = es_index_name_limiter( start_date=filter_settings["start_date"], end_date=filter_settings["end_date"], ixtypes=["reports"], ) if not index_names: return [] result = Datastores.es.search( body=es_query, index=index_names, doc_type="log", size=0 ) series = [] for bucket in result["aggregations"]["parent_agg"]["buckets"]: point = { "x": datetime.utcfromtimestamp(int(bucket["key"]) / 1000), "report": 0, "not_found": 0, "slow_report": 0, } for subbucket in bucket["types"]["sub_agg"]["buckets"]: if subbucket["key"] == "slow": point["slow_report"] = subbucket["doc_count"] elif subbucket["key"] == "error": point["report"] = subbucket["doc_count"] elif subbucket["key"] == "not_found": point["not_found"] = subbucket["doc_count"] series.append(point) return series