# -*- coding: utf-8 -*- # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging import sqlalchemy as sa from datetime import datetime, timedelta from pyramid.threadlocal import get_current_request from sqlalchemy.dialects.postgresql import JSON from ziggurat_foundations.models.base import BaseModel from appenlight.models import Base, get_db_session, Datastores from appenlight.lib.enums import ReportType from appenlight.lib.rule import Rule from appenlight.lib.redis_keys import REDIS_KEYS from appenlight.models.report import REPORT_TYPE_MATRIX log = logging.getLogger(__name__) class ReportGroup(Base, BaseModel): __tablename__ = "reports_groups" __table_args__ = {"implicit_returning": False} id = sa.Column(sa.BigInteger(), nullable=False, primary_key=True) resource_id = sa.Column( sa.Integer(), sa.ForeignKey( "applications.resource_id", onupdate="CASCADE", ondelete="CASCADE" ), nullable=False, index=True, ) priority = sa.Column( sa.Integer, nullable=False, index=True, default=5, server_default="5" ) first_timestamp = sa.Column( sa.DateTime(), default=datetime.utcnow, server_default=sa.func.now() ) last_timestamp = sa.Column( sa.DateTime(), default=datetime.utcnow, server_default=sa.func.now() ) error = sa.Column(sa.UnicodeText(), index=True) grouping_hash = sa.Column(sa.String(40), default="") triggered_postprocesses_ids = sa.Column(JSON(), nullable=False, default=list) report_type = sa.Column(sa.Integer, default=1) total_reports = sa.Column(sa.Integer, default=1) last_report = sa.Column(sa.Integer) occurences = sa.Column(sa.Integer, default=1) average_duration = sa.Column(sa.Float, default=0) summed_duration = sa.Column(sa.Float, default=0) read = sa.Column(sa.Boolean(), index=True, default=False) fixed = sa.Column(sa.Boolean(), index=True, default=False) notified = sa.Column(sa.Boolean(), index=True, default=False) public = sa.Column(sa.Boolean(), index=True, default=False) reports = sa.orm.relationship( "Report", lazy="dynamic", backref="report_group", cascade="all, delete-orphan", passive_deletes=True, passive_updates=True, ) comments = sa.orm.relationship( "ReportComment", lazy="dynamic", backref="report", cascade="all, delete-orphan", passive_deletes=True, passive_updates=True, order_by="ReportComment.comment_id", ) assigned_users = sa.orm.relationship( "User", backref=sa.orm.backref( "assigned_reports_relation", lazy="dynamic", order_by=sa.desc(sa.text("reports_groups.id")), ), passive_deletes=True, passive_updates=True, secondary="reports_assignments", order_by="User.user_name", ) stats = sa.orm.relationship( "ReportStat", lazy="dynamic", backref="report", passive_deletes=True, passive_updates=True, ) last_report_ref = sa.orm.relationship( "Report", uselist=False, primaryjoin="ReportGroup.last_report " "== Report.id", foreign_keys="Report.id", cascade="all, delete-orphan", passive_deletes=True, passive_updates=True, ) def __repr__(self): return "".format(self.id) def get_report(self, report_id=None, public=False): """ Gets report with specific id or latest report if id was not specified """ from .report import Report if not report_id: return self.last_report_ref else: return self.reports.filter(Report.id == report_id).first() def get_public_url(self, request, _app_url=None): url = request.route_url("/", _app_url=_app_url) return (url + "ui/report/%s") % self.id def run_postprocessing(self, report): """ Alters report group priority based on postprocessing configuration """ request = get_current_request() get_db_session(None, self).flush() for action in self.application.postprocess_conf: get_db_session(None, self).flush() rule_obj = Rule(action.rule, REPORT_TYPE_MATRIX) report_dict = report.get_dict(request) # if was not processed yet if ( rule_obj.match(report_dict) and action.pkey not in self.triggered_postprocesses_ids ): action.postprocess(self) # this way sqla can track mutation of list self.triggered_postprocesses_ids = self.triggered_postprocesses_ids + [ action.pkey ] get_db_session(None, self).flush() # do not go out of bounds if self.priority < 1: self.priority = 1 if self.priority > 10: self.priority = 10 def get_dict(self, request): instance_dict = super(ReportGroup, self).get_dict() instance_dict["server_name"] = self.get_report().tags.get("server_name") instance_dict["view_name"] = self.get_report().tags.get("view_name") instance_dict["resource_name"] = self.application.resource_name instance_dict["report_type"] = self.get_report().report_type instance_dict["url_path"] = self.get_report().url_path instance_dict["front_url"] = self.get_report().get_public_url(request) del instance_dict["triggered_postprocesses_ids"] return instance_dict def es_doc(self): return { "_id": str(self.id), "group_id": str(self.id), "resource_id": self.resource_id, "error": self.error, "fixed": self.fixed, "public": self.public, "read": self.read, "priority": self.priority, "occurences": self.occurences, "average_duration": self.average_duration, "summed_duration": self.summed_duration, "first_timestamp": self.first_timestamp, "last_timestamp": self.last_timestamp, "type": "report_group", "join_field": { "name": "report_group" }, } def set_notification_info(self, notify_10=False, notify_100=False): """ Update redis notification maps for notification job """ current_time = datetime.utcnow().replace(second=0, microsecond=0) # global app counter key = REDIS_KEYS["counters"]["reports_per_type"].format( self.report_type, current_time ) redis_pipeline = Datastores.redis.pipeline() redis_pipeline.incr(key) redis_pipeline.expire(key, 3600 * 24) # detailed app notification for alerts and notifications redis_pipeline.sadd(REDIS_KEYS["apps_that_had_reports"], self.resource_id) redis_pipeline.sadd( REDIS_KEYS["apps_that_had_reports_alerting"], self.resource_id ) # only notify for exceptions here if self.report_type == ReportType.error: redis_pipeline.sadd(REDIS_KEYS["apps_that_had_reports"], self.resource_id) redis_pipeline.sadd( REDIS_KEYS["apps_that_had_error_reports_alerting"], self.resource_id ) key = REDIS_KEYS["counters"]["report_group_occurences"].format(self.id) redis_pipeline.incr(key) redis_pipeline.expire(key, 3600 * 24) key = REDIS_KEYS["counters"]["report_group_occurences_alerting"].format(self.id) redis_pipeline.incr(key) redis_pipeline.expire(key, 3600 * 24) if notify_10: key = REDIS_KEYS["counters"]["report_group_occurences_10th"].format(self.id) redis_pipeline.setex(key, 3600 * 24, 1) if notify_100: key = REDIS_KEYS["counters"]["report_group_occurences_100th"].format( self.id ) redis_pipeline.setex(key, 3600 * 24, 1) key = REDIS_KEYS["reports_to_notify_per_type_per_app"].format( self.report_type, self.resource_id ) redis_pipeline.sadd(key, self.id) redis_pipeline.expire(key, 3600 * 24) key = REDIS_KEYS["reports_to_notify_per_type_per_app_alerting"].format( self.report_type, self.resource_id ) redis_pipeline.sadd(key, self.id) redis_pipeline.expire(key, 3600 * 24) redis_pipeline.execute() @property def partition_id(self): return "rcae_r_%s" % self.first_timestamp.strftime("%Y_%m") def partition_range(self): start_date = self.first_timestamp.date().replace(day=1) end_date = start_date + timedelta(days=40) end_date = end_date.replace(day=1) return start_date, end_date def after_insert(mapper, connection, target): if not hasattr(target, "_skip_ft_index"): data = target.es_doc() data.pop("_id", None) Datastores.es.index(target.partition_id, "report", data, id=target.id) def after_update(mapper, connection, target): if not hasattr(target, "_skip_ft_index"): data = target.es_doc() data.pop("_id", None) Datastores.es.index(target.partition_id, "report", data, id=target.id) def after_delete(mapper, connection, target): query = {"query": {"term": {"group_id": target.id}}} # delete by query Datastores.es.delete_by_query( index=target.partition_id, doc_type="report", body=query, conflicts="proceed" ) sa.event.listen(ReportGroup, "after_insert", after_insert) sa.event.listen(ReportGroup, "after_update", after_update) sa.event.listen(ReportGroup, "after_delete", after_delete)