# -*- coding: utf-8 -*- # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from datetime import datetime import appenlight.lib.helpers as h from appenlight.models import get_db_session, Datastores from appenlight.models.services.base import BaseService from appenlight.lib.enums import ReportType from appenlight.lib.utils import es_index_name_limiter try: from ae_uptime_ce.models.services.uptime_metric import \ UptimeMetricService except ImportError: UptimeMetricService = None def check_key(key, stats, uptime, total_seconds): if key not in stats: stats[key] = {'name': key, 'requests': 0, 'errors': 0, 'tolerated_requests': 0, 'frustrating_requests': 0, 'satisfying_requests': 0, 'total_minutes': total_seconds / 60.0, 'uptime': uptime, 'apdex': 0, 'rpm': 0, 'response_time': 0, 'avg_response_time': 0} class RequestMetricService(BaseService): @classmethod def get_metrics_stats(cls, request, filter_settings, db_session=None): delta = filter_settings['end_date'] - filter_settings['start_date'] if delta < h.time_deltas.get('12h')['delta']: interval = '1m' elif delta <= h.time_deltas.get('3d')['delta']: interval = '5m' elif delta >= h.time_deltas.get('2w')['delta']: interval = '24h' else: interval = '1h' filter_settings['namespace'] = ['appenlight.request_metric'] es_query = { 'aggs': { 'parent_agg': { 'aggs': {'custom': {'aggs': {'sub_agg': { 'sum': {'field': 'tags.custom.numeric_values'}}}, 'filter': {'exists': { 'field': 'tags.custom.numeric_values'}}}, 'main': {'aggs': {'sub_agg': {'sum': { 'field': 'tags.main.numeric_values'}}}, 'filter': {'exists': { 'field': 'tags.main.numeric_values'}}}, 'nosql': {'aggs': {'sub_agg': {'sum': { 'field': 'tags.nosql.numeric_values'}}}, 'filter': {'exists': { 'field': 'tags.nosql.numeric_values'}}}, 'remote': {'aggs': {'sub_agg': {'sum': { 'field': 'tags.remote.numeric_values'}}}, 'filter': {'exists': { 'field': 'tags.remote.numeric_values'}}}, 'requests': {'aggs': {'sub_agg': {'sum': { 'field': 'tags.requests.numeric_values'}}}, 'filter': {'exists': { 'field': 'tags.requests.numeric_values'}}}, 'sql': {'aggs': {'sub_agg': { 'sum': {'field': 'tags.sql.numeric_values'}}}, 'filter': {'exists': { 'field': 'tags.sql.numeric_values'}}}, 'tmpl': {'aggs': {'sub_agg': {'sum': { 'field': 'tags.tmpl.numeric_values'}}}, 'filter': {'exists': { 'field': 'tags.tmpl.numeric_values'}}}}, 'date_histogram': {'extended_bounds': { 'max': filter_settings['end_date'], 'min': filter_settings['start_date']}, 'field': 'timestamp', 'interval': interval, 'min_doc_count': 0}}}, 'query': {'filtered': { 'filter': {'and': [{'terms': { 'resource_id': [filter_settings['resource'][0]]}}, {'range': {'timestamp': { 'gte': filter_settings['start_date'], 'lte': filter_settings['end_date']}}}, {'terms': {'namespace': [ 'appenlight.request_metric']}}]}}}} index_names = es_index_name_limiter( start_date=filter_settings['start_date'], end_date=filter_settings['end_date'], ixtypes=['metrics']) if not index_names: return [] result = Datastores.es.search(body=es_query, index=index_names, doc_type='log', size=0) plot_data = [] for item in result['aggregations']['parent_agg']['buckets']: x_time = datetime.utcfromtimestamp(int(item['key']) / 1000) point = {"x": x_time} for key in ['custom', 'main', 'nosql', 'remote', 'requests', 'sql', 'tmpl']: value = item[key]['sub_agg']['value'] point[key] = round(value, 3) if value else 0 plot_data.append(point) return plot_data @classmethod def get_requests_breakdown(cls, request, filter_settings, db_session=None): db_session = get_db_session(db_session) # fetch total time of all requests in this time range index_names = es_index_name_limiter( start_date=filter_settings['start_date'], end_date=filter_settings['end_date'], ixtypes=['metrics']) if index_names and filter_settings['resource']: es_query = { 'aggs': {'main': {'aggs': { 'sub_agg': {'sum': {'field': 'tags.main.numeric_values'}}}, 'filter': {'exists': { 'field': 'tags.main.numeric_values'}}}}, 'query': {'filtered': { 'filter': {'and': [ {'terms': { 'resource_id': [filter_settings['resource'][0]]}}, {'range': {'timestamp': { 'gte': filter_settings['start_date'], 'lte': filter_settings['end_date']}}}, {'terms': {'namespace': [ 'appenlight.request_metric']}}]}}}} result = Datastores.es.search(body=es_query, index=index_names, doc_type='log', size=0) total_time_spent = result['aggregations']['main']['sub_agg'][ 'value'] else: total_time_spent = 0 script_text = "doc['tags.main.numeric_values'].value / {}".format( total_time_spent) if index_names and filter_settings['resource']: es_query = { 'aggs': { 'parent_agg': { 'aggs': {'main': {'aggs': { 'sub_agg': { 'sum': {'field': 'tags.main.numeric_values'}}}, 'filter': { 'exists': { 'field': 'tags.main.numeric_values'}}}, 'percentage': { 'aggs': {'sub_agg': { 'sum': { 'lang': 'expression', 'script': script_text}}}, 'filter': { 'exists': { 'field': 'tags.main.numeric_values'}}}, 'requests': {'aggs': {'sub_agg': { 'sum': { 'field': 'tags.requests.numeric_values'}}}, 'filter': {'exists': { 'field': 'tags.requests.numeric_values'}}}}, 'terms': {'field': 'tags.view_name.values', 'order': {'percentage>sub_agg': 'desc'}, 'size': 15}}}, 'query': {'filtered': {'filter': {'and': [ {'terms': { 'resource_id': [filter_settings['resource'][0]]}}, {'range': { 'timestamp': {'gte': filter_settings['start_date'], 'lte': filter_settings['end_date'] } } } ]} }} } result = Datastores.es.search(body=es_query, index=index_names, doc_type='log', size=0) series = result['aggregations']['parent_agg']['buckets'] else: series = [] and_part = [ {"term": {"resource_id": filter_settings['resource'][0]}}, {"terms": {"tags.view_name.values": [row['key'] for row in series]}}, {"term": {"report_type": str(ReportType.slow)}} ] query = { "aggs": { "top_reports": { "terms": { "field": "tags.view_name.values", "size": len(series) }, "aggs": { "top_calls_hits": { "top_hits": { "sort": {"start_time": "desc"}, "size": 5 } } } } }, "query": { "filtered": { "filter": { "and": and_part } } } } details = {} index_names = es_index_name_limiter(ixtypes=['reports']) if index_names and series: result = Datastores.es.search( body=query, doc_type='report', size=0, index=index_names) for bucket in result['aggregations']['top_reports']['buckets']: details[bucket['key']] = [] for hit in bucket['top_calls_hits']['hits']['hits']: details[bucket['key']].append( {'report_id': hit['_source']['pg_id'], 'group_id': hit['_source']['group_id']} ) results = [] for row in series: result = { 'key': row['key'], 'main': row['main']['sub_agg']['value'], 'requests': row['requests']['sub_agg']['value'] } # es can return 'infinity' try: result['percentage'] = float( row['percentage']['sub_agg']['value']) except ValueError: result['percentage'] = 0 result['latest_details'] = details.get(row['key']) or [] results.append(result) return results @classmethod def get_apdex_stats(cls, request, filter_settings, threshold=1, db_session=None): """ Returns information and calculates APDEX score per server for dashboard server information (upper right stats boxes) """ # Apdex t = (Satisfied Count + Tolerated Count / 2) / Total Samples db_session = get_db_session(db_session) index_names = es_index_name_limiter( start_date=filter_settings['start_date'], end_date=filter_settings['end_date'], ixtypes=['metrics']) requests_series = [] if index_names and filter_settings['resource']: es_query = { 'aggs': { 'parent_agg': {'aggs': { 'frustrating': {'aggs': {'sub_agg': { 'sum': {'field': 'tags.requests.numeric_values'}}}, 'filter': {'and': [ {'range': { 'tags.main.numeric_values': {'gte': '4'}}}, {'exists': { 'field': 'tags.requests.numeric_values'}}] } }, 'main': {'aggs': {'sub_agg': {'sum': { 'field': 'tags.main.numeric_values'}}}, 'filter': {'exists': { 'field': 'tags.main.numeric_values'}}}, 'requests': {'aggs': {'sub_agg': { 'sum': { 'field': 'tags.requests.numeric_values'}}}, 'filter': {'exists': { 'field': 'tags.requests.numeric_values'}}}, 'tolerated': {'aggs': {'sub_agg': { 'sum': { 'field': 'tags.requests.numeric_values'}}}, 'filter': {'and': [ {'range': { 'tags.main.numeric_values': {'gte': '1'}}}, {'range': { 'tags.main.numeric_values': {'lt': '4'}}}, {'exists': { 'field': 'tags.requests.numeric_values'}}]} } }, 'terms': {'field': 'tags.server_name.values', 'size': 999999}}}, 'query': { 'filtered': { 'filter': {'and': [{'terms': { 'resource_id': [ filter_settings['resource'][0]]}}, {'range': {'timestamp': { 'gte': filter_settings['start_date'], 'lte': filter_settings['end_date']}}}, {'terms': {'namespace': [ 'appenlight.request_metric']}}]}}}} result = Datastores.es.search(body=es_query, index=index_names, doc_type='log', size=0) for bucket in result['aggregations']['parent_agg']['buckets']: requests_series.append({ 'frustrating': bucket['frustrating']['sub_agg']['value'], 'main': bucket['main']['sub_agg']['value'], 'requests': bucket['requests']['sub_agg']['value'], 'tolerated': bucket['tolerated']['sub_agg']['value'], 'key': bucket['key'] }) since_when = filter_settings['start_date'] until = filter_settings['end_date'] # total errors index_names = es_index_name_limiter( start_date=filter_settings['start_date'], end_date=filter_settings['end_date'], ixtypes=['reports']) report_series = [] if index_names and filter_settings['resource']: report_type = ReportType.key_from_value(ReportType.error) es_query = { 'aggs': { 'parent_agg': {'aggs': {'errors': {'aggs': {'sub_agg': { 'sum': { 'field': 'tags.occurences.numeric_values'}}}, 'filter': {'and': [ {'terms': { 'tags.type.values': [report_type]}}, {'exists': { 'field': 'tags.occurences.numeric_values'}}] } }}, 'terms': {'field': 'tags.server_name.values', 'size': 999999}}}, 'query': {'filtered': { 'filter': {'and': [ {'terms': { 'resource_id': [filter_settings['resource'][0]]}}, {'range': { 'timestamp': {'gte': filter_settings['start_date'], 'lte': filter_settings['end_date']}} }, {'terms': {'namespace': ['appenlight.error']}}] } }} } result = Datastores.es.search(body=es_query, index=index_names, doc_type='log', size=0) for bucket in result['aggregations']['parent_agg']['buckets']: report_series.append( {'key': bucket['key'], 'errors': bucket['errors']['sub_agg']['value'] } ) stats = {} if UptimeMetricService is not None: uptime = UptimeMetricService.get_uptime_by_app( filter_settings['resource'][0], since_when=since_when, until=until) else: uptime = 0 total_seconds = (until - since_when).total_seconds() for stat in requests_series: check_key(stat['key'], stats, uptime, total_seconds) stats[stat['key']]['requests'] = int(stat['requests']) stats[stat['key']]['response_time'] = stat['main'] stats[stat['key']]['tolerated_requests'] = stat['tolerated'] stats[stat['key']]['frustrating_requests'] = stat['frustrating'] for server in report_series: check_key(server['key'], stats, uptime, total_seconds) stats[server['key']]['errors'] = server['errors'] server_stats = list(stats.values()) for stat in server_stats: stat['satisfying_requests'] = stat['requests'] - stat['errors'] \ - stat['frustrating_requests'] - \ stat['tolerated_requests'] if stat['satisfying_requests'] < 0: stat['satisfying_requests'] = 0 if stat['requests']: stat['avg_response_time'] = round(stat['response_time'] / stat['requests'], 3) qual_requests = stat['satisfying_requests'] + \ stat['tolerated_requests'] / 2.0 stat['apdex'] = round((qual_requests / stat['requests']) * 100, 2) stat['rpm'] = round(stat['requests'] / stat['total_minutes'], 2) return sorted(server_stats, key=lambda x: x['name'])