|
|
# -*- coding: utf-8 -*-
|
|
|
|
|
|
# Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
|
|
|
#
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
# you may not use this file except in compliance with the License.
|
|
|
# You may obtain a copy of the License at
|
|
|
#
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
#
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
# See the License for the specific language governing permissions and
|
|
|
# limitations under the License.
|
|
|
|
|
|
from datetime import datetime
|
|
|
|
|
|
import appenlight.lib.helpers as h
|
|
|
from appenlight.models import get_db_session, Datastores
|
|
|
from appenlight.models.services.base import BaseService
|
|
|
from appenlight.lib.enums import ReportType
|
|
|
from appenlight.lib.utils import es_index_name_limiter
|
|
|
|
|
|
try:
|
|
|
from ae_uptime_ce.models.services.uptime_metric import \
|
|
|
UptimeMetricService
|
|
|
except ImportError:
|
|
|
UptimeMetricService = None
|
|
|
|
|
|
|
|
|
def check_key(key, stats, uptime, total_seconds):
|
|
|
if key not in stats:
|
|
|
stats[key] = {'name': key,
|
|
|
'requests': 0,
|
|
|
'errors': 0,
|
|
|
'tolerated_requests': 0,
|
|
|
'frustrating_requests': 0,
|
|
|
'satisfying_requests': 0,
|
|
|
'total_minutes': total_seconds / 60.0,
|
|
|
'uptime': uptime,
|
|
|
'apdex': 0,
|
|
|
'rpm': 0,
|
|
|
'response_time': 0,
|
|
|
'avg_response_time': 0}
|
|
|
|
|
|
|
|
|
class RequestMetricService(BaseService):
|
|
|
@classmethod
|
|
|
def get_metrics_stats(cls, request, filter_settings, db_session=None):
|
|
|
delta = filter_settings['end_date'] - filter_settings['start_date']
|
|
|
if delta < h.time_deltas.get('12h')['delta']:
|
|
|
interval = '1m'
|
|
|
elif delta <= h.time_deltas.get('3d')['delta']:
|
|
|
interval = '5m'
|
|
|
elif delta >= h.time_deltas.get('2w')['delta']:
|
|
|
interval = '24h'
|
|
|
else:
|
|
|
interval = '1h'
|
|
|
|
|
|
filter_settings['namespace'] = ['appenlight.request_metric']
|
|
|
|
|
|
es_query = {
|
|
|
'aggs': {
|
|
|
'parent_agg': {
|
|
|
'aggs': {'custom': {'aggs': {'sub_agg': {
|
|
|
'sum': {'field': 'tags.custom.numeric_values'}}},
|
|
|
'filter': {'exists': {
|
|
|
'field': 'tags.custom.numeric_values'}}},
|
|
|
'main': {'aggs': {'sub_agg': {'sum': {
|
|
|
'field': 'tags.main.numeric_values'}}},
|
|
|
'filter': {'exists': {
|
|
|
'field': 'tags.main.numeric_values'}}},
|
|
|
'nosql': {'aggs': {'sub_agg': {'sum': {
|
|
|
'field': 'tags.nosql.numeric_values'}}},
|
|
|
'filter': {'exists': {
|
|
|
'field': 'tags.nosql.numeric_values'}}},
|
|
|
'remote': {'aggs': {'sub_agg': {'sum': {
|
|
|
'field': 'tags.remote.numeric_values'}}},
|
|
|
'filter': {'exists': {
|
|
|
'field': 'tags.remote.numeric_values'}}},
|
|
|
'requests': {'aggs': {'sub_agg': {'sum': {
|
|
|
'field': 'tags.requests.numeric_values'}}},
|
|
|
'filter': {'exists': {
|
|
|
'field': 'tags.requests.numeric_values'}}},
|
|
|
'sql': {'aggs': {'sub_agg': {
|
|
|
'sum': {'field': 'tags.sql.numeric_values'}}},
|
|
|
'filter': {'exists': {
|
|
|
'field': 'tags.sql.numeric_values'}}},
|
|
|
'tmpl': {'aggs': {'sub_agg': {'sum': {
|
|
|
'field': 'tags.tmpl.numeric_values'}}},
|
|
|
'filter': {'exists': {
|
|
|
'field': 'tags.tmpl.numeric_values'}}}},
|
|
|
'date_histogram': {'extended_bounds': {
|
|
|
'max': filter_settings['end_date'],
|
|
|
'min': filter_settings['start_date']},
|
|
|
'field': 'timestamp',
|
|
|
'interval': interval,
|
|
|
'min_doc_count': 0}}},
|
|
|
'query': {'filtered': {
|
|
|
'filter': {'and': [{'terms': {
|
|
|
'resource_id': [filter_settings['resource'][0]]}},
|
|
|
{'range': {'timestamp': {
|
|
|
'gte': filter_settings['start_date'],
|
|
|
'lte': filter_settings['end_date']}}},
|
|
|
{'terms': {'namespace': [
|
|
|
'appenlight.request_metric']}}]}}}}
|
|
|
|
|
|
index_names = es_index_name_limiter(
|
|
|
start_date=filter_settings['start_date'],
|
|
|
end_date=filter_settings['end_date'],
|
|
|
ixtypes=['metrics'])
|
|
|
if not index_names:
|
|
|
return []
|
|
|
|
|
|
result = Datastores.es.search(es_query,
|
|
|
index=index_names,
|
|
|
doc_type='log',
|
|
|
size=0)
|
|
|
|
|
|
plot_data = []
|
|
|
for item in result['aggregations']['parent_agg']['buckets']:
|
|
|
x_time = datetime.utcfromtimestamp(int(item['key']) / 1000)
|
|
|
point = {"x": x_time}
|
|
|
for key in ['custom', 'main', 'nosql', 'remote',
|
|
|
'requests', 'sql', 'tmpl']:
|
|
|
value = item[key]['sub_agg']['value']
|
|
|
point[key] = round(value, 3) if value else 0
|
|
|
plot_data.append(point)
|
|
|
|
|
|
return plot_data
|
|
|
|
|
|
@classmethod
|
|
|
def get_requests_breakdown(cls, request, filter_settings,
|
|
|
db_session=None):
|
|
|
db_session = get_db_session(db_session)
|
|
|
|
|
|
# fetch total time of all requests in this time range
|
|
|
index_names = es_index_name_limiter(
|
|
|
start_date=filter_settings['start_date'],
|
|
|
end_date=filter_settings['end_date'],
|
|
|
ixtypes=['metrics'])
|
|
|
|
|
|
if index_names and filter_settings['resource']:
|
|
|
es_query = {
|
|
|
'aggs': {'main': {'aggs': {
|
|
|
'sub_agg': {'sum': {'field': 'tags.main.numeric_values'}}},
|
|
|
'filter': {'exists': {
|
|
|
'field': 'tags.main.numeric_values'}}}},
|
|
|
'query': {'filtered': {
|
|
|
'filter': {'and': [
|
|
|
{'terms': {
|
|
|
'resource_id': [filter_settings['resource'][0]]}},
|
|
|
{'range': {'timestamp': {
|
|
|
'gte': filter_settings['start_date'],
|
|
|
'lte': filter_settings['end_date']}}},
|
|
|
{'terms': {'namespace': [
|
|
|
'appenlight.request_metric']}}]}}}}
|
|
|
result = Datastores.es.search(es_query,
|
|
|
index=index_names,
|
|
|
doc_type='log',
|
|
|
size=0)
|
|
|
total_time_spent = result['aggregations']['main']['sub_agg'][
|
|
|
'value']
|
|
|
else:
|
|
|
total_time_spent = 0
|
|
|
script_text = "doc['tags.main.numeric_values'].value / {}".format(
|
|
|
total_time_spent)
|
|
|
|
|
|
if index_names and filter_settings['resource']:
|
|
|
es_query = {
|
|
|
'aggs': {
|
|
|
'parent_agg': {
|
|
|
'aggs': {'main': {'aggs': {
|
|
|
'sub_agg': {
|
|
|
'sum': {'field': 'tags.main.numeric_values'}}},
|
|
|
'filter': {
|
|
|
'exists': {
|
|
|
'field': 'tags.main.numeric_values'}}},
|
|
|
'percentage': {
|
|
|
'aggs': {'sub_agg': {
|
|
|
'sum': {
|
|
|
'lang': 'expression',
|
|
|
'script': script_text}}},
|
|
|
'filter': {
|
|
|
'exists': {
|
|
|
'field': 'tags.main.numeric_values'}}},
|
|
|
'requests': {'aggs': {'sub_agg': {
|
|
|
'sum': {
|
|
|
'field': 'tags.requests.numeric_values'}}},
|
|
|
'filter': {'exists': {
|
|
|
'field': 'tags.requests.numeric_values'}}}},
|
|
|
'terms': {'field': 'tags.view_name.values',
|
|
|
'order': {'percentage>sub_agg': 'desc'},
|
|
|
'size': 15}}},
|
|
|
'query': {'filtered': {'filter': {'and': [
|
|
|
{'terms': {
|
|
|
'resource_id': [filter_settings['resource'][0]]}},
|
|
|
{'range': {
|
|
|
'timestamp': {'gte': filter_settings['start_date'],
|
|
|
'lte': filter_settings['end_date']
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
]}
|
|
|
}}
|
|
|
}
|
|
|
result = Datastores.es.search(es_query,
|
|
|
index=index_names,
|
|
|
doc_type='log',
|
|
|
size=0)
|
|
|
series = result['aggregations']['parent_agg']['buckets']
|
|
|
else:
|
|
|
series = []
|
|
|
|
|
|
and_part = [
|
|
|
{"term": {"resource_id": filter_settings['resource'][0]}},
|
|
|
{"terms": {"tags.view_name.values": [row['key'] for
|
|
|
row in series]}},
|
|
|
{"term": {"report_type": str(ReportType.slow)}}
|
|
|
]
|
|
|
query = {
|
|
|
"aggs": {
|
|
|
"top_reports": {
|
|
|
"terms": {
|
|
|
"field": "tags.view_name.values",
|
|
|
"size": len(series)
|
|
|
},
|
|
|
"aggs": {
|
|
|
"top_calls_hits": {
|
|
|
"top_hits": {
|
|
|
"sort": {"start_time": "desc"},
|
|
|
"size": 5
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
},
|
|
|
|
|
|
"query": {
|
|
|
"filtered": {
|
|
|
"filter": {
|
|
|
"and": and_part
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
details = {}
|
|
|
index_names = es_index_name_limiter(ixtypes=['reports'])
|
|
|
if index_names and series:
|
|
|
result = Datastores.es.search(
|
|
|
query, doc_type='report', size=0, index=index_names)
|
|
|
for bucket in result['aggregations']['top_reports']['buckets']:
|
|
|
details[bucket['key']] = []
|
|
|
|
|
|
for hit in bucket['top_calls_hits']['hits']['hits']:
|
|
|
details[bucket['key']].append(
|
|
|
{'report_id': hit['_source']['pg_id'],
|
|
|
'group_id': hit['_source']['group_id']}
|
|
|
)
|
|
|
|
|
|
results = []
|
|
|
for row in series:
|
|
|
result = {
|
|
|
'key': row['key'],
|
|
|
'main': row['main']['sub_agg']['value'],
|
|
|
'requests': row['requests']['sub_agg']['value']
|
|
|
}
|
|
|
# es can return 'infinity'
|
|
|
try:
|
|
|
result['percentage'] = float(
|
|
|
row['percentage']['sub_agg']['value'])
|
|
|
except ValueError:
|
|
|
result['percentage'] = 0
|
|
|
|
|
|
result['latest_details'] = details.get(row['key']) or []
|
|
|
results.append(result)
|
|
|
|
|
|
return results
|
|
|
|
|
|
@classmethod
|
|
|
def get_apdex_stats(cls, request, filter_settings,
|
|
|
threshold=1, db_session=None):
|
|
|
"""
|
|
|
Returns information and calculates APDEX score per server for dashboard
|
|
|
server information (upper right stats boxes)
|
|
|
"""
|
|
|
# Apdex t = (Satisfied Count + Tolerated Count / 2) / Total Samples
|
|
|
db_session = get_db_session(db_session)
|
|
|
index_names = es_index_name_limiter(
|
|
|
start_date=filter_settings['start_date'],
|
|
|
end_date=filter_settings['end_date'], ixtypes=['metrics'])
|
|
|
|
|
|
requests_series = []
|
|
|
|
|
|
if index_names and filter_settings['resource']:
|
|
|
es_query = {
|
|
|
'aggs': {
|
|
|
'parent_agg': {'aggs': {
|
|
|
'frustrating': {'aggs': {'sub_agg': {
|
|
|
'sum': {'field': 'tags.requests.numeric_values'}}},
|
|
|
'filter': {'and': [
|
|
|
{'range': {
|
|
|
'tags.main.numeric_values': {'gte': '4'}}},
|
|
|
{'exists': {
|
|
|
'field': 'tags.requests.numeric_values'}}]
|
|
|
}
|
|
|
},
|
|
|
'main': {'aggs': {'sub_agg': {'sum': {
|
|
|
'field': 'tags.main.numeric_values'}}},
|
|
|
'filter': {'exists': {
|
|
|
'field': 'tags.main.numeric_values'}}},
|
|
|
'requests': {'aggs': {'sub_agg': {
|
|
|
'sum': {
|
|
|
'field': 'tags.requests.numeric_values'}}},
|
|
|
'filter': {'exists': {
|
|
|
'field': 'tags.requests.numeric_values'}}},
|
|
|
'tolerated': {'aggs': {'sub_agg': {
|
|
|
'sum': {
|
|
|
'field': 'tags.requests.numeric_values'}}},
|
|
|
'filter': {'and': [
|
|
|
{'range': {
|
|
|
'tags.main.numeric_values': {'gte': '1'}}},
|
|
|
{'range': {
|
|
|
'tags.main.numeric_values': {'lt': '4'}}},
|
|
|
{'exists': {
|
|
|
'field': 'tags.requests.numeric_values'}}]}
|
|
|
}
|
|
|
},
|
|
|
'terms': {'field': 'tags.server_name.values',
|
|
|
'size': 999999}}},
|
|
|
'query': {
|
|
|
'filtered': {
|
|
|
'filter': {'and': [{'terms': {
|
|
|
'resource_id': [
|
|
|
filter_settings['resource'][0]]}},
|
|
|
{'range': {'timestamp': {
|
|
|
'gte': filter_settings['start_date'],
|
|
|
'lte': filter_settings['end_date']}}},
|
|
|
{'terms': {'namespace': [
|
|
|
'appenlight.request_metric']}}]}}}}
|
|
|
|
|
|
result = Datastores.es.search(es_query,
|
|
|
index=index_names,
|
|
|
doc_type='log',
|
|
|
size=0)
|
|
|
for bucket in result['aggregations']['parent_agg']['buckets']:
|
|
|
requests_series.append({
|
|
|
'frustrating': bucket['frustrating']['sub_agg']['value'],
|
|
|
'main': bucket['main']['sub_agg']['value'],
|
|
|
'requests': bucket['requests']['sub_agg']['value'],
|
|
|
'tolerated': bucket['tolerated']['sub_agg']['value'],
|
|
|
'key': bucket['key']
|
|
|
})
|
|
|
|
|
|
since_when = filter_settings['start_date']
|
|
|
until = filter_settings['end_date']
|
|
|
|
|
|
# total errors
|
|
|
|
|
|
index_names = es_index_name_limiter(
|
|
|
start_date=filter_settings['start_date'],
|
|
|
end_date=filter_settings['end_date'], ixtypes=['reports'])
|
|
|
|
|
|
report_series = []
|
|
|
if index_names and filter_settings['resource']:
|
|
|
report_type = ReportType.key_from_value(ReportType.error)
|
|
|
es_query = {
|
|
|
'aggs': {
|
|
|
'parent_agg': {'aggs': {'errors': {'aggs': {'sub_agg': {
|
|
|
'sum': {
|
|
|
'field': 'tags.occurences.numeric_values'}}},
|
|
|
'filter': {'and': [
|
|
|
{'terms': {
|
|
|
'tags.type.values': [report_type]}},
|
|
|
{'exists': {
|
|
|
'field': 'tags.occurences.numeric_values'}}]
|
|
|
}
|
|
|
}},
|
|
|
'terms': {'field': 'tags.server_name.values',
|
|
|
'size': 999999}}},
|
|
|
'query': {'filtered': {
|
|
|
'filter': {'and': [
|
|
|
{'terms': {
|
|
|
'resource_id': [filter_settings['resource'][0]]}},
|
|
|
{'range': {
|
|
|
'timestamp': {'gte': filter_settings['start_date'],
|
|
|
'lte': filter_settings['end_date']}}
|
|
|
},
|
|
|
{'terms': {'namespace': ['appenlight.error']}}]
|
|
|
}
|
|
|
}}
|
|
|
}
|
|
|
result = Datastores.es.search(es_query,
|
|
|
index=index_names,
|
|
|
doc_type='log',
|
|
|
size=0)
|
|
|
for bucket in result['aggregations']['parent_agg']['buckets']:
|
|
|
report_series.append(
|
|
|
{'key': bucket['key'],
|
|
|
'errors': bucket['errors']['sub_agg']['value']
|
|
|
}
|
|
|
)
|
|
|
|
|
|
stats = {}
|
|
|
if UptimeMetricService is not None:
|
|
|
uptime = UptimeMetricService.get_uptime_by_app(
|
|
|
filter_settings['resource'][0],
|
|
|
since_when=since_when, until=until)
|
|
|
else:
|
|
|
uptime = 0
|
|
|
|
|
|
total_seconds = (until - since_when).total_seconds()
|
|
|
|
|
|
for stat in requests_series:
|
|
|
check_key(stat['key'], stats, uptime, total_seconds)
|
|
|
stats[stat['key']]['requests'] = int(stat['requests'])
|
|
|
stats[stat['key']]['response_time'] = stat['main']
|
|
|
stats[stat['key']]['tolerated_requests'] = stat['tolerated']
|
|
|
stats[stat['key']]['frustrating_requests'] = stat['frustrating']
|
|
|
for server in report_series:
|
|
|
check_key(server['key'], stats, uptime, total_seconds)
|
|
|
stats[server['key']]['errors'] = server['errors']
|
|
|
|
|
|
server_stats = list(stats.values())
|
|
|
for stat in server_stats:
|
|
|
stat['satisfying_requests'] = stat['requests'] - stat['errors'] \
|
|
|
- stat['frustrating_requests'] - \
|
|
|
stat['tolerated_requests']
|
|
|
if stat['satisfying_requests'] < 0:
|
|
|
stat['satisfying_requests'] = 0
|
|
|
|
|
|
if stat['requests']:
|
|
|
stat['avg_response_time'] = round(stat['response_time'] /
|
|
|
stat['requests'], 3)
|
|
|
qual_requests = stat['satisfying_requests'] + \
|
|
|
stat['tolerated_requests'] / 2.0
|
|
|
stat['apdex'] = round((qual_requests / stat['requests']) * 100,
|
|
|
2)
|
|
|
stat['rpm'] = round(stat['requests'] / stat['total_minutes'],
|
|
|
2)
|
|
|
|
|
|
return sorted(server_stats, key=lambda x: x['name'])
|
|
|
|
