diff --git a/backend/src/appenlight/celery/tasks.py b/backend/src/appenlight/celery/tasks.py index 182fbf3..8041de8 100644 --- a/backend/src/appenlight/celery/tasks.py +++ b/backend/src/appenlight/celery/tasks.py @@ -325,10 +325,9 @@ def add_logs(resource_id, request_params, dataset, **kwargs): query = {"query": {"terms": {"delete_hash": batch}}} try: - Datastores.es.transport.perform_request( - "DELETE", - "/{}/{}/_query".format(es_index, "log"), - body=query, + Datastores.es.delete_by_query( + index=es_index, doc_type="log", + body=query, conflicts="proceed" ) except elasticsearch.exceptions.NotFoundError as exc: msg = "skipping index {}".format(es_index) @@ -703,6 +702,6 @@ def logs_cleanup(resource_id, filter_settings): ) query.delete(synchronize_session=False) request.tm.commit() - Datastores.es.transport.perform_request( - "DELETE", "/{}/{}/_query".format("rcae_l_*", "log"), body=es_query + Datastores.es.delete_by_query( + index="rcae_l_*", doc_type="log", body=es_query, conflicts="proceed" ) diff --git a/backend/src/appenlight/models/report.py b/backend/src/appenlight/models/report.py index ec093a4..8ea4d6c 100644 --- a/backend/src/appenlight/models/report.py +++ b/backend/src/appenlight/models/report.py @@ -519,8 +519,8 @@ def after_update(mapper, connection, target): def after_delete(mapper, connection, target): if not hasattr(target, "_skip_ft_index"): query = {"query": {"term": {"pg_id": target.id}}} - Datastores.es.transport.perform_request( - "DELETE", "/{}/{}/_query".format(target.partition_id, "report"), body=query + Datastores.es.delete_by_query( + index=target.partition_id, doc_type="report", body=query, conflicts="proceed" ) diff --git a/backend/src/appenlight/models/report_group.py b/backend/src/appenlight/models/report_group.py index afc04ec..f7e91c5 100644 --- a/backend/src/appenlight/models/report_group.py +++ b/backend/src/appenlight/models/report_group.py @@ -271,14 +271,12 @@ def after_update(mapper, connection, target): def after_delete(mapper, connection, target): query = {"query": {"term": {"group_id": target.id}}} # delete by query - Datastores.es.transport.perform_request( - "DELETE", "/{}/{}/_query".format(target.partition_id, "report"), body=query + Datastores.es.delete_by_query( + index=target.partition_id, doc_type="report", body=query, conflicts="proceed" ) query = {"query": {"term": {"pg_id": target.id}}} - Datastores.es.transport.perform_request( - "DELETE", - "/{}/{}/_query".format(target.partition_id, "report_group"), - body=query, + Datastores.es.delete_by_query( + index=target.partition_id, doc_type="report_group", body=query, conflicts="proceed" ) diff --git a/backend/src/appenlight/models/services/report_group.py b/backend/src/appenlight/models/services/report_group.py index 426548e..193fd50 100644 --- a/backend/src/appenlight/models/services/report_group.py +++ b/backend/src/appenlight/models/services/report_group.py @@ -64,13 +64,13 @@ class ReportGroupService(BaseService): "groups": { "aggs": { "sub_agg": { - "value_count": {"field": "tags.group_id.values"} + "value_count": {"field": "tags.group_id.values.keyword"} } }, "filter": {"exists": {"field": "tags.group_id.values"}}, } }, - "terms": {"field": "tags.group_id.values", "size": limit}, + "terms": {"field": "tags.group_id.values.keyword", "size": limit}, } }, "query": { @@ -143,7 +143,7 @@ class ReportGroupService(BaseService): "top_groups": { "terms": { "size": 5000, - "field": "_parent", + "field": "_parent#report_group", "order": {"newest": "desc"}, }, "aggs": { @@ -445,10 +445,12 @@ class ReportGroupService(BaseService): "aggs": { "types": { "aggs": { - "sub_agg": {"terms": {"field": "tags.type.values"}} + "sub_agg": {"terms": {"field": "tags.type.values.keyword"}} }, "filter": { - "and": [{"exists": {"field": "tags.type.values"}}] + "bool": { + "filter": [{"exists": {"field": "tags.type.values"}}] + } }, } }, @@ -485,7 +487,7 @@ class ReportGroupService(BaseService): } if group_id: parent_agg = es_query["aggs"]["parent_agg"] - filters = parent_agg["aggs"]["types"]["filter"]["and"] + filters = parent_agg["aggs"]["types"]["filter"]["bool"]["filter"] filters.append({"terms": {"tags.group_id.values": [group_id]}}) index_names = es_index_name_limiter( diff --git a/backend/src/appenlight/models/services/report_stat.py b/backend/src/appenlight/models/services/report_stat.py index d536e23..4ab4b45 100644 --- a/backend/src/appenlight/models/services/report_stat.py +++ b/backend/src/appenlight/models/services/report_stat.py @@ -34,10 +34,12 @@ class ReportStatService(BaseService): "sub_agg": {"value_count": {"field": "tags.group_id.values"}} }, "filter": { - "and": [ - {"terms": {"resource_id": [resource_id]}}, - {"exists": {"field": "tags.group_id.values"}}, - ] + "bool": { + "filter": [ + {"terms": {"resource_id": [resource_id]}}, + {"exists": {"field": "tags.group_id.values"}}, + ] + } }, } }, diff --git a/backend/src/appenlight/models/services/request_metric.py b/backend/src/appenlight/models/services/request_metric.py index 245732b..02b0992 100644 --- a/backend/src/appenlight/models/services/request_metric.py +++ b/backend/src/appenlight/models/services/request_metric.py @@ -235,6 +235,8 @@ class RequestMetricService(BaseService): script_text = "doc['tags.main.numeric_values'].value / {}".format( total_time_spent ) + if total_time_spent == 0: + script_text = '0' if index_names and filter_settings["resource"]: es_query = { @@ -255,7 +257,6 @@ class RequestMetricService(BaseService): "aggs": { "sub_agg": { "sum": { - "lang": "expression", "script": script_text, } } @@ -276,7 +277,7 @@ class RequestMetricService(BaseService): }, }, "terms": { - "field": "tags.view_name.values", + "field": "tags.view_name.values.keyword", "order": {"percentage>sub_agg": "desc"}, "size": 15, }, @@ -317,7 +318,7 @@ class RequestMetricService(BaseService): query = { "aggs": { "top_reports": { - "terms": {"field": "tags.view_name.values", "size": len(series)}, + "terms": {"field": "tags.view_name.values.keyword", "size": len(series)}, "aggs": { "top_calls_hits": { "top_hits": {"sort": {"start_time": "desc"}, "size": 5} @@ -390,18 +391,20 @@ class RequestMetricService(BaseService): } }, "filter": { - "and": [ - { - "range": { - "tags.main.numeric_values": {"gte": "4"} - } - }, - { - "exists": { - "field": "tags.requests.numeric_values" - } - }, - ] + "bool": { + "filter": [ + { + "range": { + "tags.main.numeric_values": {"gte": "4"} + } + }, + { + "exists": { + "field": "tags.requests.numeric_values" + } + }, + ] + } }, }, "main": { @@ -431,7 +434,7 @@ class RequestMetricService(BaseService): } }, "filter": { - "and": [ + "bool": {"filter": [ { "range": { "tags.main.numeric_values": {"gte": "1"} @@ -447,11 +450,11 @@ class RequestMetricService(BaseService): "field": "tags.requests.numeric_values" } }, - ] + ]} }, }, }, - "terms": {"field": "tags.server_name.values", "size": 999999}, + "terms": {"field": "tags.server_name.values.keyword", "size": 999999}, } }, "query": { @@ -517,18 +520,20 @@ class RequestMetricService(BaseService): } }, "filter": { - "and": [ - {"terms": {"tags.type.values": [report_type]}}, - { - "exists": { - "field": "tags.occurences.numeric_values" - } - }, - ] + "bool": { + "filter": [ + {"terms": {"tags.type.values": [report_type]}}, + { + "exists": { + "field": "tags.occurences.numeric_values" + } + }, + ] + } }, } }, - "terms": {"field": "tags.server_name.values", "size": 999999}, + "terms": {"field": "tags.server_name.values.keyword", "size": 999999}, } }, "query": { @@ -586,10 +591,10 @@ class RequestMetricService(BaseService): server_stats = list(stats.values()) for stat in server_stats: stat["satisfying_requests"] = ( - stat["requests"] - - stat["errors"] - - stat["frustrating_requests"] - - stat["tolerated_requests"] + stat["requests"] + - stat["errors"] + - stat["frustrating_requests"] + - stat["tolerated_requests"] ) if stat["satisfying_requests"] < 0: stat["satisfying_requests"] = 0 @@ -599,7 +604,7 @@ class RequestMetricService(BaseService): stat["response_time"] / stat["requests"], 3 ) qual_requests = ( - stat["satisfying_requests"] + stat["tolerated_requests"] / 2.0 + stat["satisfying_requests"] + stat["tolerated_requests"] / 2.0 ) stat["apdex"] = round((qual_requests / stat["requests"]) * 100, 2) stat["rpm"] = round(stat["requests"] / stat["total_minutes"], 2) diff --git a/backend/src/appenlight/models/services/slow_call.py b/backend/src/appenlight/models/services/slow_call.py index 189e6b5..e074fbd 100644 --- a/backend/src/appenlight/models/services/slow_call.py +++ b/backend/src/appenlight/models/services/slow_call.py @@ -50,7 +50,7 @@ class SlowCallService(BaseService): "aggs": { "sub_agg": { "value_count": { - "field": "tags.statement_hash.values" + "field": "tags.statement_hash.values.keyword" } } }, @@ -60,7 +60,7 @@ class SlowCallService(BaseService): }, }, "terms": { - "field": "tags.statement_hash.values", + "field": "tags.statement_hash.values.keyword", "order": {"duration>sub_agg": "desc"}, "size": 15, }, @@ -98,7 +98,7 @@ class SlowCallService(BaseService): calls_query = { "aggs": { "top_calls": { - "terms": {"field": "tags.statement_hash.values", "size": 15}, + "terms": {"field": "tags.statement_hash.values.keyword", "size": 15}, "aggs": { "top_calls_hits": { "top_hits": {"sort": {"timestamp": "desc"}, "size": 5} diff --git a/backend/src/appenlight/scripts/reindex_elasticsearch.py b/backend/src/appenlight/scripts/reindex_elasticsearch.py index ed205b6..9ba2ec9 100644 --- a/backend/src/appenlight/scripts/reindex_elasticsearch.py +++ b/backend/src/appenlight/scripts/reindex_elasticsearch.py @@ -139,7 +139,13 @@ def update_template(): "mapping": { "type": "object", "properties": { - "values": {"type": "string", "analyzer": "tag_value"}, + "values": {"type": "text", "analyzer": "tag_value", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + }}, "numeric_values": {"type": "float"}, }, }, @@ -177,10 +183,10 @@ def update_template(): "_all": {"enabled": False}, "dynamic_templates": tag_templates, "properties": { - "pg_id": {"type": "string", "index": "not_analyzed"}, + "pg_id": {"type": "keyword", "index": True}, "resource_id": {"type": "integer"}, "priority": {"type": "integer"}, - "error": {"type": "string", "analyzer": "simple"}, + "error": {"type": "text", "analyzer": "simple"}, "read": {"type": "boolean"}, "occurences": {"type": "integer"}, "fixed": {"type": "boolean"}, @@ -195,21 +201,27 @@ def update_template(): "_all": {"enabled": False}, "dynamic_templates": tag_templates, "properties": { - "pg_id": {"type": "string", "index": "not_analyzed"}, + "pg_id": {"type": "keyword", "index": True}, "resource_id": {"type": "integer"}, - "group_id": {"type": "string"}, + "group_id": {"type": "keyword"}, "http_status": {"type": "integer"}, - "ip": {"type": "string", "index": "not_analyzed"}, - "url_domain": {"type": "string", "analyzer": "simple"}, - "url_path": {"type": "string", "analyzer": "url_path"}, - "error": {"type": "string", "analyzer": "simple"}, + "ip": {"type": "keyword", "index": True}, + "url_domain": {"type": "text", "analyzer": "simple"}, + "url_path": {"type": "text", "analyzer": "url_path"}, + "error": {"type": "text", "analyzer": "simple"}, "report_type": {"type": "integer"}, "start_time": {"type": "date"}, - "request_id": {"type": "string", "index": "not_analyzed"}, + "request_id": {"type": "keyword", "index": True}, "end_time": {"type": "date"}, "duration": {"type": "float"}, "tags": {"type": "object"}, - "tag_list": {"type": "string", "analyzer": "tag_value"}, + "tag_list": {"type": "text", "analyzer": "tag_value", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + }}, "extra": {"type": "object"}, }, "_parent": {"type": "report_group"}, @@ -218,17 +230,26 @@ def update_template(): "_all": {"enabled": False}, "dynamic_templates": tag_templates, "properties": { - "pg_id": {"type": "string", "index": "not_analyzed"}, - "delete_hash": {"type": "string", "index": "not_analyzed"}, + "pg_id": {"type": "keyword", "index": True}, + "delete_hash": {"type": "keyword", "index": True}, "resource_id": {"type": "integer"}, "timestamp": {"type": "date"}, "permanent": {"type": "boolean"}, - "request_id": {"type": "string", "index": "not_analyzed"}, - "log_level": {"type": "string", "analyzer": "simple"}, - "message": {"type": "string", "analyzer": "simple"}, - "namespace": {"type": "string", "index": "not_analyzed"}, + "request_id": {"type": "keyword", "index": True}, + "log_level": {"type": "text", "analyzer": "simple"}, + "message": {"type": "text", "analyzer": "simple"}, + "namespace": { + "type": "text", + "fields": {"keyword": {"type": "keyword", "ignore_above": 256}}, + }, "tags": {"type": "object"}, - "tag_list": {"type": "string", "analyzer": "tag_value"}, + "tag_list": {"type": "text", "analyzer": "tag_value", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + }}, }, }, }, diff --git a/backend/src/appenlight/views/logs.py b/backend/src/appenlight/views/logs.py index b45dd73..96aaa8a 100644 --- a/backend/src/appenlight/views/logs.py +++ b/backend/src/appenlight/views/logs.py @@ -171,13 +171,13 @@ def common_tags(request): if namespaces: filter_part.append({"terms": {"namespace": namespaces}}) - query["aggs"] = {"sub_agg": {"terms": {"field": "tag_list", "size": 50}}} + query["aggs"] = {"sub_agg": {"terms": {"field": "tag_list.keyword", "size": 50}}} # tags index_names = es_index_name_limiter(ixtypes=[config.get("datasource", "logs")]) result = Datastores.es.search(body=query, index=index_names, doc_type="log", size=0) tag_buckets = result["aggregations"]["sub_agg"].get("buckets", []) # namespaces - query["aggs"] = {"sub_agg": {"terms": {"field": "namespace", "size": 50}}} + query["aggs"] = {"sub_agg": {"terms": {"field": "namespace.keyword", "size": 50}}} result = Datastores.es.search(body=query, index=index_names, doc_type="log", size=0) namespaces_buckets = result["aggregations"]["sub_agg"].get("buckets", []) return {