From 02b699bc943584bc7553b58ed36f8089f6791458 2019-03-17 17:37:27 From: Marcin Lulek Date: 2019-03-17 17:37:27 Subject: [PATCH] normalize bool.filter format for elasticsearch 5.x --- diff --git a/backend/src/appenlight/celery/tasks.py b/backend/src/appenlight/celery/tasks.py index 8ba091a..182fbf3 100644 --- a/backend/src/appenlight/celery/tasks.py +++ b/backend/src/appenlight/celery/tasks.py @@ -691,14 +691,14 @@ def logs_cleanup(resource_id, filter_settings): request.tm.begin() es_query = { "query": { - "bool": {"filter": {"and": [{"term": {"resource_id": resource_id}}]}} + "bool": {"filter": [{"term": {"resource_id": resource_id}}]} } } query = DBSession.query(Log).filter(Log.resource_id == resource_id) if filter_settings["namespace"]: query = query.filter(Log.namespace == filter_settings["namespace"][0]) - es_query["query"]["bool"]["filter"]["and"].append( + es_query["query"]["bool"]["filter"].append( {"term": {"namespace": filter_settings["namespace"][0]}} ) query.delete(synchronize_session=False) diff --git a/backend/src/appenlight/models/report.py b/backend/src/appenlight/models/report.py index 5338925..fc27517 100644 --- a/backend/src/appenlight/models/report.py +++ b/backend/src/appenlight/models/report.py @@ -312,12 +312,10 @@ class Report(Base, BaseModel): "size": 1, "query": { "bool": { - "filter": { - "and": [ - {"term": {"group_id": self.group_id}}, - {"range": {"pg_id": {"lt": self.id}}}, - ] - } + "filter": [ + {"term": {"group_id": self.group_id}}, + {"range": {"pg_id": {"lt": self.id}}}, + ] } }, "sort": [{"_doc": {"order": "desc"}}], @@ -333,12 +331,10 @@ class Report(Base, BaseModel): "size": 1, "query": { "bool": { - "filter": { - "and": [ - {"term": {"group_id": self.group_id}}, - {"range": {"pg_id": {"gt": self.id}}}, - ] - } + "filter": [ + {"term": {"group_id": self.group_id}}, + {"range": {"pg_id": {"gt": self.id}}}, + ] } }, "sort": [{"_doc": {"order": "asc"}}], diff --git a/backend/src/appenlight/models/services/log.py b/backend/src/appenlight/models/services/log.py index 8536f42..620c9a2 100644 --- a/backend/src/appenlight/models/services/log.py +++ b/backend/src/appenlight/models/services/log.py @@ -58,14 +58,14 @@ class LogService(BaseService): query = { "query": { "bool": { - "filter": {"and": [{"terms": {"resource_id": list(app_ids)}}]} + "filter": [{"terms": {"resource_id": list(app_ids)}}] } } } start_date = filter_settings.get("start_date") end_date = filter_settings.get("end_date") - filter_part = query["query"]["bool"]["filter"]["and"] + filter_part = query["query"]["bool"]["filter"] for tag in filter_settings.get("tags", []): tag_values = [v.lower() for v in tag["value"]] diff --git a/backend/src/appenlight/models/services/report_group.py b/backend/src/appenlight/models/services/report_group.py index 1acf8f1..426548e 100644 --- a/backend/src/appenlight/models/services/report_group.py +++ b/backend/src/appenlight/models/services/report_group.py @@ -75,28 +75,26 @@ class ReportGroupService(BaseService): }, "query": { "bool": { - "filter": { - "and": [ - { - "terms": { - "resource_id": [filter_settings["resource"][0]] - } - }, - { - "range": { - "timestamp": { - "gte": filter_settings["start_date"], - "lte": filter_settings["end_date"], - } + "filter": [ + { + "terms": { + "resource_id": [filter_settings["resource"][0]] + } + }, + { + "range": { + "timestamp": { + "gte": filter_settings["start_date"], + "lte": filter_settings["end_date"], } - }, - ] - } + } + }, + ] } }, } if tags: - es_query["query"]["bool"]["filter"]["and"].extend(tags) + es_query["query"]["bool"]["filter"].extend(tags) result = Datastores.es.search( body=es_query, index=index_names, doc_type="log", size=0 @@ -138,7 +136,7 @@ class ReportGroupService(BaseService): "bool": { "must": [], "should": [], - "filter": {"and": [{"terms": {"resource_id": list(app_ids)}}]} + "filter": [{"terms": {"resource_id": list(app_ids)}}] } }, "aggs": { @@ -160,7 +158,7 @@ class ReportGroupService(BaseService): start_date = filter_settings.get("start_date") end_date = filter_settings.get("end_date") - filter_part = query["query"]["bool"]["filter"]["and"] + filter_part = query["query"]["bool"]["filter"] date_range = {"range": {"start_time": {}}} if start_date: date_range["range"]["start_time"]["gte"] = start_date @@ -467,23 +465,21 @@ class ReportGroupService(BaseService): }, "query": { "bool": { - "filter": { - "and": [ - { - "terms": { - "resource_id": [filter_settings["resource"][0]] - } - }, - { - "range": { - "timestamp": { - "gte": filter_settings["start_date"], - "lte": filter_settings["end_date"], - } + "filter": [ + { + "terms": { + "resource_id": [filter_settings["resource"][0]] + } + }, + { + "range": { + "timestamp": { + "gte": filter_settings["start_date"], + "lte": filter_settings["end_date"], } - }, - ] - } + } + }, + ] } }, } diff --git a/backend/src/appenlight/models/services/report_stat.py b/backend/src/appenlight/models/services/report_stat.py index ab0b6d0..0289d22 100644 --- a/backend/src/appenlight/models/services/report_stat.py +++ b/backend/src/appenlight/models/services/report_stat.py @@ -33,23 +33,19 @@ class ReportStatService(BaseService): "aggs": { "sub_agg": {"value_count": {"field": "tags.group_id.values"}} }, - "filter": { - "and": [ - {"terms": {"resource_id": [resource_id]}}, - {"exists": {"field": "tags.group_id.values"}}, - ] - }, + "filter": [ + {"terms": {"resource_id": [resource_id]}}, + {"exists": {"field": "tags.group_id.values"}}, + ], } }, "query": { "bool": { - "filter": { - "and": [ - {"terms": {"resource_id": [resource_id]}}, - {"terms": {"tags.type.values": [report_type]}}, - {"range": {"timestamp": {"gte": since_when}}}, - ] - } + "filter": [ + {"terms": {"resource_id": [resource_id]}}, + {"terms": {"tags.type.values": [report_type]}}, + {"range": {"timestamp": {"gte": since_when}}}, + ] } }, } diff --git a/backend/src/appenlight/models/services/request_metric.py b/backend/src/appenlight/models/services/request_metric.py index b20b776..245732b 100644 --- a/backend/src/appenlight/models/services/request_metric.py +++ b/backend/src/appenlight/models/services/request_metric.py @@ -141,24 +141,22 @@ class RequestMetricService(BaseService): }, "query": { "bool": { - "filter": { - "and": [ - { - "terms": { - "resource_id": [filter_settings["resource"][0]] - } - }, - { - "range": { - "timestamp": { - "gte": filter_settings["start_date"], - "lte": filter_settings["end_date"], - } + "filter": [ + { + "terms": { + "resource_id": [filter_settings["resource"][0]] + } + }, + { + "range": { + "timestamp": { + "gte": filter_settings["start_date"], + "lte": filter_settings["end_date"], } - }, - {"terms": {"namespace": ["appenlight.request_metric"]}}, - ] - } + } + }, + {"terms": {"namespace": ["appenlight.request_metric"]}}, + ] } }, } @@ -209,24 +207,22 @@ class RequestMetricService(BaseService): }, "query": { "bool": { - "filter": { - "and": [ - { - "terms": { - "resource_id": [filter_settings["resource"][0]] - } - }, - { - "range": { - "timestamp": { - "gte": filter_settings["start_date"], - "lte": filter_settings["end_date"], - } + "filter": [ + { + "terms": { + "resource_id": [filter_settings["resource"][0]] + } + }, + { + "range": { + "timestamp": { + "gte": filter_settings["start_date"], + "lte": filter_settings["end_date"], } - }, - {"terms": {"namespace": ["appenlight.request_metric"]}}, - ] - } + } + }, + {"terms": {"namespace": ["appenlight.request_metric"]}}, + ] } }, } @@ -288,23 +284,21 @@ class RequestMetricService(BaseService): }, "query": { "bool": { - "filter": { - "and": [ - { - "terms": { - "resource_id": [filter_settings["resource"][0]] - } - }, - { - "range": { - "timestamp": { - "gte": filter_settings["start_date"], - "lte": filter_settings["end_date"], - } + "filter": [ + { + "terms": { + "resource_id": [filter_settings["resource"][0]] + } + }, + { + "range": { + "timestamp": { + "gte": filter_settings["start_date"], + "lte": filter_settings["end_date"], } - }, - ] - } + } + }, + ] } }, } @@ -331,7 +325,7 @@ class RequestMetricService(BaseService): }, } }, - "query": {"bool": {"filter": {"and": and_part}}}, + "query": {"bool": {"filter": and_part}}, } details = {} index_names = es_index_name_limiter(ixtypes=["reports"]) @@ -462,24 +456,22 @@ class RequestMetricService(BaseService): }, "query": { "bool": { - "filter": { - "and": [ - { - "terms": { - "resource_id": [filter_settings["resource"][0]] - } - }, - { - "range": { - "timestamp": { - "gte": filter_settings["start_date"], - "lte": filter_settings["end_date"], - } + "filter": [ + { + "terms": { + "resource_id": [filter_settings["resource"][0]] + } + }, + { + "range": { + "timestamp": { + "gte": filter_settings["start_date"], + "lte": filter_settings["end_date"], } - }, - {"terms": {"namespace": ["appenlight.request_metric"]}}, - ] - } + } + }, + {"terms": {"namespace": ["appenlight.request_metric"]}}, + ] } }, } @@ -541,24 +533,22 @@ class RequestMetricService(BaseService): }, "query": { "bool": { - "filter": { - "and": [ - { - "terms": { - "resource_id": [filter_settings["resource"][0]] - } - }, - { - "range": { - "timestamp": { - "gte": filter_settings["start_date"], - "lte": filter_settings["end_date"], - } + "filter": [ + { + "terms": { + "resource_id": [filter_settings["resource"][0]] + } + }, + { + "range": { + "timestamp": { + "gte": filter_settings["start_date"], + "lte": filter_settings["end_date"], } - }, - {"terms": {"namespace": ["appenlight.error"]}}, - ] - } + } + }, + {"terms": {"namespace": ["appenlight.error"]}}, + ] } }, } diff --git a/backend/src/appenlight/models/services/slow_call.py b/backend/src/appenlight/models/services/slow_call.py index 0ddc5f1..189e6b5 100644 --- a/backend/src/appenlight/models/services/slow_call.py +++ b/backend/src/appenlight/models/services/slow_call.py @@ -68,23 +68,21 @@ class SlowCallService(BaseService): }, "query": { "bool": { - "filter": { - "and": [ - { - "terms": { - "resource_id": [filter_settings["resource"][0]] - } - }, - { - "range": { - "timestamp": { - "gte": filter_settings["start_date"], - "lte": filter_settings["end_date"], - } + "filter": [ + { + "terms": { + "resource_id": [filter_settings["resource"][0]] + } + }, + { + "range": { + "timestamp": { + "gte": filter_settings["start_date"], + "lte": filter_settings["end_date"], } - }, - ] - } + } + }, + ] } }, } @@ -110,24 +108,22 @@ class SlowCallService(BaseService): }, "query": { "bool": { - "filter": { - "and": [ - { - "terms": { - "resource_id": [filter_settings["resource"][0]] - } - }, - {"terms": {"tags.statement_hash.values": hashes}}, - { - "range": { - "timestamp": { - "gte": filter_settings["start_date"], - "lte": filter_settings["end_date"], - } + "filter": [ + { + "terms": { + "resource_id": [filter_settings["resource"][0]] + } + }, + {"terms": {"tags.statement_hash.values": hashes}}, + { + "range": { + "timestamp": { + "gte": filter_settings["start_date"], + "lte": filter_settings["end_date"], } - }, - ] - } + } + }, + ] } }, } diff --git a/backend/src/appenlight/views/logs.py b/backend/src/appenlight/views/logs.py index f42bf51..b45dd73 100644 --- a/backend/src/appenlight/views/logs.py +++ b/backend/src/appenlight/views/logs.py @@ -148,13 +148,13 @@ def common_tags(request): query = { "query": { "bool": { - "filter": {"and": [{"terms": {"resource_id": list(resources)}}]} + "filter": [{"terms": {"resource_id": list(resources)}}] } } } start_date = filter_settings.get("start_date") end_date = filter_settings.get("end_date") - filter_part = query["query"]["bool"]["filter"]["and"] + filter_part = query["query"]["bool"]["filter"] date_range = {"range": {"timestamp": {}}} if start_date: @@ -207,9 +207,7 @@ def common_values(request): query = { "query": { "bool": { - "filter": { - "and": and_part - } + "filter": and_part } } }