From ba532c1708dcf6201ebb3bd716da198bf14e1611 2019-03-17 14:56:56 From: Marcin Lulek Date: 2019-03-17 14:56:56 Subject: [PATCH] elasticsearch: replace "filtered" with "bool" clause --- diff --git a/backend/src/appenlight/celery/tasks.py b/backend/src/appenlight/celery/tasks.py index f768f5e..8ba091a 100644 --- a/backend/src/appenlight/celery/tasks.py +++ b/backend/src/appenlight/celery/tasks.py @@ -691,14 +691,14 @@ def logs_cleanup(resource_id, filter_settings): request.tm.begin() es_query = { "query": { - "filtered": {"filter": {"and": [{"term": {"resource_id": resource_id}}]}} + "bool": {"filter": {"and": [{"term": {"resource_id": resource_id}}]}} } } query = DBSession.query(Log).filter(Log.resource_id == resource_id) if filter_settings["namespace"]: query = query.filter(Log.namespace == filter_settings["namespace"][0]) - es_query["query"]["filtered"]["filter"]["and"].append( + es_query["query"]["bool"]["filter"]["and"].append( {"term": {"namespace": filter_settings["namespace"][0]}} ) query.delete(synchronize_session=False) diff --git a/backend/src/appenlight/models/report.py b/backend/src/appenlight/models/report.py index c9aa3fd..5338925 100644 --- a/backend/src/appenlight/models/report.py +++ b/backend/src/appenlight/models/report.py @@ -311,7 +311,7 @@ class Report(Base, BaseModel): query = { "size": 1, "query": { - "filtered": { + "bool": { "filter": { "and": [ {"term": {"group_id": self.group_id}}, @@ -332,7 +332,7 @@ class Report(Base, BaseModel): query = { "size": 1, "query": { - "filtered": { + "bool": { "filter": { "and": [ {"term": {"group_id": self.group_id}}, diff --git a/backend/src/appenlight/models/services/log.py b/backend/src/appenlight/models/services/log.py index a73185e..8536f42 100644 --- a/backend/src/appenlight/models/services/log.py +++ b/backend/src/appenlight/models/services/log.py @@ -57,7 +57,7 @@ class LogService(BaseService): query = { "query": { - "filtered": { + "bool": { "filter": {"and": [{"terms": {"resource_id": list(app_ids)}}]} } } @@ -65,7 +65,7 @@ class LogService(BaseService): start_date = filter_settings.get("start_date") end_date = filter_settings.get("end_date") - filter_part = query["query"]["filtered"]["filter"]["and"] + filter_part = query["query"]["bool"]["filter"]["and"] for tag in filter_settings.get("tags", []): tag_values = [v.lower() for v in tag["value"]] @@ -93,7 +93,7 @@ class LogService(BaseService): messages = filter_settings.get("message") if messages: - query["query"]["filtered"]["query"] = { + query["query"]["bool"]["must"] = { "match": {"message": {"query": " ".join(messages), "operator": "and"}} } return query @@ -132,13 +132,13 @@ class LogService(BaseService): @classmethod def get_search_iterator( - cls, - app_ids=None, - page=1, - items_per_page=50, - order_by=None, - filter_settings=None, - limit=None, + cls, + app_ids=None, + page=1, + items_per_page=50, + order_by=None, + filter_settings=None, + limit=None, ): if not app_ids: return {}, 0 @@ -171,15 +171,15 @@ class LogService(BaseService): @classmethod def get_paginator_by_app_ids( - cls, - app_ids=None, - page=1, - item_count=None, - items_per_page=50, - order_by=None, - filter_settings=None, - exclude_columns=None, - db_session=None, + cls, + app_ids=None, + page=1, + item_count=None, + items_per_page=50, + order_by=None, + filter_settings=None, + exclude_columns=None, + db_session=None, ): if not filter_settings: filter_settings = {} diff --git a/backend/src/appenlight/models/services/report_group.py b/backend/src/appenlight/models/services/report_group.py index c82122d..1acf8f1 100644 --- a/backend/src/appenlight/models/services/report_group.py +++ b/backend/src/appenlight/models/services/report_group.py @@ -74,7 +74,7 @@ class ReportGroupService(BaseService): } }, "query": { - "filtered": { + "bool": { "filter": { "and": [ { @@ -96,7 +96,7 @@ class ReportGroupService(BaseService): }, } if tags: - es_query["query"]["filtered"]["filter"]["and"].extend(tags) + es_query["query"]["bool"]["filter"]["and"].extend(tags) result = Datastores.es.search( body=es_query, index=index_names, doc_type="log", size=0 @@ -135,7 +135,9 @@ class ReportGroupService(BaseService): query = { "size": 0, "query": { - "filtered": { + "bool": { + "must": [], + "should": [], "filter": {"and": [{"terms": {"resource_id": list(app_ids)}}]} } }, @@ -158,7 +160,7 @@ class ReportGroupService(BaseService): start_date = filter_settings.get("start_date") end_date = filter_settings.get("end_date") - filter_part = query["query"]["filtered"]["filter"]["and"] + filter_part = query["query"]["bool"]["filter"]["and"] date_range = {"range": {"start_time": {}}} if start_date: date_range["range"]["start_time"]["gte"] = start_date @@ -224,19 +226,19 @@ class ReportGroupService(BaseService): messages = filter_settings.get("message") if messages: condition = {"match": {"message": " ".join(messages)}} - query["query"]["filtered"]["query"] = condition + query["query"]["bool"]["must"].append(condition) errors = filter_settings.get("error") if errors: condition = {"match": {"error": " ".join(errors)}} - query["query"]["filtered"]["query"] = condition + query["query"]["bool"]["must"].append(condition) url_domains = filter_settings.get("url_domain") if url_domains: condition = {"terms": {"url_domain": url_domains}} - query["query"]["filtered"]["query"] = condition + query["query"]["bool"]["must"].append(condition) url_paths = filter_settings.get("url_path") if url_paths: condition = {"terms": {"url_path": url_paths}} - query["query"]["filtered"]["query"] = condition + query["query"]["bool"]["must"].append(condition) if filter_settings.get("report_status"): for status in filter_settings.get("report_status"): @@ -464,7 +466,7 @@ class ReportGroupService(BaseService): } }, "query": { - "filtered": { + "bool": { "filter": { "and": [ { diff --git a/backend/src/appenlight/models/services/report_stat.py b/backend/src/appenlight/models/services/report_stat.py index cb54d83..ab0b6d0 100644 --- a/backend/src/appenlight/models/services/report_stat.py +++ b/backend/src/appenlight/models/services/report_stat.py @@ -42,7 +42,7 @@ class ReportStatService(BaseService): } }, "query": { - "filtered": { + "bool": { "filter": { "and": [ {"terms": {"resource_id": [resource_id]}}, diff --git a/backend/src/appenlight/models/services/request_metric.py b/backend/src/appenlight/models/services/request_metric.py index b93f15a..b20b776 100644 --- a/backend/src/appenlight/models/services/request_metric.py +++ b/backend/src/appenlight/models/services/request_metric.py @@ -140,7 +140,7 @@ class RequestMetricService(BaseService): } }, "query": { - "filtered": { + "bool": { "filter": { "and": [ { @@ -208,7 +208,7 @@ class RequestMetricService(BaseService): } }, "query": { - "filtered": { + "bool": { "filter": { "and": [ { @@ -287,7 +287,7 @@ class RequestMetricService(BaseService): } }, "query": { - "filtered": { + "bool": { "filter": { "and": [ { @@ -331,7 +331,7 @@ class RequestMetricService(BaseService): }, } }, - "query": {"filtered": {"filter": {"and": and_part}}}, + "query": {"bool": {"filter": {"and": and_part}}}, } details = {} index_names = es_index_name_limiter(ixtypes=["reports"]) @@ -461,7 +461,7 @@ class RequestMetricService(BaseService): } }, "query": { - "filtered": { + "bool": { "filter": { "and": [ { @@ -540,7 +540,7 @@ class RequestMetricService(BaseService): } }, "query": { - "filtered": { + "bool": { "filter": { "and": [ { diff --git a/backend/src/appenlight/models/services/slow_call.py b/backend/src/appenlight/models/services/slow_call.py index 8c6e201..0ddc5f1 100644 --- a/backend/src/appenlight/models/services/slow_call.py +++ b/backend/src/appenlight/models/services/slow_call.py @@ -67,7 +67,7 @@ class SlowCallService(BaseService): } }, "query": { - "filtered": { + "bool": { "filter": { "and": [ { @@ -109,7 +109,7 @@ class SlowCallService(BaseService): } }, "query": { - "filtered": { + "bool": { "filter": { "and": [ { diff --git a/backend/src/appenlight/views/logs.py b/backend/src/appenlight/views/logs.py index 196c956..f42bf51 100644 --- a/backend/src/appenlight/views/logs.py +++ b/backend/src/appenlight/views/logs.py @@ -147,14 +147,14 @@ def common_tags(request): resources = list(filter_settings["resource"]) query = { "query": { - "filtered": { + "bool": { "filter": {"and": [{"terms": {"resource_id": list(resources)}}]} } } } start_date = filter_settings.get("start_date") end_date = filter_settings.get("end_date") - filter_part = query["query"]["filtered"]["filter"]["and"] + filter_part = query["query"]["bool"]["filter"]["and"] date_range = {"range": {"timestamp": {}}} if start_date: @@ -206,7 +206,7 @@ def common_values(request): and_part.append({"terms": {"namespace": filter_settings["namespace"]}}) query = { "query": { - "filtered": { + "bool": { "filter": { "and": and_part }