elasticsearch: migrate to ES 5.x

backend/src/appenlight/celery/tasks.py

@@ -325,10 +325,9 b' def add_logs(resource_id, request_params, dataset, **kwargs):'
                         query = {"query": {"terms": {"delete_hash": batch}}}
 
                         try:
-                            Datastores.es.transport.perform_request(
-                                "DELETE",
-                                "/{}/{}/_query".format(es_index, "log"),
-                                body=query,
+                            Datastores.es.delete_by_query(
+                                index=es_index, doc_type="log",
+                                body=query, conflicts="proceed"
                             )
                         except elasticsearch.exceptions.NotFoundError as exc:
                             msg = "skipping index {}".format(es_index)
@@ -703,6 +702,6 b' def logs_cleanup(resource_id, filter_settings):'
         )
     query.delete(synchronize_session=False)
     request.tm.commit()
-    Datastores.es.transport.perform_request(
-        "DELETE", "/{}/{}/_query".format("rcae_l_*", "log"), body=es_query
+    Datastores.es.delete_by_query(
+        index="rcae_l_*",  doc_type="log", body=es_query, conflicts="proceed"
     )

backend/src/appenlight/models/report.py

@@ -519,8 +519,8 b' def after_update(mapper, connection, target):'
 def after_delete(mapper, connection, target):
     if not hasattr(target, "_skip_ft_index"):
         query = {"query": {"term": {"pg_id": target.id}}}
-        Datastores.es.transport.perform_request(
-            "DELETE", "/{}/{}/_query".format(target.partition_id, "report"), body=query
+        Datastores.es.delete_by_query(
+            index=target.partition_id, doc_type="report", body=query, conflicts="proceed"
         )
 
 

backend/src/appenlight/models/report_group.py

@@ -271,14 +271,12 b' def after_update(mapper, connection, target):'
 def after_delete(mapper, connection, target):
     query = {"query": {"term": {"group_id": target.id}}}
     # delete by query
-    Datastores.es.transport.perform_request(
-        "DELETE", "/{}/{}/_query".format(target.partition_id, "report"), body=query
+    Datastores.es.delete_by_query(
+        index=target.partition_id, doc_type="report", body=query, conflicts="proceed"
     )
     query = {"query": {"term": {"pg_id": target.id}}}
-    Datastores.es.transport.perform_request(
-        "DELETE",
-        "/{}/{}/_query".format(target.partition_id, "report_group"),
-        body=query,
+    Datastores.es.delete_by_query(
+        index=target.partition_id, doc_type="report_group", body=query, conflicts="proceed"
     )
 
 

backend/src/appenlight/models/services/report_group.py

@@ -64,13 +64,13 b' class ReportGroupService(BaseService):'
                         "groups": {
                             "aggs": {
                                 "sub_agg": {
-                                    "value_count": {"field": "tags.group_id.values"}
+                                    "value_count": {"field": "tags.group_id.values.keyword"}
                                 }
                             },
                             "filter": {"exists": {"field": "tags.group_id.values"}},
                         }
                     },
-                    "terms": {"field": "tags.group_id.values", "size": limit},
+                    "terms": {"field": "tags.group_id.values.keyword", "size": limit},
                 }
             },
             "query": {
@@ -143,7 +143,7 b' class ReportGroupService(BaseService):'
                 "top_groups": {
                     "terms": {
                         "size": 5000,
-                        "field": "_parent",
+                        "field": "_parent#report_group",
                         "order": {"newest": "desc"},
                     },
                     "aggs": {
@@ -445,10 +445,12 b' class ReportGroupService(BaseService):'
                     "aggs": {
                         "types": {
                             "aggs": {
-                                "sub_agg": {"terms": {"field": "tags.type.values"}}
+                                "sub_agg": {"terms": {"field": "tags.type.values.keyword"}}
                             },
                             "filter": {
-                                "and": [{"exists": {"field": "tags.type.values"}}]
+                            "bool": {
+                                "filter": [{"exists": {"field": "tags.type.values"}}]
+                            }
                             },
                         }
                     },
@@ -485,7 +487,7 b' class ReportGroupService(BaseService):'
         }
         if group_id:
             parent_agg = es_query["aggs"]["parent_agg"]
-            filters = parent_agg["aggs"]["types"]["filter"]["and"]
+            filters = parent_agg["aggs"]["types"]["filter"]["bool"]["filter"]
             filters.append({"terms": {"tags.group_id.values": [group_id]}})
 
         index_names = es_index_name_limiter(

backend/src/appenlight/models/services/report_stat.py

@@ -34,10 +34,12 b' class ReportStatService(BaseService):'
                         "sub_agg": {"value_count": {"field": "tags.group_id.values"}}
                     },
                     "filter": {
-                        "and": [
-                            {"terms": {"resource_id": [resource_id]}},
-                            {"exists": {"field": "tags.group_id.values"}},
-                        ]
+                        "bool": {
+                            "filter": [
+                                {"terms": {"resource_id": [resource_id]}},
+                                {"exists": {"field": "tags.group_id.values"}},
+                            ]
+                        }
                     },
                 }
             },

backend/src/appenlight/models/services/request_metric.py

@@ -235,6 +235,8 b' class RequestMetricService(BaseService):'
         script_text = "doc['tags.main.numeric_values'].value / {}".format(
             total_time_spent
         )
+        if total_time_spent == 0:
+            script_text = '0'
 
         if index_names and filter_settings["resource"]:
             es_query = {
@@ -255,7 +257,6 b' class RequestMetricService(BaseService):'
                                 "aggs": {
                                     "sub_agg": {
                                         "sum": {
-                                            "lang": "expression",
                                             "script": script_text,
                                         }
                                     }
@@ -276,7 +277,7 b' class RequestMetricService(BaseService):'
                             },
                         },
                         "terms": {
-                            "field": "tags.view_name.values",
+                            "field": "tags.view_name.values.keyword",
                             "order": {"percentage>sub_agg": "desc"},
                             "size": 15,
                         },
@@ -317,7 +318,7 b' class RequestMetricService(BaseService):'
         query = {
             "aggs": {
                 "top_reports": {
-                    "terms": {"field": "tags.view_name.values", "size": len(series)},
+                    "terms": {"field": "tags.view_name.values.keyword", "size": len(series)},
                     "aggs": {
                         "top_calls_hits": {
                             "top_hits": {"sort": {"start_time": "desc"}, "size": 5}
@@ -390,18 +391,20 b' class RequestMetricService(BaseService):'
                                     }
                                 },
                                 "filter": {
-                                    "and": [
-                                        {
-                                            "range": {
-                                                "tags.main.numeric_values": {"gte": "4"}
-                                            }
-                                        },
-                                        {
-                                            "exists": {
-                                                "field": "tags.requests.numeric_values"
-                                            }
-                                        },
-                                    ]
+                                    "bool": {
+                                        "filter": [
+                                            {
+                                                "range": {
+                                                    "tags.main.numeric_values": {"gte": "4"}
+                                                }
+                                            },
+                                            {
+                                                "exists": {
+                                                    "field": "tags.requests.numeric_values"
+                                                }
+                                            },
+                                        ]
+                                    }
                                 },
                             },
                             "main": {
@@ -431,7 +434,7 b' class RequestMetricService(BaseService):'
                                     }
                                 },
                                 "filter": {
-                                    "and": [
+                                    "bool": {"filter": [
                                         {
                                             "range": {
                                                 "tags.main.numeric_values": {"gte": "1"}
@@ -447,11 +450,11 b' class RequestMetricService(BaseService):'
                                                 "field": "tags.requests.numeric_values"
                                             }
                                         },
-                                    ]
+                                    ]}
                                 },
                             },
                         },
-                        "terms": {"field": "tags.server_name.values", "size": 999999},
+                        "terms": {"field": "tags.server_name.values.keyword", "size": 999999},
                     }
                 },
                 "query": {
@@ -517,18 +520,20 b' class RequestMetricService(BaseService):'
                                     }
                                 },
                                 "filter": {
-                                    "and": [
-                                        {"terms": {"tags.type.values": [report_type]}},
-                                        {
-                                            "exists": {
-                                                "field": "tags.occurences.numeric_values"
-                                            }
-                                        },
-                                    ]
+                                    "bool": {
+                                        "filter": [
+                                            {"terms": {"tags.type.values": [report_type]}},
+                                            {
+                                                "exists": {
+                                                    "field": "tags.occurences.numeric_values"
+                                                }
+                                            },
+                                        ]
+                                    }
                                 },
                             }
                         },
-                        "terms": {"field": "tags.server_name.values", "size": 999999},
+                        "terms": {"field": "tags.server_name.values.keyword", "size": 999999},
                     }
                 },
                 "query": {
@@ -586,10 +591,10 b' class RequestMetricService(BaseService):'
         server_stats = list(stats.values())
         for stat in server_stats:
             stat["satisfying_requests"] = (
-                stat["requests"]
-                - stat["errors"]
-                - stat["frustrating_requests"]
-                - stat["tolerated_requests"]
+                    stat["requests"]
+                    - stat["errors"]
+                    - stat["frustrating_requests"]
+                    - stat["tolerated_requests"]
             )
             if stat["satisfying_requests"] < 0:
                 stat["satisfying_requests"] = 0
@@ -599,7 +604,7 b' class RequestMetricService(BaseService):'
                     stat["response_time"] / stat["requests"], 3
                 )
                 qual_requests = (
-                    stat["satisfying_requests"] + stat["tolerated_requests"] / 2.0
+                        stat["satisfying_requests"] + stat["tolerated_requests"] / 2.0
                 )
                 stat["apdex"] = round((qual_requests / stat["requests"]) * 100, 2)
                 stat["rpm"] = round(stat["requests"] / stat["total_minutes"], 2)

backend/src/appenlight/models/services/slow_call.py

@@ -50,7 +50,7 b' class SlowCallService(BaseService):'
                                 "aggs": {
                                     "sub_agg": {
                                         "value_count": {
-                                            "field": "tags.statement_hash.values"
+                                            "field": "tags.statement_hash.values.keyword"
                                         }
                                     }
                                 },
@@ -60,7 +60,7 b' class SlowCallService(BaseService):'
                             },
                         },
                         "terms": {
-                            "field": "tags.statement_hash.values",
+                            "field": "tags.statement_hash.values.keyword",
                             "order": {"duration>sub_agg": "desc"},
                             "size": 15,
                         },
@@ -98,7 +98,7 b' class SlowCallService(BaseService):'
         calls_query = {
             "aggs": {
                 "top_calls": {
-                    "terms": {"field": "tags.statement_hash.values", "size": 15},
+                    "terms": {"field": "tags.statement_hash.values.keyword", "size": 15},
                     "aggs": {
                         "top_calls_hits": {
                             "top_hits": {"sort": {"timestamp": "desc"}, "size": 5}

backend/src/appenlight/scripts/reindex_elasticsearch.py

@@ -139,7 +139,13 b' def update_template():'
                 "mapping": {
                     "type": "object",
                     "properties": {
-                        "values": {"type": "string", "analyzer": "tag_value"},
+                        "values": {"type": "text", "analyzer": "tag_value",
+                                   "fields": {
+                                       "keyword": {
+                                           "type": "keyword",
+                                           "ignore_above": 256
+                                       }
+                                   }},
                         "numeric_values": {"type": "float"},
                     },
                 },
@@ -177,10 +183,10 b' def update_template():'
                 "_all": {"enabled": False},
                 "dynamic_templates": tag_templates,
                 "properties": {
-                    "pg_id": {"type": "string", "index": "not_analyzed"},
+                    "pg_id": {"type": "keyword", "index": True},
                     "resource_id": {"type": "integer"},
                     "priority": {"type": "integer"},
-                    "error": {"type": "string", "analyzer": "simple"},
+                    "error": {"type": "text", "analyzer": "simple"},
                     "read": {"type": "boolean"},
                     "occurences": {"type": "integer"},
                     "fixed": {"type": "boolean"},
@@ -195,21 +201,27 b' def update_template():'
                 "_all": {"enabled": False},
                 "dynamic_templates": tag_templates,
                 "properties": {
-                    "pg_id": {"type": "string", "index": "not_analyzed"},
+                    "pg_id": {"type": "keyword", "index": True},
                     "resource_id": {"type": "integer"},
-                    "group_id": {"type": "string"},
+                    "group_id": {"type": "keyword"},
                     "http_status": {"type": "integer"},
-                    "ip": {"type": "string", "index": "not_analyzed"},
-                    "url_domain": {"type": "string", "analyzer": "simple"},
-                    "url_path": {"type": "string", "analyzer": "url_path"},
-                    "error": {"type": "string", "analyzer": "simple"},
+                    "ip": {"type": "keyword", "index": True},
+                    "url_domain": {"type": "text", "analyzer": "simple"},
+                    "url_path": {"type": "text", "analyzer": "url_path"},
+                    "error": {"type": "text", "analyzer": "simple"},
                     "report_type": {"type": "integer"},
                     "start_time": {"type": "date"},
-                    "request_id": {"type": "string", "index": "not_analyzed"},
+                    "request_id": {"type": "keyword", "index": True},
                     "end_time": {"type": "date"},
                     "duration": {"type": "float"},
                     "tags": {"type": "object"},
-                    "tag_list": {"type": "string", "analyzer": "tag_value"},
+                    "tag_list": {"type": "text", "analyzer": "tag_value",
+                                 "fields": {
+                                     "keyword": {
+                                         "type": "keyword",
+                                         "ignore_above": 256
+                                     }
+                                 }},
                     "extra": {"type": "object"},
                 },
                 "_parent": {"type": "report_group"},
@@ -218,17 +230,26 b' def update_template():'
                 "_all": {"enabled": False},
                 "dynamic_templates": tag_templates,
                 "properties": {
-                    "pg_id": {"type": "string", "index": "not_analyzed"},
-                    "delete_hash": {"type": "string", "index": "not_analyzed"},
+                    "pg_id": {"type": "keyword", "index": True},
+                    "delete_hash": {"type": "keyword", "index": True},
                     "resource_id": {"type": "integer"},
                     "timestamp": {"type": "date"},
                     "permanent": {"type": "boolean"},
-                    "request_id": {"type": "string", "index": "not_analyzed"},
-                    "log_level": {"type": "string", "analyzer": "simple"},
-                    "message": {"type": "string", "analyzer": "simple"},
-                    "namespace": {"type": "string", "index": "not_analyzed"},
+                    "request_id": {"type": "keyword", "index": True},
+                    "log_level": {"type": "text", "analyzer": "simple"},
+                    "message": {"type": "text", "analyzer": "simple"},
+                    "namespace": {
+                        "type": "text",
+                        "fields": {"keyword": {"type": "keyword", "ignore_above": 256}},
+                    },
                     "tags": {"type": "object"},
-                    "tag_list": {"type": "string", "analyzer": "tag_value"},
+                    "tag_list": {"type": "text", "analyzer": "tag_value",
+                                 "fields": {
+                                     "keyword": {
+                                         "type": "keyword",
+                                         "ignore_above": 256
+                                     }
+                                 }},
                 },
             },
         },

backend/src/appenlight/views/logs.py

@@ -171,13 +171,13 b' def common_tags(request):'
     if namespaces:
         filter_part.append({"terms": {"namespace": namespaces}})
 
-    query["aggs"] = {"sub_agg": {"terms": {"field": "tag_list", "size": 50}}}
+    query["aggs"] = {"sub_agg": {"terms": {"field": "tag_list.keyword", "size": 50}}}
     # tags
     index_names = es_index_name_limiter(ixtypes=[config.get("datasource", "logs")])
     result = Datastores.es.search(body=query, index=index_names, doc_type="log", size=0)
     tag_buckets = result["aggregations"]["sub_agg"].get("buckets", [])
     # namespaces
-    query["aggs"] = {"sub_agg": {"terms": {"field": "namespace", "size": 50}}}
+    query["aggs"] = {"sub_agg": {"terms": {"field": "namespace.keyword", "size": 50}}}
     result = Datastores.es.search(body=query, index=index_names, doc_type="log", size=0)
     namespaces_buckets = result["aggregations"]["sub_agg"].get("buckets", [])
     return {