From 765d965d661ccc665b4659dfd37131a8c8da28e3 2019-04-07 13:30:54
From: Marcin Lulek <info@webreactor.eu>
Date: 2019-04-07 13:30:54
Subject: [PATCH] reformat: black

---

diff --git a/backend/src/appenlight/celery/tasks.py b/backend/src/appenlight/celery/tasks.py
index 416f8f3..46b7063 100644
--- a/backend/src/appenlight/celery/tasks.py
+++ b/backend/src/appenlight/celery/tasks.py
@@ -326,8 +326,10 @@ def add_logs(resource_id, request_params, dataset, **kwargs):
 
                         try:
                             Datastores.es.delete_by_query(
-                                index=es_index, doc_type="log",
-                                body=query, conflicts="proceed"
+                                index=es_index,
+                                doc_type="log",
+                                body=query,
+                                conflicts="proceed",
                             )
                         except elasticsearch.exceptions.NotFoundError as exc:
                             msg = "skipping index {}".format(es_index)
@@ -688,11 +690,7 @@ def alerting_reports():
 def logs_cleanup(resource_id, filter_settings):
     request = get_current_request()
     request.tm.begin()
-    es_query = {
-        "query": {
-            "bool": {"filter": [{"term": {"resource_id": resource_id}}]}
-        }
-    }
+    es_query = {"query": {"bool": {"filter": [{"term": {"resource_id": resource_id}}]}}}
 
     query = DBSession.query(Log).filter(Log.resource_id == resource_id)
     if filter_settings["namespace"]:
@@ -703,5 +701,5 @@ def logs_cleanup(resource_id, filter_settings):
     query.delete(synchronize_session=False)
     request.tm.commit()
     Datastores.es.delete_by_query(
-        index="rcae_l_*",  doc_type="log", body=es_query, conflicts="proceed"
+        index="rcae_l_*", doc_type="log", body=es_query, conflicts="proceed"
     )
diff --git a/backend/src/appenlight/lib/utils/__init__.py b/backend/src/appenlight/lib/utils/__init__.py
index f1cb9ca..7b9a07e 100644
--- a/backend/src/appenlight/lib/utils/__init__.py
+++ b/backend/src/appenlight/lib/utils/__init__.py
@@ -552,7 +552,9 @@ def get_es_info(cache_regions, es_conn):
     @cache_regions.memory_min_10.cache_on_arguments()
     def get_es_info_cached():
         returned_info = {"raw_info": es_conn.info()}
-        returned_info["version"] = returned_info["raw_info"]["version"]["number"].split('.')
+        returned_info["version"] = returned_info["raw_info"]["version"]["number"].split(
+            "."
+        )
         return returned_info
 
     return get_es_info_cached()
diff --git a/backend/src/appenlight/models/report.py b/backend/src/appenlight/models/report.py
index 4767574..8cceba6 100644
--- a/backend/src/appenlight/models/report.py
+++ b/backend/src/appenlight/models/report.py
@@ -483,13 +483,10 @@ class Report(Base, BaseModel):
             "ip": self.ip,
             "group_id": str(self.group_id),
             "type": "report",
-            "join_field": {
-                "name": "report",
-                "parent": str(self.group_id)
-            },
+            "join_field": {"name": "report", "parent": str(self.group_id)},
             "tags": tags,
             "tag_list": tag_list,
-            "_routing": str(self.group_id)
+            "_routing": str(self.group_id),
         }
 
     @property
@@ -525,7 +522,10 @@ def after_delete(mapper, connection, target):
     if not hasattr(target, "_skip_ft_index"):
         query = {"query": {"term": {"report_id": target.id}}}
         Datastores.es.delete_by_query(
-            index=target.partition_id, doc_type="report", body=query, conflicts="proceed"
+            index=target.partition_id,
+            doc_type="report",
+            body=query,
+            conflicts="proceed",
         )
 
 
diff --git a/backend/src/appenlight/models/report_group.py b/backend/src/appenlight/models/report_group.py
index 9ea8b94..54b9219 100644
--- a/backend/src/appenlight/models/report_group.py
+++ b/backend/src/appenlight/models/report_group.py
@@ -191,9 +191,7 @@ class ReportGroup(Base, BaseModel):
             "first_timestamp": self.first_timestamp,
             "last_timestamp": self.last_timestamp,
             "type": "report_group",
-            "join_field": {
-                "name": "report_group"
-            },
+            "join_field": {"name": "report_group"},
         }
 
     def set_notification_info(self, notify_10=False, notify_100=False):
diff --git a/backend/src/appenlight/models/services/log.py b/backend/src/appenlight/models/services/log.py
index 3a93c4a..c3da530 100644
--- a/backend/src/appenlight/models/services/log.py
+++ b/backend/src/appenlight/models/services/log.py
@@ -56,11 +56,7 @@ class LogService(BaseService):
             filter_settings = {}
 
         query = {
-            "query": {
-                "bool": {
-                    "filter": [{"terms": {"resource_id": list(app_ids)}}]
-                }
-            }
+            "query": {"bool": {"filter": [{"terms": {"resource_id": list(app_ids)}}]}}
         }
 
         start_date = filter_settings.get("start_date")
@@ -132,13 +128,13 @@ class LogService(BaseService):
 
     @classmethod
     def get_search_iterator(
-            cls,
-            app_ids=None,
-            page=1,
-            items_per_page=50,
-            order_by=None,
-            filter_settings=None,
-            limit=None,
+        cls,
+        app_ids=None,
+        page=1,
+        items_per_page=50,
+        order_by=None,
+        filter_settings=None,
+        limit=None,
     ):
         if not app_ids:
             return {}, 0
@@ -171,15 +167,15 @@ class LogService(BaseService):
 
     @classmethod
     def get_paginator_by_app_ids(
-            cls,
-            app_ids=None,
-            page=1,
-            item_count=None,
-            items_per_page=50,
-            order_by=None,
-            filter_settings=None,
-            exclude_columns=None,
-            db_session=None,
+        cls,
+        app_ids=None,
+        page=1,
+        item_count=None,
+        items_per_page=50,
+        order_by=None,
+        filter_settings=None,
+        exclude_columns=None,
+        db_session=None,
     ):
         if not filter_settings:
             filter_settings = {}
diff --git a/backend/src/appenlight/models/services/report_group.py b/backend/src/appenlight/models/services/report_group.py
index d1e8ba7..b90e3eb 100644
--- a/backend/src/appenlight/models/services/report_group.py
+++ b/backend/src/appenlight/models/services/report_group.py
@@ -64,7 +64,9 @@ class ReportGroupService(BaseService):
                         "groups": {
                             "aggs": {
                                 "sub_agg": {
-                                    "value_count": {"field": "tags.group_id.values.keyword"}
+                                    "value_count": {
+                                        "field": "tags.group_id.values.keyword"
+                                    }
                                 }
                             },
                             "filter": {"exists": {"field": "tags.group_id.values"}},
@@ -76,11 +78,7 @@ class ReportGroupService(BaseService):
             "query": {
                 "bool": {
                     "filter": [
-                        {
-                            "terms": {
-                                "resource_id": [filter_settings["resource"][0]]
-                            }
-                        },
+                        {"terms": {"resource_id": [filter_settings["resource"][0]]}},
                         {
                             "range": {
                                 "timestamp": {
@@ -136,7 +134,7 @@ class ReportGroupService(BaseService):
                 "bool": {
                     "must": [],
                     "should": [],
-                    "filter": [{"terms": {"resource_id": list(app_ids)}}]
+                    "filter": [{"terms": {"resource_id": list(app_ids)}}],
                 }
             },
             "aggs": {
@@ -315,7 +313,9 @@ class ReportGroupService(BaseService):
         ordered_ids = []
         if results:
             for item in results["top_groups"]["buckets"]:
-                pg_id = item["top_reports_hits"]["hits"]["hits"][0]["_source"]["report_id"]
+                pg_id = item["top_reports_hits"]["hits"]["hits"][0]["_source"][
+                    "report_id"
+                ]
                 ordered_ids.append(pg_id)
         log.info(filter_settings)
         paginator = paginate.Page(
@@ -445,12 +445,16 @@ class ReportGroupService(BaseService):
                     "aggs": {
                         "types": {
                             "aggs": {
-                                "sub_agg": {"terms": {"field": "tags.type.values.keyword"}}
+                                "sub_agg": {
+                                    "terms": {"field": "tags.type.values.keyword"}
+                                }
                             },
                             "filter": {
-                            "bool": {
-                                "filter": [{"exists": {"field": "tags.type.values"}}]
-                            }
+                                "bool": {
+                                    "filter": [
+                                        {"exists": {"field": "tags.type.values"}}
+                                    ]
+                                }
                             },
                         }
                     },
@@ -468,11 +472,7 @@ class ReportGroupService(BaseService):
             "query": {
                 "bool": {
                     "filter": [
-                        {
-                            "terms": {
-                                "resource_id": [filter_settings["resource"][0]]
-                            }
-                        },
+                        {"terms": {"resource_id": [filter_settings["resource"][0]]}},
                         {
                             "range": {
                                 "timestamp": {
diff --git a/backend/src/appenlight/models/services/report_stat.py b/backend/src/appenlight/models/services/report_stat.py
index 87d3377..811ebe4 100644
--- a/backend/src/appenlight/models/services/report_stat.py
+++ b/backend/src/appenlight/models/services/report_stat.py
@@ -31,7 +31,9 @@ class ReportStatService(BaseService):
             "aggs": {
                 "reports": {
                     "aggs": {
-                        "sub_agg": {"value_count": {"field": "tags.group_id.values.keyword"}}
+                        "sub_agg": {
+                            "value_count": {"field": "tags.group_id.values.keyword"}
+                        }
                     },
                     "filter": {
                         "bool": {
diff --git a/backend/src/appenlight/models/services/request_metric.py b/backend/src/appenlight/models/services/request_metric.py
index c80a00c..f6271cc 100644
--- a/backend/src/appenlight/models/services/request_metric.py
+++ b/backend/src/appenlight/models/services/request_metric.py
@@ -142,11 +142,7 @@ class RequestMetricService(BaseService):
             "query": {
                 "bool": {
                     "filter": [
-                        {
-                            "terms": {
-                                "resource_id": [filter_settings["resource"][0]]
-                            }
-                        },
+                        {"terms": {"resource_id": [filter_settings["resource"][0]]}},
                         {
                             "range": {
                                 "timestamp": {
@@ -236,7 +232,7 @@ class RequestMetricService(BaseService):
             total_time_spent
         )
         if total_time_spent == 0:
-            script_text = '0'
+            script_text = "0"
 
         if index_names and filter_settings["resource"]:
             es_query = {
@@ -254,13 +250,7 @@ class RequestMetricService(BaseService):
                                 },
                             },
                             "percentage": {
-                                "aggs": {
-                                    "sub_agg": {
-                                        "sum": {
-                                            "script": script_text,
-                                        }
-                                    }
-                                },
+                                "aggs": {"sub_agg": {"sum": {"script": script_text}}},
                                 "filter": {
                                     "exists": {"field": "tags.main.numeric_values"}
                                 },
@@ -318,7 +308,10 @@ class RequestMetricService(BaseService):
         query = {
             "aggs": {
                 "top_reports": {
-                    "terms": {"field": "tags.view_name.values.keyword", "size": len(series)},
+                    "terms": {
+                        "field": "tags.view_name.values.keyword",
+                        "size": len(series),
+                    },
                     "aggs": {
                         "top_calls_hits": {
                             "top_hits": {"sort": {"start_time": "desc"}, "size": 5}
@@ -395,7 +388,9 @@ class RequestMetricService(BaseService):
                                         "filter": [
                                             {
                                                 "range": {
-                                                    "tags.main.numeric_values": {"gte": "4"}
+                                                    "tags.main.numeric_values": {
+                                                        "gte": "4"
+                                                    }
                                                 }
                                             },
                                             {
@@ -434,27 +429,36 @@ class RequestMetricService(BaseService):
                                     }
                                 },
                                 "filter": {
-                                    "bool": {"filter": [
-                                        {
-                                            "range": {
-                                                "tags.main.numeric_values": {"gte": "1"}
-                                            }
-                                        },
-                                        {
-                                            "range": {
-                                                "tags.main.numeric_values": {"lt": "4"}
-                                            }
-                                        },
-                                        {
-                                            "exists": {
-                                                "field": "tags.requests.numeric_values"
-                                            }
-                                        },
-                                    ]}
+                                    "bool": {
+                                        "filter": [
+                                            {
+                                                "range": {
+                                                    "tags.main.numeric_values": {
+                                                        "gte": "1"
+                                                    }
+                                                }
+                                            },
+                                            {
+                                                "range": {
+                                                    "tags.main.numeric_values": {
+                                                        "lt": "4"
+                                                    }
+                                                }
+                                            },
+                                            {
+                                                "exists": {
+                                                    "field": "tags.requests.numeric_values"
+                                                }
+                                            },
+                                        ]
+                                    }
                                 },
                             },
                         },
-                        "terms": {"field": "tags.server_name.values.keyword", "size": 999999},
+                        "terms": {
+                            "field": "tags.server_name.values.keyword",
+                            "size": 999999,
+                        },
                     }
                 },
                 "query": {
@@ -522,7 +526,11 @@ class RequestMetricService(BaseService):
                                 "filter": {
                                     "bool": {
                                         "filter": [
-                                            {"terms": {"tags.type.values": [report_type]}},
+                                            {
+                                                "terms": {
+                                                    "tags.type.values": [report_type]
+                                                }
+                                            },
                                             {
                                                 "exists": {
                                                     "field": "tags.occurences.numeric_values"
@@ -533,7 +541,10 @@ class RequestMetricService(BaseService):
                                 },
                             }
                         },
-                        "terms": {"field": "tags.server_name.values.keyword", "size": 999999},
+                        "terms": {
+                            "field": "tags.server_name.values.keyword",
+                            "size": 999999,
+                        },
                     }
                 },
                 "query": {
@@ -591,10 +602,10 @@ class RequestMetricService(BaseService):
         server_stats = list(stats.values())
         for stat in server_stats:
             stat["satisfying_requests"] = (
-                    stat["requests"]
-                    - stat["errors"]
-                    - stat["frustrating_requests"]
-                    - stat["tolerated_requests"]
+                stat["requests"]
+                - stat["errors"]
+                - stat["frustrating_requests"]
+                - stat["tolerated_requests"]
             )
             if stat["satisfying_requests"] < 0:
                 stat["satisfying_requests"] = 0
@@ -604,7 +615,7 @@ class RequestMetricService(BaseService):
                     stat["response_time"] / stat["requests"], 3
                 )
                 qual_requests = (
-                        stat["satisfying_requests"] + stat["tolerated_requests"] / 2.0
+                    stat["satisfying_requests"] + stat["tolerated_requests"] / 2.0
                 )
                 stat["apdex"] = round((qual_requests / stat["requests"]) * 100, 2)
                 stat["rpm"] = round(stat["requests"] / stat["total_minutes"], 2)
diff --git a/backend/src/appenlight/models/services/slow_call.py b/backend/src/appenlight/models/services/slow_call.py
index e074fbd..8de1105 100644
--- a/backend/src/appenlight/models/services/slow_call.py
+++ b/backend/src/appenlight/models/services/slow_call.py
@@ -98,7 +98,10 @@ class SlowCallService(BaseService):
         calls_query = {
             "aggs": {
                 "top_calls": {
-                    "terms": {"field": "tags.statement_hash.values.keyword", "size": 15},
+                    "terms": {
+                        "field": "tags.statement_hash.values.keyword",
+                        "size": 15,
+                    },
                     "aggs": {
                         "top_calls_hits": {
                             "top_hits": {"sort": {"timestamp": "desc"}, "size": 5}
@@ -109,11 +112,7 @@ class SlowCallService(BaseService):
             "query": {
                 "bool": {
                     "filter": [
-                        {
-                            "terms": {
-                                "resource_id": [filter_settings["resource"][0]]
-                            }
-                        },
+                        {"terms": {"resource_id": [filter_settings["resource"][0]]}},
                         {"terms": {"tags.statement_hash.values": hashes}},
                         {
                             "range": {
diff --git a/backend/src/appenlight/scripts/reindex_elasticsearch.py b/backend/src/appenlight/scripts/reindex_elasticsearch.py
index 4bac808..746bb6e 100644
--- a/backend/src/appenlight/scripts/reindex_elasticsearch.py
+++ b/backend/src/appenlight/scripts/reindex_elasticsearch.py
@@ -152,13 +152,13 @@ def update_template():
                 "mapping": {
                     "type": "object",
                     "properties": {
-                        "values": {"type": "text", "analyzer": "tag_value",
-                                   "fields": {
-                                       "keyword": {
-                                           "type": "keyword",
-                                           "ignore_above": 256
-                                       }
-                                   }},
+                        "values": {
+                            "type": "text",
+                            "analyzer": "tag_value",
+                            "fields": {
+                                "keyword": {"type": "keyword", "ignore_above": 256}
+                            },
+                        },
                         "numeric_values": {"type": "float"},
                     },
                 },
@@ -200,13 +200,11 @@ def update_template():
                 "fields": {"keyword": {"type": "keyword", "ignore_above": 256}},
             },
             "tags": {"type": "object"},
-            "tag_list": {"type": "text", "analyzer": "tag_value",
-                         "fields": {
-                             "keyword": {
-                                 "type": "keyword",
-                                 "ignore_above": 256
-                             }
-                         }},
+            "tag_list": {
+                "type": "text",
+                "analyzer": "tag_value",
+                "fields": {"keyword": {"type": "keyword", "ignore_above": 256}},
+            },
         },
     }
 
@@ -215,7 +213,7 @@ def update_template():
         "settings": {
             "index": {
                 "refresh_interval": "5s",
-                "translog": {"sync_interval": "5s", "durability": "async"}
+                "translog": {"sync_interval": "5s", "durability": "async"},
             },
             "number_of_shards": 5,
             "analysis": shared_analysis,
@@ -240,7 +238,6 @@ def update_template():
                     "summed_duration": {"type": "float"},
                     "public": {"type": "boolean"},
                     # report
-
                     "report_id": {"type": "keyword", "index": True},
                     "http_status": {"type": "integer"},
                     "ip": {"type": "keyword", "index": True},
@@ -252,17 +249,13 @@ def update_template():
                     "end_time": {"type": "date"},
                     "duration": {"type": "float"},
                     "tags": {"type": "object"},
-                    "tag_list": {"type": "text", "analyzer": "tag_value",
-                                 "fields": {
-                                     "keyword": {
-                                         "type": "keyword",
-                                         "ignore_above": 256
-                                     }
-                                 }},
+                    "tag_list": {
+                        "type": "text",
+                        "analyzer": "tag_value",
+                        "fields": {"keyword": {"type": "keyword", "ignore_above": 256}},
+                    },
                     "extra": {"type": "object"},
-
                     # report stats
-
                     "report_stat_id": {"type": "keyword", "index": True},
                     "timestamp": {"type": "date"},
                     "permanent": {"type": "boolean"},
@@ -272,17 +265,13 @@ def update_template():
                         "type": "text",
                         "fields": {"keyword": {"type": "keyword", "ignore_above": 256}},
                     },
-
                     "join_field": {
                         "type": "join",
-                        "relations": {
-                            "report_group": ["report", "report_stat"]
-                        }
-                    }
-
+                        "relations": {"report_group": ["report", "report_stat"]},
+                    },
                 },
             }
-        }
+        },
     }
 
     Datastores.es.indices.put_template("rcae_reports", body=report_schema)
@@ -301,15 +290,15 @@ def update_template():
             "number_of_shards": 5,
             "analysis": shared_analysis,
         },
-        "mappings": {
-            "log": logs_mapping,
-        },
+        "mappings": {"log": logs_mapping},
     }
 
     Datastores.es.indices.put_template("rcae_logs", body=log_template)
 
     slow_call_mapping = copy.deepcopy(shared_log_mapping)
-    slow_call_mapping["properties"]["slow_call_id"] = slow_call_mapping["properties"]["pg_id"]
+    slow_call_mapping["properties"]["slow_call_id"] = slow_call_mapping["properties"][
+        "pg_id"
+    ]
     del slow_call_mapping["properties"]["pg_id"]
 
     slow_call_template = {
@@ -322,9 +311,7 @@ def update_template():
             "number_of_shards": 5,
             "analysis": shared_analysis,
         },
-        "mappings": {
-            "log": slow_call_mapping,
-        },
+        "mappings": {"log": slow_call_mapping},
     }
 
     Datastores.es.indices.put_template("rcae_slow_calls", body=slow_call_template)
@@ -343,15 +330,15 @@ def update_template():
             "number_of_shards": 5,
             "analysis": shared_analysis,
         },
-        "mappings": {
-            "log": metric_mapping,
-        },
+        "mappings": {"log": metric_mapping},
     }
 
     Datastores.es.indices.put_template("rcae_metrics", body=metrics_template)
 
     uptime_metric_mapping = copy.deepcopy(shared_log_mapping)
-    uptime_metric_mapping["properties"]["uptime_id"] = uptime_metric_mapping["properties"]["pg_id"]
+    uptime_metric_mapping["properties"]["uptime_id"] = uptime_metric_mapping[
+        "properties"
+    ]["pg_id"]
     del uptime_metric_mapping["properties"]["pg_id"]
 
     uptime_metrics_template = {
@@ -364,12 +351,12 @@ def update_template():
             "number_of_shards": 5,
             "analysis": shared_analysis,
         },
-        "mappings": {
-            "log": shared_log_mapping,
-        },
+        "mappings": {"log": shared_log_mapping},
     }
 
-    Datastores.es.indices.put_template("rcae_uptime_metrics", body=uptime_metrics_template)
+    Datastores.es.indices.put_template(
+        "rcae_uptime_metrics", body=uptime_metrics_template
+    )
 
 
 def reindex_reports():
diff --git a/backend/src/appenlight/views/logs.py b/backend/src/appenlight/views/logs.py
index 96aaa8a..0a92aef 100644
--- a/backend/src/appenlight/views/logs.py
+++ b/backend/src/appenlight/views/logs.py
@@ -146,11 +146,7 @@ def common_tags(request):
 
     resources = list(filter_settings["resource"])
     query = {
-        "query": {
-            "bool": {
-                "filter": [{"terms": {"resource_id": list(resources)}}]
-            }
-        }
+        "query": {"bool": {"filter": [{"terms": {"resource_id": list(resources)}}]}}
     }
     start_date = filter_settings.get("start_date")
     end_date = filter_settings.get("end_date")
@@ -199,18 +195,10 @@ def common_values(request):
     resources = list(filter_settings["resource"])
     tag_name = filter_settings["tags"][0]["value"][0]
 
-    and_part = [
-        {"terms": {"resource_id": list(resources)}},
-    ]
+    and_part = [{"terms": {"resource_id": list(resources)}}]
     if filter_settings["namespace"]:
         and_part.append({"terms": {"namespace": filter_settings["namespace"]}})
-    query = {
-        "query": {
-            "bool": {
-                "filter": and_part
-            }
-        }
-    }
+    query = {"query": {"bool": {"filter": and_part}}}
     query["aggs"] = {
         "sub_agg": {"terms": {"field": "tags.{}.values".format(tag_name), "size": 50}}
     }