##// END OF EJS Templates
reformat: black
ergo -
Show More
@@ -326,8 +326,10 b' def add_logs(resource_id, request_params, dataset, **kwargs):'
326
326
327 try:
327 try:
328 Datastores.es.delete_by_query(
328 Datastores.es.delete_by_query(
329 index=es_index, doc_type="log",
329 index=es_index,
330 body=query, conflicts="proceed"
330 doc_type="log",
331 body=query,
332 conflicts="proceed",
331 )
333 )
332 except elasticsearch.exceptions.NotFoundError as exc:
334 except elasticsearch.exceptions.NotFoundError as exc:
333 msg = "skipping index {}".format(es_index)
335 msg = "skipping index {}".format(es_index)
@@ -688,11 +690,7 b' def alerting_reports():'
688 def logs_cleanup(resource_id, filter_settings):
690 def logs_cleanup(resource_id, filter_settings):
689 request = get_current_request()
691 request = get_current_request()
690 request.tm.begin()
692 request.tm.begin()
691 es_query = {
693 es_query = {"query": {"bool": {"filter": [{"term": {"resource_id": resource_id}}]}}}
692 "query": {
693 "bool": {"filter": [{"term": {"resource_id": resource_id}}]}
694 }
695 }
696
694
697 query = DBSession.query(Log).filter(Log.resource_id == resource_id)
695 query = DBSession.query(Log).filter(Log.resource_id == resource_id)
698 if filter_settings["namespace"]:
696 if filter_settings["namespace"]:
@@ -552,7 +552,9 b' def get_es_info(cache_regions, es_conn):'
552 @cache_regions.memory_min_10.cache_on_arguments()
552 @cache_regions.memory_min_10.cache_on_arguments()
553 def get_es_info_cached():
553 def get_es_info_cached():
554 returned_info = {"raw_info": es_conn.info()}
554 returned_info = {"raw_info": es_conn.info()}
555 returned_info["version"] = returned_info["raw_info"]["version"]["number"].split('.')
555 returned_info["version"] = returned_info["raw_info"]["version"]["number"].split(
556 "."
557 )
556 return returned_info
558 return returned_info
557
559
558 return get_es_info_cached()
560 return get_es_info_cached()
@@ -483,13 +483,10 b' class Report(Base, BaseModel):'
483 "ip": self.ip,
483 "ip": self.ip,
484 "group_id": str(self.group_id),
484 "group_id": str(self.group_id),
485 "type": "report",
485 "type": "report",
486 "join_field": {
486 "join_field": {"name": "report", "parent": str(self.group_id)},
487 "name": "report",
488 "parent": str(self.group_id)
489 },
490 "tags": tags,
487 "tags": tags,
491 "tag_list": tag_list,
488 "tag_list": tag_list,
492 "_routing": str(self.group_id)
489 "_routing": str(self.group_id),
493 }
490 }
494
491
495 @property
492 @property
@@ -525,7 +522,10 b' def after_delete(mapper, connection, target):'
525 if not hasattr(target, "_skip_ft_index"):
522 if not hasattr(target, "_skip_ft_index"):
526 query = {"query": {"term": {"report_id": target.id}}}
523 query = {"query": {"term": {"report_id": target.id}}}
527 Datastores.es.delete_by_query(
524 Datastores.es.delete_by_query(
528 index=target.partition_id, doc_type="report", body=query, conflicts="proceed"
525 index=target.partition_id,
526 doc_type="report",
527 body=query,
528 conflicts="proceed",
529 )
529 )
530
530
531
531
@@ -191,9 +191,7 b' class ReportGroup(Base, BaseModel):'
191 "first_timestamp": self.first_timestamp,
191 "first_timestamp": self.first_timestamp,
192 "last_timestamp": self.last_timestamp,
192 "last_timestamp": self.last_timestamp,
193 "type": "report_group",
193 "type": "report_group",
194 "join_field": {
194 "join_field": {"name": "report_group"},
195 "name": "report_group"
196 },
197 }
195 }
198
196
199 def set_notification_info(self, notify_10=False, notify_100=False):
197 def set_notification_info(self, notify_10=False, notify_100=False):
@@ -56,11 +56,7 b' class LogService(BaseService):'
56 filter_settings = {}
56 filter_settings = {}
57
57
58 query = {
58 query = {
59 "query": {
59 "query": {"bool": {"filter": [{"terms": {"resource_id": list(app_ids)}}]}}
60 "bool": {
61 "filter": [{"terms": {"resource_id": list(app_ids)}}]
62 }
63 }
64 }
60 }
65
61
66 start_date = filter_settings.get("start_date")
62 start_date = filter_settings.get("start_date")
@@ -64,7 +64,9 b' class ReportGroupService(BaseService):'
64 "groups": {
64 "groups": {
65 "aggs": {
65 "aggs": {
66 "sub_agg": {
66 "sub_agg": {
67 "value_count": {"field": "tags.group_id.values.keyword"}
67 "value_count": {
68 "field": "tags.group_id.values.keyword"
69 }
68 }
70 }
69 },
71 },
70 "filter": {"exists": {"field": "tags.group_id.values"}},
72 "filter": {"exists": {"field": "tags.group_id.values"}},
@@ -76,11 +78,7 b' class ReportGroupService(BaseService):'
76 "query": {
78 "query": {
77 "bool": {
79 "bool": {
78 "filter": [
80 "filter": [
79 {
81 {"terms": {"resource_id": [filter_settings["resource"][0]]}},
80 "terms": {
81 "resource_id": [filter_settings["resource"][0]]
82 }
83 },
84 {
82 {
85 "range": {
83 "range": {
86 "timestamp": {
84 "timestamp": {
@@ -136,7 +134,7 b' class ReportGroupService(BaseService):'
136 "bool": {
134 "bool": {
137 "must": [],
135 "must": [],
138 "should": [],
136 "should": [],
139 "filter": [{"terms": {"resource_id": list(app_ids)}}]
137 "filter": [{"terms": {"resource_id": list(app_ids)}}],
140 }
138 }
141 },
139 },
142 "aggs": {
140 "aggs": {
@@ -315,7 +313,9 b' class ReportGroupService(BaseService):'
315 ordered_ids = []
313 ordered_ids = []
316 if results:
314 if results:
317 for item in results["top_groups"]["buckets"]:
315 for item in results["top_groups"]["buckets"]:
318 pg_id = item["top_reports_hits"]["hits"]["hits"][0]["_source"]["report_id"]
316 pg_id = item["top_reports_hits"]["hits"]["hits"][0]["_source"][
317 "report_id"
318 ]
319 ordered_ids.append(pg_id)
319 ordered_ids.append(pg_id)
320 log.info(filter_settings)
320 log.info(filter_settings)
321 paginator = paginate.Page(
321 paginator = paginate.Page(
@@ -445,11 +445,15 b' class ReportGroupService(BaseService):'
445 "aggs": {
445 "aggs": {
446 "types": {
446 "types": {
447 "aggs": {
447 "aggs": {
448 "sub_agg": {"terms": {"field": "tags.type.values.keyword"}}
448 "sub_agg": {
449 "terms": {"field": "tags.type.values.keyword"}
450 }
449 },
451 },
450 "filter": {
452 "filter": {
451 "bool": {
453 "bool": {
452 "filter": [{"exists": {"field": "tags.type.values"}}]
454 "filter": [
455 {"exists": {"field": "tags.type.values"}}
456 ]
453 }
457 }
454 },
458 },
455 }
459 }
@@ -468,11 +472,7 b' class ReportGroupService(BaseService):'
468 "query": {
472 "query": {
469 "bool": {
473 "bool": {
470 "filter": [
474 "filter": [
471 {
475 {"terms": {"resource_id": [filter_settings["resource"][0]]}},
472 "terms": {
473 "resource_id": [filter_settings["resource"][0]]
474 }
475 },
476 {
476 {
477 "range": {
477 "range": {
478 "timestamp": {
478 "timestamp": {
@@ -31,7 +31,9 b' class ReportStatService(BaseService):'
31 "aggs": {
31 "aggs": {
32 "reports": {
32 "reports": {
33 "aggs": {
33 "aggs": {
34 "sub_agg": {"value_count": {"field": "tags.group_id.values.keyword"}}
34 "sub_agg": {
35 "value_count": {"field": "tags.group_id.values.keyword"}
36 }
35 },
37 },
36 "filter": {
38 "filter": {
37 "bool": {
39 "bool": {
@@ -142,11 +142,7 b' class RequestMetricService(BaseService):'
142 "query": {
142 "query": {
143 "bool": {
143 "bool": {
144 "filter": [
144 "filter": [
145 {
145 {"terms": {"resource_id": [filter_settings["resource"][0]]}},
146 "terms": {
147 "resource_id": [filter_settings["resource"][0]]
148 }
149 },
150 {
146 {
151 "range": {
147 "range": {
152 "timestamp": {
148 "timestamp": {
@@ -236,7 +232,7 b' class RequestMetricService(BaseService):'
236 total_time_spent
232 total_time_spent
237 )
233 )
238 if total_time_spent == 0:
234 if total_time_spent == 0:
239 script_text = '0'
235 script_text = "0"
240
236
241 if index_names and filter_settings["resource"]:
237 if index_names and filter_settings["resource"]:
242 es_query = {
238 es_query = {
@@ -254,13 +250,7 b' class RequestMetricService(BaseService):'
254 },
250 },
255 },
251 },
256 "percentage": {
252 "percentage": {
257 "aggs": {
253 "aggs": {"sub_agg": {"sum": {"script": script_text}}},
258 "sub_agg": {
259 "sum": {
260 "script": script_text,
261 }
262 }
263 },
264 "filter": {
254 "filter": {
265 "exists": {"field": "tags.main.numeric_values"}
255 "exists": {"field": "tags.main.numeric_values"}
266 },
256 },
@@ -318,7 +308,10 b' class RequestMetricService(BaseService):'
318 query = {
308 query = {
319 "aggs": {
309 "aggs": {
320 "top_reports": {
310 "top_reports": {
321 "terms": {"field": "tags.view_name.values.keyword", "size": len(series)},
311 "terms": {
312 "field": "tags.view_name.values.keyword",
313 "size": len(series),
314 },
322 "aggs": {
315 "aggs": {
323 "top_calls_hits": {
316 "top_calls_hits": {
324 "top_hits": {"sort": {"start_time": "desc"}, "size": 5}
317 "top_hits": {"sort": {"start_time": "desc"}, "size": 5}
@@ -395,7 +388,9 b' class RequestMetricService(BaseService):'
395 "filter": [
388 "filter": [
396 {
389 {
397 "range": {
390 "range": {
398 "tags.main.numeric_values": {"gte": "4"}
391 "tags.main.numeric_values": {
392 "gte": "4"
393 }
399 }
394 }
400 },
395 },
401 {
396 {
@@ -434,15 +429,20 b' class RequestMetricService(BaseService):'
434 }
429 }
435 },
430 },
436 "filter": {
431 "filter": {
437 "bool": {"filter": [
432 "bool": {
433 "filter": [
438 {
434 {
439 "range": {
435 "range": {
440 "tags.main.numeric_values": {"gte": "1"}
436 "tags.main.numeric_values": {
437 "gte": "1"
438 }
441 }
439 }
442 },
440 },
443 {
441 {
444 "range": {
442 "range": {
445 "tags.main.numeric_values": {"lt": "4"}
443 "tags.main.numeric_values": {
444 "lt": "4"
445 }
446 }
446 }
447 },
447 },
448 {
448 {
@@ -450,11 +450,15 b' class RequestMetricService(BaseService):'
450 "field": "tags.requests.numeric_values"
450 "field": "tags.requests.numeric_values"
451 }
451 }
452 },
452 },
453 ]}
453 ]
454 }
455 },
454 },
456 },
455 },
457 },
458 "terms": {
459 "field": "tags.server_name.values.keyword",
460 "size": 999999,
456 },
461 },
457 "terms": {"field": "tags.server_name.values.keyword", "size": 999999},
458 }
462 }
459 },
463 },
460 "query": {
464 "query": {
@@ -522,7 +526,11 b' class RequestMetricService(BaseService):'
522 "filter": {
526 "filter": {
523 "bool": {
527 "bool": {
524 "filter": [
528 "filter": [
525 {"terms": {"tags.type.values": [report_type]}},
529 {
530 "terms": {
531 "tags.type.values": [report_type]
532 }
533 },
526 {
534 {
527 "exists": {
535 "exists": {
528 "field": "tags.occurences.numeric_values"
536 "field": "tags.occurences.numeric_values"
@@ -533,7 +541,10 b' class RequestMetricService(BaseService):'
533 },
541 },
534 }
542 }
535 },
543 },
536 "terms": {"field": "tags.server_name.values.keyword", "size": 999999},
544 "terms": {
545 "field": "tags.server_name.values.keyword",
546 "size": 999999,
547 },
537 }
548 }
538 },
549 },
539 "query": {
550 "query": {
@@ -98,7 +98,10 b' class SlowCallService(BaseService):'
98 calls_query = {
98 calls_query = {
99 "aggs": {
99 "aggs": {
100 "top_calls": {
100 "top_calls": {
101 "terms": {"field": "tags.statement_hash.values.keyword", "size": 15},
101 "terms": {
102 "field": "tags.statement_hash.values.keyword",
103 "size": 15,
104 },
102 "aggs": {
105 "aggs": {
103 "top_calls_hits": {
106 "top_calls_hits": {
104 "top_hits": {"sort": {"timestamp": "desc"}, "size": 5}
107 "top_hits": {"sort": {"timestamp": "desc"}, "size": 5}
@@ -109,11 +112,7 b' class SlowCallService(BaseService):'
109 "query": {
112 "query": {
110 "bool": {
113 "bool": {
111 "filter": [
114 "filter": [
112 {
115 {"terms": {"resource_id": [filter_settings["resource"][0]]}},
113 "terms": {
114 "resource_id": [filter_settings["resource"][0]]
115 }
116 },
117 {"terms": {"tags.statement_hash.values": hashes}},
116 {"terms": {"tags.statement_hash.values": hashes}},
118 {
117 {
119 "range": {
118 "range": {
@@ -152,13 +152,13 b' def update_template():'
152 "mapping": {
152 "mapping": {
153 "type": "object",
153 "type": "object",
154 "properties": {
154 "properties": {
155 "values": {"type": "text", "analyzer": "tag_value",
155 "values": {
156 "type": "text",
157 "analyzer": "tag_value",
156 "fields": {
158 "fields": {
157 "keyword": {
159 "keyword": {"type": "keyword", "ignore_above": 256}
158 "type": "keyword",
160 },
159 "ignore_above": 256
161 },
160 }
161 }},
162 "numeric_values": {"type": "float"},
162 "numeric_values": {"type": "float"},
163 },
163 },
164 },
164 },
@@ -200,13 +200,11 b' def update_template():'
200 "fields": {"keyword": {"type": "keyword", "ignore_above": 256}},
200 "fields": {"keyword": {"type": "keyword", "ignore_above": 256}},
201 },
201 },
202 "tags": {"type": "object"},
202 "tags": {"type": "object"},
203 "tag_list": {"type": "text", "analyzer": "tag_value",
203 "tag_list": {
204 "fields": {
204 "type": "text",
205 "keyword": {
205 "analyzer": "tag_value",
206 "type": "keyword",
206 "fields": {"keyword": {"type": "keyword", "ignore_above": 256}},
207 "ignore_above": 256
207 },
208 }
209 }},
210 },
208 },
211 }
209 }
212
210
@@ -215,7 +213,7 b' def update_template():'
215 "settings": {
213 "settings": {
216 "index": {
214 "index": {
217 "refresh_interval": "5s",
215 "refresh_interval": "5s",
218 "translog": {"sync_interval": "5s", "durability": "async"}
216 "translog": {"sync_interval": "5s", "durability": "async"},
219 },
217 },
220 "number_of_shards": 5,
218 "number_of_shards": 5,
221 "analysis": shared_analysis,
219 "analysis": shared_analysis,
@@ -240,7 +238,6 b' def update_template():'
240 "summed_duration": {"type": "float"},
238 "summed_duration": {"type": "float"},
241 "public": {"type": "boolean"},
239 "public": {"type": "boolean"},
242 # report
240 # report
243
244 "report_id": {"type": "keyword", "index": True},
241 "report_id": {"type": "keyword", "index": True},
245 "http_status": {"type": "integer"},
242 "http_status": {"type": "integer"},
246 "ip": {"type": "keyword", "index": True},
243 "ip": {"type": "keyword", "index": True},
@@ -252,17 +249,13 b' def update_template():'
252 "end_time": {"type": "date"},
249 "end_time": {"type": "date"},
253 "duration": {"type": "float"},
250 "duration": {"type": "float"},
254 "tags": {"type": "object"},
251 "tags": {"type": "object"},
255 "tag_list": {"type": "text", "analyzer": "tag_value",
252 "tag_list": {
256 "fields": {
253 "type": "text",
257 "keyword": {
254 "analyzer": "tag_value",
258 "type": "keyword",
255 "fields": {"keyword": {"type": "keyword", "ignore_above": 256}},
259 "ignore_above": 256
256 },
260 }
261 }},
262 "extra": {"type": "object"},
257 "extra": {"type": "object"},
263
264 # report stats
258 # report stats
265
266 "report_stat_id": {"type": "keyword", "index": True},
259 "report_stat_id": {"type": "keyword", "index": True},
267 "timestamp": {"type": "date"},
260 "timestamp": {"type": "date"},
268 "permanent": {"type": "boolean"},
261 "permanent": {"type": "boolean"},
@@ -272,17 +265,13 b' def update_template():'
272 "type": "text",
265 "type": "text",
273 "fields": {"keyword": {"type": "keyword", "ignore_above": 256}},
266 "fields": {"keyword": {"type": "keyword", "ignore_above": 256}},
274 },
267 },
275
276 "join_field": {
268 "join_field": {
277 "type": "join",
269 "type": "join",
278 "relations": {
270 "relations": {"report_group": ["report", "report_stat"]},
279 "report_group": ["report", "report_stat"]
271 },
280 }
281 }
282
283 },
272 },
284 }
273 }
285 }
274 },
286 }
275 }
287
276
288 Datastores.es.indices.put_template("rcae_reports", body=report_schema)
277 Datastores.es.indices.put_template("rcae_reports", body=report_schema)
@@ -301,15 +290,15 b' def update_template():'
301 "number_of_shards": 5,
290 "number_of_shards": 5,
302 "analysis": shared_analysis,
291 "analysis": shared_analysis,
303 },
292 },
304 "mappings": {
293 "mappings": {"log": logs_mapping},
305 "log": logs_mapping,
306 },
307 }
294 }
308
295
309 Datastores.es.indices.put_template("rcae_logs", body=log_template)
296 Datastores.es.indices.put_template("rcae_logs", body=log_template)
310
297
311 slow_call_mapping = copy.deepcopy(shared_log_mapping)
298 slow_call_mapping = copy.deepcopy(shared_log_mapping)
312 slow_call_mapping["properties"]["slow_call_id"] = slow_call_mapping["properties"]["pg_id"]
299 slow_call_mapping["properties"]["slow_call_id"] = slow_call_mapping["properties"][
300 "pg_id"
301 ]
313 del slow_call_mapping["properties"]["pg_id"]
302 del slow_call_mapping["properties"]["pg_id"]
314
303
315 slow_call_template = {
304 slow_call_template = {
@@ -322,9 +311,7 b' def update_template():'
322 "number_of_shards": 5,
311 "number_of_shards": 5,
323 "analysis": shared_analysis,
312 "analysis": shared_analysis,
324 },
313 },
325 "mappings": {
314 "mappings": {"log": slow_call_mapping},
326 "log": slow_call_mapping,
327 },
328 }
315 }
329
316
330 Datastores.es.indices.put_template("rcae_slow_calls", body=slow_call_template)
317 Datastores.es.indices.put_template("rcae_slow_calls", body=slow_call_template)
@@ -343,15 +330,15 b' def update_template():'
343 "number_of_shards": 5,
330 "number_of_shards": 5,
344 "analysis": shared_analysis,
331 "analysis": shared_analysis,
345 },
332 },
346 "mappings": {
333 "mappings": {"log": metric_mapping},
347 "log": metric_mapping,
348 },
349 }
334 }
350
335
351 Datastores.es.indices.put_template("rcae_metrics", body=metrics_template)
336 Datastores.es.indices.put_template("rcae_metrics", body=metrics_template)
352
337
353 uptime_metric_mapping = copy.deepcopy(shared_log_mapping)
338 uptime_metric_mapping = copy.deepcopy(shared_log_mapping)
354 uptime_metric_mapping["properties"]["uptime_id"] = uptime_metric_mapping["properties"]["pg_id"]
339 uptime_metric_mapping["properties"]["uptime_id"] = uptime_metric_mapping[
340 "properties"
341 ]["pg_id"]
355 del uptime_metric_mapping["properties"]["pg_id"]
342 del uptime_metric_mapping["properties"]["pg_id"]
356
343
357 uptime_metrics_template = {
344 uptime_metrics_template = {
@@ -364,12 +351,12 b' def update_template():'
364 "number_of_shards": 5,
351 "number_of_shards": 5,
365 "analysis": shared_analysis,
352 "analysis": shared_analysis,
366 },
353 },
367 "mappings": {
354 "mappings": {"log": shared_log_mapping},
368 "log": shared_log_mapping,
369 },
370 }
355 }
371
356
372 Datastores.es.indices.put_template("rcae_uptime_metrics", body=uptime_metrics_template)
357 Datastores.es.indices.put_template(
358 "rcae_uptime_metrics", body=uptime_metrics_template
359 )
373
360
374
361
375 def reindex_reports():
362 def reindex_reports():
@@ -146,11 +146,7 b' def common_tags(request):'
146
146
147 resources = list(filter_settings["resource"])
147 resources = list(filter_settings["resource"])
148 query = {
148 query = {
149 "query": {
149 "query": {"bool": {"filter": [{"terms": {"resource_id": list(resources)}}]}}
150 "bool": {
151 "filter": [{"terms": {"resource_id": list(resources)}}]
152 }
153 }
154 }
150 }
155 start_date = filter_settings.get("start_date")
151 start_date = filter_settings.get("start_date")
156 end_date = filter_settings.get("end_date")
152 end_date = filter_settings.get("end_date")
@@ -199,18 +195,10 b' def common_values(request):'
199 resources = list(filter_settings["resource"])
195 resources = list(filter_settings["resource"])
200 tag_name = filter_settings["tags"][0]["value"][0]
196 tag_name = filter_settings["tags"][0]["value"][0]
201
197
202 and_part = [
198 and_part = [{"terms": {"resource_id": list(resources)}}]
203 {"terms": {"resource_id": list(resources)}},
204 ]
205 if filter_settings["namespace"]:
199 if filter_settings["namespace"]:
206 and_part.append({"terms": {"namespace": filter_settings["namespace"]}})
200 and_part.append({"terms": {"namespace": filter_settings["namespace"]}})
207 query = {
201 query = {"query": {"bool": {"filter": and_part}}}
208 "query": {
209 "bool": {
210 "filter": and_part
211 }
212 }
213 }
214 query["aggs"] = {
202 query["aggs"] = {
215 "sub_agg": {"terms": {"field": "tags.{}.values".format(tag_name), "size": 50}}
203 "sub_agg": {"terms": {"field": "tags.{}.values".format(tag_name), "size": 50}}
216 }
204 }
General Comments 4
Under Review
author

Auto status change to "Under Review"

Under Review
author

Auto status change to "Under Review"

You need to be logged in to leave comments. Login now