##// END OF EJS Templates
appenlight
RSS

Clone from https://github.com/Appenlight/appenlight.git

reformat: black
ergo -
Show More
Show file before | Show file after | Hide comments
@@ -326,8 +326,10 b' def add_logs(resource_id, request_params, dataset, **kwargs):'
326 326
327 327 try:
328 328 Datastores.es.delete_by_query(
329 index=es_index, doc_type="log",
330 body=query, conflicts="proceed"
329 index=es_index,
330 doc_type="log",
331 body=query,
332 conflicts="proceed",
331 333 )
332 334 except elasticsearch.exceptions.NotFoundError as exc:
333 335 msg = "skipping index {}".format(es_index)
@@ -688,11 +690,7 b' def alerting_reports():'
688 690 def logs_cleanup(resource_id, filter_settings):
689 691 request = get_current_request()
690 692 request.tm.begin()
691 es_query = {
692 "query": {
693 "bool": {"filter": [{"term": {"resource_id": resource_id}}]}
694 }
695 }
693 es_query = {"query": {"bool": {"filter": [{"term": {"resource_id": resource_id}}]}}}
696 694
697 695 query = DBSession.query(Log).filter(Log.resource_id == resource_id)
698 696 if filter_settings["namespace"]:
@@ -703,5 +701,5 b' def logs_cleanup(resource_id, filter_settings):'
703 701 query.delete(synchronize_session=False)
704 702 request.tm.commit()
705 703 Datastores.es.delete_by_query(
706 index="rcae_l_*", doc_type="log", body=es_query, conflicts="proceed"
704 index="rcae_l_*", doc_type="log", body=es_query, conflicts="proceed"
707 705 )
Show file before | Show file after | Hide comments
@@ -552,7 +552,9 b' def get_es_info(cache_regions, es_conn):'
552 552 @cache_regions.memory_min_10.cache_on_arguments()
553 553 def get_es_info_cached():
554 554 returned_info = {"raw_info": es_conn.info()}
555 returned_info["version"] = returned_info["raw_info"]["version"]["number"].split('.')
555 returned_info["version"] = returned_info["raw_info"]["version"]["number"].split(
556 "."
557 )
556 558 return returned_info
557 559
558 560 return get_es_info_cached()
Show file before | Show file after | Hide comments
@@ -483,13 +483,10 b' class Report(Base, BaseModel):'
483 483 "ip": self.ip,
484 484 "group_id": str(self.group_id),
485 485 "type": "report",
486 "join_field": {
487 "name": "report",
488 "parent": str(self.group_id)
489 },
486 "join_field": {"name": "report", "parent": str(self.group_id)},
490 487 "tags": tags,
491 488 "tag_list": tag_list,
492 "_routing": str(self.group_id)
489 "_routing": str(self.group_id),
493 490 }
494 491
495 492 @property
@@ -525,7 +522,10 b' def after_delete(mapper, connection, target):'
525 522 if not hasattr(target, "_skip_ft_index"):
526 523 query = {"query": {"term": {"report_id": target.id}}}
527 524 Datastores.es.delete_by_query(
528 index=target.partition_id, doc_type="report", body=query, conflicts="proceed"
525 index=target.partition_id,
526 doc_type="report",
527 body=query,
528 conflicts="proceed",
529 529 )
530 530
531 531
Show file before | Show file after | Hide comments
@@ -191,9 +191,7 b' class ReportGroup(Base, BaseModel):'
191 191 "first_timestamp": self.first_timestamp,
192 192 "last_timestamp": self.last_timestamp,
193 193 "type": "report_group",
194 "join_field": {
195 "name": "report_group"
196 },
194 "join_field": {"name": "report_group"},
197 195 }
198 196
199 197 def set_notification_info(self, notify_10=False, notify_100=False):
Show file before | Show file after | Hide comments
@@ -56,11 +56,7 b' class LogService(BaseService):'
56 56 filter_settings = {}
57 57
58 58 query = {
59 "query": {
60 "bool": {
61 "filter": [{"terms": {"resource_id": list(app_ids)}}]
62 }
63 }
59 "query": {"bool": {"filter": [{"terms": {"resource_id": list(app_ids)}}]}}
64 60 }
65 61
66 62 start_date = filter_settings.get("start_date")
@@ -132,13 +128,13 b' class LogService(BaseService):'
132 128
133 129 @classmethod
134 130 def get_search_iterator(
135 cls,
136 app_ids=None,
137 page=1,
138 items_per_page=50,
139 order_by=None,
140 filter_settings=None,
141 limit=None,
131 cls,
132 app_ids=None,
133 page=1,
134 items_per_page=50,
135 order_by=None,
136 filter_settings=None,
137 limit=None,
142 138 ):
143 139 if not app_ids:
144 140 return {}, 0
@@ -171,15 +167,15 b' class LogService(BaseService):'
171 167
172 168 @classmethod
173 169 def get_paginator_by_app_ids(
174 cls,
175 app_ids=None,
176 page=1,
177 item_count=None,
178 items_per_page=50,
179 order_by=None,
180 filter_settings=None,
181 exclude_columns=None,
182 db_session=None,
170 cls,
171 app_ids=None,
172 page=1,
173 item_count=None,
174 items_per_page=50,
175 order_by=None,
176 filter_settings=None,
177 exclude_columns=None,
178 db_session=None,
183 179 ):
184 180 if not filter_settings:
185 181 filter_settings = {}
Show file before | Show file after | Hide comments
@@ -64,7 +64,9 b' class ReportGroupService(BaseService):'
64 64 "groups": {
65 65 "aggs": {
66 66 "sub_agg": {
67 "value_count": {"field": "tags.group_id.values.keyword"}
67 "value_count": {
68 "field": "tags.group_id.values.keyword"
69 }
68 70 }
69 71 },
70 72 "filter": {"exists": {"field": "tags.group_id.values"}},
@@ -76,11 +78,7 b' class ReportGroupService(BaseService):'
76 78 "query": {
77 79 "bool": {
78 80 "filter": [
79 {
80 "terms": {
81 "resource_id": [filter_settings["resource"][0]]
82 }
83 },
81 {"terms": {"resource_id": [filter_settings["resource"][0]]}},
84 82 {
85 83 "range": {
86 84 "timestamp": {
@@ -136,7 +134,7 b' class ReportGroupService(BaseService):'
136 134 "bool": {
137 135 "must": [],
138 136 "should": [],
139 "filter": [{"terms": {"resource_id": list(app_ids)}}]
137 "filter": [{"terms": {"resource_id": list(app_ids)}}],
140 138 }
141 139 },
142 140 "aggs": {
@@ -315,7 +313,9 b' class ReportGroupService(BaseService):'
315 313 ordered_ids = []
316 314 if results:
317 315 for item in results["top_groups"]["buckets"]:
318 pg_id = item["top_reports_hits"]["hits"]["hits"][0]["_source"]["report_id"]
316 pg_id = item["top_reports_hits"]["hits"]["hits"][0]["_source"][
317 "report_id"
318 ]
319 319 ordered_ids.append(pg_id)
320 320 log.info(filter_settings)
321 321 paginator = paginate.Page(
@@ -445,12 +445,16 b' class ReportGroupService(BaseService):'
445 445 "aggs": {
446 446 "types": {
447 447 "aggs": {
448 "sub_agg": {"terms": {"field": "tags.type.values.keyword"}}
448 "sub_agg": {
449 "terms": {"field": "tags.type.values.keyword"}
450 }
449 451 },
450 452 "filter": {
451 "bool": {
452 "filter": [{"exists": {"field": "tags.type.values"}}]
453 }
453 "bool": {
454 "filter": [
455 {"exists": {"field": "tags.type.values"}}
456 ]
457 }
454 458 },
455 459 }
456 460 },
@@ -468,11 +472,7 b' class ReportGroupService(BaseService):'
468 472 "query": {
469 473 "bool": {
470 474 "filter": [
471 {
472 "terms": {
473 "resource_id": [filter_settings["resource"][0]]
474 }
475 },
475 {"terms": {"resource_id": [filter_settings["resource"][0]]}},
476 476 {
477 477 "range": {
478 478 "timestamp": {
Show file before | Show file after | Hide comments
@@ -31,7 +31,9 b' class ReportStatService(BaseService):'
31 31 "aggs": {
32 32 "reports": {
33 33 "aggs": {
34 "sub_agg": {"value_count": {"field": "tags.group_id.values.keyword"}}
34 "sub_agg": {
35 "value_count": {"field": "tags.group_id.values.keyword"}
36 }
35 37 },
36 38 "filter": {
37 39 "bool": {
Show file before | Show file after | Hide comments
@@ -142,11 +142,7 b' class RequestMetricService(BaseService):'
142 142 "query": {
143 143 "bool": {
144 144 "filter": [
145 {
146 "terms": {
147 "resource_id": [filter_settings["resource"][0]]
148 }
149 },
145 {"terms": {"resource_id": [filter_settings["resource"][0]]}},
150 146 {
151 147 "range": {
152 148 "timestamp": {
@@ -236,7 +232,7 b' class RequestMetricService(BaseService):'
236 232 total_time_spent
237 233 )
238 234 if total_time_spent == 0:
239 script_text = '0'
235 script_text = "0"
240 236
241 237 if index_names and filter_settings["resource"]:
242 238 es_query = {
@@ -254,13 +250,7 b' class RequestMetricService(BaseService):'
254 250 },
255 251 },
256 252 "percentage": {
257 "aggs": {
258 "sub_agg": {
259 "sum": {
260 "script": script_text,
261 }
262 }
263 },
253 "aggs": {"sub_agg": {"sum": {"script": script_text}}},
264 254 "filter": {
265 255 "exists": {"field": "tags.main.numeric_values"}
266 256 },
@@ -318,7 +308,10 b' class RequestMetricService(BaseService):'
318 308 query = {
319 309 "aggs": {
320 310 "top_reports": {
321 "terms": {"field": "tags.view_name.values.keyword", "size": len(series)},
311 "terms": {
312 "field": "tags.view_name.values.keyword",
313 "size": len(series),
314 },
322 315 "aggs": {
323 316 "top_calls_hits": {
324 317 "top_hits": {"sort": {"start_time": "desc"}, "size": 5}
@@ -395,7 +388,9 b' class RequestMetricService(BaseService):'
395 388 "filter": [
396 389 {
397 390 "range": {
398 "tags.main.numeric_values": {"gte": "4"}
391 "tags.main.numeric_values": {
392 "gte": "4"
393 }
399 394 }
400 395 },
401 396 {
@@ -434,27 +429,36 b' class RequestMetricService(BaseService):'
434 429 }
435 430 },
436 431 "filter": {
437 "bool": {"filter": [
438 {
439 "range": {
440 "tags.main.numeric_values": {"gte": "1"}
441 }
442 },
443 {
444 "range": {
445 "tags.main.numeric_values": {"lt": "4"}
446 }
447 },
448 {
449 "exists": {
450 "field": "tags.requests.numeric_values"
451 }
452 },
453 ]}
432 "bool": {
433 "filter": [
434 {
435 "range": {
436 "tags.main.numeric_values": {
437 "gte": "1"
438 }
439 }
440 },
441 {
442 "range": {
443 "tags.main.numeric_values": {
444 "lt": "4"
445 }
446 }
447 },
448 {
449 "exists": {
450 "field": "tags.requests.numeric_values"
451 }
452 },
453 ]
454 }
454 455 },
455 456 },
456 457 },
457 "terms": {"field": "tags.server_name.values.keyword", "size": 999999},
458 "terms": {
459 "field": "tags.server_name.values.keyword",
460 "size": 999999,
461 },
458 462 }
459 463 },
460 464 "query": {
@@ -522,7 +526,11 b' class RequestMetricService(BaseService):'
522 526 "filter": {
523 527 "bool": {
524 528 "filter": [
525 {"terms": {"tags.type.values": [report_type]}},
529 {
530 "terms": {
531 "tags.type.values": [report_type]
532 }
533 },
526 534 {
527 535 "exists": {
528 536 "field": "tags.occurences.numeric_values"
@@ -533,7 +541,10 b' class RequestMetricService(BaseService):'
533 541 },
534 542 }
535 543 },
536 "terms": {"field": "tags.server_name.values.keyword", "size": 999999},
544 "terms": {
545 "field": "tags.server_name.values.keyword",
546 "size": 999999,
547 },
537 548 }
538 549 },
539 550 "query": {
@@ -591,10 +602,10 b' class RequestMetricService(BaseService):'
591 602 server_stats = list(stats.values())
592 603 for stat in server_stats:
593 604 stat["satisfying_requests"] = (
594 stat["requests"]
595 - stat["errors"]
596 - stat["frustrating_requests"]
597 - stat["tolerated_requests"]
605 stat["requests"]
606 - stat["errors"]
607 - stat["frustrating_requests"]
608 - stat["tolerated_requests"]
598 609 )
599 610 if stat["satisfying_requests"] < 0:
600 611 stat["satisfying_requests"] = 0
@@ -604,7 +615,7 b' class RequestMetricService(BaseService):'
604 615 stat["response_time"] / stat["requests"], 3
605 616 )
606 617 qual_requests = (
607 stat["satisfying_requests"] + stat["tolerated_requests"] / 2.0
618 stat["satisfying_requests"] + stat["tolerated_requests"] / 2.0
608 619 )
609 620 stat["apdex"] = round((qual_requests / stat["requests"]) * 100, 2)
610 621 stat["rpm"] = round(stat["requests"] / stat["total_minutes"], 2)
Show file before | Show file after | Hide comments
@@ -98,7 +98,10 b' class SlowCallService(BaseService):'
98 98 calls_query = {
99 99 "aggs": {
100 100 "top_calls": {
101 "terms": {"field": "tags.statement_hash.values.keyword", "size": 15},
101 "terms": {
102 "field": "tags.statement_hash.values.keyword",
103 "size": 15,
104 },
102 105 "aggs": {
103 106 "top_calls_hits": {
104 107 "top_hits": {"sort": {"timestamp": "desc"}, "size": 5}
@@ -109,11 +112,7 b' class SlowCallService(BaseService):'
109 112 "query": {
110 113 "bool": {
111 114 "filter": [
112 {
113 "terms": {
114 "resource_id": [filter_settings["resource"][0]]
115 }
116 },
115 {"terms": {"resource_id": [filter_settings["resource"][0]]}},
117 116 {"terms": {"tags.statement_hash.values": hashes}},
118 117 {
119 118 "range": {
Show file before | Show file after | Hide comments
@@ -152,13 +152,13 b' def update_template():'
152 152 "mapping": {
153 153 "type": "object",
154 154 "properties": {
155 "values": {"type": "text", "analyzer": "tag_value",
156 "fields": {
157 "keyword": {
158 "type": "keyword",
159 "ignore_above": 256
160 }
161 }},
155 "values": {
156 "type": "text",
157 "analyzer": "tag_value",
158 "fields": {
159 "keyword": {"type": "keyword", "ignore_above": 256}
160 },
161 },
162 162 "numeric_values": {"type": "float"},
163 163 },
164 164 },
@@ -200,13 +200,11 b' def update_template():'
200 200 "fields": {"keyword": {"type": "keyword", "ignore_above": 256}},
201 201 },
202 202 "tags": {"type": "object"},
203 "tag_list": {"type": "text", "analyzer": "tag_value",
204 "fields": {
205 "keyword": {
206 "type": "keyword",
207 "ignore_above": 256
208 }
209 }},
203 "tag_list": {
204 "type": "text",
205 "analyzer": "tag_value",
206 "fields": {"keyword": {"type": "keyword", "ignore_above": 256}},
207 },
210 208 },
211 209 }
212 210
@@ -215,7 +213,7 b' def update_template():'
215 213 "settings": {
216 214 "index": {
217 215 "refresh_interval": "5s",
218 "translog": {"sync_interval": "5s", "durability": "async"}
216 "translog": {"sync_interval": "5s", "durability": "async"},
219 217 },
220 218 "number_of_shards": 5,
221 219 "analysis": shared_analysis,
@@ -240,7 +238,6 b' def update_template():'
240 238 "summed_duration": {"type": "float"},
241 239 "public": {"type": "boolean"},
242 240 # report
243
244 241 "report_id": {"type": "keyword", "index": True},
245 242 "http_status": {"type": "integer"},
246 243 "ip": {"type": "keyword", "index": True},
@@ -252,17 +249,13 b' def update_template():'
252 249 "end_time": {"type": "date"},
253 250 "duration": {"type": "float"},
254 251 "tags": {"type": "object"},
255 "tag_list": {"type": "text", "analyzer": "tag_value",
256 "fields": {
257 "keyword": {
258 "type": "keyword",
259 "ignore_above": 256
260 }
261 }},
252 "tag_list": {
253 "type": "text",
254 "analyzer": "tag_value",
255 "fields": {"keyword": {"type": "keyword", "ignore_above": 256}},
256 },
262 257 "extra": {"type": "object"},
263
264 258 # report stats
265
266 259 "report_stat_id": {"type": "keyword", "index": True},
267 260 "timestamp": {"type": "date"},
268 261 "permanent": {"type": "boolean"},
@@ -272,17 +265,13 b' def update_template():'
272 265 "type": "text",
273 266 "fields": {"keyword": {"type": "keyword", "ignore_above": 256}},
274 267 },
275
276 268 "join_field": {
277 269 "type": "join",
278 "relations": {
279 "report_group": ["report", "report_stat"]
280 }
281 }
282
270 "relations": {"report_group": ["report", "report_stat"]},
271 },
283 272 },
284 273 }
285 }
274 },
286 275 }
287 276
288 277 Datastores.es.indices.put_template("rcae_reports", body=report_schema)
@@ -301,15 +290,15 b' def update_template():'
301 290 "number_of_shards": 5,
302 291 "analysis": shared_analysis,
303 292 },
304 "mappings": {
305 "log": logs_mapping,
306 },
293 "mappings": {"log": logs_mapping},
307 294 }
308 295
309 296 Datastores.es.indices.put_template("rcae_logs", body=log_template)
310 297
311 298 slow_call_mapping = copy.deepcopy(shared_log_mapping)
312 slow_call_mapping["properties"]["slow_call_id"] = slow_call_mapping["properties"]["pg_id"]
299 slow_call_mapping["properties"]["slow_call_id"] = slow_call_mapping["properties"][
300 "pg_id"
301 ]
313 302 del slow_call_mapping["properties"]["pg_id"]
314 303
315 304 slow_call_template = {
@@ -322,9 +311,7 b' def update_template():'
322 311 "number_of_shards": 5,
323 312 "analysis": shared_analysis,
324 313 },
325 "mappings": {
326 "log": slow_call_mapping,
327 },
314 "mappings": {"log": slow_call_mapping},
328 315 }
329 316
330 317 Datastores.es.indices.put_template("rcae_slow_calls", body=slow_call_template)
@@ -343,15 +330,15 b' def update_template():'
343 330 "number_of_shards": 5,
344 331 "analysis": shared_analysis,
345 332 },
346 "mappings": {
347 "log": metric_mapping,
348 },
333 "mappings": {"log": metric_mapping},
349 334 }
350 335
351 336 Datastores.es.indices.put_template("rcae_metrics", body=metrics_template)
352 337
353 338 uptime_metric_mapping = copy.deepcopy(shared_log_mapping)
354 uptime_metric_mapping["properties"]["uptime_id"] = uptime_metric_mapping["properties"]["pg_id"]
339 uptime_metric_mapping["properties"]["uptime_id"] = uptime_metric_mapping[
340 "properties"
341 ]["pg_id"]
355 342 del uptime_metric_mapping["properties"]["pg_id"]
356 343
357 344 uptime_metrics_template = {
@@ -364,12 +351,12 b' def update_template():'
364 351 "number_of_shards": 5,
365 352 "analysis": shared_analysis,
366 353 },
367 "mappings": {
368 "log": shared_log_mapping,
369 },
354 "mappings": {"log": shared_log_mapping},
370 355 }
371 356
372 Datastores.es.indices.put_template("rcae_uptime_metrics", body=uptime_metrics_template)
357 Datastores.es.indices.put_template(
358 "rcae_uptime_metrics", body=uptime_metrics_template
359 )
373 360
374 361
375 362 def reindex_reports():
Show file before | Show file after | Hide comments
@@ -146,11 +146,7 b' def common_tags(request):'
146 146
147 147 resources = list(filter_settings["resource"])
148 148 query = {
149 "query": {
150 "bool": {
151 "filter": [{"terms": {"resource_id": list(resources)}}]
152 }
153 }
149 "query": {"bool": {"filter": [{"terms": {"resource_id": list(resources)}}]}}
154 150 }
155 151 start_date = filter_settings.get("start_date")
156 152 end_date = filter_settings.get("end_date")
@@ -199,18 +195,10 b' def common_values(request):'
199 195 resources = list(filter_settings["resource"])
200 196 tag_name = filter_settings["tags"][0]["value"][0]
201 197
202 and_part = [
203 {"terms": {"resource_id": list(resources)}},
204 ]
198 and_part = [{"terms": {"resource_id": list(resources)}}]
205 199 if filter_settings["namespace"]:
206 200 and_part.append({"terms": {"namespace": filter_settings["namespace"]}})
207 query = {
208 "query": {
209 "bool": {
210 "filter": and_part
211 }
212 }
213 }
201 query = {"query": {"bool": {"filter": and_part}}}
214 202 query["aggs"] = {
215 203 "sub_agg": {"terms": {"field": "tags.{}.values".format(tag_name), "size": 50}}
216 204 }
General Comments 4
Under Review
vaingmuny
author
Comment #23545
Edit
Delete
|

Auto status change to "Under Review"

Under Review
vaingmuny
author
Comment #23711
Edit
Delete
|

Auto status change to "Under Review"

You need to be logged in to leave comments. Login now