##// END OF EJS Templates
appenlight
RSS

Clone from https://github.com/Appenlight/appenlight.git

elasticsearch: migrate to ES 5.x
ergo -
Show More
Show file before | Show file after | Hide comments
@@ -325,10 +325,9 b' def add_logs(resource_id, request_params, dataset, **kwargs):'
325 query = {"query": {"terms": {"delete_hash": batch}}}
325 query = {"query": {"terms": {"delete_hash": batch}}}
326
326
327 try:
327 try:
328 Datastores.es.transport.perform_request(
328 Datastores.es.delete_by_query(
329 "DELETE",
329 index=es_index, doc_type="log",
330 "/{}/{}/_query".format(es_index, "log"),
330 body=query, conflicts="proceed"
331 body=query,
332 )
331 )
333 except elasticsearch.exceptions.NotFoundError as exc:
332 except elasticsearch.exceptions.NotFoundError as exc:
334 msg = "skipping index {}".format(es_index)
333 msg = "skipping index {}".format(es_index)
@@ -703,6 +702,6 b' def logs_cleanup(resource_id, filter_settings):'
703 )
702 )
704 query.delete(synchronize_session=False)
703 query.delete(synchronize_session=False)
705 request.tm.commit()
704 request.tm.commit()
706 Datastores.es.transport.perform_request(
705 Datastores.es.delete_by_query(
707 "DELETE", "/{}/{}/_query".format("rcae_l_*", "log"), body=es_query
706 index="rcae_l_*", doc_type="log", body=es_query, conflicts="proceed"
708 )
707 )
Show file before | Show file after | Hide comments
@@ -519,8 +519,8 b' def after_update(mapper, connection, target):'
519 def after_delete(mapper, connection, target):
519 def after_delete(mapper, connection, target):
520 if not hasattr(target, "_skip_ft_index"):
520 if not hasattr(target, "_skip_ft_index"):
521 query = {"query": {"term": {"pg_id": target.id}}}
521 query = {"query": {"term": {"pg_id": target.id}}}
522 Datastores.es.transport.perform_request(
522 Datastores.es.delete_by_query(
523 "DELETE", "/{}/{}/_query".format(target.partition_id, "report"), body=query
523 index=target.partition_id, doc_type="report", body=query, conflicts="proceed"
524 )
524 )
525
525
526
526
Show file before | Show file after | Hide comments
@@ -271,14 +271,12 b' def after_update(mapper, connection, target):'
271 def after_delete(mapper, connection, target):
271 def after_delete(mapper, connection, target):
272 query = {"query": {"term": {"group_id": target.id}}}
272 query = {"query": {"term": {"group_id": target.id}}}
273 # delete by query
273 # delete by query
274 Datastores.es.transport.perform_request(
274 Datastores.es.delete_by_query(
275 "DELETE", "/{}/{}/_query".format(target.partition_id, "report"), body=query
275 index=target.partition_id, doc_type="report", body=query, conflicts="proceed"
276 )
276 )
277 query = {"query": {"term": {"pg_id": target.id}}}
277 query = {"query": {"term": {"pg_id": target.id}}}
278 Datastores.es.transport.perform_request(
278 Datastores.es.delete_by_query(
279 "DELETE",
279 index=target.partition_id, doc_type="report_group", body=query, conflicts="proceed"
280 "/{}/{}/_query".format(target.partition_id, "report_group"),
281 body=query,
282 )
280 )
283
281
284
282
Show file before | Show file after | Hide comments
@@ -64,13 +64,13 b' class ReportGroupService(BaseService):'
64 "groups": {
64 "groups": {
65 "aggs": {
65 "aggs": {
66 "sub_agg": {
66 "sub_agg": {
67 "value_count": {"field": "tags.group_id.values"}
67 "value_count": {"field": "tags.group_id.values.keyword"}
68 }
68 }
69 },
69 },
70 "filter": {"exists": {"field": "tags.group_id.values"}},
70 "filter": {"exists": {"field": "tags.group_id.values"}},
71 }
71 }
72 },
72 },
73 "terms": {"field": "tags.group_id.values", "size": limit},
73 "terms": {"field": "tags.group_id.values.keyword", "size": limit},
74 }
74 }
75 },
75 },
76 "query": {
76 "query": {
@@ -143,7 +143,7 b' class ReportGroupService(BaseService):'
143 "top_groups": {
143 "top_groups": {
144 "terms": {
144 "terms": {
145 "size": 5000,
145 "size": 5000,
146 "field": "_parent",
146 "field": "_parent#report_group",
147 "order": {"newest": "desc"},
147 "order": {"newest": "desc"},
148 },
148 },
149 "aggs": {
149 "aggs": {
@@ -445,10 +445,12 b' class ReportGroupService(BaseService):'
445 "aggs": {
445 "aggs": {
446 "types": {
446 "types": {
447 "aggs": {
447 "aggs": {
448 "sub_agg": {"terms": {"field": "tags.type.values"}}
448 "sub_agg": {"terms": {"field": "tags.type.values.keyword"}}
449 },
449 },
450 "filter": {
450 "filter": {
451 "and": [{"exists": {"field": "tags.type.values"}}]
451 "bool": {
452 "filter": [{"exists": {"field": "tags.type.values"}}]
453 }
452 },
454 },
453 }
455 }
454 },
456 },
@@ -485,7 +487,7 b' class ReportGroupService(BaseService):'
485 }
487 }
486 if group_id:
488 if group_id:
487 parent_agg = es_query["aggs"]["parent_agg"]
489 parent_agg = es_query["aggs"]["parent_agg"]
488 filters = parent_agg["aggs"]["types"]["filter"]["and"]
490 filters = parent_agg["aggs"]["types"]["filter"]["bool"]["filter"]
489 filters.append({"terms": {"tags.group_id.values": [group_id]}})
491 filters.append({"terms": {"tags.group_id.values": [group_id]}})
490
492
491 index_names = es_index_name_limiter(
493 index_names = es_index_name_limiter(
Show file before | Show file after | Hide comments
@@ -34,10 +34,12 b' class ReportStatService(BaseService):'
34 "sub_agg": {"value_count": {"field": "tags.group_id.values"}}
34 "sub_agg": {"value_count": {"field": "tags.group_id.values"}}
35 },
35 },
36 "filter": {
36 "filter": {
37 "and": [
37 "bool": {
38 "filter": [
38 {"terms": {"resource_id": [resource_id]}},
39 {"terms": {"resource_id": [resource_id]}},
39 {"exists": {"field": "tags.group_id.values"}},
40 {"exists": {"field": "tags.group_id.values"}},
40 ]
41 ]
42 }
41 },
43 },
42 }
44 }
43 },
45 },
Show file before | Show file after | Hide comments
@@ -235,6 +235,8 b' class RequestMetricService(BaseService):'
235 script_text = "doc['tags.main.numeric_values'].value / {}".format(
235 script_text = "doc['tags.main.numeric_values'].value / {}".format(
236 total_time_spent
236 total_time_spent
237 )
237 )
238 if total_time_spent == 0:
239 script_text = '0'
238
240
239 if index_names and filter_settings["resource"]:
241 if index_names and filter_settings["resource"]:
240 es_query = {
242 es_query = {
@@ -255,7 +257,6 b' class RequestMetricService(BaseService):'
255 "aggs": {
257 "aggs": {
256 "sub_agg": {
258 "sub_agg": {
257 "sum": {
259 "sum": {
258 "lang": "expression",
259 "script": script_text,
260 "script": script_text,
260 }
261 }
261 }
262 }
@@ -276,7 +277,7 b' class RequestMetricService(BaseService):'
276 },
277 },
277 },
278 },
278 "terms": {
279 "terms": {
279 "field": "tags.view_name.values",
280 "field": "tags.view_name.values.keyword",
280 "order": {"percentage>sub_agg": "desc"},
281 "order": {"percentage>sub_agg": "desc"},
281 "size": 15,
282 "size": 15,
282 },
283 },
@@ -317,7 +318,7 b' class RequestMetricService(BaseService):'
317 query = {
318 query = {
318 "aggs": {
319 "aggs": {
319 "top_reports": {
320 "top_reports": {
320 "terms": {"field": "tags.view_name.values", "size": len(series)},
321 "terms": {"field": "tags.view_name.values.keyword", "size": len(series)},
321 "aggs": {
322 "aggs": {
322 "top_calls_hits": {
323 "top_calls_hits": {
323 "top_hits": {"sort": {"start_time": "desc"}, "size": 5}
324 "top_hits": {"sort": {"start_time": "desc"}, "size": 5}
@@ -390,7 +391,8 b' class RequestMetricService(BaseService):'
390 }
391 }
391 },
392 },
392 "filter": {
393 "filter": {
393 "and": [
394 "bool": {
395 "filter": [
394 {
396 {
395 "range": {
397 "range": {
396 "tags.main.numeric_values": {"gte": "4"}
398 "tags.main.numeric_values": {"gte": "4"}
@@ -402,6 +404,7 b' class RequestMetricService(BaseService):'
402 }
404 }
403 },
405 },
404 ]
406 ]
407 }
405 },
408 },
406 },
409 },
407 "main": {
410 "main": {
@@ -431,7 +434,7 b' class RequestMetricService(BaseService):'
431 }
434 }
432 },
435 },
433 "filter": {
436 "filter": {
434 "and": [
437 "bool": {"filter": [
435 {
438 {
436 "range": {
439 "range": {
437 "tags.main.numeric_values": {"gte": "1"}
440 "tags.main.numeric_values": {"gte": "1"}
@@ -447,11 +450,11 b' class RequestMetricService(BaseService):'
447 "field": "tags.requests.numeric_values"
450 "field": "tags.requests.numeric_values"
448 }
451 }
449 },
452 },
450 ]
453 ]}
451 },
454 },
452 },
455 },
453 },
456 },
454 "terms": {"field": "tags.server_name.values", "size": 999999},
457 "terms": {"field": "tags.server_name.values.keyword", "size": 999999},
455 }
458 }
456 },
459 },
457 "query": {
460 "query": {
@@ -517,7 +520,8 b' class RequestMetricService(BaseService):'
517 }
520 }
518 },
521 },
519 "filter": {
522 "filter": {
520 "and": [
523 "bool": {
524 "filter": [
521 {"terms": {"tags.type.values": [report_type]}},
525 {"terms": {"tags.type.values": [report_type]}},
522 {
526 {
523 "exists": {
527 "exists": {
@@ -525,10 +529,11 b' class RequestMetricService(BaseService):'
525 }
529 }
526 },
530 },
527 ]
531 ]
532 }
528 },
533 },
529 }
534 }
530 },
535 },
531 "terms": {"field": "tags.server_name.values", "size": 999999},
536 "terms": {"field": "tags.server_name.values.keyword", "size": 999999},
532 }
537 }
533 },
538 },
534 "query": {
539 "query": {
Show file before | Show file after | Hide comments
@@ -50,7 +50,7 b' class SlowCallService(BaseService):'
50 "aggs": {
50 "aggs": {
51 "sub_agg": {
51 "sub_agg": {
52 "value_count": {
52 "value_count": {
53 "field": "tags.statement_hash.values"
53 "field": "tags.statement_hash.values.keyword"
54 }
54 }
55 }
55 }
56 },
56 },
@@ -60,7 +60,7 b' class SlowCallService(BaseService):'
60 },
60 },
61 },
61 },
62 "terms": {
62 "terms": {
63 "field": "tags.statement_hash.values",
63 "field": "tags.statement_hash.values.keyword",
64 "order": {"duration>sub_agg": "desc"},
64 "order": {"duration>sub_agg": "desc"},
65 "size": 15,
65 "size": 15,
66 },
66 },
@@ -98,7 +98,7 b' class SlowCallService(BaseService):'
98 calls_query = {
98 calls_query = {
99 "aggs": {
99 "aggs": {
100 "top_calls": {
100 "top_calls": {
101 "terms": {"field": "tags.statement_hash.values", "size": 15},
101 "terms": {"field": "tags.statement_hash.values.keyword", "size": 15},
102 "aggs": {
102 "aggs": {
103 "top_calls_hits": {
103 "top_calls_hits": {
104 "top_hits": {"sort": {"timestamp": "desc"}, "size": 5}
104 "top_hits": {"sort": {"timestamp": "desc"}, "size": 5}
Show file before | Show file after | Hide comments
@@ -139,7 +139,13 b' def update_template():'
139 "mapping": {
139 "mapping": {
140 "type": "object",
140 "type": "object",
141 "properties": {
141 "properties": {
142 "values": {"type": "string", "analyzer": "tag_value"},
142 "values": {"type": "text", "analyzer": "tag_value",
143 "fields": {
144 "keyword": {
145 "type": "keyword",
146 "ignore_above": 256
147 }
148 }},
143 "numeric_values": {"type": "float"},
149 "numeric_values": {"type": "float"},
144 },
150 },
145 },
151 },
@@ -177,10 +183,10 b' def update_template():'
177 "_all": {"enabled": False},
183 "_all": {"enabled": False},
178 "dynamic_templates": tag_templates,
184 "dynamic_templates": tag_templates,
179 "properties": {
185 "properties": {
180 "pg_id": {"type": "string", "index": "not_analyzed"},
186 "pg_id": {"type": "keyword", "index": True},
181 "resource_id": {"type": "integer"},
187 "resource_id": {"type": "integer"},
182 "priority": {"type": "integer"},
188 "priority": {"type": "integer"},
183 "error": {"type": "string", "analyzer": "simple"},
189 "error": {"type": "text", "analyzer": "simple"},
184 "read": {"type": "boolean"},
190 "read": {"type": "boolean"},
185 "occurences": {"type": "integer"},
191 "occurences": {"type": "integer"},
186 "fixed": {"type": "boolean"},
192 "fixed": {"type": "boolean"},
@@ -195,21 +201,27 b' def update_template():'
195 "_all": {"enabled": False},
201 "_all": {"enabled": False},
196 "dynamic_templates": tag_templates,
202 "dynamic_templates": tag_templates,
197 "properties": {
203 "properties": {
198 "pg_id": {"type": "string", "index": "not_analyzed"},
204 "pg_id": {"type": "keyword", "index": True},
199 "resource_id": {"type": "integer"},
205 "resource_id": {"type": "integer"},
200 "group_id": {"type": "string"},
206 "group_id": {"type": "keyword"},
201 "http_status": {"type": "integer"},
207 "http_status": {"type": "integer"},
202 "ip": {"type": "string", "index": "not_analyzed"},
208 "ip": {"type": "keyword", "index": True},
203 "url_domain": {"type": "string", "analyzer": "simple"},
209 "url_domain": {"type": "text", "analyzer": "simple"},
204 "url_path": {"type": "string", "analyzer": "url_path"},
210 "url_path": {"type": "text", "analyzer": "url_path"},
205 "error": {"type": "string", "analyzer": "simple"},
211 "error": {"type": "text", "analyzer": "simple"},
206 "report_type": {"type": "integer"},
212 "report_type": {"type": "integer"},
207 "start_time": {"type": "date"},
213 "start_time": {"type": "date"},
208 "request_id": {"type": "string", "index": "not_analyzed"},
214 "request_id": {"type": "keyword", "index": True},
209 "end_time": {"type": "date"},
215 "end_time": {"type": "date"},
210 "duration": {"type": "float"},
216 "duration": {"type": "float"},
211 "tags": {"type": "object"},
217 "tags": {"type": "object"},
212 "tag_list": {"type": "string", "analyzer": "tag_value"},
218 "tag_list": {"type": "text", "analyzer": "tag_value",
219 "fields": {
220 "keyword": {
221 "type": "keyword",
222 "ignore_above": 256
223 }
224 }},
213 "extra": {"type": "object"},
225 "extra": {"type": "object"},
214 },
226 },
215 "_parent": {"type": "report_group"},
227 "_parent": {"type": "report_group"},
@@ -218,17 +230,26 b' def update_template():'
218 "_all": {"enabled": False},
230 "_all": {"enabled": False},
219 "dynamic_templates": tag_templates,
231 "dynamic_templates": tag_templates,
220 "properties": {
232 "properties": {
221 "pg_id": {"type": "string", "index": "not_analyzed"},
233 "pg_id": {"type": "keyword", "index": True},
222 "delete_hash": {"type": "string", "index": "not_analyzed"},
234 "delete_hash": {"type": "keyword", "index": True},
223 "resource_id": {"type": "integer"},
235 "resource_id": {"type": "integer"},
224 "timestamp": {"type": "date"},
236 "timestamp": {"type": "date"},
225 "permanent": {"type": "boolean"},
237 "permanent": {"type": "boolean"},
226 "request_id": {"type": "string", "index": "not_analyzed"},
238 "request_id": {"type": "keyword", "index": True},
227 "log_level": {"type": "string", "analyzer": "simple"},
239 "log_level": {"type": "text", "analyzer": "simple"},
228 "message": {"type": "string", "analyzer": "simple"},
240 "message": {"type": "text", "analyzer": "simple"},
229 "namespace": {"type": "string", "index": "not_analyzed"},
241 "namespace": {
242 "type": "text",
243 "fields": {"keyword": {"type": "keyword", "ignore_above": 256}},
244 },
230 "tags": {"type": "object"},
245 "tags": {"type": "object"},
231 "tag_list": {"type": "string", "analyzer": "tag_value"},
246 "tag_list": {"type": "text", "analyzer": "tag_value",
247 "fields": {
248 "keyword": {
249 "type": "keyword",
250 "ignore_above": 256
251 }
252 }},
232 },
253 },
233 },
254 },
234 },
255 },
Show file before | Show file after | Hide comments
@@ -171,13 +171,13 b' def common_tags(request):'
171 if namespaces:
171 if namespaces:
172 filter_part.append({"terms": {"namespace": namespaces}})
172 filter_part.append({"terms": {"namespace": namespaces}})
173
173
174 query["aggs"] = {"sub_agg": {"terms": {"field": "tag_list", "size": 50}}}
174 query["aggs"] = {"sub_agg": {"terms": {"field": "tag_list.keyword", "size": 50}}}
175 # tags
175 # tags
176 index_names = es_index_name_limiter(ixtypes=[config.get("datasource", "logs")])
176 index_names = es_index_name_limiter(ixtypes=[config.get("datasource", "logs")])
177 result = Datastores.es.search(body=query, index=index_names, doc_type="log", size=0)
177 result = Datastores.es.search(body=query, index=index_names, doc_type="log", size=0)
178 tag_buckets = result["aggregations"]["sub_agg"].get("buckets", [])
178 tag_buckets = result["aggregations"]["sub_agg"].get("buckets", [])
179 # namespaces
179 # namespaces
180 query["aggs"] = {"sub_agg": {"terms": {"field": "namespace", "size": 50}}}
180 query["aggs"] = {"sub_agg": {"terms": {"field": "namespace.keyword", "size": 50}}}
181 result = Datastores.es.search(body=query, index=index_names, doc_type="log", size=0)
181 result = Datastores.es.search(body=query, index=index_names, doc_type="log", size=0)
182 namespaces_buckets = result["aggregations"]["sub_agg"].get("buckets", [])
182 namespaces_buckets = result["aggregations"]["sub_agg"].get("buckets", [])
183 return {
183 return {
General Comments 4
Under Review
vaingmuny
author
Comment #23545
Edit
Delete
|

Auto status change to "Under Review"

Under Review
vaingmuny
author
Comment #23711
Edit
Delete
|

Auto status change to "Under Review"

You need to be logged in to leave comments. Login now